diff --git a/.github/skills/sentinel-xdr-dependency-helper/SKILL.md b/.github/skills/sentinel-xdr-dependency-helper/SKILL.md new file mode 100644 index 00000000000..a30d120b1d3 --- /dev/null +++ b/.github/skills/sentinel-xdr-dependency-helper/SKILL.md @@ -0,0 +1,162 @@ +--- +name: sentinel-xdr-dependency-helper +description: Help authors review and fill XDR YAML workload and connector dependencies using exact-query Log Analytics lab metadata. Conversion remains offline. +--- + +# Review detection dependency provenance + +For workbook gaps, use the same explicit accept/change/defer experience with +individual query pointers and hashes. Workbook proposals/configuration are +separate from the detection-only `dependency-plan` command. Static/manual review +starts with `workbook-dependency-plan --solution "" --run-id ""`. +Use `workbook-review` with `--plan`, `--decisions`, `--plan-sha256`, `--actor`, +`--reference`, and a new run-local `--output-dir` to emit reviewed conversion +configuration and a receipt. These commands do not edit source content or +authenticate. A reviewed removal request is a pending parser-change action, +not proof of an applied removal. `convert --overwrite-workbooks` can replace +unchanged hash-bound managed workbook drafts without authorizing detection +overwrites. Static/manual review +and explicitly approved lab metadata discovery are available in either profile. +Do not execute unresolved workbook parameters or reuse detection consent for +workbook queries. Show underlying data types, workload mappings, connector +alternatives, function prerequisites, and all remaining technical failures. +Prefer modern connectors over deprecated alternatives only with reviewed +recommendation/lifecycle evidence. Never require both by default. + +Reviewed Sentinel workspace functions can be preserved for workbooks and AH, +not Custom Detections. Standard built-in ASIM parsers starting with `_` (such +as `_ASim_*` and `_Im_*`) are accepted by Defender XDR Advanced Hunting and can +be used in both Custom Detections and Defender Workbooks; do not reject them as +unsupported workspace-saved functions. Keep exact packaged function-definition +hashes in `contentProvenance.requiredWorkspaceFunctions`; function aliases are +not tables. +`_GetWatchlist` is a supported built-in in Sentinel/AH; do not surface it as a +missing parser or add saved-search installation dependencies for it. +`Watchlist` is platform reference data, not a connector-fed telemetry data type. +Preserve raw lineage but exclude it from required workload/connector data types; +do not ask the user to choose a connector for it. +Required parsers must be installed before the workbook. Optional vendor branches +can remain unavailable with explicit warnings and incomplete coverage. Offer +keep/remove/defer; removal needs a scoped parser-change preview and approval, +not deletion of shared functions or automatic clearance of technical blockers. + +Use this skill when YAML prerequisites are missing or uncertain, or when a user +requests lab-assisted dependency validation. This is a validation helper, not +deployment, ingestion, qualification, or permission to change toolkit code. + +## Offline proposal + +Run `sentinel-xdr-migration dependency-plan --solution ""`. +It reads the original Sentinel queries, converted queries, source connector +declarations, and current table mappings. It makes no Azure call and does not +modify YAML. +Conversion never requires Log Analytics authentication or availability. + +Inspect each proposal and unresolved reason. The workload vocabulary is: +`mde`, `mdo`, `mdi`, `mda`, `mdc`, `entraid`, `purview`, `appgovernance`, +`sentinel`, `mtp`. These are toolkit telemetry prerequisites, not Graph +`serviceSource` values or a claimed public contentProvenance API enum. +Only independently reviewed mappings are built in. Do not copy private catalogs. +In particular, `SecurityAlert` does not establish a specific Defender product +or connector. Preserve authoritative source connector declarations. + +## Approved lab metadata + +For Authoring or Qualification, ask for explicit bounded read-only approval +covering the exact original queries, tenant, subscription, and lab workspace +before executing them. Use the existing locked target in Qualification; +Authoring requires an explicitly selected lab. Do not reinterpret existing +Advanced Hunting consent as Sentinel-query permission. No authentication, +permission changes, environment writes, or Qualification gates are authorized. + +Inspect advertised official Sentinel Triage MCP tools first. Use one only if +it can request `Prefer: include-dataSources=true` and return input metadata. +If that capability is absent or the provider fails, use the Azure CLI runner +skill for the approved Log Analytics REST query: + +```text +POST https://api.loganalytics.io/v1/workspaces//query +Prefer: include-dataSources=true +Content-Type: application/json +{"query": ""} +``` + +Use a run-local JSON body file, not shell interpolation of KQL. Run only against +the approved workspace with an already authenticated identity verified against +the approved tenant/subscription. Do not sign in automatically. Do not replace +the original query with `getschema`, add a time range, or otherwise change it. +Do not fall back after a genuine KQL failure, partial result, auth failure from +the direct provider, or any failed REST response. Record the failure explicitly. + +The SDK source-validation runner also requests input metadata, but its existing +Qualification target and consent gates remain unchanged. + +Keep raw successful metadata and provider/target evidence in ignored run-local +files. Normalize one result per detection: + +```json +{ + "results": [{ + "detection": "Example.yaml", + "status": "passed", + "workspaceId": "", + "querySha256": "", + "dataSources": [{"tables": [{"name": "DeviceEvents"}]}], + "error": null + }] +} +``` + +Never use response `tables` (output rows/schema) as input lineage. Import the +complete current family with: + +```text +sentinel-xdr-migration dependency-plan --solution "" --results "" --workspace-id "" +``` + +The backend accepts 1-10 distinct underlying tables. More than 10, or missing/ +empty metadata from a successful query, triggers the conservative scanner. +It records the method, fallback reason, original table list and query hash. +It also scans the converted query for added table references and reports +`targetDataTypes` and `targetDiscoveryMethod`. Source API metadata is not a +claim that Graph returned input lineage for the converted query. +Malformed metadata and failed/stale/cross-workspace results are errors, not +successful empty dependency lists. + +## Review and YAML update + +Present proposed records, fallback use, unknown mappings, and live comparison +errors. Ask the user to supply unresolved data types, workload mappings and +connectors. If nothing resolves, leave arrays empty and explicitly mark the +provenance `dependencyReviewRequired: true`; never describe it as verified. +Native XDR rules can legitimately have no Sentinel connector prerequisite. +Reviewed no-prerequisite arrays are valid; unknown empty arrays are not a +review decision. + +After explicit approval, update only the detection YAML's +`contentProvenance.requiredWorkloads` and +`contentProvenance.requiredDataConnectors`, preserving `source`, `conversion`, +identities, versions, source rule bytes, tactics and disabled status. Clear +`dependencyReviewRequired` only after the missing information is reviewed. +Records have the following form: + +```yaml +contentProvenance: + requiredWorkloads: + - workload: mde + dataTypes: [DeviceEvents] + requiredDataConnectors: + - connectorId: MicrosoftThreatProtection + dataTypes: [DeviceEvents] +``` + +Do not add packageId/packageVersion/contentId/contentVersion to authored YAML. +Packaging derives them from solution metadata and the canonical detection +identity/version. Dependencies belong to the full Content Hub record, not the +lightweight deployed rule record. + +Run offline `validate`, then repeat the imported lab comparison when applicable. +Retain evidence tied to the exact query and current declarations. YAML edits +invalidate previous package/validation evidence. Metadata verifies table +prerequisites, not connector installation or availability, alert parity, +marketplace registration, or public Graph support for contentProvenance. diff --git a/.github/skills/sentinel-xdr-detection-validator/SKILL.md b/.github/skills/sentinel-xdr-detection-validator/SKILL.md index fce1faca206..86bcca5c206 100644 --- a/.github/skills/sentinel-xdr-detection-validator/SKILL.md +++ b/.github/skills/sentinel-xdr-detection-validator/SKILL.md @@ -5,6 +5,23 @@ description: Validate generated XDR Detection YAML structurally and validate sou # Validate XDR Detection YAML +Offline validation requires `contentProvenance.requiredWorkloads` records +(`workload`, `dataTypes`) and `contentProvenance.requiredDataConnectors` records +(`connectorId`, `dataTypes`). Missing fields, malformed/unknown workloads and +`dependencyReviewRequired: true` block validation. Use +`sentinel-xdr-dependency-helper` to review unresolved prerequisites against an +explicitly approved lab. Conversion never calls or requires Log Analytics. +Reviewed empty connector arrays are valid for native XDR-only content. + +Live source validation requests `Prefer: include-dataSources=true`, retains +input metadata, and compares it with authored workload/connector declarations. +It uses API metadata for 1-10 distinct tables; successful missing metadata or +more than 10 tables falls back to the conservative scanner. Failed or partial +queries never become empty successful dependencies. Source connector +declarations are compared without inventing connectors from shared tables. +Authoring can use the helper's explicitly approved lab reads and offline +metadata import without enabling the Qualification-only SDK execution route. + Use the `sentinel-xdr-migration` CLI for deterministic planning, structural validation, fallback execution, and reporting. Runtime query execution prefers: diff --git a/.github/skills/sentinel-xdr-solution-packager/SKILL.md b/.github/skills/sentinel-xdr-solution-packager/SKILL.md index e7c52996656..374049d48e3 100644 --- a/.github/skills/sentinel-xdr-solution-packager/SKILL.md +++ b/.github/skills/sentinel-xdr-solution-packager/SKILL.md @@ -5,8 +5,8 @@ description: Package a complete Microsoft Sentinel and Defender XDR solution thr # Package a Sentinel and Defender XDR solution -Use this skill after solution content and any `XDR Detections/*.yaml` files are -complete and structurally validated. +Use this skill after solution content, any `XDR Detections/*.yaml`, and selected +`XDR Workbooks/*.json` files are complete and structurally validated. The XDR Solution Manager workflow always runs V3.1. Legacy V3 remains available outside this workflow for legacy Sentinel-only packaging and intentionally ignores XDR @@ -19,6 +19,44 @@ Detections are declared, it also supports hybrid installation: - `DeployCustomDetection=true` (default false) installs Custom Detections and their Content Hub registrations independently. Both selections may be true. - Custom Detections are always packaged disabled. +- `DeployDefenderWorkbook=true` (default false) installs Defender Workbooks and + their registrations independently of Sentinel `DeployWorkbook`. + +## Defender Workbook candidate boundary + +Use the explicit solution Data `XDR Workbooks` selection. An empty array emits no +Defender Workbook resources or deployment flag; never scan to repopulate it. +Each selected file retains a complete `Notebook/1.0`, distinct canonical content +and installation GUIDs, exact raw source-file and query provenance, resolved +prerequisites, and explicit false review markers. Content versions inherit the +selected solution release; the Notebook format version is unchanged. + +Installation and stored installation templates carry exactly the resolved +package/content identity quartet. Full registration also retains prerequisites. +Reviewed workspace functions are declared in `requiredWorkspaceFunctions` with +`functionAlias`, solution-relative `sourcePath`, and current `sourceSha256`. +Required parser installation is mandatory: the workbook's outer installation +deployment must depend on the actual unconditional +`Microsoft.OperationalInsights/workspaces/savedSearches` resources emitted in +the same enclosing template. Do not substitute parser catalog registration for +installation, invent dependencies on external resources, or allow selection +flags to skip required parsers. Missing/stale/ambiguous prerequisites block +packaging before template mutation. Parser overwrites are expected for this route. +Keep workspace-function prerequisites in full registration; installed content +provenance remains the quartet. ARM ordering does not prove AH visibility or +query/runtime acceptance. +Do not classify `_GetWatchlist` as an installable parser or `Watchlist` as +connector-fed telemetry. Raw reference lineage is separate from required +workload/connector data types. +Do not rewrite source workbooks, remove panels, silently resolve blockers, or +treat conversion findings as permission to change toolkit implementation. + +The installation/registration provider contract is unverified. A generated +candidate is not accepted packaging. Require independent approved lab +installation, registration/provenance readback, repeat-install and update +evidence before claiming contract acceptance. ARM success, ARM-TTK, and local +tests do not establish workbook UI or query qualification. No packaging command +implicitly authorizes environment writes. ## Scope @@ -44,8 +82,9 @@ Use `none` for repeat local validation. Before producing a package intended for submission, ask which release increment to apply. Never bump the version more than once for the same package attempt. -V3.1 requires effective solution and XDR content release versions in -`[3.1.0, 4.0.0)`. Other content release versions must be below `4.0.0`; their +V3.1 requires effective solution release versions in `[3.2.0, 4.0.0)`. +Independent XDR Custom Detection release versions remain in `[3.1.0, 4.0.0)`. +Other content release versions must be below `4.0.0`; their legacy lower versions remain valid. Out-of-range versions are hard errors, not automatic promotions. A major bump from `3.x` is therefore rejected. XDR requires its own top-level YAML `version` (`major.minor.patch`); new @@ -90,8 +129,14 @@ authored detection YAML or provenance. Installation and registration IDs and references must use the same normalized identity. Existing deployed identities are not migrated or deleted. An optional read-only identity review may help diagnose known legacy IDs or collisions, but it is not a mandatory -pre-deployment existence check. Stop for manual reconciliation if an observed -ID's ownership or content is uncertain; never overwrite unrelated content. +pre-deployment existence check. Follow the canonical toolkit-owner managed +update policy: approved same-ID workbook and XDR detection overwrites are normal, +including in labs. Do not require a previous remote-content hash match or an +extra overwrite decision solely because existing content/version differs. +Stop for wrong or ambiguously mapped identities and out-of-scope resources, +not for an intended managed same-ID update. Personalized workbook copies use +separate IDs and are not selected for toolkit updates. Preserve deployment +consent, disabled detection payloads and exact post-write acceptance checks. `Include XDR Content Registration` is deprecated: any supplied value warns and is ignored. V3.1 emits installation and registration together under diff --git a/.script/local-validation/build-and-validate.ps1 b/.script/local-validation/build-and-validate.ps1 index e0f82e0e0e0..2955afe4b2a 100644 --- a/.script/local-validation/build-and-validate.ps1 +++ b/.script/local-validation/build-and-validate.ps1 @@ -29,6 +29,10 @@ Exact V3.1 packaging report under the selected solution. When supplied, the runner requires validation-only mode and emits hash-bound evidence for that package. +.PARAMETER ArmValidationMethod + arm-ttk (legacy default) runs ARM-TTK. lab-deployment replaces only ARM-TTK + with pending external approved deployment validation; its report is not acceptance. + .PARAMETER VersionBump How to increment the version. Options: patch (default), minor, major. @@ -86,7 +90,10 @@ param( [string]$ReleaseNotes = "", - [string]$PackageReportPath + [string]$PackageReportPath, + + [ValidateSet("arm-ttk", "lab-deployment")] + [string]$ArmValidationMethod = "arm-ttk" ) # ============================================================================= @@ -94,6 +101,19 @@ param( # ============================================================================= $ErrorActionPreference = "Stop" +if ($ArmValidationMethod -eq "lab-deployment") { + if (-not $SkipPackaging -or $SkipValidation -or -not $PackageReportPath) { + Write-Host "❌ lab-deployment requires -SkipPackaging, -PackageReportPath and all static validators." -ForegroundColor Red + exit 2 + } + $labAllowedSkips = @("kql", "detection-schema", "non-ascii", "arm-ttk") + if (@($ValidationSkip -split ',' | Where-Object { $_ -and $_ -notin $labAllowedSkips }).Count -gt 0) { + Write-Host "❌ lab-deployment cannot skip other static validators." -ForegroundColor Red + exit 2 + } + # These checks run natively below; only ARM-TTK is replaced by the lab route. + $ValidationSkip = "kql,detection-schema,non-ascii,arm-ttk" +} # The toolkit root is anchored to this script, never to the caller's current directory. $toolkitRoot = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot "../..")) @@ -1127,7 +1147,16 @@ if (-not $SkipValidation) { # ARM-TTK Validator (local — calls run-arm-ttk.ps1, same checks as entrypoint.ps1) # ----------------------------------------------------------------- $packageDir = Join-Path $solutionPath "Package" - if (Test-Path $packageDir) { + if ($ArmValidationMethod -eq "lab-deployment") { + Write-Host " ⏳ ARM deployment validation pending; candidate is NOT accepted." -ForegroundColor Yellow + $null = $reportEntries.Add(@{ + Step = 3; Name = "ARM-TTK"; Status = "skipped" + Passed = 0; Failed = 0; Skipped = 1; FileCount = 0 + Duration = [TimeSpan]::Zero + SkipReason = "Replaced only by pending approved lab deployment validation; not acceptance" + }) + } + elseif (Test-Path $packageDir) { $armTtkScript = Join-Path $toolkitRoot ".script/local-validation/run-arm-ttk.ps1" if (-not (Test-Path $armTtkScript)) { Write-Host " ⏭ ARM-TTK validation skipped — run-arm-ttk.ps1 not found." -ForegroundColor Yellow @@ -1566,6 +1595,13 @@ if ($PackageReportPath -and $null -ne $validationEvidenceBefore) { -PackagingSkipped ([bool]$SkipPackaging) ` -ValidationSkipped ([bool]$SkipValidation) ` -EvidenceRunPath $evidenceRunPath + $validationBinding.armValidationMethod = $ArmValidationMethod + $validationBinding.armValidationStatus = if ($ArmValidationMethod -eq "lab-deployment") { "pending" } else { + if (@($reportEntries | Where-Object { $_.Name -eq "ARM-TTK" -and $_.Status -eq "passed" }).Count -eq 1) { "passed" } else { "not-passed" } + } + $validationBinding.packageLifecycleStatus = if ($ArmValidationMethod -eq "lab-deployment") { "candidate-generated" } else { "validated" } + $validationBinding.packageAccepted = $false + if ($ArmValidationMethod -eq "lab-deployment") { $validationBinding.workbookCapabilityVersion = 1 } $bindingReceipt = Write-ValidationEvidenceBinding ` -EvidenceRunPath $evidenceRunPath ` -Binding $validationBinding @@ -1723,7 +1759,10 @@ function Write-BuildReport { Add-Line "Version: $versionStr" Add-Line "Date: $dateStr" Add-Line "Duration: $durationStr" - Add-Line "Result: $(if ($Success) { 'PASSED' } else { 'FAILED' })" + Add-Line "ARM validation method: $ArmValidationMethod" + Add-Line "ARM validation status: $(if ($ArmValidationMethod -eq 'lab-deployment') { 'pending' } else { 'see validator results' })" + Add-Line "Package accepted: false" + Add-Line "Result: $(if (-not $Success) { 'FAILED' } elseif ($ArmValidationMethod -eq 'lab-deployment') { 'CANDIDATE_GENERATED' } else { 'PASSED' })" if ($Meta.ValidationBindingPath) { Add-Line "Validation binding: $($Meta.ValidationBindingPath)" Add-Line "Validation binding SHA256: $($Meta.ValidationBindingSha256)" @@ -1799,7 +1838,11 @@ function Write-BuildReport { Add-Line "" Add-Line "Total: $totalValidators validators | $totalPassed passed | $totalFailed failed | $totalSkipped skipped" Add-Line "" - if ($Success) { + if ($Success -and $ArmValidationMethod -eq "lab-deployment") { + Add-Line "**CANDIDATE GENERATED — NOT ACCEPTED**" + Add-Line "All reported static checks completed; approved deployment and workbook contract readback remain pending." + } + elseif ($Success) { Add-Line "**BUILD SUCCESSFUL**" } else { diff --git a/Tools/Create-Azure-Sentinel-Solution/V3/README-V3_1.md b/Tools/Create-Azure-Sentinel-Solution/V3/README-V3_1.md index d246e407991..652bbab2faa 100644 --- a/Tools/Create-Azure-Sentinel-Solution/V3/README-V3_1.md +++ b/Tools/Create-Azure-Sentinel-Solution/V3/README-V3_1.md @@ -8,7 +8,7 @@ Both entry points live in the V3 directory: and remains the CI entry point. Both entry points use `common\commonFunctions.ps1` for the core packaging -implementation. Only V3.1 enables Defender XDR Custom Detection injection. +implementation. Only V3.1 enables Defender XDR Custom Detection and Defender Workbook injection. They are alternative entry points, not two required packaging steps. Use V3 for a legacy Sentinel-only package and V3.1 when the solution contains XDR @@ -65,12 +65,15 @@ For a solution carrying the five corresponding families: | `DeployWorkbook` | `true` | Sentinel Workbook and WorkbookTemplate content | | `DeployDataConnector` | `true` | DataConnector, ResourceDataConnector/ResourcesDataConnector, DataType, connector metadata and catalog resources | | `DeployCustomDetection` | `false` | Both Custom Detection installation and Content Hub registration | +| `DeployDefenderWorkbook` | `false` | Both Defender Workbook installation and Content Hub registration | `common/contentDeploymentParameters.json` is the shared vocabulary used by generation, UI, Python artifact checks, and tests. HuntingQuery, InvestigationQuery, Parser, Watchlist/WatchlistTemplate, Notebook, AutomationRule and SummaryRule have their own `Deploy` controls **when emitted**, not speculative empty controls. -No DefenderWorkbook emitter/control is added. Classification uses content kinds, +Required workspace parsers declared by selected Defender Workbooks remain +unconditional prerequisites, independent of optional content-family selections. +Classification uses content kinds, resource types and explicit emitter ownership, not display names, counts or query text. Unclassified content fails with its resource name/type/kind. @@ -111,6 +114,31 @@ uninstall operation. There is no new all-off runtime check. A selection can leave prerequisites unmet: the booleans do not rewrite dependency semantics, guarantee Content Hub acceptance, or provision missing external dependencies. +### Defender Workbook parser prerequisites + +Authored workbooks declare reviewed function prerequisites in +`contentProvenance.requiredWorkspaceFunctions`: each record contains the exact +`functionAlias`, packaged solution-relative `sourcePath`, and current +`sourceSha256`. V3.1 resolves these records to the actual top-level +`Microsoft.OperationalInsights/workspaces/savedSearches` resources, not parser +content templates. The outer Defender Workbook installation deployment adds +`dependsOn` edges to those resources, preserving its content-package dependency. +Required parser installation is mandatory and supports normal same-resource +updates. Missing, duplicate, conditional, or stale prerequisites are errors, +not dependencies on nonexistent resources or permission to skip a parser. + +`_GetWatchlist` is a supported built-in, not an installable parser prerequisite. +Its `Watchlist` input is platform reference data: preserve raw lineage, but +reject it in connector-fed telemetry data types in required workload/connector +records. + +Full workbook registration carries reviewed function prerequisites. Installed +notebook provenance remains exactly the package/content identity/version +quartet. Deployment ordering does not verify connected-workspace access, AH +propagation, nested optional vendor coverage, query correctness, or the preview +workbook provider contract. Reviewed Sentinel workspace functions can be used +in AH and workbooks; this does not make them supported in Custom Detection rules. + **Compatibility:** `E5Flavor` and `RegisterE5Content` are no longer generated. Update external parameter files to use only controls declared by that package; omitting a declared parameter uses its default, whereas explicit `false` skips @@ -129,6 +157,27 @@ Live provider support for CustomDetection registration is **unverified**. Template emission and offline tests do not establish deployment/Marketplace acceptance. Any subsequent environment write requires separate approval. +### Custom Detection content provenance + +Authored YAML must contain `contentProvenance.requiredWorkloads` records with +`workload` and `dataTypes`, and `requiredDataConnectors` records with `connectorId` +and `dataTypes`. Supported workload values are `mde`, `mdo`, `mdi`, `mda`, `mdc`, +`entraid`, `purview`, `appgovernance`, `sentinel`, `mtp`. Malformed/missing arrays, +unknown workloads and explicitly unresolved dependency review block packaging. +The legacy `conversion.requiredWorkloads` strings are not used as dependency +records. + +The packager resolves `packageId` and `packageVersion` from the current solution, +and `contentId`/`contentVersion` from the canonical packaged detection ID and +authored XDR version. It stamps the same four-key `contentProvenance` on the +installation resource and the registration's stored installation resource. +Resolved literals are used in inner-scope templates, not references to outer +variables. Registration properties contain that quartet plus the YAML +`requiredWorkloads` and `requiredDataConnectors` arrays. Dependencies are never +placed on the deployed Graph object. Authored YAML and source rules are not +mutated. These emitted contract fields remain subject to live provider +verification; offline tests are not registration/deployment acceptance. + ### First authored tactic in V3.1 ARM payloads Authored YAML may retain multiple tactics. V3.1 validates a supplied, nonempty @@ -196,13 +245,17 @@ V3.1 applies these hard guards in local, catalog, and positional pipeline modes: | Release version | Accepted range | |---|---| -| Effective solution/package version, even without XDR content | `>= 3.1.0` and `< 4.0.0` | +| Effective solution/package version, even without XDR content | `>= 3.2.0` and `< 4.0.0` | | Effective XDR Custom Detection content version | `>= 3.1.0` and `< 4.0.0` | | Every other packaged content-item release version | `< 4.0.0`; existing `1.x`/`2.x` versions remain valid | The guard reports the content type/path, effective version, and required range. It never promotes versions automatically. Explicit local bumps are evaluated -before output: for example, `major` from `3.1.0` selects `4.0.0` and is rejected. +before output: for example, `major` from `3.2.0` selects `4.0.0` and is rejected. +An older source solution may reach the supported range through an explicitly +approved target or bump: `minor` from `3.1.x` selects `3.2.0`, whereas a `patch` +that remains in `3.1.x` is rejected. This solution floor does not change +independent content, Notebook, schema, API or provider version checks. All version checks run before generated JSON/ZIP files are overwritten. Local bump persistence is deferred until version validation and package generation succeed; rejected versions leave the source versions and existing outputs intact. diff --git a/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3.ps1 b/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3.ps1 index 171d57535b0..fee0b09ee04 100644 --- a/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3.ps1 @@ -4,9 +4,22 @@ param( [ValidateSet("catalog", "local")] [string]$VersionMode = "catalog", [ValidateSet("patch", "minor", "major")] - [string]$VersionBump = "patch" + [string]$VersionBump = "patch", + [ValidateSet("arm-ttk", "lab-deployment")] + [string]$ArmValidationMethod = "arm-ttk" ) +if ($ArmValidationMethod -eq 'lab-deployment') { + & (Join-Path $PSScriptRoot '..\common\createSolutionLocal.ps1') ` + -SolutionDataFolderPath $SolutionDataFolderPath ` + -VersionMode $VersionMode ` + -VersionBump $VersionBump ` + -EntryPointName 'V3' ` + -DefaultInputPath (Join-Path $PSScriptRoot 'input') ` + -ArmValidationMethod $ArmValidationMethod + return +} + Write-Host '=======Starting Package Creation using V3 tool=========' Write-Host "Version Mode: $VersionMode" if ($VersionMode -eq "local") { diff --git a/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3_1.ps1 b/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3_1.ps1 index 791b1a24ae8..04a0f6b8283 100644 --- a/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3_1.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3_1.ps1 @@ -35,6 +35,8 @@ param ( [string]$TargetVersion = $null, [Parameter(ParameterSetName = "Local")] [string]$ToolkitRootPath = $null, + [ValidateSet("arm-ttk", "lab-deployment")] + [string]$ArmValidationMethod = "arm-ttk", [Parameter(ParameterSetName = "Local")] [Parameter(ParameterSetName = "Pipeline")] [switch]$SkipAttributionLookup @@ -64,6 +66,7 @@ if ($PSCmdlet.ParameterSetName -eq "Local") { -EntryPointName "V3.1" ` -DefaultInputPath (Join-Path $PSScriptRoot "input") ` -ToolkitRootPath $ToolkitRootPath ` + -ArmValidationMethod $ArmValidationMethod ` -IncludeXdrDetections $true ` -EnforceV31Versions $true ` -SkipAttributionLookup:$SkipAttributionLookup diff --git a/Tools/Create-Azure-Sentinel-Solution/common/commonFunctions.ps1 b/Tools/Create-Azure-Sentinel-Solution/common/commonFunctions.ps1 index 734fe19bd52..47d498e688d 100644 --- a/Tools/Create-Azure-Sentinel-Solution/common/commonFunctions.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/common/commonFunctions.ps1 @@ -105,7 +105,7 @@ function Convert-SolutionDataContentArrays { $encodedArrayNames = @( 'parsers', 'data connectors', 'dataconnectors', 'playbooks', 'workbooks', 'analytic rules', 'analytics rules', 'hunting queries', 'watchlists', - 'summary rules', 'summaryrules', 'xdr detections', 'custom detections', + 'summary rules', 'summaryrules', 'xdr detections', 'custom detections', 'xdr workbooks', 'savedsearches' ) foreach ($property in $Content.PSObject.Properties) { @@ -164,7 +164,27 @@ $baseCreateUiDefinitionPath = "$PSScriptRoot/templating/baseCreateUiDefinition.j $global:baseMainTemplate = Get-Content -Raw $baseMainTemplatePath | Out-String | ConvertFrom-Json $global:baseCreateUiDefinition = Get-Content -Raw $baseCreateUiDefinitionPath | Out-String | ConvertFrom-Json . "$PSScriptRoot/customDetections.ps1" +. "$PSScriptRoot/defenderWorkbooks.ps1" . "$PSScriptRoot/contentDeploymentParameters.ps1" +$script:baseSolutionResourceSelection = (Get-Command Get-SolutionResourceSelection).ScriptBlock + +function Get-SolutionResourceSelection { + param([psobject]$Resource) + if ($Resource.type -eq 'Microsoft.Resources/deployments') { + $inner = $Resource.properties.template + $workbook = $inner.resources.workbook + if ($inner.languageVersion -eq '2.0' -and + $inner.imports.MicrosoftSecurity.provider -eq 'MicrosoftSecurity' -and + $inner.imports.MicrosoftSecurity.version -eq '1.0.0' -and + $workbook.import -eq 'MicrosoftSecurity' -and + $workbook.type -eq 'Microsoft.Security/workbooks@2026-06-01-preview' -and + @($inner.resources.PSObject.Properties).Count -eq 1) { + return 'DeployDefenderWorkbook' + } + } + return (& $script:baseSolutionResourceSelection -Resource $Resource) +} + . "$PSScriptRoot/customerUsageAttribution.ps1" . "$PSScriptRoot/versionPolicy.ps1" $v31VersionPolicy = $null @@ -195,6 +215,7 @@ $ContentKindDict.Add("Playbook", "pl") $ContentKindDict.Add("PlaybookTemplate", "plt") $ContentKindDict.Add("Workbook", "wb") $ContentKindDict.Add("WorkbookTemplate", "wbt") +$ContentKindDict.Add("DefenderWorkbook", "dwb") $ContentKindDict.Add("Watchlist", "wl") $ContentKindDict.Add("WatchlistTemplate", "wlt") $ContentKindDict.Add("Notebook", "nb") @@ -606,6 +627,21 @@ function Update-XdrDescriptionCount([int]$Count) { ) } +function Update-DefenderWorkbookDescriptionCount([int]$Count) { + if ($Count -le 0) { return } + $pattern = '(\{\{SolutionDescription\}\}\r?\n\r?\n)(?[^\r\n]*)' + $global:baseCreateUiDefinition.parameters.config.basics.description = [regex]::Replace( + $global:baseCreateUiDefinition.parameters.config.basics.description, + $pattern, + [System.Text.RegularExpressions.MatchEvaluator]{ + param($match) + $counts = @($match.Groups['counts'].Value.TrimEnd(', '), "**XDR Workbooks:** $Count") | + Where-Object { $_ -ne '' } + return $match.Groups[1].Value + ($counts -join ', ') + } + ) +} + function GetContentSchemaVersion($defaultPackageVersion, $dataInputVersion) { # DEPENDING OF VERSION WE SHOULD SET THE CONTENTSCHEMAVERSION if ($null -eq $defaultPackageVersion -and $null -eq $dataInputVersion) { @@ -3227,9 +3263,13 @@ function GeneratePackage( ) { Assert-V31SolutionVersion -SolutionName $solutionName -Version $contentToImport.Version -CalculatedVersion $calculatedBuildPipelinePackageVersion Assert-V31VersionChecksPassed + $packageSerializationDepth = $jsonConversionDepth if ($IncludeXdrDetections) { $xdrDetectionCount = Add-XdrCustomDetectionsToSolution -SolutionName $solutionName -ContentToImport $contentToImport -Template $global:baseMainTemplate Update-XdrDescriptionCount -Count $xdrDetectionCount + $defenderWorkbookCount = Add-XdrDefenderWorkbooksToSolution -SolutionName $solutionName -ContentToImport $contentToImport -Template $global:baseMainTemplate + if ($defenderWorkbookCount -gt 0) { $packageSerializationDepth = 100 } + Update-DefenderWorkbookDescriptionCount -Count $defenderWorkbookCount Add-SolutionContentDeploymentParameters -Template $global:baseMainTemplate -UiDefinition $global:baseCreateUiDefinition } @@ -3288,15 +3328,15 @@ function GeneratePackage( if ($IncludeXdrDetections) { # Persist only after content/version preparation, after metadata filtering and before output. - Add-CustomerUsageAttribution -SolutionMetadataPath $SolutionMetadataPath -Template $global:baseMainTemplate -WarnIfMissing ($xdrDetectionCount -gt 0) -SkipAttributionLookup:$SkipAttributionLookup + Add-CustomerUsageAttribution -SolutionMetadataPath $SolutionMetadataPath -Template $global:baseMainTemplate -WarnIfMissing ($xdrDetectionCount -gt 0 -or $defenderWorkbookCount -gt 0) -SkipAttributionLookup:$SkipAttributionLookup } try { - $baseMainTemplate | ConvertTo-Json -Depth $jsonConversionDepth | Out-File $mainTemplateOutputPath -Encoding utf8 + $baseMainTemplate | ConvertTo-Json -Depth $packageSerializationDepth | Out-File $mainTemplateOutputPath -Encoding utf8 # create testParameters json file which will be used for test case automation $testParametersOutputPath = "$solutionFolder/testParameters.json" - $baseMainTemplate.parameters | ConvertTo-Json -Depth $jsonConversionDepth | Out-File $testParametersOutputPath -Encoding utf8 + $baseMainTemplate.parameters | ConvertTo-Json -Depth $packageSerializationDepth | Out-File $testParametersOutputPath -Encoding utf8 } catch { Write-Host "Failed to write output file $mainTemplateOutputPath" -ForegroundColor Red @@ -3316,7 +3356,7 @@ function GeneratePackage( # when there is no content then create ui fails as step is null $global:baseCreateUiDefinition.parameters.steps = @(@{}) # [{}] } - $global:baseCreateUiDefinition | ConvertTo-Json -Depth $jsonConversionDepth | Out-File $createUiDefinitionOutputPath -Encoding utf8 + $global:baseCreateUiDefinition | ConvertTo-Json -Depth $packageSerializationDepth | Out-File $createUiDefinitionOutputPath -Encoding utf8 } catch { Write-Host "Failed to write output file $createUiDefinitionOutputPath" -ForegroundColor Red diff --git a/Tools/Create-Azure-Sentinel-Solution/common/contentDeploymentParameters.json b/Tools/Create-Azure-Sentinel-Solution/common/contentDeploymentParameters.json index cd2793c8233..c4f4f262bcc 100644 --- a/Tools/Create-Azure-Sentinel-Solution/common/contentDeploymentParameters.json +++ b/Tools/Create-Azure-Sentinel-Solution/common/contentDeploymentParameters.json @@ -14,6 +14,11 @@ "defaultValue": true, "description": "Deploy the Microsoft Sentinel workbook content and its Content Hub registration carried by this solution (ContentKind Workbook and WorkbookTemplate)." }, + "DeployDefenderWorkbook": { + "kinds": ["DefenderWorkbook"], + "defaultValue": false, + "description": "Deploy Defender Workbooks and their Content Hub registrations independently of Microsoft Sentinel DeployWorkbook. Candidate provider contract requires approved lab installation, readback and update verification." + }, "DeployDataConnector": { "kinds": ["DataConnector", "ResourceDataConnector", "ResourcesDataConnector", "DataType"], "defaultValue": true, diff --git a/Tools/Create-Azure-Sentinel-Solution/common/contentDeploymentParameters.ps1 b/Tools/Create-Azure-Sentinel-Solution/common/contentDeploymentParameters.ps1 index 49b7d6b31c2..0aecd881f37 100644 --- a/Tools/Create-Azure-Sentinel-Solution/common/contentDeploymentParameters.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/common/contentDeploymentParameters.ps1 @@ -1,22 +1,31 @@ $script:contentDeploymentContract = Get-Content -LiteralPath (Join-Path $PSScriptRoot 'contentDeploymentParameters.json') -Raw | ConvertFrom-Json $script:solutionResourceKinds = [System.Collections.Generic.Dictionary[object,string]]::new([System.Collections.Generic.ReferenceEqualityComparer]::Instance) +$script:solutionRequiredResources = [System.Collections.Generic.HashSet[object]]::new([System.Collections.Generic.ReferenceEqualityComparer]::Instance) + +function Register-SolutionRequiredResources { + param([object[]]$Resources) + foreach ($resource in $Resources) { + [void]$script:solutionRequiredResources.Add($resource) + } +} function Register-SolutionResourceKind { param([object[]]$Resources, [string]$Kind) foreach ($resource in $Resources) { - $script:solutionResourceKinds[$resource.PSObject.BaseObject] = $Kind + $script:solutionResourceKinds[$resource] = $Kind } } function Get-SolutionResourceSelection { param([psobject]$Resource) + if ($script:solutionRequiredResources.Contains($Resource)) { return $null } $kind = [string]$Resource.properties.contentKind if (-not $kind -and $Resource.type -eq 'Microsoft.OperationalInsights/workspaces/providers/metadata') { $kind = [string]$Resource.properties.kind } if (-not $kind) { - $kind = $script:solutionResourceKinds[$Resource.PSObject.BaseObject] + $kind = $script:solutionResourceKinds[$Resource] } if (-not $kind) { $kind = switch ($Resource.type) { diff --git a/Tools/Create-Azure-Sentinel-Solution/common/createSolutionLocal.ps1 b/Tools/Create-Azure-Sentinel-Solution/common/createSolutionLocal.ps1 index 1e479cb9591..71749213c03 100644 --- a/Tools/Create-Azure-Sentinel-Solution/common/createSolutionLocal.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/common/createSolutionLocal.ps1 @@ -11,7 +11,9 @@ param( [string]$ToolkitRootPath = $null, [bool]$IncludeXdrDetections = $false, [bool]$EnforceV31Versions = $false, - [switch]$SkipAttributionLookup + [switch]$SkipAttributionLookup, + [ValidateSet("arm-ttk", "lab-deployment")] + [string]$ArmValidationMethod = "arm-ttk" ) Write-Host "=======Starting Package Creation using $EntryPointName tool=========" @@ -607,7 +609,12 @@ try { if ($EnforceV31Versions -and $VersionMode -eq 'local' -and ($VersionBump -ne 'none' -or -not [string]::IsNullOrWhiteSpace($TargetVersion))) { Save-V31LocalPackageVersion -DataFilePath $inputFile.FullName -Version $contentToImport.Version } - RunArmTtkOnPackage -solutionName $solutionName -isPipelineRun $false; + if ($ArmValidationMethod -eq "arm-ttk") { + RunArmTtkOnPackage -solutionName $solutionName -isPipelineRun $false; + } + else { + Write-Host "ARM-VALIDATION: Method=lab-deployment Status=pending CandidateGenerated=true Accepted=false" + } # check if mainTemplate and createUiDefinition json files are valid or not CheckJsonIsValid($solutionFolderBasePath) diff --git a/Tools/Create-Azure-Sentinel-Solution/common/customDetections.ps1 b/Tools/Create-Azure-Sentinel-Solution/common/customDetections.ps1 index c97750f90be..1ec1b6473f3 100644 --- a/Tools/Create-Azure-Sentinel-Solution/common/customDetections.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/common/customDetections.ps1 @@ -330,6 +330,9 @@ function New-CustomDetectionInstallDeployment { [string]$ExtensionVersion ) + if ($null -eq $DetectionResource.properties.contentProvenance) { + throw 'Custom Detection installation requires resolved content provenance.' + } $deploymentDetection = Copy-CustomDetectionObject -InputObject $DetectionResource $deploymentDetection.properties.id = ConvertTo-CustomDetectionArmLiteral -Value ([string]$deploymentDetection.properties.id) foreach ($mappingGroup in $deploymentDetection.properties.detectionAction.alertTemplate.entityMappings.PSObject.Properties) { @@ -423,12 +426,10 @@ function New-CustomDetectionRegistration { version = $ContentVersion } - $requiredWorkloads = @($DetectionDocument.contentProvenance.conversion.requiredWorkloads) - if ($requiredWorkloads.Count -gt 0) { - $properties['dependency'] = [pscustomobject]@{ - requiredWorkloads = $requiredWorkloads - } - } + $fullProvenance = Copy-CustomDetectionObject -InputObject $DetectionResource.properties.contentProvenance + $fullProvenance | Add-Member -NotePropertyName requiredWorkloads -NotePropertyValue @($DetectionDocument.contentProvenance.requiredWorkloads) + $fullProvenance | Add-Member -NotePropertyName requiredDataConnectors -NotePropertyValue @($DetectionDocument.contentProvenance.requiredDataConnectors) + $properties['contentProvenance'] = $fullProvenance return [pscustomobject]@{ type = 'Microsoft.OperationalInsights/workspaces/providers/contentTemplates' @@ -608,9 +609,6 @@ function Add-XdrCustomDetectionsToSolution { throw "Expected exactly one AnalyticsRule content template for Custom Detection source id '$sourceId'; found $($matches.Count)." } - $deploymentName = Get-CustomDetectionDeploymentName -SolutionName $SolutionName -DetectionId $detectionId - $Template.resources += New-CustomDetectionInstallDeployment -DeploymentName $deploymentName -DetectionResource $detectionResource -ExtensionVersion $extensionVersion - $contentVersion = [string]$detectionDocument.version if ($v31VersionPolicy) { Assert-V31ReleaseVersion -Version $contentVersion -ContentKind CustomDetection -ContentPath $configuredPath @@ -622,6 +620,47 @@ function Add-XdrCustomDetectionsToSolution { $contentVersion = $defaultContentVersion } } + $workloads = @('mde', 'mdo', 'mdi', 'mda', 'mdc', 'entraid', 'purview', 'appgovernance', 'sentinel', 'mtp') + foreach ($field in @('requiredWorkloads', 'requiredDataConnectors')) { + $fieldValue = $detectionDocument.contentProvenance.PSObject.Properties[$field] + if ($null -eq $fieldValue -or $fieldValue.Value -isnot [System.Array]) { + throw "Custom Detection '$detectionId' requires contentProvenance.$field as an array." + } + $identityField = if ($field -eq 'requiredWorkloads') { 'workload' } else { 'connectorId' } + foreach ($record in $fieldValue.Value) { + if ($null -eq $record -or $record.$identityField -isnot [string] -or + [string]::IsNullOrWhiteSpace($record.$identityField) -or + $record.dataTypes -isnot [System.Array]) { + throw "Custom Detection '$detectionId' has malformed contentProvenance.$field." + } + if ($field -eq 'requiredWorkloads' -and $record.workload -cnotin $workloads) { + throw "Custom Detection '$detectionId' has unknown workload '$($record.workload)'." + } + foreach ($dataType in $record.dataTypes) { + if ($dataType -isnot [string] -or [string]::IsNullOrWhiteSpace($dataType)) { + throw "Custom Detection '$detectionId' has invalid dependency dataTypes." + } + } + } + } + if ($detectionDocument.contentProvenance.dependencyReviewRequired -eq $true) { + throw "Custom Detection '$detectionId' has unresolved dependency provenance." + } + $packageId = [string]$Template.variables.solutionId + $packageVersion = [string]$Template.variables._solutionVersion + if ([string]::IsNullOrWhiteSpace($packageId) -or $packageId.StartsWith('[') -or + [string]::IsNullOrWhiteSpace($packageVersion) -or $packageVersion.StartsWith('[') -or + [string]::IsNullOrWhiteSpace($contentVersion)) { + throw "Custom Detection '$detectionId' requires resolved package and content identity/version." + } + $detectionResource.properties | Add-Member -NotePropertyName contentProvenance -NotePropertyValue ([pscustomobject]@{ + packageId = $packageId + packageVersion = $packageVersion + contentId = $detectionId + contentVersion = $contentVersion + }) -Force + $deploymentName = Get-CustomDetectionDeploymentName -SolutionName $SolutionName -DetectionId $detectionId + $Template.resources += New-CustomDetectionInstallDeployment -DeploymentName $deploymentName -DetectionResource $detectionResource -ExtensionVersion $extensionVersion $registration = New-CustomDetectionRegistration -DeploymentName $deploymentName -DetectionResource $detectionResource -DetectionDocument $detectionDocument -ContentVersion $contentVersion -ExtensionVersion $extensionVersion $Template.resources += $registration $dependencyCriteria += [pscustomobject]@{ diff --git a/Tools/Create-Azure-Sentinel-Solution/common/defenderWorkbooks.ps1 b/Tools/Create-Azure-Sentinel-Solution/common/defenderWorkbooks.ps1 new file mode 100644 index 00000000000..3ec8af6e74e --- /dev/null +++ b/Tools/Create-Azure-Sentinel-Solution/common/defenderWorkbooks.ps1 @@ -0,0 +1,944 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE in the repository root. + +function Get-DefenderWorkbookSelection { + param([psobject]$ContentToImport, [string]$SolutionRoot, [string]$SolutionName) + + $property = $ContentToImport.PSObject.Properties['XDR Workbooks'] + if ($null -eq $property) { + $directory = Join-Path $SolutionRoot 'XDR Workbooks' + if (Test-Path -LiteralPath $directory) { + Assert-NoSolutionContentSymlinks -SolutionRoot $SolutionRoot + if (@(Get-ChildItem -LiteralPath $directory -Recurse -File -Force | + Where-Object { $_.Extension -ieq '.json' }).Count -gt 0) { + throw "Existing XDR Workbooks require an explicit 'XDR Workbooks' selection array; use [] to exclude them." + } + } + return + } + if ($property.Value -isnot [array]) { + throw "Solution Data 'XDR Workbooks' must be an explicit array." + } + $seen = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + foreach ($reference in $property.Value) { + if ($reference -isnot [string] -or [string]::IsNullOrWhiteSpace($reference)) { + throw "XDR Workbooks selection must contain nonempty local path strings." + } + $resolved = Resolve-SolutionContentPath -SolutionRoot $SolutionRoot -SolutionName $SolutionName ` + -ContentPath $reference -RequireFile + if (-not $resolved.RelativePath.StartsWith('XDR Workbooks/', [System.StringComparison]::Ordinal) -or + [System.IO.Path]::GetExtension($resolved.FullPath) -ine '.json') { + throw "Defender Workbook selection must reference JSON files under XDR Workbooks: $reference" + } + if (-not $seen.Add($resolved.FullPath)) { + throw "Duplicate Defender Workbook selection: $reference" + } + $resolved + } +} + +function Assert-DefenderWorkbookReleaseVersion { + param([string]$Version) + if ($Version -cnotmatch '\A3\.[1-9][0-9]*\.(?:0|[1-9][0-9]*)\z') { + throw "Defender Workbook release version '$Version' must be canonical major.minor.patch in [3.1.0, 4.0.0)." + } +} + +function Assert-DefenderWorkbookDocument { + param([psobject]$Document, [string]$SolutionRoot, [string]$SolutionName, [string]$ContentPath) + + if ($Document -isnot [pscustomobject] -or $Document.version -cne 'Notebook/1.0' -or + $Document.items -isnot [array]) { + throw "Defender Workbook '$ContentPath' must be a complete Notebook/1.0 object with an items array." + } + Assert-DefenderWorkbookSerializationDepth -Value $Document + $metadata = $Document.xdrMetadata + $guidPattern = '\A[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}\z' + if ($metadata -isnot [pscustomobject] -or $metadata.schemaVersion -cne '1.0.0' -or + $metadata.contentId -isnot [string] -or $metadata.contentId -cnotmatch $guidPattern -or + $metadata.instanceId -isnot [string] -or $metadata.instanceId -cnotmatch $guidPattern -or + $metadata.contentId -ieq $metadata.instanceId -or + $metadata.displayName -isnot [string] -or [string]::IsNullOrWhiteSpace($metadata.displayName)) { + throw "Defender Workbook '$ContentPath' requires distinct canonical contentId/instanceId GUIDs and reviewed displayName." + } + Assert-DefenderWorkbookReleaseVersion -Version $metadata.version + $provenance = $Document.contentProvenance + if ($provenance -isnot [pscustomobject] -or + $provenance.dependencyReviewRequired -isnot [bool] -or $provenance.dependencyReviewRequired -or + $provenance.conversion -isnot [pscustomobject] -or + $provenance.conversion.tool -isnot [string] -or [string]::IsNullOrWhiteSpace($provenance.conversion.tool) -or + $provenance.conversion.version -isnot [string] -or [string]::IsNullOrWhiteSpace($provenance.conversion.version) -or + $provenance.conversion.reviewRequired -isnot [bool] -or $provenance.conversion.reviewRequired) { + throw "Defender Workbook '$ContentPath' requires explicitly reviewed conversion and dependency provenance." + } + $source = $provenance.source + if ($source -isnot [pscustomobject] -or $source.path -isnot [string] -or + $source.sha256 -isnot [string] -or $source.sha256 -cnotmatch '\A[0-9a-fA-F]{64}\z') { + throw "Defender Workbook '$ContentPath' requires its current source path and SHA256." + } + $resolvedSource = Resolve-SolutionContentPath -SolutionRoot $SolutionRoot -SolutionName $SolutionName ` + -ContentPath $source.path -RequireFile + if (-not $resolvedSource.RelativePath.StartsWith('Workbooks/', [System.StringComparison]::Ordinal) -or + [System.IO.Path]::GetExtension($resolvedSource.FullPath) -ine '.json' -or + (Get-FileHash -LiteralPath $resolvedSource.FullPath -Algorithm SHA256).Hash -ine $source.sha256) { + throw "Defender Workbook '$ContentPath' source identity/hash is stale or outside Workbooks." + } + if ($provenance.conversion.queries -isnot [array]) { + throw "Defender Workbook '$ContentPath' requires conversion.queries as an array." + } + $sourceNotebook = Get-Content -LiteralPath $resolvedSource.FullPath -Raw -ErrorAction Stop | + ConvertFrom-Json -ErrorAction Stop + if ($sourceNotebook -isnot [pscustomobject] -or $sourceNotebook.version -cne 'Notebook/1.0' -or + $sourceNotebook.items -isnot [array]) { + throw "Defender Workbook '$ContentPath' source must be a complete Notebook/1.0." + } + $sourcePointers = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + $targetPointers = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($query in $provenance.conversion.queries) { + if ($query -isnot [pscustomobject] -or + $query.status -cnotin @('converted', 'removed-scope-helper') -or + $query.sourcePath -isnot [string] -or -not $query.sourcePath.StartsWith('/') -or + $query.sourceSha256 -isnot [string] -or $query.sourceSha256 -cnotmatch '\A[0-9a-fA-F]{64}\z' -or + -not $sourcePointers.Add($query.sourcePath)) { + throw "Defender Workbook '$ContentPath' has unreviewed or malformed query provenance." + } + $sourceQuery = Get-DefenderWorkbookJsonPointer -Document $sourceNotebook -Pointer $query.sourcePath + if ($sourceQuery -isnot [string]) { + throw "Defender Workbook '$ContentPath' source query pointer must resolve to a string." + } + $sourceBytes = [System.Text.Encoding]::UTF8.GetBytes($sourceQuery) + $sourceHash = [System.Convert]::ToHexString([System.Security.Cryptography.SHA256]::HashData($sourceBytes)) + if ($sourceHash -ine $query.sourceSha256) { + throw "Defender Workbook '$ContentPath' source query hash is stale." + } + if ($query.status -ceq 'removed-scope-helper') { + if ($null -ne $query.outputPath -or $null -ne $query.targetSha256) { + throw "Defender Workbook '$ContentPath' removed scope query must have null output identity." + } + continue + } + if ($query.outputPath -isnot [string] -or -not $query.outputPath.StartsWith('/') -or + $query.targetSha256 -isnot [string] -or $query.targetSha256 -cnotmatch '\A[0-9a-fA-F]{64}\z' -or + -not $targetPointers.Add($query.outputPath)) { + throw "Defender Workbook '$ContentPath' requires current target query provenance." + } + $targetQuery = Get-DefenderWorkbookJsonPointer -Document $Document -Pointer $query.outputPath + if ($targetQuery -isnot [string]) { + throw "Defender Workbook '$ContentPath' target query pointer must resolve to a string." + } + $bytes = [System.Text.Encoding]::UTF8.GetBytes($targetQuery) + $hash = [System.Convert]::ToHexString([System.Security.Cryptography.SHA256]::HashData($bytes)) + if ($hash -ine $query.targetSha256) { + throw "Defender Workbook '$ContentPath' target query hash is stale." + } + } + $workloads = @('mde', 'mdo', 'mdi', 'mda', 'mdc', 'entraid', 'purview', 'appgovernance', 'sentinel', 'mtp') + foreach ($field in @('requiredWorkloads', 'requiredDataConnectors')) { + $value = $provenance.PSObject.Properties[$field] + if ($null -eq $value -or $value.Value -isnot [array]) { + throw "Defender Workbook '$ContentPath' requires contentProvenance.$field as an array." + } + $identityField = if ($field -eq 'requiredWorkloads') { 'workload' } else { 'connectorId' } + $seen = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + foreach ($record in $value.Value) { + if ($record -isnot [pscustomobject] -or $record.$identityField -isnot [string] -or + [string]::IsNullOrWhiteSpace($record.$identityField) -or + $record.dataTypes -isnot [array] -or -not $seen.Add($record.$identityField)) { + throw "Defender Workbook '$ContentPath' has malformed or duplicate $field records." + } + if ($field -eq 'requiredWorkloads' -and $record.workload -cnotin $workloads) { + throw "Defender Workbook '$ContentPath' has unknown workload '$($record.workload)'." + } + $types = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + foreach ($dataType in $record.dataTypes) { + if ($dataType -isnot [string] -or [string]::IsNullOrWhiteSpace($dataType) -or + -not $types.Add($dataType)) { + throw "Defender Workbook '$ContentPath' has malformed or duplicate dependency dataTypes." + } + if ($dataType -ieq 'Watchlist') { + throw "Defender Workbook '$ContentPath' Watchlist is platform reference data, not a telemetry dependency." + } + } + } + } +} + +function Assert-DefenderWorkbookSerializationDepth { + param([AllowNull()][object]$Value, [int]$Depth = 0) + if ($Depth -gt 85) { + throw 'Defender Workbook exceeds safe package JSON serialization depth; content cannot be truncated.' + } + if ($Value -is [pscustomobject]) { + foreach ($property in $Value.PSObject.Properties) { + Assert-DefenderWorkbookSerializationDepth -Value $property.Value -Depth ($Depth + 1) + } + } + elseif ($Value -is [array]) { + foreach ($item in $Value) { + Assert-DefenderWorkbookSerializationDepth -Value $item -Depth ($Depth + 1) + } + } +} + +function Get-DefenderWorkbookJsonPointer { + param([psobject]$Document, [string]$Pointer) + $value = $Document + foreach ($part in $Pointer.Substring(1).Split('/')) { + if ($part -cmatch '~(?:[^01]|$)') { throw "Invalid Defender Workbook query JSON pointer: $Pointer" } + $key = $part.Replace('~1', '/').Replace('~0', '~') + if ($value -is [array] -and $key -cmatch '\A(?:0|[1-9][0-9]*)\z' -and + [System.Numerics.BigInteger]::Parse($key) -lt $value.Count) { + $value = $value[[int]$key] + } + elseif ($value -is [pscustomobject] -and $null -ne $value.PSObject.Properties[$key]) { + $value = $value.PSObject.Properties[$key].Value + } + else { throw "Unresolvable Defender Workbook query JSON pointer: $Pointer" } + } + return $value +} + +function Get-DefenderWorkbookDeploymentName { + param([string]$SolutionName, [string]$ContentId) + $bytes = [System.Text.Encoding]::UTF8.GetBytes("$SolutionName/$ContentId") + $hash = [System.Convert]::ToHexString([System.Security.Cryptography.SHA256]::HashData($bytes)).ToLowerInvariant() + return 'DefenderWorkbook-' + $hash.Substring(0, 32) +} + +function Get-DefenderWorkbookKnownCatalogTables { + if ($null -eq $script:DefenderWorkbookKnownCatalogTables) { + $set = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + $catalogDir = Join-Path $PSScriptRoot '../../SentinelToXDRMigration/sentinel_xdr_migration/catalog' + foreach ($file in @('xdr_only_tables.json', 'table_mappings.json')) { + $full = Join-Path $catalogDir $file + if (Test-Path -LiteralPath $full -PathType Leaf) { + $json = Get-Content -LiteralPath $full -Raw | ConvertFrom-Json + if ($null -ne $json.tables) { + foreach ($prop in $json.tables.PSObject.Properties) { + [void]$set.Add($prop.Name) + } + } + } + } + $script:DefenderWorkbookKnownCatalogTables = $set + } + return ,$script:DefenderWorkbookKnownCatalogTables +} + +function Get-DefenderWorkbookImplicitFunctions { + param( + [AllowNull()][object]$Value, + [AllowNull()][System.Collections.Generic.HashSet[string]]$AllowedBareTables = $null, + [AllowNull()][System.Collections.Generic.HashSet[string]]$KnownFunctionAliases = $null + ) + if ($Value -is [pscustomobject]) { + foreach ($property in $Value.PSObject.Properties) { + if ($property.Name -ceq 'query' -and $property.Value -is [string]) { + $queryText = $property.Value + if ($Value.queryType -eq 18 -and $queryText.TrimStart().StartsWith('{')) { + try { $settings = $queryText | ConvertFrom-Json -ErrorAction Stop } + catch { throw 'Malformed Defender Workbook query wrapper.' } + if ($settings -isnot [pscustomobject] -or + $settings.__advancedHuntingSettings -isnot [bool] -or + -not $settings.__advancedHuntingSettings -or $settings.query -isnot [string] -or + @($settings.PSObject.Properties.Name | Where-Object { + $_ -cnotin @('__advancedHuntingSettings', 'query', 'selectedWorkspaceIds') + }).Count -gt 0) { + throw 'Unrecognized Defender Workbook query wrapper.' + } + $queryText = $settings.query + } + # Ignore single-line // comments first (/* */ is NOT a KQL comment). + $noComments = [regex]::Replace($queryText, '//[^\r\n]*', ' ') + $strPat = '@"(?:""|[^"])*"|@''(?:''''|[^''])*''|"(?:\\.|[^"\\])*"|''(?:\\.|[^''\\])*''' + $bracketPat = "(?s)\[\s*($strPat)\s*\]" + $tableCallPat = "(?i)(?= 3.1.0 and < 4.0.0' } else { '< 4.0.0' } + $minimum = if ($ContentKind -eq 'Solution') { [version]'3.2.0.0' } elseif ($ContentKind -eq 'CustomDetection') { [version]'3.1.0.0' } else { [version]'0.0.0.0' } + $range = if ($ContentKind -eq 'Solution') { '>= 3.2.0 and < 4.0.0' } elseif ($ContentKind -eq 'CustomDetection') { '>= 3.1.0 and < 4.0.0' } else { '< 4.0.0' } $parsed = $null $valid = $Version -cmatch '\A[0-9]+\.[0-9]+(?:\.[0-9]+){0,2}\z' -and [version]::TryParse($Version, [ref]$parsed) @@ -53,11 +53,11 @@ function Assert-V31TargetVersion { param([AllowNull()][AllowEmptyString()][string]$Version) if ($Version -cnotmatch '\A(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\z') { - throw "V3.1 target version '$Version' must be canonical major.minor.patch in the range >= 3.1.0 and < 4.0.0." + throw "V3.1 target version '$Version' must be canonical major.minor.patch in the range >= 3.2.0 and < 4.0.0." } $parts = $Version.Split(".") - if ($parts[0] -cne "3" -or [System.Numerics.BigInteger]::Parse($parts[1]) -lt [System.Numerics.BigInteger]::One) { - throw "V3.1 target version '$Version' must be in the range >= 3.1.0 and < 4.0.0." + if ($parts[0] -cne "3" -or [System.Numerics.BigInteger]::Parse($parts[1]) -lt 2) { + throw "V3.1 target version '$Version' must be in the range >= 3.2.0 and < 4.0.0." } } diff --git a/Tools/SentinelToXDRMigration/FOLLOW_UPS.md b/Tools/SentinelToXDRMigration/FOLLOW_UPS.md index 6c43720e92d..21149684e6c 100644 --- a/Tools/SentinelToXDRMigration/FOLLOW_UPS.md +++ b/Tools/SentinelToXDRMigration/FOLLOW_UPS.md @@ -17,6 +17,13 @@ themselves establish a toolkit fix is required. ## Candidate follow-ups +- **Make workbook depends on parsers during packaging.** Approved toolkit + implementation work: bind reviewed workbook function prerequisites to actual + `Microsoft.OperationalInsights/workspaces/savedSearches` resources in the + same enclosing ARM template and add workbook-installer `dependsOn` edges. + Required parser installation is mandatory; same-resource parser overwrites + are expected. Keep this tracked until packaging regressions confirm exact + resource mapping, prerequisite freshness, and unconditional installation. - **Validate entity identity bindings.** Reject filename-only bindings and synthetic text hashes where a genuine content SHA-1/SHA-256 is required; define and test the supported identity contract. @@ -27,6 +34,15 @@ themselves establish a toolkit fix is required. analytic rules, hunting queries, and other versioned content the version of the solution release containing the update. Leave unchanged content versions intact and keep source provenance consistent with the actual source version. +- **Default hourly Sentinel rules to the native XDR lookback.** For supported + rules with a one-hour Sentinel frequency and lookback, retain the hourly + Custom Detection frequency (`PT1H`) and accept the native four-hour + ingestion-based lookback without pausing for timing-only approval. Four + hours is the lookback, not the execution frequency. Preserve the query + without adding a time filter, and save the source/target timing, exact + source-document and target-query hashes, applied default, and timing or + aggregation differences as evidence. Do not claim parity or clear unrelated + mapping, dependency, KQL, validation, packaging, or environment-write gates. - **Reconcile Qualification readiness.** Review existing authorization and access checks, then separately assess deployment and ingestion permission readiness before proposing new implementation. @@ -36,16 +52,53 @@ themselves establish a toolkit fix is required. - **Improve operator visibility.** Consider concise progress and preflight reporting, a workflow diagram, watchdog summaries, and suppression of no-change notices. +- **Add scheduled migration status reminders.** Consider a scheduled agent + that reminds the top coordinator to surface what is being worked on, what + comes next, and why a step is taking longer, including larger detection + batches. For example: "Applying your decisions to the XDR detections before + validation." Bind updates to actual workflow and subagent evidence; never + invent progress, completion estimates, or approvals. Coordinate with the + existing single-owner watchdog, avoid repetitive notices, and do not create + a second workflow executor. Review the cadence and design separately before + implementing an agent or enabling a schedule. +- **Collect review feedback in one consolidated batch.** Prefer one solution-wide + review packet and one feedback request over serial per-rule or per-workbook + approval prompts. Inventory all outstanding detection, workbook, dependency, + timing, entity, version, and content findings before asking. Group repeated + findings, retain exact affected identities, query pointers and hashes, and + offer accept/change/defer with per-item exceptions. Preserve previous current + decisions and record all new decisions as evidence before applying them + through the supported deterministic commands. Keep technical failures and + deferred items visible; batch approval does not establish runtime correctness + or parity, or authorize packaging or environment writes. This is the preferred + review experience, not a claim that a native unified batch-review CLI exists. +- **Guide users to the next migration blocker review.** End migration status + updates with a focused next-step recommendation, such as "Next migration step + can be reviewing blockers A and B." Identify the next blockers and recommended + review actions rather than overwhelm users with the full issue list, so they + can stay focused on progressing through the migration. +- **Make sure validation scripts handle XDR Detections and Defender Workbooks + correctly.** Verify that the official solution validation runner checks both + content families, reports applicable schema and provenance failures, and keeps + skipped or unsupported checks visible rather than treating them as passes. - **Forward explicit approval safely.** Route chat approval to the accountable owner while preserving the exact approved operation, scope, and bindings. - **Clarify execution ownership.** Use appropriate delegation with one owner and no duplicate executors for the same operation. - **Set release-note dates predictably.** Default to today's date unless the user explicitly supplies an override. -- **Evaluate an Author and Validate mode.** A separately approved, - deployment-only check in a non-production environment could verify - deployment mechanics. It would not constitute functional/runtime - qualification, alert-parity evidence, or broader Qualification. +- **Change the validation tools to verify ARM templates through lab deployment.** + Make deployment of the generated `mainTemplate` to an explicitly approved + non-production lab the default ARM validation route for both Authoring and + Qualification, skipping ARM-TTK on that route and using deployment results + to verify ARM template deployment validity. Allow Authoring to select + ARM-TTK instead of lab deployment verification. Save deployment findings, + including failures and target/artifact bindings, as workflow evidence. + Ask in advance during preflight for preapproval to deploy the solution ARM + templates to the selected lab, stating the expected content items: + X Sentinel rules, Y workbooks, and Z custom detections. + Deployment verification alone does not establish functional/runtime + qualification or alert parity. Any future environment write remains separately gated by explicit approval immediately before the exact operation. No cloud action is authorized by this diff --git a/Tools/SentinelToXDRMigration/README.md b/Tools/SentinelToXDRMigration/README.md index 282fad85a2a..e01d9c868d7 100644 --- a/Tools/SentinelToXDRMigration/README.md +++ b/Tools/SentinelToXDRMigration/README.md @@ -1,7 +1,8 @@ # Microsoft Sentinel to Defender XDR migration toolkit -This toolkit converts the analytic rules in a Microsoft Sentinel solution into -versioned Defender XDR Custom Detection YAML files. +This toolkit converts Microsoft Sentinel analytic rules into versioned Defender +XDR Custom Detection YAML files and provides offline workbook conversion with +candidate V3.1 Defender Workbook packaging. See [tracked follow-up areas](FOLLOW_UPS.md) for curated capability and operator-experience work that may be considered separately. @@ -13,8 +14,9 @@ XDR solutions at scale and with quality. Once a user defines the solution, the toolkit should prepare the best-supported migration candidate, run the applicable automated checks, and clearly surface assumptions and evidence for human review. For optional Qualification, the user must also define an approved -non-production test environment and provide the required permissions. Offline -Authoring does not require a test environment. +non-production test environment and provide the required permissions. Legacy +offline Authoring does not require a test environment. New CLI runs +default to the versioned lab ARM-validation route described below. The intended operator flow is: @@ -47,9 +49,9 @@ The intended operator flow is: 5. **Package and, only if approved, qualify.** Build the package from the reviewed, current candidate inputs through the V3.1 route. Packaging, deployment readiness, and runtime qualification remain separate. Deployment - and ingestion are Qualification-only writes requiring explicit, - exact-scope approval immediately before the operation; candidate acceptance - does not authorize them. + requires explicit approval under the selected versioned ARM route; ingestion + remains Qualification-only. Neither candidate generation nor review acceptance + authorizes environment writes. **Implementation status:** This is the desired product direction, not a claim that end-to-end autonomous semantic mapping, general uncertainty handling, @@ -60,6 +62,263 @@ Unresolved review and technical gates still block packaging. Existing commands do not yet provide the general batch-review decisions described above; see the capability matrix below for implemented behavior and pending work. +## Dependency provenance + +Conversion remains offline. Generated YAML declares telemetry prerequisites in +`contentProvenance.requiredWorkloads` (`workload`, `dataTypes`) and +`contentProvenance.requiredDataConnectors` (`connectorId`, `dataTypes`), preserving +the separate source/conversion records and source connector metadata. +Offline validation requires these fields and rejects unresolved +`dependencyReviewRequired` records. Workloads use the toolkit vocabulary +`mde`, `mdo`, `mdi`, `mda`, `mdc`, `entraid`, `purview`, `appgovernance`, +`sentinel`, `mtp`; this is not a claimed public Microsoft enum. + +Use the `sentinel-xdr-dependency-helper` skill for explicitly approved +lab-assisted review in either profile. `dependency-plan --solution ` +proposes records without network access or YAML writes. Import a complete, +successful, workspace/query-bound metadata family using +`--results --workspace-id `. The source-validation +SDK requests `Prefer: include-dataSources=true` and compares returned underlying +tables with YAML. It retains its existing Qualification execution/consent gates. +1-10 distinct tables use API metadata; successful missing metadata or more than +10 tables uses the conservative scanner. Query/partial/provider failures are +never empty-success fallbacks. Unknown dependencies require user review. +Tables added by the converted query are also included through a separate +offline reference scan. Source API lineage is not Graph input-lineage evidence. +Shared `SecurityAlert` never establishes a specific connector or Defender +product by itself. Returned output `tables` are not input lineage. + +Packaging and actual Graph deployment stamp the four resolved identity/version +keys, without adding them to authored YAML. V3.1 installation and stored +installation templates carry the same lightweight record; the Content Hub +registration additionally contains the authored dependency arrays. Direct +Graph deployment obtains real package identity from solution Data/Metadata and +requires matching provenance on readback. Unbound Graph payload helpers are +planning-only. Offline emission does not prove live Graph/Content Hub support +for this candidate contract. Reviewed YAML dependency edits invalidate package +evidence; reconversion preserves declarations only while source/target queries +and source connector metadata remain unchanged. + +## Defender Workbook candidate contract + +Workbook support retains the complete `Notebook/1.0` document under +`XDR Workbooks/.json`. Source `Workbooks/` files are not +rewritten. Authored `xdrMetadata` distinguishes the canonical content GUID from +the stable installation GUID; its content version inherits the selected solution +release. The Notebook format version and original source metadata version remain +separate. + +`sentinel_xdr_migration.workbooks` exposes `inventory_workbooks`, +`convert_workbooks`, `validate_workbooks`, and `selected_workbook_files`. +Conversion is offline. Reviewed resolution records bind each original query +JSON pointer to its exact raw query hash, complete source tables, and evidence. +Unknown dependencies, unreviewed saved functions, unsupported providers, stale +resolution, and unresolved review findings remain blockers. The converter records all +source items in `workbooks-manifest.json`, separate from detection `manifest.json`. +Queries removed as proven scope-only helpers remain in provenance. + +The explicit solution Data `XDR Workbooks` array selects deployable content. +An empty array selects none; packaging never repopulates it by scanning files. +V3.1 emits workbook installation and Content Hub registration together under +`DeployDefenderWorkbook=false`, independent of Sentinel `DeployWorkbook`. +Deployment copies omit authored `xdrMetadata` and carry exactly the four resolved +package/content identity and version keys. Full registration also retains +`requiredWorkloads` and `requiredDataConnectors`. Layout and ARM-looking literals +are preserved; literal escaping applies only to package copies. + +### Workbook gap review and workspace functions + +Create a read-only proposal with +`workbook-dependency-plan --solution --run-id `. +The JSON plan carries `planSha256`, complete source/query inventory, connector +alternatives, and function candidates. Optional `--results +--workspace-id ` imports approved lab evidence; planning +and import never execute queries or authenticate. + +Apply explicit decisions with `workbook-review --solution --run-id +--plan --decisions --plan-sha256 --actor +--reference --output-dir `. +The review emits supported conversion configuration and a review receipt, not +source workbook or parser edits. Use the emitted configuration with `convert`. +`convert --overwrite-workbooks` replaces only unchanged, hash-bound managed +workbook drafts; it never authorizes overwriting existing detections. The +ordinary `--overwrite` option remains a separate, explicit content decision. + +Present dependency gaps before asking for acceptance: show the workbook, +query/picker JSON pointer, exact query hash, proposed data types, evidence, +workload mappings, connector alternatives, and remaining technical failures. +Offer static/manual review or explicitly approved lab discovery in either +Authoring or Qualification. Static scanner proposals are not accepted evidence. +Lab metadata identifies underlying inputs, not output columns or installed +connectors. Parameterized queries need explicit bindings before execution; +never send unresolved workbook placeholders or silently choose a time range. + +The reviewer can accept, change, or defer each proposal. Preserve separate +per-query records even when identical proposals are accepted as a batch. +Changed query templates or prerequisite definitions invalidate prior reviews. +Deferral leaves a gap unresolved; acceptance cannot waive query errors, +unsupported providers, incomplete source evidence, or runtime/UI gates. + +Where reviewed connector alternatives satisfy the same requirement, prefer +the modern connector over a deprecated route. Cite authoritative recommendation +or lifecycle evidence; do not infer deprecation from a connector's name, require +both alternatives by default, or claim that a selected connector is installed. + +[Sentinel workspace functions are supported in Advanced Hunting and workbooks](https://learn.microsoft.com/defender-xdr/advanced-hunting-custom-functions#custom-functions-imported-from-microsoft-sentinel-log-analytics), +but not in Custom Detection rules. Preserve reviewed function invocations +instead of requiring inlining solely because the source is a workspace function. +Authored `contentProvenance.requiredWorkspaceFunctions` records bind each +`functionAlias` to its packaged `sourcePath` and exact `sourceSha256`. +Function aliases are not table names: underlying source data types require +separate evidence. The connected workspace, access, nested dependencies, +external data, and exact parameterized execution remain runtime concerns. +`_GetWatchlist` is a supported Sentinel built-in available in AH, not a custom +parser to install or an unsupported-function gap. Preserve its calls and +review underlying data inputs separately; only actual execution/data failures +are findings. +`Watchlist` is platform reference data, not connector-fed telemetry. Retain it +in raw source-lineage evidence when present, but do not propose a workload or +connector telemetry requirement for it or ask the user to install a connector. +This exemption does not hide unrelated unknown data types or actual query errors. + +Distinguish required functions from reviewed optional vendor branches. A missing +optional fuzzy-union branch can leave the query usable but its coverage incomplete. +Keep warnings visible; unavailable branches are not verified zero-event results. +Offer keep, remove, or defer. A removal request requires an explicit parser-change +preview and fresh review/validation; it must not silently delete shared parser +resources, unrelated branches, or panels. Catalog joins may reintroduce removed +products and need coordinated changes. + +V3.1 installs required workspace parsers unconditionally and orders workbook +installation after their actual `Microsoft.OperationalInsights/workspaces/savedSearches` +resources in the same enclosing ARM template. Parser content-template registration +alone is not installation. Missing, stale, ambiguous, or conditional required +resources block packaging. Full workbook registration retains function prerequisites; +the installed notebook retains only the identity/version quartet. ARM ordering +does not establish AH propagation, query correctness, or provider/UI acceptance. + +**Provider contract unverified:** generated templates and successful local tests +are candidate evidence, not accepted installation or publication evidence. +Independent, approved lab installation, registration/provenance readback, repeat +installation, and update evidence are required for contract acceptance. ARM +success alone does not establish those checks or workbook UI/query correctness. +The known provenance `packageId` ARM-TTK finding remains a failure; no broad +validator waiver or identifier rewrite is applied. + +The Microsoft-authored conversion slice is package-local and MIT-licensed. +It has no sibling-checkout, authentication, or live resolver dependency. +`_workbook_core/import-manifest.json` records the imported file hashes and +adaptations. Refreshing this slice is a deliberate maintainer change requiring +reviewed import evidence and the differential workbook fixture tests, not an +automatic runtime fetch. + +### Toolkit-owner managed update policy + +Toolkit-managed Defender Workbooks and XDR detections are updated by stable ID. +Same-ID overwrites are expected, including in the approved lab; a different +existing content hash or version is not an overwrite blocker. Managed workbooks +are intended to be consumed read-only, with personalization in separate copies +using different IDs. Toolkit updates must not overwrite those copies. + +This does not waive target/identity binding, effective-access checks, current +candidate evidence, deployment consent, immediate write confirmation or exact +post-write provenance/readback. Detections remain deployed disabled. It neither +authorizes other resources/deletion nor relaxes local source-file safeguards. +Read-only portal/provider enforcement remains something to verify, not a +capability established by this policy. See the +[canonical owner policy](agent-workflows/end-to-end-migration.md#toolkit-owner-policy-managed-same-id-updates). + +### Versioned ARM route and workbook QA + +New CLI runs persist workbook capability 1 and default to lab ARM validation in +either explicitly selected profile. Authoring can select +`--arm-validation-method arm-ttk`; existing runs retain their scope and method. +Direct library initialization retains legacy defaults unless explicitly opted +in. The [canonical workflow](agent-workflows/end-to-end-migration.md) defines +the versioned candidate, consent and acceptance boundaries. + +Static content validation precedes V3.1 candidate generation. A draft cannot +satisfy accepted Packaging. Independent approved ARM and workbook-contract +evidence is required; missing consent or provider support blocks the route, +without fallback or another version increment. Evidence-based metadata preflight +now checks the selected lab, authenticated ARM principal, effective control-plane +access, applicable role/deny assignments, disabled content and exact resource +identities. Unresolved evidence blocks dispatch rather than asserting readiness. +The MicrosoftSecurity extension authentication/write contract remains unverified, +so the current default adapter still blocks Defender workbook deployment. + +After `workflow-arm-plan`, use `workflow-arm-preflight-plan` to display the +bounded metadata read operations, then `workflow-arm-preflight-review` with +`--preflight-plan-sha256`, `--actor`, `--reference` and `--decision approved`. +Execute with `workflow-arm-preflight-execute --preflight-plan-sha256 `; +all commands also require the solution and explicit run ID. +`workflow-arm-preflight-revoke` withdraws that read approval. + +Metadata-read approval is separate from deployment consent and immediate write +confirmation. Same-target, same-operation scope can retain read consent across +managed same-ID content updates, but plans and execution evidence must be current. +Changed read URLs, identities or target require renewed approval. The default +reader uses the existing pinned Azure CLI profile and correlates actual ARM +responses with the same bearer. Consent explicitly permits acquisition/refresh +of the scoped ARM token through the existing cached identity, not sign-in, new +credentials, permission grants or Graph/AH token reuse. +Graph extension observations remain explicitly unverified, without borrowing +an Advanced Hunting token. Prior receipts are archived, blocked runs return +nonzero, and these commands never write to the environment. + +Workbook-only Qualification records `validationMode=workbook-content` before +candidate generation. Exact query and UI evidence is mandatory before final +Report, not before installation. Mixed solutions preserve detection runtime +prerequisites. Frozen workbook inventory and exclusions never inflate detection +coverage. + +Qualification query commands are `workbook-query-plan`, `workbook-query-review`, +`workbook-query-execute` and `workbook-query-revoke`. The reviewed Linux Machines +fixture runs 202 harness queries through independent Sentinel/AH providers with +exact result assertions and no ingestion. + +The optional `browser` extra provides Playwright. Provision it with a supported +interpreter and `python -m playwright install chromium` only when requested. +`workbook-ui-plan` additionally accepts a reviewed `--browser-config`, containing +exact DOM assertions, observed tenant/content/instance identities, session/error +guards, request/navigation allowlists, timeouts and viewport. All seven UI check +kinds and their expected values must be reviewed. No production selectors are +supplied or presumed verified. + +```bash +sentinel-xdr-migration workbook-ui-review \ + --solution "$SOLUTION" --run-id "$RUN_ID" \ + --plan "$PLAN_PATH" --plan-sha256 "$PLAN_SHA256" \ + --actor "$REVIEWER" --reference "$APPROVAL_REFERENCE" --decision approved + +sentinel-xdr-migration workbook-ui-execute \ + --solution "$SOLUTION" --run-id "$RUN_ID" \ + --plan "$PLAN_PATH" --plan-sha256 "$PLAN_SHA256" \ + --storage-state "Logs/sentinel-xdr-migration/$RUN_ID/private/browser-state.json" +``` + +Plan/input paths are solution-relative; keep private inputs under the selected +run's excluded evidence directory. Session state must already exist, match the +reviewed hash, be user-owned and have no group/other permissions or symlinks. +Ownership enforcement is currently POSIX-only. No automatic authentication, +installation, Save/Edit, permission changes or cleanup occurs. Missing browser +support fails explicitly. Current UI assertions compare reviewed expectations, +not automatically fetched API results. Network classifications are reviewed +attestations, not a browser-wide no-write sandbox. + +Receipts under `Logs/sentinel-xdr-migration//workbook-ui..json` +contain sanitized events and hashed observations; failed retries replace current +success and archive previous evidence. No screenshots, raw trace, HAR or session +state are exported. `workbook-ui-import` remains available for reviewed external +captures. Live portal execution still requires separate exact-scope approval. + +[Public Graph beta metadata](https://graph.microsoft.com/beta/$metadata) confirms +`security/workbooks` and the `microsoft.graph.security.workbook` model's +`name`/open `content` properties. Write permissions, ID allocation/update +semantics and runtime acceptance remain unverified. Direct Graph deployment +is not implemented; retain ARM until that contract is confirmed. A future +direct-API smoke test will not replace full V3.1 Content Hub acceptance. + ## Requirements - Python 3.11 or 3.12 @@ -1451,7 +1710,7 @@ evidence. An explicit target version may be selected instead of a bump when the release must move directly to a particular supported V3.1 version. Targets must be canonical numeric `major.minor.patch` versions in the inclusive range -`>=3.1.0` and `<4.0.0` (for example, `3.999.999`); the target does not have to +`>=3.2.0` and `<4.0.0` (for example, `3.999.999`); the target does not have to be the next consecutive version. Select the exact target at initialization, or bind it to the current package-input hash when planning a later release. Use `--version-bump none` with `--target-version`; a non-`none` bump and target @@ -1462,12 +1721,12 @@ sentinel-xdr-migration workflow-init ` --solution "Solutions\" ` --workflow-profile authoring ` --version-bump none ` - --target-version 3.1.2 + --target-version 3.2.1 sentinel-xdr-migration package-v3-1 ` --solution "Solutions\" ` --version-bump none ` - --target-version 3.1.2 + --target-version 3.2.1 ``` For a later release after inputs change, use @@ -1538,7 +1797,7 @@ exactly, stop rather than infer missing file hashes or changed paths. sentinel-xdr-migration workflow-plan-same-version-content-rebuild ` --solution "Solutions\" ` --run-id "" ` - --target-version 3.1.2 ` + --target-version 3.2.1 ` --expected-package-input-sha256 "" ` --changed-input "XDR Detections\.yaml" @@ -1559,7 +1818,7 @@ permitted: sentinel-xdr-migration workflow-plan-same-version-content-rebuild ` --solution "Solutions\" ` --run-id "" ` - --target-version 3.1.2 ` + --target-version 3.2.1 ` --expected-package-input-sha256 "" ` --changed-input "XDR Detections\.yaml" ` --failed-candidate-evidence "Logs\sentinel-xdr-migration\\packaging-arm-ttk-failure-handoff.json" ` @@ -1578,7 +1837,7 @@ to the plan command and, for a pre-existing plan, to every package attempt: sentinel-xdr-migration workflow-plan-same-version-content-rebuild ` --solution "Solutions\" ` --run-id "" ` - --target-version 3.1.2 ` + --target-version 3.2.1 ` --expected-package-input-sha256 "" ` --changed-input "XDR Detections\.yaml" ` --legacy-baseline-evidence "Logs\sentinel-xdr-migration\\same-version-content-rebuild\receipt-current-input-comparison.json" @@ -1683,11 +1942,14 @@ run/provider/target execution bindings; sidecar authorization cannot make an unbound report eligible. Current source structure accepts both Scheduled and NRT kinds with their corresponding required/forbidden scheduling fields. -Package-version selection is unchanged, but V3.1 -rejects effective solution and XDR release versions outside `[3.1.0, 4.0.0)`, +Package-version selection is unchanged, but V3.1 rejects effective solution +release versions outside `[3.2.0, 4.0.0)`, independent XDR Custom Detection +release versions outside `[3.1.0, 4.0.0)`, and other content-item release versions `>= 4.0.0`. Explicit bumps that select an out-of-range version fail without overwriting package files or persisting the bumped source version. These are release bounds, not ARM API/schema bounds. +Older source solutions require an explicitly selected supported target or bump; +no-bump rebuilds below the solution floor are rejected rather than promoted. XDR uses only its top-level YAML `version`, including when registration is disabled. Registration metadata and content product IDs use that same version. Data `XDR Detection Version`, solution version, and source provenance are not @@ -3076,7 +3338,7 @@ python -m unittest discover -s Tools\SentinelToXDRMigration\tests -p "test_*.py" ## Planned milestones -- workbook conversion; +- approved live Defender Workbook contract and query/UI qualification evidence; - solution migration dashboard; - optional retrieval-assisted guidance for schemas, prior art, and historical findings. diff --git a/Tools/SentinelToXDRMigration/agent-workflows/end-to-end-migration.md b/Tools/SentinelToXDRMigration/agent-workflows/end-to-end-migration.md index a42fdf44da4..3f9168b91a9 100644 --- a/Tools/SentinelToXDRMigration/agent-workflows/end-to-end-migration.md +++ b/Tools/SentinelToXDRMigration/agent-workflows/end-to-end-migration.md @@ -1,7 +1,8 @@ # Sentinel to Defender XDR migration workflow This is the canonical, agent-neutral workflow for authoring and migrating -Microsoft Sentinel solution content to Defender XDR Custom Detections. Agent +Microsoft Sentinel solution content to Defender XDR Custom Detections and +Defender Workbooks. Agent adapters must reference this document rather than redefine the process. Deterministic behavior belongs to: @@ -68,8 +69,141 @@ disabled-by-default, and exact-scope authorization gates remain in force. `Discovery -> Conversion -> Validation -> Packaging -> Report` -This is the ISV publishing workflow. It does not require a tenant, workspace, -deployment, mock ingestion, or alert-parity test. +Legacy offline Authoring does not require a tenant, workspace, deployment, mock +ingestion, or alert-parity test. New capability-version-1 CLI runs use the +candidate ARM lifecycle below; their default lab route requires an approved +target and independently verified deployment evidence before package acceptance. + +### Versioned workbook and candidate ARM lifecycle + +New CLI runs persist `workbookCapabilityVersion=1` and default to +`armValidationMethod=lab-deployment`, in either explicitly selected profile. +Authoring may explicitly choose `--arm-validation-method arm-ttk`. Existing +runs never silently acquire workbook scope or change their validation method; +direct library initialization retains legacy defaults unless capability 1 is +explicitly requested. + +The order is: + +`Discovery -> Conversion -> content/static Validation -> V3.1 candidate -> +remaining official checks -> selected ARM validation -> accepted Packaging` + +Keep frozen recursive workbook inventory and explicit selections separate from +detection coverage. Workbook-only solutions need no invented Analytic Rules, +rule IDs, runtime manifests, ingestion or alert parity. Exclusions remain +visible, and explicit empty `XDR Workbooks` selections stay empty. + +A candidate is not accepted packaging or a successful release. Preserve its +artifacts and consumed version action for same-version retries. Use the official +`build-and-validate.ps1` receipt bound to the current candidate. Lab validation +replaces only ARM-TTK; remaining applicable official checks are mandatory. +For selected Defender Workbooks, both methods additionally require independent +installation, registration/provenance readback and repeat-install/update +contract evidence. ARM-TTK alone cannot establish that contract. + +Versioned lab consent binds the selected run/lab, deployment operation and +allowed content/resource types. Hash/count changes within that scope require +fresh evidence, not automatically broader approval. Changed labs, operations +or type sets require renewed approval. Revocation blocks dispatch. The executor +also requires explicit immediate confirmation and verified access, disabled +content and ownership; consent never grants permissions or authorizes ingestion, +activation or cleanup. This is not a reinterpretation of legacy consent. + +**Defender workbook dispatch remains blocked:** the evidence-based preflight +adapter checks target, authenticated ARM principal, effective control-plane +access, applicable role/deny assignments, disabled content and exact identities, +but the MicrosoftSecurity extension authentication/write contract is unverified. +Unknown evidence blocks dispatch. Do not replace this blocker with asserted +readiness flags, infer support from a mock, reuse an unrelated Graph/AH token +as the extension identity, or treat local tests as provider acceptance. + +#### Candidate ARM metadata preflight + +After the current candidate deployment plan exists, display the exact read plan +with `workflow-arm-preflight-plan`. Obtain separate affirmative metadata-read +approval through `workflow-arm-preflight-review`, bound to the displayed +`--preflight-plan-sha256` and recorded actor/reference. Execute the current plan +through `workflow-arm-preflight-execute`; withdraw approval through +`workflow-arm-preflight-revoke`. All commands require the explicit run ID. + +Read consent binds the run, target and exact operation scope independently of +candidate/template hashes. Managed same-ID body/version changes require a fresh +plan and evidence, not automatically renewed approval of unchanged reads. +Changed IDs, URLs, operations or targets require renewed read approval. +Deployment consent and immediate write confirmation remain separate gates. + +The production reader uses existing cached authentication in the pinned Azure +CLI profile, makes only planned metadata GETs, and correlates successful ARM +observations with that same authenticated bearer. Token claims or a local +account alone do not prove effective access. It never signs in, grants +permissions or attempts Graph extension reads with unverified credentials. +Read consent explicitly permits scoped ARM token acquisition/refresh through +the existing cached identity, not new credentials or Graph/AH token access. +Receipts preserve blocked/unknown findings and prior attempts; execution +rechecks current consent and candidate freshness. No preflight operation +authorizes installation, activation, ingestion, cleanup or browser execution. + +New workbook-only Qualification runs record `validationMode=workbook-content` +before packaging, with query/UI checks not run. They require current independent +query and UI evidence before final Report. Mixed solutions retain all detection +runtime prerequisites before candidate generation; workbook UI proof remains +post-installation. A passed ARM candidate contract can supply workbook-only +Deployment evidence; do not run a detection-only deployer or invent detections. + +### Toolkit-owner policy: managed same-ID updates + +Toolkit-managed Defender Workbooks and XDR Custom Detections use stable-ID +upserts. Deploying approved content to the same ID intentionally replaces the +existing managed content; this is expected update behavior, including in labs. +Do not block an intended same-ID update merely because the remote content, +version or hash differs from the candidate or a previous deployment. + +The intended workbook consumption model is read-only managed content. +Personalization uses a separate copy with its own identity; toolkit updates +must not discover or overwrite those copies. This policy defines the product +contract, not proof that the preview provider or portal currently enforces +read-only behavior. + +Preflight must bind the selected target and intended resource identities, +verify effective access, and reject wrong returned IDs, ambiguous identity +mappings or operations outside the approved scope. A managed same-ID update +does not itself require an additional overwrite decision or previous-content +hash match. It still requires current candidate/review evidence, applicable +deployment consent and immediate write confirmation. Lab overwrite approval +does not authorize production, other identities, activation, deletion or +permission grants; XDR detections remain deployed disabled. + +Keep exact post-write provenance/readback and package acceptance checks. +This managed cloud-update policy does not relax local source-file protection, +edited generated-file overwrite safeguards, or the QA prohibition on Save/Edit. + +### Workbook Qualification and browser QA + +`workbook-query-plan/review/execute` bind the exact reviewed fixture harness +family to the locked Qualification target. Linux Machines has 202 queries +across independent Sentinel and Advanced Hunting providers. Exact result +assertions, including schema, cells, multiplicity and ordering, are mandatory. +This fixture route does not authorize ingestion or prove live-panel telemetry. + +`workbook-ui-plan/review/execute` use a reviewed, hash-bound browser configuration, +an existing private session and independently observed tenant, canonical content +and installed instance IDs. Require all seven check kinds: pickers, dependent +refresh, filters, drill-down, reload, layout and conditional visibility. +Selectors, expected values, request URLs, navigation and timeouts require review; +no guessed production DOM bindings or automatic sign-in. Save/Edit and cleanup +are prohibited. Browser/network checks are not a browser-wide no-write sandbox. + +The optional `browser` extra provides Playwright. UI assertions compare reviewed +expectations, not automatically fetched API results. Query evidence, browser +evidence and full ARM/Content Hub acceptance remain separate. Existing +`workbook-ui-import` accepts explicitly reviewed external captures; it does not +launch a browser. Live browser execution needs separate exact-scope approval. + +Public Graph beta metadata exposes `security/workbooks` and the security +workbook `name`/open `content` model. It does not establish write permissions, +create/update ID semantics or provider acceptance. Retain the ARM candidate +route until those are verified; a future direct-API smoke test cannot replace +full V3.1 package acceptance. #### Optional explicitly approved Advanced Hunting read validation @@ -92,6 +226,75 @@ blocks dispatch. This route cannot satisfy Qualification's source/runtime, target, consent, dispatch, completeness, schema, packaging, deployment, or parity gates; source SDK validation remains Qualification-only. +#### Dependency provenance validation + +Conversion is always offline. Each XDR YAML requires +`contentProvenance.requiredWorkloads` and `requiredDataConnectors` record arrays. +Structural validation checks the workload vocabulary and record shape; +unresolved `dependencyReviewRequired` records block validation. +The legacy `conversion.requiredWorkloads` labels are not deployment dependencies. + +The `sentinel-xdr-dependency-helper` skill may help users fill and review these +records from a Log Analytics lab in either profile. This optional validation +capability requires fresh explicit read-only approval for the exact original +queries and selected tenant/subscription/workspace. It does not reinterpret +Authoring Advanced Hunting consent, select Qualification, or broaden SDK, +deployment, authentication, ingestion, or parity authorization. +Prefer an advertised official Triage capability supporting input metadata; +otherwise use the approved REST read route. Request +`Prefer: include-dataSources=true`. The offline `dependency-plan` importer +checks exact current query hashes and workspace binding before comparing YAML. +Use API input metadata for 1-10 distinct tables, otherwise a recorded +conservative scanner fallback after a successful query. Failures and partial +results remain failures. Unknown tables/connectors require user input. + +Keep dependency arrays in authored provenance; do not add the deployment +package/content identity quartet to YAML. V3.1 packaging and actual Graph +deployment resolve that quartet from package metadata and canonical content +identity/version. Only full Content Hub registration includes dependencies. +Preserve disabled status, source content and classification. + +#### Workbook dependency review + +For a workbook gap, show each query/picker pointer and exact hash, source evidence, +proposed data types, workload/connector mappings, and unresolved technical issues. +Offer static/manual review or separately approved bounded lab metadata discovery +in either profile. Acceptance, changes, and deferrals are explicit and hash-bound; +batch acceptance must retain individual query records. Static suggestions are not +automatically reviewed facts. Query errors, partial failures, stale evidence, +unknown functions, and unsupported providers remain blockers after acceptance. +Parameterized lab queries need approved concrete bindings, not unresolved workbook +tokens or an invented time range. Source metadata is not output-schema lineage or +proof that a connector is installed. + +When reviewed connector alternatives are equivalent for the declared requirement, +recommend the modern connector over a deprecated route, citing recommendation or +lifecycle evidence. Do not require both merely because both appear in the solution. + +Preserve explicitly reviewed Sentinel workspace function invocations for workbooks +and AH queries; this support does not apply to Custom Detections. Standard +built-in ASIM parsers starting with `_` (such as `_ASim_*` and `_Im_*`) are +accepted by Defender XDR Advanced Hunting and can be used in both Custom +Detections and Defender Workbooks; do not reject them as unsupported +workspace-saved functions. Bind required +functions to packaged definition paths, aliases, and current file hashes in +`contentProvenance.requiredWorkspaceFunctions`. Resolve underlying tables separately. +Treat `_GetWatchlist` as a supported built-in, not a missing workspace parser, +an installation prerequisite, or an AH function-support blocker. +Keep `Watchlist` platform-reference inputs in raw lineage, but omit them from +telemetry data types in required workload/connector records. They are not fed +by a data connector and must not cause connector-discovery prompts. +During V3.1 packaging, required parser installation is unconditional and the outer +workbook installer depends on the actual saved-search resources in the enclosing +template. Content registration is not installation; deployment order is not AH +visibility or runtime acceptance evidence. + +Reviewed optional vendor branches may be absent with visible warnings and incomplete +coverage. Offer keep/remove/defer; removal requires a scoped parser-change preview, +explicit approval, current definition evidence and fresh validation. Never silently +remove shared parser resources or treat missing branches as confirmed no telemetry. +Accepting a removal request does not itself establish that the function was edited. + #### Optional resumable full-inventory authoring The CLI also provides a separate, explicit `--yolo` mode for durable local @@ -180,21 +383,21 @@ does not waive technical or entity-review gates: unresolved `reviewRequired` flags continue to block packaging. An author may select a canonical target version instead of a patch/minor/major -bump. The target must satisfy `>=3.1.0` and `<4.0.0` and need not be consecutive +bump. The target must satisfy `>=3.2.0` and `<4.0.0` and need not be consecutive with the current version. For example, an initial release may go directly from -`3.0.7` to `3.1.2`: +`3.0.7` to `3.2.1`: ```powershell sentinel-xdr-migration workflow-init ` --solution "Solutions\" ` --workflow-profile authoring ` --version-bump none ` - --target-version 3.1.2 + --target-version 3.2.1 sentinel-xdr-migration package-v3-1 ` --solution "Solutions\" ` --version-bump none ` - --target-version 3.1.2 + --target-version 3.2.1 ``` For changed package inputs, plan the exact target against the current input @@ -207,6 +410,11 @@ cannot be combined with a non-`none` bump. The persisted target/action and release plan retain the same review, freshness, campaign, and one-time consumption protections as bump actions. +The resulting solution release must be `>=3.2.0` and `<4.0.0` for V3.1 +packaging, including no-bump rebuilds and pipeline-calculated versions. +An older source version is not silently promoted; select a supported target or +bump explicitly. Independent content-version checks remain unchanged. + For offline Authoring only, if the successful package is already at the approved version but its release-note row is missing, use `workflow-plan-release-notes-refresh` rather than selecting another version @@ -1123,8 +1331,10 @@ full inventory. If initialization selected `version-bump=none` but a first versioned release is required, inspect the current package-input SHA-256 and explicitly plan one `patch`, `minor`, or `major` action against that exact hash before invoking V3.1. Do not reuse stale selection, skip review gates, or -consume a second version action. Qualification still requires its exact -runtime-bound selection and all Qualification gates before packaging. +consume a second version action. Detection-bearing Qualification still requires +its exact runtime-bound selection before packaging. The versioned workbook +route described above requires workbook query/UI evidence after installation, +before final Qualification completion. ### Advance deployment-scope planning and consent diff --git a/Tools/SentinelToXDRMigration/pyproject.toml b/Tools/SentinelToXDRMigration/pyproject.toml index 9f867b88105..3ddae2ecfe8 100644 --- a/Tools/SentinelToXDRMigration/pyproject.toml +++ b/Tools/SentinelToXDRMigration/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "sentinel-xdr-migration" version = "0.1.0" -description = "Convert Microsoft Sentinel analytic rules into Defender XDR Custom Detection YAML." +description = "Convert Microsoft Sentinel analytic rules and workbooks into Defender XDR solution content." requires-python = ">=3.11,<3.13" classifiers = [ "Programming Language :: Python :: 3.11", @@ -18,9 +18,15 @@ dependencies = [ "jsonschema>=4.0,<5" ] +[project.optional-dependencies] +browser = ["playwright>=1.49,<2"] + [project.scripts] sentinel-xdr-migrate = "sentinel_xdr_migration.cli:main" sentinel-xdr-migration = "sentinel_xdr_migration.cli:main" [tool.setuptools.packages.find] include = ["sentinel_xdr_migration*"] + +[tool.setuptools.package-data] +"sentinel_xdr_migration._workbook_core" = ["data/*.json", "LICENSE", "import-manifest.json"] diff --git a/Tools/SentinelToXDRMigration/schema/workflow-state.schema.json b/Tools/SentinelToXDRMigration/schema/workflow-state.schema.json index 1140afb1140..22c78e70251 100644 --- a/Tools/SentinelToXDRMigration/schema/workflow-state.schema.json +++ b/Tools/SentinelToXDRMigration/schema/workflow-state.schema.json @@ -3,6 +3,29 @@ "title": "Sentinel to XDR workflow state", "type": "object", "additionalProperties": false, + "allOf": [ + { + "if": {"properties": {"context": {"required": ["workbookCapabilityVersion"]}}}, + "then": { + "required": ["workbooks"], + "properties": {"context": {"required": ["armValidationMethod"]}} + }, + "else": { + "not": {"required": ["workbooks"]}, + "properties": {"context": {"not": {"anyOf": [ + {"required": ["armValidationMethod"]}, {"required": ["workbookFullInventory"]}, + {"required": ["workbookQualificationAfterArm"]} + ]}}} + } + }, + { + "if": {"properties": {"context": {"required": ["workbookQualificationAfterArm"]}}}, + "then": {"properties": {"context": {"properties": { + "workflowProfile": {"const": "qualification"}, + "targetContextLocked": {"const": true} + }, "required": ["targetContextLocked"]}}} + } + ], "required": [ "schemaVersion", "solution", @@ -13,6 +36,49 @@ "stages" ], "properties": { + "workbooks": { + "type": "object", + "additionalProperties": false, + "required": ["schemaVersion", "inventory", "inventorySha256", "selection", "selectionSha256", "selectionExplicit"], + "properties": { + "schemaVersion": {"const": "1.0.0"}, + "inventorySha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "selectionSha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "selectionExplicit": {"type": "boolean"}, + "inventory": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["path", "sha256"], + "properties": { + "path": {"type": "string", "minLength": 1}, + "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"} + } + } + }, + "selection": { + "type": "object", + "additionalProperties": false, + "required": ["included", "excluded"], + "properties": { + "included": {"type": "array", "uniqueItems": true, "items": {"type": "string", "minLength": 1}}, + "excluded": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["path", "reason"], + "properties": { + "path": {"type": "string", "minLength": 1}, + "reason": {"type": "string", "minLength": 1} + } + } + } + } + } + } + }, "runId": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]{0,95}$", @@ -79,6 +145,10 @@ "targetContextLocked": { "type": "boolean" }, + "workbookQualificationAfterArm": { + "const": true, + "description": "New workbook-only Qualification runs validate content before ARM acceptance; query/UI proof remains mandatory before completion." + }, "versionBump": { "enum": [ "patch", @@ -103,6 +173,15 @@ }, "profileSelectionConfirmed": { "type": "boolean" + }, + "workbookCapabilityVersion": { + "const": 1 + }, + "armValidationMethod": { + "enum": ["arm-ttk", "lab-deployment"] + }, + "workbookFullInventory": { + "const": true } } }, diff --git a/Tools/SentinelToXDRMigration/schema/xdr-detection.schema.json b/Tools/SentinelToXDRMigration/schema/xdr-detection.schema.json index 0c1fa21313b..9201d955ed3 100644 --- a/Tools/SentinelToXDRMigration/schema/xdr-detection.schema.json +++ b/Tools/SentinelToXDRMigration/schema/xdr-detection.schema.json @@ -122,9 +122,45 @@ "type": "object", "required": [ "source", - "conversion" + "conversion", + "requiredWorkloads", + "requiredDataConnectors" ], "properties": { + "requiredWorkloads": { + "description": "Authored runtime telemetry prerequisites, not legacy conversion workload labels.", + "type": "array", + "items": { + "type": "object", + "required": ["workload", "dataTypes"], + "properties": { + "workload": { + "enum": ["mde", "mdo", "mdi", "mda", "mdc", "entraid", "purview", "appgovernance", "sentinel", "mtp"] + }, + "dataTypes": { + "type": "array", + "items": {"type": "string", "minLength": 1} + } + }, + "additionalProperties": false + } + }, + "requiredDataConnectors": { + "type": "array", + "items": { + "type": "object", + "required": ["connectorId", "dataTypes"], + "properties": { + "connectorId": {"type": "string", "minLength": 1}, + "dataTypes": { + "type": "array", + "items": {"type": "string", "minLength": 1} + } + }, + "additionalProperties": true + } + }, + "dependencyReviewRequired": {"type": "boolean"}, "source": { "type": "object", "required": [ diff --git a/Tools/SentinelToXDRMigration/schema/xdr-workbook.schema.json b/Tools/SentinelToXDRMigration/schema/xdr-workbook.schema.json new file mode 100644 index 00000000000..3441a69c598 --- /dev/null +++ b/Tools/SentinelToXDRMigration/schema/xdr-workbook.schema.json @@ -0,0 +1,107 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Authored Defender XDR Workbook", + "description": "Complete Notebook with additive authoring identity and provenance. Not an installed deployment payload.", + "type": "object", + "required": ["version", "items", "xdrMetadata", "contentProvenance"], + "properties": { + "version": {"const": "Notebook/1.0"}, + "items": {"type": "array", "items": {"type": "object"}}, + "xdrMetadata": { + "type": "object", + "required": ["contentId", "instanceId", "version", "schemaVersion", "displayName"], + "properties": { + "contentId": {"$ref": "#/$defs/guid"}, + "instanceId": {"$ref": "#/$defs/guid"}, + "version": {"type": "string", "pattern": "^3\\.[1-9][0-9]*\\.(0|[1-9][0-9]*)$"}, + "schemaVersion": {"const": "1.0.0"}, + "displayName": {"type": "string", "minLength": 1} + }, + "additionalProperties": false + }, + "contentProvenance": { + "type": "object", + "required": ["source", "conversion", "requiredWorkloads", "requiredDataConnectors", "dependencyReviewRequired"], + "properties": { + "source": { + "type": "object", + "required": ["path", "sha256"], + "properties": { + "path": {"type": "string", "pattern": "^Workbooks/[^\\\\]+\\.[jJ][sS][oO][nN]$"}, + "sha256": {"$ref": "#/$defs/sha256"}, + "contentId": {"type": ["string", "null"]}, + "originalMetadataVersion": {"type": ["string", "null"]} + } + }, + "conversion": { + "type": "object", + "required": ["tool", "version", "reviewRequired", "queries"], + "properties": { + "tool": {"const": "sentinel-xdr-migration"}, + "version": {"type": "string", "minLength": 1}, + "reviewRequired": {"const": false}, + "queries": {"type": "array", "items": {"$ref": "#/$defs/query"}} + } + }, + "requiredWorkloads": { + "type": "array", + "items": { + "type": "object", + "required": ["workload", "dataTypes"], + "properties": { + "workload": {"enum": ["mde", "mdo", "mdi", "mda", "mdc", "entraid", "purview", "appgovernance", "sentinel", "mtp"]}, + "dataTypes": {"$ref": "#/$defs/dataTypes"} + } + } + }, + "requiredDataConnectors": { + "type": "array", + "items": { + "type": "object", + "required": ["connectorId", "dataTypes"], + "properties": { + "connectorId": {"type": "string", "minLength": 1}, + "dataTypes": {"$ref": "#/$defs/dataTypes"} + } + } + }, + "requiredWorkspaceFunctions": {"type": "array", "items": {"$ref": "#/$defs/workspaceFunction"}}, + "dependencyReviewRequired": {"const": false} + }, + "additionalProperties": false + } + }, + "$defs": { + "workspaceFunction": { + "type": "object", + "required": ["functionAlias", "sourcePath", "sourceSha256"], + "properties": { + "functionAlias": {"type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]*$"}, + "sourcePath": {"type": "string", "pattern": "^(Parsers|Workspace Functions)/(?!/|\\.{1,2}/)(?!.*(?:/\\.{1,2}/|//))[^\\\\]+\\.yaml$"}, + "sourceSha256": {"$ref": "#/$defs/sha256"} + }, + "additionalProperties": false + }, + "guid": {"type": "string", "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"}, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "dataTypes": {"type": "array", "items": {"type": "string", "minLength": 1, "not": {"const": "Watchlist"}}}, + "query": { + "type": "object", + "required": ["sourcePath", "outputPath", "sourceSha256", "targetSha256", "status"], + "properties": { + "sourcePath": {"type": "string", "pattern": "^/"}, + "outputPath": {"type": ["string", "null"]}, + "sourceSha256": {"$ref": "#/$defs/sha256"}, + "targetSha256": {"anyOf": [{"$ref": "#/$defs/sha256"}, {"type": "null"}]}, + "status": {"enum": ["converted", "removed-scope-helper"]}, + "workspaceFunctions": {"type": "array", "uniqueItems": true, "items": {"type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]*$"}}, + "referenceInputs": {"type": "array", "uniqueItems": true, "items": {"const": "Watchlist"}} + }, + "allOf": [{ + "if": {"properties": {"status": {"const": "removed-scope-helper"}}}, + "then": {"properties": {"outputPath": {"type": "null"}, "targetSha256": {"type": "null"}}}, + "else": {"properties": {"outputPath": {"type": "string", "pattern": "^/"}, "targetSha256": {"$ref": "#/$defs/sha256"}}} + }] + } + } +} diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/LICENSE b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/LICENSE new file mode 100644 index 00000000000..22aed37e650 --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Microsoft Corporation. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/__init__.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/__init__.py new file mode 100644 index 00000000000..aa440a4ad60 --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/__init__.py @@ -0,0 +1,5 @@ +"""Offline workbook adaptation and shared deterministic KQL conversion. + +Catalog mappings are conversion candidates, not evidence of table visibility +or runtime parity in any particular environment. +""" diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/asim_functions.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/asim_functions.py new file mode 100644 index 00000000000..40a7e2895b8 --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/asim_functions.py @@ -0,0 +1,276 @@ +"""Deterministic rewrite of Sentinel ASIM unifying-parser function calls +(`_Im_(...)`) into their AH-native `ASimLogs` table equivalent. + +Why this exists: `_Im_WebSession`, `_Im_NetworkSession`, `_Im_Dns`, etc. are +Sentinel workspace-scoped saved-query functions (union + normalize several +vendor parsers under one ASIM schema, with convenience named parameters like +`starttime=`, `url_has_any=`). Advanced Hunting has no workspace-function +namespace, so any call to one of these fails at runtime with something like +"Operator source expression should be table or column" - AH can't resolve the +identifier, and doesn't support the named-parameter call syntax at all. + +The fix: the unified SecOps AH surface ships the equivalent data pre-normalized +directly in a native `ASimLogs` table (confirmed live via +tenant_resolved_tables.json - see data/mapping_catalog.json's +`asimFunctionRewrites._source` for the evidence trail). Swap the function call +for the table, and inline whatever named parameters we have a verified, +semantics-preserving `| where` translation for (see +`asimFunctionRewrites.paramRewrites` in the catalog). + +Conservative by design: a call is only auto-rewritten when EVERY named +parameter it uses has a known translation. If any parameter is unrecognized, +the call is left untouched and flagged for manual review instead of guessing. +""" +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +from . import mapping_catalog + + +def _split_top_level_args(argtext: str) -> List[str]: + """Split a call's argument text on top-level commas (respecting nested + parens/brackets and string literals).""" + parts: List[str] = [] + depth, start, i, n = 0, 0, 0, len(argtext) + in_str: Optional[str] = None + while i < n: + c = argtext[i] + if in_str: + if c == "\\" and in_str == '"' and not (i > 0 and argtext[i - 1] == "@"): + i += 2 + continue + if c == in_str: + in_str = None + i += 1 + continue + if c in ("'", '"'): + in_str = c + elif c in "([{": + depth += 1 + elif c in ")]}": + depth -= 1 + elif c == "," and depth == 0: + parts.append(argtext[start:i]) + start = i + 1 + i += 1 + tail = argtext[start:] + if tail.strip(): + parts.append(tail) + return parts + + +def _find_call(kql: str, func_name: str, pos: int) -> Optional[Tuple[int, int, str]]: + """If `func_name` at `pos` is followed (allowing whitespace) by a balanced + `(...)`, return (call_start, call_end, argtext). Else None (bare reference).""" + j = pos + len(func_name) + while j < len(kql) and kql[j] in " \t\r\n": + j += 1 + if j >= len(kql) or kql[j] != "(": + return None + depth, k, n = 1, j + 1, len(kql) + in_str: Optional[str] = None + while k < n and depth > 0: + c = kql[k] + if in_str: + if c == "\\" and in_str == '"' and not (k > 0 and kql[k - 1] == "@"): + k += 2 + continue + if c == in_str: + in_str = None + elif c in ("'", '"'): + in_str = c + elif c == "(": + depth += 1 + elif c == ")": + depth -= 1 + k += 1 + return pos, k, kql[j + 1:k - 1] + + +def _comment_mask(text: str) -> List[bool]: + """[ASIM-FIX 2026-08-02] Per-character `True` if that position is inside a + `//...` KQL comment (from `//` to end of line), string-literal-aware so a `//` inside a + quoted string isn't mistaken for a comment start.""" + mask = [False] * len(text) + in_str: Optional[str] = None + i, n = 0, len(text) + while i < n: + c = text[i] + if in_str: + if c == "\\" and in_str == '"' and not (i > 0 and text[i - 1] == "@"): + i += 2 + continue + if c == in_str: + in_str = None + i += 1 + continue + if c in ("'", '"'): + in_str = c + i += 1 + continue + if c == "/" and i + 1 < n and text[i + 1] == "/": + j = i + while j < n and text[j] != "\n": + mask[j] = True + j += 1 + i = j + continue + i += 1 + return mask + + +def rewrite_im_calls(kql: str, catalog: Optional[dict] = None) -> dict: + """Rewrite every `_Im_(...)` / bare `_Im_` reference in + `kql` that this catalog knows how to translate. + + Returns: {"convertedKql": str, "diff": [...], "flags": [...]}. + """ + cat = catalog or mapping_catalog.load_mapping_catalog() + asim = cat.get("asimFunctionRewrites", {}) + functions: Dict[str, dict] = asim.get("functions", {}) + param_rewrites: Dict[str, dict] = asim.get("paramRewrites", {}) + + if not functions: + return {"convertedKql": kql, "diff": [], "flags": []} + + names = sorted(functions, key=len, reverse=True) + pat = re.compile(r"\b(" + "|".join(re.escape(n) for n in names) + r")\b") + + diff: List[dict] = [] + flags: List[dict] = [] + out = kql + # Process rightmost-first so earlier offsets stay valid as we splice. + # Skip matches inside `//` comments - dead/inert text (e.g. an author's + # commented-out "if you enable _Im_Dns, uncomment this" suggestion) is not + # a live call and must not be rewritten or flagged for manual review. + comment_mask = _comment_mask(out) + matches = [m for m in pat.finditer(out) if not comment_mask[m.start()]] + for m in reversed(matches): + func = m.group(1) + ah_table = functions[func]["ahTable"] + call = _find_call(out, func, m.start()) + if call is None: + # Bare reference (no parens) - safe straight rename, no params to lose. + out = out[:m.start()] + ah_table + out[m.end():] + diff.append({"kind": "asim-function-rewrite", "before": func, + "after": ah_table, "count": 1, "source": "deterministic", + "func": func, "ahTable": ah_table, + "reason": "ASIM unifying-parser function call replaced by " + "its AH-native normalized table (no params to translate)."}) + continue + + start, end, argtext = call + args = _split_top_level_args(argtext) + clauses: List[str] = [] + unknown: List[str] = [] + for a in args: + a = a.strip() + if not a or "=" not in a: + unknown.append(a) + continue + key, value = a.split("=", 1) + key, value = key.strip(), value.strip() + spec = param_rewrites.get(key) + if spec is None: + unknown.append(a) + continue + if spec.get("drop"): + continue + clauses.append(spec["template"].format(value=value)) + + if unknown: + # Don't guess - leave this call exactly as-is, flag for manual review. + flags.append({"kind": "asim-function-unrewritten", "name": func, + "reason": f"Call to {func}(...) uses parameter(s) " + f"{unknown} with no verified AH translation; " + f"left unchanged. Manual review required " + f"(equivalent table is {ah_table})."}) + continue + + replacement = ah_table + if clauses: + replacement += "\n | where " + "\n | where ".join(clauses) + out = out[:start] + replacement + out[end:] + diff.append({"kind": "asim-function-rewrite", "before": f"{func}({argtext})", + "after": replacement, "count": 1, "source": "deterministic", + "func": func, "ahTable": ah_table, + "reason": f"ASIM unifying-parser function call replaced by " + f"{ah_table} with its named parameters inlined as " + f"where clauses."}) + + return {"convertedKql": out, "diff": diff, "flags": flags} + + +def flag_im_calls(kql: str, catalog: Optional[dict] = None) -> List[dict]: + """[ASIM-FIX-TAG 2026-08-03] Detection-only counterpart to `rewrite_im_calls`, + used while `rewrite.ASIM_REWRITE_ENABLED = False`. Every live `_Im_(...)` + / bare `_Im_` reference is left completely untouched in the KQL and + flagged for manual review - same flag shape (`asim-function-unrewritten`) the + rewriter already produces for calls with an untranslatable parameter, so the + existing needs-review gate in rewrite.py handles this path identically without + any gate changes. Does not rewrite anything and returns no `diff`. + + [ASIM-FIX-TAG 2026-08-03b] Each flag also carries `func`, `ahTable`, and a + `translatable` bool distinguishing two risk levels, so callers (batch_xdr.py) + can decide whether a passing live validation may be trusted: + - `translatable=True`: every named parameter used (if any) has a verified + catalog translation - i.e. this call WOULD have been safely auto-rewritten + had the rewrite been enabled. The only remaining unknown is whether AH has + this Sentinel-workspace function at all on a generic tenant, not whether + the data it returns (if it resolves) is correct. + - `translatable=False`: uses a parameter with NO verified translation - the + exact 2026-08-02 finding scenario (`_Im_FileEvent` silently binding to + Blob-storage-log-shaped columns). A passing live bind/run must NOT be + trusted here regardless of outcome. + """ + cat = catalog or mapping_catalog.load_mapping_catalog() + asim = cat.get("asimFunctionRewrites", {}) + functions: Dict[str, dict] = asim.get("functions", {}) + param_rewrites: Dict[str, dict] = asim.get("paramRewrites", {}) + if not functions: + return [] + + names = sorted(functions, key=len, reverse=True) + pat = re.compile(r"\b(" + "|".join(re.escape(n) for n in names) + r")\b") + comment_mask = _comment_mask(kql) + + flags: List[dict] = [] + seen: set = set() + for m in pat.finditer(kql): + if comment_mask[m.start()]: + continue + func = m.group(1) + if func in seen: + continue + seen.add(func) + ah_table = functions[func]["ahTable"] + + translatable = True + call = _find_call(kql, func, m.start()) + if call is not None: + _, _, argtext = call + for a in _split_top_level_args(argtext): + a = a.strip() + if not a: + continue + if "=" not in a or a.split("=", 1)[0].strip() not in param_rewrites: + translatable = False + break + + reason = (f"ASIM rewrite currently disabled (AH does not yet natively " + f"support unifying parsers for generic tenants); {func}(...) " + f"left unchanged. Manual review required (equivalent table " + f"would be {ah_table}).") if translatable else ( + f"Call to {func}(...) uses a parameter with no verified AH " + f"translation; ASIM rewrite disabled, so it is left unchanged " + f"and must not be trusted even if it appears to bind/run " + f"(equivalent table would be {ah_table}).") + flags.append({"kind": "asim-function-unrewritten", "name": func, + "func": func, "ahTable": ah_table, "translatable": translatable, + "reason": reason}) + return flags diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/ah_schema_catalog.json b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/ah_schema_catalog.json new file mode 100644 index 00000000000..97b93d1c18c --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/ah_schema_catalog.json @@ -0,0 +1,6491 @@ +{ + "AADManagedIdentitySignInLogs": { + "AADTenantId": "string", + "Agent": "string", + "AppId": "string", + "AppOwnerTenantId": "string", + "AuthenticationContextClassReferences": "string", + "AuthenticationProcessingDetails": "string", + "Category": "string", + "ClientCredentialType": "string", + "ConditionalAccessAudiences": "string", + "ConditionalAccessPolicies": "string", + "ConditionalAccessPoliciesV2": "dynamic", + "ConditionalAccessStatus": "string", + "CorrelationId": "string", + "CreatedDateTime": "datetime", + "DurationMs": "long", + "FederatedCredentialId": "string", + "IPAddress": "string", + "Id": "string", + "Identity": "string", + "Level": "string", + "Location": "string", + "LocationDetails": "string", + "ManagedServiceIdentity": "string", + "NetworkLocationDetails": "string", + "OperationName": "string", + "OperationVersion": "string", + "ResourceDisplayName": "string", + "ResourceGroup": "string", + "ResourceIdentity": "string", + "ResourceOwnerTenantId": "string", + "ResourceServicePrincipalId": "string", + "ResultDescription": "string", + "ResultSignature": "string", + "ResultType": "string", + "ServicePrincipalCredentialKeyId": "string", + "ServicePrincipalCredentialThumbprint": "string", + "ServicePrincipalId": "string", + "ServicePrincipalName": "string", + "SessionId": "string", + "SourceAppClientId": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "UniqueTokenIdentifier": "string", + "UserAgent": "string" + }, + "AADNonInteractiveUserSignInLogs": { + "AADTenantId": "string", + "Agent": "string", + "AlternateSignInName": "string", + "AppDisplayName": "string", + "AppId": "string", + "AppOwnerTenantId": "string", + "AppliedEventListeners": "dynamic", + "AuthenticationContextClassReferences": "string", + "AuthenticationDetails": "string", + "AuthenticationMethodsUsed": "string", + "AuthenticationProcessingDetails": "string", + "AuthenticationProtocol": "string", + "AuthenticationRequirement": "string", + "AuthenticationRequirementPolicies": "string", + "AuthenticatorAppLocation": "string", + "AutonomousSystemNumber": "string", + "Category": "string", + "ClientAppUsed": "string", + "ClientCredentialType": "string", + "ClientSessionId": "string", + "ConditionalAccessAudiences": "string", + "ConditionalAccessPolicies": "string", + "ConditionalAccessPoliciesV2": "dynamic", + "ConditionalAccessStatus": "string", + "CorrelationId": "string", + "CreatedDateTime": "datetime", + "CrossTenantAccessType": "string", + "DeviceDetail": "string", + "DurationMs": "long", + "FederatedCredentialId": "string", + "GlobalSecureAccessIpAddress": "string", + "HomeTenantId": "string", + "HomeTenantName": "string", + "IPAddress": "string", + "Id": "string", + "Identity": "string", + "IncomingTokenType": "string", + "IsInteractive": "bool", + "IsRisky": "bool", + "IsTenantRestricted": "bool", + "IsThroughGlobalSecureAccess": "bool", + "Level": "string", + "Location": "string", + "LocationDetails": "string", + "MfaDetail": "string", + "NetworkLocationDetails": "string", + "OperationName": "string", + "OperationVersion": "string", + "OriginalRequestId": "string", + "OriginalTransferMethod": "string", + "ProcessingTimeInMs": "string", + "ResourceDisplayName": "string", + "ResourceGroup": "string", + "ResourceIdentity": "string", + "ResourceOwnerTenantId": "string", + "ResourceServicePrincipalId": "string", + "ResourceTenantId": "string", + "ResultDescription": "string", + "ResultSignature": "string", + "ResultType": "string", + "RiskDetail": "string", + "RiskEventTypes": "string", + "RiskEventTypes_V2": "string", + "RiskLevelAggregated": "string", + "RiskLevelDuringSignIn": "string", + "RiskState": "string", + "ServicePrincipalId": "string", + "SessionId": "string", + "SessionLifetimePolicies": "string", + "SignInEventTypes": "string", + "SignInIdentifierType": "string", + "SourceSystem": "string", + "Status": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TokenIssuerName": "string", + "TokenIssuerType": "string", + "TokenProtectionStatusDetails": "string", + "Type": "string", + "UniqueTokenIdentifier": "string", + "UserAgent": "string", + "UserDisplayName": "string", + "UserId": "string", + "UserPrincipalName": "string", + "UserType": "string" + }, + "AADServicePrincipalSignInLogs": { + "AADTenantId": "string", + "Agent": "string", + "AppId": "string", + "AppOwnerTenantId": "string", + "AuthenticationContextClassReferences": "string", + "AuthenticationProcessingDetails": "string", + "AutonomousSystemNumber": "string", + "Category": "string", + "ClientCredentialType": "string", + "ConditionalAccessAudiences": "string", + "ConditionalAccessPolicies": "string", + "ConditionalAccessPoliciesV2": "dynamic", + "ConditionalAccessStatus": "string", + "CorrelationId": "string", + "CreatedDateTime": "datetime", + "DurationMs": "long", + "FederatedCredentialId": "string", + "IPAddress": "string", + "Id": "string", + "Identity": "string", + "Level": "string", + "Location": "string", + "LocationDetails": "string", + "NetworkLocationDetails": "string", + "OperationName": "string", + "OperationVersion": "string", + "ResourceDisplayName": "string", + "ResourceGroup": "string", + "ResourceIdentity": "string", + "ResourceOwnerTenantId": "string", + "ResourceServicePrincipalId": "string", + "ResultDescription": "string", + "ResultSignature": "string", + "ResultType": "string", + "ServicePrincipalCredentialKeyId": "string", + "ServicePrincipalCredentialThumbprint": "string", + "ServicePrincipalId": "string", + "ServicePrincipalName": "string", + "SessionId": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "UniqueTokenIdentifier": "string", + "UserAgent": "string" + }, + "ASimAgentEventLogs": { + "ActingAppId": "string", + "ActingAppName": "string", + "ActingAppType": "string", + "ActingApplicationEntityKey": "string", + "ActorUserAdditionalIds": "dynamic", + "ActorUserEntityKey": "string", + "ActorUserId": "string", + "ActorUserIdType": "string", + "ActorUserScope": "string", + "ActorUserScopeId": "string", + "ActorUsername": "string", + "ActorUsernameType": "string", + "AdditionalEntities": "dynamic", + "AdditionalFields": "dynamic", + "DvcEntityKey": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventErrorDetails": "string", + "EventFinishReasons": "dynamic", + "EventOriginalErrorType": "string", + "EventOriginalRequestDetails": "string", + "EventOriginalResultDetails": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventOutputType": "string", + "EventProduct": "string", + "EventRequestFrequencyPenalty": "real", + "EventRequestId": "string", + "EventRequestPresencePenalty": "real", + "EventRequestSeed": "long", + "EventRequestTemperature": "real", + "EventRequestTopP": "real", + "EventResponseId": "string", + "EventSchema": "string", + "EventSchemaVersion": "string", + "EventSessionId": "string", + "EventSessionName": "string", + "EventStartTime": "datetime", + "EventThoughtProcessDetails": "string", + "EventThoughtProcessId": "string", + "EventType": "string", + "EventUid": "string", + "EventVendor": "string", + "InputTokensUsed": "long", + "ModelName": "string", + "ModelProviderName": "string", + "OutputTokensUsed": "long", + "PlatformTargetAgentDescription": "string", + "PlatformTargetAgentId": "string", + "PlatformTargetAgentName": "string", + "PlatformTargetOriginalAgentType": "string", + "SourceSystem": "string", + "SrcAgentBlueprintId": "string", + "SrcAgentDescription": "string", + "SrcAgentEntityKey": "string", + "SrcAgentId": "string", + "SrcAgentName": "string", + "SrcAgentOriginalType": "string", + "SrcFQDN": "string", + "SrcIpAddr": "string", + "SrcPortNumber": "int", + "TargetAgentBlueprintId": "string", + "TargetAgentDescription": "string", + "TargetAgentEntityKey": "string", + "TargetAgentId": "string", + "TargetAgentName": "string", + "TargetAgentOriginalType": "string", + "TargetAgentUserId": "string", + "TargetAgentUsername": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "ToolDescription": "string", + "ToolId": "string", + "ToolName": "string", + "ToolOriginalType": "string", + "Type": "string", + "_ResourceId": "string" + }, + "ASimAlertEventLogs": { + "AdditionalEntities": "dynamic", + "AdditionalFields": "dynamic", + "AlertDescription": "string", + "AlertId": "string", + "AlertName": "string", + "AlertOriginalStatus": "string", + "AlertStatus": "string", + "AlertVerdict": "string", + "AttackRemediationSteps": "string", + "AttackTactics": "string", + "AttackTechniques": "string", + "DetectionMethod": "string", + "DvcAction": "string", + "DvcDescription": "string", + "DvcDomain": "string", + "DvcDomainType": "string", + "DvcEntityKey": "string", + "DvcFQDN": "string", + "DvcHostname": "string", + "DvcId": "string", + "DvcIdType": "string", + "DvcInterface": "string", + "DvcIpAddr": "string", + "DvcMacAddr": "string", + "DvcOriginalAction": "string", + "DvcOs": "string", + "DvcOsVersion": "string", + "DvcScope": "string", + "DvcScopeId": "string", + "DvcZone": "string", + "EmailMessageId": "string", + "EmailSubject": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventMessage": "string", + "EventOriginalResultDetails": "string", + "EventOriginalSeverity": "string", + "EventOriginalSubType": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventOwner": "string", + "EventProduct": "string", + "EventProductVersion": "string", + "EventReportUrl": "string", + "EventResult": "string", + "EventResultDetails": "string", + "EventSchemaVersion": "string", + "EventSeverity": "string", + "EventStartTime": "datetime", + "EventSubType": "string", + "EventType": "string", + "EventUid": "string", + "EventVendor": "string", + "FileEntityKey": "string", + "FileMD5": "string", + "FileName": "string", + "FilePath": "string", + "FileSHA1": "string", + "FileSHA256": "string", + "FileSize": "string", + "IndicatorAssociation": "string", + "IndicatorType": "string", + "OriginalUserType": "string", + "ProcessCommandLine": "string", + "ProcessEntityKey": "string", + "ProcessFileCompany": "string", + "ProcessId": "string", + "ProcessName": "string", + "RegistryKey": "string", + "RegistryValue": "string", + "RegistryValueData": "string", + "RegistryValueType": "string", + "Rule": "string", + "RuleDescription": "string", + "RuleName": "string", + "RuleNumber": "int", + "SourceSystem": "string", + "TenantId": "string", + "ThreatCategory": "string", + "ThreatConfidence": "int", + "ThreatFirstReportedTime": "datetime", + "ThreatId": "string", + "ThreatIsActive": "bool", + "ThreatLastReportedTime": "datetime", + "ThreatName": "string", + "ThreatOriginalCategory": "string", + "ThreatOriginalConfidence": "string", + "ThreatOriginalRiskLevel": "string", + "ThreatRiskLevel": "int", + "TimeGenerated": "datetime", + "Type": "string", + "Url": "string", + "User": "string", + "UserAdditionalIds": "dynamic", + "UserEntityKey": "string", + "UserId": "string", + "UserIdType": "string", + "UserScope": "string", + "UserScopeId": "string", + "UserSessionId": "string", + "UserType": "string", + "Username": "string", + "UsernameType": "string", + "_ResourceId": "string" + }, + "ASimAssetEntityLogs": { + "AADTenantId": "string", + "AdditionalAssetOwners": "dynamic", + "AdditionalFields": "dynamic", + "AssetClassificationLastScanDateTime": "datetime", + "AssetIsProtectedByDlp": "bool", + "AssetOriginalDataClassificationType": "dynamic", + "AssetOriginalPermissions": "dynamic", + "AssetOriginalRiskDetails": "dynamic", + "AssetOriginalRiskLevel": "string", + "AssetOriginalSensitivityLevel": "string", + "AssetOriginalType": "string", + "AssetOwnerId": "string", + "AssetOwnerIdType": "string", + "AssetOwnerScope": "string", + "AssetOwnerScopeId": "string", + "AssetOwnerType": "string", + "AssetPath": "string", + "AssetRelatedIndicators": "dynamic", + "AssetRiskFirstReportedTime": "datetime", + "AssetRiskLastReportedTime": "datetime", + "AssetRiskLevel": "string", + "AssetRiskName": "string", + "AssetSensitivityLabel": "string", + "AssetType": "string", + "EntityCreatedTime": "datetime", + "EntityFeedType": "string", + "EntityId": "string", + "EntityIdType": "string", + "EntityIngestionTime": "datetime", + "EntityIsDeleted": "bool", + "EntityKey": "string", + "EntityLastAccessedTime": "datetime", + "EntityLastModifiedTime": "datetime", + "EntityName": "string", + "EntityNameType": "string", + "EntityOriginalId": "string", + "EntityOriginalSource": "string", + "EntityProduct": "string", + "EntitySchema": "string", + "EntitySchemaVersion": "string", + "EntitySnapshotId": "string", + "EntitySource": "string", + "EntitySubProduct": "string", + "EntityUpdatedTime": "datetime", + "EntityVendor": "string", + "ExternalUsersCount": "int", + "FileExtension": "string", + "FileIsSignatureValid": "bool", + "FileMD5": "string", + "FilePath": "string", + "FileSHA1": "string", + "FileSHA256": "string", + "FileSHA512": "string", + "FileSignatureDetails": "string", + "FileSize": "long", + "IdentityDirectoryId": "string", + "IdentityDirectoryName": "string", + "InternalUsersCount": "int", + "SitePath": "string", + "SitePrimaryUri": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "User": "string", + "_ResourceId": "string" + }, + "ASimAuditEventLogs": { + "ActingAppId": "string", + "ActingAppName": "string", + "ActingAppType": "string", + "ActingApplicationEntityKey": "string", + "ActingOriginalAppType": "string", + "ActorOriginalUserType": "string", + "ActorScope": "string", + "ActorScopeId": "string", + "ActorSessionId": "string", + "ActorUserAadId": "string", + "ActorUserAdditionalIds": "dynamic", + "ActorUserEntityKey": "string", + "ActorUserId": "string", + "ActorUserIdType": "string", + "ActorUserSid": "string", + "ActorUserType": "string", + "ActorUsername": "string", + "ActorUsernameType": "string", + "AdditionalEntities": "dynamic", + "AdditionalFields": "dynamic", + "DvcAction": "string", + "DvcDescription": "string", + "DvcDomain": "string", + "DvcDomainType": "string", + "DvcEntityKey": "string", + "DvcFQDN": "string", + "DvcHostname": "string", + "DvcId": "string", + "DvcIdType": "string", + "DvcInterface": "string", + "DvcIpAddr": "string", + "DvcMacAddr": "string", + "DvcOriginalAction": "string", + "DvcOs": "string", + "DvcOsVersion": "string", + "DvcScope": "string", + "DvcScopeId": "string", + "DvcZone": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventMessage": "string", + "EventOriginalResultDetails": "string", + "EventOriginalSeverity": "string", + "EventOriginalSubType": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventOwner": "string", + "EventProduct": "string", + "EventProductVersion": "string", + "EventReportUrl": "string", + "EventResult": "string", + "EventResultDetails": "string", + "EventSchemaVersion": "string", + "EventSeverity": "string", + "EventStartTime": "datetime", + "EventSubType": "string", + "EventType": "string", + "EventVendor": "string", + "HttpUserAgent": "string", + "NewValue": "string", + "Object": "string", + "ObjectId": "string", + "ObjectType": "string", + "OldValue": "string", + "Operation": "string", + "OriginalObjectType": "string", + "RuleName": "string", + "RuleNumber": "int", + "SourceSystem": "string", + "SrcDescription": "string", + "SrcDeviceType": "string", + "SrcDomain": "string", + "SrcDomainType": "string", + "SrcDvcId": "string", + "SrcDvcIdType": "string", + "SrcDvcScope": "string", + "SrcDvcScopeId": "string", + "SrcFQDN": "string", + "SrcGeoCity": "string", + "SrcGeoCountry": "string", + "SrcGeoLatitude": "real", + "SrcGeoLongitude": "real", + "SrcGeoRegion": "string", + "SrcHostname": "string", + "SrcIpAddr": "string", + "SrcOriginalRiskLevel": "string", + "SrcPortNumber": "int", + "SrcRiskLevel": "int", + "SrcSystemEntityKey": "string", + "TargetAppId": "string", + "TargetAppName": "string", + "TargetAppType": "string", + "TargetApplicationEntityKey": "string", + "TargetDescription": "string", + "TargetDeviceType": "string", + "TargetDomain": "string", + "TargetDomainType": "string", + "TargetDvcId": "string", + "TargetDvcIdType": "string", + "TargetDvcOs": "string", + "TargetDvcScope": "string", + "TargetDvcScopeId": "string", + "TargetFQDN": "string", + "TargetGeoCity": "string", + "TargetGeoCountry": "string", + "TargetGeoLatitude": "real", + "TargetGeoLongitude": "real", + "TargetGeoRegion": "string", + "TargetHostname": "string", + "TargetIpAddr": "string", + "TargetOriginalAppType": "string", + "TargetOriginalRiskLevel": "string", + "TargetPortNumber": "int", + "TargetRiskLevel": "int", + "TargetSystemEntityKey": "string", + "TargetUrl": "string", + "TenantId": "string", + "ThreatCategory": "string", + "ThreatConfidence": "int", + "ThreatField": "string", + "ThreatFirstReportedTime": "datetime", + "ThreatId": "string", + "ThreatIpAddr": "string", + "ThreatIsActive": "bool", + "ThreatLastReportedTime": "datetime", + "ThreatName": "string", + "ThreatOriginalConfidence": "string", + "ThreatOriginalRiskLevel": "string", + "ThreatRiskLevel": "int", + "TimeGenerated": "datetime", + "Type": "string", + "ValueType": "string", + "_ResourceId": "string" + }, + "ASimAuthenticationEventLogs": { + "ActingAppId": "string", + "ActingAppName": "string", + "ActingAppType": "string", + "ActingApplicationEntityKey": "string", + "ActingOriginalAppType": "string", + "ActorOriginalUserType": "string", + "ActorScope": "string", + "ActorScopeId": "string", + "ActorSessionId": "string", + "ActorUserAdditionalIds": "dynamic", + "ActorUserEntityKey": "string", + "ActorUserId": "string", + "ActorUserIdType": "string", + "ActorUserType": "string", + "ActorUsername": "string", + "ActorUsernameType": "string", + "AdditionalEntities": "dynamic", + "AdditionalFields": "dynamic", + "DvcAction": "string", + "DvcDescription": "string", + "DvcDomain": "string", + "DvcDomainType": "string", + "DvcEntityKey": "string", + "DvcFQDN": "string", + "DvcHostname": "string", + "DvcId": "string", + "DvcIdType": "string", + "DvcInterface": "string", + "DvcIpAddr": "string", + "DvcMacAddr": "string", + "DvcOriginalAction": "string", + "DvcOs": "string", + "DvcOsVersion": "string", + "DvcScope": "string", + "DvcScopeId": "string", + "DvcZone": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventMessage": "string", + "EventOriginalResultDetails": "string", + "EventOriginalSeverity": "string", + "EventOriginalSubType": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventOwner": "string", + "EventProduct": "string", + "EventProductVersion": "string", + "EventReportUrl": "string", + "EventResult": "string", + "EventResultDetails": "string", + "EventSchemaVersion": "string", + "EventSeverity": "string", + "EventStartTime": "datetime", + "EventSubType": "string", + "EventType": "string", + "EventVendor": "string", + "HttpUserAgent": "string", + "LogonMethod": "string", + "LogonProtocol": "string", + "RuleName": "string", + "RuleNumber": "int", + "SourceSystem": "string", + "SrcDescription": "string", + "SrcDeviceType": "string", + "SrcDomain": "string", + "SrcDomainType": "string", + "SrcDvcId": "string", + "SrcDvcIdType": "string", + "SrcDvcOs": "string", + "SrcDvcScope": "string", + "SrcDvcScopeId": "string", + "SrcFQDN": "string", + "SrcGeoCity": "string", + "SrcGeoCountry": "string", + "SrcGeoLatitude": "real", + "SrcGeoLongitude": "real", + "SrcGeoRegion": "string", + "SrcHostname": "string", + "SrcIpAddr": "string", + "SrcIsp": "string", + "SrcOriginalRiskLevel": "string", + "SrcPortNumber": "int", + "SrcRiskLevel": "int", + "SrcSystemEntityKey": "string", + "TargetAppId": "string", + "TargetAppName": "string", + "TargetAppType": "string", + "TargetApplicationEntityKey": "string", + "TargetDescription": "string", + "TargetDeviceType": "string", + "TargetDomain": "string", + "TargetDomainType": "string", + "TargetDvcId": "string", + "TargetDvcIdType": "string", + "TargetDvcOs": "string", + "TargetDvcScope": "string", + "TargetDvcScopeId": "string", + "TargetFQDN": "string", + "TargetGeoCity": "string", + "TargetGeoCountry": "string", + "TargetGeoLatitude": "real", + "TargetGeoLongitude": "real", + "TargetGeoRegion": "string", + "TargetHostname": "string", + "TargetIpAddr": "string", + "TargetOriginalAppType": "string", + "TargetOriginalRiskLevel": "string", + "TargetOriginalUserType": "string", + "TargetPortNumber": "int", + "TargetRiskLevel": "int", + "TargetSessionId": "string", + "TargetSystemEntityKey": "string", + "TargetUrl": "string", + "TargetUserAdditionalIds": "dynamic", + "TargetUserEntityKey": "string", + "TargetUserId": "string", + "TargetUserIdType": "string", + "TargetUserScope": "string", + "TargetUserScopeId": "string", + "TargetUserType": "string", + "TargetUsername": "string", + "TargetUsernameType": "string", + "TenantId": "string", + "ThreatCategory": "string", + "ThreatConfidence": "int", + "ThreatField": "string", + "ThreatFirstReportedTime": "datetime", + "ThreatId": "string", + "ThreatIpAddr": "string", + "ThreatIsActive": "bool", + "ThreatLastReportedTime": "datetime", + "ThreatName": "string", + "ThreatOriginalConfidence": "string", + "ThreatOriginalRiskLevel": "string", + "ThreatRiskLevel": "int", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "ASimDhcpEventLogs": { + "AdditionalEntities": "dynamic", + "AdditionalFields": "dynamic", + "DhcpCircuitId": "string", + "DhcpLeaseDuration": "int", + "DhcpSessionDuration": "int", + "DhcpSessionId": "string", + "DhcpSrcDHCId": "string", + "DhcpSubscriberId": "string", + "DhcpUserClass": "string", + "DhcpUserClassId": "string", + "DhcpVendorClass": "string", + "DhcpVendorClassId": "string", + "DvcAction": "string", + "DvcDescription": "string", + "DvcDomain": "string", + "DvcDomainType": "string", + "DvcEntityKey": "string", + "DvcFQDN": "string", + "DvcHostname": "string", + "DvcId": "string", + "DvcIdType": "string", + "DvcInterface": "string", + "DvcIpAddr": "string", + "DvcMacAddr": "string", + "DvcOriginalAction": "string", + "DvcOs": "string", + "DvcOsVersion": "string", + "DvcScope": "string", + "DvcScopeId": "string", + "DvcZone": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventMessage": "string", + "EventOriginalResultDetails": "string", + "EventOriginalSeverity": "string", + "EventOriginalSubType": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventOwner": "string", + "EventProduct": "string", + "EventProductVersion": "string", + "EventReportUrl": "string", + "EventResult": "string", + "EventResultDetails": "string", + "EventSchema": "string", + "EventSchemaVersion": "string", + "EventSeverity": "string", + "EventStartTime": "datetime", + "EventSubType": "string", + "EventType": "string", + "EventVendor": "string", + "RequestedIpAddr": "string", + "RuleName": "string", + "RuleNumber": "int", + "SourceSystem": "string", + "SrcDescription": "string", + "SrcDeviceType": "string", + "SrcDomain": "string", + "SrcDomainType": "string", + "SrcDvcId": "string", + "SrcDvcIdType": "string", + "SrcDvcScope": "string", + "SrcDvcScopeId": "string", + "SrcFQDN": "string", + "SrcGeoCity": "string", + "SrcGeoCountry": "string", + "SrcGeoLatitude": "real", + "SrcGeoLongitude": "real", + "SrcGeoRegion": "string", + "SrcHostname": "string", + "SrcIpAddr": "string", + "SrcMacAddr": "string", + "SrcOriginalRiskLevel": "string", + "SrcOriginalUserType": "string", + "SrcPortNumber": "int", + "SrcRiskLevel": "int", + "SrcSystemEntityKey": "string", + "SrcUserAdditionalIds": "dynamic", + "SrcUserEntityKey": "string", + "SrcUserId": "string", + "SrcUserIdType": "string", + "SrcUserScope": "string", + "SrcUserScopeId": "string", + "SrcUserSessionId": "string", + "SrcUserType": "string", + "SrcUserUid": "string", + "SrcUsername": "string", + "SrcUsernameType": "string", + "TenantId": "string", + "ThreatCategory": "string", + "ThreatConfidence": "int", + "ThreatField": "string", + "ThreatFirstReportedTime": "datetime", + "ThreatId": "string", + "ThreatIsActive": "bool", + "ThreatLastReportedTime": "datetime", + "ThreatName": "string", + "ThreatOriginalConfidence": "string", + "ThreatOriginalRiskLevel": "string", + "ThreatRiskLevel": "int", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "ASimDnsActivityLogs": { + "AdditionalEntities": "dynamic", + "AdditionalFields": "dynamic", + "DnsFlags": "string", + "DnsFlagsAuthenticated": "bool", + "DnsFlagsAuthoritative": "bool", + "DnsFlagsCheckingDisabled": "bool", + "DnsFlagsRecursionAvailable": "bool", + "DnsFlagsRecursionDesired": "bool", + "DnsFlagsTruncated": "bool", + "DnsFlagsZ": "bool", + "DnsNetworkDuration": "int", + "DnsQuery": "string", + "DnsQueryClass": "int", + "DnsQueryClassName": "string", + "DnsQueryType": "int", + "DnsQueryTypeName": "string", + "DnsResponseCode": "int", + "DnsResponseIpCity": "string", + "DnsResponseIpCountry": "string", + "DnsResponseIpLatitude": "real", + "DnsResponseIpLongitude": "real", + "DnsResponseIpRegion": "string", + "DnsResponseName": "string", + "DnsSessionId": "string", + "Dst": "string", + "DstDescription": "string", + "DstDeviceType": "string", + "DstDomain": "string", + "DstDomainType": "string", + "DstDvcId": "string", + "DstDvcIdType": "string", + "DstDvcScope": "string", + "DstDvcScopeId": "string", + "DstFQDN": "string", + "DstGeoCity": "string", + "DstGeoCountry": "string", + "DstGeoLatitude": "real", + "DstGeoLongitude": "real", + "DstGeoRegion": "string", + "DstHostname": "string", + "DstIpAddr": "string", + "DstOriginalRiskLevel": "string", + "DstPortNumber": "int", + "DstRiskLevel": "int", + "DstSystemEntityKey": "string", + "Dvc": "string", + "DvcAction": "string", + "DvcDescription": "string", + "DvcDomain": "string", + "DvcDomainType": "string", + "DvcEntityKey": "string", + "DvcFQDN": "string", + "DvcHostname": "string", + "DvcId": "string", + "DvcIdType": "string", + "DvcInterface": "string", + "DvcIpAddr": "string", + "DvcMacAddr": "string", + "DvcOriginalAction": "string", + "DvcOs": "string", + "DvcOsVersion": "string", + "DvcScope": "string", + "DvcScopeId": "string", + "DvcZone": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventMessage": "string", + "EventOriginalSeverity": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventOwner": "string", + "EventProduct": "string", + "EventProductVersion": "string", + "EventReportUrl": "string", + "EventResult": "string", + "EventResultDetails": "string", + "EventSchemaVersion": "string", + "EventSeverity": "string", + "EventStartTime": "datetime", + "EventSubType": "string", + "EventType": "string", + "EventVendor": "string", + "NetworkProtocol": "string", + "NetworkProtocolVersion": "string", + "RuleName": "string", + "RuleNumber": "int", + "SentinelScope_CF": "string", + "SourceSystem": "string", + "Src": "string", + "SrcDescription": "string", + "SrcDeviceType": "string", + "SrcDomain": "string", + "SrcDomainType": "string", + "SrcDvcId": "string", + "SrcDvcIdType": "string", + "SrcDvcScope": "string", + "SrcDvcScopeId": "string", + "SrcFQDN": "string", + "SrcGeoCity": "string", + "SrcGeoCountry": "string", + "SrcGeoLatitude": "real", + "SrcGeoLongitude": "real", + "SrcGeoRegion": "string", + "SrcHostname": "string", + "SrcIpAddr": "string", + "SrcOriginalRiskLevel": "string", + "SrcOriginalUserType": "string", + "SrcPortNumber": "int", + "SrcProcessEntityKey": "string", + "SrcProcessGuid": "string", + "SrcProcessId": "string", + "SrcProcessName": "string", + "SrcRiskLevel": "int", + "SrcSystemEntityKey": "string", + "SrcUserAdditionalIds": "dynamic", + "SrcUserEntityKey": "string", + "SrcUserId": "string", + "SrcUserIdType": "string", + "SrcUserScope": "string", + "SrcUserScopeId": "string", + "SrcUserSessionId": "string", + "SrcUserType": "string", + "SrcUsername": "string", + "SrcUsernameType": "string", + "TenantId": "string", + "ThreatCategory": "string", + "ThreatConfidence": "int", + "ThreatField": "string", + "ThreatFirstReportedTime": "string", + "ThreatFirstReportedTime_d": "datetime", + "ThreatId": "string", + "ThreatIpAddr": "string", + "ThreatIsActive": "bool", + "ThreatLastReportedTime": "string", + "ThreatLastReportedTime_d": "datetime", + "ThreatName": "string", + "ThreatOriginalConfidence": "string", + "ThreatOriginalRiskLevel": "int", + "ThreatOriginalRiskLevel_s": "string", + "ThreatRiskLevel": "int", + "TimeGenerated": "datetime", + "TransactionIdHex": "string", + "Type": "string", + "UrlCategory": "string", + "_ResourceId": "string" + }, + "ASimFileEventLogs": { + "ActingApplicationEntityKey": "string", + "ActingProcessCommandLine": "string", + "ActingProcessGuid": "string", + "ActingProcessId": "string", + "ActingProcessName": "string", + "ActorOriginalUserType": "string", + "ActorScope": "string", + "ActorScopeId": "string", + "ActorSessionId": "string", + "ActorUserAadId": "string", + "ActorUserAdditionalIds": "dynamic", + "ActorUserEntityKey": "string", + "ActorUserId": "string", + "ActorUserIdType": "string", + "ActorUserSid": "string", + "ActorUserType": "string", + "ActorUsername": "string", + "ActorUsernameType": "string", + "AdditionalEntities": "dynamic", + "AdditionalFields": "dynamic", + "DvcAction": "string", + "DvcDescription": "string", + "DvcDomain": "string", + "DvcDomainType": "string", + "DvcEntityKey": "string", + "DvcFQDN": "string", + "DvcHostname": "string", + "DvcId": "string", + "DvcIdType": "string", + "DvcInterface": "string", + "DvcIpAddr": "string", + "DvcMacAddr": "string", + "DvcOriginalAction": "string", + "DvcOs": "string", + "DvcOsVersion": "string", + "DvcScope": "string", + "DvcScopeId": "string", + "DvcZone": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventMessage": "string", + "EventOriginalResultDetails": "string", + "EventOriginalSeverity": "string", + "EventOriginalSubType": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventOwner": "string", + "EventProduct": "string", + "EventProductVersion": "string", + "EventReportUrl": "string", + "EventResult": "string", + "EventResultDetails": "string", + "EventSchema": "string", + "EventSchemaVersion": "string", + "EventSeverity": "string", + "EventStartTime": "datetime", + "EventSubType": "string", + "EventType": "string", + "EventVendor": "string", + "HashType": "string", + "HttpUserAgent": "string", + "NetworkApplicationProtocol": "string", + "RuleName": "string", + "RuleNumber": "int", + "SourceSystem": "string", + "SrcDescription": "string", + "SrcDeviceType": "string", + "SrcDomain": "string", + "SrcDomainType": "string", + "SrcDvcId": "string", + "SrcDvcIdType": "string", + "SrcDvcScope": "string", + "SrcDvcScopeId": "string", + "SrcFQDN": "string", + "SrcFileCreationTime": "datetime", + "SrcFileDirectory": "string", + "SrcFileEntityKey": "string", + "SrcFileExtension": "string", + "SrcFileMD5": "string", + "SrcFileMimeType": "string", + "SrcFileName": "string", + "SrcFilePath": "string", + "SrcFilePathType": "string", + "SrcFileSHA1": "string", + "SrcFileSHA256": "string", + "SrcFileSHA512": "string", + "SrcFileSize": "long", + "SrcGeoCity": "string", + "SrcGeoCountry": "string", + "SrcGeoLatitude": "real", + "SrcGeoLongitude": "real", + "SrcGeoRegion": "string", + "SrcHostname": "string", + "SrcIpAddr": "string", + "SrcMacAddr": "string", + "SrcOriginalRiskLevel": "string", + "SrcPortNumber": "int", + "SrcProcessEntityKey": "string", + "SrcRiskLevel": "int", + "SrcSystemEntityKey": "string", + "TargetAppId": "string", + "TargetAppName": "string", + "TargetAppType": "string", + "TargetApplicationEntityKey": "string", + "TargetFileCreationTime": "datetime", + "TargetFileDirectory": "string", + "TargetFileEntityKey": "string", + "TargetFileExtension": "string", + "TargetFileMD5": "string", + "TargetFileMimeType": "string", + "TargetFileName": "string", + "TargetFilePath": "string", + "TargetFilePathType": "string", + "TargetFileSHA1": "string", + "TargetFileSHA256": "string", + "TargetFileSHA512": "string", + "TargetFileSize": "long", + "TargetOriginalAppType": "string", + "TargetUrl": "string", + "TenantId": "string", + "ThreatCategory": "string", + "ThreatConfidence": "int", + "ThreatField": "string", + "ThreatFilePath": "string", + "ThreatFirstReportedTime": "datetime", + "ThreatId": "string", + "ThreatIsActive": "bool", + "ThreatLastReportedTime": "datetime", + "ThreatName": "string", + "ThreatOriginalConfidence": "string", + "ThreatOriginalRiskLevel": "string", + "ThreatRiskLevel": "int", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "ASimNetworkSessionLogs": { + "AdditionalEntities": "dynamic", + "AdditionalFields": "dynamic", + "DstAppId": "string", + "DstAppName": "string", + "DstAppType": "string", + "DstApplicationEntityKey": "string", + "DstBytes": "long", + "DstDescription": "string", + "DstDeviceType": "string", + "DstDomain": "string", + "DstDomainType": "string", + "DstDvcId": "string", + "DstDvcIdType": "string", + "DstFQDN": "string", + "DstGeoCity": "string", + "DstGeoCountry": "string", + "DstGeoLatitude": "real", + "DstGeoLongitude": "real", + "DstGeoRegion": "string", + "DstHostname": "string", + "DstInterfaceGuid": "string", + "DstInterfaceName": "string", + "DstIpAddr": "string", + "DstMacAddr": "string", + "DstNatIpAddr": "string", + "DstNatPortNumber": "int", + "DstOriginalUserType": "string", + "DstPackets": "long", + "DstPortNumber": "int", + "DstSubscriptionId": "string", + "DstSystemEntityKey": "string", + "DstUserAdditionalIds": "dynamic", + "DstUserEntityKey": "string", + "DstUserId": "string", + "DstUserIdType": "string", + "DstUserType": "string", + "DstUsername": "string", + "DstUsernameType": "string", + "DstVlanId": "string", + "DstZone": "string", + "Dvc": "string", + "DvcAction": "string", + "DvcDescription": "string", + "DvcDomain": "string", + "DvcDomainType": "string", + "DvcEntityKey": "string", + "DvcFQDN": "string", + "DvcHostname": "string", + "DvcId": "string", + "DvcIdType": "string", + "DvcInboundInterface": "string", + "DvcInterface": "string", + "DvcIpAddr": "string", + "DvcMacAddr": "string", + "DvcOriginalAction": "string", + "DvcOs": "string", + "DvcOsVersion": "string", + "DvcOutboundInterface": "string", + "DvcSubscriptionId": "string", + "DvcZone": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventMessage": "string", + "EventOriginalResultDetails": "string", + "EventOriginalSeverity": "string", + "EventOriginalSubType": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventProduct": "string", + "EventProductVersion": "string", + "EventReportUrl": "string", + "EventResult": "string", + "EventResultDetails": "string", + "EventSchemaVersion": "string", + "EventSeverity": "string", + "EventStartTime": "datetime", + "EventSubType": "string", + "EventType": "string", + "EventVendor": "string", + "NetworkApplicationProtocol": "string", + "NetworkBytes": "long", + "NetworkConnectionHistory": "string", + "NetworkDirection": "string", + "NetworkDuration": "int", + "NetworkIcmpCode": "int", + "NetworkIcmpType": "string", + "NetworkPackets": "long", + "NetworkProtocol": "string", + "NetworkProtocolVersion": "string", + "NetworkRuleName": "string", + "NetworkRuleNumber": "int", + "NetworkSessionId": "string", + "SourceSystem": "string", + "SrcAppId": "string", + "SrcAppName": "string", + "SrcAppType": "string", + "SrcApplicationEntityKey": "string", + "SrcBytes": "long", + "SrcDescription": "string", + "SrcDeviceType": "string", + "SrcDomain": "string", + "SrcDomainType": "string", + "SrcDvcId": "string", + "SrcDvcIdType": "string", + "SrcFQDN": "string", + "SrcGeoCity": "string", + "SrcGeoCountry": "string", + "SrcGeoLatitude": "real", + "SrcGeoLongitude": "real", + "SrcGeoRegion": "string", + "SrcHostname": "string", + "SrcInterfaceGuid": "string", + "SrcInterfaceName": "string", + "SrcIpAddr": "string", + "SrcMacAddr": "string", + "SrcNatIpAddr": "string", + "SrcNatPortNumber": "int", + "SrcOriginalUserType": "string", + "SrcPackets": "long", + "SrcPortNumber": "int", + "SrcSubscriptionId": "string", + "SrcSystemEntityKey": "string", + "SrcUserAdditionalIds": "dynamic", + "SrcUserEntityKey": "string", + "SrcUserId": "string", + "SrcUserIdType": "string", + "SrcUserType": "string", + "SrcUsername": "string", + "SrcUsernameType": "string", + "SrcVlanId": "string", + "SrcZone": "string", + "TcpFlagsAck": "bool", + "TcpFlagsFin": "bool", + "TcpFlagsPsh": "bool", + "TcpFlagsRst": "bool", + "TcpFlagsSyn": "bool", + "TcpFlagsUrg": "bool", + "TenantId": "string", + "ThreatCategory": "string", + "ThreatConfidence": "int", + "ThreatField": "string", + "ThreatFirstReportedTime": "datetime", + "ThreatId": "string", + "ThreatIpAddr": "string", + "ThreatIsActive": "bool", + "ThreatLastReportedTime": "datetime", + "ThreatName": "string", + "ThreatOriginalConfidence": "string", + "ThreatOriginalRiskLevel": "string", + "ThreatRiskLevel": "int", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "ASimProcessEventLogs": { + "ActingProcessCommandLine": "string", + "ActingProcessCreationTime": "datetime", + "ActingProcessEntityKey": "string", + "ActingProcessFileCompany": "string", + "ActingProcessFileDescription": "string", + "ActingProcessFileInternalName": "string", + "ActingProcessFileOriginalName": "string", + "ActingProcessFileProduct": "string", + "ActingProcessFileSize": "long", + "ActingProcessFileVersion": "string", + "ActingProcessFilename": "string", + "ActingProcessGuid": "string", + "ActingProcessIMPHASH": "string", + "ActingProcessId": "string", + "ActingProcessInjectedAddress": "string", + "ActingProcessIntegrityLevel": "string", + "ActingProcessIsHidden": "bool", + "ActingProcessMD5": "string", + "ActingProcessName": "string", + "ActingProcessSHA1": "string", + "ActingProcessSHA256": "string", + "ActingProcessSHA512": "string", + "ActingProcessTokenElevation": "string", + "ActorOriginalUserType": "string", + "ActorScope": "string", + "ActorScopeId": "string", + "ActorSessionId": "string", + "ActorUserAdditionalIds": "dynamic", + "ActorUserEntityKey": "string", + "ActorUserId": "string", + "ActorUserIdType": "string", + "ActorUserType": "string", + "ActorUsername": "string", + "ActorUsernameType": "string", + "AdditionalEntities": "dynamic", + "AdditionalFields": "dynamic", + "DvcAction": "string", + "DvcDescription": "string", + "DvcDomain": "string", + "DvcDomainType": "string", + "DvcEntityKey": "string", + "DvcFQDN": "string", + "DvcHostname": "string", + "DvcId": "string", + "DvcIdType": "string", + "DvcInterface": "string", + "DvcIpAddr": "string", + "DvcMacAddr": "string", + "DvcOriginalAction": "string", + "DvcOs": "string", + "DvcOsVersion": "string", + "DvcScope": "string", + "DvcScopeId": "string", + "DvcZone": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventMessage": "string", + "EventOriginalResultDetails": "string", + "EventOriginalSeverity": "string", + "EventOriginalSubType": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventOwner": "string", + "EventProduct": "string", + "EventProductVersion": "string", + "EventReportUrl": "string", + "EventResult": "string", + "EventResultDetails": "string", + "EventSchemaVersion": "string", + "EventSeverity": "string", + "EventStartTime": "datetime", + "EventSubType": "string", + "EventType": "string", + "EventVendor": "string", + "ParentProcessCreationTime": "datetime", + "ParentProcessEntityKey": "string", + "ParentProcessFileCompany": "string", + "ParentProcessFileDescription": "string", + "ParentProcessFileProduct": "string", + "ParentProcessFileVersion": "string", + "ParentProcessGuid": "string", + "ParentProcessIMPHASH": "string", + "ParentProcessId": "string", + "ParentProcessInjectedAddress": "string", + "ParentProcessIntegrityLevel": "string", + "ParentProcessIsHidden": "bool", + "ParentProcessMD5": "string", + "ParentProcessName": "string", + "ParentProcessSHA1": "string", + "ParentProcessSHA256": "string", + "ParentProcessSHA512": "string", + "ParentProcessTokenElevation": "string", + "RuleName": "string", + "RuleNumber": "int", + "SourceSystem": "string", + "TargetOriginalUserType": "string", + "TargetProcessCommandLine": "string", + "TargetProcessCreationTime": "datetime", + "TargetProcessCurrentDirectory": "string", + "TargetProcessEntityKey": "string", + "TargetProcessFileCompany": "string", + "TargetProcessFileDescription": "string", + "TargetProcessFileInternalName": "string", + "TargetProcessFileOriginalName": "string", + "TargetProcessFileProduct": "string", + "TargetProcessFileSize": "long", + "TargetProcessFileVersion": "string", + "TargetProcessFilename": "string", + "TargetProcessGuid": "string", + "TargetProcessIMPHASH": "string", + "TargetProcessId": "string", + "TargetProcessInjectedAddress": "string", + "TargetProcessIntegrityLevel": "string", + "TargetProcessIsHidden": "bool", + "TargetProcessMD5": "string", + "TargetProcessName": "string", + "TargetProcessSHA1": "string", + "TargetProcessSHA256": "string", + "TargetProcessSHA512": "string", + "TargetProcessStatusCode": "string", + "TargetProcessTokenElevation": "string", + "TargetScope": "string", + "TargetScopeId": "string", + "TargetUserAdditionalIds": "dynamic", + "TargetUserEntityKey": "string", + "TargetUserId": "string", + "TargetUserIdType": "string", + "TargetUserSessionGuid": "string", + "TargetUserSessionId": "string", + "TargetUserType": "string", + "TargetUsername": "string", + "TargetUsernameType": "string", + "TenantId": "string", + "ThreatCategory": "string", + "ThreatConfidence": "int", + "ThreatField": "string", + "ThreatFirstReportedTime": "datetime", + "ThreatId": "string", + "ThreatIsActive": "bool", + "ThreatLastReportedTime": "datetime", + "ThreatName": "string", + "ThreatOriginalConfidence": "string", + "ThreatOriginalRiskLevel": "string", + "ThreatRiskLevel": "int", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "ASimRegistryEventLogs": { + "ActingProcessCommandLine": "string", + "ActingProcessEntityKey": "string", + "ActingProcessGuid": "string", + "ActingProcessId": "string", + "ActingProcessName": "string", + "ActorOriginalUserType": "string", + "ActorScope": "string", + "ActorScopeId": "string", + "ActorSessionId": "string", + "ActorUserAadId": "string", + "ActorUserAdditionalIds": "dynamic", + "ActorUserEntityKey": "string", + "ActorUserId": "string", + "ActorUserIdType": "string", + "ActorUserSid": "string", + "ActorUserType": "string", + "ActorUsername": "string", + "ActorUsernameType": "string", + "AdditionalEntities": "dynamic", + "AdditionalFields": "dynamic", + "DvcAction": "string", + "DvcDescription": "string", + "DvcDomain": "string", + "DvcDomainType": "string", + "DvcEntityKey": "string", + "DvcFQDN": "string", + "DvcHostname": "string", + "DvcId": "string", + "DvcIdType": "string", + "DvcInterface": "string", + "DvcIpAddr": "string", + "DvcMacAddr": "string", + "DvcOriginalAction": "string", + "DvcOs": "string", + "DvcOsVersion": "string", + "DvcScope": "string", + "DvcScopeId": "string", + "DvcZone": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventMessage": "string", + "EventOriginalResultDetails": "string", + "EventOriginalSeverity": "string", + "EventOriginalSubType": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventOwner": "string", + "EventProduct": "string", + "EventProductVersion": "string", + "EventReportUrl": "string", + "EventResult": "string", + "EventResultDetails": "string", + "EventSchema": "string", + "EventSchemaVersion": "string", + "EventSeverity": "string", + "EventStartTime": "datetime", + "EventSubType": "string", + "EventType": "string", + "EventVendor": "string", + "ParentProcessCommandLine": "string", + "ParentProcessEntityKey": "string", + "ParentProcessGuid": "string", + "ParentProcessId": "string", + "ParentProcessName": "string", + "RegistryKey": "string", + "RegistryPreviousKey": "string", + "RegistryPreviousValue": "string", + "RegistryPreviousValueData": "string", + "RegistryPreviousValueType": "string", + "RegistryValue": "string", + "RegistryValueData": "string", + "RegistryValueType": "string", + "RuleName": "string", + "RuleNumber": "int", + "SourceSystem": "string", + "TenantId": "string", + "ThreatCategory": "string", + "ThreatConfidence": "int", + "ThreatField": "string", + "ThreatFirstReportedTime": "datetime", + "ThreatId": "string", + "ThreatIsActive": "bool", + "ThreatLastReportedTime": "datetime", + "ThreatName": "string", + "ThreatOriginalConfidence": "string", + "ThreatOriginalRiskLevel": "string", + "ThreatRiskLevel": "int", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "ASimUserManagementActivityLogs": { + "ActingAppId": "string", + "ActingAppName": "string", + "ActingAppType": "string", + "ActingApplicationEntityKey": "string", + "ActingOriginalAppType": "string", + "ActorOriginalUserType": "string", + "ActorScope": "string", + "ActorScopeId": "string", + "ActorSessionId": "string", + "ActorUserAadId": "string", + "ActorUserAdditionalIds": "dynamic", + "ActorUserEntityKey": "string", + "ActorUserId": "string", + "ActorUserIdType": "string", + "ActorUserSid": "string", + "ActorUserType": "string", + "ActorUsername": "string", + "ActorUsernameType": "string", + "AdditionalEntities": "dynamic", + "AdditionalFields": "dynamic", + "DvcAction": "string", + "DvcDescription": "string", + "DvcDomain": "string", + "DvcDomainType": "string", + "DvcEntityKey": "string", + "DvcFQDN": "string", + "DvcHostname": "string", + "DvcId": "string", + "DvcIdType": "string", + "DvcInterface": "string", + "DvcIpAddr": "string", + "DvcMacAddr": "string", + "DvcOriginalAction": "string", + "DvcOs": "string", + "DvcOsVersion": "string", + "DvcScope": "string", + "DvcScopeId": "string", + "DvcZone": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventMessage": "string", + "EventOriginalResultDetails": "string", + "EventOriginalSeverity": "string", + "EventOriginalSubType": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventOwner": "string", + "EventProduct": "string", + "EventProductVersion": "string", + "EventReportUrl": "string", + "EventResult": "string", + "EventResultDetails": "string", + "EventSchema": "string", + "EventSchemaVersion": "string", + "EventSeverity": "string", + "EventStartTime": "datetime", + "EventSubType": "string", + "EventType": "string", + "EventVendor": "string", + "GroupId": "string", + "GroupIdType": "string", + "GroupName": "string", + "GroupNameType": "string", + "GroupOriginalType": "string", + "GroupType": "string", + "HttpUserAgent": "string", + "NewPropertyValue": "string", + "PreviousPropertyValue": "string", + "RuleName": "string", + "RuleNumber": "int", + "SourceSystem": "string", + "SrcDescription": "string", + "SrcDeviceType": "string", + "SrcDomain": "string", + "SrcDomainType": "string", + "SrcDvcId": "string", + "SrcDvcIdType": "string", + "SrcDvcScope": "string", + "SrcDvcScopeId": "string", + "SrcFQDN": "string", + "SrcGeoCity": "string", + "SrcGeoCountry": "string", + "SrcGeoLatitude": "real", + "SrcGeoLongitude": "real", + "SrcGeoRegion": "string", + "SrcHostname": "string", + "SrcIpAddr": "string", + "SrcMacAddr": "string", + "SrcOriginalRiskLevel": "string", + "SrcPortNumber": "int", + "SrcRiskLevel": "int", + "SrcSystemEntityKey": "string", + "TargetOriginalUserType": "string", + "TargetUserAdditionalIds": "dynamic", + "TargetUserEntityKey": "string", + "TargetUserId": "string", + "TargetUserIdType": "string", + "TargetUserScope": "string", + "TargetUserScopeId": "string", + "TargetUserSessionId": "string", + "TargetUserType": "string", + "TargetUserUid": "string", + "TargetUsername": "string", + "TargetUsernameType": "string", + "TenantId": "string", + "ThreatCategory": "string", + "ThreatConfidence": "int", + "ThreatField": "string", + "ThreatFirstReportedTime": "datetime", + "ThreatId": "string", + "ThreatIsActive": "bool", + "ThreatLastReportedTime": "datetime", + "ThreatName": "string", + "ThreatOriginalConfidence": "string", + "ThreatOriginalRiskLevel": "string", + "ThreatRiskLevel": "int", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "ASimWebSessionLogs": { + "AdditionalEntities": "dynamic", + "AdditionalFields": "dynamic", + "DstAppId": "string", + "DstAppName": "string", + "DstAppType": "string", + "DstApplicationEntityKey": "string", + "DstBytes": "long", + "DstDeviceType": "string", + "DstDomain": "string", + "DstDomainType": "string", + "DstDvcId": "string", + "DstDvcIdType": "string", + "DstDvcScope": "string", + "DstDvcScopeId": "string", + "DstFQDN": "string", + "DstGeoCity": "string", + "DstGeoCountry": "string", + "DstGeoLatitude": "real", + "DstGeoLongitude": "real", + "DstGeoRegion": "string", + "DstHostname": "string", + "DstIpAddr": "string", + "DstMacAddr": "string", + "DstNatIpAddr": "string", + "DstNatPortNumber": "int", + "DstOriginalUserType": "string", + "DstPackets": "long", + "DstPortNumber": "int", + "DstSystemEntityKey": "string", + "DstUserAdditionalIds": "dynamic", + "DstUserEntityKey": "string", + "DstUserId": "string", + "DstUserIdType": "string", + "DstUserType": "string", + "DstUsername": "string", + "DstUsernameType": "string", + "Dvc": "string", + "DvcAction": "string", + "DvcDomain": "string", + "DvcDomainType": "string", + "DvcEntityKey": "string", + "DvcFQDN": "string", + "DvcHostname": "string", + "DvcId": "string", + "DvcIdType": "string", + "DvcIpAddr": "string", + "DvcOriginalAction": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventMessage": "string", + "EventOriginalResultDetails": "string", + "EventOriginalSeverity": "string", + "EventOriginalSubType": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventOwner": "string", + "EventProduct": "string", + "EventProductVersion": "string", + "EventReportUrl": "string", + "EventResult": "string", + "EventResultDetails": "string", + "EventSchemaVersion": "string", + "EventSeverity": "string", + "EventStartTime": "datetime", + "EventSubType": "string", + "EventType": "string", + "EventVendor": "string", + "FileContentType": "string", + "FileMD5": "string", + "FileName": "string", + "FileSHA1": "string", + "FileSHA256": "string", + "FileSHA512": "string", + "FileSize": "int", + "HttpContentFormat": "string", + "HttpContentType": "string", + "HttpHost": "string", + "HttpReferrer": "string", + "HttpRequestMethod": "string", + "HttpRequestTime": "int", + "HttpRequestXff": "string", + "HttpResponseTime": "int", + "HttpUserAgent": "string", + "HttpVersion": "string", + "NetworkApplicationProtocol": "string", + "NetworkBytes": "long", + "NetworkConnectionHistory": "string", + "NetworkDirection": "string", + "NetworkDuration": "int", + "NetworkIcmpCode": "int", + "NetworkIcmpType": "string", + "NetworkPackets": "long", + "NetworkProtocol": "string", + "NetworkProtocolVersion": "string", + "NetworkSessionId": "string", + "Rule": "string", + "RuleName": "string", + "RuleNumber": "int", + "SourceSystem": "string", + "SrcAppId": "string", + "SrcAppName": "string", + "SrcAppType": "string", + "SrcApplicationEntityKey": "string", + "SrcBytes": "long", + "SrcDeviceType": "string", + "SrcDomain": "string", + "SrcDomainType": "string", + "SrcDvcId": "string", + "SrcDvcIdType": "string", + "SrcDvcScope": "string", + "SrcDvcScopeId": "string", + "SrcFQDN": "string", + "SrcGeoCity": "string", + "SrcGeoCountry": "string", + "SrcGeoLatitude": "real", + "SrcGeoLongitude": "real", + "SrcGeoRegion": "string", + "SrcHostname": "string", + "SrcIpAddr": "string", + "SrcMacAddr": "string", + "SrcNatIpAddr": "string", + "SrcNatPortNumber": "int", + "SrcOriginalUserType": "string", + "SrcPackets": "long", + "SrcPortNumber": "int", + "SrcProcessGuid": "string", + "SrcProcessId": "string", + "SrcProcessName": "string", + "SrcSystemEntityKey": "string", + "SrcUserAdditionalIds": "dynamic", + "SrcUserEntityKey": "string", + "SrcUserId": "string", + "SrcUserIdType": "string", + "SrcUserScope": "string", + "SrcUserScopeId": "string", + "SrcUserType": "string", + "SrcUsername": "string", + "SrcUsernameType": "string", + "TenantId": "string", + "ThreatCategory": "string", + "ThreatConfidence": "int", + "ThreatField": "string", + "ThreatFirstReportedTime": "datetime", + "ThreatId": "string", + "ThreatIpAddr": "string", + "ThreatIsActive": "bool", + "ThreatLastReportedTime": "datetime", + "ThreatName": "string", + "ThreatOriginalConfidence": "string", + "ThreatOriginalRiskLevel": "string", + "ThreatRiskLevel": "int", + "TimeGenerated": "datetime", + "Type": "string", + "Url": "string", + "UrlCategory": "string", + "UrlOriginal": "string", + "_ResourceId": "string" + }, + "AWSCloudTrail": { + "APIVersion": "string", + "AWSRegion": "string", + "AdditionalEventData": "string", + "AwsEventId": "string", + "AwsRequestId": "string", + "AwsRequestId_": "string", + "Category": "string", + "CidrIp": "string", + "CipherSuite": "string", + "ClientProvidedHostHeader": "string", + "DestinationPort": "string", + "EC2RoleDelivery": "string", + "ErrorCode": "string", + "ErrorMessage": "string", + "EventName": "string", + "EventSource": "string", + "EventTypeName": "string", + "EventVersion": "string", + "IPAddress_CF": "string", + "IpProtocol": "string", + "ManagementEvent": "bool", + "OperationName": "string", + "ReadOnly": "bool", + "RecipientAccountId": "string", + "RequestParameters": "string", + "Resources": "string", + "ResponseElements": "string", + "ServiceEventDetails": "string", + "SessionCreationDate": "datetime", + "SessionIssuerAccountId": "string", + "SessionIssuerArn": "string", + "SessionIssuerPrincipalId": "string", + "SessionIssuerType": "string", + "SessionIssuerUserName": "string", + "SessionMfaAuthenticated": "bool", + "SharedEventId": "string", + "SourceIpAddress": "string", + "SourcePort": "string", + "SourceSystem": "string", + "SubnetMask_CF": "int", + "TenantId": "string", + "TimeGenerated": "datetime", + "TlsVersion": "string", + "Type": "string", + "UserAgent": "string", + "UserIdentityAccessKeyId": "string", + "UserIdentityAccountId": "string", + "UserIdentityArn": "string", + "UserIdentityInvokedBy": "string", + "UserIdentityPrincipalid": "string", + "UserIdentityStoreArn": "string", + "UserIdentityType": "string", + "UserIdentityUserId": "string", + "UserIdentityUserName": "string", + "VpcEndpointId": "string" + }, + "AWSGuardDuty": { + "AccountId": "string", + "ActivityType": "string", + "Arn": "string", + "Description": "string", + "Id": "string", + "Partition": "string", + "Region": "string", + "ResourceDetails": "dynamic", + "SchemaVersion": "string", + "ServiceDetails": "dynamic", + "Severity": "int", + "SourceSystem": "string", + "TenantId": "string", + "TimeCreated": "datetime", + "TimeGenerated": "datetime", + "Title": "string", + "Type": "string" + }, + "AWSVPCFlow": { + "AccountId": "string", + "Action": "string", + "AzId": "string", + "Bytes": "long", + "DstAddr": "string", + "DstPort": "int", + "EcsClusterArn": "string", + "EcsClusterName": "string", + "EcsContainerId": "string", + "EcsContainerInstanceArn": "string", + "EcsContainerInstanceId": "string", + "EcsSecondContainerId": "string", + "EcsServiceName": "string", + "EcsTaskArn": "string", + "EcsTaskDefinitionArn": "string", + "EcsTaskId": "string", + "End": "datetime", + "FlowDirection": "string", + "InstanceId": "string", + "InterfaceId": "string", + "LogStatus": "string", + "Packets": "int", + "PktDstAddr": "string", + "PktDstAwsService": "string", + "PktSrcAddr": "string", + "PktSrcAwsService": "string", + "Protocol": "int", + "Region": "string", + "SourceSystem": "string", + "SrcAddr": "string", + "SrcPort": "int", + "Start": "datetime", + "SublocationId": "string", + "SublocationType": "string", + "SubnetId": "string", + "TcpFlags": "int", + "TenantId": "string", + "TimeGenerated": "datetime", + "TrafficPath": "string", + "TrafficType": "string", + "Type": "string", + "Version": "int", + "VpcId": "string" + }, + "AWSWAF": { + "Action": "string", + "Args": "string", + "CaptchaResponse": "dynamic", + "ChallengeResponse": "dynamic", + "ClientIp": "string", + "Country": "string", + "ExcludedRules": "dynamic", + "FormatVersion": "string", + "Headers": "dynamic", + "HttpMethod": "string", + "HttpRequest": "dynamic", + "HttpSourceId": "string", + "HttpSourceName": "string", + "HttpVersion": "string", + "Ja3Fingerprint": "string", + "Labels": "dynamic", + "NonTerminatingMatchingRules": "dynamic", + "OversizeFields": "dynamic", + "RateBasedRuleList": "dynamic", + "RequestHeadersInserted": "dynamic", + "RequestId": "string", + "ResponseCodeSent": "int", + "RuleGroupId": "string", + "RuleGroupList": "dynamic", + "SourceSystem": "string", + "TenantId": "string", + "TerminatingRule": "dynamic", + "TerminatingRuleId": "string", + "TerminatingRuleMatchDetails": "dynamic", + "TerminatingRuleType": "string", + "TimeGenerated": "datetime", + "Type": "string", + "Uri": "string", + "WebAclId": "string" + }, + "AZFWApplicationRule": { + "Action": "string", + "ActionReason": "string", + "DestinationPort": "int", + "Fqdn": "string", + "IsExplicitProxyRequest": "bool", + "IsTlsInspected": "bool", + "Policy": "string", + "Protocol": "string", + "Rule": "string", + "RuleCollection": "string", + "RuleCollectionGroup": "string", + "SourceIp": "string", + "SourcePort": "int", + "SourceSystem": "string", + "TargetUrl": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "WebCategory": "string", + "_ResourceId": "string" + }, + "AZFWDnsQuery": { + "DnssecOkBit": "bool", + "EDNS0BufferSize": "int", + "ErrorMessage": "string", + "ErrorNumber": "int", + "Protocol": "string", + "QueryClass": "string", + "QueryId": "int", + "QueryName": "string", + "QueryType": "string", + "RequestDurationSecs": "real", + "RequestSize": "int", + "ResponseCode": "string", + "ResponseFlags": "string", + "ResponseSize": "int", + "SourceIp": "string", + "SourcePort": "int", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "AZFWIdpsSignature": { + "Action": "string", + "Category": "string", + "Description": "string", + "DestinationIp": "string", + "DestinationPort": "int", + "Protocol": "string", + "Severity": "int", + "SignatureId": "string", + "SourceIp": "string", + "SourcePort": "int", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "AZFWNatRule": { + "DestinationIp": "string", + "DestinationPort": "int", + "Policy": "string", + "Protocol": "string", + "Rule": "string", + "RuleCollection": "string", + "RuleCollectionGroup": "string", + "SourceIp": "string", + "SourcePort": "int", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TranslatedIp": "string", + "TranslatedPort": "int", + "Type": "string", + "_ResourceId": "string" + }, + "AZFWNetworkRule": { + "Action": "string", + "ActionReason": "string", + "DestinationIp": "string", + "DestinationPort": "int", + "Policy": "string", + "Protocol": "string", + "Rule": "string", + "RuleCollection": "string", + "RuleCollectionGroup": "string", + "SourceIp": "string", + "SourcePort": "int", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "AZFWThreatIntel": { + "Action": "string", + "DestinationIp": "string", + "DestinationPort": "int", + "Fqdn": "string", + "IsTlsInspected": "bool", + "Protocol": "string", + "SourceIp": "string", + "SourcePort": "int", + "SourceSystem": "string", + "TargetUrl": "string", + "TenantId": "string", + "ThreatDescription": "string", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "Alert": { + "AlertContext": "string", + "AlertDescription": "string", + "AlertError": "string", + "AlertId": "string", + "AlertName": "string", + "AlertPriority": "string", + "AlertRuleId": "string", + "AlertRuleInstanceId": "string", + "AlertSeverity": "string", + "AlertState": "string", + "AlertStatus": "int", + "AlertTypeDescription": "string", + "AlertTypeNumber": "int", + "AlertUniqueId": "string", + "AlertValue": "int", + "Comments": "string", + "Computer": "string", + "Custom1": "string", + "Custom10": "string", + "Custom2": "string", + "Custom3": "string", + "Custom4": "string", + "Custom5": "string", + "Custom6": "string", + "Custom7": "string", + "Custom8": "string", + "Custom9": "string", + "Expression": "string", + "Flags": "int", + "FlagsDescription": "string", + "HostName": "string", + "LastModifiedBy": "string", + "LinkToSearchResults": "string", + "MG": "string", + "ManagementGroupName": "string", + "ObjectDisplayName": "string", + "PriorityNumber": "int", + "Query": "string", + "QueryExecutionEndTime": "datetime", + "QueryExecutionStartTime": "datetime", + "RemediationJobId": "string", + "RemediationRunbookName": "string", + "RepeatCount": "int", + "ResolvedBy": "string", + "ResourceId": "string", + "ResourceType": "string", + "ResourceValue": "string", + "RootObjectName": "string", + "ServiceDeskConnectionName": "string", + "ServiceDeskId": "string", + "ServiceDeskWorkItemLink": "string", + "ServiceDeskWorkItemType": "string", + "SourceDisplayName": "string", + "SourceFullName": "string", + "SourceSystem": "string", + "StateType": "string", + "StatusDescription": "string", + "TemplateId": "string", + "TenantId": "string", + "ThresholdOperator": "string", + "ThresholdValue": "int", + "TicketId": "string", + "TimeGenerated": "datetime", + "TimeLastModified": "datetime", + "TimeRaised": "datetime", + "TimeResolved": "datetime", + "TriggerId": "string", + "Type": "string", + "Url": "string", + "ValueDescription": "string", + "ValueFlags": "int", + "ValueFlagsDescription": "string", + "_ResourceId": "string" + }, + "AlertEvidence": { + "AccountDomain": "string", + "AccountName": "string", + "AccountObjectId": "string", + "AccountSid": "string", + "AccountUpn": "string", + "AdditionalFields": "string", + "AlertId": "string", + "Application": "string", + "ApplicationId": "int", + "AttackTechniques": "string", + "Categories": "string", + "CloudPlatform": "string", + "CloudResource": "string", + "DetectionSource": "string", + "DeviceId": "string", + "DeviceName": "string", + "EmailSubject": "string", + "EntityType": "string", + "EvidenceDirection": "string", + "EvidenceRole": "string", + "FileName": "string", + "FileSize": "long", + "FolderPath": "string", + "LocalIP": "string", + "MachineGroup": "string", + "NetworkMessageId": "string", + "OAuthApplicationId": "string", + "ProcessCommandLine": "string", + "RegistryKey": "string", + "RegistryValueData": "string", + "RegistryValueName": "string", + "RemoteIP": "string", + "RemoteUrl": "string", + "ResourceID": "string", + "ResourceType": "string", + "SHA1": "string", + "SHA256": "string", + "SentinelWorkspaceIds": "string", + "ServiceSource": "string", + "Severity": "string", + "SourceSystem": "string", + "SubscriptionId": "string", + "TenantId": "string", + "ThreatFamily": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Title": "string", + "Type": "string" + }, + "AlertInfo": { + "AlertId": "string", + "AttackTechniques": "string", + "Category": "string", + "DetectionSource": "string", + "MachineGroup": "string", + "SentinelWorkspaceIds": "string", + "ServiceSource": "string", + "Severity": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Title": "string", + "Type": "string" + }, + "Anomalies": { + "ActivityInsights": "dynamic", + "AnomalyDetails": "dynamic", + "AnomalyReasons": "dynamic", + "AnomalyTemplateId": "string", + "AnomalyTemplateName": "string", + "AnomalyTemplateVersion": "string", + "Description": "string", + "DestinationDevice": "string", + "DestinationIpAddress": "string", + "DestinationLocation": "dynamic", + "DeviceInsights": "dynamic", + "EndTime": "datetime", + "Entities": "dynamic", + "ExtendedLinks": "dynamic", + "ExtendedProperties": "dynamic", + "Id": "string", + "RuleConfigVersion": "string", + "RuleId": "string", + "RuleName": "string", + "RuleStatus": "string", + "Score": "real", + "SourceDevice": "string", + "SourceIpAddress": "string", + "SourceLocation": "dynamic", + "SourceSystem": "string", + "StartTime": "datetime", + "Tactics": "string", + "Techniques": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "UserInsights": "dynamic", + "UserName": "string", + "UserPrincipalName": "string", + "VendorName": "string", + "WorkspaceId": "string" + }, + "AuditLogs": { + "AADOperationType": "string", + "AADTenantId": "string", + "ActivityDateTime": "datetime", + "ActivityDisplayName": "string", + "AdditionalDetails": "dynamic", + "Category": "string", + "CorrelationId": "string", + "DurationMs": "long", + "Id": "string", + "Identity": "string", + "InitiatedBy": "dynamic", + "Level": "string", + "Location": "string", + "LoggedByService": "string", + "OperationName": "string", + "OperationVersion": "string", + "Resource": "string", + "ResourceGroup": "string", + "ResourceId": "string", + "ResourceProvider": "string", + "Result": "string", + "ResultDescription": "string", + "ResultReason": "string", + "ResultSignature": "string", + "ResultType": "string", + "SourceSystem": "string", + "TargetResources": "dynamic", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string" + }, + "AzureActivity": { + "ActivityStatus": "string", + "ActivityStatusValue": "string", + "ActivitySubstatus": "string", + "ActivitySubstatusValue": "string", + "Authorization": "string", + "Authorization_d": "dynamic", + "Caller": "string", + "CallerIpAddress": "string", + "Category": "string", + "CategoryValue": "string", + "Claims": "string", + "Claims_d": "dynamic", + "CorrelationId": "string", + "EventDataId": "string", + "EventSubmissionTimestamp": "datetime", + "HTTPRequest": "string", + "Hierarchy": "string", + "Level": "string", + "OperationId": "string", + "OperationName": "string", + "OperationNameValue": "string", + "Properties": "string", + "Properties_d": "dynamic", + "Resource": "string", + "ResourceGroup": "string", + "ResourceId": "string", + "ResourceProvider": "string", + "ResourceProviderValue": "string", + "SourceSystem": "string", + "SubscriptionId": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "AzureDiagnostics": { + "ActivityId_g": "string", + "ActivityName_s": "string", + "AdditionalFields": "dynamic", + "Age_DatFrozenxid_d": "real", + "Age_DatMinmxid_d": "real", + "Analyze_count_d": "real", + "AppImage_s": "string", + "AppType_s": "string", + "AppVersion_s": "string", + "Application_name_s": "string", + "AssetIdentity_g": "string", + "AttackVectors_s": "string", + "AuthId_g": "string", + "AuthId_s": "string", + "AuthType_s": "string", + "Autoanalyze_count_d": "real", + "Autovacuum_count_d": "real", + "Autovacuum_freeze_max_age_d": "real", + "Autovacuum_multixact_freeze_max_age_d": "real", + "Backend_start_t": "datetime", + "Backend_type_s": "string", + "Backend_xid_d": "real", + "Backend_xid_s": "string", + "Backend_xmin_d": "real", + "Backend_xmin_s": "string", + "CallerIPAddress": "string", + "Caller_s": "string", + "Category": "string", + "ChildEntityName_s": "string", + "ChildEntityType_s": "string", + "ClientIp_s": "string", + "ClientOperationId_g": "string", + "Client_addr_s": "string", + "Collection_time_t": "datetime", + "Computer": "string", + "CorrelationId": "string", + "CorrelationRequestId_g": "string", + "Count_d": "real", + "DNSLatencyMicroseconds_s": "string", + "DatConnLimit_d": "real", + "DatFrozenxid_d": "real", + "DatMinmxid_d": "real", + "DatabaseName_s": "string", + "Database_name_s": "string", + "Datdba_d": "real", + "Datid_d": "real", + "Datid_s": "string", + "DeploymentVersion_s": "string", + "Description_s": "string", + "Documents_d": "real", + "DropReasons_s": "string", + "DurationMs": "long", + "ElasticPoolName_s": "string", + "EntityName_s": "string", + "EntityType_s": "string", + "Environment_s": "string", + "ErrorInfo_s": "string", + "EvaluationDetails_s": "string", + "EventName_s": "string", + "GatewayManagerVersion_s": "string", + "IPAddress": "string", + "Idx_scan_d": "real", + "Idx_scan_s": "string", + "Idx_tup_fetch_d": "real", + "Idx_tup_fetch_s": "string", + "IndexName_s": "string", + "IsComplianceCheck_b": "bool", + "JobId_g": "string", + "Level": "string", + "Level_d": "real", + "LogicalServerName_s": "string", + "MG": "string", + "ManagementGroupName": "string", + "Message": "string", + "MitigationPeriodEnd_t": "datetime", + "MitigationPeriodStart_t": "datetime", + "N_dead_tup_d": "real", + "N_live_tup_d": "real", + "N_mod_since_analyze_d": "real", + "N_tup_del_d": "real", + "N_tup_hot_upd_d": "real", + "N_tup_ins_d": "real", + "N_tup_upd_d": "real", + "NamespaceName_s": "string", + "NetworkType_s": "string", + "ObjectName_s": "string", + "ObjectType_s": "string", + "OperationName": "string", + "OperationVersion": "string", + "OriginalPrimaryServerName_s": "string", + "POP_s": "string", + "PartitionId_s": "string", + "Pid_d": "real", + "Properties_s": "string", + "Protocol_s": "string", + "Protocols_s": "string", + "Query_s": "string", + "Query_start_s": "string", + "Query_start_t": "datetime", + "RawData": "string", + "Region_s": "string", + "Remaining_mxids_till_emergency_autovacuum_d": "real", + "Remaining_mxids_till_wraparound_d": "real", + "Remaining_xids_till_emergency_autovacuum_d": "real", + "Remaining_xids_till_wraparound_d": "real", + "ReplicaRole_s": "string", + "ReportType_s": "string", + "Resource": "string", + "ResourceGroup": "string", + "ResourceId": "string", + "ResourceProvider": "string", + "ResourceType": "string", + "ResultDescription": "string", + "ResultSignature": "string", + "ResultType": "string", + "RunOn_s": "string", + "RunbookName_s": "string", + "ScaleUnit_s": "string", + "Schemaname_s": "string", + "Seq_scan_d": "real", + "Seq_tup_read_d": "real", + "ServerLocation_s": "string", + "ServerType_s": "string", + "ServerVersion_s": "string", + "Session_duration_s": "string", + "Sku_s": "string", + "SourceContinents_s": "string", + "SourceSystem": "string", + "State_change_s": "string", + "State_change_t": "datetime", + "State_s": "string", + "Status_s": "string", + "StreamType_s": "string", + "SubscriptionId": "string", + "Tables_analyzed_d": "real", + "Tables_autoanalyzed_d": "real", + "Tables_autovacuumed_d": "real", + "Tables_counter_d": "real", + "Tables_vacuumed_d": "real", + "TaskName_s": "string", + "TenantId": "string", + "Tenant_g": "string", + "Tenant_s": "string", + "TimeGenerated": "datetime", + "TopSourceASNs_s": "string", + "TopSourceCountriesForDroppedPackets_s": "string", + "TopSourceCountries_s": "string", + "Total_remaining_mxids_d": "real", + "Total_remaining_xids_d": "real", + "TrafficOverview_s": "string", + "Type": "string", + "Usesysid_d": "real", + "Usesysid_s": "string", + "Vacuum_count_d": "real", + "Vacuum_freeze_min_age_d": "real", + "Vacuum_multixact_freeze_min_age_d": "real", + "WAFEvaluationTime_s": "string", + "WAFMode_s": "string", + "WAFPolicyID_s": "string", + "Wait_event_s": "string", + "Wait_event_type_s": "string", + "Xact_start_s": "string", + "Xact_start_t": "datetime", + "_ResourceId": "string", + "_schema_s": "string", + "aadAppliedRoleAssignmentId_g": "string", + "aadAppliedRoleAssignmentId_s": "string", + "aadPrincipalId_g": "string", + "aadPrincipalId_s": "string", + "actionCount_d": "real", + "action_s": "string", + "activityId_g": "string", + "addedAccessPolicy_ObjectId_g": "string", + "addedAccessPolicy_Permissions_certificates_s": "string", + "addedAccessPolicy_Permissions_keys_s": "string", + "addedAccessPolicy_Permissions_secrets_s": "string", + "addedAccessPolicy_Permissions_storage_s": "string", + "addedAccessPolicy_TenantId_g": "string", + "addrAuthType_s": "string", + "apiId_s": "string", + "apiKindResourceType_s": "string", + "apiKind_s": "string", + "apiRevision_s": "string", + "apimSubscriptionId_s": "string", + "appliedAssignmentId_g": "string", + "authTokenType_s": "string", + "backendMethod_s": "string", + "backendPoolName_s": "string", + "backendProtocol_s": "string", + "backendResponseCode_d": "real", + "backendSettingName_s": "string", + "backendTime_d": "real", + "backendUrl_s": "string", + "cacheStatus_s": "string", + "cacheTime_d": "real", + "cache_s": "string", + "certificateEnrollmentProperties_attributes_created_d": "real", + "certificateEnrollmentProperties_attributes_enabled_b": "bool", + "certificateEnrollmentProperties_attributes_updated_d": "real", + "certificateEnrollmentProperties_certificateProperties_exp_t": "datetime", + "certificateEnrollmentProperties_certificateProperties_nbf_t": "datetime", + "certificateEnrollmentProperties_certificateProperties_sha1_s": "string", + "certificateEnrollmentProperties_certificateProperties_sha256_s": "string", + "certificateEnrollmentProperties_certificateProperties_subject_s": "string", + "certificateEnrollmentProperties_id_s": "string", + "certificateEnrollmentProperties_keyProperties_size_d": "real", + "certificateEnrollmentProperties_keyProperties_type_s": "string", + "certificateEnrollmentProperties_secretProperties_type_s": "string", + "certificatePolicyProperties_attributes_enabled_b": "bool", + "certificatePolicyProperties_certificateIssuerProperties_name_s": "string", + "certificatePolicyProperties_certificateProperties_subject_s": "string", + "certificatePolicyProperties_certificateProperties_validityInMonths_d": "real", + "certificatePolicyProperties_keyProperties_export_b": "bool", + "certificatePolicyProperties_keyProperties_reuse_b": "bool", + "certificatePolicyProperties_keyProperties_size_d": "real", + "certificatePolicyProperties_keyProperties_type_s": "string", + "certificatePolicyProperties_secretProperties_type_s": "string", + "certificateProperties_attributes_enabled_b": "bool", + "certificateProperties_exp_t": "datetime", + "certificateProperties_nbf_t": "datetime", + "certificateProperties_sha1_s": "string", + "certificateProperties_sha256_s": "string", + "certificateProperties_subject_s": "string", + "clientCountry_s": "string", + "clientIP_s": "string", + "clientInfo_AppId_g": "string", + "clientInfo_ClientRequestId_g": "string", + "clientInfo_IpAddress_s": "string", + "clientInfo_Issuer_s": "string", + "clientInfo_ObjectId_g": "string", + "clientInfo_PrincipalName_s": "string", + "clientInfo_TenantId_g": "string", + "clientInfo_s": "string", + "clientIpAddress_s": "string", + "clientIp_s": "string", + "clientJA4FingerPrint_s": "string", + "clientPort_d": "real", + "clientPort_s": "string", + "clientProtocol_s": "string", + "clientResponseTime_d": "real", + "clientTime_d": "real", + "clientTlsCipherSuite_s": "string", + "clientTlsVersion_s": "string", + "code_s": "string", + "collectionName_s": "string", + "collectionRid_s": "string", + "conditions_None_s": "string", + "conditions_destinationIP_s": "string", + "conditions_destinationPortRange_s": "string", + "conditions_protocols_s": "string", + "conditions_sourceIP_s": "string", + "conditions_sourcePortRange_s": "string", + "connectionLatencyMilliseconds_s": "string", + "connectionMode_s": "string", + "connectionSerialNumber_d": "real", + "contentType_s": "string", + "correlation_actionTrackingId_g": "string", + "correlation_clientKeywords_s": "string", + "correlation_clientTrackingId_s": "string", + "correlation_parentFlowId_g": "string", + "correlation_parentRunId_s": "string", + "cors_s": "string", + "count_d": "real", + "databaseName_s": "string", + "databaseRid_s": "string", + "defaultConsistencyLevel_s": "string", + "destPort_s": "string", + "destPublicIpAddress_s": "string", + "detail_log_s": "string", + "details_data_s": "string", + "details_file_s": "string", + "details_line_s": "string", + "details_matches_s": "string", + "details_message_s": "string", + "details_msg_s": "string", + "direction_s": "string", + "domain_s": "string", + "duration_s": "string", + "enableAllLogsProperty_s": "string", + "enableAuditProperty_s": "string", + "enableAutomaticFailover_s": "string", + "enableCassandraRequestsTrace_s": "string", + "enableControlPlaneRequestsTrace_s": "string", + "enableDataPlaneRequestsTrace15M_s": "string", + "enableDataPlaneRequestsTrace5M_s": "string", + "enableDataPlaneRequestsTrace_s": "string", + "enableGremlinRequestsTrace_s": "string", + "enableMongoRequestsTrace_s": "string", + "enableMultipleWriteLocations_s": "string", + "enablePrivateEndpointConnection_s": "string", + "enableVirtualNetworkFilter_s": "string", + "endTime_t": "datetime", + "endpoint_s": "string", + "engine_s": "string", + "errorInfo_s": "string", + "errorLevel_s": "string", + "error_code_s": "string", + "error_info_s": "string", + "error_message_s": "string", + "errors_s": "string", + "eventTimestamp_s": "string", + "event_s": "string", + "executionClusterType_s": "string", + "fired_b": "bool", + "healthProbeId_g": "string", + "hint_s": "string", + "hostName_s": "string", + "host_s": "string", + "hostname_s": "string", + "httpMethod_s": "string", + "httpStatusCode_d": "real", + "httpStatusCode_s": "string", + "httpStatusDetails_s": "string", + "httpStatus_d": "real", + "httpVerb_s": "string", + "httpVersion_s": "string", + "httpstatusCode_s": "string", + "id_s": "string", + "identity_claim_amr_s": "string", + "identity_claim_appid_g": "string", + "identity_claim_appidacr_s": "string", + "identity_claim_http_schemas_microsoft_com_claims_authnmethodsreferences_s": "string", + "identity_claim_http_schemas_microsoft_com_identity_claims_objectidentifier_g": "string", + "identity_claim_http_schemas_microsoft_com_identity_claims_scope_s": "string", + "identity_claim_http_schemas_xmlsoap_org_ws_2005_05_identity_claims_name_s": "string", + "identity_claim_http_schemas_xmlsoap_org_ws_2005_05_identity_claims_upn_s": "string", + "identity_claim_idtyp_s": "string", + "identity_claim_ipaddr_s": "string", + "identity_claim_oid_g": "string", + "identity_claim_scp_s": "string", + "identity_claim_unique_name_s": "string", + "identity_claim_upn_s": "string", + "identity_claim_xms_az_nwperimid_s": "string", + "identity_claim_xms_az_rid_s": "string", + "identity_claim_xms_az_tm_s": "string", + "identity_claim_xms_mirid_s": "string", + "instanceId_s": "string", + "instance_s": "string", + "ipRangeFilter_s": "string", + "isAccessPolicyMatch_b": "bool", + "isAddressAuthorized_b": "bool", + "isRbacAuthorized_b": "bool", + "isRequestSuccess_b": "bool", + "isV2Threshold_b": "bool", + "keyProperties_attributes_enabled_b": "bool", + "keyProperties_attributes_exportable_b": "bool", + "keyProperties_attributes_hsmPlatform_s": "string", + "keyProperties_curve_s": "string", + "keyProperties_operations_s": "string", + "keyProperties_size_d": "real", + "keyProperties_type_s": "string", + "keyType_s": "string", + "lastError_elapsed_d": "real", + "lastError_message_s": "string", + "lastError_path_s": "string", + "lastError_reason_s": "string", + "lastError_section_s": "string", + "lastError_source_s": "string", + "listenerName_s": "string", + "location_s": "string", + "macAddress_s": "string", + "matchedConnections_d": "real", + "maxStalenessIntervalInSeconds_s": "string", + "maxStalenessPrefix_s": "string", + "metadataOverflowContentLength_d": "real", + "method_s": "string", + "msg_s": "string", + "noOfConnectionRequests_d": "real", + "operationId_s": "string", + "operationType_s": "string", + "originCryptProtocol_s": "string", + "originIP_s": "string", + "originIp_s": "string", + "originName_s": "string", + "originUrl_s": "string", + "originalHost_s": "string", + "originalRequestUriWithArgs_s": "string", + "partitionId_s": "string", + "policyId_s": "string", + "policyMode_s": "string", + "policyScopeName_s": "string", + "policyScope_s": "string", + "policy_s": "string", + "pop_s": "string", + "primaryIPv4Address_s": "string", + "priority_d": "real", + "privateEndpointArmUrl_s": "string", + "privateEndpointConnections_s": "string", + "probeURL_s": "string", + "probeUrl_s": "string", + "processId_d": "real", + "properties_enableRbacAuthorization_b": "bool", + "properties_enableSoftDelete_b": "bool", + "properties_enabledForDeployment_b": "bool", + "properties_enabledForDiskEncryption_b": "bool", + "properties_enabledForTemplateDeployment_b": "bool", + "properties_networkAcls_bypass_s": "string", + "properties_networkAcls_defaultAction_s": "string", + "properties_s": "string", + "properties_sku_Family_s": "string", + "properties_sku_Name_s": "string", + "properties_softDeleteRetentionInDays_d": "real", + "properties_tenantId_g": "string", + "protocol_s": "string", + "publicIpAddress_s": "string", + "query_hash_s": "string", + "receivedBytes_d": "real", + "referer_s": "string", + "region_s": "string", + "removedAccessPolicy_ObjectId_g": "string", + "removedAccessPolicy_Permissions_certificates_s": "string", + "removedAccessPolicy_Permissions_keys_s": "string", + "removedAccessPolicy_Permissions_secrets_s": "string", + "removedAccessPolicy_Permissions_storage_s": "string", + "removedAccessPolicy_TenantId_g": "string", + "requestBytes_s": "string", + "requestCharge_s": "string", + "requestLength_s": "string", + "requestProtocol_s": "string", + "requestQuery_s": "string", + "requestResourceId_s": "string", + "requestResourceType_s": "string", + "requestSize_d": "real", + "requestUri_s": "string", + "resourceDetails_s": "string", + "resourceTokenPermissionId_s": "string", + "resourceTokenPermissionMode_s": "string", + "resourceTokenUserRid_s": "string", + "resourceUri_s": "string", + "resource_actionName_s": "string", + "resource_location_s": "string", + "resource_originRunId_s": "string", + "resource_resourceGroupName_s": "string", + "resource_runId_s": "string", + "resource_sourceTriggerHistoryName_s": "string", + "resource_subscriptionId_g": "string", + "resource_triggerName_s": "string", + "resource_workflowId_g": "string", + "resource_workflowName_s": "string", + "responseBytes_s": "string", + "responseCode_d": "real", + "responseLength_s": "string", + "responseSize_d": "real", + "resultDescription_ChildJobs_s": "string", + "resultDescription_ErrorJobs_s": "string", + "resultDescription_Summary_DurationInMinutes_d": "real", + "resultDescription_Summary_DurationInMinutes_s": "string", + "resultDescription_Summary_EndDateTimeUtc_s": "string", + "resultDescription_Summary_EndDateTimeUtc_t": "datetime", + "resultDescription_Summary_InitialRequiredUpdatesCount_d": "real", + "resultDescription_Summary_InstallPercentage_d": "real", + "resultDescription_Summary_MachineId_s": "string", + "resultDescription_Summary_MachineName_s": "string", + "resultDescription_Summary_RebootRequired_b": "bool", + "resultDescription_Summary_ScheduleName_s": "string", + "resultDescription_Summary_StartDateTimeUtc_t": "datetime", + "resultDescription_Summary_StatusDescription_s": "string", + "resultDescription_Summary_Status_s": "string", + "resultDescription_Summary_TotalUpdatesFailed_d": "real", + "resultDescription_Summary_TotalUpdatesInstalled_d": "real", + "resultSignature_d": "real", + "result_s": "string", + "retryHistory_s": "string", + "routingRuleName_s": "string", + "ruleGroup_s": "string", + "ruleId_s": "string", + "ruleName_s": "string", + "ruleSetType_s": "string", + "ruleSetVersion_s": "string", + "rulesEngineMatchNames_s": "string", + "secretProperties_attributes_enabled_b": "bool", + "secretProperties_attributes_exp_d": "real", + "secretProperties_type_s": "string", + "securityCipher_s": "string", + "securityCurves_s": "string", + "securityProtocol_s": "string", + "sentBytes_d": "real", + "serverConnectTime_s": "string", + "serverHeaderTime_s": "string", + "serverResponseLatency_s": "string", + "serverRouted_s": "string", + "serverStatus_s": "string", + "sni_s": "string", + "socketIP_s": "string", + "socketIp_s": "string", + "sourcePort_s": "string", + "sourcePublicIpAddress_s": "string", + "sqlQueryTextTraceType_s": "string", + "sqlerrcode_s": "string", + "sslCipher_s": "string", + "sslClientCertificateFingerprint_s": "string", + "sslClientCertificateIssuerName_s": "string", + "sslClientVerify_s": "string", + "sslEnabled_s": "string", + "sslProtocol_s": "string", + "startTime_t": "datetime", + "statusCode_s": "string", + "status_s": "string", + "subnetId_s": "string", + "subnetPrefix_s": "string", + "systemId_g": "string", + "tags_CreatedDate_s": "string", + "tags_Created_By_s": "string", + "tags_CreatorUPN_s": "string", + "tags_LogicAppsCategory_s": "string", + "tags_Source_s": "string", + "tags_hidden_SentinelTemplateName_s": "string", + "tags_hidden_SentinelTemplateVersion_s": "string", + "tags_hidden_SentinelWorkspaceId_s": "string", + "targetResources_JobId_g": "string", + "targetResources_JobName_g": "string", + "targetResources_ModuleId_g": "string", + "targetResources_ModuleName_s": "string", + "targetResources_Resource_s": "string", + "targetResources_RunbookId_g": "string", + "targetResources_RunbookName_s": "string", + "targetResources_Uri_s": "string", + "targetResources_WebhookName_s": "string", + "timeGeneratedEndTime_t": "datetime", + "timeStamp_t": "datetime", + "timeTaken_d": "real", + "timeTaken_s": "string", + "timeToFirstByte_s": "string", + "timestamp_s": "string", + "tlsVersion_s": "string", + "totalLatencyMilliseconds_s": "string", + "trackingReference_s": "string", + "transactionId_g": "string", + "truncated_d": "real", + "trustedService_s": "string", + "type_s": "string", + "upstreamSourcePort_s": "string", + "url_s": "string", + "userAgent_s": "string", + "virtualNetworkResourceEntries_s": "string", + "vnetResourceGuid_g": "string", + "workflowId_s": "string" + }, + "AzureMetrics": { + "AdditionalInformation": "string", + "Average": "real", + "CallerIpAddress": "string", + "Category": "string", + "Confidence": "string", + "CorrelationId": "string", + "Count": "real", + "Description": "string", + "DurationMs": "long", + "FirstReportedDateTime": "string", + "IndicatorThreatType": "string", + "IsActive": "string", + "LastReportedDateTime": "string", + "MaliciousIP": "string", + "Maximum": "real", + "MetricName": "string", + "Minimum": "real", + "OperationName": "string", + "OperationVersion": "string", + "RemoteIPCountry": "string", + "RemoteIPLatitude": "real", + "RemoteIPLongitude": "real", + "ReportReferenceLink": "string", + "Resource": "string", + "ResourceGroup": "string", + "ResourceId": "string", + "ResourceProvider": "string", + "ResultDescription": "string", + "ResultSignature": "string", + "ResultType": "string", + "Severity": "int", + "SourceSystem": "string", + "SubscriptionId": "string", + "TLPLevel": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TimeGrain": "string", + "Total": "real", + "Type": "string", + "UnitName": "string", + "_ResourceId": "string" + }, + "CloudAppEvents": { + "AccountDisplayName": "string", + "AccountId": "string", + "AccountObjectId": "string", + "AccountType": "string", + "ActionType": "string", + "ActivityObjects": "dynamic", + "ActivityType": "string", + "AdditionalFields": "dynamic", + "AppInstanceId": "int", + "Application": "string", + "ApplicationId": "int", + "AuditSource": "string", + "City": "string", + "CountryCode": "string", + "DeviceType": "string", + "IPAddress": "string", + "IPCategory": "string", + "IPTags": "dynamic", + "ISP": "string", + "IsAdminOperation": "bool", + "IsAnonymousProxy": "bool", + "IsExternalUser": "bool", + "IsImpersonated": "bool", + "LastSeenForUser": "dynamic", + "OAuthAppId": "string", + "OSPlatform": "string", + "ObjectId": "string", + "ObjectName": "string", + "ObjectType": "string", + "RawEventData": "dynamic", + "ReportId": "string", + "SessionData": "dynamic", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string", + "UncommonForUser": "dynamic", + "UserAgent": "string", + "UserAgentTags": "dynamic" + }, + "CommonSecurityLog": { + "Activity": "string", + "AdditionalExtensions": "string", + "ApplicationProtocol": "string", + "CollectorHostName": "string", + "CommunicationDirection": "string", + "Computer": "string", + "DestinationDnsDomain": "string", + "DestinationHostName": "string", + "DestinationIP": "string", + "DestinationMACAddress": "string", + "DestinationNTDomain": "string", + "DestinationPort": "int", + "DestinationProcessId": "int", + "DestinationProcessName": "string", + "DestinationServiceName": "string", + "DestinationTranslatedAddress": "string", + "DestinationTranslatedPort": "int", + "DestinationUserID": "string", + "DestinationUserName": "string", + "DestinationUserPrivileges": "string", + "DeviceAction": "string", + "DeviceAddress": "string", + "DeviceCustomDate1": "string", + "DeviceCustomDate1Label": "string", + "DeviceCustomDate2": "string", + "DeviceCustomDate2Label": "string", + "DeviceCustomFloatingPoint1": "real", + "DeviceCustomFloatingPoint1Label": "string", + "DeviceCustomFloatingPoint2": "real", + "DeviceCustomFloatingPoint2Label": "string", + "DeviceCustomFloatingPoint3": "real", + "DeviceCustomFloatingPoint3Label": "string", + "DeviceCustomFloatingPoint4": "real", + "DeviceCustomFloatingPoint4Label": "string", + "DeviceCustomIPv6Address1": "string", + "DeviceCustomIPv6Address1Label": "string", + "DeviceCustomIPv6Address2": "string", + "DeviceCustomIPv6Address2Label": "string", + "DeviceCustomIPv6Address3": "string", + "DeviceCustomIPv6Address3Label": "string", + "DeviceCustomIPv6Address4": "string", + "DeviceCustomIPv6Address4Label": "string", + "DeviceCustomNumber1": "int", + "DeviceCustomNumber1Label": "string", + "DeviceCustomNumber2": "int", + "DeviceCustomNumber2Label": "string", + "DeviceCustomNumber3": "int", + "DeviceCustomNumber3Label": "string", + "DeviceCustomString1": "string", + "DeviceCustomString1Label": "string", + "DeviceCustomString2": "string", + "DeviceCustomString2Label": "string", + "DeviceCustomString3": "string", + "DeviceCustomString3Label": "string", + "DeviceCustomString4": "string", + "DeviceCustomString4Label": "string", + "DeviceCustomString5": "string", + "DeviceCustomString5Label": "string", + "DeviceCustomString6": "string", + "DeviceCustomString6Label": "string", + "DeviceDnsDomain": "string", + "DeviceEventCategory": "string", + "DeviceEventClassID": "string", + "DeviceExternalID": "string", + "DeviceFacility": "string", + "DeviceInboundInterface": "string", + "DeviceMacAddress": "string", + "DeviceName": "string", + "DeviceNtDomain": "string", + "DeviceOutboundInterface": "string", + "DevicePayloadId": "string", + "DeviceProduct": "string", + "DeviceTimeZone": "string", + "DeviceTranslatedAddress": "string", + "DeviceVendor": "string", + "DeviceVersion": "string", + "EndTime": "datetime", + "EventCount": "int", + "EventOutcome": "string", + "EventType": "int", + "ExtID": "string", + "ExternalID": "int", + "FieldDeviceCustomNumber1": "long", + "FieldDeviceCustomNumber2": "long", + "FieldDeviceCustomNumber3": "long", + "FileCreateTime": "string", + "FileHash": "string", + "FileID": "string", + "FileModificationTime": "string", + "FileName": "string", + "FilePath": "string", + "FilePermission": "string", + "FileSize": "int", + "FileType": "string", + "FlexDate1": "string", + "FlexDate1Label": "string", + "FlexNumber1": "int", + "FlexNumber1Label": "string", + "FlexNumber2": "int", + "FlexNumber2Label": "string", + "FlexString1": "string", + "FlexString1Label": "string", + "FlexString2": "string", + "FlexString2Label": "string", + "IndicatorThreatType": "string", + "LogSeverity": "string", + "MaliciousIP": "string", + "MaliciousIPCountry": "string", + "MaliciousIPLatitude": "real", + "MaliciousIPLongitude": "real", + "Message": "string", + "OldFileCreateTime": "string", + "OldFileHash": "string", + "OldFileID": "string", + "OldFileModificationTime": "string", + "OldFileName": "string", + "OldFilePath": "string", + "OldFilePermission": "string", + "OldFileSize": "int", + "OldFileType": "string", + "OriginalLogSeverity": "string", + "ProcessID": "int", + "ProcessName": "string", + "Protocol": "string", + "Reason": "string", + "ReceiptTime": "string", + "ReceivedBytes": "long", + "RemoteIP": "string", + "RemotePort": "string", + "ReportReferenceLink": "string", + "RequestClientApplication": "string", + "RequestContext": "string", + "RequestCookies": "string", + "RequestMethod": "string", + "RequestURL": "string", + "SentBytes": "long", + "SimplifiedDeviceAction": "string", + "SourceDnsDomain": "string", + "SourceHostName": "string", + "SourceIP": "string", + "SourceMACAddress": "string", + "SourceNTDomain": "string", + "SourcePort": "int", + "SourceProcessId": "int", + "SourceProcessName": "string", + "SourceServiceName": "string", + "SourceSystem": "string", + "SourceTranslatedAddress": "string", + "SourceTranslatedPort": "int", + "SourceUserID": "string", + "SourceUserName": "string", + "SourceUserPrivileges": "string", + "StartTime": "datetime", + "TenantId": "string", + "ThreatConfidence": "string", + "ThreatDescription": "string", + "ThreatSeverity": "int", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "ConfidentialWatchlist": { + "AzureTenantId": "string", + "CorrelationId": "string", + "CreatedBy": "dynamic", + "CreatedTimeUTC": "datetime", + "DefaultDuration": "string", + "EntityMapping": "dynamic", + "LastUpdatedTimeUTC": "datetime", + "Notes": "string", + "Provider": "string", + "SearchKey": "string", + "Source": "string", + "SourceSystem": "string", + "Tags": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TimeToLive": "datetime", + "Type": "string", + "UpdatedBy": "dynamic", + "WatchlistAlias": "string", + "WatchlistCategory": "string", + "WatchlistId": "string", + "WatchlistItem": "dynamic", + "WatchlistItemId": "string", + "WatchlistName": "string", + "_DTItemId": "string", + "_DTItemStatus": "string", + "_DTItemType": "string", + "_DTTimestamp": "datetime" + }, + "ContainerInventory": { + "Command": "string", + "ComposeGroup": "string", + "Computer": "string", + "ContainerHostname": "string", + "ContainerID": "string", + "ContainerState": "string", + "CreatedTime": "datetime", + "EnvironmentVar": "string", + "ExitCode": "int", + "FinishedTime": "datetime", + "Image": "string", + "ImageID": "string", + "ImageTag": "string", + "Links": "string", + "Name": "string", + "Ports": "string", + "Repository": "string", + "SourceSystem": "string", + "StartedTime": "datetime", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "DeviceBehaviorEntities": { + "AccountDomain": "string", + "AccountName": "string", + "AccountObjectId": "string", + "AccountSid": "string", + "AccountUpn": "string", + "ActionType": "string", + "AdditionalFields": "string", + "Application": "string", + "ApplicationId": "string", + "BehaviorId": "string", + "Categories": "string", + "DataSources": "string", + "DetailedEntityRole": "string", + "DetectionSource": "string", + "DeviceId": "string", + "DeviceName": "string", + "EmailClusterId": "string", + "EmailSubject": "string", + "EntityRole": "string", + "EntityType": "string", + "FileName": "string", + "FileSize": "long", + "FolderPath": "string", + "LocalIP": "string", + "NetworkMessageId": "string", + "OAuthApplicationId": "string", + "ProcessCommandLine": "string", + "RegistryKey": "string", + "RegistryValueData": "string", + "RegistryValueName": "string", + "RemoteIP": "string", + "RemoteUrl": "string", + "SHA1": "string", + "SHA256": "string", + "ServiceSource": "string", + "SourceSystem": "string", + "TenantId": "string", + "ThreatFamily": "string", + "TimeGenerated": "datetime", + "Type": "string" + }, + "DeviceBehaviorInfo": { + "AccountObjectId": "string", + "AccountUpn": "string", + "ActionType": "string", + "AdditionalFields": "string", + "AttackTechniques": "string", + "BehaviorId": "string", + "Categories": "string", + "DataSources": "string", + "Description": "string", + "DetectionSource": "string", + "DeviceId": "string", + "EndTime": "datetime", + "ServiceSource": "string", + "SourceSystem": "string", + "StartTime": "datetime", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string" + }, + "DeviceCustomFileEvents": { + "ActionType": "string", + "AdditionalFields": "dynamic", + "AppGuardContainerId": "string", + "DeviceId": "string", + "DeviceName": "string", + "FileName": "string", + "FileOriginIP": "string", + "FileOriginReferrerUrl": "string", + "FileOriginUrl": "string", + "FileSize": "long", + "FolderPath": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessIntegrityLevel": "string", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessTokenElevation": "string", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsAzureInfoProtectionApplied": "bool", + "IsInitiatingProcessRemoteSession": "bool", + "MD5": "string", + "MachineGroup": "string", + "PreviousFileName": "string", + "PreviousFolderPath": "string", + "ReportId": "long", + "RequestAccountDomain": "string", + "RequestAccountName": "string", + "RequestAccountSid": "string", + "RequestProtocol": "string", + "RequestSourceIP": "string", + "RequestSourcePort": "int", + "RuleLastModificationTime": "datetime", + "RuleName": "string", + "SHA1": "string", + "SHA256": "string", + "SensitivityLabel": "string", + "SensitivitySubLabel": "string", + "ShareName": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceCustomImageLoadEvents": { + "ActionType": "string", + "AppGuardContainerId": "string", + "DeviceId": "string", + "DeviceName": "string", + "FileName": "string", + "FileSize": "long", + "FolderPath": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessIntegrityLevel": "string", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessTokenElevation": "string", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsInitiatingProcessRemoteSession": "bool", + "MD5": "string", + "MachineGroup": "string", + "ReportId": "long", + "RuleLastModificationTime": "datetime", + "RuleName": "string", + "SHA1": "string", + "SHA256": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceCustomNetworkEvents": { + "ActionType": "string", + "AdditionalFields": "dynamic", + "AppGuardContainerId": "string", + "DeviceId": "string", + "DeviceName": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessIntegrityLevel": "string", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessTokenElevation": "string", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsInitiatingProcessRemoteSession": "bool", + "LocalIP": "string", + "LocalIPType": "string", + "LocalPort": "int", + "MachineGroup": "string", + "Protocol": "string", + "RemoteIP": "string", + "RemoteIPType": "string", + "RemotePort": "int", + "RemoteUrl": "string", + "ReportId": "long", + "RuleLastModificationTime": "datetime", + "RuleName": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceCustomProcessEvents": { + "AccountDomain": "string", + "AccountName": "string", + "AccountObjectId": "string", + "AccountSid": "string", + "AccountUpn": "string", + "ActionType": "string", + "AdditionalFields": "dynamic", + "AppGuardContainerId": "string", + "CreatedProcessSessionId": "long", + "DeviceId": "string", + "DeviceName": "string", + "FileName": "string", + "FileSize": "long", + "FolderPath": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessIntegrityLevel": "string", + "InitiatingProcessLogonId": "long", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessSignatureStatus": "string", + "InitiatingProcessSignerType": "string", + "InitiatingProcessTokenElevation": "string", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsInitiatingProcessRemoteSession": "bool", + "IsProcessRemoteSession": "bool", + "LogonId": "long", + "MD5": "string", + "MachineGroup": "string", + "ProcessCommandLine": "string", + "ProcessCreationTime": "datetime", + "ProcessId": "long", + "ProcessIntegrityLevel": "string", + "ProcessRemoteSessionDeviceName": "string", + "ProcessRemoteSessionIP": "string", + "ProcessTokenElevation": "string", + "ProcessUniqueId": "string", + "ProcessVersionInfoCompanyName": "string", + "ProcessVersionInfoFileDescription": "string", + "ProcessVersionInfoInternalFileName": "string", + "ProcessVersionInfoOriginalFileName": "string", + "ProcessVersionInfoProductName": "string", + "ProcessVersionInfoProductVersion": "string", + "ReportId": "long", + "RuleLastModificationTime": "datetime", + "RuleName": "string", + "SHA1": "string", + "SHA256": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceCustomRegistryEvents": { + "ActionType": "string", + "AppGuardContainerId": "string", + "DeviceId": "string", + "DeviceName": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessIntegrityLevel": "string", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessTokenElevation": "string", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsInitiatingProcessRemoteSession": "bool", + "MachineGroup": "string", + "PreviousRegistryKey": "string", + "PreviousRegistryValueData": "string", + "PreviousRegistryValueName": "string", + "RegistryKey": "string", + "RegistryValueData": "string", + "RegistryValueName": "string", + "RegistryValueType": "string", + "ReportId": "long", + "RuleLastModificationTime": "datetime", + "RuleName": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceCustomScriptEvents": { + "ActionType": "string", + "DeviceId": "string", + "DeviceName": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessSignatureStatus": "string", + "InitiatingProcessSignerType": "string", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsInitiatingProcessRemoteSession": "bool", + "ReportId": "long", + "RuleLastModificationTime": "datetime", + "RuleName": "string", + "ScriptContent": "string", + "ScriptContentSHA256": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceEvents": { + "AccountDomain": "string", + "AccountName": "string", + "AccountSid": "string", + "ActionType": "string", + "AdditionalFields": "string", + "AppGuardContainerId": "string", + "CreatedProcessSessionId": "long", + "DeviceId": "string", + "DeviceName": "string", + "FileName": "string", + "FileOriginIP": "string", + "FileOriginUrl": "string", + "FileSize": "long", + "FolderPath": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessLogonId": "long", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsInitiatingProcessRemoteSession": "bool", + "IsProcessRemoteSession": "bool", + "LocalIP": "string", + "LocalPort": "int", + "LogonId": "long", + "MD5": "string", + "MachineGroup": "string", + "ProcessCommandLine": "string", + "ProcessCreationTime": "datetime", + "ProcessId": "long", + "ProcessRemoteSessionDeviceName": "string", + "ProcessRemoteSessionIP": "string", + "ProcessTokenElevation": "string", + "RegistryKey": "string", + "RegistryValueData": "string", + "RegistryValueName": "string", + "RemoteDeviceName": "string", + "RemoteIP": "string", + "RemotePort": "int", + "RemoteUrl": "string", + "ReportId": "long", + "SHA1": "string", + "SHA256": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceFileCertificateInfo": { + "CertificateCountersignatureTime": "datetime", + "CertificateCreationTime": "datetime", + "CertificateExpirationTime": "datetime", + "CertificateSerialNumber": "string", + "CrlDistributionPointUrls": "string", + "DeviceId": "string", + "DeviceName": "string", + "IsRootSignerMicrosoft": "bool", + "IsSigned": "bool", + "IsTrusted": "bool", + "Issuer": "string", + "IssuerHash": "string", + "MachineGroup": "string", + "ReportId": "long", + "SHA1": "string", + "SignatureType": "string", + "Signer": "string", + "SignerHash": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceFileEvents": { + "ActionType": "string", + "AdditionalFields": "string", + "AppGuardContainerId": "string", + "DeviceId": "string", + "DeviceName": "string", + "FileName": "string", + "FileOriginIP": "string", + "FileOriginReferrerUrl": "string", + "FileOriginUrl": "string", + "FileSize": "long", + "FolderPath": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessIntegrityLevel": "string", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessTokenElevation": "string", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsAzureInfoProtectionApplied": "bool", + "IsInitiatingProcessRemoteSession": "bool", + "MD5": "string", + "MachineGroup": "string", + "PreviousFileName": "string", + "PreviousFolderPath": "string", + "ReportId": "long", + "RequestAccountDomain": "string", + "RequestAccountName": "string", + "RequestAccountSid": "string", + "RequestProtocol": "string", + "RequestSourceIP": "string", + "RequestSourcePort": "int", + "SHA1": "string", + "SHA256": "string", + "SensitivityLabel": "string", + "SensitivitySubLabel": "string", + "ShareName": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceImageLoadEvents": { + "ActionType": "string", + "AppGuardContainerId": "string", + "DeviceId": "string", + "DeviceName": "string", + "FileName": "string", + "FileSize": "long", + "FolderPath": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessIntegrityLevel": "string", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessTokenElevation": "string", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsInitiatingProcessRemoteSession": "bool", + "MD5": "string", + "MachineGroup": "string", + "ReportId": "long", + "SHA1": "string", + "SHA256": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceInfo": { + "AadDeviceId": "string", + "AdditionalFields": "string", + "AssetValue": "string", + "AwsResourceName": "string", + "AzureResourceId": "string", + "AzureVmId": "string", + "AzureVmSubscriptionId": "string", + "ClientVersion": "string", + "CloudPlatforms": "string", + "ConnectivityType": "string", + "DeviceCategory": "string", + "DeviceDynamicTags": "string", + "DeviceId": "string", + "DeviceManualTags": "string", + "DeviceName": "string", + "DeviceSubtype": "string", + "DeviceType": "string", + "DiscoverySources": "string", + "DlpInfo": "string", + "ExclusionReason": "string", + "ExposureLevel": "string", + "FirmwareVersions": "string", + "GcpFullResourceName": "string", + "HardwareUuid": "string", + "HostDeviceId": "string", + "IsAzureADJoined": "bool", + "IsExcluded": "bool", + "IsInternetFacing": "bool", + "IsTransient": "bool", + "JoinType": "string", + "LoggedOnUsers": "string", + "MachineGroup": "string", + "MergedDeviceIds": "string", + "MergedToDeviceId": "string", + "MitigationStatus": "string", + "Model": "string", + "OSArchitecture": "string", + "OSBuild": "long", + "OSDistribution": "string", + "OSPlatform": "string", + "OSVersion": "string", + "OSVersionInfo": "string", + "OnboardingStatus": "string", + "OsBuildRevision": "string", + "PublicIP": "string", + "RegistryDeviceTag": "string", + "ReportId": "long", + "RestrictedDeviceSecurityOperations": "string", + "SensorHealthState": "string", + "Site": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string", + "Vendor": "string" + }, + "DeviceLogonEvents": { + "AccountDomain": "string", + "AccountName": "string", + "AccountSid": "string", + "ActionType": "string", + "AdditionalFields": "string", + "AppGuardContainerId": "string", + "DeviceId": "string", + "DeviceName": "string", + "FailureReason": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessIntegrityLevel": "string", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessTokenElevation": "string", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsInitiatingProcessRemoteSession": "bool", + "IsLocalAdmin": "bool", + "LogonId": "long", + "LogonType": "string", + "MachineGroup": "string", + "Protocol": "string", + "RemoteDeviceName": "string", + "RemoteIP": "string", + "RemoteIPType": "string", + "RemotePort": "int", + "ReportId": "long", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceNetworkEvents": { + "ActionType": "string", + "AdditionalFields": "string", + "AppGuardContainerId": "string", + "DeviceId": "string", + "DeviceName": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessIntegrityLevel": "string", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessTokenElevation": "string", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsInitiatingProcessRemoteSession": "bool", + "LocalIP": "string", + "LocalIPType": "string", + "LocalPort": "int", + "MachineGroup": "string", + "Protocol": "string", + "RemoteIP": "string", + "RemoteIPType": "string", + "RemotePort": "int", + "RemoteUrl": "string", + "ReportId": "long", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceNetworkInfo": { + "ConnectedNetworks": "string", + "DefaultGateways": "string", + "DeviceId": "string", + "DeviceName": "string", + "DnsAddresses": "string", + "IPAddresses": "string", + "IPv4Dhcp": "string", + "IPv6Dhcp": "string", + "MacAddress": "string", + "MachineGroup": "string", + "NetworkAdapterName": "string", + "NetworkAdapterStatus": "string", + "NetworkAdapterType": "string", + "NetworkAdapterVendor": "string", + "ReportId": "long", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "TunnelType": "string", + "Type": "string" + }, + "DeviceProcessEvents": { + "AccountDomain": "string", + "AccountName": "string", + "AccountObjectId": "string", + "AccountSid": "string", + "AccountUpn": "string", + "ActionType": "string", + "AdditionalFields": "string", + "AppGuardContainerId": "string", + "CreatedProcessSessionId": "long", + "DeviceId": "string", + "DeviceName": "string", + "FileName": "string", + "FileSize": "long", + "FolderPath": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessIntegrityLevel": "string", + "InitiatingProcessLogonId": "long", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessSignatureStatus": "string", + "InitiatingProcessSignerType": "string", + "InitiatingProcessTokenElevation": "string", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsInitiatingProcessRemoteSession": "bool", + "IsProcessRemoteSession": "bool", + "LogonId": "long", + "MD5": "string", + "MachineGroup": "string", + "ProcessCommandLine": "string", + "ProcessCreationTime": "datetime", + "ProcessId": "long", + "ProcessIntegrityLevel": "string", + "ProcessRemoteSessionDeviceName": "string", + "ProcessRemoteSessionIP": "string", + "ProcessTokenElevation": "string", + "ProcessUniqueId": "string", + "ProcessVersionInfoCompanyName": "string", + "ProcessVersionInfoFileDescription": "string", + "ProcessVersionInfoInternalFileName": "string", + "ProcessVersionInfoOriginalFileName": "string", + "ProcessVersionInfoProductName": "string", + "ProcessVersionInfoProductVersion": "string", + "ReportId": "long", + "SHA1": "string", + "SHA256": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceRegistryEvents": { + "ActionType": "string", + "AppGuardContainerId": "string", + "DeviceId": "string", + "DeviceName": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessIntegrityLevel": "string", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessTokenElevation": "string", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsInitiatingProcessRemoteSession": "bool", + "MachineGroup": "string", + "PreviousRegistryKey": "string", + "PreviousRegistryValueData": "string", + "PreviousRegistryValueName": "string", + "RegistryKey": "string", + "RegistryValueData": "string", + "RegistryValueName": "string", + "RegistryValueType": "string", + "ReportId": "long", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceTvmSecureConfigurationAssessment": { + "ConfigurationCategory": "string", + "ConfigurationId": "string", + "ConfigurationImpact": "real", + "ConfigurationSubcategory": "string", + "Context": "dynamic", + "DeviceId": "string", + "DeviceName": "string", + "IsApplicable": "bool", + "IsCompliant": "bool", + "IsExpectedUserImpact": "bool", + "MachineGroup": "string", + "OSPlatform": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceTvmSecureConfigurationAssessmentKB": { + "ConfigurationBenchmarks": "dynamic", + "ConfigurationCategory": "string", + "ConfigurationDescription": "string", + "ConfigurationId": "string", + "ConfigurationImpact": "real", + "ConfigurationName": "string", + "ConfigurationSubcategory": "string", + "RemediationOptions": "string", + "RiskDescription": "string", + "SourceSystem": "string", + "Tags": "dynamic", + "TenantId": "string", + "Type": "string" + }, + "DeviceTvmSoftwareInventory": { + "DeviceId": "string", + "DeviceName": "string", + "EndOfSupportDate": "datetime", + "EndOfSupportStatus": "string", + "MachineGroup": "string", + "OSArchitecture": "string", + "OSPlatform": "string", + "OSVersion": "string", + "ProductCodeCpe": "string", + "SoftwareName": "string", + "SoftwareVendor": "string", + "SoftwareVersion": "string", + "SourceSystem": "string", + "TenantId": "string", + "Type": "string" + }, + "DeviceTvmSoftwareVulnerabilities": { + "AadDeviceId": "string", + "CveId": "string", + "CveMitigationStatus": "string", + "CveTags": "dynamic", + "DeviceId": "string", + "DeviceName": "string", + "MachineGroup": "string", + "OSArchitecture": "string", + "OSPlatform": "string", + "OSVersion": "string", + "RecommendedSecurityUpdate": "string", + "RecommendedSecurityUpdateId": "string", + "SoftwareName": "string", + "SoftwareVendor": "string", + "SoftwareVersion": "string", + "SourceSystem": "string", + "TenantId": "string", + "Type": "string", + "VulnerabilitySeverityLevel": "string" + }, + "DeviceTvmSoftwareVulnerabilitiesKB": { + "AffectedSoftware": "dynamic", + "CveId": "string", + "CveSupportability": "string", + "CvssScore": "real", + "CvssVector": "string", + "EpssScore": "real", + "IsExploitAvailable": "bool", + "LastModifiedTime": "datetime", + "PublishedDate": "datetime", + "SourceSystem": "string", + "TenantId": "string", + "Type": "string", + "VulnerabilityDescription": "string", + "VulnerabilitySeverityLevel": "string" + }, + "DnsEvents": { + "ClientIP": "string", + "Computer": "string", + "Confidence": "string", + "Description": "string", + "EventId": "int", + "IPAddresses": "string", + "IndicatorThreatType": "string", + "MaliciousIP": "string", + "Message": "string", + "Name": "string", + "QueryType": "string", + "RemoteIPCountry": "string", + "RemoteIPLatitude": "real", + "RemoteIPLongitude": "real", + "ReportReferenceLink": "string", + "Result": "string", + "ResultCode": "int", + "Severity": "int", + "SourceSystem": "string", + "SubType": "string", + "TaskCategory": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "EmailAttachmentInfo": { + "DetectionMethods": "string", + "FileExtension": "string", + "FileName": "string", + "FileSize": "long", + "FileType": "string", + "NetworkMessageId": "string", + "RecipientEmailAddress": "string", + "RecipientObjectId": "string", + "ReportId": "string", + "SHA256": "string", + "SenderDisplayName": "string", + "SenderFromAddress": "string", + "SenderObjectId": "string", + "SourceSystem": "string", + "TenantId": "string", + "ThreatNames": "string", + "ThreatTypes": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "EmailEvents": { + "AdditionalFields": "string", + "AttachmentCount": "int", + "AuthenticationDetails": "string", + "BulkComplaintLevel": "int", + "Cc": "dynamic", + "ConfidenceLevel": "string", + "Connectors": "string", + "Context": "string", + "DeliveryAction": "string", + "DeliveryLocation": "string", + "DetectionMethods": "string", + "DistributionList": "string", + "EmailAction": "string", + "EmailActionPolicy": "string", + "EmailActionPolicyGuid": "string", + "EmailClusterId": "long", + "EmailDirection": "string", + "EmailLanguage": "string", + "EmailSize": "long", + "ExchangeTransportRule": "string", + "ForwardingInformation": "string", + "InternetMessageId": "string", + "IsFirstContact": "bool", + "LatestDeliveryAction": "string", + "LatestDeliveryLocation": "string", + "NetworkMessageId": "string", + "OrgLevelAction": "string", + "OrgLevelPolicy": "string", + "RecipientDomain": "string", + "RecipientEmailAddress": "string", + "RecipientObjectId": "string", + "ReportId": "string", + "SenderDisplayName": "string", + "SenderFromAddress": "string", + "SenderFromDomain": "string", + "SenderIPv4": "string", + "SenderIPv6": "string", + "SenderMailFromAddress": "string", + "SenderMailFromDomain": "string", + "SenderObjectId": "string", + "SourceSystem": "string", + "Subject": "string", + "TenantId": "string", + "ThreatClassification": "string", + "ThreatNames": "string", + "ThreatTypes": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "To": "dynamic", + "Type": "string", + "UrlCount": "int", + "UserLevelAction": "string", + "UserLevelPolicy": "string" + }, + "EmailPostDeliveryEvents": { + "Action": "string", + "ActionResult": "string", + "ActionTrigger": "string", + "ActionType": "string", + "DeliveryLocation": "string", + "DetectionMethods": "string", + "EmailDirection": "string", + "InternetMessageId": "string", + "NetworkMessageId": "string", + "RecipientEmailAddress": "string", + "ReportId": "string", + "SenderFromAddress": "string", + "SourceLocation": "string", + "SourceSystem": "string", + "TenantId": "string", + "ThreatTypes": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "EmailUrlInfo": { + "NetworkMessageId": "string", + "ReportId": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string", + "Url": "string", + "UrlChainId": "string", + "UrlChainPosition": "int", + "UrlDomain": "string", + "UrlLocation": "string" + }, + "Event": { + "AzureDeploymentID": "string", + "Computer": "string", + "EventCategory": "int", + "EventData": "string", + "EventID": "int", + "EventLevel": "int", + "EventLevelName": "string", + "EventLog": "string", + "MG": "string", + "ManagementGroupName": "string", + "Message": "string", + "ParameterXml": "string", + "RenderedDescription": "string", + "Role": "string", + "Source": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "UserName": "string", + "_ResourceId": "string" + }, + "FileMaliciousContentInfo": { + "DetectionMethods": "dynamic", + "DocumentID": "string", + "FileCreationTime": "datetime", + "FileName": "string", + "FileOwnerDisplayName": "string", + "FileOwnerUpn": "string", + "FileSize": "long", + "FolderPath": "string", + "LastModifiedTime": "datetime", + "LastModifyingAccountUpn": "string", + "ReportId": "string", + "SHA256": "string", + "SourceSystem": "string", + "TenantId": "string", + "ThreatNames": "string", + "ThreatTypes": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string", + "Workload": "string" + }, + "GCPDNS": { + "AuthAnswer": "bool", + "AuthenticationInfoPrincipalEmail": "string", + "AuthenticationInfoPrincipalSubject": "string", + "AuthorizationInfo": "string", + "DestinationIP": "string", + "GCPResourceName": "string", + "GCPResourceType": "string", + "InsertId": "string", + "LogName": "string", + "MethodName": "string", + "PayloadType": "string", + "Protocol": "string", + "QueryName": "string", + "QueryType": "string", + "Rdata": "string", + "ReqManZoneCloudLogConfigEnableLogging": "bool", + "ReqManZonePrivVisibConfigNetworks": "string", + "ReqmetaCallerSuppliedUserAgent": "string", + "ReqmetaRequestAttributesTime": "datetime", + "RequestAPIType": "string", + "RequestChangeAdditions": "string", + "RequestChangeDeletions": "string", + "RequestManagedZone": "string", + "RequestManagedZoneDescription": "string", + "RequestManagedZoneDnsName": "string", + "RequestManagedZoneName": "string", + "RequestManagedZoneVisibility": "string", + "RequestMetadataCallerIp": "string", + "RequestName": "string", + "RequestProject": "string", + "RequestType": "string", + "ResManZoneCloudLogConfigEnableLogging": "bool", + "ResourceLabelsLocation": "string", + "ResourceLabelsPolicyName": "string", + "ResourceLabelsProjectId": "string", + "ResourceLabelsSourceType": "string", + "ResourceLabelsTargetName": "string", + "ResourceLabelsTargetType": "string", + "ResourceLabelsZoneName": "string", + "RespManZonePrivVisibConfigNetworks": "string", + "ResponseChangeAdditions": "string", + "ResponseChangeDeletions": "string", + "ResponseChangeId": "string", + "ResponseChangeStartTime": "datetime", + "ResponseChangeStatus": "string", + "ResponseCode": "string", + "ResponseManagedZoneCreationTime": "datetime", + "ResponseManagedZoneDescription": "string", + "ResponseManagedZoneDnsName": "string", + "ResponseManagedZoneFingerprint": "string", + "ResponseManagedZoneId": "string", + "ResponseManagedZoneName": "string", + "ResponseManagedZoneNameServers": "string", + "ResponseManagedZoneRrsetCount": "string", + "ResponseManagedZoneVisibility": "string", + "ResponseOpZoneContextNewValue": "string", + "ResponseOpZoneContextOldValue": "string", + "ResponseOperationId": "string", + "ResponseOperationStartTime": "datetime", + "ResponseOperationStatus": "string", + "ResponseOperationType": "string", + "ResponseOperationUser": "string", + "ResponseType": "string", + "ServerLatency": "string", + "ServiceName": "string", + "Severity": "string", + "SourceIP": "string", + "SourceNetwork": "string", + "SourceSystem": "string", + "StatusCode": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string", + "VmInstanceId": "string", + "VmInstanceIdString": "string", + "VmInstanceName": "string", + "VmProjectId": "string", + "VmZoneName": "string" + }, + "Heartbeat": { + "Category": "string", + "Computer": "string", + "ComputerEnvironment": "string", + "ComputerIP": "string", + "ComputerPrivateIPs": "dynamic", + "IsGatewayInstalled": "bool", + "MG": "string", + "ManagementGroupName": "string", + "OSMajorVersion": "string", + "OSMinorVersion": "string", + "OSName": "string", + "OSType": "string", + "RemoteIPCountry": "string", + "RemoteIPLatitude": "real", + "RemoteIPLongitude": "real", + "Resource": "string", + "ResourceGroup": "string", + "ResourceId": "string", + "ResourceProvider": "string", + "ResourceType": "string", + "SCAgentChannel": "string", + "Solutions": "string", + "SourceComputerId": "string", + "SourceSystem": "string", + "SubscriptionId": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "VMUUID": "string", + "Version": "string", + "_ResourceId": "string" + }, + "IdentityDirectoryEvents": { + "AccountDisplayName": "string", + "AccountDomain": "string", + "AccountName": "string", + "AccountObjectId": "string", + "AccountSid": "string", + "AccountUpn": "string", + "ActionType": "string", + "AdditionalFields": "dynamic", + "Application": "string", + "DestinationDeviceName": "string", + "DestinationIPAddress": "string", + "DestinationPort": "int", + "DeviceName": "string", + "IPAddress": "string", + "ISP": "string", + "Location": "string", + "Port": "int", + "Protocol": "string", + "ReportId": "string", + "SourceSystem": "string", + "TargetAccountDisplayName": "string", + "TargetAccountUpn": "string", + "TargetDeviceName": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "IdentityLogonEvents": { + "AccountDisplayName": "string", + "AccountDomain": "string", + "AccountName": "string", + "AccountObjectId": "string", + "AccountSid": "string", + "AccountUpn": "string", + "ActionType": "string", + "AdditionalFields": "dynamic", + "Application": "string", + "DestinationDeviceName": "string", + "DestinationIPAddress": "string", + "DestinationPort": "int", + "DeviceName": "string", + "DeviceType": "string", + "FailureReason": "string", + "IPAddress": "string", + "ISP": "string", + "LastSeenForUser": "dynamic", + "Location": "string", + "LogonType": "string", + "OSPlatform": "string", + "Port": "int", + "Protocol": "string", + "ReportId": "string", + "SourceSystem": "string", + "TargetAccountDisplayName": "string", + "TargetDeviceName": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string", + "UncommonForUser": "dynamic" + }, + "IdentityQueryEvents": { + "AccountDisplayName": "string", + "AccountDomain": "string", + "AccountName": "string", + "AccountObjectId": "string", + "AccountSid": "string", + "AccountUpn": "string", + "ActionType": "string", + "AdditionalFields": "dynamic", + "Application": "string", + "DestinationDeviceName": "string", + "DestinationIPAddress": "string", + "DestinationPort": "int", + "DeviceName": "string", + "IPAddress": "string", + "Location": "string", + "Port": "int", + "Protocol": "string", + "Query": "string", + "QueryTarget": "string", + "QueryType": "string", + "ReportId": "string", + "SourceSystem": "string", + "TargetAccountDisplayName": "string", + "TargetAccountUpn": "string", + "TargetDeviceName": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "InsightsMetrics": { + "AgentId": "string", + "Computer": "string", + "Name": "string", + "Namespace": "string", + "Origin": "string", + "SourceSystem": "string", + "Tags": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "Val": "real", + "Value": "string", + "_ResourceId": "string" + }, + "NTANetAnalytics": { + "AclGroup": "string", + "AclRule": "string", + "Action": "string", + "AllowedInFlows": "long", + "AllowedOutFlows": "long", + "AzureRegion": "string", + "BytesDestToSrc": "long", + "BytesSrcToDest": "long", + "CompletedFlows": "long", + "ConnectingVnets": "string", + "ConnectionName": "string", + "ConnectionType": "string", + "Country": "string", + "DeniedInFlows": "long", + "DeniedOutFlows": "long", + "Description": "string", + "DestApplicationGateway": "string", + "DestCloud": "string", + "DestEnvironment": "string", + "DestExpressRouteCircuit": "string", + "DestIp": "string", + "DestLoadBalancer": "string", + "DestLocalNetworkGateway": "string", + "DestNetworkGroups": "string", + "DestNic": "string", + "DestPort": "int", + "DestPublicIps": "string", + "DestRegion": "string", + "DestServiceId": "string", + "DestServiceName": "string", + "DestServiceTags": "string", + "DestSubnet": "string", + "DestSubscription": "string", + "DestTenantId": "string", + "DestTenantName": "string", + "DestVm": "string", + "ExpressRouteCircuitPeeringType": "string", + "FaSchemaVersion": "string", + "FlowDirection": "string", + "FlowEncryption": "string", + "FlowEndTime": "datetime", + "FlowIntervalEndTime": "datetime", + "FlowIntervalStartTime": "datetime", + "FlowLogResourceId": "string", + "FlowStartTime": "datetime", + "FlowStatus": "string", + "FlowType": "string", + "IsFlowCapturedAtUdrHop": "bool", + "L4Protocol": "string", + "L7Protocol": "string", + "LogResourceId": "string", + "MacAddress": "string", + "NsgList": "string", + "NsgRule": "string", + "NsgRuleType": "string", + "PacketsDestToSrc": "long", + "PacketsSrcToDest": "long", + "PrivateEndpointResourceId": "string", + "PrivateLinkResourceId": "string", + "PrivateLinkResourceName": "string", + "RecordCount": "long", + "SourceSystem": "string", + "SrcApplicationGateway": "string", + "SrcCloud": "string", + "SrcEnvironment": "string", + "SrcExpressRouteCircuit": "string", + "SrcIp": "string", + "SrcLoadBalancer": "string", + "SrcLocalNetworkGateway": "string", + "SrcNetworkGroups": "string", + "SrcNic": "string", + "SrcPorts": "string", + "SrcPublicIps": "string", + "SrcRegion": "string", + "SrcServiceId": "string", + "SrcServiceName": "string", + "SrcServiceTags": "string", + "SrcSubnet": "string", + "SrcSubscription": "string", + "SrcTenantId": "string", + "SrcTenantName": "string", + "SrcVm": "string", + "Status": "string", + "SubType": "string", + "TargetResourceId": "string", + "TargetResourceType": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TimeProcessed": "datetime", + "TrafficAnalyticsResourceId": "string", + "Type": "string" + }, + "NetworkAccessAlerts": { + "AlertType": "string", + "ComponentName": "string", + "CreationDateTime": "datetime", + "Description": "string", + "DetectionTechnology": "string", + "DisplayName": "string", + "ExtendedProperties": "dynamic", + "FirstActivityDateTime": "datetime", + "Id": "string", + "IsPreview": "bool", + "LastActivityDateTime": "datetime", + "PolicyId": "string", + "ProductName": "string", + "RelatedResources": "dynamic", + "Severity": "string", + "SourceSystem": "string", + "SubTechniques": "string", + "Techniques": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "VendorName": "string" + }, + "NetworkAccessConnectionEvents": { + "AgentVersion": "string", + "AppId": "string", + "ClientDeviceName": "string", + "ConnectionId": "string", + "CrossTenantAccessType": "string", + "DestinationFqdn": "string", + "DestinationIp": "string", + "DestinationPort": "int", + "DeviceCategory": "string", + "DeviceId": "string", + "DeviceJoinType": "string", + "DeviceOperatingSystem": "string", + "DeviceOperatingSystemVersion": "string", + "EventType": "string", + "HomeTenantId": "string", + "InitiatingProcessName": "string", + "IsLocal": "bool", + "NetworkProtocol": "string", + "PopProcessingRegion": "string", + "RemoteNetworkId": "string", + "RemoteNetworkSourceIp": "string", + "SecurityPolicyId": "string", + "SecurityPolicyName": "string", + "SecurityPolicyVersion": "string", + "SecurityProfileId": "string", + "SecurityProfileName": "string", + "SecurityProfileVersion": "string", + "SecurityRuleId": "string", + "SourceIp": "string", + "SourceIpAutonomousSystemNumber": "string", + "SourceIpCarrier": "string", + "SourceIpCity": "string", + "SourceIpCountryCode": "string", + "SourceIpLatitude": "real", + "SourceIpLongitude": "real", + "SourceIpState": "string", + "SourceIpStateCode": "string", + "SourcePort": "int", + "SourcePrivateIp": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Token3PExpiry": "datetime", + "Token3PIssuedAt": "datetime", + "Token3PUniqueId": "string", + "Token3PValidFrom": "datetime", + "TrafficType": "string", + "TransportProtocol": "string", + "Type": "string", + "UserId": "string", + "UserPrincipalName": "string" + }, + "NetworkAccessGenerativeAIInsights": { + "Activity": "string", + "Content": "string", + "DestinationUrl": "string", + "EventId": "string", + "EventType": "string", + "McpClientName": "string", + "McpServerName": "string", + "SessionId": "string", + "SourceSystem": "string", + "SubActivity": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TransactionId": "string", + "Type": "string", + "UserPrincipalName": "string" + }, + "NetworkAccessTraffic": { + "AIAgentId": "string", + "AIAgentName": "string", + "AIDetectionConfidence": "int", + "AIDetectionEvidence": "string", + "AccessType": "string", + "Action": "string", + "AgentVersion": "string", + "AppId": "string", + "AppSegmentId": "string", + "CloudAppCatalogId": "string", + "CloudAppCategories": "string", + "CloudAppCategory": "string", + "CloudAppComplianceScore": "int", + "CloudAppGeneralScore": "int", + "CloudAppLegalScore": "int", + "CloudAppLoginUser": "string", + "CloudAppName": "string", + "CloudAppRiskScore": "int", + "ConnectionId": "string", + "ConnectionStatus": "string", + "ConnectorId": "string", + "ConnectorIp": "string", + "ConnectorName": "string", + "Description": "string", + "DestinationFqdn": "string", + "DestinationIp": "string", + "DestinationPort": "int", + "DestinationUrl": "string", + "DestinationWebCategories": "string", + "DeviceCategory": "string", + "DeviceId": "string", + "DeviceOperatingSystem": "string", + "DeviceOperatingSystemVersion": "string", + "DnsResponseOrigin": "string", + "FilteringProfileId": "string", + "FilteringProfileName": "string", + "HttpMethod": "string", + "HttpRequestContentType": "string", + "HttpResponseContentType": "string", + "HttpUserAgent": "string", + "InitiatingProcessName": "string", + "IsAgentic": "bool", + "IsTokenAgentic": "bool", + "KerberosClientName": "string", + "KerberosErrorCode": "int", + "KerberosErrorCodeName": "string", + "KerberosErrorText": "string", + "KerberosMsgType": "string", + "KerberosNonce": "int", + "NetworkProtocol": "string", + "OnPremAccount": "string", + "OriginHeader": "string", + "PolicyId": "string", + "PolicyName": "string", + "PolicyRuleId": "string", + "ProcessArgs": "string", + "ProcessTree": "string", + "ProcessingRegion": "string", + "Realm": "string", + "ReceivedBytes": "long", + "ReferrerHeader": "string", + "RemoteNetworkId": "string", + "ResourceTenantId": "string", + "ResponseCode": "int", + "RuleName": "string", + "SentBytes": "long", + "ServicePrincipalName": "string", + "SessionId": "string", + "SourceIp": "string", + "SourcePort": "int", + "SourceSystem": "string", + "TenantId": "string", + "ThreatType": "string", + "TimeGenerated": "datetime", + "TlsAction": "string", + "TlsPolicyId": "string", + "TlsPolicyName": "string", + "TlsRuleId": "string", + "TlsRuleName": "string", + "TlsStatus": "string", + "Token3PExpiry": "datetime", + "Token3PIssuedAt": "datetime", + "Token3PUniqueId": "string", + "Token3PValidFrom": "datetime", + "TrafficType": "string", + "TransactionId": "string", + "TransportProtocol": "string", + "Type": "string", + "UniqueTokenId": "string", + "UserId": "string", + "UserPrincipalName": "string", + "VendorNames": "string", + "XForwardedFor": "string" + }, + "NetworkSessions": { + "AdditionalFields": "dynamic", + "CloudAppId": "string", + "CloudAppName": "string", + "CloudAppOperation": "string", + "CloudAppRiskLevel": "string", + "DstBytes": "long", + "DstDomainHostname": "string", + "DstDvcDomain": "string", + "DstDvcFqdn": "string", + "DstDvcHostname": "string", + "DstDvcIpAddr": "string", + "DstDvcMacAddr": "string", + "DstGeoCity": "string", + "DstGeoCountry": "string", + "DstGeoLatitude": "real", + "DstGeoLongitude": "real", + "DstGeoRegion": "string", + "DstInterfaceGuid": "string", + "DstInterfaceName": "string", + "DstIpAddr": "string", + "DstMacAddr": "string", + "DstNatIpAddr": "string", + "DstNatPortNumber": "int", + "DstPackets": "long", + "DstPortNumber": "int", + "DstResourceId": "string", + "DstUserAadId": "string", + "DstUserDomain": "string", + "DstUserName": "string", + "DstUserSid": "string", + "DstUserUpn": "string", + "DstZone": "string", + "DvcAction": "string", + "DvcHostname": "string", + "DvcInboundInterface": "string", + "DvcIpAddr": "string", + "DvcMacAddr": "string", + "DvcOutboundInterface": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventMessage": "string", + "EventOriginalUid": "string", + "EventProduct": "string", + "EventProductVersion": "string", + "EventReportUrl": "string", + "EventResourceId": "string", + "EventResult": "string", + "EventResultDetails": "string", + "EventSchemaVersion": "string", + "EventSeverity": "string", + "EventStartTime": "datetime", + "EventSubType": "string", + "EventTimeIngested": "datetime", + "EventType": "string", + "EventUid": "string", + "EventVendor": "string", + "FileExtension": "string", + "FileHashMd5": "string", + "FileHashSha1": "string", + "FileHashSha256": "string", + "FileHashSha512": "string", + "FileMimeType": "string", + "FileName": "string", + "FilePath": "string", + "FileSize": "int", + "HttpContentType": "string", + "HttpReferrerOriginal": "string", + "HttpRequestMethod": "string", + "HttpRequestTime": "int", + "HttpRequestXff": "string", + "HttpResponseTime": "int", + "HttpStatusCode": "string", + "HttpUserAgentOriginal": "string", + "HttpVersion": "string", + "NetworkApplicationProtocol": "string", + "NetworkBytes": "long", + "NetworkDirection": "string", + "NetworkDuration": "int", + "NetworkIcmpCode": "int", + "NetworkIcmpType": "string", + "NetworkPackets": "long", + "NetworkProtocol": "string", + "NetworkRuleName": "string", + "NetworkRuleNumber": "int", + "NetworkSessionId": "string", + "SourceSystem": "string", + "SrcBytes": "long", + "SrcDvcDomain": "string", + "SrcDvcFqdn": "string", + "SrcDvcHostname": "string", + "SrcDvcIpAddr": "string", + "SrcDvcMacAddr": "string", + "SrcDvcModelName": "string", + "SrcDvcModelNumber": "string", + "SrcDvcOs": "string", + "SrcDvcType": "string", + "SrcGeoCity": "string", + "SrcGeoCountry": "string", + "SrcGeoLatitude": "real", + "SrcGeoLongitude": "real", + "SrcGeoRegion": "string", + "SrcInterfaceGuid": "string", + "SrcInterfaceName": "string", + "SrcIpAddr": "string", + "SrcMacAddr": "string", + "SrcNatIpAddr": "string", + "SrcNatPortNumber": "int", + "SrcPackets": "long", + "SrcPortNumber": "int", + "SrcResourceId": "string", + "SrcUserAadId": "string", + "SrcUserDomain": "string", + "SrcUserName": "string", + "SrcUserSid": "string", + "SrcUserUpn": "string", + "SrcZone": "string", + "TenantId": "string", + "ThreatCategory": "string", + "ThreatId": "string", + "ThreatName": "string", + "TimeGenerated": "datetime", + "Type": "string", + "UrlCategory": "string", + "UrlHostname": "string", + "UrlOriginal": "string" + }, + "OfficeActivity": { + "AADGroupId": "string", + "AADTarget": "string", + "Activity": "string", + "Actor": "string", + "ActorContextId": "string", + "ActorContextId_": "string", + "ActorIpAddress": "string", + "AddOnGuid": "string", + "AddOnType": "string", + "AddonName": "string", + "AffectedItems": "string", + "AppAccessContext": "dynamic", + "AppDistributionMode": "string", + "AppId": "string", + "AppPoolName": "string", + "Application": "string", + "ApplicationId": "string", + "ArtifactsShared": "dynamic", + "Attendees": "dynamic", + "AzureADAppId": "string", + "AzureActiveDirectory_EventType": "string", + "ChannelGuid": "string", + "ChannelName": "string", + "ChannelType": "string", + "ChatName": "string", + "ChatThreadId": "string", + "Client": "string", + "ClientAppId": "string", + "ClientIP": "string", + "ClientIP_": "string", + "ClientInfoString": "string", + "ClientMachineName": "string", + "ClientProcessName": "string", + "ClientVersion": "string", + "Client_IPAddress": "string", + "CommunicationType": "string", + "CrossMailboxOperations": "bool", + "CustomEvent": "string", + "DataCenterSecurityEventType": "int", + "DestFolder": "string", + "DestMailboxId": "string", + "DestMailboxOwnerMasterAccountSid": "string", + "DestMailboxOwnerSid": "string", + "DestMailboxOwnerUPN": "string", + "DestinationFileExtension": "string", + "DestinationFileName": "string", + "DestinationRelativeUrl": "string", + "DeviceInformation": "string", + "EffectiveOrganization": "string", + "ElevationApprovedTime": "datetime", + "ElevationApprover": "string", + "ElevationDuration": "int", + "ElevationRequestId": "string", + "ElevationRole": "string", + "ElevationTime": "datetime", + "EventSource": "string", + "Event_Data": "string", + "ExtendedProperties": "string", + "ExternalAccess": "string", + "ExtraProperties": "dynamic", + "Folder": "string", + "Folders": "string", + "GenericInfo": "string", + "InterSystemsId": "string", + "InternalLogonType": "int", + "IntraSystemId": "string", + "IsJoinedFromLobby": "bool", + "IsManagedDevice": "bool", + "IssuedAtTime": "datetime", + "Item": "string", + "ItemName": "string", + "ItemType": "string", + "JoinTime": "datetime", + "LeaveTime": "datetime", + "ListItemUniqueId": "string", + "LoginStatus": "int", + "LogonUserDisplayName": "string", + "LogonUserSid": "string", + "Logon_Type": "string", + "MachineDomainInfo": "string", + "MachineId": "string", + "MailboxGuid": "string", + "MailboxOwnerMasterAccountSid": "string", + "MailboxOwnerSid": "string", + "MailboxOwnerUPN": "string", + "MeetingDetailId": "string", + "Members": "dynamic", + "MessageId": "string", + "ModifiedObjectResolvedName": "string", + "ModifiedProperties": "string", + "Name": "string", + "NewValue": "string", + "OfficeId": "string", + "OfficeObjectId": "string", + "OfficeTenantId": "string", + "OfficeTenantId_": "string", + "OfficeWorkload": "string", + "OldValue": "string", + "Operation": "string", + "OperationProperties": "dynamic", + "OperationScope": "string", + "OrganizationId": "string", + "OrganizationId_": "string", + "OrganizationName": "string", + "OriginatingServer": "string", + "Parameters": "string", + "RecordType": "string", + "ResultReasonType": "string", + "ResultStatus": "string", + "SRPolicyId": "string", + "SRPolicyName": "string", + "SRRuleMatchDetails": "dynamic", + "Scope": "string", + "SendAsUserMailboxGuid": "string", + "SendAsUserSmtp": "string", + "SendOnBehalfOfUserSmtp": "string", + "SendonBehalfOfUserMailboxGuid": "string", + "SensitivityLabelId": "string", + "SharingType": "string", + "Site_": "string", + "Site_Url": "string", + "Site_Url_": "string", + "SourceFileExtension": "string", + "SourceFileName": "string", + "SourceFileName_": "string", + "SourceRecordId": "string", + "SourceRelativeUrl": "string", + "SourceRelativeUrl_": "string", + "SourceSystem": "string", + "Source_Name": "string", + "Start_Time": "datetime", + "SupportTicketId": "string", + "TabType": "string", + "TargetContextId": "string", + "TargetUserId": "string", + "TargetUserOrGroupName": "string", + "TargetUserOrGroupType": "string", + "TeamGuid": "string", + "TeamName": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "UniqueSharingId": "string", + "UniqueTokenId": "string", + "UserAgent": "string", + "UserDomain": "string", + "UserId": "string", + "UserId_": "string", + "UserKey": "string", + "UserSharedWith": "string", + "UserType": "string", + "_ResourceId": "string" + }, + "Operation": { + "Computer": "string", + "CorrelationId": "string", + "Detail": "string", + "ErrorId": "string", + "HelpLink": "string", + "MG": "string", + "ManagementGroupName": "string", + "OperationCategory": "string", + "OperationKey": "string", + "OperationStatus": "string", + "Solution": "string", + "SourceComputerId": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "Perf": { + "BucketEndTime": "datetime", + "BucketStartTime": "datetime", + "Computer": "string", + "CounterName": "string", + "CounterPath": "string", + "CounterValue": "real", + "InstanceName": "string", + "MG": "string", + "Max": "real", + "Min": "real", + "ObjectName": "string", + "SampleCount": "int", + "SourceSystem": "string", + "StandardDeviation": "real", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "PowerPlatformAdminActivity": { + "ActorName": "string", + "ActorUserId": "string", + "ActorUserType": "string", + "EnvironmentId": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventResult": "string", + "OrganizationId": "string", + "Properties": "dynamic", + "PropertyCollection": "dynamic", + "RecordType": "string", + "RequiresCustomerKeyEncryption": "bool", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "Workload": "string" + }, + "RemoteNetworkHealthLogs": { + "BgpRoutesAdvertisedCount": "int", + "CreatedDateTime": "datetime", + "Description": "string", + "DestinationIp": "string", + "Id": "string", + "ReceivedBytes": "long", + "RemoteNetworkId": "string", + "SentBytes": "long", + "SourceIp": "string", + "SourceSystem": "string", + "Status": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string" + }, + "SecurityAlert": { + "AlertLink": "string", + "AlertName": "string", + "AlertSeverity": "string", + "AlertType": "string", + "CompromisedEntity": "string", + "ConfidenceLevel": "string", + "ConfidenceScore": "real", + "Description": "string", + "DisplayName": "string", + "EndTime": "datetime", + "Entities": "string", + "ExtendedLinks": "string", + "ExtendedProperties": "string", + "IsIncident": "bool", + "ProcessingEndTime": "datetime", + "ProductComponentName": "string", + "ProductName": "string", + "ProviderName": "string", + "RemediationSteps": "string", + "ResourceId": "string", + "SourceComputerId": "string", + "SourceSystem": "string", + "StartTime": "datetime", + "Status": "string", + "SubTechniques": "string", + "SystemAlertId": "string", + "Tactics": "string", + "Techniques": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "VendorName": "string", + "VendorOriginalId": "string", + "WorkspaceResourceGroup": "string", + "WorkspaceSubscriptionId": "string" + }, + "SecurityEvent": { + "AccessList": "string", + "AccessMask": "string", + "AccessReason": "string", + "Account": "string", + "AccountDomain": "string", + "AccountExpires": "string", + "AccountName": "string", + "AccountSessionIdentifier": "string", + "AccountType": "string", + "Activity": "string", + "AdditionalInfo": "string", + "AdditionalInfo2": "string", + "AllowedToDelegateTo": "string", + "Attributes": "string", + "AuditPolicyChanges": "string", + "AuditsDiscarded": "int", + "AuthenticationLevel": "int", + "AuthenticationPackageName": "string", + "AuthenticationProvider": "string", + "AuthenticationServer": "string", + "AuthenticationService": "int", + "AuthenticationType": "string", + "AzureDeploymentID": "string", + "AzureTableName": "string", + "CACertificateHash": "string", + "CAPublicKeyHash": "string", + "CalledStationID": "string", + "CallerProcessId": "string", + "CallerProcessName": "string", + "CallingStationID": "string", + "CategoryId": "string", + "CertificateDatabaseHash": "string", + "Channel": "string", + "ClassId": "string", + "ClassName": "string", + "ClientAddress": "string", + "ClientIPAddress": "string", + "ClientName": "string", + "CommandLine": "string", + "CompatibleIds": "string", + "Computer": "string", + "Correlation": "string", + "DCDNSName": "string", + "DeviceDescription": "string", + "DeviceId": "string", + "DisplayName": "string", + "Disposition": "string", + "DomainBehaviorVersion": "string", + "DomainName": "string", + "DomainPolicyChanged": "string", + "DomainSid": "string", + "EAPType": "string", + "ElevatedToken": "string", + "ErrorCode": "int", + "EventData": "string", + "EventID": "int", + "EventLevelName": "string", + "EventOriginId": "string", + "EventRecordId": "string", + "EventSourceName": "string", + "ExtendedQuarantineState": "string", + "FailureReason": "string", + "FileHash": "string", + "FilePath": "string", + "FilePathNoUser": "string", + "Filter": "string", + "ForceLogoff": "string", + "Fqbn": "string", + "FullyQualifiedSubjectMachineName": "string", + "FullyQualifiedSubjectUserName": "string", + "GroupMembership": "string", + "HandleId": "string", + "HardwareIds": "string", + "HomeDirectory": "string", + "HomePath": "string", + "ImpersonationLevel": "string", + "InterfaceUuid": "string", + "IpAddress": "string", + "IpPort": "string", + "KeyLength": "int", + "Keywords": "string", + "Level": "string", + "LmPackageName": "string", + "LocationInformation": "string", + "LockoutDuration": "string", + "LockoutObservationWindow": "string", + "LockoutThreshold": "string", + "LoggingResult": "string", + "LogonGuid": "string", + "LogonHours": "string", + "LogonID": "string", + "LogonProcessName": "string", + "LogonType": "int", + "LogonTypeName": "string", + "MG": "string", + "MachineAccountQuota": "string", + "MachineInventory": "string", + "MachineLogon": "string", + "ManagementGroupName": "string", + "MandatoryLabel": "string", + "MaxPasswordAge": "string", + "MemberName": "string", + "MemberSid": "string", + "MinPasswordAge": "string", + "MinPasswordLength": "string", + "MixedDomainMode": "string", + "NASIPv4Address": "string", + "NASIPv6Address": "string", + "NASIdentifier": "string", + "NASPort": "string", + "NASPortType": "string", + "NetworkPolicyName": "string", + "NewDate": "string", + "NewMaxUsers": "string", + "NewProcessId": "string", + "NewProcessName": "string", + "NewRemark": "string", + "NewShareFlags": "string", + "NewTime": "string", + "NewUacValue": "string", + "NewValue": "string", + "NewValueType": "string", + "ObjectName": "string", + "ObjectServer": "string", + "ObjectType": "string", + "ObjectValueName": "string", + "OemInformation": "string", + "OldMaxUsers": "string", + "OldRemark": "string", + "OldShareFlags": "string", + "OldUacValue": "string", + "OldValue": "string", + "OldValueType": "string", + "Opcode": "string", + "OperationType": "string", + "PackageName": "string", + "ParentProcessName": "string", + "PartitionKey": "string", + "PasswordHistoryLength": "string", + "PasswordLastSet": "string", + "PasswordProperties": "string", + "PreviousDate": "string", + "PreviousTime": "string", + "PrimaryGroupId": "string", + "PrivateKeyUsageCount": "string", + "PrivilegeList": "string", + "Process": "string", + "ProcessId": "string", + "ProcessName": "string", + "ProfilePath": "string", + "Properties": "string", + "ProtocolSequence": "string", + "ProxyPolicyName": "string", + "QuarantineHelpURL": "string", + "QuarantineSessionID": "string", + "QuarantineSessionIdentifier": "string", + "QuarantineState": "string", + "QuarantineSystemHealthResult": "string", + "RelativeTargetName": "string", + "RemoteIpAddress": "string", + "RemotePort": "string", + "RequestId": "string", + "Requester": "string", + "RestrictedAdminMode": "string", + "RowKey": "string", + "RowsDeleted": "string", + "SamAccountName": "string", + "ScriptPath": "string", + "SecurityDescriptor": "string", + "ServiceAccount": "string", + "ServiceFileName": "string", + "ServiceName": "string", + "ServiceStartType": "int", + "ServiceType": "string", + "SessionName": "string", + "ShareLocalPath": "string", + "ShareName": "string", + "SidHistory": "string", + "SourceComputerId": "string", + "SourceSystem": "string", + "Status": "string", + "StorageAccount": "string", + "SubStatus": "string", + "SubcategoryGuid": "string", + "SubcategoryId": "string", + "Subject": "string", + "SubjectAccount": "string", + "SubjectDomainName": "string", + "SubjectKeyIdentifier": "string", + "SubjectLogonId": "string", + "SubjectMachineName": "string", + "SubjectMachineSID": "string", + "SubjectUserName": "string", + "SubjectUserSid": "string", + "SystemProcessId": "int", + "SystemThreadId": "int", + "SystemUserId": "string", + "TableId": "string", + "TargetAccount": "string", + "TargetDomainName": "string", + "TargetInfo": "string", + "TargetLinkedLogonId": "string", + "TargetLogonGuid": "string", + "TargetLogonId": "string", + "TargetOutboundDomainName": "string", + "TargetOutboundUserName": "string", + "TargetServerName": "string", + "TargetSid": "string", + "TargetUser": "string", + "TargetUserName": "string", + "TargetUserSid": "string", + "Task": "int", + "TemplateContent": "string", + "TemplateDSObjectFQDN": "string", + "TemplateInternalName": "string", + "TemplateOID": "string", + "TemplateSchemaVersion": "string", + "TemplateVersion": "string", + "TenantId": "string", + "TimeCollected": "datetime", + "TimeGenerated": "datetime", + "TokenElevationType": "string", + "TransmittedServices": "string", + "Type": "string", + "UserAccountControl": "string", + "UserParameters": "string", + "UserPrincipalName": "string", + "UserWorkstations": "string", + "VendorIds": "string", + "Version": "int", + "VirtualAccount": "string", + "Workstation": "string", + "WorkstationName": "string", + "_ResourceId": "string" + }, + "SecurityIncident": { + "AdditionalData": "dynamic", + "AlertIds": "dynamic", + "BookmarkIds": "dynamic", + "Classification": "string", + "ClassificationComment": "string", + "ClassificationReason": "string", + "ClosedTime": "datetime", + "Comments": "dynamic", + "CreatedTime": "datetime", + "Description": "string", + "FirstActivityTime": "datetime", + "FirstModifiedTime": "datetime", + "IncidentName": "string", + "IncidentNumber": "int", + "IncidentUrl": "string", + "Labels": "dynamic", + "LastActivityTime": "datetime", + "LastModifiedTime": "datetime", + "ModifiedBy": "string", + "Owner": "dynamic", + "ProviderIncidentId": "string", + "ProviderName": "string", + "RelatedAnalyticRuleIds": "dynamic", + "Severity": "string", + "SourceSystem": "string", + "Status": "string", + "Tasks": "dynamic", + "TenantId": "string", + "TimeGenerated": "datetime", + "Title": "string", + "Type": "string" + }, + "SentinelAlibabaCloudAPIGatewayLogs": { + "ApiGroupName": "string", + "ApiGroupUid": "string", + "ApiName": "string", + "ApiStageName": "string", + "ApiStageUid": "string", + "ApiUid": "string", + "AppId": "string", + "AppName": "string", + "ClientIp": "string", + "ClientNonce": "string", + "ConsumerAppKey": "string", + "CustomTraceId": "string", + "Domain": "string", + "ErrorCode": "string", + "ErrorMessage": "string", + "Exception": "string", + "ExchangeTimestamp": "string", + "HttpMethod": "string", + "InitialRequestId": "string", + "InstanceId": "string", + "JwtClaims": "string", + "LogProject": "string", + "LogRegion": "string", + "LogStore": "string", + "Path": "string", + "Plugin": "string", + "ProviderAliUid": "string", + "Region": "string", + "RequestBody": "string", + "RequestHandleTime": "datetime", + "RequestHeaders": "string", + "RequestId": "string", + "RequestProtocol": "string", + "RequestQueryString": "string", + "RequestSize": "string", + "ResponseBody": "string", + "ResponseHeaders": "string", + "ResponseSize": "string", + "ServiceLatency": "real", + "SourceSystem": "string", + "StatusCode": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TotalLatency": "real", + "Type": "string" + }, + "SentinelAlibabaCloudVPCFlowLogs": { + "AccountId": "string", + "Action": "string", + "Bytes": "string", + "Direction": "string", + "DstAddr": "string", + "DstPort": "string", + "DstType": "string", + "End": "string", + "EniId": "string", + "LogProject": "string", + "LogRegion": "string", + "LogStatus": "string", + "LogStore": "string", + "Packets": "string", + "Protocol": "string", + "Region": "string", + "SourceSystem": "string", + "SrcAddr": "string", + "SrcPort": "string", + "SrcType": "string", + "Start": "string", + "TcpFlags": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TrafficPath": "string", + "Type": "string", + "Version": "string", + "VmId": "string", + "VpcId": "string", + "VswitchId": "string" + }, + "SentinelAlibabaCloudWAFLogs": { + "AccountAction": "string", + "AccountRuleId": "string", + "AccountTest": "bool", + "AclAction": "string", + "AclRuleId": "string", + "AclRuleType": "string", + "AclTest": "bool", + "AlgorithmRuleId": "string", + "AntiscanAction": "string", + "AntiscanRuleId": "string", + "AntiscanRuleType": "string", + "AntiscanTest": "bool", + "BlockAction": "string", + "BodyBytesSent": "long", + "BypassMatchedIds": "string", + "CcAction": "string", + "CcBlocks": "string", + "CcRuleId": "string", + "CcRuleType": "string", + "CcTest": "bool", + "ContentType": "string", + "DeeplearningAction": "string", + "DeeplearningRuleId": "string", + "DeeplearningRuleType": "string", + "DeeplearningTest": "bool", + "DlpRuleId": "string", + "DlpTest": "bool", + "FinalAction": "string", + "FinalPlugin": "string", + "FinalRuleId": "string", + "FinalRuleType": "string", + "Host": "string", + "HttpCookie": "string", + "HttpReferer": "string", + "HttpUserAgent": "string", + "HttpXForwardedFor": "string", + "Https": "bool", + "LogProject": "string", + "LogRegion": "string", + "LogStore": "string", + "MatchedHost": "string", + "NormalizedAction": "string", + "NormalizedRuleId": "string", + "NormalizedRuleType": "string", + "NormalizedTest": "bool", + "OwnerId": "string", + "Querystring": "string", + "RealClientIp": "string", + "Region": "string", + "RemoteAddr": "string", + "RemotePort": "int", + "RequestLength": "long", + "RequestMethod": "string", + "RequestPath": "string", + "RequestTimeMsec": "real", + "RequestTraceid": "string", + "SceneAction": "string", + "SceneId": "string", + "SceneRuleId": "string", + "SceneRuleType": "string", + "SceneTest": "bool", + "ServerPort": "int", + "ServerProtocol": "string", + "SourceSystem": "string", + "SslCipher": "string", + "SslProtocol": "string", + "Status": "int", + "TenantId": "string", + "Time": "datetime", + "TimeGenerated": "datetime", + "Type": "string", + "UaBrowser": "string", + "UaBrowserFamily": "string", + "UaBrowserType": "string", + "UaBrowserVersion": "string", + "UaDeviceType": "string", + "UaOs": "string", + "UaOsFamily": "string", + "UpstreamAddr": "string", + "UpstreamResponseTime": "real", + "UpstreamStatus": "int", + "UserId": "string", + "WafAction": "string", + "WafRuleId": "string", + "WafRuleType": "string", + "WafTest": "bool" + }, + "SentinelAudit": { + "CorrelationId": "string", + "Description": "string", + "ExtendedProperties": "dynamic", + "OperationName": "string", + "SentinelResourceId": "string", + "SentinelResourceKind": "string", + "SentinelResourceName": "string", + "SentinelResourceType": "string", + "SourceSystem": "string", + "Status": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "WorkspaceId": "string" + }, + "SentinelBehaviorEntities": { + "AccountDomain": "string", + "AccountName": "string", + "AccountObjectId": "string", + "AccountSid": "string", + "AccountUpn": "string", + "ActionType": "string", + "AdditionalFields": "string", + "Application": "string", + "ApplicationId": "int", + "BehaviorId": "string", + "Categories": "string", + "CloudPlatform": "string", + "CloudResource": "string", + "CloudResourceId": "string", + "CloudResourceType": "string", + "CloudSubscriptionId": "string", + "DataSources": "string", + "DetailedEntityRole": "string", + "DetectionSource": "string", + "DeviceId": "string", + "DeviceName": "string", + "EmailClusterId": "string", + "EmailSubject": "string", + "EntityRole": "string", + "EntityType": "string", + "FileName": "string", + "FileSize": "long", + "FolderPath": "string", + "LocalIP": "string", + "NetworkMessageId": "string", + "OAuthApplicationId": "string", + "ProcessCommandLine": "string", + "RegistryKey": "string", + "RegistryValueData": "string", + "RegistryValueName": "string", + "RemoteIP": "string", + "RemoteUrl": "string", + "SHA1": "string", + "SHA256": "string", + "ServiceSource": "string", + "SourceSystem": "string", + "TenantId": "string", + "ThreatFamily": "string", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "SentinelBehaviorInfo": { + "AccountObjectId": "string", + "AccountUpn": "string", + "ActionType": "string", + "AdditionalFields": "string", + "AttackTechniques": "string", + "BehaviorId": "string", + "Categories": "string", + "DataSources": "string", + "Description": "string", + "DetectionSource": "string", + "DeviceId": "string", + "EndTime": "datetime", + "Insights": "string", + "ServiceSource": "string", + "SourceSystem": "string", + "StartTime": "datetime", + "TenantId": "string", + "TimeGenerated": "datetime", + "Title": "string", + "Type": "string", + "_ResourceId": "string" + }, + "SentinelHealth": { + "Description": "string", + "ExtendedProperties": "dynamic", + "OperationName": "string", + "Reason": "string", + "RecordId": "string", + "SentinelResourceId": "string", + "SentinelResourceKind": "string", + "SentinelResourceName": "string", + "SentinelResourceType": "string", + "SourceSystem": "string", + "Status": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "WorkspaceId": "string" + }, + "SentinelImpervaWAFCloudV2Logs": { + "Act": "string", + "AdditionalReqHeaders": "string", + "AdditionalResHeaders": "string", + "AdditionalRuleInfo": "string", + "App": "string", + "AttackName": "string", + "AttackSeverity": "string", + "CCode": "string", + "CapSupport": "string", + "CiCode": "string", + "ClApp": "string", + "ClAppSig": "string", + "Cn1": "string", + "CoSupport": "string", + "Cpt": "string", + "Customer": "string", + "DeliveryRuleDetails": "string", + "DeviceExternalId": "string", + "DeviceFacility": "string", + "DeviceVersion": "string", + "Dproc": "string", + "End": "string", + "EventProduct": "string", + "EventType": "string", + "EventVendor": "string", + "FileId": "string", + "FilePermission": "string", + "FileType": "string", + "In": "string", + "JavascriptSupport": "string", + "Latitude": "string", + "LogVersion": "string", + "Longitude": "string", + "PostBody": "string", + "QStr": "string", + "Ref": "string", + "Request": "string", + "RequestClientApplication": "string", + "RequestMethod": "string", + "RuleName": "string", + "SIP": "string", + "SignatureId": "string", + "SiteId": "string", + "SiteTag": "string", + "SourceServiceName": "string", + "SourceSystem": "string", + "Spt": "string", + "Src": "string", + "Start": "string", + "Suid": "string", + "Tag": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "VID": "string", + "Ver": "string", + "Xff": "string" + }, + "SigninLogs": { + "AADTenantId": "string", + "Agent": "dynamic", + "AlternateSignInName": "string", + "AppDisplayName": "string", + "AppId": "string", + "AppOwnerTenantId": "string", + "AppliedConditionalAccessPolicies": "string", + "AppliedEventListeners": "dynamic", + "AuthenticationAppDeviceDetails": "string", + "AuthenticationAppPolicyEvaluationDetails": "string", + "AuthenticationContextClassReferences": "string", + "AuthenticationDetails": "string", + "AuthenticationMethodsUsed": "string", + "AuthenticationProcessingDetails": "string", + "AuthenticationProtocol": "string", + "AuthenticationRequirement": "string", + "AuthenticationRequirementPolicies": "string", + "AuthenticatorAppLocation": "string", + "AutonomousSystemNumber": "string", + "Category": "string", + "ClientAppUsed": "string", + "ClientCredentialType": "string", + "ConditionalAccessAudiences": "string", + "ConditionalAccessPolicies": "dynamic", + "ConditionalAccessStatus": "string", + "CorrelationId": "string", + "CreatedDateTime": "datetime", + "CrossTenantAccessType": "string", + "DeviceDetail": "dynamic", + "DurationMs": "long", + "FederatedCredentialId": "string", + "FlaggedForReview": "bool", + "GlobalSecureAccessIpAddress": "string", + "HomeTenantId": "string", + "HomeTenantName": "string", + "IPAddress": "string", + "IPAddressFromResourceProvider": "string", + "Id": "string", + "Identity": "string", + "IncomingTokenType": "string", + "IsInteractive": "bool", + "IsRisky": "bool", + "IsTenantRestricted": "bool", + "IsThroughGlobalSecureAccess": "bool", + "Level": "string", + "Location": "string", + "LocationDetails": "dynamic", + "MfaDetail": "dynamic", + "NetworkLocationDetails": "string", + "OperationName": "string", + "OperationVersion": "string", + "OriginalRequestId": "string", + "OriginalTransferMethod": "string", + "ProcessingTimeInMilliseconds": "string", + "Resource": "string", + "ResourceDisplayName": "string", + "ResourceGroup": "string", + "ResourceId": "string", + "ResourceIdentity": "string", + "ResourceOwnerTenantId": "string", + "ResourceProvider": "string", + "ResourceServicePrincipalId": "string", + "ResourceTenantId": "string", + "ResultDescription": "string", + "ResultSignature": "string", + "ResultType": "string", + "RiskDetail": "string", + "RiskEventTypes": "string", + "RiskEventTypes_V2": "string", + "RiskLevel": "string", + "RiskLevelAggregated": "string", + "RiskLevelDuringSignIn": "string", + "RiskState": "string", + "ServicePrincipalId": "string", + "ServicePrincipalName": "string", + "SessionId": "string", + "SessionLifetimePolicies": "string", + "SignInIdentifier": "string", + "SignInIdentifierType": "string", + "SourceAppClientId": "string", + "SourceSystem": "string", + "Status": "dynamic", + "TenantId": "string", + "TimeGenerated": "datetime", + "TokenIssuerName": "string", + "TokenIssuerType": "string", + "TokenProtectionStatusDetails": "dynamic", + "Type": "string", + "UniqueTokenIdentifier": "string", + "UserAgent": "string", + "UserDisplayName": "string", + "UserId": "string", + "UserPrincipalName": "string", + "UserType": "string" + }, + "StorageBlobLogs": { + "AccessTier": "string", + "AccountName": "string", + "AuthenticationHash": "string", + "AuthenticationType": "string", + "AuthorizationDetails": "dynamic", + "CallerIpAddress": "string", + "Category": "string", + "ClientRequestId": "string", + "ConditionsUsed": "string", + "ContentLengthHeader": "long", + "CorrelationId": "string", + "DestinationUri": "string", + "DurationMs": "real", + "Etag": "string", + "LastModifiedTime": "datetime", + "Location": "string", + "MetricResponseType": "string", + "ObjectKey": "string", + "OperationCount": "int", + "OperationName": "string", + "OperationVersion": "string", + "Protocol": "string", + "ReferrerHeader": "string", + "RehydratePriority": "string", + "RequestBodySize": "long", + "RequestHeaderSize": "long", + "RequestMd5": "string", + "RequesterAppId": "string", + "RequesterAudience": "string", + "RequesterObjectId": "string", + "RequesterTenantId": "string", + "RequesterTokenIssuer": "string", + "RequesterUpn": "string", + "ResponseBodySize": "long", + "ResponseHeaderSize": "long", + "ResponseMd5": "string", + "SasExpiryStatus": "string", + "SchemaVersion": "string", + "ServerLatencyMs": "real", + "ServiceType": "string", + "SourceAccessTier": "string", + "SourceSystem": "string", + "SourceUri": "string", + "StatusCode": "string", + "StatusText": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TlsVersion": "string", + "Type": "string", + "Uri": "string", + "UserAgentHeader": "string", + "_ResourceId": "string" + }, + "StorageFileLogs": { + "AccountName": "string", + "AuthenticationHash": "string", + "AuthenticationType": "string", + "AuthorizationDetails": "dynamic", + "CallerIpAddress": "string", + "Category": "string", + "ClientRequestId": "string", + "ConditionsUsed": "string", + "ContentLengthHeader": "long", + "CorrelationId": "string", + "DurationMs": "real", + "Etag": "string", + "LastModifiedTime": "datetime", + "Location": "string", + "MetricResponseType": "string", + "ObjectKey": "string", + "OperationCount": "int", + "OperationName": "string", + "OperationVersion": "string", + "Protocol": "string", + "ReferrerHeader": "string", + "RequestBodySize": "long", + "RequestHeaderSize": "long", + "RequestMd5": "string", + "RequesterAppId": "string", + "RequesterAudience": "string", + "RequesterObjectId": "string", + "RequesterTenantId": "string", + "RequesterTokenIssuer": "string", + "RequesterUpn": "string", + "RequesterUserName": "string", + "ResponseBodySize": "long", + "ResponseHeaderSize": "long", + "ResponseMd5": "string", + "SasExpiryStatus": "string", + "SchemaVersion": "string", + "ServerLatencyMs": "real", + "ServiceType": "string", + "SmbCommandDetail": "string", + "SmbCommandMajor": "int", + "SmbCommandMinor": "string", + "SmbCreditsConsumed": "int", + "SmbFileId": "string", + "SmbMessageID": "string", + "SmbPersistentHandleID": "string", + "SmbPrimarySID": "string", + "SmbSessionID": "string", + "SmbStatusCode": "string", + "SmbTreeConnectID": "string", + "SmbVolatileHandleID": "string", + "SourceSystem": "string", + "StatusCode": "string", + "StatusText": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TlsVersion": "string", + "Type": "string", + "Uri": "string", + "UserAgentHeader": "string", + "_ResourceId": "string" + }, + "StorageQueueLogs": { + "AccountName": "string", + "AuthenticationHash": "string", + "AuthenticationType": "string", + "AuthorizationDetails": "dynamic", + "CallerIpAddress": "string", + "Category": "string", + "ClientRequestId": "string", + "ConditionsUsed": "string", + "ContentLengthHeader": "long", + "CorrelationId": "string", + "DurationMs": "real", + "Etag": "string", + "LastModifiedTime": "datetime", + "Location": "string", + "MetricResponseType": "string", + "ObjectKey": "string", + "OperationCount": "int", + "OperationName": "string", + "OperationVersion": "string", + "Protocol": "string", + "ReferrerHeader": "string", + "RequestBodySize": "long", + "RequestHeaderSize": "long", + "RequestMd5": "string", + "RequesterAppId": "string", + "RequesterAudience": "string", + "RequesterObjectId": "string", + "RequesterTenantId": "string", + "RequesterTokenIssuer": "string", + "RequesterUpn": "string", + "ResponseBodySize": "long", + "ResponseHeaderSize": "long", + "ResponseMd5": "string", + "SasExpiryStatus": "string", + "SchemaVersion": "string", + "ServerLatencyMs": "real", + "ServiceType": "string", + "SourceSystem": "string", + "StatusCode": "string", + "StatusText": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TlsVersion": "string", + "Type": "string", + "Uri": "string", + "UserAgentHeader": "string", + "_ResourceId": "string" + }, + "StorageTableLogs": { + "AccountName": "string", + "AuthenticationHash": "string", + "AuthenticationType": "string", + "AuthorizationDetails": "dynamic", + "CallerIpAddress": "string", + "Category": "string", + "ClientRequestId": "string", + "ConditionsUsed": "string", + "ContentLengthHeader": "long", + "CorrelationId": "string", + "DurationMs": "real", + "Etag": "string", + "LastModifiedTime": "datetime", + "Location": "string", + "MetricResponseType": "string", + "ObjectKey": "string", + "OperationCount": "int", + "OperationName": "string", + "OperationVersion": "string", + "Protocol": "string", + "ReferrerHeader": "string", + "RequestBodySize": "long", + "RequestHeaderSize": "long", + "RequestMd5": "string", + "RequesterAppId": "string", + "RequesterAudience": "string", + "RequesterObjectId": "string", + "RequesterTenantId": "string", + "RequesterTokenIssuer": "string", + "RequesterUpn": "string", + "ResponseBodySize": "long", + "ResponseHeaderSize": "long", + "ResponseMd5": "string", + "SasExpiryStatus": "string", + "SchemaVersion": "string", + "ServerLatencyMs": "real", + "ServiceType": "string", + "SourceSystem": "string", + "StatusCode": "string", + "StatusText": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TlsVersion": "string", + "Type": "string", + "Uri": "string", + "UserAgentHeader": "string", + "_ResourceId": "string" + }, + "Syslog": { + "CollectorHostName": "string", + "Computer": "string", + "EventTime": "datetime", + "Facility": "string", + "HostIP": "string", + "HostName": "string", + "MG": "string", + "ProcessID": "int", + "ProcessName": "string", + "SeverityLevel": "string", + "SourceSystem": "string", + "SyslogMessage": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "ThreatIntelExportOperation": { + "DestinationInfo": "dynamic", + "ErrorDetails": "string", + "ExportDuration": "int", + "ExportId": "string", + "ExportType": "string", + "ExportedBy": "string", + "ExternallyExportedId": "string", + "Id": "string", + "SourceSystem": "string", + "Status": "string", + "StixId": "string", + "TenantId": "string", + "TimeExported": "datetime", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "ThreatIntelIndicators": { + "AdditionalFields": "dynamic", + "AzureTenantId": "string", + "Confidence": "int", + "Created": "datetime", + "Data": "dynamic", + "Id": "string", + "IsActive": "bool", + "IsDeleted": "bool", + "LastUpdateMethod": "string", + "Modified": "datetime", + "ObservableKey": "string", + "ObservableValue": "string", + "Pattern": "string", + "Revoked": "bool", + "SourceSystem": "string", + "Tags": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "ValidFrom": "datetime", + "ValidUntil": "datetime", + "WorkspaceId": "string", + "_ResourceId": "string" + }, + "ThreatIntelObjects": { + "AdditionalFields": "dynamic", + "AzureTenantId": "string", + "Data": "dynamic", + "Id": "string", + "IsDeleted": "bool", + "LastUpdateMethod": "string", + "SourceSystem": "string", + "StixType": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "WorkspaceId": "string", + "_ResourceId": "string" + }, + "ThreatIntelligenceIndicator": { + "Action": "string", + "Active": "bool", + "ActivityGroupNames": "string", + "AdditionalInformation": "string", + "ApplicationId": "string", + "AzureTenantId": "string", + "ConfidenceScore": "real", + "Description": "string", + "DiamondModel": "string", + "DomainName": "string", + "EmailEncoding": "string", + "EmailLanguage": "string", + "EmailRecipient": "string", + "EmailSenderAddress": "string", + "EmailSenderName": "string", + "EmailSourceDomain": "string", + "EmailSourceIpAddress": "string", + "EmailSubject": "string", + "EmailXMailer": "string", + "ExpirationDateTime": "datetime", + "ExternalIndicatorId": "string", + "FileCompileDateTime": "datetime", + "FileCreatedDateTime": "datetime", + "FileHashType": "string", + "FileHashValue": "string", + "FileMutexName": "string", + "FileName": "string", + "FilePacker": "string", + "FilePath": "string", + "FileSize": "int", + "FileType": "string", + "IndicatorId": "string", + "IndicatorProvider": "string", + "KillChainActions": "bool", + "KillChainC2": "bool", + "KillChainDelivery": "bool", + "KillChainExploitation": "bool", + "KillChainReconnaissance": "bool", + "KillChainWeaponization": "bool", + "KnownFalsePositives": "string", + "MalwareNames": "string", + "NetworkCidrBlock": "string", + "NetworkDestinationAsn": "int", + "NetworkDestinationCidrBlock": "string", + "NetworkDestinationIP": "string", + "NetworkDestinationPort": "int", + "NetworkIP": "string", + "NetworkPort": "int", + "NetworkProtocol": "int", + "NetworkSourceAsn": "int", + "NetworkSourceCidrBlock": "string", + "NetworkSourceIP": "string", + "NetworkSourcePort": "int", + "PassiveOnly": "bool", + "SourceSystem": "string", + "Tags": "string", + "TenantId": "string", + "ThreatSeverity": "int", + "ThreatType": "string", + "TimeGenerated": "datetime", + "TrafficLightProtocolLevel": "string", + "Type": "string", + "Url": "string", + "UserAgent": "string" + }, + "UrlClickEvents": { + "AccountUpn": "string", + "ActionType": "string", + "AppName": "string", + "AppVersion": "string", + "DetectionMethods": "string", + "IPAddress": "string", + "IsClickedThrough": "bool", + "NetworkMessageId": "string", + "ReportId": "string", + "SourceId": "string", + "SourceSystem": "string", + "TenantId": "string", + "ThreatTypes": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string", + "Url": "string", + "UrlChain": "string", + "Workload": "string" + }, + "Usage": { + "AvgLatencyInSeconds": "real", + "BatchesCapped": "long", + "BatchesOutsideSla": "long", + "BatchesWithinSla": "long", + "Computer": "string", + "DataType": "string", + "EndTime": "datetime", + "IsBillable": "bool", + "LinkedMeterId": "string", + "LinkedResourceUri": "string", + "MeterId": "string", + "Plan": "string", + "Quantity": "real", + "QuantityUnit": "string", + "ResourceUri": "string", + "Solution": "string", + "SourceSystem": "string", + "StartTime": "datetime", + "TenantId": "string", + "TimeGenerated": "datetime", + "TotalBatches": "long", + "Type": "string" + }, + "VMConnection": { + "AdditionalInformation": "string", + "AgentId": "string", + "BytesReceived": "long", + "BytesSent": "long", + "Computer": "string", + "Confidence": "string", + "ConnectionId": "string", + "Description": "string", + "DestinationIp": "string", + "DestinationPort": "int", + "Direction": "string", + "FirstReportedDateTime": "string", + "IndicatorThreatType": "string", + "IsActive": "string", + "LastReportedDateTime": "string", + "LinksEstablished": "long", + "LinksFailed": "long", + "LinksLive": "long", + "LinksTerminated": "long", + "Machine": "string", + "MaliciousIp": "string", + "Process": "string", + "ProcessName": "string", + "Protocol": "string", + "RemoteClassification": "string", + "RemoteCountry": "string", + "RemoteDnsCanonicalNames": "string", + "RemoteDnsQuestions": "string", + "RemoteIp": "string", + "RemoteLatitude": "real", + "RemoteLongitude": "real", + "ReportReferenceLink": "string", + "ResponseTimeMax": "long", + "ResponseTimeMin": "long", + "ResponseTimeSum": "long", + "Responses": "long", + "Severity": "int", + "SourceIp": "string", + "SourceSystem": "string", + "TLPLevel": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "W3CIISLog": { + "AdditionalInformation": "string", + "AzureDeploymentID": "string", + "Computer": "string", + "Confidence": "string", + "Date": "string", + "Description": "string", + "FileOffset": "int", + "FileUri": "string", + "FirstReportedDateTime": "string", + "IndicatorThreatType": "string", + "IsActive": "string", + "LastReportedDateTime": "string", + "MG": "string", + "MaliciousIP": "string", + "ManagementGroupName": "string", + "RemoteIPCountry": "string", + "RemoteIPLatitude": "real", + "RemoteIPLongitude": "real", + "ReportReferenceLink": "string", + "Role": "string", + "RoleInstance": "string", + "Severity": "int", + "SourceSystem": "string", + "StorageAccount": "string", + "TLPLevel": "string", + "TenantId": "string", + "Time": "string", + "TimeGenerated": "datetime", + "TimeTaken": "long", + "Type": "string", + "_ResourceId": "string", + "cIP": "string", + "csBytes": "long", + "csCookie": "string", + "csHost": "string", + "csMethod": "string", + "csReferer": "string", + "csUriQuery": "string", + "csUriStem": "string", + "csUserAgent": "string", + "csUserName": "string", + "csVersion": "string", + "sComputerName": "string", + "sIP": "string", + "sPort": "int", + "sSiteName": "string", + "scBytes": "long", + "scStatus": "string", + "scSubStatus": "string", + "scWin32Status": "string" + }, + "Watchlist": { + "AzureTenantId": "string", + "CorrelationId": "string", + "CreatedBy": "dynamic", + "CreatedTimeUTC": "datetime", + "DefaultDuration": "string", + "EntityMapping": "dynamic", + "LastUpdatedTimeUTC": "datetime", + "Notes": "string", + "Provider": "string", + "SearchKey": "string", + "Source": "string", + "SourceSystem": "string", + "Tags": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TimeToLive": "datetime", + "Type": "string", + "UpdatedBy": "dynamic", + "WatchlistAlias": "string", + "WatchlistCategory": "string", + "WatchlistId": "string", + "WatchlistItem": "dynamic", + "WatchlistItemId": "string", + "WatchlistName": "string", + "_DTItemId": "string", + "_DTItemStatus": "string", + "_DTItemType": "string", + "_DTTimestamp": "datetime" + }, + "WindowsEvent": { + "Channel": "string", + "Computer": "string", + "Correlation": "string", + "Data": "dynamic", + "EventData": "dynamic", + "EventID": "int", + "EventLevel": "int", + "EventLevelName": "string", + "EventOriginId": "string", + "EventRecordId": "string", + "Keywords": "string", + "ManagementGroupName": "string", + "Opcode": "string", + "Provider": "string", + "RawEventData": "string", + "SourceSystem": "string", + "SystemProcessId": "int", + "SystemThreadId": "int", + "SystemUserId": "string", + "Task": "int", + "TenantId": "string", + "TimeCreated": "datetime", + "TimeGenerated": "datetime", + "Type": "string", + "Version": "int", + "_ResourceId": "string" + } +} diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/la_schema_catalog.json b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/la_schema_catalog.json new file mode 100644 index 00000000000..8a230b55c6c --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/la_schema_catalog.json @@ -0,0 +1,6052 @@ +{ + "AADManagedIdentitySignInLogs": { + "AADTenantId": "string", + "Agent": "string", + "AppId": "string", + "AppOwnerTenantId": "string", + "AuthenticationContextClassReferences": "string", + "AuthenticationProcessingDetails": "string", + "Category": "string", + "ClientCredentialType": "string", + "ConditionalAccessAudiences": "string", + "ConditionalAccessPolicies": "string", + "ConditionalAccessPoliciesV2": "dynamic", + "ConditionalAccessStatus": "string", + "CorrelationId": "string", + "CreatedDateTime": "datetime", + "DurationMs": "long", + "FederatedCredentialId": "string", + "IPAddress": "string", + "Id": "string", + "Identity": "string", + "Level": "string", + "Location": "string", + "LocationDetails": "string", + "ManagedServiceIdentity": "string", + "NetworkLocationDetails": "string", + "OperationName": "string", + "OperationVersion": "string", + "ResourceDisplayName": "string", + "ResourceGroup": "string", + "ResourceIdentity": "string", + "ResourceOwnerTenantId": "string", + "ResourceServicePrincipalId": "string", + "ResultDescription": "string", + "ResultSignature": "string", + "ResultType": "string", + "ServicePrincipalCredentialKeyId": "string", + "ServicePrincipalCredentialThumbprint": "string", + "ServicePrincipalId": "string", + "ServicePrincipalName": "string", + "SessionId": "string", + "SourceAppClientId": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "UniqueTokenIdentifier": "string", + "UserAgent": "string" + }, + "AADNonInteractiveUserSignInLogs": { + "AADTenantId": "string", + "Agent": "string", + "AlternateSignInName": "string", + "AppDisplayName": "string", + "AppId": "string", + "AppOwnerTenantId": "string", + "AppliedEventListeners": "dynamic", + "AuthenticationContextClassReferences": "string", + "AuthenticationDetails": "string", + "AuthenticationMethodsUsed": "string", + "AuthenticationProcessingDetails": "string", + "AuthenticationProtocol": "string", + "AuthenticationRequirement": "string", + "AuthenticationRequirementPolicies": "string", + "AuthenticatorAppLocation": "string", + "AutonomousSystemNumber": "string", + "Category": "string", + "ClientAppUsed": "string", + "ClientCredentialType": "string", + "ClientSessionId": "string", + "ConditionalAccessAudiences": "string", + "ConditionalAccessPolicies": "string", + "ConditionalAccessPoliciesV2": "dynamic", + "ConditionalAccessStatus": "string", + "CorrelationId": "string", + "CreatedDateTime": "datetime", + "CrossTenantAccessType": "string", + "DeviceDetail": "string", + "DurationMs": "long", + "FederatedCredentialId": "string", + "GlobalSecureAccessIpAddress": "string", + "HomeTenantId": "string", + "HomeTenantName": "string", + "IPAddress": "string", + "Id": "string", + "Identity": "string", + "IncomingTokenType": "string", + "IsInteractive": "bool", + "IsRisky": "bool", + "IsTenantRestricted": "bool", + "IsThroughGlobalSecureAccess": "bool", + "Level": "string", + "Location": "string", + "LocationDetails": "string", + "MfaDetail": "string", + "NetworkLocationDetails": "string", + "OperationName": "string", + "OperationVersion": "string", + "OriginalRequestId": "string", + "OriginalTransferMethod": "string", + "ProcessingTimeInMs": "string", + "ResourceDisplayName": "string", + "ResourceGroup": "string", + "ResourceIdentity": "string", + "ResourceOwnerTenantId": "string", + "ResourceServicePrincipalId": "string", + "ResourceTenantId": "string", + "ResultDescription": "string", + "ResultSignature": "string", + "ResultType": "string", + "RiskDetail": "string", + "RiskEventTypes": "string", + "RiskEventTypes_V2": "string", + "RiskLevelAggregated": "string", + "RiskLevelDuringSignIn": "string", + "RiskState": "string", + "ServicePrincipalId": "string", + "SessionId": "string", + "SessionLifetimePolicies": "string", + "SignInEventTypes": "string", + "SignInIdentifierType": "string", + "SourceSystem": "string", + "Status": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TokenIssuerName": "string", + "TokenIssuerType": "string", + "TokenProtectionStatusDetails": "string", + "Type": "string", + "UniqueTokenIdentifier": "string", + "UserAgent": "string", + "UserDisplayName": "string", + "UserId": "string", + "UserPrincipalName": "string", + "UserType": "string" + }, + "AADServicePrincipalSignInLogs": { + "AADTenantId": "string", + "Agent": "string", + "AppId": "string", + "AppOwnerTenantId": "string", + "AuthenticationContextClassReferences": "string", + "AuthenticationProcessingDetails": "string", + "AutonomousSystemNumber": "string", + "Category": "string", + "ClientCredentialType": "string", + "ConditionalAccessAudiences": "string", + "ConditionalAccessPolicies": "string", + "ConditionalAccessPoliciesV2": "dynamic", + "ConditionalAccessStatus": "string", + "CorrelationId": "string", + "CreatedDateTime": "datetime", + "DurationMs": "long", + "FederatedCredentialId": "string", + "IPAddress": "string", + "Id": "string", + "Identity": "string", + "Level": "string", + "Location": "string", + "LocationDetails": "string", + "NetworkLocationDetails": "string", + "OperationName": "string", + "OperationVersion": "string", + "ResourceDisplayName": "string", + "ResourceGroup": "string", + "ResourceIdentity": "string", + "ResourceOwnerTenantId": "string", + "ResourceServicePrincipalId": "string", + "ResultDescription": "string", + "ResultSignature": "string", + "ResultType": "string", + "ServicePrincipalCredentialKeyId": "string", + "ServicePrincipalCredentialThumbprint": "string", + "ServicePrincipalId": "string", + "ServicePrincipalName": "string", + "SessionId": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "UniqueTokenIdentifier": "string", + "UserAgent": "string" + }, + "ASimAgentEventLogs": { + "ActingAppId": "string", + "ActingAppName": "string", + "ActingAppType": "string", + "ActingApplicationEntityKey": "string", + "ActorUserAdditionalIds": "dynamic", + "ActorUserEntityKey": "string", + "ActorUserId": "string", + "ActorUserIdType": "string", + "ActorUserScope": "string", + "ActorUserScopeId": "string", + "ActorUsername": "string", + "ActorUsernameType": "string", + "AdditionalEntities": "dynamic", + "AdditionalFields": "dynamic", + "DvcEntityKey": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventErrorDetails": "string", + "EventFinishReasons": "dynamic", + "EventOriginalErrorType": "string", + "EventOriginalRequestDetails": "string", + "EventOriginalResultDetails": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventOutputType": "string", + "EventProduct": "string", + "EventRequestFrequencyPenalty": "real", + "EventRequestId": "string", + "EventRequestPresencePenalty": "real", + "EventRequestSeed": "long", + "EventRequestTemperature": "real", + "EventRequestTopP": "real", + "EventResponseId": "string", + "EventSchema": "string", + "EventSchemaVersion": "string", + "EventSessionId": "string", + "EventSessionName": "string", + "EventStartTime": "datetime", + "EventThoughtProcessDetails": "string", + "EventThoughtProcessId": "string", + "EventType": "string", + "EventUid": "string", + "EventVendor": "string", + "InputTokensUsed": "long", + "ModelName": "string", + "ModelProviderName": "string", + "OutputTokensUsed": "long", + "PlatformTargetAgentDescription": "string", + "PlatformTargetAgentId": "string", + "PlatformTargetAgentName": "string", + "PlatformTargetOriginalAgentType": "string", + "SourceSystem": "string", + "SrcAgentBlueprintId": "string", + "SrcAgentDescription": "string", + "SrcAgentEntityKey": "string", + "SrcAgentId": "string", + "SrcAgentName": "string", + "SrcAgentOriginalType": "string", + "SrcFQDN": "string", + "SrcIpAddr": "string", + "SrcPortNumber": "int", + "TargetAgentBlueprintId": "string", + "TargetAgentDescription": "string", + "TargetAgentEntityKey": "string", + "TargetAgentId": "string", + "TargetAgentName": "string", + "TargetAgentOriginalType": "string", + "TargetAgentUserId": "string", + "TargetAgentUsername": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "ToolDescription": "string", + "ToolId": "string", + "ToolName": "string", + "ToolOriginalType": "string", + "Type": "string", + "_ResourceId": "string" + }, + "ASimAlertEventLogs": { + "AdditionalEntities": "dynamic", + "AdditionalFields": "dynamic", + "AlertDescription": "string", + "AlertId": "string", + "AlertName": "string", + "AlertOriginalStatus": "string", + "AlertStatus": "string", + "AlertVerdict": "string", + "AttackRemediationSteps": "string", + "AttackTactics": "string", + "AttackTechniques": "string", + "DetectionMethod": "string", + "DvcAction": "string", + "DvcDescription": "string", + "DvcDomain": "string", + "DvcDomainType": "string", + "DvcEntityKey": "string", + "DvcFQDN": "string", + "DvcHostname": "string", + "DvcId": "string", + "DvcIdType": "string", + "DvcInterface": "string", + "DvcIpAddr": "string", + "DvcMacAddr": "string", + "DvcOriginalAction": "string", + "DvcOs": "string", + "DvcOsVersion": "string", + "DvcScope": "string", + "DvcScopeId": "string", + "DvcZone": "string", + "EmailMessageId": "string", + "EmailSubject": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventMessage": "string", + "EventOriginalResultDetails": "string", + "EventOriginalSeverity": "string", + "EventOriginalSubType": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventOwner": "string", + "EventProduct": "string", + "EventProductVersion": "string", + "EventReportUrl": "string", + "EventResult": "string", + "EventResultDetails": "string", + "EventSchemaVersion": "string", + "EventSeverity": "string", + "EventStartTime": "datetime", + "EventSubType": "string", + "EventType": "string", + "EventUid": "string", + "EventVendor": "string", + "FileEntityKey": "string", + "FileMD5": "string", + "FileName": "string", + "FilePath": "string", + "FileSHA1": "string", + "FileSHA256": "string", + "FileSize": "string", + "IndicatorAssociation": "string", + "IndicatorType": "string", + "OriginalUserType": "string", + "ProcessCommandLine": "string", + "ProcessEntityKey": "string", + "ProcessFileCompany": "string", + "ProcessId": "string", + "ProcessName": "string", + "RegistryKey": "string", + "RegistryValue": "string", + "RegistryValueData": "string", + "RegistryValueType": "string", + "Rule": "string", + "RuleDescription": "string", + "RuleName": "string", + "RuleNumber": "int", + "SourceSystem": "string", + "TenantId": "string", + "ThreatCategory": "string", + "ThreatConfidence": "int", + "ThreatFirstReportedTime": "datetime", + "ThreatId": "string", + "ThreatIsActive": "bool", + "ThreatLastReportedTime": "datetime", + "ThreatName": "string", + "ThreatOriginalCategory": "string", + "ThreatOriginalConfidence": "string", + "ThreatOriginalRiskLevel": "string", + "ThreatRiskLevel": "int", + "TimeGenerated": "datetime", + "Type": "string", + "Url": "string", + "User": "string", + "UserAdditionalIds": "dynamic", + "UserEntityKey": "string", + "UserId": "string", + "UserIdType": "string", + "UserScope": "string", + "UserScopeId": "string", + "UserSessionId": "string", + "UserType": "string", + "Username": "string", + "UsernameType": "string", + "_ResourceId": "string" + }, + "ASimAssetEntityLogs": { + "AADTenantId": "string", + "AdditionalAssetOwners": "dynamic", + "AdditionalFields": "dynamic", + "AssetClassificationLastScanDateTime": "datetime", + "AssetIsProtectedByDlp": "bool", + "AssetOriginalDataClassificationType": "dynamic", + "AssetOriginalPermissions": "dynamic", + "AssetOriginalRiskDetails": "dynamic", + "AssetOriginalRiskLevel": "string", + "AssetOriginalSensitivityLevel": "string", + "AssetOriginalType": "string", + "AssetOwnerId": "string", + "AssetOwnerIdType": "string", + "AssetOwnerScope": "string", + "AssetOwnerScopeId": "string", + "AssetOwnerType": "string", + "AssetPath": "string", + "AssetRelatedIndicators": "dynamic", + "AssetRiskFirstReportedTime": "datetime", + "AssetRiskLastReportedTime": "datetime", + "AssetRiskLevel": "string", + "AssetRiskName": "string", + "AssetSensitivityLabel": "string", + "AssetType": "string", + "EntityCreatedTime": "datetime", + "EntityFeedType": "string", + "EntityId": "string", + "EntityIdType": "string", + "EntityIngestionTime": "datetime", + "EntityIsDeleted": "bool", + "EntityKey": "string", + "EntityLastAccessedTime": "datetime", + "EntityLastModifiedTime": "datetime", + "EntityName": "string", + "EntityNameType": "string", + "EntityOriginalId": "string", + "EntityOriginalSource": "string", + "EntityProduct": "string", + "EntitySchema": "string", + "EntitySchemaVersion": "string", + "EntitySnapshotId": "string", + "EntitySource": "string", + "EntitySubProduct": "string", + "EntityUpdatedTime": "datetime", + "EntityVendor": "string", + "ExternalUsersCount": "int", + "FileExtension": "string", + "FileIsSignatureValid": "bool", + "FileMD5": "string", + "FilePath": "string", + "FileSHA1": "string", + "FileSHA256": "string", + "FileSHA512": "string", + "FileSignatureDetails": "string", + "FileSize": "long", + "IdentityDirectoryId": "string", + "IdentityDirectoryName": "string", + "InternalUsersCount": "int", + "SitePath": "string", + "SitePrimaryUri": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "User": "string", + "_ResourceId": "string" + }, + "ASimAuditEventLogs": { + "ActingAppId": "string", + "ActingAppName": "string", + "ActingAppType": "string", + "ActingApplicationEntityKey": "string", + "ActingOriginalAppType": "string", + "ActorOriginalUserType": "string", + "ActorScope": "string", + "ActorScopeId": "string", + "ActorSessionId": "string", + "ActorUserAadId": "string", + "ActorUserAdditionalIds": "dynamic", + "ActorUserEntityKey": "string", + "ActorUserId": "string", + "ActorUserIdType": "string", + "ActorUserSid": "string", + "ActorUserType": "string", + "ActorUsername": "string", + "ActorUsernameType": "string", + "AdditionalEntities": "dynamic", + "AdditionalFields": "dynamic", + "DvcAction": "string", + "DvcDescription": "string", + "DvcDomain": "string", + "DvcDomainType": "string", + "DvcEntityKey": "string", + "DvcFQDN": "string", + "DvcHostname": "string", + "DvcId": "string", + "DvcIdType": "string", + "DvcInterface": "string", + "DvcIpAddr": "string", + "DvcMacAddr": "string", + "DvcOriginalAction": "string", + "DvcOs": "string", + "DvcOsVersion": "string", + "DvcScope": "string", + "DvcScopeId": "string", + "DvcZone": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventMessage": "string", + "EventOriginalResultDetails": "string", + "EventOriginalSeverity": "string", + "EventOriginalSubType": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventOwner": "string", + "EventProduct": "string", + "EventProductVersion": "string", + "EventReportUrl": "string", + "EventResult": "string", + "EventResultDetails": "string", + "EventSchemaVersion": "string", + "EventSeverity": "string", + "EventStartTime": "datetime", + "EventSubType": "string", + "EventType": "string", + "EventVendor": "string", + "HttpUserAgent": "string", + "NewValue": "string", + "Object": "string", + "ObjectId": "string", + "ObjectType": "string", + "OldValue": "string", + "Operation": "string", + "OriginalObjectType": "string", + "RuleName": "string", + "RuleNumber": "int", + "SourceSystem": "string", + "SrcDescription": "string", + "SrcDeviceType": "string", + "SrcDomain": "string", + "SrcDomainType": "string", + "SrcDvcId": "string", + "SrcDvcIdType": "string", + "SrcDvcScope": "string", + "SrcDvcScopeId": "string", + "SrcFQDN": "string", + "SrcGeoCity": "string", + "SrcGeoCountry": "string", + "SrcGeoLatitude": "real", + "SrcGeoLongitude": "real", + "SrcGeoRegion": "string", + "SrcHostname": "string", + "SrcIpAddr": "string", + "SrcOriginalRiskLevel": "string", + "SrcPortNumber": "int", + "SrcRiskLevel": "int", + "SrcSystemEntityKey": "string", + "TargetAppId": "string", + "TargetAppName": "string", + "TargetAppType": "string", + "TargetApplicationEntityKey": "string", + "TargetDescription": "string", + "TargetDeviceType": "string", + "TargetDomain": "string", + "TargetDomainType": "string", + "TargetDvcId": "string", + "TargetDvcIdType": "string", + "TargetDvcOs": "string", + "TargetDvcScope": "string", + "TargetDvcScopeId": "string", + "TargetFQDN": "string", + "TargetGeoCity": "string", + "TargetGeoCountry": "string", + "TargetGeoLatitude": "real", + "TargetGeoLongitude": "real", + "TargetGeoRegion": "string", + "TargetHostname": "string", + "TargetIpAddr": "string", + "TargetOriginalAppType": "string", + "TargetOriginalRiskLevel": "string", + "TargetPortNumber": "int", + "TargetRiskLevel": "int", + "TargetSystemEntityKey": "string", + "TargetUrl": "string", + "TenantId": "string", + "ThreatCategory": "string", + "ThreatConfidence": "int", + "ThreatField": "string", + "ThreatFirstReportedTime": "datetime", + "ThreatId": "string", + "ThreatIpAddr": "string", + "ThreatIsActive": "bool", + "ThreatLastReportedTime": "datetime", + "ThreatName": "string", + "ThreatOriginalConfidence": "string", + "ThreatOriginalRiskLevel": "string", + "ThreatRiskLevel": "int", + "TimeGenerated": "datetime", + "Type": "string", + "ValueType": "string", + "_ResourceId": "string" + }, + "ASimAuthenticationEventLogs": { + "ActingAppId": "string", + "ActingAppName": "string", + "ActingAppType": "string", + "ActingApplicationEntityKey": "string", + "ActingOriginalAppType": "string", + "ActorOriginalUserType": "string", + "ActorScope": "string", + "ActorScopeId": "string", + "ActorSessionId": "string", + "ActorUserAdditionalIds": "dynamic", + "ActorUserEntityKey": "string", + "ActorUserId": "string", + "ActorUserIdType": "string", + "ActorUserType": "string", + "ActorUsername": "string", + "ActorUsernameType": "string", + "AdditionalEntities": "dynamic", + "AdditionalFields": "dynamic", + "DvcAction": "string", + "DvcDescription": "string", + "DvcDomain": "string", + "DvcDomainType": "string", + "DvcEntityKey": "string", + "DvcFQDN": "string", + "DvcHostname": "string", + "DvcId": "string", + "DvcIdType": "string", + "DvcInterface": "string", + "DvcIpAddr": "string", + "DvcMacAddr": "string", + "DvcOriginalAction": "string", + "DvcOs": "string", + "DvcOsVersion": "string", + "DvcScope": "string", + "DvcScopeId": "string", + "DvcZone": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventMessage": "string", + "EventOriginalResultDetails": "string", + "EventOriginalSeverity": "string", + "EventOriginalSubType": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventOwner": "string", + "EventProduct": "string", + "EventProductVersion": "string", + "EventReportUrl": "string", + "EventResult": "string", + "EventResultDetails": "string", + "EventSchemaVersion": "string", + "EventSeverity": "string", + "EventStartTime": "datetime", + "EventSubType": "string", + "EventType": "string", + "EventVendor": "string", + "HttpUserAgent": "string", + "LogonMethod": "string", + "LogonProtocol": "string", + "RuleName": "string", + "RuleNumber": "int", + "SourceSystem": "string", + "SrcDescription": "string", + "SrcDeviceType": "string", + "SrcDomain": "string", + "SrcDomainType": "string", + "SrcDvcId": "string", + "SrcDvcIdType": "string", + "SrcDvcOs": "string", + "SrcDvcScope": "string", + "SrcDvcScopeId": "string", + "SrcFQDN": "string", + "SrcGeoCity": "string", + "SrcGeoCountry": "string", + "SrcGeoLatitude": "real", + "SrcGeoLongitude": "real", + "SrcGeoRegion": "string", + "SrcHostname": "string", + "SrcIpAddr": "string", + "SrcIsp": "string", + "SrcOriginalRiskLevel": "string", + "SrcPortNumber": "int", + "SrcRiskLevel": "int", + "SrcSystemEntityKey": "string", + "TargetAppId": "string", + "TargetAppName": "string", + "TargetAppType": "string", + "TargetApplicationEntityKey": "string", + "TargetDescription": "string", + "TargetDeviceType": "string", + "TargetDomain": "string", + "TargetDomainType": "string", + "TargetDvcId": "string", + "TargetDvcIdType": "string", + "TargetDvcOs": "string", + "TargetDvcScope": "string", + "TargetDvcScopeId": "string", + "TargetFQDN": "string", + "TargetGeoCity": "string", + "TargetGeoCountry": "string", + "TargetGeoLatitude": "real", + "TargetGeoLongitude": "real", + "TargetGeoRegion": "string", + "TargetHostname": "string", + "TargetIpAddr": "string", + "TargetOriginalAppType": "string", + "TargetOriginalRiskLevel": "string", + "TargetOriginalUserType": "string", + "TargetPortNumber": "int", + "TargetRiskLevel": "int", + "TargetSessionId": "string", + "TargetSystemEntityKey": "string", + "TargetUrl": "string", + "TargetUserAdditionalIds": "dynamic", + "TargetUserEntityKey": "string", + "TargetUserId": "string", + "TargetUserIdType": "string", + "TargetUserScope": "string", + "TargetUserScopeId": "string", + "TargetUserType": "string", + "TargetUsername": "string", + "TargetUsernameType": "string", + "TenantId": "string", + "ThreatCategory": "string", + "ThreatConfidence": "int", + "ThreatField": "string", + "ThreatFirstReportedTime": "datetime", + "ThreatId": "string", + "ThreatIpAddr": "string", + "ThreatIsActive": "bool", + "ThreatLastReportedTime": "datetime", + "ThreatName": "string", + "ThreatOriginalConfidence": "string", + "ThreatOriginalRiskLevel": "string", + "ThreatRiskLevel": "int", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "ASimDhcpEventLogs": { + "AdditionalEntities": "dynamic", + "AdditionalFields": "dynamic", + "DhcpCircuitId": "string", + "DhcpLeaseDuration": "int", + "DhcpSessionDuration": "int", + "DhcpSessionId": "string", + "DhcpSrcDHCId": "string", + "DhcpSubscriberId": "string", + "DhcpUserClass": "string", + "DhcpUserClassId": "string", + "DhcpVendorClass": "string", + "DhcpVendorClassId": "string", + "DvcAction": "string", + "DvcDescription": "string", + "DvcDomain": "string", + "DvcDomainType": "string", + "DvcEntityKey": "string", + "DvcFQDN": "string", + "DvcHostname": "string", + "DvcId": "string", + "DvcIdType": "string", + "DvcInterface": "string", + "DvcIpAddr": "string", + "DvcMacAddr": "string", + "DvcOriginalAction": "string", + "DvcOs": "string", + "DvcOsVersion": "string", + "DvcScope": "string", + "DvcScopeId": "string", + "DvcZone": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventMessage": "string", + "EventOriginalResultDetails": "string", + "EventOriginalSeverity": "string", + "EventOriginalSubType": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventOwner": "string", + "EventProduct": "string", + "EventProductVersion": "string", + "EventReportUrl": "string", + "EventResult": "string", + "EventResultDetails": "string", + "EventSchema": "string", + "EventSchemaVersion": "string", + "EventSeverity": "string", + "EventStartTime": "datetime", + "EventSubType": "string", + "EventType": "string", + "EventVendor": "string", + "RequestedIpAddr": "string", + "RuleName": "string", + "RuleNumber": "int", + "SourceSystem": "string", + "SrcDescription": "string", + "SrcDeviceType": "string", + "SrcDomain": "string", + "SrcDomainType": "string", + "SrcDvcId": "string", + "SrcDvcIdType": "string", + "SrcDvcScope": "string", + "SrcDvcScopeId": "string", + "SrcFQDN": "string", + "SrcGeoCity": "string", + "SrcGeoCountry": "string", + "SrcGeoLatitude": "real", + "SrcGeoLongitude": "real", + "SrcGeoRegion": "string", + "SrcHostname": "string", + "SrcIpAddr": "string", + "SrcMacAddr": "string", + "SrcOriginalRiskLevel": "string", + "SrcOriginalUserType": "string", + "SrcPortNumber": "int", + "SrcRiskLevel": "int", + "SrcSystemEntityKey": "string", + "SrcUserAdditionalIds": "dynamic", + "SrcUserEntityKey": "string", + "SrcUserId": "string", + "SrcUserIdType": "string", + "SrcUserScope": "string", + "SrcUserScopeId": "string", + "SrcUserSessionId": "string", + "SrcUserType": "string", + "SrcUserUid": "string", + "SrcUsername": "string", + "SrcUsernameType": "string", + "TenantId": "string", + "ThreatCategory": "string", + "ThreatConfidence": "int", + "ThreatField": "string", + "ThreatFirstReportedTime": "datetime", + "ThreatId": "string", + "ThreatIsActive": "bool", + "ThreatLastReportedTime": "datetime", + "ThreatName": "string", + "ThreatOriginalConfidence": "string", + "ThreatOriginalRiskLevel": "string", + "ThreatRiskLevel": "int", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "ASimDnsActivityLogs": { + "AdditionalEntities": "dynamic", + "AdditionalFields": "dynamic", + "DnsFlags": "string", + "DnsFlagsAuthenticated": "bool", + "DnsFlagsAuthoritative": "bool", + "DnsFlagsCheckingDisabled": "bool", + "DnsFlagsRecursionAvailable": "bool", + "DnsFlagsRecursionDesired": "bool", + "DnsFlagsTruncated": "bool", + "DnsFlagsZ": "bool", + "DnsNetworkDuration": "int", + "DnsQuery": "string", + "DnsQueryClass": "int", + "DnsQueryClassName": "string", + "DnsQueryType": "int", + "DnsQueryTypeName": "string", + "DnsResponseCode": "int", + "DnsResponseIpCity": "string", + "DnsResponseIpCountry": "string", + "DnsResponseIpLatitude": "real", + "DnsResponseIpLongitude": "real", + "DnsResponseIpRegion": "string", + "DnsResponseName": "string", + "DnsSessionId": "string", + "Dst": "string", + "DstDescription": "string", + "DstDeviceType": "string", + "DstDomain": "string", + "DstDomainType": "string", + "DstDvcId": "string", + "DstDvcIdType": "string", + "DstDvcScope": "string", + "DstDvcScopeId": "string", + "DstFQDN": "string", + "DstGeoCity": "string", + "DstGeoCountry": "string", + "DstGeoLatitude": "real", + "DstGeoLongitude": "real", + "DstGeoRegion": "string", + "DstHostname": "string", + "DstIpAddr": "string", + "DstOriginalRiskLevel": "string", + "DstPortNumber": "int", + "DstRiskLevel": "int", + "DstSystemEntityKey": "string", + "Dvc": "string", + "DvcAction": "string", + "DvcDescription": "string", + "DvcDomain": "string", + "DvcDomainType": "string", + "DvcEntityKey": "string", + "DvcFQDN": "string", + "DvcHostname": "string", + "DvcId": "string", + "DvcIdType": "string", + "DvcInterface": "string", + "DvcIpAddr": "string", + "DvcMacAddr": "string", + "DvcOriginalAction": "string", + "DvcOs": "string", + "DvcOsVersion": "string", + "DvcScope": "string", + "DvcScopeId": "string", + "DvcZone": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventMessage": "string", + "EventOriginalSeverity": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventOwner": "string", + "EventProduct": "string", + "EventProductVersion": "string", + "EventReportUrl": "string", + "EventResult": "string", + "EventResultDetails": "string", + "EventSchemaVersion": "string", + "EventSeverity": "string", + "EventStartTime": "datetime", + "EventSubType": "string", + "EventType": "string", + "EventVendor": "string", + "NetworkProtocol": "string", + "NetworkProtocolVersion": "string", + "RuleName": "string", + "RuleNumber": "int", + "SourceSystem": "string", + "Src": "string", + "SrcDescription": "string", + "SrcDeviceType": "string", + "SrcDomain": "string", + "SrcDomainType": "string", + "SrcDvcId": "string", + "SrcDvcIdType": "string", + "SrcDvcScope": "string", + "SrcDvcScopeId": "string", + "SrcFQDN": "string", + "SrcGeoCity": "string", + "SrcGeoCountry": "string", + "SrcGeoLatitude": "real", + "SrcGeoLongitude": "real", + "SrcGeoRegion": "string", + "SrcHostname": "string", + "SrcIpAddr": "string", + "SrcOriginalRiskLevel": "string", + "SrcOriginalUserType": "string", + "SrcPortNumber": "int", + "SrcProcessEntityKey": "string", + "SrcProcessGuid": "string", + "SrcProcessId": "string", + "SrcProcessName": "string", + "SrcRiskLevel": "int", + "SrcSystemEntityKey": "string", + "SrcUserAdditionalIds": "dynamic", + "SrcUserEntityKey": "string", + "SrcUserId": "string", + "SrcUserIdType": "string", + "SrcUserScope": "string", + "SrcUserScopeId": "string", + "SrcUserSessionId": "string", + "SrcUserType": "string", + "SrcUsername": "string", + "SrcUsernameType": "string", + "TenantId": "string", + "ThreatCategory": "string", + "ThreatConfidence": "int", + "ThreatField": "string", + "ThreatFirstReportedTime": "string", + "ThreatFirstReportedTime_d": "datetime", + "ThreatId": "string", + "ThreatIpAddr": "string", + "ThreatIsActive": "bool", + "ThreatLastReportedTime": "string", + "ThreatLastReportedTime_d": "datetime", + "ThreatName": "string", + "ThreatOriginalConfidence": "string", + "ThreatOriginalRiskLevel": "int", + "ThreatOriginalRiskLevel_s": "string", + "ThreatRiskLevel": "int", + "TimeGenerated": "datetime", + "TransactionIdHex": "string", + "Type": "string", + "UrlCategory": "string", + "_ResourceId": "string" + }, + "ASimFileEventLogs": { + "ActingApplicationEntityKey": "string", + "ActingProcessCommandLine": "string", + "ActingProcessGuid": "string", + "ActingProcessId": "string", + "ActingProcessName": "string", + "ActorOriginalUserType": "string", + "ActorScope": "string", + "ActorScopeId": "string", + "ActorSessionId": "string", + "ActorUserAadId": "string", + "ActorUserAdditionalIds": "dynamic", + "ActorUserEntityKey": "string", + "ActorUserId": "string", + "ActorUserIdType": "string", + "ActorUserSid": "string", + "ActorUserType": "string", + "ActorUsername": "string", + "ActorUsernameType": "string", + "AdditionalEntities": "dynamic", + "AdditionalFields": "dynamic", + "DvcAction": "string", + "DvcDescription": "string", + "DvcDomain": "string", + "DvcDomainType": "string", + "DvcEntityKey": "string", + "DvcFQDN": "string", + "DvcHostname": "string", + "DvcId": "string", + "DvcIdType": "string", + "DvcInterface": "string", + "DvcIpAddr": "string", + "DvcMacAddr": "string", + "DvcOriginalAction": "string", + "DvcOs": "string", + "DvcOsVersion": "string", + "DvcScope": "string", + "DvcScopeId": "string", + "DvcZone": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventMessage": "string", + "EventOriginalResultDetails": "string", + "EventOriginalSeverity": "string", + "EventOriginalSubType": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventOwner": "string", + "EventProduct": "string", + "EventProductVersion": "string", + "EventReportUrl": "string", + "EventResult": "string", + "EventResultDetails": "string", + "EventSchema": "string", + "EventSchemaVersion": "string", + "EventSeverity": "string", + "EventStartTime": "datetime", + "EventSubType": "string", + "EventType": "string", + "EventVendor": "string", + "HashType": "string", + "HttpUserAgent": "string", + "NetworkApplicationProtocol": "string", + "RuleName": "string", + "RuleNumber": "int", + "SourceSystem": "string", + "SrcDescription": "string", + "SrcDeviceType": "string", + "SrcDomain": "string", + "SrcDomainType": "string", + "SrcDvcId": "string", + "SrcDvcIdType": "string", + "SrcDvcScope": "string", + "SrcDvcScopeId": "string", + "SrcFQDN": "string", + "SrcFileCreationTime": "datetime", + "SrcFileDirectory": "string", + "SrcFileEntityKey": "string", + "SrcFileExtension": "string", + "SrcFileMD5": "string", + "SrcFileMimeType": "string", + "SrcFileName": "string", + "SrcFilePath": "string", + "SrcFilePathType": "string", + "SrcFileSHA1": "string", + "SrcFileSHA256": "string", + "SrcFileSHA512": "string", + "SrcFileSize": "long", + "SrcGeoCity": "string", + "SrcGeoCountry": "string", + "SrcGeoLatitude": "real", + "SrcGeoLongitude": "real", + "SrcGeoRegion": "string", + "SrcHostname": "string", + "SrcIpAddr": "string", + "SrcMacAddr": "string", + "SrcOriginalRiskLevel": "string", + "SrcPortNumber": "int", + "SrcProcessEntityKey": "string", + "SrcRiskLevel": "int", + "SrcSystemEntityKey": "string", + "TargetAppId": "string", + "TargetAppName": "string", + "TargetAppType": "string", + "TargetApplicationEntityKey": "string", + "TargetFileCreationTime": "datetime", + "TargetFileDirectory": "string", + "TargetFileEntityKey": "string", + "TargetFileExtension": "string", + "TargetFileMD5": "string", + "TargetFileMimeType": "string", + "TargetFileName": "string", + "TargetFilePath": "string", + "TargetFilePathType": "string", + "TargetFileSHA1": "string", + "TargetFileSHA256": "string", + "TargetFileSHA512": "string", + "TargetFileSize": "long", + "TargetOriginalAppType": "string", + "TargetUrl": "string", + "TenantId": "string", + "ThreatCategory": "string", + "ThreatConfidence": "int", + "ThreatField": "string", + "ThreatFilePath": "string", + "ThreatFirstReportedTime": "datetime", + "ThreatId": "string", + "ThreatIsActive": "bool", + "ThreatLastReportedTime": "datetime", + "ThreatName": "string", + "ThreatOriginalConfidence": "string", + "ThreatOriginalRiskLevel": "string", + "ThreatRiskLevel": "int", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "ASimNetworkSessionLogs": { + "AdditionalEntities": "dynamic", + "AdditionalFields": "dynamic", + "DstAppId": "string", + "DstAppName": "string", + "DstAppType": "string", + "DstApplicationEntityKey": "string", + "DstBytes": "long", + "DstDescription": "string", + "DstDeviceType": "string", + "DstDomain": "string", + "DstDomainType": "string", + "DstDvcId": "string", + "DstDvcIdType": "string", + "DstFQDN": "string", + "DstGeoCity": "string", + "DstGeoCountry": "string", + "DstGeoLatitude": "real", + "DstGeoLongitude": "real", + "DstGeoRegion": "string", + "DstHostname": "string", + "DstInterfaceGuid": "string", + "DstInterfaceName": "string", + "DstIpAddr": "string", + "DstMacAddr": "string", + "DstNatIpAddr": "string", + "DstNatPortNumber": "int", + "DstOriginalUserType": "string", + "DstPackets": "long", + "DstPortNumber": "int", + "DstSubscriptionId": "string", + "DstSystemEntityKey": "string", + "DstUserAdditionalIds": "dynamic", + "DstUserEntityKey": "string", + "DstUserId": "string", + "DstUserIdType": "string", + "DstUserType": "string", + "DstUsername": "string", + "DstUsernameType": "string", + "DstVlanId": "string", + "DstZone": "string", + "Dvc": "string", + "DvcAction": "string", + "DvcDescription": "string", + "DvcDomain": "string", + "DvcDomainType": "string", + "DvcEntityKey": "string", + "DvcFQDN": "string", + "DvcHostname": "string", + "DvcId": "string", + "DvcIdType": "string", + "DvcInboundInterface": "string", + "DvcInterface": "string", + "DvcIpAddr": "string", + "DvcMacAddr": "string", + "DvcOriginalAction": "string", + "DvcOs": "string", + "DvcOsVersion": "string", + "DvcOutboundInterface": "string", + "DvcSubscriptionId": "string", + "DvcZone": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventMessage": "string", + "EventOriginalResultDetails": "string", + "EventOriginalSeverity": "string", + "EventOriginalSubType": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventProduct": "string", + "EventProductVersion": "string", + "EventReportUrl": "string", + "EventResult": "string", + "EventResultDetails": "string", + "EventSchemaVersion": "string", + "EventSeverity": "string", + "EventStartTime": "datetime", + "EventSubType": "string", + "EventType": "string", + "EventVendor": "string", + "NetworkApplicationProtocol": "string", + "NetworkBytes": "long", + "NetworkConnectionHistory": "string", + "NetworkDirection": "string", + "NetworkDuration": "int", + "NetworkIcmpCode": "int", + "NetworkIcmpType": "string", + "NetworkPackets": "long", + "NetworkProtocol": "string", + "NetworkProtocolVersion": "string", + "NetworkRuleName": "string", + "NetworkRuleNumber": "int", + "NetworkSessionId": "string", + "SourceSystem": "string", + "SrcAppId": "string", + "SrcAppName": "string", + "SrcAppType": "string", + "SrcApplicationEntityKey": "string", + "SrcBytes": "long", + "SrcDescription": "string", + "SrcDeviceType": "string", + "SrcDomain": "string", + "SrcDomainType": "string", + "SrcDvcId": "string", + "SrcDvcIdType": "string", + "SrcFQDN": "string", + "SrcGeoCity": "string", + "SrcGeoCountry": "string", + "SrcGeoLatitude": "real", + "SrcGeoLongitude": "real", + "SrcGeoRegion": "string", + "SrcHostname": "string", + "SrcInterfaceGuid": "string", + "SrcInterfaceName": "string", + "SrcIpAddr": "string", + "SrcMacAddr": "string", + "SrcNatIpAddr": "string", + "SrcNatPortNumber": "int", + "SrcOriginalUserType": "string", + "SrcPackets": "long", + "SrcPortNumber": "int", + "SrcSubscriptionId": "string", + "SrcSystemEntityKey": "string", + "SrcUserAdditionalIds": "dynamic", + "SrcUserEntityKey": "string", + "SrcUserId": "string", + "SrcUserIdType": "string", + "SrcUserType": "string", + "SrcUsername": "string", + "SrcUsernameType": "string", + "SrcVlanId": "string", + "SrcZone": "string", + "TcpFlagsAck": "bool", + "TcpFlagsFin": "bool", + "TcpFlagsPsh": "bool", + "TcpFlagsRst": "bool", + "TcpFlagsSyn": "bool", + "TcpFlagsUrg": "bool", + "TenantId": "string", + "ThreatCategory": "string", + "ThreatConfidence": "int", + "ThreatField": "string", + "ThreatFirstReportedTime": "datetime", + "ThreatId": "string", + "ThreatIpAddr": "string", + "ThreatIsActive": "bool", + "ThreatLastReportedTime": "datetime", + "ThreatName": "string", + "ThreatOriginalConfidence": "string", + "ThreatOriginalRiskLevel": "string", + "ThreatRiskLevel": "int", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "ASimProcessEventLogs": { + "ActingProcessCommandLine": "string", + "ActingProcessCreationTime": "datetime", + "ActingProcessEntityKey": "string", + "ActingProcessFileCompany": "string", + "ActingProcessFileDescription": "string", + "ActingProcessFileInternalName": "string", + "ActingProcessFileOriginalName": "string", + "ActingProcessFileProduct": "string", + "ActingProcessFileSize": "long", + "ActingProcessFileVersion": "string", + "ActingProcessFilename": "string", + "ActingProcessGuid": "string", + "ActingProcessIMPHASH": "string", + "ActingProcessId": "string", + "ActingProcessInjectedAddress": "string", + "ActingProcessIntegrityLevel": "string", + "ActingProcessIsHidden": "bool", + "ActingProcessMD5": "string", + "ActingProcessName": "string", + "ActingProcessSHA1": "string", + "ActingProcessSHA256": "string", + "ActingProcessSHA512": "string", + "ActingProcessTokenElevation": "string", + "ActorOriginalUserType": "string", + "ActorScope": "string", + "ActorScopeId": "string", + "ActorSessionId": "string", + "ActorUserAdditionalIds": "dynamic", + "ActorUserEntityKey": "string", + "ActorUserId": "string", + "ActorUserIdType": "string", + "ActorUserType": "string", + "ActorUsername": "string", + "ActorUsernameType": "string", + "AdditionalEntities": "dynamic", + "AdditionalFields": "dynamic", + "DvcAction": "string", + "DvcDescription": "string", + "DvcDomain": "string", + "DvcDomainType": "string", + "DvcEntityKey": "string", + "DvcFQDN": "string", + "DvcHostname": "string", + "DvcId": "string", + "DvcIdType": "string", + "DvcInterface": "string", + "DvcIpAddr": "string", + "DvcMacAddr": "string", + "DvcOriginalAction": "string", + "DvcOs": "string", + "DvcOsVersion": "string", + "DvcScope": "string", + "DvcScopeId": "string", + "DvcZone": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventMessage": "string", + "EventOriginalResultDetails": "string", + "EventOriginalSeverity": "string", + "EventOriginalSubType": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventOwner": "string", + "EventProduct": "string", + "EventProductVersion": "string", + "EventReportUrl": "string", + "EventResult": "string", + "EventResultDetails": "string", + "EventSchemaVersion": "string", + "EventSeverity": "string", + "EventStartTime": "datetime", + "EventSubType": "string", + "EventType": "string", + "EventVendor": "string", + "ParentProcessCreationTime": "datetime", + "ParentProcessEntityKey": "string", + "ParentProcessFileCompany": "string", + "ParentProcessFileDescription": "string", + "ParentProcessFileProduct": "string", + "ParentProcessFileVersion": "string", + "ParentProcessGuid": "string", + "ParentProcessIMPHASH": "string", + "ParentProcessId": "string", + "ParentProcessInjectedAddress": "string", + "ParentProcessIntegrityLevel": "string", + "ParentProcessIsHidden": "bool", + "ParentProcessMD5": "string", + "ParentProcessName": "string", + "ParentProcessSHA1": "string", + "ParentProcessSHA256": "string", + "ParentProcessSHA512": "string", + "ParentProcessTokenElevation": "string", + "RuleName": "string", + "RuleNumber": "int", + "SourceSystem": "string", + "TargetOriginalUserType": "string", + "TargetProcessCommandLine": "string", + "TargetProcessCreationTime": "datetime", + "TargetProcessCurrentDirectory": "string", + "TargetProcessEntityKey": "string", + "TargetProcessFileCompany": "string", + "TargetProcessFileDescription": "string", + "TargetProcessFileInternalName": "string", + "TargetProcessFileOriginalName": "string", + "TargetProcessFileProduct": "string", + "TargetProcessFileSize": "long", + "TargetProcessFileVersion": "string", + "TargetProcessFilename": "string", + "TargetProcessGuid": "string", + "TargetProcessIMPHASH": "string", + "TargetProcessId": "string", + "TargetProcessInjectedAddress": "string", + "TargetProcessIntegrityLevel": "string", + "TargetProcessIsHidden": "bool", + "TargetProcessMD5": "string", + "TargetProcessName": "string", + "TargetProcessSHA1": "string", + "TargetProcessSHA256": "string", + "TargetProcessSHA512": "string", + "TargetProcessStatusCode": "string", + "TargetProcessTokenElevation": "string", + "TargetScope": "string", + "TargetScopeId": "string", + "TargetUserAdditionalIds": "dynamic", + "TargetUserEntityKey": "string", + "TargetUserId": "string", + "TargetUserIdType": "string", + "TargetUserSessionGuid": "string", + "TargetUserSessionId": "string", + "TargetUserType": "string", + "TargetUsername": "string", + "TargetUsernameType": "string", + "TenantId": "string", + "ThreatCategory": "string", + "ThreatConfidence": "int", + "ThreatField": "string", + "ThreatFirstReportedTime": "datetime", + "ThreatId": "string", + "ThreatIsActive": "bool", + "ThreatLastReportedTime": "datetime", + "ThreatName": "string", + "ThreatOriginalConfidence": "string", + "ThreatOriginalRiskLevel": "string", + "ThreatRiskLevel": "int", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "ASimRegistryEventLogs": { + "ActingProcessCommandLine": "string", + "ActingProcessEntityKey": "string", + "ActingProcessGuid": "string", + "ActingProcessId": "string", + "ActingProcessName": "string", + "ActorOriginalUserType": "string", + "ActorScope": "string", + "ActorScopeId": "string", + "ActorSessionId": "string", + "ActorUserAadId": "string", + "ActorUserAdditionalIds": "dynamic", + "ActorUserEntityKey": "string", + "ActorUserId": "string", + "ActorUserIdType": "string", + "ActorUserSid": "string", + "ActorUserType": "string", + "ActorUsername": "string", + "ActorUsernameType": "string", + "AdditionalEntities": "dynamic", + "AdditionalFields": "dynamic", + "DvcAction": "string", + "DvcDescription": "string", + "DvcDomain": "string", + "DvcDomainType": "string", + "DvcEntityKey": "string", + "DvcFQDN": "string", + "DvcHostname": "string", + "DvcId": "string", + "DvcIdType": "string", + "DvcInterface": "string", + "DvcIpAddr": "string", + "DvcMacAddr": "string", + "DvcOriginalAction": "string", + "DvcOs": "string", + "DvcOsVersion": "string", + "DvcScope": "string", + "DvcScopeId": "string", + "DvcZone": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventMessage": "string", + "EventOriginalResultDetails": "string", + "EventOriginalSeverity": "string", + "EventOriginalSubType": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventOwner": "string", + "EventProduct": "string", + "EventProductVersion": "string", + "EventReportUrl": "string", + "EventResult": "string", + "EventResultDetails": "string", + "EventSchema": "string", + "EventSchemaVersion": "string", + "EventSeverity": "string", + "EventStartTime": "datetime", + "EventSubType": "string", + "EventType": "string", + "EventVendor": "string", + "ParentProcessCommandLine": "string", + "ParentProcessEntityKey": "string", + "ParentProcessGuid": "string", + "ParentProcessId": "string", + "ParentProcessName": "string", + "RegistryKey": "string", + "RegistryPreviousKey": "string", + "RegistryPreviousValue": "string", + "RegistryPreviousValueData": "string", + "RegistryPreviousValueType": "string", + "RegistryValue": "string", + "RegistryValueData": "string", + "RegistryValueType": "string", + "RuleName": "string", + "RuleNumber": "int", + "SourceSystem": "string", + "TenantId": "string", + "ThreatCategory": "string", + "ThreatConfidence": "int", + "ThreatField": "string", + "ThreatFirstReportedTime": "datetime", + "ThreatId": "string", + "ThreatIsActive": "bool", + "ThreatLastReportedTime": "datetime", + "ThreatName": "string", + "ThreatOriginalConfidence": "string", + "ThreatOriginalRiskLevel": "string", + "ThreatRiskLevel": "int", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "ASimUserManagementActivityLogs": { + "ActingAppId": "string", + "ActingAppName": "string", + "ActingAppType": "string", + "ActingApplicationEntityKey": "string", + "ActingOriginalAppType": "string", + "ActorOriginalUserType": "string", + "ActorScope": "string", + "ActorScopeId": "string", + "ActorSessionId": "string", + "ActorUserAadId": "string", + "ActorUserAdditionalIds": "dynamic", + "ActorUserEntityKey": "string", + "ActorUserId": "string", + "ActorUserIdType": "string", + "ActorUserSid": "string", + "ActorUserType": "string", + "ActorUsername": "string", + "ActorUsernameType": "string", + "AdditionalEntities": "dynamic", + "AdditionalFields": "dynamic", + "DvcAction": "string", + "DvcDescription": "string", + "DvcDomain": "string", + "DvcDomainType": "string", + "DvcEntityKey": "string", + "DvcFQDN": "string", + "DvcHostname": "string", + "DvcId": "string", + "DvcIdType": "string", + "DvcInterface": "string", + "DvcIpAddr": "string", + "DvcMacAddr": "string", + "DvcOriginalAction": "string", + "DvcOs": "string", + "DvcOsVersion": "string", + "DvcScope": "string", + "DvcScopeId": "string", + "DvcZone": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventMessage": "string", + "EventOriginalResultDetails": "string", + "EventOriginalSeverity": "string", + "EventOriginalSubType": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventOwner": "string", + "EventProduct": "string", + "EventProductVersion": "string", + "EventReportUrl": "string", + "EventResult": "string", + "EventResultDetails": "string", + "EventSchema": "string", + "EventSchemaVersion": "string", + "EventSeverity": "string", + "EventStartTime": "datetime", + "EventSubType": "string", + "EventType": "string", + "EventVendor": "string", + "GroupId": "string", + "GroupIdType": "string", + "GroupName": "string", + "GroupNameType": "string", + "GroupOriginalType": "string", + "GroupType": "string", + "HttpUserAgent": "string", + "NewPropertyValue": "string", + "PreviousPropertyValue": "string", + "RuleName": "string", + "RuleNumber": "int", + "SourceSystem": "string", + "SrcDescription": "string", + "SrcDeviceType": "string", + "SrcDomain": "string", + "SrcDomainType": "string", + "SrcDvcId": "string", + "SrcDvcIdType": "string", + "SrcDvcScope": "string", + "SrcDvcScopeId": "string", + "SrcFQDN": "string", + "SrcGeoCity": "string", + "SrcGeoCountry": "string", + "SrcGeoLatitude": "real", + "SrcGeoLongitude": "real", + "SrcGeoRegion": "string", + "SrcHostname": "string", + "SrcIpAddr": "string", + "SrcMacAddr": "string", + "SrcOriginalRiskLevel": "string", + "SrcPortNumber": "int", + "SrcRiskLevel": "int", + "SrcSystemEntityKey": "string", + "TargetOriginalUserType": "string", + "TargetUserAdditionalIds": "dynamic", + "TargetUserEntityKey": "string", + "TargetUserId": "string", + "TargetUserIdType": "string", + "TargetUserScope": "string", + "TargetUserScopeId": "string", + "TargetUserSessionId": "string", + "TargetUserType": "string", + "TargetUserUid": "string", + "TargetUsername": "string", + "TargetUsernameType": "string", + "TenantId": "string", + "ThreatCategory": "string", + "ThreatConfidence": "int", + "ThreatField": "string", + "ThreatFirstReportedTime": "datetime", + "ThreatId": "string", + "ThreatIsActive": "bool", + "ThreatLastReportedTime": "datetime", + "ThreatName": "string", + "ThreatOriginalConfidence": "string", + "ThreatOriginalRiskLevel": "string", + "ThreatRiskLevel": "int", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "ASimWebSessionLogs": { + "AdditionalEntities": "dynamic", + "AdditionalFields": "dynamic", + "DstAppId": "string", + "DstAppName": "string", + "DstAppType": "string", + "DstApplicationEntityKey": "string", + "DstBytes": "long", + "DstDeviceType": "string", + "DstDomain": "string", + "DstDomainType": "string", + "DstDvcId": "string", + "DstDvcIdType": "string", + "DstDvcScope": "string", + "DstDvcScopeId": "string", + "DstFQDN": "string", + "DstGeoCity": "string", + "DstGeoCountry": "string", + "DstGeoLatitude": "real", + "DstGeoLongitude": "real", + "DstGeoRegion": "string", + "DstHostname": "string", + "DstIpAddr": "string", + "DstMacAddr": "string", + "DstNatIpAddr": "string", + "DstNatPortNumber": "int", + "DstOriginalUserType": "string", + "DstPackets": "long", + "DstPortNumber": "int", + "DstSystemEntityKey": "string", + "DstUserAdditionalIds": "dynamic", + "DstUserEntityKey": "string", + "DstUserId": "string", + "DstUserIdType": "string", + "DstUserType": "string", + "DstUsername": "string", + "DstUsernameType": "string", + "Dvc": "string", + "DvcAction": "string", + "DvcDomain": "string", + "DvcDomainType": "string", + "DvcEntityKey": "string", + "DvcFQDN": "string", + "DvcHostname": "string", + "DvcId": "string", + "DvcIdType": "string", + "DvcIpAddr": "string", + "DvcOriginalAction": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventMessage": "string", + "EventOriginalResultDetails": "string", + "EventOriginalSeverity": "string", + "EventOriginalSubType": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventOwner": "string", + "EventProduct": "string", + "EventProductVersion": "string", + "EventReportUrl": "string", + "EventResult": "string", + "EventResultDetails": "string", + "EventSchemaVersion": "string", + "EventSeverity": "string", + "EventStartTime": "datetime", + "EventSubType": "string", + "EventType": "string", + "EventVendor": "string", + "FileContentType": "string", + "FileMD5": "string", + "FileName": "string", + "FileSHA1": "string", + "FileSHA256": "string", + "FileSHA512": "string", + "FileSize": "int", + "HttpContentFormat": "string", + "HttpContentType": "string", + "HttpHost": "string", + "HttpReferrer": "string", + "HttpRequestMethod": "string", + "HttpRequestTime": "int", + "HttpRequestXff": "string", + "HttpResponseTime": "int", + "HttpUserAgent": "string", + "HttpVersion": "string", + "NetworkApplicationProtocol": "string", + "NetworkBytes": "long", + "NetworkConnectionHistory": "string", + "NetworkDirection": "string", + "NetworkDuration": "int", + "NetworkIcmpCode": "int", + "NetworkIcmpType": "string", + "NetworkPackets": "long", + "NetworkProtocol": "string", + "NetworkProtocolVersion": "string", + "NetworkSessionId": "string", + "Rule": "string", + "RuleName": "string", + "RuleNumber": "int", + "SourceSystem": "string", + "SrcAppId": "string", + "SrcAppName": "string", + "SrcAppType": "string", + "SrcApplicationEntityKey": "string", + "SrcBytes": "long", + "SrcDeviceType": "string", + "SrcDomain": "string", + "SrcDomainType": "string", + "SrcDvcId": "string", + "SrcDvcIdType": "string", + "SrcDvcScope": "string", + "SrcDvcScopeId": "string", + "SrcFQDN": "string", + "SrcGeoCity": "string", + "SrcGeoCountry": "string", + "SrcGeoLatitude": "real", + "SrcGeoLongitude": "real", + "SrcGeoRegion": "string", + "SrcHostname": "string", + "SrcIpAddr": "string", + "SrcMacAddr": "string", + "SrcNatIpAddr": "string", + "SrcNatPortNumber": "int", + "SrcOriginalUserType": "string", + "SrcPackets": "long", + "SrcPortNumber": "int", + "SrcProcessGuid": "string", + "SrcProcessId": "string", + "SrcProcessName": "string", + "SrcSystemEntityKey": "string", + "SrcUserAdditionalIds": "dynamic", + "SrcUserEntityKey": "string", + "SrcUserId": "string", + "SrcUserIdType": "string", + "SrcUserScope": "string", + "SrcUserScopeId": "string", + "SrcUserType": "string", + "SrcUsername": "string", + "SrcUsernameType": "string", + "TenantId": "string", + "ThreatCategory": "string", + "ThreatConfidence": "int", + "ThreatField": "string", + "ThreatFirstReportedTime": "datetime", + "ThreatId": "string", + "ThreatIpAddr": "string", + "ThreatIsActive": "bool", + "ThreatLastReportedTime": "datetime", + "ThreatName": "string", + "ThreatOriginalConfidence": "string", + "ThreatOriginalRiskLevel": "string", + "ThreatRiskLevel": "int", + "TimeGenerated": "datetime", + "Type": "string", + "Url": "string", + "UrlCategory": "string", + "UrlOriginal": "string", + "_ResourceId": "string" + }, + "AWSCloudTrail": { + "APIVersion": "string", + "AWSRegion": "string", + "AdditionalEventData": "string", + "AwsEventId": "string", + "AwsRequestId": "string", + "AwsRequestId_": "string", + "Category": "string", + "CidrIp": "string", + "CipherSuite": "string", + "ClientProvidedHostHeader": "string", + "DestinationPort": "string", + "EC2RoleDelivery": "string", + "ErrorCode": "string", + "ErrorMessage": "string", + "EventName": "string", + "EventSource": "string", + "EventTypeName": "string", + "EventVersion": "string", + "IpProtocol": "string", + "ManagementEvent": "bool", + "OperationName": "string", + "ReadOnly": "bool", + "RecipientAccountId": "string", + "RequestParameters": "string", + "Resources": "string", + "ResponseElements": "string", + "ServiceEventDetails": "string", + "SessionCreationDate": "datetime", + "SessionIssuerAccountId": "string", + "SessionIssuerArn": "string", + "SessionIssuerPrincipalId": "string", + "SessionIssuerType": "string", + "SessionIssuerUserName": "string", + "SessionMfaAuthenticated": "bool", + "SharedEventId": "string", + "SourceIpAddress": "string", + "SourcePort": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TlsVersion": "string", + "Type": "string", + "UserAgent": "string", + "UserIdentityAccessKeyId": "string", + "UserIdentityAccountId": "string", + "UserIdentityArn": "string", + "UserIdentityInvokedBy": "string", + "UserIdentityPrincipalid": "string", + "UserIdentityStoreArn": "string", + "UserIdentityType": "string", + "UserIdentityUserId": "string", + "UserIdentityUserName": "string", + "VpcEndpointId": "string" + }, + "AWSGuardDuty": { + "AccountId": "string", + "ActivityType": "string", + "Arn": "string", + "Description": "string", + "Id": "string", + "Partition": "string", + "Region": "string", + "ResourceDetails": "dynamic", + "SchemaVersion": "string", + "ServiceDetails": "dynamic", + "Severity": "int", + "SourceSystem": "string", + "TenantId": "string", + "TimeCreated": "datetime", + "TimeGenerated": "datetime", + "Title": "string", + "Type": "string" + }, + "AWSVPCFlow": { + "AccountId": "string", + "Action": "string", + "AzId": "string", + "Bytes": "long", + "DstAddr": "string", + "DstPort": "int", + "EcsClusterArn": "string", + "EcsClusterName": "string", + "EcsContainerId": "string", + "EcsContainerInstanceArn": "string", + "EcsContainerInstanceId": "string", + "EcsSecondContainerId": "string", + "EcsServiceName": "string", + "EcsTaskArn": "string", + "EcsTaskDefinitionArn": "string", + "EcsTaskId": "string", + "End": "datetime", + "FlowDirection": "string", + "InstanceId": "string", + "InterfaceId": "string", + "LogStatus": "string", + "Packets": "int", + "PktDstAddr": "string", + "PktDstAwsService": "string", + "PktSrcAddr": "string", + "PktSrcAwsService": "string", + "Protocol": "int", + "Region": "string", + "SourceSystem": "string", + "SrcAddr": "string", + "SrcPort": "int", + "Start": "datetime", + "SublocationId": "string", + "SublocationType": "string", + "SubnetId": "string", + "TcpFlags": "int", + "TenantId": "string", + "TimeGenerated": "datetime", + "TrafficPath": "string", + "TrafficType": "string", + "Type": "string", + "Version": "int", + "VpcId": "string" + }, + "AWSWAF": { + "Action": "string", + "Args": "string", + "CaptchaResponse": "dynamic", + "ChallengeResponse": "dynamic", + "ClientIp": "string", + "Country": "string", + "ExcludedRules": "dynamic", + "FormatVersion": "string", + "Headers": "dynamic", + "HttpMethod": "string", + "HttpRequest": "dynamic", + "HttpSourceId": "string", + "HttpSourceName": "string", + "HttpVersion": "string", + "Ja3Fingerprint": "string", + "Labels": "dynamic", + "NonTerminatingMatchingRules": "dynamic", + "OversizeFields": "dynamic", + "RateBasedRuleList": "dynamic", + "RequestHeadersInserted": "dynamic", + "RequestId": "string", + "ResponseCodeSent": "int", + "RuleGroupId": "string", + "RuleGroupList": "dynamic", + "SourceSystem": "string", + "TenantId": "string", + "TerminatingRule": "dynamic", + "TerminatingRuleId": "string", + "TerminatingRuleMatchDetails": "dynamic", + "TerminatingRuleType": "string", + "TimeGenerated": "datetime", + "Type": "string", + "Uri": "string", + "WebAclId": "string" + }, + "AZFWApplicationRule": { + "Action": "string", + "ActionReason": "string", + "DestinationPort": "int", + "Fqdn": "string", + "IsExplicitProxyRequest": "bool", + "IsTlsInspected": "bool", + "Policy": "string", + "Protocol": "string", + "Rule": "string", + "RuleCollection": "string", + "RuleCollectionGroup": "string", + "SourceIp": "string", + "SourcePort": "int", + "SourceSystem": "string", + "TargetUrl": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "WebCategory": "string", + "_ResourceId": "string" + }, + "AZFWDnsQuery": { + "DnssecOkBit": "bool", + "EDNS0BufferSize": "int", + "ErrorMessage": "string", + "ErrorNumber": "int", + "Protocol": "string", + "QueryClass": "string", + "QueryId": "int", + "QueryName": "string", + "QueryType": "string", + "RequestDurationSecs": "real", + "RequestSize": "int", + "ResponseCode": "string", + "ResponseFlags": "string", + "ResponseSize": "int", + "SourceIp": "string", + "SourcePort": "int", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "AZFWIdpsSignature": { + "Action": "string", + "Category": "string", + "Description": "string", + "DestinationIp": "string", + "DestinationPort": "int", + "Protocol": "string", + "Severity": "int", + "SignatureId": "string", + "SourceIp": "string", + "SourcePort": "int", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "AZFWNatRule": { + "DestinationIp": "string", + "DestinationPort": "int", + "Policy": "string", + "Protocol": "string", + "Rule": "string", + "RuleCollection": "string", + "RuleCollectionGroup": "string", + "SourceIp": "string", + "SourcePort": "int", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TranslatedIp": "string", + "TranslatedPort": "int", + "Type": "string", + "_ResourceId": "string" + }, + "AZFWNetworkRule": { + "Action": "string", + "ActionReason": "string", + "DestinationIp": "string", + "DestinationPort": "int", + "Policy": "string", + "Protocol": "string", + "Rule": "string", + "RuleCollection": "string", + "RuleCollectionGroup": "string", + "SourceIp": "string", + "SourcePort": "int", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "AZFWThreatIntel": { + "Action": "string", + "DestinationIp": "string", + "DestinationPort": "int", + "Fqdn": "string", + "IsTlsInspected": "bool", + "Protocol": "string", + "SourceIp": "string", + "SourcePort": "int", + "SourceSystem": "string", + "TargetUrl": "string", + "TenantId": "string", + "ThreatDescription": "string", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "Alert": { + "AlertContext": "string", + "AlertDescription": "string", + "AlertError": "string", + "AlertId": "string", + "AlertName": "string", + "AlertPriority": "string", + "AlertRuleId": "string", + "AlertRuleInstanceId": "string", + "AlertSeverity": "string", + "AlertState": "string", + "AlertStatus": "int", + "AlertTypeDescription": "string", + "AlertTypeNumber": "int", + "AlertUniqueId": "string", + "AlertValue": "int", + "Comments": "string", + "Computer": "string", + "Custom1": "string", + "Custom10": "string", + "Custom2": "string", + "Custom3": "string", + "Custom4": "string", + "Custom5": "string", + "Custom6": "string", + "Custom7": "string", + "Custom8": "string", + "Custom9": "string", + "Expression": "string", + "Flags": "int", + "FlagsDescription": "string", + "HostName": "string", + "LastModifiedBy": "string", + "LinkToSearchResults": "string", + "MG": "string", + "ManagementGroupName": "string", + "ObjectDisplayName": "string", + "PriorityNumber": "int", + "Query": "string", + "QueryExecutionEndTime": "datetime", + "QueryExecutionStartTime": "datetime", + "RemediationJobId": "string", + "RemediationRunbookName": "string", + "RepeatCount": "int", + "ResolvedBy": "string", + "ResourceId": "string", + "ResourceType": "string", + "ResourceValue": "string", + "RootObjectName": "string", + "ServiceDeskConnectionName": "string", + "ServiceDeskId": "string", + "ServiceDeskWorkItemLink": "string", + "ServiceDeskWorkItemType": "string", + "SourceDisplayName": "string", + "SourceFullName": "string", + "SourceSystem": "string", + "StateType": "string", + "StatusDescription": "string", + "TemplateId": "string", + "TenantId": "string", + "ThresholdOperator": "string", + "ThresholdValue": "int", + "TicketId": "string", + "TimeGenerated": "datetime", + "TimeLastModified": "datetime", + "TimeRaised": "datetime", + "TimeResolved": "datetime", + "TriggerId": "string", + "Type": "string", + "Url": "string", + "ValueDescription": "string", + "ValueFlags": "int", + "ValueFlagsDescription": "string", + "_ResourceId": "string" + }, + "AlertEvidence": { + "AccountDomain": "string", + "AccountName": "string", + "AccountObjectId": "string", + "AccountSid": "string", + "AccountUpn": "string", + "AdditionalFields": "dynamic", + "AlertId": "string", + "Application": "string", + "ApplicationId": "int", + "AttackTechniques": "string", + "Categories": "string", + "CloudPlatform": "string", + "CloudResource": "string", + "DetectionSource": "string", + "DeviceId": "string", + "DeviceName": "string", + "EmailSubject": "string", + "EntityType": "string", + "EvidenceDirection": "string", + "EvidenceRole": "string", + "FileName": "string", + "FileSize": "long", + "FolderPath": "string", + "LocalIP": "string", + "NetworkMessageId": "string", + "OAuthApplicationId": "string", + "ProcessCommandLine": "string", + "RegistryKey": "string", + "RegistryValueData": "string", + "RegistryValueName": "string", + "RemoteIP": "string", + "RemoteUrl": "string", + "SHA1": "string", + "SHA256": "string", + "ServiceSource": "string", + "Severity": "string", + "SourceSystem": "string", + "TenantId": "string", + "ThreatFamily": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Title": "string", + "Type": "string" + }, + "AlertInfo": { + "AlertId": "string", + "AttackTechniques": "string", + "Category": "string", + "DetectionSource": "string", + "ServiceSource": "string", + "Severity": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Title": "string", + "Type": "string" + }, + "Anomalies": { + "ActivityInsights": "dynamic", + "AnomalyDetails": "dynamic", + "AnomalyReasons": "dynamic", + "AnomalyTemplateId": "string", + "AnomalyTemplateName": "string", + "AnomalyTemplateVersion": "string", + "Description": "string", + "DestinationDevice": "string", + "DestinationIpAddress": "string", + "DestinationLocation": "dynamic", + "DeviceInsights": "dynamic", + "EndTime": "datetime", + "Entities": "dynamic", + "ExtendedLinks": "dynamic", + "ExtendedProperties": "dynamic", + "Id": "string", + "RuleConfigVersion": "string", + "RuleId": "string", + "RuleName": "string", + "RuleStatus": "string", + "Score": "real", + "SourceDevice": "string", + "SourceIpAddress": "string", + "SourceLocation": "dynamic", + "SourceSystem": "string", + "StartTime": "datetime", + "Tactics": "string", + "Techniques": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "UserInsights": "dynamic", + "UserName": "string", + "UserPrincipalName": "string", + "VendorName": "string", + "WorkspaceId": "string" + }, + "AuditLogs": { + "AADOperationType": "string", + "AADTenantId": "string", + "ActivityDateTime": "datetime", + "ActivityDisplayName": "string", + "AdditionalDetails": "dynamic", + "Category": "string", + "CorrelationId": "string", + "DurationMs": "long", + "Id": "string", + "Identity": "string", + "InitiatedBy": "dynamic", + "Level": "string", + "Location": "string", + "LoggedByService": "string", + "OperationName": "string", + "OperationVersion": "string", + "Resource": "string", + "ResourceGroup": "string", + "ResourceId": "string", + "ResourceProvider": "string", + "Result": "string", + "ResultDescription": "string", + "ResultReason": "string", + "ResultSignature": "string", + "ResultType": "string", + "SourceSystem": "string", + "TargetResources": "dynamic", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string" + }, + "AzureActivity": { + "ActivityStatus": "string", + "ActivityStatusValue": "string", + "ActivitySubstatus": "string", + "ActivitySubstatusValue": "string", + "Authorization": "string", + "Authorization_d": "dynamic", + "Caller": "string", + "CallerIpAddress": "string", + "Category": "string", + "CategoryValue": "string", + "Claims": "string", + "Claims_d": "dynamic", + "CorrelationId": "string", + "EventDataId": "string", + "EventSubmissionTimestamp": "datetime", + "HTTPRequest": "string", + "Hierarchy": "string", + "Level": "string", + "OperationId": "string", + "OperationName": "string", + "OperationNameValue": "string", + "Properties": "string", + "Properties_d": "dynamic", + "Resource": "string", + "ResourceGroup": "string", + "ResourceId": "string", + "ResourceProvider": "string", + "ResourceProviderValue": "string", + "SourceSystem": "string", + "SubscriptionId": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "AzureDiagnostics": { + "CallerIPAddress": "string", + "Caller_s": "string", + "Category": "string", + "CorrelationId": "string", + "DatabaseName_s": "string", + "DurationMs": "long", + "ElasticPoolName_s": "string", + "EventName_s": "string", + "JobId_g": "string", + "Level": "string", + "LogicalServerName_s": "string", + "Message": "string", + "OperationName": "string", + "OperationVersion": "string", + "Resource": "string", + "ResourceGroup": "string", + "ResourceId": "string", + "ResourceProvider": "string", + "ResourceType": "string", + "ResultDescription": "string", + "ResultSignature": "string", + "ResultType": "string", + "RunOn_s": "string", + "RunbookName_s": "string", + "SourceSystem": "string", + "StreamType_s": "string", + "SubscriptionId": "string", + "TenantId": "string", + "Tenant_g": "string", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string", + "_schema_s": "string", + "clientIP_s": "string", + "clientInfo_s": "string", + "clientPort_d": "real", + "code_s": "string", + "conditions_None_s": "string", + "conditions_destinationIP_s": "string", + "conditions_destinationPortRange_s": "string", + "conditions_protocols_s": "string", + "conditions_sourceIP_s": "string", + "conditions_sourcePortRange_s": "string", + "correlation_actionTrackingId_g": "string", + "correlation_clientTrackingId_s": "string", + "direction_s": "string", + "endTime_t": "datetime", + "host_s": "string", + "httpMethod_s": "string", + "httpStatusCode_d": "real", + "httpStatus_d": "real", + "httpVersion_s": "string", + "id_s": "string", + "identity_claim_appid_g": "string", + "identity_claim_http_schemas_microsoft_com_claims_authnmethodsreferences_s": "string", + "identity_claim_http_schemas_microsoft_com_identity_claims_objectidentifier_g": "string", + "identity_claim_http_schemas_microsoft_com_identity_claims_scope_s": "string", + "identity_claim_http_schemas_xmlsoap_org_ws_2005_05_identity_claims_upn_s": "string", + "identity_claim_ipaddr_s": "string", + "instanceId_s": "string", + "isAccessPolicyMatch_b": "bool", + "macAddress_s": "string", + "matchedConnections_d": "real", + "primaryIPv4Address_s": "string", + "priority_d": "real", + "properties_enabledForDeployment_b": "bool", + "properties_enabledForDiskEncryption_b": "bool", + "properties_enabledForTemplateDeployment_b": "bool", + "properties_sku_Family_s": "string", + "properties_sku_Name_s": "string", + "properties_tenantId_g": "string", + "query_hash_s": "string", + "receivedBytes_d": "real", + "requestQuery_s": "string", + "requestUri_s": "string", + "resource_actionName_s": "string", + "resource_location_s": "string", + "resource_originRunId_s": "string", + "resource_resourceGroupName_s": "string", + "resource_runId_s": "string", + "resource_subscriptionId_g": "string", + "resource_triggerName_s": "string", + "resource_workflowId_g": "string", + "resource_workflowName_s": "string", + "resultDescription_ChildJobs_s": "string", + "resultDescription_ErrorJobs_s": "string", + "resultDescription_Summary_DurationInMinutes_d": "real", + "resultDescription_Summary_DurationInMinutes_s": "string", + "resultDescription_Summary_EndDateTimeUtc_s": "string", + "resultDescription_Summary_EndDateTimeUtc_t": "datetime", + "resultDescription_Summary_InitialRequiredUpdatesCount_d": "real", + "resultDescription_Summary_InstallPercentage_d": "real", + "resultDescription_Summary_MachineId_s": "string", + "resultDescription_Summary_MachineName_s": "string", + "resultDescription_Summary_RebootRequired_b": "bool", + "resultDescription_Summary_ScheduleName_s": "string", + "resultDescription_Summary_StartDateTimeUtc_t": "datetime", + "resultDescription_Summary_StatusDescription_s": "string", + "resultDescription_Summary_Status_s": "string", + "resultDescription_Summary_TotalUpdatesFailed_d": "real", + "resultDescription_Summary_TotalUpdatesInstalled_d": "real", + "ruleName_s": "string", + "sentBytes_d": "real", + "sslEnabled_s": "string", + "startTime_t": "datetime", + "status_s": "string", + "subnetId_s": "string", + "subnetPrefix_s": "string", + "systemId_g": "string", + "timeTaken_d": "real", + "type_s": "string", + "userAgent_s": "string", + "vnetResourceGuid_g": "string", + "workflowId_s": "string" + }, + "AzureMetrics": { + "AdditionalInformation": "string", + "Average": "real", + "CallerIpAddress": "string", + "Category": "string", + "Confidence": "string", + "CorrelationId": "string", + "Count": "real", + "Description": "string", + "DurationMs": "long", + "FirstReportedDateTime": "string", + "IndicatorThreatType": "string", + "IsActive": "string", + "LastReportedDateTime": "string", + "MaliciousIP": "string", + "Maximum": "real", + "MetricName": "string", + "Minimum": "real", + "OperationName": "string", + "OperationVersion": "string", + "RemoteIPCountry": "string", + "RemoteIPLatitude": "real", + "RemoteIPLongitude": "real", + "ReportReferenceLink": "string", + "Resource": "string", + "ResourceGroup": "string", + "ResourceId": "string", + "ResourceProvider": "string", + "ResultDescription": "string", + "ResultSignature": "string", + "ResultType": "string", + "Severity": "int", + "SourceSystem": "string", + "SubscriptionId": "string", + "TLPLevel": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TimeGrain": "string", + "Total": "real", + "Type": "string", + "UnitName": "string", + "_ResourceId": "string" + }, + "CloudAppEvents": { + "AccountDisplayName": "string", + "AccountId": "string", + "AccountObjectId": "string", + "AccountType": "string", + "ActionType": "string", + "ActivityObjects": "dynamic", + "ActivityType": "string", + "AdditionalFields": "dynamic", + "AppInstanceId": "int", + "Application": "string", + "ApplicationId": "int", + "AuditSource": "string", + "City": "string", + "CountryCode": "string", + "DeviceType": "string", + "IPAddress": "string", + "IPCategory": "string", + "IPTags": "dynamic", + "ISP": "string", + "IsAdminOperation": "bool", + "IsAnonymousProxy": "bool", + "IsExternalUser": "bool", + "IsImpersonated": "bool", + "LastSeenForUser": "dynamic", + "OAuthAppId": "string", + "OSPlatform": "string", + "ObjectId": "string", + "ObjectName": "string", + "ObjectType": "string", + "RawEventData": "dynamic", + "ReportId": "string", + "SessionData": "dynamic", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string", + "UncommonForUser": "dynamic", + "UserAgent": "string", + "UserAgentTags": "dynamic" + }, + "CommonSecurityLog": { + "Activity": "string", + "AdditionalExtensions": "string", + "ApplicationProtocol": "string", + "CollectorHostName": "string", + "CommunicationDirection": "string", + "Computer": "string", + "DestinationDnsDomain": "string", + "DestinationHostName": "string", + "DestinationIP": "string", + "DestinationMACAddress": "string", + "DestinationNTDomain": "string", + "DestinationPort": "int", + "DestinationProcessId": "int", + "DestinationProcessName": "string", + "DestinationServiceName": "string", + "DestinationTranslatedAddress": "string", + "DestinationTranslatedPort": "int", + "DestinationUserID": "string", + "DestinationUserName": "string", + "DestinationUserPrivileges": "string", + "DeviceAction": "string", + "DeviceAddress": "string", + "DeviceCustomDate1": "string", + "DeviceCustomDate1Label": "string", + "DeviceCustomDate2": "string", + "DeviceCustomDate2Label": "string", + "DeviceCustomFloatingPoint1": "real", + "DeviceCustomFloatingPoint1Label": "string", + "DeviceCustomFloatingPoint2": "real", + "DeviceCustomFloatingPoint2Label": "string", + "DeviceCustomFloatingPoint3": "real", + "DeviceCustomFloatingPoint3Label": "string", + "DeviceCustomFloatingPoint4": "real", + "DeviceCustomFloatingPoint4Label": "string", + "DeviceCustomIPv6Address1": "string", + "DeviceCustomIPv6Address1Label": "string", + "DeviceCustomIPv6Address2": "string", + "DeviceCustomIPv6Address2Label": "string", + "DeviceCustomIPv6Address3": "string", + "DeviceCustomIPv6Address3Label": "string", + "DeviceCustomIPv6Address4": "string", + "DeviceCustomIPv6Address4Label": "string", + "DeviceCustomNumber1": "int", + "DeviceCustomNumber1Label": "string", + "DeviceCustomNumber2": "int", + "DeviceCustomNumber2Label": "string", + "DeviceCustomNumber3": "int", + "DeviceCustomNumber3Label": "string", + "DeviceCustomString1": "string", + "DeviceCustomString1Label": "string", + "DeviceCustomString2": "string", + "DeviceCustomString2Label": "string", + "DeviceCustomString3": "string", + "DeviceCustomString3Label": "string", + "DeviceCustomString4": "string", + "DeviceCustomString4Label": "string", + "DeviceCustomString5": "string", + "DeviceCustomString5Label": "string", + "DeviceCustomString6": "string", + "DeviceCustomString6Label": "string", + "DeviceDnsDomain": "string", + "DeviceEventCategory": "string", + "DeviceEventClassID": "string", + "DeviceExternalID": "string", + "DeviceFacility": "string", + "DeviceInboundInterface": "string", + "DeviceMacAddress": "string", + "DeviceName": "string", + "DeviceNtDomain": "string", + "DeviceOutboundInterface": "string", + "DevicePayloadId": "string", + "DeviceProduct": "string", + "DeviceTimeZone": "string", + "DeviceTranslatedAddress": "string", + "DeviceVendor": "string", + "DeviceVersion": "string", + "EndTime": "datetime", + "EventCount": "int", + "EventOutcome": "string", + "EventType": "int", + "ExtID": "string", + "ExternalID": "int", + "FieldDeviceCustomNumber1": "long", + "FieldDeviceCustomNumber2": "long", + "FieldDeviceCustomNumber3": "long", + "FileCreateTime": "string", + "FileHash": "string", + "FileID": "string", + "FileModificationTime": "string", + "FileName": "string", + "FilePath": "string", + "FilePermission": "string", + "FileSize": "int", + "FileType": "string", + "FlexDate1": "string", + "FlexDate1Label": "string", + "FlexNumber1": "int", + "FlexNumber1Label": "string", + "FlexNumber2": "int", + "FlexNumber2Label": "string", + "FlexString1": "string", + "FlexString1Label": "string", + "FlexString2": "string", + "FlexString2Label": "string", + "IndicatorThreatType": "string", + "LogSeverity": "string", + "MaliciousIP": "string", + "MaliciousIPCountry": "string", + "MaliciousIPLatitude": "real", + "MaliciousIPLongitude": "real", + "Message": "string", + "OldFileCreateTime": "string", + "OldFileHash": "string", + "OldFileID": "string", + "OldFileModificationTime": "string", + "OldFileName": "string", + "OldFilePath": "string", + "OldFilePermission": "string", + "OldFileSize": "int", + "OldFileType": "string", + "OriginalLogSeverity": "string", + "ProcessID": "int", + "ProcessName": "string", + "Protocol": "string", + "Reason": "string", + "ReceiptTime": "string", + "ReceivedBytes": "long", + "RemoteIP": "string", + "RemotePort": "string", + "ReportReferenceLink": "string", + "RequestClientApplication": "string", + "RequestContext": "string", + "RequestCookies": "string", + "RequestMethod": "string", + "RequestURL": "string", + "SentBytes": "long", + "SimplifiedDeviceAction": "string", + "SourceDnsDomain": "string", + "SourceHostName": "string", + "SourceIP": "string", + "SourceMACAddress": "string", + "SourceNTDomain": "string", + "SourcePort": "int", + "SourceProcessId": "int", + "SourceProcessName": "string", + "SourceServiceName": "string", + "SourceSystem": "string", + "SourceTranslatedAddress": "string", + "SourceTranslatedPort": "int", + "SourceUserID": "string", + "SourceUserName": "string", + "SourceUserPrivileges": "string", + "StartTime": "datetime", + "TenantId": "string", + "ThreatConfidence": "string", + "ThreatDescription": "string", + "ThreatSeverity": "int", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "ConfidentialWatchlist": { + "AzureTenantId": "string", + "CorrelationId": "string", + "CreatedBy": "dynamic", + "CreatedTimeUTC": "datetime", + "DefaultDuration": "string", + "EntityMapping": "dynamic", + "LastUpdatedTimeUTC": "datetime", + "Notes": "string", + "Provider": "string", + "SearchKey": "string", + "Source": "string", + "SourceSystem": "string", + "Tags": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TimeToLive": "datetime", + "Type": "string", + "UpdatedBy": "dynamic", + "WatchlistAlias": "string", + "WatchlistCategory": "string", + "WatchlistId": "string", + "WatchlistItem": "dynamic", + "WatchlistItemId": "string", + "WatchlistName": "string", + "_DTItemId": "string", + "_DTItemStatus": "string", + "_DTItemType": "string", + "_DTTimestamp": "datetime" + }, + "ContainerInventory": { + "Command": "string", + "ComposeGroup": "string", + "Computer": "string", + "ContainerHostname": "string", + "ContainerID": "string", + "ContainerState": "string", + "CreatedTime": "datetime", + "EnvironmentVar": "string", + "ExitCode": "int", + "FinishedTime": "datetime", + "Image": "string", + "ImageID": "string", + "ImageTag": "string", + "Links": "string", + "Name": "string", + "Ports": "string", + "Repository": "string", + "SourceSystem": "string", + "StartedTime": "datetime", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "DeviceBehaviorEntities": { + "AccountDomain": "string", + "AccountName": "string", + "AccountObjectId": "string", + "AccountSid": "string", + "AccountUpn": "string", + "ActionType": "string", + "AdditionalFields": "string", + "Application": "string", + "ApplicationId": "string", + "BehaviorId": "string", + "Categories": "string", + "DataSources": "string", + "DetailedEntityRole": "string", + "DetectionSource": "string", + "DeviceId": "string", + "DeviceName": "string", + "EmailClusterId": "string", + "EmailSubject": "string", + "EntityRole": "string", + "EntityType": "string", + "FileName": "string", + "FileSize": "long", + "FolderPath": "string", + "LocalIP": "string", + "NetworkMessageId": "string", + "OAuthApplicationId": "string", + "ProcessCommandLine": "string", + "RegistryKey": "string", + "RegistryValueData": "string", + "RegistryValueName": "string", + "RemoteIP": "string", + "RemoteUrl": "string", + "SHA1": "string", + "SHA256": "string", + "ServiceSource": "string", + "SourceSystem": "string", + "TenantId": "string", + "ThreatFamily": "string", + "TimeGenerated": "datetime", + "Type": "string" + }, + "DeviceBehaviorInfo": { + "AccountObjectId": "string", + "AccountUpn": "string", + "ActionType": "string", + "AdditionalFields": "string", + "AttackTechniques": "string", + "BehaviorId": "string", + "Categories": "string", + "DataSources": "string", + "Description": "string", + "DetectionSource": "string", + "DeviceId": "string", + "EndTime": "datetime", + "ServiceSource": "string", + "SourceSystem": "string", + "StartTime": "datetime", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string" + }, + "DeviceCustomFileEvents": { + "ActionType": "string", + "AdditionalFields": "dynamic", + "AppGuardContainerId": "string", + "DeviceId": "string", + "DeviceName": "string", + "FileName": "string", + "FileOriginIP": "string", + "FileOriginReferrerUrl": "string", + "FileOriginUrl": "string", + "FileSize": "long", + "FolderPath": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessIntegrityLevel": "string", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessTokenElevation": "string", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsAzureInfoProtectionApplied": "bool", + "IsInitiatingProcessRemoteSession": "bool", + "MD5": "string", + "MachineGroup": "string", + "PreviousFileName": "string", + "PreviousFolderPath": "string", + "ReportId": "long", + "RequestAccountDomain": "string", + "RequestAccountName": "string", + "RequestAccountSid": "string", + "RequestProtocol": "string", + "RequestSourceIP": "string", + "RequestSourcePort": "int", + "RuleLastModificationTime": "datetime", + "RuleName": "string", + "SHA1": "string", + "SHA256": "string", + "SensitivityLabel": "string", + "SensitivitySubLabel": "string", + "ShareName": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceCustomImageLoadEvents": { + "ActionType": "string", + "AppGuardContainerId": "string", + "DeviceId": "string", + "DeviceName": "string", + "FileName": "string", + "FileSize": "long", + "FolderPath": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessIntegrityLevel": "string", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessTokenElevation": "string", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsInitiatingProcessRemoteSession": "bool", + "MD5": "string", + "MachineGroup": "string", + "ReportId": "long", + "RuleLastModificationTime": "datetime", + "RuleName": "string", + "SHA1": "string", + "SHA256": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceCustomNetworkEvents": { + "ActionType": "string", + "AdditionalFields": "dynamic", + "AppGuardContainerId": "string", + "DeviceId": "string", + "DeviceName": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessIntegrityLevel": "string", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessTokenElevation": "string", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsInitiatingProcessRemoteSession": "bool", + "LocalIP": "string", + "LocalIPType": "string", + "LocalPort": "int", + "MachineGroup": "string", + "Protocol": "string", + "RemoteIP": "string", + "RemoteIPType": "string", + "RemotePort": "int", + "RemoteUrl": "string", + "ReportId": "long", + "RuleLastModificationTime": "datetime", + "RuleName": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceCustomProcessEvents": { + "AccountDomain": "string", + "AccountName": "string", + "AccountObjectId": "string", + "AccountSid": "string", + "AccountUpn": "string", + "ActionType": "string", + "AdditionalFields": "dynamic", + "AppGuardContainerId": "string", + "CreatedProcessSessionId": "long", + "DeviceId": "string", + "DeviceName": "string", + "FileName": "string", + "FileSize": "long", + "FolderPath": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessIntegrityLevel": "string", + "InitiatingProcessLogonId": "long", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessSignatureStatus": "string", + "InitiatingProcessSignerType": "string", + "InitiatingProcessTokenElevation": "string", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsInitiatingProcessRemoteSession": "bool", + "IsProcessRemoteSession": "bool", + "LogonId": "long", + "MD5": "string", + "MachineGroup": "string", + "ProcessCommandLine": "string", + "ProcessCreationTime": "datetime", + "ProcessId": "long", + "ProcessIntegrityLevel": "string", + "ProcessRemoteSessionDeviceName": "string", + "ProcessRemoteSessionIP": "string", + "ProcessTokenElevation": "string", + "ProcessUniqueId": "string", + "ProcessVersionInfoCompanyName": "string", + "ProcessVersionInfoFileDescription": "string", + "ProcessVersionInfoInternalFileName": "string", + "ProcessVersionInfoOriginalFileName": "string", + "ProcessVersionInfoProductName": "string", + "ProcessVersionInfoProductVersion": "string", + "ReportId": "long", + "RuleLastModificationTime": "datetime", + "RuleName": "string", + "SHA1": "string", + "SHA256": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceCustomRegistryEvents": { + "ActionType": "string", + "AppGuardContainerId": "string", + "DeviceId": "string", + "DeviceName": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessIntegrityLevel": "string", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessTokenElevation": "string", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsInitiatingProcessRemoteSession": "bool", + "MachineGroup": "string", + "PreviousRegistryKey": "string", + "PreviousRegistryValueData": "string", + "PreviousRegistryValueName": "string", + "RegistryKey": "string", + "RegistryValueData": "string", + "RegistryValueName": "string", + "RegistryValueType": "string", + "ReportId": "long", + "RuleLastModificationTime": "datetime", + "RuleName": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceCustomScriptEvents": { + "ActionType": "string", + "DeviceId": "string", + "DeviceName": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessSignatureStatus": "string", + "InitiatingProcessSignerType": "string", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsInitiatingProcessRemoteSession": "bool", + "ReportId": "long", + "RuleLastModificationTime": "datetime", + "RuleName": "string", + "ScriptContent": "string", + "ScriptContentSHA256": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceEvents": { + "AccountDomain": "string", + "AccountName": "string", + "AccountSid": "string", + "ActionType": "string", + "AdditionalFields": "dynamic", + "AppGuardContainerId": "string", + "CreatedProcessSessionId": "long", + "DeviceId": "string", + "DeviceName": "string", + "FileName": "string", + "FileOriginIP": "string", + "FileOriginUrl": "string", + "FileSize": "long", + "FolderPath": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessLogonId": "long", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsInitiatingProcessRemoteSession": "bool", + "IsProcessRemoteSession": "bool", + "LocalIP": "string", + "LocalPort": "int", + "LogonId": "long", + "MD5": "string", + "MachineGroup": "string", + "ProcessCommandLine": "string", + "ProcessCreationTime": "datetime", + "ProcessId": "long", + "ProcessRemoteSessionDeviceName": "string", + "ProcessRemoteSessionIP": "string", + "ProcessTokenElevation": "string", + "RegistryKey": "string", + "RegistryValueData": "string", + "RegistryValueName": "string", + "RemoteDeviceName": "string", + "RemoteIP": "string", + "RemotePort": "int", + "RemoteUrl": "string", + "ReportId": "long", + "SHA1": "string", + "SHA256": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceFileCertificateInfo": { + "CertificateCountersignatureTime": "datetime", + "CertificateCreationTime": "datetime", + "CertificateExpirationTime": "datetime", + "CertificateSerialNumber": "string", + "CrlDistributionPointUrls": "string", + "DeviceId": "string", + "DeviceName": "string", + "IsRootSignerMicrosoft": "bool", + "IsSigned": "bool", + "IsTrusted": "bool", + "Issuer": "string", + "IssuerHash": "string", + "MachineGroup": "string", + "ReportId": "long", + "SHA1": "string", + "SignatureType": "string", + "Signer": "string", + "SignerHash": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceFileEvents": { + "ActionType": "string", + "AdditionalFields": "dynamic", + "AppGuardContainerId": "string", + "DeviceId": "string", + "DeviceName": "string", + "FileName": "string", + "FileOriginIP": "string", + "FileOriginReferrerUrl": "string", + "FileOriginUrl": "string", + "FileSize": "long", + "FolderPath": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessIntegrityLevel": "string", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessTokenElevation": "string", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsAzureInfoProtectionApplied": "bool", + "IsInitiatingProcessRemoteSession": "bool", + "MD5": "string", + "MachineGroup": "string", + "PreviousFileName": "string", + "PreviousFolderPath": "string", + "ReportId": "long", + "RequestAccountDomain": "string", + "RequestAccountName": "string", + "RequestAccountSid": "string", + "RequestProtocol": "string", + "RequestSourceIP": "string", + "RequestSourcePort": "int", + "SHA1": "string", + "SHA256": "string", + "SensitivityLabel": "string", + "SensitivitySubLabel": "string", + "ShareName": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceImageLoadEvents": { + "ActionType": "string", + "AppGuardContainerId": "string", + "DeviceId": "string", + "DeviceName": "string", + "FileName": "string", + "FileSize": "long", + "FolderPath": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessIntegrityLevel": "string", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessTokenElevation": "string", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsInitiatingProcessRemoteSession": "bool", + "MD5": "string", + "MachineGroup": "string", + "ReportId": "long", + "SHA1": "string", + "SHA256": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceInfo": { + "AadDeviceId": "string", + "AdditionalFields": "dynamic", + "AssetValue": "string", + "AwsResourceName": "string", + "AzureResourceId": "string", + "AzureVmId": "string", + "AzureVmSubscriptionId": "string", + "ClientVersion": "string", + "CloudPlatforms": "string", + "DeviceCategory": "string", + "DeviceDynamicTags": "string", + "DeviceId": "string", + "DeviceManualTags": "string", + "DeviceName": "string", + "DeviceObjectId": "string", + "DeviceSubtype": "string", + "DeviceType": "string", + "ExclusionReason": "string", + "ExposureLevel": "string", + "GcpFullResourceName": "string", + "HardwareUuid": "string", + "HostDeviceId": "string", + "IsAzureADJoined": "bool", + "IsExcluded": "bool", + "IsInternetFacing": "bool", + "IsTransient": "bool", + "JoinType": "string", + "LoggedOnUsers": "dynamic", + "MachineGroup": "string", + "MergedDeviceIds": "string", + "MergedToDeviceId": "string", + "MitigationStatus": "string", + "Model": "string", + "OSArchitecture": "string", + "OSBuild": "long", + "OSDistribution": "string", + "OSPlatform": "string", + "OSVersion": "string", + "OSVersionInfo": "string", + "OnboardingStatus": "string", + "OsBuildRevision": "string", + "PublicIP": "string", + "RegistryDeviceTag": "string", + "ReportId": "long", + "RestrictedDeviceSecurityOperations": "string", + "SensorHealthState": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string", + "Vendor": "string" + }, + "DeviceLogonEvents": { + "AccountDomain": "string", + "AccountName": "string", + "AccountSid": "string", + "ActionType": "string", + "AdditionalFields": "dynamic", + "AppGuardContainerId": "string", + "DeviceId": "string", + "DeviceName": "string", + "FailureReason": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessIntegrityLevel": "string", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessTokenElevation": "string", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsInitiatingProcessRemoteSession": "bool", + "IsLocalAdmin": "bool", + "LogonId": "long", + "LogonType": "string", + "MachineGroup": "string", + "Protocol": "string", + "RemoteDeviceName": "string", + "RemoteIP": "string", + "RemoteIPType": "string", + "RemotePort": "int", + "ReportId": "long", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceNetworkEvents": { + "ActionType": "string", + "AdditionalFields": "dynamic", + "AppGuardContainerId": "string", + "DeviceId": "string", + "DeviceName": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessIntegrityLevel": "string", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessTokenElevation": "string", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsInitiatingProcessRemoteSession": "bool", + "LocalIP": "string", + "LocalIPType": "string", + "LocalPort": "int", + "MachineGroup": "string", + "Protocol": "string", + "RemoteIP": "string", + "RemoteIPType": "string", + "RemotePort": "int", + "RemoteUrl": "string", + "ReportId": "long", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceNetworkInfo": { + "ConnectedNetworks": "dynamic", + "DefaultGateways": "dynamic", + "DeviceId": "string", + "DeviceName": "string", + "DnsAddresses": "dynamic", + "IPAddresses": "dynamic", + "IPv4Dhcp": "string", + "IPv6Dhcp": "string", + "MacAddress": "string", + "MachineGroup": "string", + "NetworkAdapterName": "string", + "NetworkAdapterStatus": "string", + "NetworkAdapterType": "string", + "NetworkAdapterVendor": "string", + "ReportId": "long", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "TunnelType": "string", + "Type": "string" + }, + "DeviceProcessEvents": { + "AccountDomain": "string", + "AccountName": "string", + "AccountObjectId": "string", + "AccountSid": "string", + "AccountUpn": "string", + "ActionType": "string", + "AdditionalFields": "dynamic", + "AppGuardContainerId": "string", + "CreatedProcessSessionId": "long", + "DeviceId": "string", + "DeviceName": "string", + "FileName": "string", + "FileSize": "long", + "FolderPath": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessIntegrityLevel": "string", + "InitiatingProcessLogonId": "long", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessSignatureStatus": "string", + "InitiatingProcessSignerType": "string", + "InitiatingProcessTokenElevation": "string", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsInitiatingProcessRemoteSession": "bool", + "IsProcessRemoteSession": "bool", + "LogonId": "long", + "MD5": "string", + "MachineGroup": "string", + "ProcessCommandLine": "string", + "ProcessCreationTime": "datetime", + "ProcessId": "long", + "ProcessIntegrityLevel": "string", + "ProcessRemoteSessionDeviceName": "string", + "ProcessRemoteSessionIP": "string", + "ProcessTokenElevation": "string", + "ProcessUniqueId": "string", + "ProcessVersionInfoCompanyName": "string", + "ProcessVersionInfoFileDescription": "string", + "ProcessVersionInfoInternalFileName": "string", + "ProcessVersionInfoOriginalFileName": "string", + "ProcessVersionInfoProductName": "string", + "ProcessVersionInfoProductVersion": "string", + "ReportId": "long", + "SHA1": "string", + "SHA256": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceRegistryEvents": { + "ActionType": "string", + "AppGuardContainerId": "string", + "DeviceId": "string", + "DeviceName": "string", + "InitiatingProcessAccountDomain": "string", + "InitiatingProcessAccountName": "string", + "InitiatingProcessAccountObjectId": "string", + "InitiatingProcessAccountSid": "string", + "InitiatingProcessAccountUpn": "string", + "InitiatingProcessCommandLine": "string", + "InitiatingProcessCreationTime": "datetime", + "InitiatingProcessFileName": "string", + "InitiatingProcessFileSize": "long", + "InitiatingProcessFolderPath": "string", + "InitiatingProcessId": "long", + "InitiatingProcessIntegrityLevel": "string", + "InitiatingProcessMD5": "string", + "InitiatingProcessParentCreationTime": "datetime", + "InitiatingProcessParentFileName": "string", + "InitiatingProcessParentId": "long", + "InitiatingProcessRemoteSessionDeviceName": "string", + "InitiatingProcessRemoteSessionIP": "string", + "InitiatingProcessSHA1": "string", + "InitiatingProcessSHA256": "string", + "InitiatingProcessSessionId": "long", + "InitiatingProcessTokenElevation": "string", + "InitiatingProcessUniqueId": "string", + "InitiatingProcessVersionInfoCompanyName": "string", + "InitiatingProcessVersionInfoFileDescription": "string", + "InitiatingProcessVersionInfoInternalFileName": "string", + "InitiatingProcessVersionInfoOriginalFileName": "string", + "InitiatingProcessVersionInfoProductName": "string", + "InitiatingProcessVersionInfoProductVersion": "string", + "IsInitiatingProcessRemoteSession": "bool", + "MachineGroup": "string", + "PreviousRegistryKey": "string", + "PreviousRegistryValueData": "string", + "PreviousRegistryValueName": "string", + "RegistryKey": "string", + "RegistryValueData": "string", + "RegistryValueName": "string", + "RegistryValueType": "string", + "ReportId": "long", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceTvmSecureConfigurationAssessment": { + "ConfigurationCategory": "string", + "ConfigurationId": "string", + "ConfigurationImpact": "real", + "ConfigurationSubcategory": "string", + "Context": "dynamic", + "DeviceId": "string", + "DeviceName": "string", + "IsApplicable": "bool", + "IsCompliant": "bool", + "IsExpectedUserImpact": "bool", + "OSPlatform": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceTvmSecureConfigurationAssessmentKB": { + "ConfigurationBenchmarks": "dynamic", + "ConfigurationCategory": "string", + "ConfigurationDescription": "string", + "ConfigurationId": "string", + "ConfigurationImpact": "real", + "ConfigurationName": "string", + "ConfigurationSubcategory": "string", + "RelatedMitreTactics": "dynamic", + "RelatedMitreTechniques": "dynamic", + "RemediationOptions": "string", + "RiskDescription": "string", + "SourceSystem": "string", + "Tags": "dynamic", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "DeviceTvmSoftwareInventory": { + "DeviceId": "string", + "DeviceName": "string", + "EndOfSupportDate": "datetime", + "EndOfSupportStatus": "string", + "OSArchitecture": "string", + "OSPlatform": "string", + "OSVersion": "string", + "ProductCodeCpe": "string", + "SoftwareName": "string", + "SoftwareVendor": "string", + "SoftwareVersion": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string" + }, + "DeviceTvmSoftwareVulnerabilities": { + "CveId": "string", + "CveTags": "dynamic", + "DeviceId": "string", + "DeviceName": "string", + "OSArchitecture": "string", + "OSPlatform": "string", + "OSVersion": "string", + "RecommendedSecurityUpdate": "string", + "RecommendedSecurityUpdateId": "string", + "SoftwareName": "string", + "SoftwareVendor": "string", + "SoftwareVersion": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "VulnerabilitySeverityLevel": "string" + }, + "DeviceTvmSoftwareVulnerabilitiesKB": { + "AffectedSoftware": "dynamic", + "CveId": "string", + "CvssScore": "real", + "IsExploitAvailable": "bool", + "LastModifiedTime": "datetime", + "PublishedDate": "datetime", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string", + "VulnerabilityDescription": "string", + "VulnerabilitySeverityLevel": "string" + }, + "DnsEvents": { + "ClientIP": "string", + "Computer": "string", + "Confidence": "string", + "Description": "string", + "EventId": "int", + "IPAddresses": "string", + "IndicatorThreatType": "string", + "MaliciousIP": "string", + "Message": "string", + "Name": "string", + "QueryType": "string", + "RemoteIPCountry": "string", + "RemoteIPLatitude": "real", + "RemoteIPLongitude": "real", + "ReportReferenceLink": "string", + "Result": "string", + "ResultCode": "int", + "Severity": "int", + "SourceSystem": "string", + "SubType": "string", + "TaskCategory": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "EmailAttachmentInfo": { + "DetectionMethods": "string", + "FileExtension": "string", + "FileName": "string", + "FileSize": "long", + "FileType": "string", + "NetworkMessageId": "string", + "RecipientEmailAddress": "string", + "RecipientObjectId": "string", + "ReportId": "string", + "SHA256": "string", + "SenderDisplayName": "string", + "SenderFromAddress": "string", + "SenderObjectId": "string", + "SourceSystem": "string", + "TenantId": "string", + "ThreatNames": "string", + "ThreatTypes": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "EmailEvents": { + "AdditionalFields": "dynamic", + "AttachmentCount": "int", + "AuthenticationDetails": "string", + "BulkComplaintLevel": "int", + "Cc": "dynamic", + "ConfidenceLevel": "string", + "Connectors": "string", + "Context": "string", + "DeliveryAction": "string", + "DeliveryLocation": "string", + "DetectionMethods": "string", + "DistributionList": "string", + "EmailAction": "string", + "EmailActionPolicy": "string", + "EmailActionPolicyGuid": "string", + "EmailClusterId": "long", + "EmailDirection": "string", + "EmailLanguage": "string", + "EmailSize": "int", + "ExchangeTransportRule": "string", + "ForwardingInformation": "string", + "InternetMessageId": "string", + "IsFirstContact": "bool", + "LastEventExecutionTime": "datetime", + "LatestDeliveryAction": "string", + "LatestDeliveryLocation": "string", + "NetworkMessageId": "string", + "OrgLevelAction": "string", + "OrgLevelPolicy": "string", + "RecipientDomain": "string", + "RecipientEmailAddress": "string", + "RecipientObjectId": "string", + "ReportId": "string", + "SenderDisplayName": "string", + "SenderFromAddress": "string", + "SenderFromDomain": "string", + "SenderIPv4": "string", + "SenderIPv6": "string", + "SenderMailFromAddress": "string", + "SenderMailFromDomain": "string", + "SenderObjectId": "string", + "SourceSystem": "string", + "Subject": "string", + "TenantId": "string", + "ThreatClassification": "string", + "ThreatNames": "string", + "ThreatTypes": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "To": "dynamic", + "Type": "string", + "UrlCount": "int", + "UserLevelAction": "string", + "UserLevelPolicy": "string" + }, + "EmailPostDeliveryEvents": { + "Action": "string", + "ActionResult": "string", + "ActionTrigger": "string", + "ActionType": "string", + "DeliveryLocation": "string", + "DetectionMethods": "string", + "EmailDirection": "string", + "InternetMessageId": "string", + "NetworkMessageId": "string", + "RecipientEmailAddress": "string", + "ReportId": "string", + "SenderFromAddress": "string", + "SourceSystem": "string", + "TenantId": "string", + "ThreatTypes": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "EmailUrlInfo": { + "NetworkMessageId": "string", + "ReportId": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string", + "Url": "string", + "UrlDomain": "string", + "UrlLocation": "string" + }, + "Event": { + "AzureDeploymentID": "string", + "Computer": "string", + "EventCategory": "int", + "EventData": "string", + "EventID": "int", + "EventLevel": "int", + "EventLevelName": "string", + "EventLog": "string", + "MG": "string", + "ManagementGroupName": "string", + "Message": "string", + "ParameterXml": "string", + "RenderedDescription": "string", + "Role": "string", + "Source": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "UserName": "string", + "_ResourceId": "string" + }, + "FileMaliciousContentInfo": { + "DetectionMethods": "string", + "DocumentID": "string", + "FileCreationTime": "datetime", + "FileName": "string", + "FileOwnerDisplayName": "string", + "FileOwnerUpn": "string", + "FileSize": "long", + "FolderPath": "string", + "LastModifiedTime": "datetime", + "LastModifyingAccountUpn": "string", + "ReportId": "string", + "SHA256": "string", + "SourceSystem": "string", + "TenantId": "string", + "ThreatNames": "string", + "ThreatTypes": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string", + "Workload": "string" + }, + "GCPDNS": { + "AuthAnswer": "bool", + "AuthenticationInfoPrincipalEmail": "string", + "AuthenticationInfoPrincipalSubject": "string", + "AuthorizationInfo": "string", + "DestinationIP": "string", + "GCPResourceName": "string", + "GCPResourceType": "string", + "InsertId": "string", + "LogName": "string", + "MethodName": "string", + "PayloadType": "string", + "Protocol": "string", + "QueryName": "string", + "QueryType": "string", + "Rdata": "string", + "ReqManZoneCloudLogConfigEnableLogging": "bool", + "ReqManZonePrivVisibConfigNetworks": "string", + "ReqmetaCallerSuppliedUserAgent": "string", + "ReqmetaRequestAttributesTime": "datetime", + "RequestAPIType": "string", + "RequestChangeAdditions": "string", + "RequestChangeDeletions": "string", + "RequestManagedZone": "string", + "RequestManagedZoneDescription": "string", + "RequestManagedZoneDnsName": "string", + "RequestManagedZoneName": "string", + "RequestManagedZoneVisibility": "string", + "RequestMetadataCallerIp": "string", + "RequestName": "string", + "RequestProject": "string", + "RequestType": "string", + "ResManZoneCloudLogConfigEnableLogging": "bool", + "ResourceLabelsLocation": "string", + "ResourceLabelsPolicyName": "string", + "ResourceLabelsProjectId": "string", + "ResourceLabelsSourceType": "string", + "ResourceLabelsTargetName": "string", + "ResourceLabelsTargetType": "string", + "ResourceLabelsZoneName": "string", + "RespManZonePrivVisibConfigNetworks": "string", + "ResponseChangeAdditions": "string", + "ResponseChangeDeletions": "string", + "ResponseChangeId": "string", + "ResponseChangeStartTime": "datetime", + "ResponseChangeStatus": "string", + "ResponseCode": "string", + "ResponseManagedZoneCreationTime": "datetime", + "ResponseManagedZoneDescription": "string", + "ResponseManagedZoneDnsName": "string", + "ResponseManagedZoneFingerprint": "string", + "ResponseManagedZoneId": "string", + "ResponseManagedZoneName": "string", + "ResponseManagedZoneNameServers": "string", + "ResponseManagedZoneRrsetCount": "string", + "ResponseManagedZoneVisibility": "string", + "ResponseOpZoneContextNewValue": "string", + "ResponseOpZoneContextOldValue": "string", + "ResponseOperationId": "string", + "ResponseOperationStartTime": "datetime", + "ResponseOperationStatus": "string", + "ResponseOperationType": "string", + "ResponseOperationUser": "string", + "ResponseType": "string", + "ServerLatency": "string", + "ServiceName": "string", + "Severity": "string", + "SourceIP": "string", + "SourceNetwork": "string", + "SourceSystem": "string", + "StatusCode": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string", + "VmInstanceId": "string", + "VmInstanceIdString": "string", + "VmInstanceName": "string", + "VmProjectId": "string", + "VmZoneName": "string" + }, + "Heartbeat": { + "Category": "string", + "Computer": "string", + "ComputerEnvironment": "string", + "ComputerIP": "string", + "ComputerPrivateIPs": "dynamic", + "IsGatewayInstalled": "bool", + "MG": "string", + "ManagementGroupName": "string", + "OSMajorVersion": "string", + "OSMinorVersion": "string", + "OSName": "string", + "OSType": "string", + "RemoteIPCountry": "string", + "RemoteIPLatitude": "real", + "RemoteIPLongitude": "real", + "Resource": "string", + "ResourceGroup": "string", + "ResourceId": "string", + "ResourceProvider": "string", + "ResourceType": "string", + "SCAgentChannel": "string", + "Solutions": "string", + "SourceComputerId": "string", + "SourceSystem": "string", + "SubscriptionId": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "VMUUID": "string", + "Version": "string", + "_ResourceId": "string" + }, + "IdentityDirectoryEvents": { + "AccountDisplayName": "string", + "AccountDomain": "string", + "AccountName": "string", + "AccountObjectId": "string", + "AccountSid": "string", + "AccountUpn": "string", + "ActionType": "string", + "AdditionalFields": "dynamic", + "Application": "string", + "DestinationDeviceName": "string", + "DestinationIPAddress": "string", + "DestinationPort": "string", + "DeviceName": "string", + "IPAddress": "string", + "ISP": "string", + "Location": "string", + "Port": "string", + "Protocol": "string", + "ReportId": "string", + "SourceSystem": "string", + "TargetAccountDisplayName": "string", + "TargetAccountUpn": "string", + "TargetDeviceName": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "IdentityLogonEvents": { + "AccountDisplayName": "string", + "AccountDomain": "string", + "AccountName": "string", + "AccountObjectId": "string", + "AccountSid": "string", + "AccountUpn": "string", + "ActionType": "string", + "AdditionalFields": "dynamic", + "Application": "string", + "DestinationDeviceName": "string", + "DestinationIPAddress": "string", + "DestinationPort": "string", + "DeviceName": "string", + "DeviceType": "string", + "FailureReason": "string", + "IPAddress": "string", + "ISP": "string", + "LastSeenForUser": "dynamic", + "Location": "string", + "LogonType": "string", + "OSPlatform": "string", + "Port": "string", + "Protocol": "string", + "ReportId": "string", + "SourceSystem": "string", + "TargetAccountDisplayName": "string", + "TargetDeviceName": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string", + "UncommonForUser": "dynamic" + }, + "IdentityQueryEvents": { + "AccountDisplayName": "string", + "AccountDomain": "string", + "AccountName": "string", + "AccountObjectId": "string", + "AccountSid": "string", + "AccountUpn": "string", + "ActionType": "string", + "AdditionalFields": "dynamic", + "Application": "string", + "DestinationDeviceName": "string", + "DestinationIPAddress": "string", + "DestinationPort": "string", + "DeviceName": "string", + "IPAddress": "string", + "Location": "string", + "Port": "string", + "Protocol": "string", + "Query": "string", + "QueryTarget": "string", + "QueryType": "string", + "ReportId": "string", + "SourceSystem": "string", + "TargetAccountDisplayName": "string", + "TargetAccountUpn": "string", + "TargetDeviceName": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string" + }, + "InsightsMetrics": { + "AgentId": "string", + "Computer": "string", + "Name": "string", + "Namespace": "string", + "Origin": "string", + "SourceSystem": "string", + "Tags": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "Val": "real", + "Value": "string", + "_ResourceId": "string" + }, + "NTANetAnalytics": { + "AclGroup": "string", + "AclRule": "string", + "Action": "string", + "AllowedInFlows": "long", + "AllowedOutFlows": "long", + "AzureRegion": "string", + "BytesDestToSrc": "long", + "BytesSrcToDest": "long", + "CompletedFlows": "long", + "ConnectingVnets": "string", + "ConnectionName": "string", + "ConnectionType": "string", + "Country": "string", + "DeniedInFlows": "long", + "DeniedOutFlows": "long", + "Description": "string", + "DestApplicationGateway": "string", + "DestCloud": "string", + "DestEnvironment": "string", + "DestExpressRouteCircuit": "string", + "DestIp": "string", + "DestLoadBalancer": "string", + "DestLocalNetworkGateway": "string", + "DestNetworkGroups": "string", + "DestNic": "string", + "DestPort": "int", + "DestPublicIps": "string", + "DestRegion": "string", + "DestServiceId": "string", + "DestServiceName": "string", + "DestServiceTags": "string", + "DestSubnet": "string", + "DestSubscription": "string", + "DestTenantId": "string", + "DestTenantName": "string", + "DestVm": "string", + "ExpressRouteCircuitPeeringType": "string", + "FaSchemaVersion": "string", + "FlowDirection": "string", + "FlowEncryption": "string", + "FlowEndTime": "datetime", + "FlowIntervalEndTime": "datetime", + "FlowIntervalStartTime": "datetime", + "FlowLogResourceId": "string", + "FlowStartTime": "datetime", + "FlowStatus": "string", + "FlowType": "string", + "IsFlowCapturedAtUdrHop": "bool", + "L4Protocol": "string", + "L7Protocol": "string", + "LogResourceId": "string", + "MacAddress": "string", + "NsgList": "string", + "NsgRule": "string", + "NsgRuleType": "string", + "PacketsDestToSrc": "long", + "PacketsSrcToDest": "long", + "PrivateEndpointResourceId": "string", + "PrivateLinkResourceId": "string", + "PrivateLinkResourceName": "string", + "RecordCount": "long", + "SourceSystem": "string", + "SrcApplicationGateway": "string", + "SrcCloud": "string", + "SrcEnvironment": "string", + "SrcExpressRouteCircuit": "string", + "SrcIp": "string", + "SrcLoadBalancer": "string", + "SrcLocalNetworkGateway": "string", + "SrcNetworkGroups": "string", + "SrcNic": "string", + "SrcPorts": "string", + "SrcPublicIps": "string", + "SrcRegion": "string", + "SrcServiceId": "string", + "SrcServiceName": "string", + "SrcServiceTags": "string", + "SrcSubnet": "string", + "SrcSubscription": "string", + "SrcTenantId": "string", + "SrcTenantName": "string", + "SrcVm": "string", + "Status": "string", + "SubType": "string", + "TargetResourceId": "string", + "TargetResourceType": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TimeProcessed": "datetime", + "TrafficAnalyticsResourceId": "string", + "Type": "string" + }, + "NetworkAccessAlerts": { + "AlertType": "string", + "ComponentName": "string", + "CreationDateTime": "datetime", + "Description": "string", + "DetectionTechnology": "string", + "DisplayName": "string", + "ExtendedProperties": "dynamic", + "FirstActivityDateTime": "datetime", + "Id": "string", + "IsPreview": "bool", + "LastActivityDateTime": "datetime", + "PolicyId": "string", + "ProductName": "string", + "RelatedResources": "dynamic", + "Severity": "string", + "SourceSystem": "string", + "SubTechniques": "string", + "Techniques": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "VendorName": "string" + }, + "NetworkAccessConnectionEvents": { + "AgentVersion": "string", + "AppId": "string", + "ClientDeviceName": "string", + "ConnectionId": "string", + "CrossTenantAccessType": "string", + "DestinationFqdn": "string", + "DestinationIp": "string", + "DestinationPort": "int", + "DeviceCategory": "string", + "DeviceId": "string", + "DeviceJoinType": "string", + "DeviceOperatingSystem": "string", + "DeviceOperatingSystemVersion": "string", + "EventType": "string", + "HomeTenantId": "string", + "InitiatingProcessName": "string", + "IsLocal": "bool", + "NetworkProtocol": "string", + "PopProcessingRegion": "string", + "RemoteNetworkId": "string", + "RemoteNetworkSourceIp": "string", + "SecurityPolicyId": "string", + "SecurityPolicyName": "string", + "SecurityPolicyVersion": "string", + "SecurityProfileId": "string", + "SecurityProfileName": "string", + "SecurityProfileVersion": "string", + "SecurityRuleId": "string", + "SourceIp": "string", + "SourceIpAutonomousSystemNumber": "string", + "SourceIpCarrier": "string", + "SourceIpCity": "string", + "SourceIpCountryCode": "string", + "SourceIpLatitude": "real", + "SourceIpLongitude": "real", + "SourceIpState": "string", + "SourceIpStateCode": "string", + "SourcePort": "int", + "SourcePrivateIp": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Token3PExpiry": "datetime", + "Token3PIssuedAt": "datetime", + "Token3PUniqueId": "string", + "Token3PValidFrom": "datetime", + "TrafficType": "string", + "TransportProtocol": "string", + "Type": "string", + "UserId": "string", + "UserPrincipalName": "string" + }, + "NetworkAccessGenerativeAIInsights": { + "Activity": "string", + "Content": "string", + "DestinationUrl": "string", + "EventId": "string", + "EventType": "string", + "McpClientName": "string", + "McpServerName": "string", + "SessionId": "string", + "SourceSystem": "string", + "SubActivity": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TransactionId": "string", + "Type": "string", + "UserPrincipalName": "string" + }, + "NetworkAccessTraffic": { + "AIAgentId": "string", + "AIAgentName": "string", + "AIDetectionConfidence": "int", + "AIDetectionEvidence": "string", + "AccessType": "string", + "Action": "string", + "AgentVersion": "string", + "AppId": "string", + "AppSegmentId": "string", + "CloudAppCatalogId": "string", + "CloudAppCategories": "string", + "CloudAppCategory": "string", + "CloudAppComplianceScore": "int", + "CloudAppGeneralScore": "int", + "CloudAppLegalScore": "int", + "CloudAppLoginUser": "string", + "CloudAppName": "string", + "CloudAppRiskScore": "int", + "ConnectionId": "string", + "ConnectionStatus": "string", + "ConnectorId": "string", + "ConnectorIp": "string", + "ConnectorName": "string", + "Description": "string", + "DestinationFqdn": "string", + "DestinationIp": "string", + "DestinationPort": "int", + "DestinationUrl": "string", + "DestinationWebCategories": "string", + "DeviceCategory": "string", + "DeviceId": "string", + "DeviceOperatingSystem": "string", + "DeviceOperatingSystemVersion": "string", + "DnsResponseOrigin": "string", + "FilteringProfileId": "string", + "FilteringProfileName": "string", + "HttpMethod": "string", + "HttpRequestContentType": "string", + "HttpResponseContentType": "string", + "HttpUserAgent": "string", + "InitiatingProcessName": "string", + "IsAgentic": "bool", + "IsTokenAgentic": "bool", + "KerberosClientName": "string", + "KerberosErrorCode": "int", + "KerberosErrorCodeName": "string", + "KerberosErrorText": "string", + "KerberosMsgType": "string", + "KerberosNonce": "int", + "NetworkProtocol": "string", + "OnPremAccount": "string", + "OriginHeader": "string", + "PolicyId": "string", + "PolicyName": "string", + "PolicyRuleId": "string", + "ProcessArgs": "string", + "ProcessTree": "string", + "ProcessingRegion": "string", + "Realm": "string", + "ReceivedBytes": "long", + "ReferrerHeader": "string", + "RemoteNetworkId": "string", + "ResourceTenantId": "string", + "ResponseCode": "int", + "RuleName": "string", + "SentBytes": "long", + "ServicePrincipalName": "string", + "SessionId": "string", + "SourceIp": "string", + "SourcePort": "int", + "SourceSystem": "string", + "TenantId": "string", + "ThreatType": "string", + "TimeGenerated": "datetime", + "TlsAction": "string", + "TlsPolicyId": "string", + "TlsPolicyName": "string", + "TlsRuleId": "string", + "TlsRuleName": "string", + "TlsStatus": "string", + "Token3PExpiry": "datetime", + "Token3PIssuedAt": "datetime", + "Token3PUniqueId": "string", + "Token3PValidFrom": "datetime", + "TrafficType": "string", + "TransactionId": "string", + "TransportProtocol": "string", + "Type": "string", + "UniqueTokenId": "string", + "UserId": "string", + "UserPrincipalName": "string", + "VendorNames": "string", + "XForwardedFor": "string" + }, + "NetworkSessions": { + "AdditionalFields": "dynamic", + "CloudAppId": "string", + "CloudAppName": "string", + "CloudAppOperation": "string", + "CloudAppRiskLevel": "string", + "DstBytes": "long", + "DstDomainHostname": "string", + "DstDvcDomain": "string", + "DstDvcFqdn": "string", + "DstDvcHostname": "string", + "DstDvcIpAddr": "string", + "DstDvcMacAddr": "string", + "DstGeoCity": "string", + "DstGeoCountry": "string", + "DstGeoLatitude": "real", + "DstGeoLongitude": "real", + "DstGeoRegion": "string", + "DstInterfaceGuid": "string", + "DstInterfaceName": "string", + "DstIpAddr": "string", + "DstMacAddr": "string", + "DstNatIpAddr": "string", + "DstNatPortNumber": "int", + "DstPackets": "long", + "DstPortNumber": "int", + "DstResourceId": "string", + "DstUserAadId": "string", + "DstUserDomain": "string", + "DstUserName": "string", + "DstUserSid": "string", + "DstUserUpn": "string", + "DstZone": "string", + "DvcAction": "string", + "DvcHostname": "string", + "DvcInboundInterface": "string", + "DvcIpAddr": "string", + "DvcMacAddr": "string", + "DvcOutboundInterface": "string", + "EventCount": "int", + "EventEndTime": "datetime", + "EventMessage": "string", + "EventOriginalUid": "string", + "EventProduct": "string", + "EventProductVersion": "string", + "EventReportUrl": "string", + "EventResourceId": "string", + "EventResult": "string", + "EventResultDetails": "string", + "EventSchemaVersion": "string", + "EventSeverity": "string", + "EventStartTime": "datetime", + "EventSubType": "string", + "EventTimeIngested": "datetime", + "EventType": "string", + "EventUid": "string", + "EventVendor": "string", + "FileExtension": "string", + "FileHashMd5": "string", + "FileHashSha1": "string", + "FileHashSha256": "string", + "FileHashSha512": "string", + "FileMimeType": "string", + "FileName": "string", + "FilePath": "string", + "FileSize": "int", + "HttpContentType": "string", + "HttpReferrerOriginal": "string", + "HttpRequestMethod": "string", + "HttpRequestTime": "int", + "HttpRequestXff": "string", + "HttpResponseTime": "int", + "HttpStatusCode": "string", + "HttpUserAgentOriginal": "string", + "HttpVersion": "string", + "NetworkApplicationProtocol": "string", + "NetworkBytes": "long", + "NetworkDirection": "string", + "NetworkDuration": "int", + "NetworkIcmpCode": "int", + "NetworkIcmpType": "string", + "NetworkPackets": "long", + "NetworkProtocol": "string", + "NetworkRuleName": "string", + "NetworkRuleNumber": "int", + "NetworkSessionId": "string", + "SourceSystem": "string", + "SrcBytes": "long", + "SrcDvcDomain": "string", + "SrcDvcFqdn": "string", + "SrcDvcHostname": "string", + "SrcDvcIpAddr": "string", + "SrcDvcMacAddr": "string", + "SrcDvcModelName": "string", + "SrcDvcModelNumber": "string", + "SrcDvcOs": "string", + "SrcDvcType": "string", + "SrcGeoCity": "string", + "SrcGeoCountry": "string", + "SrcGeoLatitude": "real", + "SrcGeoLongitude": "real", + "SrcGeoRegion": "string", + "SrcInterfaceGuid": "string", + "SrcInterfaceName": "string", + "SrcIpAddr": "string", + "SrcMacAddr": "string", + "SrcNatIpAddr": "string", + "SrcNatPortNumber": "int", + "SrcPackets": "long", + "SrcPortNumber": "int", + "SrcResourceId": "string", + "SrcUserAadId": "string", + "SrcUserDomain": "string", + "SrcUserName": "string", + "SrcUserSid": "string", + "SrcUserUpn": "string", + "SrcZone": "string", + "TenantId": "string", + "ThreatCategory": "string", + "ThreatId": "string", + "ThreatName": "string", + "TimeGenerated": "datetime", + "Type": "string", + "UrlCategory": "string", + "UrlHostname": "string", + "UrlOriginal": "string" + }, + "OfficeActivity": { + "AADGroupId": "string", + "AADTarget": "string", + "Activity": "string", + "Actor": "string", + "ActorContextId": "string", + "ActorContextId_": "string", + "ActorIpAddress": "string", + "AddOnGuid": "string", + "AddOnType": "string", + "AddonName": "string", + "AffectedItems": "string", + "AppAccessContext": "dynamic", + "AppDistributionMode": "string", + "AppId": "string", + "AppPoolName": "string", + "Application": "string", + "ApplicationId": "string", + "ArtifactsShared": "dynamic", + "Attendees": "dynamic", + "AzureADAppId": "string", + "AzureActiveDirectory_EventType": "string", + "ChannelGuid": "string", + "ChannelName": "string", + "ChannelType": "string", + "ChatName": "string", + "ChatThreadId": "string", + "Client": "string", + "ClientAppId": "string", + "ClientIP": "string", + "ClientIP_": "string", + "ClientInfoString": "string", + "ClientMachineName": "string", + "ClientProcessName": "string", + "ClientVersion": "string", + "Client_IPAddress": "string", + "CommunicationType": "string", + "CrossMailboxOperations": "bool", + "CustomEvent": "string", + "DataCenterSecurityEventType": "int", + "DestFolder": "string", + "DestMailboxId": "string", + "DestMailboxOwnerMasterAccountSid": "string", + "DestMailboxOwnerSid": "string", + "DestMailboxOwnerUPN": "string", + "DestinationFileExtension": "string", + "DestinationFileName": "string", + "DestinationRelativeUrl": "string", + "DeviceInformation": "string", + "EffectiveOrganization": "string", + "ElevationApprovedTime": "datetime", + "ElevationApprover": "string", + "ElevationDuration": "int", + "ElevationRequestId": "string", + "ElevationRole": "string", + "ElevationTime": "datetime", + "EventSource": "string", + "Event_Data": "string", + "ExtendedProperties": "string", + "ExternalAccess": "string", + "ExtraProperties": "dynamic", + "Folder": "string", + "Folders": "string", + "GenericInfo": "string", + "InterSystemsId": "string", + "InternalLogonType": "int", + "IntraSystemId": "string", + "IsJoinedFromLobby": "bool", + "IsManagedDevice": "bool", + "IssuedAtTime": "datetime", + "Item": "string", + "ItemName": "string", + "ItemType": "string", + "JoinTime": "datetime", + "LeaveTime": "datetime", + "ListItemUniqueId": "string", + "LoginStatus": "int", + "LogonUserDisplayName": "string", + "LogonUserSid": "string", + "Logon_Type": "string", + "MachineDomainInfo": "string", + "MachineId": "string", + "MailboxGuid": "string", + "MailboxOwnerMasterAccountSid": "string", + "MailboxOwnerSid": "string", + "MailboxOwnerUPN": "string", + "MeetingDetailId": "string", + "Members": "dynamic", + "MessageId": "string", + "ModifiedObjectResolvedName": "string", + "ModifiedProperties": "string", + "Name": "string", + "NewValue": "string", + "OfficeId": "string", + "OfficeObjectId": "string", + "OfficeTenantId": "string", + "OfficeTenantId_": "string", + "OfficeWorkload": "string", + "OldValue": "string", + "Operation": "string", + "OperationProperties": "dynamic", + "OperationScope": "string", + "OrganizationId": "string", + "OrganizationId_": "string", + "OrganizationName": "string", + "OriginatingServer": "string", + "Parameters": "string", + "RecordType": "string", + "ResultReasonType": "string", + "ResultStatus": "string", + "SRPolicyId": "string", + "SRPolicyName": "string", + "SRRuleMatchDetails": "dynamic", + "Scope": "string", + "SendAsUserMailboxGuid": "string", + "SendAsUserSmtp": "string", + "SendOnBehalfOfUserSmtp": "string", + "SendonBehalfOfUserMailboxGuid": "string", + "SensitivityLabelId": "string", + "SharingType": "string", + "Site_": "string", + "Site_Url": "string", + "Site_Url_": "string", + "SourceFileExtension": "string", + "SourceFileName": "string", + "SourceFileName_": "string", + "SourceRecordId": "string", + "SourceRelativeUrl": "string", + "SourceRelativeUrl_": "string", + "SourceSystem": "string", + "Source_Name": "string", + "Start_Time": "datetime", + "SupportTicketId": "string", + "TabType": "string", + "TargetContextId": "string", + "TargetUserId": "string", + "TargetUserOrGroupName": "string", + "TargetUserOrGroupType": "string", + "TeamGuid": "string", + "TeamName": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "UniqueSharingId": "string", + "UniqueTokenId": "string", + "UserAgent": "string", + "UserDomain": "string", + "UserId": "string", + "UserId_": "string", + "UserKey": "string", + "UserSharedWith": "string", + "UserType": "string", + "_ResourceId": "string" + }, + "Operation": { + "Computer": "string", + "CorrelationId": "string", + "Detail": "string", + "ErrorId": "string", + "HelpLink": "string", + "MG": "string", + "ManagementGroupName": "string", + "OperationCategory": "string", + "OperationKey": "string", + "OperationStatus": "string", + "Solution": "string", + "SourceComputerId": "string", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "Perf": { + "BucketEndTime": "datetime", + "BucketStartTime": "datetime", + "Computer": "string", + "CounterName": "string", + "CounterPath": "string", + "CounterValue": "real", + "InstanceName": "string", + "MG": "string", + "Max": "real", + "Min": "real", + "ObjectName": "string", + "SampleCount": "int", + "SourceSystem": "string", + "StandardDeviation": "real", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "PowerPlatformAdminActivity": { + "ActorName": "string", + "ActorUserId": "string", + "ActorUserType": "string", + "EnvironmentId": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventResult": "string", + "OrganizationId": "string", + "Properties": "dynamic", + "PropertyCollection": "dynamic", + "RecordType": "string", + "RequiresCustomerKeyEncryption": "bool", + "SourceSystem": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "Workload": "string" + }, + "RemoteNetworkHealthLogs": { + "BgpRoutesAdvertisedCount": "int", + "CreatedDateTime": "datetime", + "Description": "string", + "DestinationIp": "string", + "Id": "string", + "ReceivedBytes": "long", + "RemoteNetworkId": "string", + "SentBytes": "long", + "SourceIp": "string", + "SourceSystem": "string", + "Status": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string" + }, + "SecurityAlert": { + "AlertLink": "string", + "AlertName": "string", + "AlertSeverity": "string", + "AlertType": "string", + "CompromisedEntity": "string", + "ConfidenceLevel": "string", + "ConfidenceScore": "real", + "Description": "string", + "DisplayName": "string", + "EndTime": "datetime", + "Entities": "string", + "ExtendedLinks": "string", + "ExtendedProperties": "string", + "IsIncident": "bool", + "ProcessingEndTime": "datetime", + "ProductComponentName": "string", + "ProductName": "string", + "ProviderName": "string", + "RemediationSteps": "string", + "ResourceId": "string", + "SourceComputerId": "string", + "SourceSystem": "string", + "StartTime": "datetime", + "Status": "string", + "SubTechniques": "string", + "SystemAlertId": "string", + "Tactics": "string", + "Techniques": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "VendorName": "string", + "VendorOriginalId": "string", + "WorkspaceResourceGroup": "string", + "WorkspaceSubscriptionId": "string" + }, + "SecurityEvent": { + "AccessList": "string", + "AccessMask": "string", + "AccessReason": "string", + "Account": "string", + "AccountDomain": "string", + "AccountExpires": "string", + "AccountName": "string", + "AccountSessionIdentifier": "string", + "AccountType": "string", + "Activity": "string", + "AdditionalInfo": "string", + "AdditionalInfo2": "string", + "AllowedToDelegateTo": "string", + "Attributes": "string", + "AuditPolicyChanges": "string", + "AuditsDiscarded": "int", + "AuthenticationLevel": "int", + "AuthenticationPackageName": "string", + "AuthenticationProvider": "string", + "AuthenticationServer": "string", + "AuthenticationService": "int", + "AuthenticationType": "string", + "AzureDeploymentID": "string", + "AzureTableName": "string", + "CACertificateHash": "string", + "CAPublicKeyHash": "string", + "CalledStationID": "string", + "CallerProcessId": "string", + "CallerProcessName": "string", + "CallingStationID": "string", + "CategoryId": "string", + "CertificateDatabaseHash": "string", + "Channel": "string", + "ClassId": "string", + "ClassName": "string", + "ClientAddress": "string", + "ClientIPAddress": "string", + "ClientName": "string", + "CommandLine": "string", + "CompatibleIds": "string", + "Computer": "string", + "Correlation": "string", + "DCDNSName": "string", + "DeviceDescription": "string", + "DeviceId": "string", + "DisplayName": "string", + "Disposition": "string", + "DomainBehaviorVersion": "string", + "DomainName": "string", + "DomainPolicyChanged": "string", + "DomainSid": "string", + "EAPType": "string", + "ElevatedToken": "string", + "ErrorCode": "int", + "EventData": "string", + "EventID": "int", + "EventLevelName": "string", + "EventOriginId": "string", + "EventRecordId": "string", + "EventSourceName": "string", + "ExtendedQuarantineState": "string", + "FailureReason": "string", + "FileHash": "string", + "FilePath": "string", + "FilePathNoUser": "string", + "Filter": "string", + "ForceLogoff": "string", + "Fqbn": "string", + "FullyQualifiedSubjectMachineName": "string", + "FullyQualifiedSubjectUserName": "string", + "GroupMembership": "string", + "HandleId": "string", + "HardwareIds": "string", + "HomeDirectory": "string", + "HomePath": "string", + "ImpersonationLevel": "string", + "InterfaceUuid": "string", + "IpAddress": "string", + "IpPort": "string", + "KeyLength": "int", + "Keywords": "string", + "Level": "string", + "LmPackageName": "string", + "LocationInformation": "string", + "LockoutDuration": "string", + "LockoutObservationWindow": "string", + "LockoutThreshold": "string", + "LoggingResult": "string", + "LogonGuid": "string", + "LogonHours": "string", + "LogonID": "string", + "LogonProcessName": "string", + "LogonType": "int", + "LogonTypeName": "string", + "MG": "string", + "MachineAccountQuota": "string", + "MachineInventory": "string", + "MachineLogon": "string", + "ManagementGroupName": "string", + "MandatoryLabel": "string", + "MaxPasswordAge": "string", + "MemberName": "string", + "MemberSid": "string", + "MinPasswordAge": "string", + "MinPasswordLength": "string", + "MixedDomainMode": "string", + "NASIPv4Address": "string", + "NASIPv6Address": "string", + "NASIdentifier": "string", + "NASPort": "string", + "NASPortType": "string", + "NetworkPolicyName": "string", + "NewDate": "string", + "NewMaxUsers": "string", + "NewProcessId": "string", + "NewProcessName": "string", + "NewRemark": "string", + "NewShareFlags": "string", + "NewTime": "string", + "NewUacValue": "string", + "NewValue": "string", + "NewValueType": "string", + "ObjectName": "string", + "ObjectServer": "string", + "ObjectType": "string", + "ObjectValueName": "string", + "OemInformation": "string", + "OldMaxUsers": "string", + "OldRemark": "string", + "OldShareFlags": "string", + "OldUacValue": "string", + "OldValue": "string", + "OldValueType": "string", + "Opcode": "string", + "OperationType": "string", + "PackageName": "string", + "ParentProcessName": "string", + "PartitionKey": "string", + "PasswordHistoryLength": "string", + "PasswordLastSet": "string", + "PasswordProperties": "string", + "PreviousDate": "string", + "PreviousTime": "string", + "PrimaryGroupId": "string", + "PrivateKeyUsageCount": "string", + "PrivilegeList": "string", + "Process": "string", + "ProcessId": "string", + "ProcessName": "string", + "ProfilePath": "string", + "Properties": "string", + "ProtocolSequence": "string", + "ProxyPolicyName": "string", + "QuarantineHelpURL": "string", + "QuarantineSessionID": "string", + "QuarantineSessionIdentifier": "string", + "QuarantineState": "string", + "QuarantineSystemHealthResult": "string", + "RelativeTargetName": "string", + "RemoteIpAddress": "string", + "RemotePort": "string", + "RequestId": "string", + "Requester": "string", + "RestrictedAdminMode": "string", + "RowKey": "string", + "RowsDeleted": "string", + "SamAccountName": "string", + "ScriptPath": "string", + "SecurityDescriptor": "string", + "ServiceAccount": "string", + "ServiceFileName": "string", + "ServiceName": "string", + "ServiceStartType": "int", + "ServiceType": "string", + "SessionName": "string", + "ShareLocalPath": "string", + "ShareName": "string", + "SidHistory": "string", + "SourceComputerId": "string", + "SourceSystem": "string", + "Status": "string", + "StorageAccount": "string", + "SubStatus": "string", + "SubcategoryGuid": "string", + "SubcategoryId": "string", + "Subject": "string", + "SubjectAccount": "string", + "SubjectDomainName": "string", + "SubjectKeyIdentifier": "string", + "SubjectLogonId": "string", + "SubjectMachineName": "string", + "SubjectMachineSID": "string", + "SubjectUserName": "string", + "SubjectUserSid": "string", + "SystemProcessId": "int", + "SystemThreadId": "int", + "SystemUserId": "string", + "TableId": "string", + "TargetAccount": "string", + "TargetDomainName": "string", + "TargetInfo": "string", + "TargetLinkedLogonId": "string", + "TargetLogonGuid": "string", + "TargetLogonId": "string", + "TargetOutboundDomainName": "string", + "TargetOutboundUserName": "string", + "TargetServerName": "string", + "TargetSid": "string", + "TargetUser": "string", + "TargetUserName": "string", + "TargetUserSid": "string", + "Task": "int", + "TemplateContent": "string", + "TemplateDSObjectFQDN": "string", + "TemplateInternalName": "string", + "TemplateOID": "string", + "TemplateSchemaVersion": "string", + "TemplateVersion": "string", + "TenantId": "string", + "TimeCollected": "datetime", + "TimeGenerated": "datetime", + "TokenElevationType": "string", + "TransmittedServices": "string", + "Type": "string", + "UserAccountControl": "string", + "UserParameters": "string", + "UserPrincipalName": "string", + "UserWorkstations": "string", + "VendorIds": "string", + "Version": "int", + "VirtualAccount": "string", + "Workstation": "string", + "WorkstationName": "string", + "_ResourceId": "string" + }, + "SecurityIncident": { + "AdditionalData": "dynamic", + "AlertIds": "dynamic", + "BookmarkIds": "dynamic", + "Classification": "string", + "ClassificationComment": "string", + "ClassificationReason": "string", + "ClosedTime": "datetime", + "Comments": "dynamic", + "CreatedTime": "datetime", + "Description": "string", + "FirstActivityTime": "datetime", + "FirstModifiedTime": "datetime", + "IncidentName": "string", + "IncidentNumber": "int", + "IncidentUrl": "string", + "Labels": "dynamic", + "LastActivityTime": "datetime", + "LastModifiedTime": "datetime", + "ModifiedBy": "string", + "Owner": "dynamic", + "ProviderIncidentId": "string", + "ProviderName": "string", + "RelatedAnalyticRuleIds": "dynamic", + "Severity": "string", + "SourceSystem": "string", + "Status": "string", + "Tasks": "dynamic", + "TenantId": "string", + "TimeGenerated": "datetime", + "Title": "string", + "Type": "string" + }, + "SentinelAlibabaCloudAPIGatewayLogs": { + "ApiGroupName": "string", + "ApiGroupUid": "string", + "ApiName": "string", + "ApiStageName": "string", + "ApiStageUid": "string", + "ApiUid": "string", + "AppId": "string", + "AppName": "string", + "ClientIp": "string", + "ClientNonce": "string", + "ConsumerAppKey": "string", + "CustomTraceId": "string", + "Domain": "string", + "ErrorCode": "string", + "ErrorMessage": "string", + "Exception": "string", + "ExchangeTimestamp": "string", + "HttpMethod": "string", + "InitialRequestId": "string", + "InstanceId": "string", + "JwtClaims": "string", + "LogProject": "string", + "LogRegion": "string", + "LogStore": "string", + "Path": "string", + "Plugin": "string", + "ProviderAliUid": "string", + "Region": "string", + "RequestBody": "string", + "RequestHandleTime": "datetime", + "RequestHeaders": "string", + "RequestId": "string", + "RequestProtocol": "string", + "RequestQueryString": "string", + "RequestSize": "string", + "ResponseBody": "string", + "ResponseHeaders": "string", + "ResponseSize": "string", + "ServiceLatency": "real", + "SourceSystem": "string", + "StatusCode": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TotalLatency": "real", + "Type": "string" + }, + "SentinelAlibabaCloudVPCFlowLogs": { + "AccountId": "string", + "Action": "string", + "Bytes": "string", + "Direction": "string", + "DstAddr": "string", + "DstPort": "string", + "DstType": "string", + "End": "string", + "EniId": "string", + "LogProject": "string", + "LogRegion": "string", + "LogStatus": "string", + "LogStore": "string", + "Packets": "string", + "Protocol": "string", + "Region": "string", + "SourceSystem": "string", + "SrcAddr": "string", + "SrcPort": "string", + "SrcType": "string", + "Start": "string", + "TcpFlags": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TrafficPath": "string", + "Type": "string", + "Version": "string", + "VmId": "string", + "VpcId": "string", + "VswitchId": "string" + }, + "SentinelAlibabaCloudWAFLogs": { + "AccountAction": "string", + "AccountRuleId": "string", + "AccountTest": "bool", + "AclAction": "string", + "AclRuleId": "string", + "AclRuleType": "string", + "AclTest": "bool", + "AlgorithmRuleId": "string", + "AntiscanAction": "string", + "AntiscanRuleId": "string", + "AntiscanRuleType": "string", + "AntiscanTest": "bool", + "BlockAction": "string", + "BodyBytesSent": "long", + "BypassMatchedIds": "string", + "CcAction": "string", + "CcBlocks": "string", + "CcRuleId": "string", + "CcRuleType": "string", + "CcTest": "bool", + "ContentType": "string", + "DeeplearningAction": "string", + "DeeplearningRuleId": "string", + "DeeplearningRuleType": "string", + "DeeplearningTest": "bool", + "DlpRuleId": "string", + "DlpTest": "bool", + "FinalAction": "string", + "FinalPlugin": "string", + "FinalRuleId": "string", + "FinalRuleType": "string", + "Host": "string", + "HttpCookie": "string", + "HttpReferer": "string", + "HttpUserAgent": "string", + "HttpXForwardedFor": "string", + "Https": "bool", + "LogProject": "string", + "LogRegion": "string", + "LogStore": "string", + "MatchedHost": "string", + "NormalizedAction": "string", + "NormalizedRuleId": "string", + "NormalizedRuleType": "string", + "NormalizedTest": "bool", + "OwnerId": "string", + "Querystring": "string", + "RealClientIp": "string", + "Region": "string", + "RemoteAddr": "string", + "RemotePort": "int", + "RequestLength": "long", + "RequestMethod": "string", + "RequestPath": "string", + "RequestTimeMsec": "real", + "RequestTraceid": "string", + "SceneAction": "string", + "SceneId": "string", + "SceneRuleId": "string", + "SceneRuleType": "string", + "SceneTest": "bool", + "ServerPort": "int", + "ServerProtocol": "string", + "SourceSystem": "string", + "SslCipher": "string", + "SslProtocol": "string", + "Status": "int", + "TenantId": "string", + "Time": "datetime", + "TimeGenerated": "datetime", + "Type": "string", + "UaBrowser": "string", + "UaBrowserFamily": "string", + "UaBrowserType": "string", + "UaBrowserVersion": "string", + "UaDeviceType": "string", + "UaOs": "string", + "UaOsFamily": "string", + "UpstreamAddr": "string", + "UpstreamResponseTime": "real", + "UpstreamStatus": "int", + "UserId": "string", + "WafAction": "string", + "WafRuleId": "string", + "WafRuleType": "string", + "WafTest": "bool" + }, + "SentinelAudit": { + "CorrelationId": "string", + "Description": "string", + "ExtendedProperties": "dynamic", + "OperationName": "string", + "SentinelResourceId": "string", + "SentinelResourceKind": "string", + "SentinelResourceName": "string", + "SentinelResourceType": "string", + "SourceSystem": "string", + "Status": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "WorkspaceId": "string" + }, + "SentinelBehaviorEntities": { + "AccountDomain": "string", + "AccountName": "string", + "AccountObjectId": "string", + "AccountSid": "string", + "AccountUpn": "string", + "ActionType": "string", + "AdditionalFields": "string", + "Application": "string", + "ApplicationId": "int", + "BehaviorId": "string", + "Categories": "string", + "CloudPlatform": "string", + "CloudResource": "string", + "CloudResourceId": "string", + "CloudResourceType": "string", + "CloudSubscriptionId": "string", + "DataSources": "string", + "DetailedEntityRole": "string", + "DetectionSource": "string", + "DeviceId": "string", + "DeviceName": "string", + "EmailClusterId": "string", + "EmailSubject": "string", + "EntityRole": "string", + "EntityType": "string", + "FileName": "string", + "FileSize": "long", + "FolderPath": "string", + "LocalIP": "string", + "NetworkMessageId": "string", + "OAuthApplicationId": "string", + "ProcessCommandLine": "string", + "RegistryKey": "string", + "RegistryValueData": "string", + "RegistryValueName": "string", + "RemoteIP": "string", + "RemoteUrl": "string", + "SHA1": "string", + "SHA256": "string", + "ServiceSource": "string", + "SourceSystem": "string", + "TenantId": "string", + "ThreatFamily": "string", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "SentinelBehaviorInfo": { + "AccountObjectId": "string", + "AccountUpn": "string", + "ActionType": "string", + "AdditionalFields": "string", + "AttackTechniques": "string", + "BehaviorId": "string", + "Categories": "string", + "DataSources": "string", + "Description": "string", + "DetectionSource": "string", + "DeviceId": "string", + "EndTime": "datetime", + "Insights": "string", + "ServiceSource": "string", + "SourceSystem": "string", + "StartTime": "datetime", + "TenantId": "string", + "TimeGenerated": "datetime", + "Title": "string", + "Type": "string", + "_ResourceId": "string" + }, + "SentinelHealth": { + "Description": "string", + "ExtendedProperties": "dynamic", + "OperationName": "string", + "Reason": "string", + "RecordId": "string", + "SentinelResourceId": "string", + "SentinelResourceKind": "string", + "SentinelResourceName": "string", + "SentinelResourceType": "string", + "SourceSystem": "string", + "Status": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "WorkspaceId": "string" + }, + "SentinelImpervaWAFCloudV2Logs": { + "Act": "string", + "AdditionalReqHeaders": "string", + "AdditionalResHeaders": "string", + "AdditionalRuleInfo": "string", + "App": "string", + "AttackName": "string", + "AttackSeverity": "string", + "CCode": "string", + "CapSupport": "string", + "CiCode": "string", + "ClApp": "string", + "ClAppSig": "string", + "Cn1": "string", + "CoSupport": "string", + "Cpt": "string", + "Customer": "string", + "DeliveryRuleDetails": "string", + "DeviceExternalId": "string", + "DeviceFacility": "string", + "DeviceVersion": "string", + "Dproc": "string", + "End": "string", + "EventProduct": "string", + "EventType": "string", + "EventVendor": "string", + "FileId": "string", + "FilePermission": "string", + "FileType": "string", + "In": "string", + "JavascriptSupport": "string", + "Latitude": "string", + "LogVersion": "string", + "Longitude": "string", + "PostBody": "string", + "QStr": "string", + "Ref": "string", + "Request": "string", + "RequestClientApplication": "string", + "RequestMethod": "string", + "RuleName": "string", + "SIP": "string", + "SignatureId": "string", + "SiteId": "string", + "SiteTag": "string", + "SourceServiceName": "string", + "SourceSystem": "string", + "Spt": "string", + "Src": "string", + "Start": "string", + "Suid": "string", + "Tag": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "VID": "string", + "Ver": "string", + "Xff": "string" + }, + "SigninLogs": { + "AADTenantId": "string", + "Agent": "dynamic", + "AlternateSignInName": "string", + "AppDisplayName": "string", + "AppId": "string", + "AppOwnerTenantId": "string", + "AppliedConditionalAccessPolicies": "string", + "AppliedEventListeners": "dynamic", + "AuthenticationAppDeviceDetails": "string", + "AuthenticationAppPolicyEvaluationDetails": "string", + "AuthenticationContextClassReferences": "string", + "AuthenticationDetails": "string", + "AuthenticationMethodsUsed": "string", + "AuthenticationProcessingDetails": "string", + "AuthenticationProtocol": "string", + "AuthenticationRequirement": "string", + "AuthenticationRequirementPolicies": "string", + "AuthenticatorAppLocation": "string", + "AutonomousSystemNumber": "string", + "Category": "string", + "ClientAppUsed": "string", + "ClientCredentialType": "string", + "ConditionalAccessAudiences": "string", + "ConditionalAccessPolicies": "dynamic", + "ConditionalAccessStatus": "string", + "CorrelationId": "string", + "CreatedDateTime": "datetime", + "CrossTenantAccessType": "string", + "DeviceDetail": "dynamic", + "DurationMs": "long", + "FederatedCredentialId": "string", + "FlaggedForReview": "bool", + "GlobalSecureAccessIpAddress": "string", + "HomeTenantId": "string", + "HomeTenantName": "string", + "IPAddress": "string", + "IPAddressFromResourceProvider": "string", + "Id": "string", + "Identity": "string", + "IncomingTokenType": "string", + "IsInteractive": "bool", + "IsRisky": "bool", + "IsTenantRestricted": "bool", + "IsThroughGlobalSecureAccess": "bool", + "Level": "string", + "Location": "string", + "LocationDetails": "dynamic", + "MfaDetail": "dynamic", + "NetworkLocationDetails": "string", + "OperationName": "string", + "OperationVersion": "string", + "OriginalRequestId": "string", + "OriginalTransferMethod": "string", + "ProcessingTimeInMilliseconds": "string", + "Resource": "string", + "ResourceDisplayName": "string", + "ResourceGroup": "string", + "ResourceId": "string", + "ResourceIdentity": "string", + "ResourceOwnerTenantId": "string", + "ResourceProvider": "string", + "ResourceServicePrincipalId": "string", + "ResourceTenantId": "string", + "ResultDescription": "string", + "ResultSignature": "string", + "ResultType": "string", + "RiskDetail": "string", + "RiskEventTypes": "string", + "RiskEventTypes_V2": "string", + "RiskLevel": "string", + "RiskLevelAggregated": "string", + "RiskLevelDuringSignIn": "string", + "RiskState": "string", + "ServicePrincipalId": "string", + "ServicePrincipalName": "string", + "SessionId": "string", + "SessionLifetimePolicies": "string", + "SignInIdentifier": "string", + "SignInIdentifierType": "string", + "SourceAppClientId": "string", + "SourceSystem": "string", + "Status": "dynamic", + "TenantId": "string", + "TimeGenerated": "datetime", + "TokenIssuerName": "string", + "TokenIssuerType": "string", + "TokenProtectionStatusDetails": "dynamic", + "Type": "string", + "UniqueTokenIdentifier": "string", + "UserAgent": "string", + "UserDisplayName": "string", + "UserId": "string", + "UserPrincipalName": "string", + "UserType": "string" + }, + "StorageBlobLogs": { + "AccessTier": "string", + "AccountName": "string", + "AuthenticationHash": "string", + "AuthenticationType": "string", + "AuthorizationDetails": "dynamic", + "CallerIpAddress": "string", + "Category": "string", + "ClientRequestId": "string", + "ConditionsUsed": "string", + "ContentLengthHeader": "long", + "CorrelationId": "string", + "DestinationUri": "string", + "DurationMs": "real", + "Etag": "string", + "LastModifiedTime": "datetime", + "Location": "string", + "MetricResponseType": "string", + "ObjectKey": "string", + "OperationCount": "int", + "OperationName": "string", + "OperationVersion": "string", + "Protocol": "string", + "ReferrerHeader": "string", + "RehydratePriority": "string", + "RequestBodySize": "long", + "RequestHeaderSize": "long", + "RequestMd5": "string", + "RequesterAppId": "string", + "RequesterAudience": "string", + "RequesterObjectId": "string", + "RequesterTenantId": "string", + "RequesterTokenIssuer": "string", + "RequesterUpn": "string", + "ResponseBodySize": "long", + "ResponseHeaderSize": "long", + "ResponseMd5": "string", + "SasExpiryStatus": "string", + "SchemaVersion": "string", + "ServerLatencyMs": "real", + "ServiceType": "string", + "SourceAccessTier": "string", + "SourceSystem": "string", + "SourceUri": "string", + "StatusCode": "string", + "StatusText": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TlsVersion": "string", + "Type": "string", + "Uri": "string", + "UserAgentHeader": "string", + "_ResourceId": "string" + }, + "StorageFileLogs": { + "AccountName": "string", + "AuthenticationHash": "string", + "AuthenticationType": "string", + "AuthorizationDetails": "dynamic", + "CallerIpAddress": "string", + "Category": "string", + "ClientRequestId": "string", + "ConditionsUsed": "string", + "ContentLengthHeader": "long", + "CorrelationId": "string", + "DurationMs": "real", + "Etag": "string", + "LastModifiedTime": "datetime", + "Location": "string", + "MetricResponseType": "string", + "ObjectKey": "string", + "OperationCount": "int", + "OperationName": "string", + "OperationVersion": "string", + "Protocol": "string", + "ReferrerHeader": "string", + "RequestBodySize": "long", + "RequestHeaderSize": "long", + "RequestMd5": "string", + "RequesterAppId": "string", + "RequesterAudience": "string", + "RequesterObjectId": "string", + "RequesterTenantId": "string", + "RequesterTokenIssuer": "string", + "RequesterUpn": "string", + "RequesterUserName": "string", + "ResponseBodySize": "long", + "ResponseHeaderSize": "long", + "ResponseMd5": "string", + "SasExpiryStatus": "string", + "SchemaVersion": "string", + "ServerLatencyMs": "real", + "ServiceType": "string", + "SmbCommandDetail": "string", + "SmbCommandMajor": "int", + "SmbCommandMinor": "string", + "SmbCreditsConsumed": "int", + "SmbFileId": "string", + "SmbMessageID": "string", + "SmbPersistentHandleID": "string", + "SmbPrimarySID": "string", + "SmbSessionID": "string", + "SmbStatusCode": "string", + "SmbTreeConnectID": "string", + "SmbVolatileHandleID": "string", + "SourceSystem": "string", + "StatusCode": "string", + "StatusText": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TlsVersion": "string", + "Type": "string", + "Uri": "string", + "UserAgentHeader": "string", + "_ResourceId": "string" + }, + "StorageQueueLogs": { + "AccountName": "string", + "AuthenticationHash": "string", + "AuthenticationType": "string", + "AuthorizationDetails": "dynamic", + "CallerIpAddress": "string", + "Category": "string", + "ClientRequestId": "string", + "ConditionsUsed": "string", + "ContentLengthHeader": "long", + "CorrelationId": "string", + "DurationMs": "real", + "Etag": "string", + "LastModifiedTime": "datetime", + "Location": "string", + "MetricResponseType": "string", + "ObjectKey": "string", + "OperationCount": "int", + "OperationName": "string", + "OperationVersion": "string", + "Protocol": "string", + "ReferrerHeader": "string", + "RequestBodySize": "long", + "RequestHeaderSize": "long", + "RequestMd5": "string", + "RequesterAppId": "string", + "RequesterAudience": "string", + "RequesterObjectId": "string", + "RequesterTenantId": "string", + "RequesterTokenIssuer": "string", + "RequesterUpn": "string", + "ResponseBodySize": "long", + "ResponseHeaderSize": "long", + "ResponseMd5": "string", + "SasExpiryStatus": "string", + "SchemaVersion": "string", + "ServerLatencyMs": "real", + "ServiceType": "string", + "SourceSystem": "string", + "StatusCode": "string", + "StatusText": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TlsVersion": "string", + "Type": "string", + "Uri": "string", + "UserAgentHeader": "string", + "_ResourceId": "string" + }, + "StorageTableLogs": { + "AccountName": "string", + "AuthenticationHash": "string", + "AuthenticationType": "string", + "AuthorizationDetails": "dynamic", + "CallerIpAddress": "string", + "Category": "string", + "ClientRequestId": "string", + "ConditionsUsed": "string", + "ContentLengthHeader": "long", + "CorrelationId": "string", + "DurationMs": "real", + "Etag": "string", + "LastModifiedTime": "datetime", + "Location": "string", + "MetricResponseType": "string", + "ObjectKey": "string", + "OperationCount": "int", + "OperationName": "string", + "OperationVersion": "string", + "Protocol": "string", + "ReferrerHeader": "string", + "RequestBodySize": "long", + "RequestHeaderSize": "long", + "RequestMd5": "string", + "RequesterAppId": "string", + "RequesterAudience": "string", + "RequesterObjectId": "string", + "RequesterTenantId": "string", + "RequesterTokenIssuer": "string", + "RequesterUpn": "string", + "ResponseBodySize": "long", + "ResponseHeaderSize": "long", + "ResponseMd5": "string", + "SasExpiryStatus": "string", + "SchemaVersion": "string", + "ServerLatencyMs": "real", + "ServiceType": "string", + "SourceSystem": "string", + "StatusCode": "string", + "StatusText": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TlsVersion": "string", + "Type": "string", + "Uri": "string", + "UserAgentHeader": "string", + "_ResourceId": "string" + }, + "Syslog": { + "CollectorHostName": "string", + "Computer": "string", + "EventTime": "datetime", + "Facility": "string", + "HostIP": "string", + "HostName": "string", + "MG": "string", + "ProcessID": "int", + "ProcessName": "string", + "SeverityLevel": "string", + "SourceSystem": "string", + "SyslogMessage": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "ThreatIntelExportOperation": { + "DestinationInfo": "dynamic", + "ErrorDetails": "string", + "ExportDuration": "int", + "ExportId": "string", + "ExportType": "string", + "ExportedBy": "string", + "ExternallyExportedId": "string", + "Id": "string", + "SourceSystem": "string", + "Status": "string", + "StixId": "string", + "TenantId": "string", + "TimeExported": "datetime", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "ThreatIntelIndicators": { + "AdditionalFields": "dynamic", + "AzureTenantId": "string", + "Confidence": "int", + "Created": "datetime", + "Data": "dynamic", + "Id": "string", + "IsActive": "bool", + "IsDeleted": "bool", + "LastUpdateMethod": "string", + "Modified": "datetime", + "ObservableKey": "string", + "ObservableValue": "string", + "Pattern": "string", + "Revoked": "bool", + "SourceSystem": "string", + "Tags": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "ValidFrom": "datetime", + "ValidUntil": "datetime", + "WorkspaceId": "string", + "_ResourceId": "string" + }, + "ThreatIntelObjects": { + "AdditionalFields": "dynamic", + "AzureTenantId": "string", + "Data": "dynamic", + "Id": "string", + "IsDeleted": "bool", + "LastUpdateMethod": "string", + "SourceSystem": "string", + "StixType": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "WorkspaceId": "string", + "_ResourceId": "string" + }, + "ThreatIntelligenceIndicator": { + "Action": "string", + "Active": "bool", + "ActivityGroupNames": "string", + "AdditionalInformation": "string", + "ApplicationId": "string", + "AzureTenantId": "string", + "ConfidenceScore": "real", + "Description": "string", + "DiamondModel": "string", + "DomainName": "string", + "EmailEncoding": "string", + "EmailLanguage": "string", + "EmailRecipient": "string", + "EmailSenderAddress": "string", + "EmailSenderName": "string", + "EmailSourceDomain": "string", + "EmailSourceIpAddress": "string", + "EmailSubject": "string", + "EmailXMailer": "string", + "ExpirationDateTime": "datetime", + "ExternalIndicatorId": "string", + "FileCompileDateTime": "datetime", + "FileCreatedDateTime": "datetime", + "FileHashType": "string", + "FileHashValue": "string", + "FileMutexName": "string", + "FileName": "string", + "FilePacker": "string", + "FilePath": "string", + "FileSize": "int", + "FileType": "string", + "IndicatorId": "string", + "IndicatorProvider": "string", + "KillChainActions": "bool", + "KillChainC2": "bool", + "KillChainDelivery": "bool", + "KillChainExploitation": "bool", + "KillChainReconnaissance": "bool", + "KillChainWeaponization": "bool", + "KnownFalsePositives": "string", + "MalwareNames": "string", + "NetworkCidrBlock": "string", + "NetworkDestinationAsn": "int", + "NetworkDestinationCidrBlock": "string", + "NetworkDestinationIP": "string", + "NetworkDestinationPort": "int", + "NetworkIP": "string", + "NetworkPort": "int", + "NetworkProtocol": "int", + "NetworkSourceAsn": "int", + "NetworkSourceCidrBlock": "string", + "NetworkSourceIP": "string", + "NetworkSourcePort": "int", + "PassiveOnly": "bool", + "SourceSystem": "string", + "Tags": "string", + "TenantId": "string", + "ThreatSeverity": "int", + "ThreatType": "string", + "TimeGenerated": "datetime", + "TrafficLightProtocolLevel": "string", + "Type": "string", + "Url": "string", + "UserAgent": "string" + }, + "UrlClickEvents": { + "AccountUpn": "string", + "ActionType": "string", + "AppName": "string", + "AppVersion": "string", + "DetectionMethods": "string", + "IPAddress": "string", + "IsClickedThrough": "bool", + "NetworkMessageId": "string", + "ReportId": "string", + "SourceId": "string", + "SourceSystem": "string", + "TenantId": "string", + "ThreatTypes": "string", + "TimeGenerated": "datetime", + "Timestamp": "datetime", + "Type": "string", + "Url": "string", + "UrlChain": "string", + "Workload": "string" + }, + "Usage": { + "AvgLatencyInSeconds": "real", + "BatchesCapped": "long", + "BatchesOutsideSla": "long", + "BatchesWithinSla": "long", + "Computer": "string", + "DataType": "string", + "EndTime": "datetime", + "IsBillable": "bool", + "LinkedMeterId": "string", + "LinkedResourceUri": "string", + "MeterId": "string", + "Plan": "string", + "Quantity": "real", + "QuantityUnit": "string", + "ResourceUri": "string", + "Solution": "string", + "SourceSystem": "string", + "StartTime": "datetime", + "TenantId": "string", + "TimeGenerated": "datetime", + "TotalBatches": "long", + "Type": "string" + }, + "VMConnection": { + "AdditionalInformation": "string", + "AgentId": "string", + "BytesReceived": "long", + "BytesSent": "long", + "Computer": "string", + "Confidence": "string", + "ConnectionId": "string", + "Description": "string", + "DestinationIp": "string", + "DestinationPort": "int", + "Direction": "string", + "FirstReportedDateTime": "string", + "IndicatorThreatType": "string", + "IsActive": "string", + "LastReportedDateTime": "string", + "LinksEstablished": "long", + "LinksFailed": "long", + "LinksLive": "long", + "LinksTerminated": "long", + "Machine": "string", + "MaliciousIp": "string", + "Process": "string", + "ProcessName": "string", + "Protocol": "string", + "RemoteClassification": "string", + "RemoteCountry": "string", + "RemoteDnsCanonicalNames": "string", + "RemoteDnsQuestions": "string", + "RemoteIp": "string", + "RemoteLatitude": "real", + "RemoteLongitude": "real", + "ReportReferenceLink": "string", + "ResponseTimeMax": "long", + "ResponseTimeMin": "long", + "ResponseTimeSum": "long", + "Responses": "long", + "Severity": "int", + "SourceIp": "string", + "SourceSystem": "string", + "TLPLevel": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "_ResourceId": "string" + }, + "W3CIISLog": { + "AdditionalInformation": "string", + "AzureDeploymentID": "string", + "Computer": "string", + "Confidence": "string", + "Date": "string", + "Description": "string", + "FileOffset": "int", + "FileUri": "string", + "FirstReportedDateTime": "string", + "IndicatorThreatType": "string", + "IsActive": "string", + "LastReportedDateTime": "string", + "MG": "string", + "MaliciousIP": "string", + "ManagementGroupName": "string", + "RemoteIPCountry": "string", + "RemoteIPLatitude": "real", + "RemoteIPLongitude": "real", + "ReportReferenceLink": "string", + "Role": "string", + "RoleInstance": "string", + "Severity": "int", + "SourceSystem": "string", + "StorageAccount": "string", + "TLPLevel": "string", + "TenantId": "string", + "Time": "string", + "TimeGenerated": "datetime", + "TimeTaken": "long", + "Type": "string", + "_ResourceId": "string", + "cIP": "string", + "csBytes": "long", + "csCookie": "string", + "csHost": "string", + "csMethod": "string", + "csReferer": "string", + "csUriQuery": "string", + "csUriStem": "string", + "csUserAgent": "string", + "csUserName": "string", + "csVersion": "string", + "sComputerName": "string", + "sIP": "string", + "sPort": "int", + "sSiteName": "string", + "scBytes": "long", + "scStatus": "string", + "scSubStatus": "string", + "scWin32Status": "string" + }, + "Watchlist": { + "AzureTenantId": "string", + "CorrelationId": "string", + "CreatedBy": "dynamic", + "CreatedTimeUTC": "datetime", + "DefaultDuration": "string", + "EntityMapping": "dynamic", + "LastUpdatedTimeUTC": "datetime", + "Notes": "string", + "Provider": "string", + "SearchKey": "string", + "Source": "string", + "SourceSystem": "string", + "Tags": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "TimeToLive": "datetime", + "Type": "string", + "UpdatedBy": "dynamic", + "WatchlistAlias": "string", + "WatchlistCategory": "string", + "WatchlistId": "string", + "WatchlistItem": "dynamic", + "WatchlistItemId": "string", + "WatchlistName": "string", + "_DTItemId": "string", + "_DTItemStatus": "string", + "_DTItemType": "string", + "_DTTimestamp": "datetime" + }, + "WindowsEvent": { + "Channel": "string", + "Computer": "string", + "Correlation": "string", + "Data": "dynamic", + "EventData": "dynamic", + "EventID": "int", + "EventLevel": "int", + "EventLevelName": "string", + "EventOriginId": "string", + "EventRecordId": "string", + "Keywords": "string", + "ManagementGroupName": "string", + "Opcode": "string", + "Provider": "string", + "RawEventData": "string", + "SourceSystem": "string", + "SystemProcessId": "int", + "SystemThreadId": "int", + "SystemUserId": "string", + "Task": "int", + "TenantId": "string", + "TimeCreated": "datetime", + "TimeGenerated": "datetime", + "Type": "string", + "Version": "int", + "_ResourceId": "string" + } +} diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/mapping_catalog.json b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/mapping_catalog.json new file mode 100644 index 00000000000..d93db7b9cb8 --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/mapping_catalog.json @@ -0,0 +1,1177 @@ +{ + "version": "1.0.0", + "_source": "Versioned offline mapping snapshot; candidate mappings require selected-environment query validation.", + "global": { + "columnRenames": {}, + "dropColumns": [ + "MG", + "SourceSystem", + "TenantId", + "Type", + "_BilledSize", + "_IsBillable", + "_ItemId", + "_ResourceId", + "_SubscriptionId", + "_TimeReceived" + ] + }, + "tables": { + "AADManagedIdentitySignInLogs": { + "ahTable": "AADManagedIdentitySignInLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "recommendAhTable": "EntraIdSpnSignInEvents", + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "AADNonInteractiveUserSignInLogs": { + "ahTable": "AADNonInteractiveUserSignInLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "recommendAhTable": "EntraIdSignInEvents", + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "AADProvisioningLogs": { + "ahTable": null, + "tier": "blocked", + "blockedReason": "No confirmed AH equivalent.", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null + }, + "AADRiskyUsers": { + "ahTable": null, + "tier": "blocked", + "blockedReason": "No confirmed AH state-table equivalent.", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null + }, + "AADServicePrincipalSignInLogs": { + "ahTable": "AADServicePrincipalSignInLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "recommendAhTable": "EntraIdSpnSignInEvents", + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "AADUserRiskEvents": { + "ahTable": null, + "tier": "blocked", + "blockedReason": "No confirmed AH event-table equivalent.", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null + }, + "ADFSSignInLogs": { + "ahTable": null, + "tier": "blocked", + "blockedReason": "No confirmed AH equivalent.", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null + }, + "ASimAgentEventLogs": { + "ahTable": "ASimAgentEventLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "ASimAlertEventLogs": { + "ahTable": "ASimAlertEventLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "ASimAssetEntityLogs": { + "ahTable": "ASimAssetEntityLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "ASimAuditEventLogs": { + "ahTable": "ASimAuditEventLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "ASimAuthenticationEventLogs": { + "ahTable": "ASimAuthenticationEventLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "ASimDhcpEventLogs": { + "ahTable": "ASimDhcpEventLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "ASimDnsActivityLogs": { + "ahTable": "ASimDnsActivityLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "ASimFileEventLogs": { + "ahTable": "ASimFileEventLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "ASimNetworkSessionLogs": { + "ahTable": "ASimNetworkSessionLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "ASimProcessEventLogs": { + "ahTable": "ASimProcessEventLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "ASimRegistryEventLogs": { + "ahTable": "ASimRegistryEventLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "ASimUserManagementActivityLogs": { + "ahTable": "ASimUserManagementActivityLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "ASimWebSessionLogs": { + "ahTable": "ASimWebSessionLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "AWSCloudTrail": { + "ahTable": "AWSCloudTrail", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "AWSGuardDuty": { + "ahTable": "AWSGuardDuty", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "AWSVPCFlow": { + "ahTable": "AWSVPCFlow", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "AWSWAF": { + "ahTable": "AWSWAF", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "AZFWApplicationRule": { + "ahTable": "AZFWApplicationRule", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "AZFWDnsQuery": { + "ahTable": "AZFWDnsQuery", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "AZFWIdpsSignature": { + "ahTable": "AZFWIdpsSignature", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "AZFWNatRule": { + "ahTable": "AZFWNatRule", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "AZFWNetworkRule": { + "ahTable": "AZFWNetworkRule", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "AZFWThreatIntel": { + "ahTable": "AZFWThreatIntel", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "Alert": { + "ahTable": "Alert", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "AlertEvidence": { + "ahTable": "AlertEvidence", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "castDynamicColumns": [ + "AdditionalFields" + ], + "removedColumns": [], + "intColumns": [], + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "AlertInfo": { + "ahTable": "AlertInfo", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "castDynamicColumns": [], + "removedColumns": [], + "intColumns": [], + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "Anomalies": { + "ahTable": "Anomalies", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "AuditLogs": { + "ahTable": "AuditLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "recommendAhTable": "EntraIdDirectoryAuditEvents", + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "AzureActivity": { + "ahTable": "AzureActivity", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "AzureDiagnostics": { + "ahTable": "AzureDiagnostics", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "AzureMetrics": { + "ahTable": "AzureMetrics", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "CloudAppEvents": { + "ahTable": "CloudAppEvents", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "castDynamicColumns": [], + "removedColumns": [], + "intColumns": [], + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "CommonSecurityLog": { + "ahTable": "CommonSecurityLog", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "ConfidentialWatchlist": { + "ahTable": "ConfidentialWatchlist", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "ContainerInventory": { + "ahTable": "ContainerInventory", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "DeviceBehaviorEntities": { + "ahTable": "DeviceBehaviorEntities", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "DeviceBehaviorInfo": { + "ahTable": "DeviceBehaviorInfo", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "DeviceCustomFileEvents": { + "ahTable": "DeviceCustomFileEvents", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "DeviceCustomImageLoadEvents": { + "ahTable": "DeviceCustomImageLoadEvents", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "DeviceCustomNetworkEvents": { + "ahTable": "DeviceCustomNetworkEvents", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "DeviceCustomProcessEvents": { + "ahTable": "DeviceCustomProcessEvents", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "DeviceCustomRegistryEvents": { + "ahTable": "DeviceCustomRegistryEvents", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "DeviceCustomScriptEvents": { + "ahTable": "DeviceCustomScriptEvents", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "DeviceEvents": { + "ahTable": "DeviceEvents", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "castDynamicColumns": [ + "AdditionalFields" + ], + "removedColumns": [], + "intColumns": [], + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "DeviceFileCertificateInfo": { + "ahTable": "DeviceFileCertificateInfo", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "castDynamicColumns": [], + "removedColumns": [], + "intColumns": [], + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "DeviceFileEvents": { + "ahTable": "DeviceFileEvents", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "castDynamicColumns": [ + "AdditionalFields" + ], + "removedColumns": [], + "intColumns": [], + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "DeviceImageLoadEvents": { + "ahTable": "DeviceImageLoadEvents", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "castDynamicColumns": [], + "removedColumns": [], + "intColumns": [], + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "DeviceInfo": { + "ahTable": "DeviceInfo", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "castDynamicColumns": [ + "AdditionalFields", + "LoggedOnUsers" + ], + "removedColumns": [ + "DeviceObjectId" + ], + "intColumns": [], + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "DeviceLogonEvents": { + "ahTable": "DeviceLogonEvents", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "castDynamicColumns": [ + "AdditionalFields" + ], + "removedColumns": [], + "intColumns": [], + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "DeviceNetworkEvents": { + "ahTable": "DeviceNetworkEvents", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "castDynamicColumns": [ + "AdditionalFields" + ], + "removedColumns": [], + "intColumns": [], + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "DeviceNetworkInfo": { + "ahTable": "DeviceNetworkInfo", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "castDynamicColumns": [ + "IPAddresses", + "DnsAddresses", + "DefaultGateways", + "ConnectedNetworks" + ], + "removedColumns": [], + "intColumns": [], + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "DeviceProcessEvents": { + "ahTable": "DeviceProcessEvents", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "castDynamicColumns": [ + "AdditionalFields" + ], + "removedColumns": [], + "intColumns": [], + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "DeviceRegistryEvents": { + "ahTable": "DeviceRegistryEvents", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "castDynamicColumns": [], + "removedColumns": [], + "intColumns": [], + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "DeviceTvmSecureConfigurationAssessment": { + "ahTable": "DeviceTvmSecureConfigurationAssessment", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "DeviceTvmSecureConfigurationAssessmentKB": { + "ahTable": "DeviceTvmSecureConfigurationAssessmentKB", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "DeviceTvmSoftwareInventory": { + "ahTable": "DeviceTvmSoftwareInventory", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "DeviceTvmSoftwareVulnerabilities": { + "ahTable": "DeviceTvmSoftwareVulnerabilities", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "DeviceTvmSoftwareVulnerabilitiesKB": { + "ahTable": "DeviceTvmSoftwareVulnerabilitiesKB", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "DnsEvents": { + "ahTable": "DnsEvents", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "EmailAttachmentInfo": { + "ahTable": "EmailAttachmentInfo", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "castDynamicColumns": [], + "removedColumns": [], + "intColumns": [], + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "EmailEvents": { + "ahTable": "EmailEvents", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "castDynamicColumns": [ + "AdditionalFields" + ], + "removedColumns": [], + "intColumns": [], + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "EmailPostDeliveryEvents": { + "ahTable": "EmailPostDeliveryEvents", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "castDynamicColumns": [], + "removedColumns": [], + "intColumns": [], + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "EmailUrlInfo": { + "ahTable": "EmailUrlInfo", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "castDynamicColumns": [], + "removedColumns": [], + "intColumns": [], + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "Event": { + "ahTable": "Event", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "FileMaliciousContentInfo": { + "ahTable": "FileMaliciousContentInfo", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "GCPDNS": { + "ahTable": "GCPDNS", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "Heartbeat": { + "ahTable": "Heartbeat", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "IdentityDirectoryEvents": { + "ahTable": "IdentityDirectoryEvents", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "castDynamicColumns": [ + "AdditionalFields" + ], + "removedColumns": [], + "intColumns": [ + "Port", + "DestinationPort" + ], + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "IdentityInfo": { + "ahTable": "IdentityInfo", + "tier": "1to1", + "columnRenames": { + "AccountUPN": "AccountUpn", + "AccountCreationTime": "CreatedDateTime", + "UserType": "TenantMembershipType" + }, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "IdentityLogonEvents": { + "ahTable": "IdentityLogonEvents", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "castDynamicColumns": [ + "AdditionalFields" + ], + "removedColumns": [], + "intColumns": [ + "Port", + "DestinationPort" + ], + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "IdentityQueryEvents": { + "ahTable": "IdentityQueryEvents", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "castDynamicColumns": [ + "AdditionalFields" + ], + "removedColumns": [], + "intColumns": [ + "Port", + "DestinationPort" + ], + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "InsightsMetrics": { + "ahTable": "InsightsMetrics", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "MicrosoftGraphActivityLogs": { + "ahTable": "MicrosoftGraphActivityLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "recommendAhTable": "GraphAPIAuditEvents", + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "NTANetAnalytics": { + "ahTable": "NTANetAnalytics", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "NetworkAccessAlerts": { + "ahTable": "NetworkAccessAlerts", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "NetworkAccessConnectionEvents": { + "ahTable": "NetworkAccessConnectionEvents", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "NetworkAccessGenerativeAIInsights": { + "ahTable": "NetworkAccessGenerativeAIInsights", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "NetworkAccessTraffic": { + "ahTable": "NetworkAccessTraffic", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "NetworkSessions": { + "ahTable": "NetworkSessions", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "OfficeActivity": { + "ahTable": "OfficeActivity", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "Operation": { + "ahTable": "Operation", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "Perf": { + "ahTable": "Perf", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "PowerPlatformAdminActivity": { + "ahTable": "PowerPlatformAdminActivity", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "RemoteNetworkHealthLogs": { + "ahTable": "RemoteNetworkHealthLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "RiskyUsers": { + "ahTable": null, + "tier": "blocked", + "blockedReason": "No confirmed AH state-table equivalent.", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null + }, + "SecurityAlert": { + "ahTable": "SecurityAlert", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "SecurityEvent": { + "ahTable": "SecurityEvent", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "SecurityIncident": { + "ahTable": "SecurityIncident", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "SentinelAlibabaCloudAPIGatewayLogs": { + "ahTable": "SentinelAlibabaCloudAPIGatewayLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "SentinelAlibabaCloudVPCFlowLogs": { + "ahTable": "SentinelAlibabaCloudVPCFlowLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "SentinelAlibabaCloudWAFLogs": { + "ahTable": "SentinelAlibabaCloudWAFLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "SentinelAudit": { + "ahTable": "SentinelAudit", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "SentinelBehaviorEntities": { + "ahTable": "SentinelBehaviorEntities", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "SentinelBehaviorInfo": { + "ahTable": "SentinelBehaviorInfo", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "SentinelHealth": { + "ahTable": "SentinelHealth", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "SentinelImpervaWAFCloudV2Logs": { + "ahTable": "SentinelImpervaWAFCloudV2Logs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "SigninLogs": { + "ahTable": "SigninLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "recommendAhTable": "EntraIdSignInEvents", + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "StorageBlobLogs": { + "ahTable": "StorageBlobLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "StorageFileLogs": { + "ahTable": "StorageFileLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "StorageQueueLogs": { + "ahTable": "StorageQueueLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "StorageTableLogs": { + "ahTable": "StorageTableLogs", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "Syslog": { + "ahTable": "Syslog", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "ThreatIntelExportOperation": { + "ahTable": "ThreatIntelExportOperation", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "ThreatIntelIndicators": { + "ahTable": "ThreatIntelIndicators", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "ThreatIntelObjects": { + "ahTable": "ThreatIntelObjects", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "ThreatIntelligenceIndicator": { + "ahTable": "ThreatIntelligenceIndicator", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "UrlClickEvents": { + "ahTable": "UrlClickEvents", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "castDynamicColumns": [], + "removedColumns": [], + "intColumns": [], + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "Usage": { + "ahTable": "Usage", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "UserRiskEvents": { + "ahTable": null, + "tier": "blocked", + "blockedReason": "No confirmed AH event-table equivalent.", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null + }, + "VMConnection": { + "ahTable": "VMConnection", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "W3CIISLog": { + "ahTable": "W3CIISLog", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "Watchlist": { + "ahTable": "Watchlist", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, + "WindowsEvent": { + "ahTable": "WindowsEvent", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + } + } +} diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/xdr-workbook.schema.json b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/xdr-workbook.schema.json new file mode 100644 index 00000000000..3441a69c598 --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/xdr-workbook.schema.json @@ -0,0 +1,107 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Authored Defender XDR Workbook", + "description": "Complete Notebook with additive authoring identity and provenance. Not an installed deployment payload.", + "type": "object", + "required": ["version", "items", "xdrMetadata", "contentProvenance"], + "properties": { + "version": {"const": "Notebook/1.0"}, + "items": {"type": "array", "items": {"type": "object"}}, + "xdrMetadata": { + "type": "object", + "required": ["contentId", "instanceId", "version", "schemaVersion", "displayName"], + "properties": { + "contentId": {"$ref": "#/$defs/guid"}, + "instanceId": {"$ref": "#/$defs/guid"}, + "version": {"type": "string", "pattern": "^3\\.[1-9][0-9]*\\.(0|[1-9][0-9]*)$"}, + "schemaVersion": {"const": "1.0.0"}, + "displayName": {"type": "string", "minLength": 1} + }, + "additionalProperties": false + }, + "contentProvenance": { + "type": "object", + "required": ["source", "conversion", "requiredWorkloads", "requiredDataConnectors", "dependencyReviewRequired"], + "properties": { + "source": { + "type": "object", + "required": ["path", "sha256"], + "properties": { + "path": {"type": "string", "pattern": "^Workbooks/[^\\\\]+\\.[jJ][sS][oO][nN]$"}, + "sha256": {"$ref": "#/$defs/sha256"}, + "contentId": {"type": ["string", "null"]}, + "originalMetadataVersion": {"type": ["string", "null"]} + } + }, + "conversion": { + "type": "object", + "required": ["tool", "version", "reviewRequired", "queries"], + "properties": { + "tool": {"const": "sentinel-xdr-migration"}, + "version": {"type": "string", "minLength": 1}, + "reviewRequired": {"const": false}, + "queries": {"type": "array", "items": {"$ref": "#/$defs/query"}} + } + }, + "requiredWorkloads": { + "type": "array", + "items": { + "type": "object", + "required": ["workload", "dataTypes"], + "properties": { + "workload": {"enum": ["mde", "mdo", "mdi", "mda", "mdc", "entraid", "purview", "appgovernance", "sentinel", "mtp"]}, + "dataTypes": {"$ref": "#/$defs/dataTypes"} + } + } + }, + "requiredDataConnectors": { + "type": "array", + "items": { + "type": "object", + "required": ["connectorId", "dataTypes"], + "properties": { + "connectorId": {"type": "string", "minLength": 1}, + "dataTypes": {"$ref": "#/$defs/dataTypes"} + } + } + }, + "requiredWorkspaceFunctions": {"type": "array", "items": {"$ref": "#/$defs/workspaceFunction"}}, + "dependencyReviewRequired": {"const": false} + }, + "additionalProperties": false + } + }, + "$defs": { + "workspaceFunction": { + "type": "object", + "required": ["functionAlias", "sourcePath", "sourceSha256"], + "properties": { + "functionAlias": {"type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]*$"}, + "sourcePath": {"type": "string", "pattern": "^(Parsers|Workspace Functions)/(?!/|\\.{1,2}/)(?!.*(?:/\\.{1,2}/|//))[^\\\\]+\\.yaml$"}, + "sourceSha256": {"$ref": "#/$defs/sha256"} + }, + "additionalProperties": false + }, + "guid": {"type": "string", "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"}, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "dataTypes": {"type": "array", "items": {"type": "string", "minLength": 1, "not": {"const": "Watchlist"}}}, + "query": { + "type": "object", + "required": ["sourcePath", "outputPath", "sourceSha256", "targetSha256", "status"], + "properties": { + "sourcePath": {"type": "string", "pattern": "^/"}, + "outputPath": {"type": ["string", "null"]}, + "sourceSha256": {"$ref": "#/$defs/sha256"}, + "targetSha256": {"anyOf": [{"$ref": "#/$defs/sha256"}, {"type": "null"}]}, + "status": {"enum": ["converted", "removed-scope-helper"]}, + "workspaceFunctions": {"type": "array", "uniqueItems": true, "items": {"type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]*$"}}, + "referenceInputs": {"type": "array", "uniqueItems": true, "items": {"const": "Watchlist"}} + }, + "allOf": [{ + "if": {"properties": {"status": {"const": "removed-scope-helper"}}}, + "then": {"properties": {"outputPath": {"type": "null"}, "targetSha256": {"type": "null"}}}, + "else": {"properties": {"outputPath": {"type": "string", "pattern": "^/"}, "targetSha256": {"$ref": "#/$defs/sha256"}}} + }] + } + } +} diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/dataframes.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/dataframes.py new file mode 100644 index 00000000000..5ae1b817048 --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/dataframes.py @@ -0,0 +1,152 @@ +"""Schema-driven `datatable()` "dataframe" stubs for tenant-independent unit tests. + +A converted query can only be validated against a live tenant if that tenant +actually *has* the referenced table. That makes validation tenant-dependent and +brittle: a table absent on the validation tenant fails to bind even when the +query is perfectly correct. + +This module removes that dependency. For any table we have a schema for, it emits + + let = datatable(, ...)[ ]; + +Because a `let` binding resolves **ahead of** any real table of the same name, +prepending this stub to a query lets it parse, bind, and run whether or not the +tenant has the table. An **empty** datatable (`[]`) binds and returns 0 rows - +a deterministic *schema/bind* unit test. Seeded rows (a later hook) turn it into +a *behavioral* test. + +Two flavours, from the two schema catalogs: + - **sentinel** side: `data/la_schema_catalog.json` (Log Analytics schema) - so + the *original* Sentinel query is self-contained and testable. + - **ah** side: `data/ah_schema_catalog.json` (Advanced Hunting schema) - so the + *converted* query is self-contained and testable. + +These stubs are **test scaffolding only**. They are never injected into the +delivered KQL - the shipped query stays byte-faithful. +""" +from __future__ import annotations + +import re +from pathlib import Path +from typing import Dict, List, Optional, Sequence + +ROOT = Path(__file__).resolve().parent +from . import mapping_catalog, migrate + +_LA_CACHE = ROOT / "data" / "la_schema_catalog.json" +_AH_CACHE = ROOT / "data" / "ah_schema_catalog.json" + +# KQL scalar types valid inside `datatable(...)`. Anything else falls back to +# `string` (a stub never needs to be type-perfect - it must only bind). +_VALID_DATATABLE_TYPES = { + "bool", "boolean", "datetime", "date", "decimal", "dynamic", "guid", + "int", "long", "real", "double", "string", "timespan", "time", +} +_TYPE_ALIASES = { + "boolean": "bool", "date": "datetime", "double": "real", "time": "timespan", +} + +_IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +def _norm_type(kql_type: Optional[str]) -> str: + t = (kql_type or "").strip().lower() + t = _TYPE_ALIASES.get(t, t) + return t if t in _VALID_DATATABLE_TYPES else "string" + + +def _col_name(name: str) -> str: + """Bare identifier when possible, else a bracketed/quoted name so odd column + names (spaces, punctuation) still form a valid `datatable` schema.""" + return name if _IDENT_RE.match(name) else '["%s"]' % name.replace('"', '\\"') + + +# --- schema lookup ----------------------------------------------------------- +def _la_cached() -> Dict[str, Dict[str, str]]: + return migrate._load(_LA_CACHE) + + +def _ah_cached() -> Dict[str, Dict[str, str]]: + return migrate._load(_AH_CACHE) + + +def _ah_table_name(sentinel_table: str) -> str: + """AH table name for a Sentinel table (identical for Wave-1 XDR tables; + catalog-mapped for renamed Wave-2 tables).""" + meta = mapping_catalog.load_mapping_catalog()["tables"].get(sentinel_table, {}) + return meta.get("ahTable") or sentinel_table + + +def schema_for(table: str, side: str = "ah") -> Optional[Dict[str, str]]: + """Return `{column: kqlType}` for `table` on the given side, or None if the + schema isn't cached. For the AH side, `table` may be either the Sentinel name + (it is mapped to its AH counterpart) or the AH name directly.""" + if side == "sentinel": + return _la_cached().get(table) + if side == "ah": + ah = _ah_cached() + return ah.get(table) or ah.get(_ah_table_name(table)) + raise ValueError(f"side must be 'sentinel' or 'ah', got {side!r}") + + +# --- stub construction ------------------------------------------------------- +def datatable_stub(table: str, schema: Dict[str, str], + rows: Optional[Sequence[Sequence[object]]] = None, + let_name: Optional[str] = None) -> str: + """Build `let
= datatable()[ ];` from a `{col: type}` + schema. `rows` is an optional list of value-tuples (behavioral hook); omit + for an empty schema/bind stub. Column order follows the schema dict order.""" + if not schema: + raise ValueError(f"empty schema for {table!r}; cannot build a stub") + name = let_name or table + cols = ", ".join(f"{_col_name(c)}:{_norm_type(t)}" for c, t in schema.items()) + body = _format_rows(schema, rows) if rows else "" + return f"let {name} = datatable({cols})[{body}];" + + +def _format_rows(schema: Dict[str, str], + rows: Sequence[Sequence[object]]) -> str: + types = [_norm_type(t) for t in schema.values()] + flat: List[str] = [] + for row in rows: + if len(row) != len(types): + raise ValueError( + f"row has {len(row)} values but schema has {len(types)} columns") + flat.extend(_literal(v, ty) for v, ty in zip(row, types)) + return "\n " + ",\n ".join( + ", ".join(flat[i:i + len(types)]) for i in range(0, len(flat), len(types)) + ) + "\n" if flat else "" + + +def _literal(value: object, kql_type: str) -> str: + if value is None: + return {"string": '""', "dynamic": "dynamic(null)"}.get( + kql_type, f"{kql_type}(null)") + if kql_type in ("int", "long", "real", "decimal"): + return str(value) + if kql_type == "bool": + return "true" if value else "false" + if kql_type == "datetime": + return f"datetime({value})" + if kql_type == "timespan": + return str(value) + if kql_type == "dynamic": + import json + return f"dynamic({json.dumps(value)})" + s = str(value).replace("\\", "\\\\").replace('"', '\\"') + return f'"{s}"' + + +def stub_for(table: str, side: str = "ah", + rows: Optional[Sequence[Sequence[object]]] = None) -> Optional[str]: + """`datatable` stub for `table` on `side`, or None if no schema is cached. + On the AH side the stub's `let` name is the AH table name (what the converted + query references).""" + schema = schema_for(table, side) + if not schema: + return None + let_name = _ah_table_name(table) if side == "ah" else table + return datatable_stub(table, schema, rows=rows, let_name=let_name) + + +# --- referenced-table detection ---------------------------------------------- diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/dynamic_casts.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/dynamic_casts.py new file mode 100644 index 00000000000..efe8b9020a6 --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/dynamic_casts.py @@ -0,0 +1,143 @@ +"""Wave-1 XDR table column gotchas: dynamic-as-string casts, a removed +column, and a string->int type change (mcp/rewrite.py, step 3 extension). + +Source: docs/Sentinel-to-XDR-KQL-Migration-V2 1.md - a per-table playbook +backed by live schema (`getschema`) and per-record value comparison on both +platforms. Three classes of gotcha, none of which the Judge's static/runtime +bind check catches (the query still binds and "passes" - it just silently +returns 0/wrong rows): + + 1. Dynamic columns (e.g. `AdditionalFields`, `LoggedOnUsers`) are delivered + as JSON **text** in Advanced Hunting. `Col.Field` member access or + `mv-expand Col` on an un-cast column fails **silently** - this module + auto-injects the documented fix (`| extend Col = todynamic(tostring(Col))`) + right after the table's first reference, deterministic and safe. + 2. `Port`/`DestinationPort` changed type string->int on the three Identity* + tables - a quoted numeric-literal comparison (`Port == "445"`) silently + returns nothing. This module auto-unquotes it (mechanical, safe: only + touches a quoted *numeric* literal). + 3. `DeviceInfo.DeviceObjectId` was removed in AH (replaced by `DeviceId`). + There's no safe mechanical rewrite for this (the query's logic may + depend on it), so it is only *flagged* for manual review, never + auto-fixed - consistent with how `rewrite.py` treats other LA-only + columns. + +Always-on (unlike bronze.py's opt-in Bronze-tier handling): these gaps are a +fixed property of the AH schema (confirmed live on the target tenant), not +tenant-dependent. +""" +from __future__ import annotations + +import re +from typing import Dict, List, Tuple + +__all__ = ["inject_dynamic_casts", "unquote_int_literals", "flag_removed_columns"] + + +def _split_statements(kql: str) -> List[str]: + return kql.split(";") + + +def _references_table(stmt: str, table: str) -> bool: + t = re.escape(table) + return bool(re.search(rf"\b{t}\b", stmt) + or re.search(rf"\[\s*[\"']{t}[\"']\s*\]", stmt)) + + +def _inject_after_table(stmt: str, table: str, clause: str) -> "str | None": + """Insert `clause` immediately after the first reference to `table` in + `stmt` (mirrors bronze.py's `_inject_after_table`).""" + m = re.search(rf"\b{re.escape(table)}\b", stmt) + if not m: + m = re.search(rf"\[\s*[\"']{re.escape(table)}[\"']\s*\]", stmt) + if not m: + return None + end = m.end() + trailing = stmt[end:] + ws = re.match(r"[ \t]*\r?\n[ \t]*", trailing) + sep = ws.group(0) if ws else "\n" + return stmt[:end] + sep + clause + " " + trailing.lstrip(" \t") + + +def _needs_cast(stmt: str, col: str) -> bool: + """True if `stmt` accesses `col.` or `mv-expand col` without an + existing `todynamic(tostring(col))` cast already present.""" + if re.search(rf"todynamic\s*\(\s*tostring\s*\(\s*{re.escape(col)}\s*\)\s*\)", stmt): + return False + return bool(re.search(rf"\b{re.escape(col)}\s*\.\s*\w", stmt) + or re.search(rf"mv-expand\s+{re.escape(col)}\b", stmt, re.I)) + + +def inject_dynamic_casts(kql: str, tables: Dict[str, List[str]]) -> Tuple[str, List[dict]]: + """For every `{table: [castColumns]}` pair, inject a `todynamic(tostring())` + cast right after the table's first reference in any statement that accesses + the column without one already present. Returns (new_kql, diff_entries).""" + if not tables: + return kql, [] + stmts = _split_statements(kql) + diffs: List[dict] = [] + for i, stmt in enumerate(stmts): + for table, cols in tables.items(): + if not cols or not _references_table(stmt, table): + continue + for col in cols: + if not _needs_cast(stmt, col): + continue + clause = f"| extend {col} = todynamic(tostring({col}))" + new_stmt = _inject_after_table(stmt, table, clause) + if new_stmt and new_stmt != stmt: + stmts[i] = new_stmt + stmt = new_stmt + diffs.append({ + "kind": "dynamic-cast", "before": col, + "after": f"{col} = todynamic(tostring({col}))", + "count": 1, "source": "deterministic", + "reason": f"`{col}` on `{table}` is delivered as a JSON " + f"string in Advanced Hunting; member access / " + f"mv-expand without a cast fails silently " + f"(0 rows, no bind error).", + }) + return ";".join(stmts), diffs + + +def unquote_int_literals(kql: str, tables: Dict[str, List[str]]) -> Tuple[str, List[dict]]: + """For tables whose columns changed type string->int in AH (e.g. + `Port`/`DestinationPort` on the Identity* tables), strip quotes off a + quoted *numeric*-literal comparison (`Port == "445"` -> `Port == 445`). + Only touches a quoted numeric literal, so it's mechanical and safe.""" + cols = sorted({c for t_cols in tables.values() for c in t_cols}) + if not cols: + return kql, [] + pat = re.compile( + r'\b(' + "|".join(re.escape(c) for c in cols) + r')' + r'(\s*(?:==|!=)\s*)"(\d+)"') + diffs: List[dict] = [] + + def repl(m: "re.Match") -> str: + col, op, num = m.group(1), m.group(2), m.group(3) + diffs.append({ + "kind": "int-literal-unquote", "before": f'{col}{op}"{num}"', + "after": f'{col}{op}{num}', "count": 1, "source": "deterministic", + "reason": f"`{col}` changed type string->int in Advanced Hunting; " + f"a quoted literal comparison silently returns no rows.", + }) + return f"{col}{op}{num}" + + new_kql = pat.sub(repl, kql) + return new_kql, diffs + + +def flag_removed_columns(kql: str, tables: Dict[str, List[str]]) -> List[dict]: + """Flag (never auto-fix) a reference to a column removed in AH - the query + may depend on it, so only a human can decide the right replacement.""" + flags: List[dict] = [] + for table, cols in tables.items(): + for col in cols: + if re.search(rf"\b{re.escape(col)}\b", kql): + flags.append({ + "kind": "removed-column", "name": col, + "reason": f"`{col}` was removed on `{table}` in Advanced " + f"Hunting; use `DeviceId` as the device key " + f"instead. Needs manual handling.", + }) + return flags diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/import-manifest.json b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/import-manifest.json new file mode 100644 index 00000000000..d67aa76819c --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/import-manifest.json @@ -0,0 +1,108 @@ +{ + "license": "MIT", + "copyright": "Microsoft Corporation", + "components": [ + { + "component": "LICENSE", + "sha256": "7df20dcdf9197e9945c14858d41c60f11b52b93e5b69e2b63416b874d598d322" + }, + { + "component": "__init__.py", + "sha256": "bef8bdb3fa35d1da1d862eeb0b765e2f554d0b2bed817d51ad6daa16f62c9627" + }, + { + "component": "asim_functions.py", + "sha256": "154b4688a429c189563e91d3ce38ade4f23fc8ac64be9fb8b1bcb0c61bfeaeea", + "originalSha256": "bae2ba74e02672836c4343ccbfa26191d192f9e73028a80dc8094d2d6db231d4" + }, + { + "component": "data/ah_schema_catalog.json", + "sha256": "017703708836cb0749c3847c587e8cf8f3ad664dd70d33d80354ec9d04a1bcf0", + "originalSha256": "a5887f8e31d1aa11c3ab013bcd3295270be09fc7dfcf1fa02d986fd8f17cdfad" + }, + { + "component": "data/la_schema_catalog.json", + "sha256": "0bc72940ef1926213409e40e3ad858853aaf1768f62ad100fb81261964b84aae", + "originalSha256": "eaceada89eb132199ea71b8efebba932d0a8379c59f9846f5a15a46b09969399" + }, + { + "component": "data/mapping_catalog.json", + "sha256": "5a76b154eb5a077219e900392eefce12f8a2e8d18c49e723441c1c4c9d3660c7", + "originalSha256": "6a3c2c8a164923ec461208f33919518e36e34242106a3a8826650c8fcde60067" + }, + { + "component": "data/xdr-workbook.schema.json", + "sha256": "32735770ee130a61cf0841adecf72a72c32ad10d808c425583642fbbcd7b6c2e" + }, + { + "component": "dataframes.py", + "sha256": "207f5c146bc38a4875c210bdfd84a88186e326d13f56eb2546bc3dbef4454a13", + "originalSha256": "d2efbc214d070f97a8813303ee678697016c4a44949c85a12644a8ff78b2cb99" + }, + { + "component": "dynamic_casts.py", + "sha256": "56fc5044ee8f3eab4167eee3383bd109e334543cce618d2e3aab7144dff77167", + "originalSha256": "56fc5044ee8f3eab4167eee3383bd109e334543cce618d2e3aab7144dff77167" + }, + { + "component": "mapping_catalog.py", + "sha256": "d72353deabbec722d3c4e8709ea6c0a60db3736ef490b25d591e5f924b41b40d", + "originalSha256": "e10b1747d318883dfe47b2ccc510ee5cfe1677bb035a2547922f4f9372d7d622" + }, + { + "component": "migrate.py", + "sha256": "80087152df2ddf98fd3a5773aaceb22ba2271f57583d185f778919291b6ecb5e" + }, + { + "component": "migrator.py", + "sha256": "694a75d820016928c127eccb51b933dae8eb5dbd95b7b9840a74dfe3ddb90c82", + "originalSha256": "4f7d9d0c126d283a1c92e7429e89c01085b08c239e7dce66b8953482b0a5e92f" + }, + { + "component": "recommend.py", + "sha256": "b7d98c9350ac86b6cc018b1df82bfa98f4b5cef4f15900ce36b9080f6a734c1d", + "originalSha256": "6251b817a28daafe0b6f8b2cab049cebb9693da88f06f251b5b98956dbb0f04e" + }, + { + "component": "rewrite.py", + "sha256": "a77ff6aaeca6c919d0ed002085f877514a47fd670b68d6d1ac2179b45640c8c2", + "originalSha256": "0c4138edf041221f0bdfc4ee00e4417d4c9d505ff8ffec7ec29d62409ef8732a" + }, + { + "component": "schema_map.py", + "sha256": "38fccdb325af19e1903fe9a206ccc07ff7b36102db53f3867f7238426cbbc3b7", + "originalSha256": "a299105355be6a6b80cf3367a954eed3482baf6d8da80291417bc4cdc7621f97" + }, + { + "component": "workbook.py", + "sha256": "283f707c82ac18ed686056177fdce4418c355a219ade1925b8cf87720b5ba4ae", + "originalSha256": "eb931b23333994a56e94d3c8975c7663db258b9b8032b0ff171b9e456e1b3942" + }, + { + "component": "workbook_fixtures.py", + "sha256": "d4352105474b531763a2f9895ca53dd1d230b11e3ade975152c10e067753eac1", + "originalSha256": "778d3c1d0a265838e6127b73ffcdc44dffd65ace843591720d64634618e77ffb" + }, + { + "component": "workbook_mock.py", + "sha256": "ad7f2e11c536573f3ffb50ef322d47b5e3bdf7a2a57ef2df4378e4f93ceb1e9a", + "originalSha256": "0a34d1fc20a97160f6a40268acf91c2a2ccc5cdc284029bd035f05a48cf0ca48" + }, + { + "component": "workbook_mock_recipes.py", + "sha256": "bba37a979fc333dc52b35fb1fbabfede42d703398409d3c88e8a965789da113c", + "originalSha256": "bba37a979fc333dc52b35fb1fbabfede42d703398409d3c88e8a965789da113c" + } + ], + "adaptations": [ + "Package-relative imports and local immutable snapshots", + "Removed CLI, live resolver/schema, automatic seed and packaging/UI routes", + "Errors preserved without silent catalog/schema fallback", + "Resolution evidence binds original raw wrapper/template strings", + "Catalog commentary neutralized; conversion mappings unchanged", + "Hash-reviewed Sentinel workspace functions preserved in workbooks with distinct source tables and runtime gates", + "_GetWatchlist recognized as built-in; Watchlist lineage recorded separately from telemetry prerequisites", + "Authored schema permits exact workspace-function prerequisites and per-query platform reference inputs", + "Externaldata is a source operator; union branch discovery remains explicitly heuristic and nonexhaustive" + ] +} diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/mapping_catalog.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/mapping_catalog.py new file mode 100644 index 00000000000..45e9b5e0a79 --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/mapping_catalog.py @@ -0,0 +1,14 @@ +"""Read the versioned package-local mapping snapshot without live resolution.""" +from functools import lru_cache +from importlib.resources import files +import json + + +@lru_cache(maxsize=1) +def load_mapping_catalog() -> dict: + return json.loads(files(__package__).joinpath("data/mapping_catalog.json").read_text("utf-8")) + + +def tier_of(table: str) -> str: + entry = load_mapping_catalog()["tables"].get(table) + return entry["tier"] if entry else "unknown" diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/migrate.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/migrate.py new file mode 100644 index 00000000000..d5917c8b5ef --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/migrate.py @@ -0,0 +1,110 @@ +"""Pure shared schema compatibility helpers.""" +from __future__ import annotations +import json +from pathlib import Path +from typing import Dict + +LA_SYSTEM_COLS = { + "TenantId", "Type", "SourceSystem", "_ResourceId", "_SubscriptionId", + "_ItemId", "_TimeReceived", "_BilledSize", "_IsBillable", "MG", +} + +LA_TO_AH_RENAME: dict[str, str] = {} + +_NUMERIC = {"int", "long", "real", "double", "decimal", "float"} + +def _type_relation(la: str, ah: str) -> str: + """Classify how an LA column type relates to its AH counterpart. + + exact - same type. + compat - functionally identical for migration (no code change needed): + * dynamic <-> string (AH getschema reports JSON `dynamic` cols as + `string`; you just parse_json() them, same as in LA). + * any numeric <-> any numeric (int/long/real...). + * guid <-> string, bool <-> boolean. + soft - a real but trivially castable change (e.g. string <-> int on Port); + the query needs a toint()/tostring(), but nothing is lost. + hard - genuinely incompatible types. + """ + a, b = (la or "").lower(), (ah or "").lower() + if a == b: + return "exact" + pair = {a, b} + if pair <= {"dynamic", "string"}: + return "compat" + if a in _NUMERIC and b in _NUMERIC: + return "compat" + if pair <= {"guid", "string"} or pair <= {"bool", "boolean"}: + return "compat" + if ("string" in pair and (a in _NUMERIC or b in _NUMERIC)): + return "soft" + return "hard" + +def _load(path: Path) -> Dict[str, Dict[str, str]]: + return json.loads(path.read_text(encoding="utf-8")) + +def diff_table(la: Dict[str, str], ah: Dict[str, str]) -> dict: + """Compare one table's LA schema against its AH schema. + + Each meaningful LA column is classified as matched / renamed / compatible + (trivial type diff) / soft (castable type diff) / typeMismatch (incompatible) + / missing (no AH counterpart - blocks that column's uses). + + Verdict: + Easy - every column matches or is trivially compatible. + Review - no missing/incompatible cols, but some need a simple cast. + Hard - at least one column is missing or has an incompatible type. + Score = fraction of meaningful LA columns usable in AH (soft counts as 0.5). + """ + matched, renamed, compatible, soft, type_mismatch, missing = [], [], [], [], [], [] + ah_lower = {k.lower(): (k, v) for k, v in ah.items()} + + meaningful = [c for c in la if c not in LA_SYSTEM_COLS] + for col in meaningful: + la_type = la[col] + target = LA_TO_AH_RENAME.get(col, col) + if target in ah: + ah_type = ah[target] + else: + _, ah_type = ah_lower.get(target.lower(), (None, None)) + if ah_type is None: + missing.append(col) + continue + + rel = _type_relation(la_type, ah_type) + entry = {"column": col, "la": la_type, "ah": ah_type} + if target != col: + entry["ahName"] = target + if rel == "exact": + (renamed if target != col else matched).append(entry if target != col else col) + elif rel == "compat": + compatible.append(entry) + elif rel == "soft": + soft.append(entry) + else: + type_mismatch.append(entry) + + considered = len(meaningful) or 1 + ok = len(matched) + len(renamed) + len(compatible) + 0.5 * len(soft) + score = round(ok / considered, 3) + extra = [c for c in ah if c not in la and c not in LA_TO_AH_RENAME.values()] + + if missing or type_mismatch: + verdict = "Hard" + elif soft: + verdict = "Review" + else: + verdict = "Easy" + return { + "verdict": verdict, + "score": score, + "matched": matched, + "renamed": renamed, + "compatible": compatible, # trivial type diffs (dynamic<->string, numeric widening) + "soft": soft, # castable type diffs (e.g. Port string<->int) + "typeMismatch": type_mismatch, # incompatible types -> blocks those uses + "missingInAh": missing, # LA columns with no AH counterpart -> blocks those uses + "extraInAh": extra, # AH-only columns (harmless for migration) + "laColumns": len(la), + "ahColumns": len(ah), + } diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/migrator.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/migrator.py new file mode 100644 index 00000000000..b7df50f9a3c --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/migrator.py @@ -0,0 +1,102 @@ +"""Migrator core - orchestrates resolve -> classify -> rewrite and emits the +Sentinel->AH conversion output contract (KQL-Conversion-Service-Spec.md §4). + +Pure, importable logic (no MCP runtime) so it is unit-testable. The MCP server +(`migrator_server.py`) is a thin wrapper over these functions. + +The Migrator fills everything in the contract EXCEPT `validation` - that is the +Judge MCP's job. `confidence` here is therefore *provisional* (pre-validation): +a clean deterministic 1:1 conversion is only high-confidence once the Judge's +static bind / runtime / parity pass. +""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Dict, List, Optional + +from . import mapping_catalog, recommend, rewrite, schema_map + +# Contract vocab: 1to1 | renamed | merged | blocked. `unknown` (table not in the +# catalog) collapses to `blocked` in the contract's mappingTier. +_CONTRACT_TIERS = {"1to1", "renamed", "merged", "blocked"} +# Provisional confidence per status, before the Judge validates. +_CONFIDENCE = {"converted": 0.7, "needs-review": 0.3, "blocked": 0.0} + + +def classify(tables: Optional[List[str]]) -> dict: + """Per-table tier + worst overall tier for a set of resolved tables.""" + cat = mapping_catalog.load_mapping_catalog()["tables"] + tiers = {t: (cat[t]["tier"] if t in cat else "unknown") for t in (tables or [])} + if not tiers: + overall = "unknown" + else: + overall = max(tiers.values(), key=lambda x: rewrite._TIER_RANK.get(x, 2)) + return {"tiers": tiers, "overallTier": overall} + + +def _status_of(tables: Optional[List[str]], res: dict) -> str: + if not tables: + return "needs-review" # unresolved data sources -> manual review + tier = res["overallTier"] + if tier in ("blocked", "unknown"): + return "blocked" + if res["deterministic"]: + return "converted" + return "needs-review" + + +def convert(kql: str, tables: Optional[List[str]] = None, + source_id: Optional[str] = None, artifact_type: str = "analyticRule", + resolve: bool = False) -> dict: + """Convert one Sentinel KQL query and return the output contract. + + tables - resolved data sources; pass them for offline/batch use. + resolve - legacy compatibility argument; True is explicitly unsupported + by this offline slice. + """ + resolve_error = None + if resolve: + raise ValueError("Live resolution is not supported by the offline converter.") + + res = rewrite.rewrite(kql, tables) + status = _status_of(tables, res) + tier = res["overallTier"] + mapping_tier = tier if tier in _CONTRACT_TIERS else "blocked" + + unsupported = [{"construct": f["name"], "kind": f["kind"], "reason": f["reason"]} + for f in res["flags"]] + if tables is None: + unsupported.append({ + "construct": "data-sources", "kind": "unresolved", + "reason": resolve_error or "Data sources could not be resolved " + "(ASIM unifying parser / dynamic field access); manual review."}) + + blocked_reason = None + if status == "blocked": + reasons = [f["reason"] for f in res["flags"] + if f["kind"] in ("blocked-table", "unknown-table")] + blocked_reason = "; ".join(dict.fromkeys(reasons)) or "No convertible AH mapping." + + # For tables with no confirmed AH mapping, recommend the best-fit AH table(s) + # so a reviewer/LLM can re-home the query instead of hitting a dead end. + unmapped = recommend.unmapped_tables(tables) + recommendations = recommend.recommend_for_tables(unmapped) if unmapped else [] + + return { + "sourceId": source_id, + "artifactType": artifact_type, + "status": status, + "convertedKql": res["convertedKql"], + "resolvedTables": tables or [], + "mappingTier": mapping_tier, + "renamedTables": res["renamedTables"], + "renamedAsimTables": res.get("renamedAsimTables") or {}, + "diff": res["diff"], + "schemaMap": schema_map.build_schema_map(tables, include_identical_columns=False), + "recommendations": recommendations, + "confidence": _CONFIDENCE[status], + "unsupported": unsupported, + "validation": None, # filled by the Judge MCP + "blockedReason": blocked_reason, + } diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/recommend.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/recommend.py new file mode 100644 index 00000000000..4075073a901 --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/recommend.py @@ -0,0 +1,262 @@ +"""Best-fit AH table recommender for Sentinel tables with **no confirmed mapping**. + +When a Sentinel table can't be converted (tier `blocked`/`unknown`), the pipeline +shouldn't just say "no equivalent" - it should point the reviewer at the AH +table(s) that come closest, so the query can be re-homed by hand or by the LLM +step. This module produces those recommendations. They are **suggestions, never +automatic conversions** - a blocked table stays `blocked`. + +Signals, strongest-first: + 1. curated category hints - domain knowledge for common Sentinel tables that + have a well-understood closest AH home (e.g. CommonSecurityLog -> network + device telemetry). High signal, human-curated. + 2. column overlap - when the source table's columns are known, score + each candidate AH table by shared column names (after LA->AH renames). The + table sharing the most columns is the most natural rewrite target. + 3. fuzzy name similarity - last resort when nothing else applies. + +Every recommendation carries a `basis` and a `reason` so the reviewer can judge +it; recommendations are ranked and capped. +""" +from __future__ import annotations + +import difflib +from pathlib import Path +from typing import Dict, List, Optional + +ROOT = Path(__file__).resolve().parent +from . import mapping_catalog, migrate + +_AH_CACHE = ROOT / "data" / "ah_schema_catalog.json" + +# A broad AH advanced-hunting table pool for name/column matching (superset of the +# 21 cached schemas). Kept explicit so recommendations aren't limited to Wave-1. +_AH_TABLE_POOL = { + # Endpoint / device + "DeviceEvents", "DeviceFileEvents", "DeviceImageLoadEvents", "DeviceLogonEvents", + "DeviceNetworkEvents", "DeviceProcessEvents", "DeviceRegistryEvents", + "DeviceFileCertificateInfo", "DeviceInfo", "DeviceNetworkInfo", + "DeviceTvmSoftwareInventory", "DeviceTvmSoftwareVulnerabilities", + "DeviceTvmSecureConfigurationAssessment", + # Email / collaboration + "EmailEvents", "EmailAttachmentInfo", "EmailPostDeliveryEvents", "EmailUrlInfo", + "UrlClickEvents", + # Cloud apps + "CloudAppEvents", + # Identity + "IdentityLogonEvents", "IdentityDirectoryEvents", "IdentityQueryEvents", "IdentityInfo", + # Alerts + "AlertInfo", "AlertEvidence", + # Entra (AH advanced-hunting Entra tables) + "EntraIdSignInEvents", "EntraIdSpnSignInEvents", "EntraIdDirectoryAuditEvents", + "GraphAPIAuditEvents", +} + +# Curated closest-AH-home hints for common Sentinel tables with no direct mapping. +# Each entry: (recommended AH tables in priority order, confidence, reason). +_CATEGORY_HINTS: Dict[str, tuple] = { + # Windows event logs -> device event families + "SecurityEvent": (["DeviceProcessEvents", "DeviceLogonEvents", "DeviceEvents"], "medium", + "Windows Security events map to endpoint device telemetry; pick the " + "family matching the EventID (process create/logon/other)."), + "WindowsEvent": (["DeviceEvents", "DeviceProcessEvents", "DeviceLogonEvents"], "medium", + "Windows event logs map to endpoint device telemetry by event type."), + "Event": (["DeviceEvents", "DeviceProcessEvents"], "low", + "Generic Windows/Sysmon events map to endpoint device telemetry by event type."), + # Firewall / CEF / syslog / network + "CommonSecurityLog": (["DeviceNetworkEvents"], "low", + "CEF firewall/appliance network logs have no exact AH equivalent; " + "device network telemetry is the closest, but coverage differs."), + "Syslog": (["DeviceEvents", "DeviceNetworkEvents"], "low", + "Linux syslog has no exact AH equivalent; device event/network telemetry is closest."), + "VMConnection": (["DeviceNetworkEvents"], "low", + "Azure Monitor VM network connections map closest to device network events."), + # DNS + "DnsEvents": (["DeviceNetworkEvents"], "low", + "No dedicated AH DNS table; DNS-over-network activity appears in device network events."), + "ASimDnsActivityLogs": (["DeviceNetworkEvents"], "low", + "ASIM DNS parser has no AH equivalent; closest is device network events."), + # IIS / web + "W3CIISLog": (["DeviceNetworkEvents"], "low", + "IIS web logs have no AH equivalent; only network-level activity is comparable."), + # Office / cloud app / AWS + "OfficeActivity": (["CloudAppEvents"], "high", + "O365 audit activity is surfaced in AH as CloudAppEvents."), + "AWSCloudTrail": (["CloudAppEvents"], "medium", + "Third-party cloud audit trails are normalized into CloudAppEvents."), + "AzureActivity": (["CloudAppEvents"], "low", + "Azure control-plane activity has no exact AH table; CloudAppEvents is the nearest."), + # Identity / sign-in + "SigninLogs": (["EntraIdSignInEvents", "IdentityLogonEvents"], "medium", + "Interactive Entra sign-ins map to EntraIdSignInEvents (needs review); " + "on-prem/hybrid logons appear in IdentityLogonEvents."), + "AADNonInteractiveUserSignInLogs": (["EntraIdSignInEvents"], "medium", + "Non-interactive Entra sign-ins merge into EntraIdSignInEvents " + "(discriminate via a sign-in-type column)."), + "AADServicePrincipalSignInLogs": (["EntraIdSpnSignInEvents"], "medium", + "Service-principal sign-ins map to EntraIdSpnSignInEvents."), + "AADManagedIdentitySignInLogs": (["EntraIdSpnSignInEvents"], "medium", + "Managed-identity sign-ins fold into EntraIdSpnSignInEvents " + "alongside service-principal sign-ins."), + "AuditLogs": (["EntraIdDirectoryAuditEvents", "IdentityDirectoryEvents"], "medium", + "Entra directory audit maps to EntraIdDirectoryAuditEvents."), + "MicrosoftGraphActivityLogs": (["GraphAPIAuditEvents"], "medium", + "Graph API activity maps to GraphAPIAuditEvents."), + # Alerts / TI + "SecurityAlert": (["AlertInfo", "AlertEvidence"], "medium", + "Sentinel alerts map to AH AlertInfo (+ AlertEvidence for entities)."), + "ThreatIntelligenceIndicator": ([], "none", + "Threat-intel indicators have no AH hunting-table equivalent."), +} + + +def _ah_schemas() -> Dict[str, Dict[str, str]]: + return migrate._load(_AH_CACHE) + + +def _candidate_pool(ah_schemas: Optional[Dict[str, Dict[str, str]]] = None) -> set[str]: + pool = set(_AH_TABLE_POOL) + pool |= set((ah_schemas or _ah_schemas()).keys()) + cat = mapping_catalog.load_mapping_catalog()["tables"] + for meta in cat.values(): + if meta.get("ahTable"): + pool.add(meta["ahTable"]) + return pool + + +def _column_overlap(source_cols: Dict[str, str], ah_cols: Dict[str, str]) -> tuple[float, list]: + """Fraction of meaningful source columns that also exist in the AH table + (case-insensitive, honoring any global column renames - currently none; + TimeGenerated exists in AH unchanged).""" + meaningful = [c for c in source_cols if c not in migrate.LA_SYSTEM_COLS] + if not meaningful: + return 0.0, [] + ah_lower = {c.lower() for c in ah_cols} + matched = [] + for c in meaningful: + target = migrate.LA_TO_AH_RENAME.get(c, c) + if target.lower() in ah_lower: + matched.append(c) + return round(len(matched) / len(meaningful), 3), matched + + +def recommend_table(table: str, source_cols: Optional[Dict[str, str]] = None, + ah_schemas: Optional[Dict[str, Dict[str, str]]] = None, + top: int = 3) -> dict: + """Rank the best-fit AH tables for one unmapped Sentinel table.""" + ah_schemas = ah_schemas if ah_schemas is not None else _ah_schemas() + pool = _candidate_pool(ah_schemas) + recs: List[dict] = [] + seen: set[str] = set() + + # 0. catalog self-mapping wins outright. A table mapped 1:1 to itself is + # already a valid AH table on the target tenant - it is its own best + # home, so never recommend a different table (kills self->self noise and + # now-obsolete category hints like WindowsEvent->DeviceEvents). + meta = mapping_catalog.load_mapping_catalog()["tables"].get(table) + if meta and meta.get("tier") == "1to1" and meta.get("ahTable") == table: + return {"sentinelTable": table, + "recommendations": [{ + "ahTable": table, "basis": "catalogSelf", "confidence": "high", + "reason": "Offline catalog maps this table to itself; selected " + "environment visibility remains unverified."}], + "hasHint": False} + + # 1. curated category hint + hint = _CATEGORY_HINTS.get(table) + if hint: + ah_list, conf, reason = hint + for ah in ah_list: + recs.append({"ahTable": ah, "basis": "category", "confidence": conf, + "reason": reason}) + seen.add(ah) + if not ah_list: # explicit "no equivalent" + return {"sentinelTable": table, "recommendations": [], + "note": reason, "hasHint": True} + + # 2. column overlap (only if we know the source schema) + if source_cols: + scored = [] + for ah in pool: + ah_cols = ah_schemas.get(ah) + if not ah_cols: + continue + score, matched = _column_overlap(source_cols, ah_cols) + if score > 0: + # An AH table with the identical name is, by definition, the + # right home - it must win ties against sibling tables whose + # columns happen to overlap just as much (e.g. Device* family). + exact = 1 if ah.lower() == table.lower() else 0 + scored.append((exact, score, ah, matched)) + scored.sort(reverse=True) + for _exact, score, ah, matched in scored: + if ah in seen: + # enrich the existing category rec with overlap evidence + for r in recs: + if r["ahTable"] == ah: + r["columnOverlap"] = score + r["matchedColumns"] = matched[:12] + continue + recs.append({"ahTable": ah, "basis": "columnOverlap", + "confidence": "high" if score >= 0.6 else + "medium" if score >= 0.3 else "low", + "reason": f"shares {int(score*100)}% of source columns with {ah}.", + "columnOverlap": score, "matchedColumns": matched[:12]}) + seen.add(ah) + + # 3. fuzzy name similarity - only as a last resort, when neither a curated + # hint nor column overlap produced anything (avoids noisy matches like + # Event -> EmailEvents). + if not recs: + for ah in difflib.get_close_matches(table, sorted(pool), n=top, cutoff=0.6): + if ah in seen: + continue + recs.append({"ahTable": ah, "basis": "nameSimilarity", "confidence": "low", + "reason": f"table name is similar to '{ah}'."}) + seen.add(ah) + + return { + "sentinelTable": table, + "recommendations": recs[:top], + "note": ("No confirmed AH equivalent; recommendations are best-effort and " + "require human/LLM review before use."), + "hasHint": bool(hint), + } + + +def recommend_for_tables(tables: List[str], + source_schemas: Optional[Dict[str, Dict[str, str]]] = None, + ah_schemas: Optional[Dict[str, Dict[str, str]]] = None, + top: int = 3) -> List[dict]: + """Recommend best-fit AH tables for each unmapped Sentinel table.""" + source_schemas = source_schemas or {} + return [recommend_table(t, source_cols=source_schemas.get(t), + ah_schemas=ah_schemas, top=top) for t in tables] + + +def unmapped_tables(tables: Optional[List[str]]) -> List[str]: + """The subset of resolved tables that have no convertible AH mapping + (catalog tier `blocked`, or absent from the catalog = `unknown`).""" + cat = mapping_catalog.load_mapping_catalog()["tables"] + out = [] + for t in (tables or []): + tier = cat[t]["tier"] if t in cat else "unknown" + if tier in ("blocked", "unknown"): + out.append(t) + return out + + +def _main(argv: Optional[List[str]] = None) -> int: + import argparse + import json + + p = argparse.ArgumentParser(description="Recommend best-fit AH tables for unmapped Sentinel tables.") + p.add_argument("tables", nargs="+", help="Sentinel table name(s)") + p.add_argument("--top", type=int, default=3) + args = p.parse_args(argv) + print(json.dumps(recommend_for_tables(args.tables, top=args.top), indent=2, default=str)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/rewrite.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/rewrite.py new file mode 100644 index 00000000000..d3bb107599c --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/rewrite.py @@ -0,0 +1,395 @@ +"""Deterministic Sentinel -> Advanced Hunting KQL rewriter (Migrator core, step 3). + +Given a Sentinel KQL query and the tables it resolves to, apply the catalog's +deterministic transforms and return the converted KQL plus a reviewable diff. + +What it does (safe, semantics-preserving only): + * table renames - when the AH table name differs (Wave-2; none for Wave 1). + * column renames - per-table renames from the mapping catalog. There is no + global time-column rename: `TimeGenerated` exists in AH + unchanged (it must NOT be rewritten to `Timestamp`). +Replacements are whole-identifier and skip string literals and `//` comments, so +tokens inside strings/comments are never touched. + +What it deliberately does NOT do: + * It never silently deletes columns. References to Log-Analytics-only columns + (`TenantId`, `_ResourceId`, ...) that don't exist in AH are *flagged* for + review, not removed - deleting them would change query semantics. The Judge's + static bind independently confirms whether the converted query binds in AH. + +Overall tier (worst across the query's tables) drives the caller's status: + 1to1 - all tables 1:1 and no LA-only-column references -> deterministic, high confidence. + renamed / merged - needs review (Wave-2, out of v1 conversion scope). + blocked - a table has no AH equivalent -> do not convert. + + python rewrite.py --query "DeviceProcessEvents | where TimeGenerated > ago(1d)" \ + --tables DeviceProcessEvents + python rewrite.py --file rule.kql --resolve # resolve tables live via Log Analytics +""" +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path +from typing import Dict, List, Optional, Tuple + + +# [ASIM-FIX-TAG 2026-08-03] Previously deliberately disabled (user decision, 2026-08-03): +# live testing on Zava (a unified-SecOps tenant) confirmed _Im_WebSession/_Im_FileEvent/ +# _Im_Dns DO resolve and return correctly-shaped ASIM data there - but only because +# that tenant links its Sentinel workspace's saved ASIM functions into AH. Generic E5 +# customers (this tool's actual target, per Business background and goal.md) are not +# guaranteed to be unified or have ASIM parsers installed, and the whole point of this +# migration is to reduce their dependency on Sentinel - so relying on `_Im_*` resolving +# was seen as at odds with the tool's own goal. +# [ASIM-FIX-TAG 2026-08-03g] Re-enabled (user decision, 2026-08-03): the mapping_catalog.json +# `asimFunctionRewrites.functions` targets were corrected to route _Im_NetworkSession/ +# _Im_WebSession/_Im_FileEvent/_Im_Dns to their live-callable `_Im_BuiltIn` AH +# functions (rather than the plain, unreliable ASimLogs tables), and these are +# now verified to work - so the rewrite is turned back on. +ASIM_REWRITE_ENABLED = True + +from . import mapping_catalog, dynamic_casts, asim_functions + +_TIER_RANK = {"1to1": 0, "renamed": 1, "merged": 1, "unknown": 2, "blocked": 3} + + +def _segments(kql: str) -> List[Tuple[str, str]]: + """Split KQL into ('code'|'str'|'comment', text) segments so renames only + touch code. Handles `//` line comments, '...'/"..." string literals with + backslash escapes, and `@"..."` verbatim strings (no escape).""" + segs: List[Tuple[str, str]] = [] + i, n, start = 0, len(kql), 0 + while i < n: + c = kql[i] + if c == "/" and i + 1 < n and kql[i + 1] == "/": + if start < i: + segs.append(("code", kql[start:i])) + j = kql.find("\n", i) + if j == -1: + j = n + segs.append(("comment", kql[i:j])) + i = start = j + elif c == '"' or c == "'": + if start < i: + segs.append(("code", kql[start:i])) + verbatim = i > 0 and kql[i - 1] == "@" + quote, j = c, i + 1 + while j < n: + if not verbatim and kql[j] == "\\": + j += 2 + continue + if kql[j] == quote: + j += 1 + break + j += 1 + segs.append(("str", kql[i:j])) + i = start = j + else: + i += 1 + if start < n: + segs.append(("code", kql[start:n])) + return segs + + +def _apply_renames(kql: str, rename_map: Dict[str, str]) -> Tuple[str, Dict[str, int]]: + """Whole-identifier replace every key with its value, in code segments only. + Returns (new_kql, {source_token: count}).""" + if not rename_map: + return kql, {} + keys = sorted(rename_map, key=len, reverse=True) + pat = re.compile(r"\b(" + "|".join(re.escape(k) for k in keys) + r")\b") + counts: Dict[str, int] = {} + + def repl(m: "re.Match") -> str: + k = m.group(1) + counts[k] = counts.get(k, 0) + 1 + return rename_map[k] + + out = [] + for kind, text in _segments(kql): + out.append(pat.sub(repl, text) if kind == "code" else text) + return "".join(out), counts + + +def _find_identifiers(kql: str, names: set) -> set: + """Return which of `names` appear as whole identifiers in code segments.""" + if not names: + return set() + pat = re.compile(r"\b(" + "|".join(re.escape(k) for k in names) + r")\b") + found = set() + for kind, text in _segments(kql): + if kind == "code": + found.update(m.group(1) for m in pat.finditer(text)) + return found + + +def _code_mask(kql: str) -> List[bool]: + """Boolean mask, True where the char is in a code segment (not string/comment).""" + mask: List[bool] = [] + for kind, text in _segments(kql): + mask.extend([kind == "code"] * len(text)) + return mask + + +def _is_word(kql: str, start: int, end: int) -> bool: + """True if kql[start:end] is bounded by non-identifier chars (whole word).""" + before = kql[start - 1] if start > 0 else "" + after = kql[end] if end < len(kql) else "" + return not (before.isalnum() or before == "_") and not (after.isalnum() or after == "_") + + +_OPEN, _CLOSE = "([{", ")]}" + + +def _dedupe_key_region(region: str, mask: List[bool], offset: int) -> Tuple[str, List[str]]: + """Split a `by` group-key region on top-level commas and drop exact + (whitespace-normalized) duplicate keys, preserving order and outer spacing.""" + parts: List[str] = [] + depth, start = 0, 0 + for idx in range(len(region)): + if not mask[offset + idx]: + continue + c = region[idx] + if c in _OPEN: + depth += 1 + elif c in _CLOSE: + depth -= 1 + elif c == "," and depth == 0: + parts.append(region[start:idx]) + start = idx + 1 + parts.append(region[start:]) + + seen: set = set() + unique: List[str] = [] + removed: List[str] = [] + for p in parts: + norm = re.sub(r"\s+", " ", p.strip()).lower() + if not norm: + continue + if norm in seen: + removed.append(p.strip()) + continue + seen.add(norm) + unique.append(p.strip()) + if not removed: + return region, [] + lead = region[:len(region) - len(region.lstrip())] + trail = region[len(region.rstrip()):] + return lead + ", ".join(unique) + trail, removed + + +def _dedupe_summarize_keys(kql: str) -> Tuple[str, List[str]]: + """Remove exact-duplicate group-by keys from every `summarize ... by ...` + clause. Repeated keys are redundant and make AH reject the query with + 'Duplicate column name(s)'. Whole-identifier / paren-aware and skips + strings & comments, so it is safe and semantics-preserving.""" + mask = _code_mask(kql) + n = len(kql) + lower = kql.lower() + repls: List[Tuple[int, int, str]] = [] + removed_all: List[str] = [] + + depth, i = 0, 0 + while i < n: + if mask[i]: + c = kql[i] + if c in _OPEN: + depth += 1 + elif c in _CLOSE: + depth -= 1 + elif depth == 0 and lower.startswith("summarize", i) and _is_word(kql, i, i + 9): + # Statement runs until the next top-level pipe (or end of query). + j, d2, stmt_end = i + 9, 0, n + while j < n: + if mask[j]: + cj = kql[j] + if cj in _OPEN: + d2 += 1 + elif cj in _CLOSE: + d2 -= 1 + elif cj == "|" and d2 == 0: + stmt_end = j + break + j += 1 + # Find the top-level `by` that separates aggregations from keys. + by_pos, d3, k = -1, 0, i + 9 + while k < stmt_end: + if mask[k]: + ck = kql[k] + if ck in _OPEN: + d3 += 1 + elif ck in _CLOSE: + d3 -= 1 + elif d3 == 0 and lower.startswith("by", k) and _is_word(kql, k, k + 2): + by_pos = k + break + k += 1 + if by_pos != -1: + key_start = by_pos + 2 + new_region, removed = _dedupe_key_region( + kql[key_start:stmt_end], mask, key_start) + if removed: + repls.append((key_start, stmt_end, new_region)) + removed_all.extend(removed) + i = stmt_end + continue + i += 1 + + if not repls: + return kql, [] + out = kql + for s, e, txt in sorted(repls, key=lambda r: r[0], reverse=True): + out = out[:s] + txt + out[e:] + return out, removed_all + + +def rewrite(kql: str, tables: Optional[List[str]], + catalog: Optional[dict] = None) -> dict: + """Deterministically rewrite a Sentinel query for Advanced Hunting. + + Returns: + convertedKql - the rewritten query. + diff - [{kind, before, after, count, source, reason}]. + renamedTables - {sentinelTable: ahTable} applied. + flags - [{kind, name, reason}] items needing review / blocking. + tiers - {table: tier}. + overallTier - worst tier across the query's tables. + deterministic - True iff every table is 1to1 and nothing was flagged. + """ + cat = catalog or mapping_catalog.load_mapping_catalog() + tbl_map = cat["tables"] + tables = list(tables or []) + + # Column renames: global + per-table for the tables present. + col_renames: Dict[str, str] = dict(cat["global"].get("columnRenames", {})) + table_renames: Dict[str, str] = {} + tiers: Dict[str, str] = {} + flags: List[dict] = [] + global_drop_cols = set(cat["global"].get("dropColumns", [])) + drop_cols: set[str] = set() + cast_cols: Dict[str, List[str]] = {} + int_cols: Dict[str, List[str]] = {} + removed_cols: Dict[str, List[str]] = {} + + for t in tables: + entry = tbl_map.get(t) + tier = entry["tier"] if entry else "unknown" + tiers[t] = tier + if not entry: + flags.append({"kind": "unknown-table", "name": t, + "reason": "Table is not in the mapping catalog " + "(custom/unsupported); cannot convert."}) + continue + if tier == "blocked": + flags.append({"kind": "blocked-table", "name": t, + "reason": entry.get("blockedReason", + "No confirmed AH equivalent.")}) + continue + if entry.get("applyGlobalDropColumns", True): + drop_cols.update(global_drop_cols) + col_renames.update(entry.get("columnRenames", {})) + drop_cols.update(entry.get("dropColumns", [])) + ah = entry.get("ahTable") + if ah and ah != t: + table_renames[t] = ah + if tier in ("renamed", "merged"): + flags.append({"kind": "non-1to1-table", "name": t, + "reason": f"Tier '{tier}': {entry.get('notes', '')}".strip()}) + if entry.get("castDynamicColumns"): + cast_cols[t] = list(entry["castDynamicColumns"]) + if entry.get("intColumns"): + int_cols[t] = list(entry["intColumns"]) + if entry.get("removedColumns"): + removed_cols[t] = list(entry["removedColumns"]) + + # Apply table + column renames in one whole-identifier pass. + rename_map = {**col_renames, **table_renames} + converted, counts = _apply_renames(kql, rename_map) + + # Drop redundant duplicate group-by keys (AH rejects duplicate output columns). + converted, dup_keys = _dedupe_summarize_keys(converted) + + # Wave-1 known column gotchas (docs/Sentinel-to-XDR-KQL-Migration-V2 1.md): + # un-cast dynamic-as-string columns, quoted int-typed literals, removed columns. + converted, cast_diffs = dynamic_casts.inject_dynamic_casts(converted, cast_cols) + converted, int_diffs = dynamic_casts.unquote_int_literals(converted, int_cols) + removed_flags = dynamic_casts.flag_removed_columns(converted, removed_cols) + flags.extend(removed_flags) + + diff: List[dict] = [] + for src, cnt in counts.items(): + kind = "table-rename" if src in table_renames else "column-rename" + reason = ("AH table name differs" if kind == "table-rename" + else "Column renamed in AH") + diff.append({"kind": kind, "before": src, "after": rename_map[src], + "count": cnt, "source": "deterministic", "reason": reason}) + for key in dup_keys: + diff.append({"kind": "dedupe-summarize-key", "before": key, "after": "(removed)", + "count": 1, "source": "deterministic", + "reason": "Duplicate summarize group-by key removed; " + "AH rejects duplicate output columns"}) + diff.extend(cast_diffs) + diff.extend(int_diffs) + + # ASIM unifying-parser function calls (_Im_WebSession(...), _Im_NetworkSession(...), + # etc.) don't resolve in AH; swap them for the AH-native ASimLogs table, + # inlining any named params we have a verified translation for. + # [ASIM-FIX-TAG 2026-08-03] Rewrite disabled per ASIM_REWRITE_ENABLED (see top of + # file) - left as an unrewritten flag instead, same as the "unknown param" path + # asim_functions.py already produces, so it still routes to manual review rather + # than silently passing through unconverted. + if ASIM_REWRITE_ENABLED: + asim_result = asim_functions.rewrite_im_calls(converted, cat) + converted = asim_result["convertedKql"] + diff.extend(asim_result["diff"]) + flags.extend(asim_result["flags"]) + else: + asim_result = {"diff": [], "flags": asim_functions.flag_im_calls(converted)} + flags.extend(asim_result["flags"]) + + # [ASIM-FIX-TAG 2026-08-02] Per the spec (KQL-Conversion-Service-Spec.md, "Step 1 - + # Resolve data sources"), an unresolvable ASIM unifying parser is allowed to be + # recorded as `dataSourcesError`/null and routed to manual review rather than + # guessed at. We instead resolve and convert these deterministically via the + # verified asimFunctionRewrites catalog - a real table substitution the spec left + # optional ("a future parser-expansion mapping", see spec §9 Open Questions). Every + # such substitution must stay visible to reviewers, so record it separately from + # `renamedTables` (which only covers ordinary 1:1/renamed LA->AH table maps) and + # surface it distinctly in the dashboard rather than silently folding it in. + renamed_asim_tables = {d["func"]: d["ahTable"] for d in asim_result["diff"] + if d["kind"] == "asim-function-rewrite"} + + diff.sort(key=lambda d: (d["kind"], d["before"])) + + # Flag references to LA-only columns that don't exist in AH (never delete). + for col in sorted(_find_identifiers(kql, drop_cols)): + flags.append({"kind": "la-only-column", "name": col, + "reason": "Log-Analytics-only column not present in AH; " + "query references it and needs manual handling."}) + + overall = "1to1" + if tables: + overall = max((tiers[t] for t in tables), key=lambda x: _TIER_RANK.get(x, 2)) + else: + overall = "unknown" + # [ASIM-FIX 2026-08-02] "asim-function-unrewritten" added to the gate below - + # previously an unrewritten _Im_(...) call (unknown param, left as-is by + # asim_functions.py) did not block `deterministic`, letting AH's silent-bind-to- + # wrong-table behavior masquerade as a passing static bind. See batch_xdr.py's + # matching [ASIM-FIX 2026-08-02] guard and mcp/tests/test_asim_functions.py. + has_review_flag = any(f["kind"] in ("la-only-column", "removed-column", + "asim-function-unrewritten") for f in flags) + deterministic = bool(tables) and overall == "1to1" and not has_review_flag + + return { + "convertedKql": converted, + "diff": diff, + "renamedTables": table_renames, + "renamedAsimTables": renamed_asim_tables, + "flags": flags, + "tiers": tiers, + "overallTier": overall, + "deterministic": deterministic, + } diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/schema_map.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/schema_map.py new file mode 100644 index 00000000000..2d6b16acf54 --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/schema_map.py @@ -0,0 +1,178 @@ +"""Column-level schema mapper for Sentinel -> Advanced Hunting migration. + +When a Sentinel table maps to an AH table whose schema is **not identical**, a +correct migration must say *how each column maps* - the exact matches, the +renames, the type casts, and the columns that have **no direct AH equivalent** +(with a best-effort suggestion, never a silent drop). This module produces that +per-column mapping so the output contract carries "the most appropriate schema". + +Sources of truth, cheapest-first: + - LA side : cached `data/la_schema_catalog.json` ({table: {col: type}}). + - AH side : cached `data/ah_schema_catalog.json` (offline, 21 Wave-1 tables), + or an explicitly supplied schema-provider callback. This + imported slice contains no live clients or authentication. + - renames : the mapping catalog's per-table `columnRenames` (authoritative + for known Wave-2 column renames). + +Relations emitted per column: + exact - same name, same/compatible-identical type; use as-is. + compatible - same name, trivially-compatible type (e.g. dynamic<->string). + renamed - different AH name (via catalog/case), compatible type. + castNeeded - real but trivially castable type diff (toint()/tostring()). + typeMismatch - genuinely incompatible types; blocks that column's uses. + missing - no AH counterpart; carries a best-effort `suggestion` or is + flagged for review. Never silently dropped. + +Reuses `migrate._type_relation` / `LA_TO_AH_RENAME` / `LA_SYSTEM_COLS` so the +type semantics match the schema-diff step exactly. +""" +from __future__ import annotations + +import difflib +from pathlib import Path +from typing import Callable, Dict, List, Optional + +ROOT = Path(__file__).resolve().parent +from . import mapping_catalog, migrate + +_LA_CACHE = ROOT / "data" / "la_schema_catalog.json" +_AH_CACHE = ROOT / "data" / "ah_schema_catalog.json" + +AhSchemaProvider = Callable[[str], Optional[Dict[str, str]]] + + +def _la_cached() -> Dict[str, Dict[str, str]]: + return migrate._load(_LA_CACHE) + + +def _ah_cached() -> Dict[str, Dict[str, str]]: + return migrate._load(_AH_CACHE) + + + + + + +def _suggest(col: str, ah_cols: Dict[str, str], + catalog_renames: Dict[str, str]) -> tuple[Optional[str], Optional[str]]: + """Best-effort AH column for an LA column with no direct match.""" + if col in catalog_renames and catalog_renames[col] in ah_cols: + return catalog_renames[col], "catalog" + lower = {c.lower(): c for c in ah_cols} + if col.lower() in lower: + return lower[col.lower()], "case" + close = difflib.get_close_matches(col, list(ah_cols), n=1, cutoff=0.82) + if close: + return close[0], "fuzzy" + return None, None + + +def map_table(la_table: str, ah_table: str, la_cols: Dict[str, str], + ah_cols: Dict[str, str], catalog_renames: Dict[str, str]) -> dict: + """Per-column mapping for one LA->AH table pair.""" + ah_lower = {c.lower(): c for c in ah_cols} + meaningful = [c for c in la_cols if c not in migrate.LA_SYSTEM_COLS] + columns: List[dict] = [] + used_ah: set[str] = set() + + for col in meaningful: + la_type = la_cols[col] + target = catalog_renames.get(col) or migrate.LA_TO_AH_RENAME.get(col, col) + was_renamed = target != col + + ah_name = ah_type = None + if target in ah_cols: + ah_name, ah_type = target, ah_cols[target] + elif target.lower() in ah_lower: + ah_name = ah_lower[target.lower()] + ah_type = ah_cols[ah_name] + + if ah_name is None: + sug, ssrc = _suggest(col, ah_cols, catalog_renames) + columns.append({ + "sentinel": col, "laType": la_type, "ah": None, "ahType": None, + "relation": "missing", "suggestion": sug, "suggestionSource": ssrc, + "note": (f"no direct AH equivalent; suggest '{sug}'" if sug + else "no direct AH equivalent; flag for review"), + }) + continue + + used_ah.add(ah_name) + rel = migrate._type_relation(la_type, ah_type) + if ah_name != col: + relation = "renamed" if rel in ("exact", "compat") else ( + "castNeeded" if rel == "soft" else "typeMismatch") + else: + relation = {"exact": "exact", "compat": "compatible", + "soft": "castNeeded", "hard": "typeMismatch"}[rel] + + entry = {"sentinel": col, "ah": ah_name, "laType": la_type, + "ahType": ah_type, "relation": relation} + if ah_name != col: + entry["renamedFrom"] = col + columns.append(entry) + + extra = [c for c in ah_cols + if c not in used_ah and c not in migrate.LA_SYSTEM_COLS] + # Consistent verdict/score with the schema-diff step. + diff = migrate.diff_table(la_cols, ah_cols) + identical = all(c["relation"] in ("exact", "compatible") for c in columns) + + return { + "sentinelTable": la_table, + "ahTable": ah_table, + "identical": identical, + "verdict": diff["verdict"], + "score": diff["score"], + "columns": columns, + "extraInAh": extra, + } + + +def build_schema_map(tables: Optional[List[str]], + ah_schema_provider: Optional[AhSchemaProvider] = None, + include_identical_columns: bool = True) -> List[dict]: + """Schema map for every resolved table. + + ``ah_schema_provider`` supplies the AH schema per AH-table-name; pass + a caller-owned schema provider, or omit to use the + offline cache. When ``include_identical_columns`` is False, tables whose + schema is identical collapse to a compact summary (keeps contracts light). + """ + la = _la_cached() + cat = mapping_catalog.load_mapping_catalog()["tables"] + ah_cache = None if ah_schema_provider else _ah_cached() + out: List[dict] = [] + + for t in (tables or []): + meta = cat.get(t, {}) + ah_table = meta.get("ahTable") or t + renames = meta.get("columnRenames") or {} + la_cols = la.get(t) + ah_cols = (ah_schema_provider(ah_table) if ah_schema_provider + else ah_cache.get(ah_table)) + + if not ah_cols and not la_cols: + out.append({"sentinelTable": t, "ahTable": ah_table, "identical": None, + "columnRenames": renames, + "note": "no cached schema for either side; catalog renames only"}) + continue + if not ah_cols: + out.append({"sentinelTable": t, "ahTable": ah_table, "identical": None, + "columnRenames": renames, + "note": "AH schema unavailable (fetch live for a full map); " + "using catalog renames"}) + continue + if not la_cols: + out.append({"sentinelTable": t, "ahTable": ah_table, "identical": None, + "ahColumns": len(ah_cols), "note": "LA schema not cached"}) + continue + + m = map_table(t, ah_table, la_cols, ah_cols, renames) + if m["identical"] and not include_identical_columns: + m = {"sentinelTable": t, "ahTable": ah_table, "identical": True, + "verdict": m["verdict"], "score": m["score"], + "columnCount": len(m["columns"]), + "note": "schema identical; no column changes"} + out.append(m) + return out diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/workbook.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/workbook.py new file mode 100644 index 00000000000..ee1496cd368 --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/workbook.py @@ -0,0 +1,738 @@ +"""Offline, loss-aware Notebook/1.0 adapter around the shared KQL migrator.""" +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +from pathlib import Path +import re +from typing import Iterator + +from . import migrator, rewrite + +AH = "advancedHunting" +LA = "microsoft.operationalinsights/workspaces" +SCOPE_KEYS = { + "crossComponentResources", "fallbackResourceIds", "resourceIds", + "workspaceId", "workspaceResourceId", "selectedWorkspaceIds", +} +PARAMETER_KEYS = { + "parameterName", "exportParameterName", "timeContextFromParameter", + "resourceContextFromParameter", +} +TOKEN = re.compile(r"\{([A-Za-z_][\w .-]*)(?::[^{}]*)?\}") + + +def walk(value, path="") -> Iterator[tuple[str, dict]]: + """Yield dictionaries with RFC 6901 JSON pointers, including nested groups.""" + if isinstance(value, dict): + yield path, value + for key, child in value.items(): + escaped = key.replace("~", "~0").replace("/", "~1") + yield from walk(child, f"{path}/{escaped}") + elif isinstance(value, list): + for i, child in enumerate(value): + yield from walk(child, f"{path}/{i}") + + +def query_hash(query: str) -> str: + return hashlib.sha256(query.encode("utf-8")).hexdigest() + + +def references(value, *, skip_scope=False) -> set[str]: + found = set() + if isinstance(value, str): + found.update(TOKEN.findall(value)) + elif isinstance(value, list): + for child in value: + found.update(references(child, skip_scope=skip_scope)) + elif isinstance(value, dict): + for key, child in value.items(): + if skip_scope and (key in SCOPE_KEYS or key == "resourceContextFromParameter"): + continue + if key in PARAMETER_KEYS and isinstance(child, str): + found.add(child) + if key == "linkTarget" and child == "parameter": + found.add(value.get("cellValue", "")) + found.update(references(child, skip_scope=skip_scope)) + return found + + +def query_surfaces(doc: dict) -> list[dict]: + """Inventory every nonempty query, honoring inherited provider settings.""" + result = [] + + def visit(value, path="", inherited=None, parameter=False): + if isinstance(value, dict): + settings = dict(inherited or {}) + settings.update({k: value[k] for k in ("queryType", "resourceType") if k in value}) + if "query" in value and value["query"] not in ("", None): + result.append({ + "path": path + "/query", "node": value, + "queryType": settings.get("queryType", 0), + "resourceType": settings.get("resourceType"), + "parameter": parameter, + }) + for key, child in value.items(): + escaped = key.replace("~", "~0").replace("/", "~1") + visit(child, f"{path}/{escaped}", settings, + parameter or key == "parameters") + elif isinstance(value, list): + for i, child in enumerate(value): + visit(child, f"{path}/{i}", inherited, parameter) + + visit(doc) + return result + + +def _scope_parameters(doc, protected=()): + entries = [(path, node) for path, node in walk(doc) + if re.search(r"/parameters/\d+$", path) and node.get("name")] + names = {node["name"] for _, node in entries} + if len(names) != len(entries): + raise ValueError("Duplicate parameter names: scope dependencies are ambiguous.") + params = {node["name"]: node for _, node in entries} + nodes = dict(walk(doc)) + inherited_resources = {} + for path, p in entries: + ancestor = path + while True: + if nodes.get(ancestor, {}).get("resourceType"): + inherited_resources[p["name"]] = nodes[ancestor]["resourceType"].lower() + break + if not ancestor: + break + ancestor = ancestor.rsplit("/", 1)[0] + surfaces = {id(s["node"]): s for s in query_surfaces(doc)} + bindings = set() + for _, node in walk(doc): + for key in SCOPE_KEYS | {"resourceContextFromParameter"}: + if key in node: + bindings |= references({key: node[key]}) + dependencies = { + name: references(p, skip_scope=surfaces.get(id(p), {}).get("queryType", 0) + in (0, None, AH)) & names + for name, p in params.items() + } + all_dependencies = {name: references(p) & names for name, p in params.items()} + + def eligible(name): + p = params[name] + # A type-5 parameter may be a KQL-backed Computer filter, not a workspace. + qt = surfaces.get(id(p), {}).get("queryType") + return not (p.get("query") and qt in (0, None, AH)) and p.get("type") != 4 + + candidates = { + name for name, p in params.items() + if eligible(name) and ( + p.get("type") == 6 + or (p.get("type") == 5 and ( + name in bindings or inherited_resources.get(name) == LA)) + or LA in json.dumps(p.get("typeSettings", {})).lower() + or (p.get("queryType") == 1 and LA in p.get("query", "").lower()) + ) + } + # Include both upstream scope helpers and hidden derived helpers (ContextFree). + while True: + expanded = candidates | { + dep for name in candidates for dep in all_dependencies[name] if eligible(dep) + } | { + name for name, p in params.items() + if eligible(name) and p.get("isHiddenWhenLocked") + and all_dependencies[name] & candidates + } + if expanded == candidates: + break + candidates = expanded + + outside = copy.deepcopy(doc) + for _, node in walk(outside): + if isinstance(node.get("parameters"), list): + node["parameters"] = [] + live = ((names - candidates) | (references(outside, skip_scope=True) & names) + | (set(protected) & names)) + while True: + expanded = live | { + dep for name in live + for dep in (all_dependencies[name] if name in candidates else dependencies[name]) + } + if expanded == live: + break + live = expanded + removed = candidates - live + return entries, removed, candidates & live + + +def _raw_query(query, query_type): + if not isinstance(query, str): + raise ValueError("query must be a string") + if query_type == AH and query.lstrip().startswith("{"): + try: + settings = json.loads(query) + except ValueError as exc: + raise ValueError("Malformed Advanced Hunting settings wrapper") from exc + if not isinstance(settings, dict) or settings.get("__advancedHuntingSettings") is not True: + raise ValueError("Unrecognized Advanced Hunting query wrapper") + if set(settings) - {"__advancedHuntingSettings", "query", "selectedWorkspaceIds"}: + raise ValueError("Unknown Advanced Hunting wrapper settings; review before removing") + if not isinstance(settings.get("query"), str): + raise ValueError("Advanced Hunting wrapper has no string query") + return settings["query"] + return query + + +_BUILTINS = { + "datatable", "range", "print", "externaldata", "external_data", "dynamic", "datetime", "timespan", + "toscalar", "tostring", "toint", "tolong", "todouble", "todatetime", "todecimal", "todynamic", "toreal", "totable", + "totimespan", "tobool", "now", "ago", "startofday", "endofday", "startofweek", "endofweek", + "startofmonth", "endofmonth", "startofyear", "endofyear", "tolower", "toupper", "coalesce", "iff", "iif", "case", + "_GetWatchlist", "workspace", "app", "resource", "cluster", "database", "table", "view", "materialize", + "abs", "acos", "asin", "atan", "atan2", "beta_cdf", "beta_inv", "beta_pdf", "bin", "bin_at", " ceiling", "ceiling", + "cos", "cot", " count", "count", "count_distinct", "count_distinctif", "countif", "countof", + " dcount", "dcount", "dcount_hll", "dcountif", "degrees", "exp", "exp10", "exp2", "floor", "gamma", + "hash", "hash_combine", "hash_many", "hash_md5", "hash_sha1", "hash_sha256", "hash_xxhash64", + "isnan", "isfinite", "isinf", "isascii", "isempty", "isnotempty", "isnull", "isnotnull", "isutf8", + "log", "log10", "log2", "loggamma", "max", "max_of", "maxif", "min", "min_of", "minif", "mod", + "not", "and", "or", "between", "in", "has", "has_all", "has_any", "has_cs", "contains", "contains_cs", + "startswith", "startswith_cs", "endswith", "endswith_cs", "matches", "regex", + "avg", "avgif", "sum", "sumif", "stdev", "stdevif", "stdevp", "variance", "varianceif", "variancep", + "percentile", "percentiles", "percentilew", "percentilesw", "arg_max", "arg_min", "any", "anyif", "take_any", "take_anyif", + "make_list", "make_list_if", "make_list_with_nulls", "makelist", "make_set", "make_set_if", "makeset", + "make_bag", "make_bag_if", "pack", "pack_all", "pack_array", "pack_dictionary", "bag_keys", "bag_pack", + "bag_pack_columns", "bag_merge", "bag_remove_keys", "bag_set_key", "bag_has_key", "bag_unpack", + "unpack_object", "treepath", "zip", "repeat", + "array_concat", "array_iif", "array_index_of", "array_iff", "array_length", "array_reverse", + "array_rotate_left", "array_rotate_right", "array_shift_left", "array_shift_right", "array_slice", + "array_sort_asc", "array_sort_desc", "array_split", "array_strcat", "array_sum", + "set_difference", "set_has_element", "set_intersect", "set_union", + "strcat", "strcat_array", "strcat_delim", "strcmp", "string_size", "strlen", "strrep", "substring", + "split", "trim", "trim_end", "trim_start", "replace", "replace_regex", "replace_string", "replace_strings", + "reverse", "translate", "url_decode", "url_encode", "url_encode_component", + "base64_decode_toarray", "base64_decode_toguid", "base64_decode_tostring", "base64_encode_fromguid", "base64_encode_tostring", + " zlib_compress_to_base64_string", "zlib_decompress_from_base64_string", "gzip_compress_to_base64_string", "gzip_decompress_from_base64_string", + "extract", "extract_all", "extractall", "extract_json", "extractjson", + "parse_command_line", "parse_csv", "parse_ipv4", "parse_ipv4_mask", "parse_ipv6", "parse_ipv6_mask", + "parse_json", "parsejson", "parse_path", "parse_url", "parse_urlquery", "parse_user_agent", "parse_version", "parse_xml", + "dynamic_to_json", "format_bytes", "format_datetime", "format_ipv4", "format_ipv4_mask", "format_timespan", + "datetime_add", "datetime_diff", "datetime_local_to_utc", "datetime_part", "datetime_utc_to_local", + "dayofmonth", "dayofweek", "dayofyear", "getmonth", "getyear", "hourofday", "monthofyear", "weekofyear", " week_of_year", + "make_datetime", "make_timespan", "unixtime_microseconds_todatetime", "unixtime_milliseconds_todatetime", + "unixtime_nanoseconds_todatetime", "unixtime_seconds_todatetime", "ingestion_time", "cursor_after", "cursor_before_or_at", "current_cursor", + "ipv4_compare", "ipv4_is_in_any_range", "ipv4_is_in_range", "ipv4_is_match", "ipv4_is_private", "ipv4_lookup", + "ipv4_netmask_suffix", "ipv6_compare", "ipv6_is_in_any_range", "ipv6_is_in_range", "ipv6_is_match", + "geo_distance_2points", "geo_info_from_ip_address", "geo_point_in_circle", "geo_point_in_polygon", "geo_point_to_geohash", "geo_point_to_s2cell", "geo_point_to_h3cell", + "indexof", "indexof_regex", "next", "prev", "row_cumsum", "row_number", "row_rank_dense", "row_rank_min", "row_window_session", + "pi", "pow", "radians", "rand", "round", "sign", "sin", "sqrt", "tan", + "series_abs", "series_acos", "series_add", "series_asin", "series_atan", "series_ ceiling", "series_cos", + "series_decompose", "series_decompose_anomalies", "series_decompose_forecast", "series_divide", "series_equals", + "series_exp", "series_fill_backward", "series_fill_const", "series_fill_forward", "series_fill_linear", + "series_fir", "series_fit_2lines", "series_fit_2lines_dynamic", "series_fit_line", "series_fit_line_dynamic", + "series_fit_poly", "series_greater", "series_greater_equals", "series_iir", "series_less", "series_less_equals", + "series_multiply", "series_not_equals", "series_outliers", "series_pearson_correlation", "series_periods_detect", + "series_periods_validate", "series_pow", "series_ seasonal", "series_sign", "series_sin", "series_stats", + "series_stats_dynamic", "series_subtract", "series_tan", + "column_ifexists", "columnifexists", "gettype", "typeof", "bool", "int", "long", "real", "double", "decimal", "string", "guid", " new_guid", + "autocluster", "basket", "diffpatterns", "narrow", "pivot", "rolling_percentile", "sql_request", "cosmosdb_sql_request", + "true", "false", "null", "by", "from", "to", "on", "with", "as", "asc", "desc", "step", "default", "kind", "in", +} +_STATEMENT_PREFIXES = { + "let", "set", "alias", "declare", "restrict", "evaluate", "access", "pattern", + "find", "search", "fork", "facet", "consume", "where", "project", "project-away", + "project-keep", "project-rename", "project-reorder", "extend", "summarize", + "order", "sort", "top", "take", "limit", "distinct", "count", "render", + "mv-expand", "mvexpand", "mv-apply", "mvapply", "parse", "parse-where", "parse-kv", + "make-series", "lookup", "join", "union", "invoke", "as", "partition", "serialize", + "sample", "sample-distinct", "scan", "reduce", "top-nested", "top-hitters", "getschema", +} + + +_QUOTED_LITERAL_RE = re.compile( + r"//[^\r\n]*|@\"(?:\"\"|[^\"])*\"|@'(?:''|[^'])*'|\"(?:\\.|[^\"\\])*\"|'(?:\\.|[^'\\])*'" +) +_BRACKETED_QUOTED_NAME_RE = re.compile( + r"\[\s*(@\"(?:\"\"|[^\"])*\"|@'(?:''|[^'])*'|\"(?:\\.|[^\"\\])*\"|'(?:\\.|[^'\\])*')\s*\]", + re.DOTALL, +) +_TABLE_LITERAL_CALL_RE = re.compile( + r"(? str: + if literal.startswith('@"') and literal.endswith('"'): + return literal[2:-1].replace('""', '"') + if literal.startswith("@'") and literal.endswith("'"): + return literal[2:-1].replace("''", "'") + if len(literal) >= 2 and literal[0] == literal[-1] and literal[0] in ('"', "'"): + inner = literal[1:-1] + return re.sub(r"\\(.)", r"\1", inner) + return literal + + +def _union_operands(code): + # Union operands are separated at the union's own depth, not by commas + # inside function arguments. Include later operands, not only the first. + for union in re.finditer(r"\bunion\b", code): + start = union.end() + flags = re.match(r"(?:\s+[A-Za-z_][\w.]*\s*=\s*[\w.*-]+)*", code[start:]) + start += flags.end() + depth, operand = 0, True + for index in range(start, len(code)): + char = code[index] + if operand and not char.isspace(): + source = re.match(r"(?:\(\s*)*([A-Za-z_]\w*)\s*(\()?", code[index:]) + if source: + yield source.group(1), bool(source.group(2)) + operand = False + if depth == 0 and char in "|;": + break + if char in "([{": + depth += 1 + elif char in ")]}": + if depth == 0: + break + depth -= 1 + elif char == "," and depth == 0: + operand = True + + +def _stripped_code(query: str, *, preserve_tokens: bool = False) -> str: + masked = _QUOTED_LITERAL_RE.sub(lambda m: " " * len(m.group(0)), query) + if preserve_tokens: + return masked + return re.sub( + r"\{[A-Za-z0-9_.:$ -]+\}", + lambda m: ( + m.group(0) + if re.search( + r"(?:\)|(? tuple[set[str], set[str]]: + calls: set[str] = set() + bare: set[str] = set() + # Mask single-line // comments only first, then inspect quoted constructs at top level. + no_comments = re.sub(r"//[^\r\n]*", lambda m: " " * len(m.group(0)), query) + protectedspans: list[tuple[int, int]] = [] + for m in _BRACKETED_QUOTED_NAME_RE.finditer(no_comments): + protectedspans.append(m.span()) + for m in _TABLE_LITERAL_CALL_RE.finditer(no_comments): + protectedspans.append(m.span()) + + # Check that each protected span does not start inside an outer string literal + valid_spans: set[tuple[int, int]] = set() + outer_string_spans: list[tuple[int, int]] = [] + idx = 0 + n = len(no_comments) + while idx < n: + for s, e in protectedspans: + if s == idx: + valid_spans.add((s, e)) + idx = e + break + else: + m_str = _QUOTED_LITERAL_RE.match(no_comments, idx) + if m_str: + outer_string_spans.append(m_str.span()) + idx = m_str.end() + continue + idx += 1 + + for m in _TABLE_LITERAL_CALL_RE.finditer(no_comments): + if m.span() in valid_spans: + name = _unquote_kql_string(m.group(1)).strip() + if name: + bare.add(name) + + # Fail closed on non-literal or unsupported table()/totable() indirection + for m in _TABLE_ANY_CALL_RE.finditer(no_comments): + if any(s <= m.start() < e for s, e in outer_string_spans): + continue + if not any(s == m.start() for s, _ in valid_spans): + calls.add("{__unsupported_table_indirection__}") + + for m in _BRACKETED_QUOTED_NAME_RE.finditer(no_comments): + if m.span() not in valid_spans: + continue + name = _unquote_kql_string(m.group(1)).strip() + if not name: + continue + before = no_comments[:m.start()] + after = no_comments[m.end():] + # Skip externaldata(...) [...] + if re.search(r"\bexternal_?data\s*\([^;|\r\n]*$", before, re.IGNORECASE): + continue + if re.match(r"^\s*\(", after): + calls.add(name) + if re.search(r"(?:[)\]}]|\.[A-Za-z_]\w*)\s*$", before): + calls.add("{__unsupported_escaped_call_context__}") + elif not re.search(r"(?:[)\]}]|\.[A-Za-z_]\w*)\s*$", before) and ( + (re.search(r"(?:^|[;{]|/\*|\*/|\bmaterialize\s*\(|\btoscalar\s*\(|\b(?:join|lookup|union)\b(?:\s+[A-Za-z_][\w.]*\s*=\s*[\w.*-]+)*)(?:\s*\()*\s*$", before, re.IGNORECASE) and re.match(r"^(?:\s*\))*\s*(?:\*/\s*)?\|", after)) + or (re.search(r"(\(\s*(?:\(\s*)*)$", before) and re.match(r"^((?:\s*\))*)\s*(?:\*/\s*)?\|", after) and re.search(r"(\(\s*(?:\(\s*)*)$", before).group(1).count("(") > re.match(r"^((?:\s*\))*)\s*(?:\*/\s*)?\|", after).group(1).count(")")) + or re.search(r"\binvoke\s*$", before) + or (re.search(r"\blet\s+[A-Za-z_]\w*\s*=(?:\s*\()*\s*$", before) and re.match(r"^(?:\s*\))*\s*[;|]", after)) + or (re.search(r"\blet\s+[A-Za-z_]\w*\s*=\s*(?:view\s*)?\([^)]*\)\s*\{(?:\s*\()*\s*$", before) and re.match(r"^(?:\s*\))*\s*[;|}]", after)) + or (re.search(r"\bview\s*\([^)]*\)\s*\{(?:\s*\()*\s*$", before) and re.match(r"^(?:\s*\))*\s*[;|}]", after)) + or (re.search(r"\bpartition\b(?:\s+hint\.[\w.]+\s*=\s*[\w.*-]+)*\s+by\s+[A-Za-z_]\w*\s*\{(?:\s*\()*\s*$", before, re.IGNORECASE) and re.match(r"^(?:\s*\))*\s*[;|}]", after)) + or (re.search(r"\bfork\b(?:\s+[A-Za-z_]\w*\s*=)?\s*\((?:\s*\()*\s*$", before, re.IGNORECASE) and re.match(r"^(?:\s*\))*\s*[|)]", after)) + or (re.search(r"\b(?:toscalar|materialize)\s*\((?:\s*\()*\s*$", before, re.IGNORECASE) and re.match(r"^(?:\s*\))*\s*[|)]", after)) + or (re.search(r"\b(?:join|lookup|union)\b(?:\s+[A-Za-z_][\w.]*\s*=\s*[\w.*-]+)*(?:\s*\()*\s*$", before) and not re.match(r"^\s*=", after)) + or re.search(r"\bunion\b[^;|]*,\s*(?:\(\s*)*$", before) + or re.search(r"\b(?:find|search)\b(?:\s+[A-Za-z_][\w.]*\s*=\s*[\w.*-]+)*\s+in\s*\([^)]*$", before, re.IGNORECASE) + ): + bare.add(name) + return calls, bare + + +def union_source_names(query: str) -> set[str]: + """Heuristic operands, not proof of a table or saved-function dependency.""" + code = _stripped_code(query) + locals = set(re.findall(r"\blet\s+([A-Za-z_]\w*)\s*=", code)) + return {name for name, _ in _union_operands(code)} - locals + + +def _bare_source_identifiers(code: str) -> set[str]: + code_no_flags = re.sub( + r"\b(?:union|join|lookup)\b(?:\s+[A-Za-z_][\w.]*\s*=\s*[\w.*-]+)+", + lambda m: re.sub(r"\s+[A-Za-z_][\w.]*\s*=\s*[\w.*-]+", " ", m.group(0)), + code, + ) + bare_matches = re.findall( + r"(?:^|[;{]|/\*|\*/|\bmaterialize\s*\(|\btoscalar\s*\(|\b(?:join|lookup|union)\b)(?:\s*\()*\s*([A-Za-z_]\w*)(?:\s*\))*\s*(?:\*/\s*)?(?=\|)" + r"|\(\s*([A-Za-z_]\w*)\s*(?:\*/\s*)?(?=\|)" + r"|\blet\s+[A-Za-z_]\w*\s*=(?:\s*\()*\s*([A-Za-z_]\w*)(?:\s*\))*\s*(?=[;|])" + r"|\blet\s+[A-Za-z_]\w*\s*=\s*(?:view\s*)?\([^)]*\)\s*\{(?:\s*\()*\s*([A-Za-z_]\w*)(?:\s*\))*\s*(?=[;|}])" + r"|\bview\s*\([^)]*\)\s*\{(?:\s*\()*\s*([A-Za-z_]\w*)(?:\s*\))*\s*(?=[;|}])" + r"|\bpartition\b(?:\s+hint\.[\w.]+\s*=\s*[\w.*-]+)*\s+by\s+[A-Za-z_]\w*\s*\{(?:\s*\()*\s*([A-Za-z_]\w*)(?:\s*\))*\s*(?=[;|}])" + r"|\b(?:toscalar|materialize)\s*\((?:\s*\()*\s*([A-Za-z_]\w*)(?:\s*\))*\s*(?=[|)])", + code_no_flags, + flags=re.IGNORECASE, + ) + bare = {name for groups in bare_matches for name in groups if name} + for m in re.finditer(r"(\(\s*(?:\(\s*)*)([A-Za-z_]\w*)((?:\s*\))*)\s*(?:\*/\s*)?(?=\|)", code_no_flags): + if m.group(1).count("(") > m.group(3).count(")"): + bare.add(m.group(2)) + for m in re.finditer(r"\bfork\b([^;|]*)", code_no_flags, re.IGNORECASE): + for bm in re.finditer(r"(?:[A-Za-z_]\w*\s*=\s*)?\((?:\s*\()*\s*([A-Za-z_]\w*)(?:\s*\))*\s*(?=[|)])", m.group(1)): + bare.add(bm.group(1)) + for op in re.finditer(r"\b(?:join|lookup)\b", code): + pos = op.end() + flags = re.match(r"(?:\s+[A-Za-z_][\w.]*\s*=\s*[\w.*-]+)*", code[pos:]) + pos += flags.end() + target = re.match(r"(?:\s*\()*\s*([A-Za-z_]\w*)\b(?!\s*[\(=.])", code[pos:]) + if target: + bare.add(target.group(1)) + for op in re.finditer(r"\b(?:find|search)\b(?:\s+[A-Za-z_][\w.]*\s*=\s*[\w.*-]+)*\s+in\s*\(([^)]*)\)", code, re.IGNORECASE): + for ident in re.findall(r"(? set[str]: + code = _stripped_code(query, preserve_tokens=True) + # Mask KQL function/view/partition brace bodies `{ ... }` so they are not misread as `{Param}` tokens. + code = re.sub( + r"((?:\)|(? m.group(3).count(")"): + tokens.add(m.group(2)) + for op in re.finditer(r"\b(?:union|join|lookup)\b", code): + pos = op.end() + flags = re.match(r"(?:\s+[A-Za-z_][\w.]*\s*=\s*[\w.*-]+)*", code[pos:]) + pos += flags.end() + target = re.match(rf"(?:\s*\()*\s*({tok_pat})", code[pos:]) + if target: + tokens.add(target.group(1)) + for op in re.finditer(r"\b(?:find|search)\b(?:\s+[A-Za-z_][\w.]*\s*=\s*[\w.*-]+)*\s+in\s*\(([^)]*)\)", code, re.IGNORECASE): + for tok in re.findall(tok_pat, op.group(1)): + tokens.add(tok) + return {tok for tok in tokens if re.search(r"[A-Za-z0-9_]", tok)} + + +def opaque_functions(query: str, *, include_bare: bool = True) -> set[str]: + """Conservative callable and bare saved-function scanner across all expressions.""" + code = _stripped_code(query) + quoted_calls, quoted_bare = _quoted_calls_and_table_sources(query) + code_no_flags = re.sub( + r"\b(?:union|join|lookup)\b(?:\s+([A-Za-z_][\w.]*)\s*=\s*([\w.*-]+))+", + lambda m: re.sub(r"\s+[A-Za-z_][\w.]*\s*=\s*[\w.*-]+", " ", m.group(0)), + code, + ) + code_no_flags = re.sub( + r"\bpartition\b(?:\s+hint\.[\w.]+\s*=\s*[\w.*-]+)*\s+by\s+[A-Za-z_]\w*\s*(?=[({])", + "partition ", + code_no_flags, + flags=re.IGNORECASE, + ) + calls = set(re.findall(r"(? dict: + """Return an intact review draft and report. Never resolves or executes live. + + resolutions maps original query pointers to {querySha256, tables, evidence}. + tables is an explicit caller-reviewed fallback, NOT a discovered table list. + """ + if source.get("version") != "Notebook/1.0" or not isinstance(source.get("items"), list): + raise ValueError("Expected a Notebook/1.0 document with an items array.") + doc = copy.deepcopy(source) + originals = query_surfaces(source) + surfaces = query_surfaces(doc) + entries, removed, retained_scope = _scope_parameters(doc) + removed_prefixes = [path + "/" for path, p in entries if p["name"] in removed] + issues, reports, gates = [], [], set() + + def issue(path, code, message): + issues.append({"path": path, "code": code, "message": message}) + + for path, p in entries: + if p["name"] in retained_scope: + issue(path, "scope-used-by-content", + f"{p['name']} is used by retained business content or a drilldown. " + "Retained; manually replace its Azure scope semantics.") + + converted_nodes = set() + for surface, original in zip(surfaces, originals): + path, node, qt = surface["path"], surface["node"], surface["queryType"] + if any(path.startswith(prefix) for prefix in removed_prefixes): + reports.append({"sourcePath": path, "status": "removed-scope-helper"}) + continue + entry = {"sourcePath": path, "parameter": surface["parameter"]} + reports.append(entry) + if qt not in (0, None, AH) or ( + qt != AH and (surface["resourceType"] or "").lower() not in ("", LA) + ): + entry["status"] = "blocked" + issue(path, "unsupported-provider", + f"Provider {qt!r}/{surface['resourceType']!r} is retained unchanged; " + "supply a reviewed provider migration.") + continue + try: + raw = _raw_query(node["query"], qt) + except ValueError as exc: + entry["status"] = "blocked" + issue(path, "query-format", str(exc)) + continue + code = "".join(text if kind == "code" else " " * len(text) + for kind, text in rewrite._segments(raw)) + if re.search(r"\b(?:workspace|app|resource|cluster|database)\s*\(", code, + re.IGNORECASE): + entry["status"] = "blocked" + issue(path, "explicit-scope-function", + "Explicit workspace/app/resource/cluster/database function cannot be " + "replaced by blank AH selection. Keep this panel and review its sources.") + continue + evidence = (resolutions or {}).get(path) + reviewed_tables = (evidence.get("tables") if isinstance(evidence, dict) + and isinstance(evidence.get("tables"), list) else tables) or () + explicit_calls = opaque_functions(raw, include_bare=False) + opaque = opaque_functions(raw) - ( + set(reviewed_tables) - explicit_calls - set(allowed_workspace_functions) + - (set(evidence.get("workspaceFunctions", ())) + if isinstance(evidence, dict) and isinstance(evidence.get("workspaceFunctions"), list) + else set()) + ) + function_names = evidence.get("workspaceFunctions", []) if isinstance(evidence, dict) else [] + if (not isinstance(function_names, list) + or any(not isinstance(name, str) for name in function_names) + or len(set(function_names)) != len(function_names) + or set(function_names) != opaque + or not opaque <= set(allowed_workspace_functions)): + entry["status"] = "blocked" + issue(path, "saved-function", + f"Source function(s) {', '.join(sorted(opaque))} need a reviewed AH " + "workspace definition, exact aliases and separate underlying table evidence.") + continue + resolved = tables + entry["resolutionEvidence"] = "caller-reviewed tables" if tables is not None else None + if evidence is not None: + if (not isinstance(evidence, dict) + or evidence.get("querySha256") != query_hash(original["node"]["query"]) + or not isinstance(evidence.get("tables"), list) + or not all(isinstance(t, str) and t for t in evidence["tables"]) + or not evidence.get("evidence")): + entry["status"] = "blocked" + issue(path, "stale-resolution", + "Supply tables, evidence and matching SHA256 of the raw query at this path.") + continue + resolved = evidence["tables"] + entry["resolutionEvidence"] = evidence["evidence"] + if opaque: + entry["workspaceFunctions"] = sorted(opaque) + if not resolved or opaque & set(resolved): + entry["status"] = "blocked" + issue(path, "function-underlying-tables", + "Supply underlying source tables separately from workspace function aliases.") + continue + contract = migrator.convert(raw, tables=resolved, source_id=path, + artifact_type="workbook", resolve=False) + entry["conversion"] = contract + if "Watchlist" in contract["resolvedTables"]: + entry["referenceInputs"] = ["Watchlist"] + entry["status"] = contract["status"] + if contract["status"] == "blocked" or resolved is None: + issue(path, "mapping-or-resolution", + contract["blockedReason"] or + "No offline source evidence. Supply --resolutions (per-path query hash, " + "tables and evidence) or --tables after reviewing ALL query sources.") + continue + if contract["unsupported"] or contract["status"] == "needs-review": + issue(path, "query-review", + "; ".join(x["reason"] for x in contract["unsupported"]) + or "Shared migrator requires source/mapping review; conversion is not deterministic.") + if opaque and contract["convertedKql"] != raw: + entry["status"] = "blocked" + issue(path, "function-query-rewrite", + "Shared conversion would modify a preserved function query; review a source change separately.") + continue + node["query"] = contract["convertedKql"] + node["queryType"] = AH + converted_nodes.add(id(node)) + if surface["parameter"]: + gates.add("ah-query-backed-parameters") + # The mapping catalog's resolve-to-self does not establish blank-selection visibility. + gates.add("blank-ah-selection-table-visibility") + if opaque: + gates.add("sentinel-workspace-function-availability-and-branch-coverage") + if re.search(r"\bexternaldata\s*[\[(]", code): + gates.add("external-source-binding-and-availability") + if references(raw) - references(node["query"]): + raise AssertionError(f"Parameter templates lost at {path}") + + # Blocked panels keep the scope inputs they still need. Revisit the graph + # before removing any helper; an unsupported sibling is not disposable. + blocked = [s for s, r in zip(surfaces, reports) if r["status"] == "blocked" + or (id(s["node"]) not in converted_nodes + and r["status"] != "removed-scope-helper")] + if blocked: + protected = set() + nodes = dict(walk(doc)) + for surface in blocked: + protected |= references(surface["node"]) + ancestor = surface["path"].rsplit("/", 1)[0] + while True: + node = nodes.get(ancestor, {}) + for key in SCOPE_KEYS | {"resourceContextFromParameter"}: + if key in node: + protected |= references({key: node[key]}) + if not ancestor: + break + ancestor = ancestor.rsplit("/", 1)[0] + entries, removed, retained_scope = _scope_parameters(doc, protected) + removed_prefixes = [path + "/" for path, p in entries if p["name"] in removed] + for surface, entry in zip(surfaces, reports): + if entry["status"] == "removed-scope-helper" and not any( + surface["path"].startswith(prefix) for prefix in removed_prefixes + ): + entry["status"] = "retained-scope-helper" + issue(surface["path"], "blocked-panel-scope", + "Helper retained because an unsupported panel still needs its scope.") + + # Keep item arrays and layout untouched, including now-empty parameter groups. + for _, node in walk(doc): + if isinstance(node.get("parameters"), list): + node["parameters"] = [p for p in node["parameters"] + if p.get("name") not in removed] + + retained_scope_nodes = {id(p) for _, p in entries if p["name"] in retained_scope} + + def clean(value, path=""): + if isinstance(value, dict): + descendants = query_surfaces(value) + safe = all(id(s["node"]) in converted_nodes for s in descendants) + binding_container = ( + bool(descendants) or not path + or "parameters" in value or "items" in value + or re.search(r"/parameters/\d+$", path) + or str(value.get("version", "")).startswith(("KqlItem/", "KqlParameterItem/", + "NotebookGroup/")) + ) + if id(value) in converted_nodes or ( + safe and binding_container and id(value) not in retained_scope_nodes + ): + for key in SCOPE_KEYS | {"resourceContextFromParameter"}: + value.pop(key, None) + if value.get("resourceType", "").lower() in ( + LA, "microsoft.resourcegraph/resources", "microsoft.insights/components" + ): + value.pop("resourceType", None) + if "parameters" in value or "items" in value: + value.pop("queryType", None) + context = value.get("context") + if isinstance(context, dict) and isinstance(context.get("ownerId"), str): + owner = context["ownerId"] + if "/subscriptions/" in owner.lower() or references(owner) & removed: + context.pop("ownerId") + for key, child in list(value.items()): + clean(child, path + "/" + key) + elif isinstance(value, list): + for index, child in enumerate(value): + clean(child, f"{path}/{index}") + + clean(doc) + output_paths = {id(s["node"]): s["path"] for s in query_surfaces(doc)} + for surface, entry in zip(surfaces, reports): + entry["outputPath"] = output_paths.get(id(surface["node"])) + return { + "workbook": doc, + "report": { + "status": "blocked" if issues else ("needs-validation" if gates else "converted"), + "queries": reports, + "removedParameters": [{"name": p["name"], "sourcePath": path} + for path, p in entries if p["name"] in removed], + "issues": issues, + "gates": [{"code": gate, "status": "unverified"} for gate in sorted(gates)], + "validation": {"localStructureOnly": True, "runtime": "not-run", "ui": "not-run"}, + }, + } diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/workbook_fixtures.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/workbook_fixtures.py new file mode 100644 index 00000000000..c278b578d38 --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/workbook_fixtures.py @@ -0,0 +1,113 @@ +"""Deterministic, actual-row workbook harnesses and exact result assertions. + +Emitting KQL is not executing KQL. No live clients are imported by this module. +""" +from __future__ import annotations + +import argparse +from collections import Counter +from datetime import datetime +import json +from pathlib import Path +import re + +from . import dataframes, workbook + + +def substitute(query: str, substitutions: dict[str, str], clock: str) -> str: + """Explicit fixture literals, never production template neutralization.""" + token = re.compile(r"\{([A-Za-z_][\w .-]*(?::[^{}]*)?)\}") + missing = set(token.findall(query)) - substitutions.keys() + if missing: + raise ValueError(f"Missing explicit fixture substitutions: {sorted(missing)}") + query = token.sub(lambda m: substitutions[m[1]], query) + # Only executable now() is fixed; strings/comments remain untouched. + result = "".join( + re.sub(r"\bnow\s*\(\s*\)", f"datetime({clock})", text) if kind == "code" else text + for kind, text in workbook.rewrite._segments(query) + ) + if any(re.search(r"\b(?:now|ago)\s*\(", text) for kind, text in + workbook.rewrite._segments(result) if kind == "code"): + raise ValueError("Express relative time against the pinned fixture clock explicitly.") + return result + + +def assert_result(expected: dict, actual: dict) -> None: + """Compare names/types, cells, nulls, aggregates and bins, not just row counts.""" + if actual.get("columns") != expected["columns"]: + raise AssertionError(f"Column mismatch: {actual.get('columns')} != {expected['columns']}") + rows = actual.get("rows") + if not isinstance(rows, list) or any(len(row) != len(expected["columns"]) for row in rows): + raise AssertionError("Invalid result row shape") + encode = lambda row: json.dumps(row, sort_keys=True, separators=(",", ":")) + if expected.get("ordered", False): + equal = list(map(encode, rows)) == list(map(encode, expected["rows"])) + else: + equal = Counter(map(encode, rows)) == Counter(map(encode, expected["rows"])) + if not equal: + raise AssertionError(f"Cell/aggregate/time-bin mismatch: {rows} != {expected['rows']}") + if expected.get("sortGroups"): + groups = expected["sortGroups"] + index = [column["name"] for column in expected["columns"]].index(groups["column"]) + ranks = [groups["ranks"][row[index]] for row in rows] + if ranks != sorted(ranks): + raise AssertionError("Result sort-group order mismatch") + + +def harness_files(manifest: dict, source: dict, converted: dict) -> dict[str, str]: + """Build artifacts in memory so callers can validate before writing.""" + clock = manifest["clock"] + if manifest.get("seed") is None: + raise ValueError("Manifest must pin a seed (rows are explicit; no random generation).") + source_queries = {s["path"]: s["node"]["query"] for s in workbook.query_surfaces(source)} + converted_queries = {s["path"]: s["node"]["query"] for s in workbook.query_surfaces(converted)} + files = {} + for case in manifest["cases"]: + case_id = case["id"] + if not re.fullmatch(r"[A-Za-z0-9_-]+", case_id): + raise ValueError("Fixture case IDs must be filesystem-safe.") + for side, queries, path in ( + ("sentinel", source_queries, case["sourcePath"]), + ("ah", converted_queries, case.get("outputPath", case["sourcePath"])), + ): + query = substitute(queries[path], case["substitutions"], clock) + stubs = [] + for table, fixture in manifest["tables"].items(): + schema, rows = fixture["schema"], fixture["rows"] + known = dataframes.schema_for(table, side) + if known and any(known.get(c) != t for c, t in schema.items()): + raise ValueError(f"Fixture schema differs from cached {side} schema: {table}") + stub = dataframes.datatable_stub(table, schema, rows=rows) + window = case.get("tableTimeRanges", {}).get(table) + if window: + column = window["column"] + if schema.get(column) != "datetime": + raise ValueError("Fixture time context must name a datetime column.") + start, end = window["start"], window["end"] + parsed = [datetime.fromisoformat(v.replace("Z", "+00:00")) for v in (start, end)] + if any(v.tzinfo is None for v in parsed) or parsed[0] > parsed[1]: + raise ValueError("Invalid fixture time context.") + # Model the workbook provider's external timespan on the input, + # never filter after an aggregation that removed TimeGenerated. + stub = stub[:-1] + ( + f"\n| where {dataframes._col_name(column)} between " + f"(datetime({start}) .. datetime({end}));") + stubs.append(stub) + filename = f"{case_id}.{side}.kql" + if filename in files: + raise ValueError("Duplicate fixture case ID.") + files[filename] = ( + "// FIXTURE ONLY: explicit rows and fixed clock; not shipped.\n" + + "\n".join(stubs) + "\n" + query + "\n") + expected = {case["id"]: case["expected"] for case in manifest["cases"]} + files["expected.json"] = json.dumps(expected, indent=2) + "\n" + return files + + +def emit(manifest: dict, source: dict, converted: dict, out: Path) -> dict: + files = harness_files(manifest, source, converted) + out.mkdir(parents=True, exist_ok=True) + for name, text in files.items(): + (out / name).write_text(text, encoding="utf-8") + return {"files": [name for name in files if name.endswith(".kql")], + "execution": "not-run", "seed": manifest["seed"], "clock": manifest["clock"]} diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/workbook_mock.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/workbook_mock.py new file mode 100644 index 00000000000..e55b3f12104 --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/workbook_mock.py @@ -0,0 +1,298 @@ +"""Generate and validate reviewed workbook mock bundles locally. No live actions.""" +from __future__ import annotations + +import argparse +from datetime import timezone +import hashlib +import json +from pathlib import Path +import re +import sys + +from . import dataframes, workbook, workbook_fixtures, workbook_mock_recipes as recipes + +VERSION = 1 +DEFAULT_CLOCK = "2026-10-07T12:00:00Z" + + +class Blocked(ValueError): + def __init__(self, blockers): + self.blockers = blockers + super().__init__("No reviewed mock contract for one or more workbook inputs.") + + +def json_bytes(value) -> bytes: + return (json.dumps(value, indent=2, ensure_ascii=True, allow_nan=False) + "\n").encode("utf-8") + + +def digest(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def configuration(seed, clock, scale, recipe, run_id=None) -> dict: + if type(seed) is not int or not 0 <= seed <= 2**32 - 1: + raise ValueError("seed must be an integer in 0..4294967295") + if type(scale) is not int or not 1 <= scale <= 10: + raise ValueError("scale must be an integer in 1..10") + if recipe not in recipes.RECIPES: + raise ValueError("Unsupported recipe; use list-recipes to see reviewed recipes.") + if not isinstance(clock, str) or not re.fullmatch(r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ", clock): + raise ValueError("clock must be a fixed UTC timestamp: YYYY-MM-DDTHH:MM:SSZ") + parsed = recipes.instant(clock) + if not 1971 <= parsed.year <= 9998: + raise ValueError("clock year must be in 1971..9998") + if run_id is not None and (not isinstance(run_id, str) or + not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,47}", run_id)): + raise ValueError("run-id must be 1..48 lowercase ASCII letters, digits or hyphens") + return {"seed": seed, "clock": clock, "scale": scale, "recipe": recipe, "runId": run_id} + + +def inventory(source: dict, recipe_id: str = "syslog-operational-v1") -> list[dict]: + if not isinstance(source, dict) or source.get("version") != "Notebook/1.0" \ + or not isinstance(source.get("items"), list): + raise ValueError("Expected a Notebook/1.0 document with an items array.") + entries, blockers = [], [] + recipe = recipes.RECIPES[recipe_id] + nodes = dict(workbook.walk(source)) + for surface in workbook.query_surfaces(source): + path, node = surface["path"], surface["node"] + inherited_time = {} + ancestor = path.rsplit("/", 1)[0] + while True: + for key in ("timeContextFromParameter", "timeContext"): + if key in nodes.get(ancestor, {}): + inherited_time.setdefault(key, nodes[ancestor][key]) + if not ancestor: + break + ancestor = ancestor.rsplit("/", 1)[0] + query = node["query"] + sha = workbook.query_hash(query) if isinstance(query, str) else None + contract = recipe.contracts.get(sha) + reason = None + if surface["queryType"] not in (0, None) or surface["resourceType"] not in (None, workbook.LA): + reason = "Unsupported source provider; reviewed contracts require Sentinel Logs KQL." + elif contract is None: + reason = ("Unreviewed query/table/function. Register an exact query dependency and " + "oracle contract plus reviewed table adapters; no generic rows are generated.") + elif any(table not in recipes.TABLE_ADAPTERS for table in contract.tables): + reason = "Missing reviewed table generator." + elif surface["parameter"] != contract.parameter: + reason = "Reviewed query moved between panel and parameter contexts." + elif inherited_time.get("timeContextFromParameter") != contract.time_parameter: + reason = "External time-context binding differs from the reviewed query contract." + elif inherited_time.get("timeContext", {}) not in ({}, {"durationMs": 0}): + reason = "Unreviewed fixed time-context override." + if reason: + blockers.append({"path": path, "querySha256": sha, "reason": reason}) + else: + entries.append({"sourcePath": path, "querySha256": sha, "kind": contract.kind, + "tables": list(contract.tables), "parameter": contract.parameter, + "timeParameter": contract.time_parameter}) + if not entries and not blockers: + blockers.append({"path": "", "reason": "Workbook contains no reviewed query surfaces."}) + parameters = {node["name"]: node for path, node in workbook.walk(source) + if re.search(r"/parameters/\d+$", path) and node.get("name")} + blockers.extend(recipe.parameter_blockers(parameters, entries)) + if blockers: + raise Blocked(blockers) + return entries + + +def validate_rows(table: str, schema: dict, rows: list[list]) -> None: + """Strict destination-table validation; never pretend this is DCR validation.""" + for side in ("sentinel", "ah"): + known = dataframes.schema_for(table, side) + if known is None or schema != known: + raise ValueError(f"Reviewed {table} schema differs from cached {side} schema.") + if not isinstance(rows, list) or len(rows) < 2: + raise ValueError(f"{table} recipe must produce multiple typed rows.") + for index, row in enumerate(rows): + if not isinstance(row, list) or len(row) != len(schema): + raise ValueError(f"{table} row {index}: invalid shape") + for (column, typ), value in zip(schema.items(), row): + valid = False + if typ == "string": + valid = isinstance(value, str) + elif typ in ("int", "long"): + bits = 32 if typ == "int" else 64 + valid = value is None or (type(value) is int and -2**(bits-1) <= value < 2**(bits-1)) + elif typ == "datetime": + if isinstance(value, str): + try: + parsed = recipes.instant(value) + valid = parsed.tzinfo is not None and parsed.utcoffset() == timezone.utc.utcoffset(parsed) + except ValueError: + pass + else: + raise ValueError(f"No reviewed type validator for {table}.{column}: {typ}") + if not valid: + raise ValueError(f"{table} row {index}: invalid {typ} in {column}") + + +def build(source_bytes: bytes, config: dict) -> dict[str, bytes]: + config = configuration(config["seed"], config["clock"], config["scale"], + config["recipe"], config.get("runId")) + recipe = recipes.RECIPES[config["recipe"]] + source = json.loads(source_bytes) + queries = inventory(source, config["recipe"]) + source_hash = digest(source_bytes) + implementation = {p.name: digest(p.read_bytes()) for p in ( + Path(__file__), Path(recipes.__file__), Path(workbook_fixtures.__file__), + Path(dataframes.__file__))} + identity = {"sourceSha256": source_hash, "config": config, + "implementation": implementation, "spec": recipe.spec} + run_id = config["runId"] or "wb-" + digest(json_bytes(identity))[:16] + tables, records_by_table = {}, {} + for table in sorted({table for query in queries for table in query["tables"]}): + adapter = recipes.TABLE_ADAPTERS[table] + records = adapter.generate(config["seed"], config["clock"], config["scale"], run_id) + if any(set(record) != set(adapter.schema) for record in records): + raise ValueError(f"{table} adapter generated missing or undeclared fields.") + rows = [[record[column] for column in adapter.schema] for record in records] + validate_rows(table, adapter.schema, rows) + tables[table] = {"schema": adapter.schema, "rows": rows} + records_by_table[table] = records + resolutions = {entry["sourcePath"]: { + "querySha256": entry["querySha256"], "tables": entry["tables"], + "evidence": f"reviewed workbook mock recipe {config['recipe']}, generator v{VERSION}", + } for entry in queries} + converted = workbook.convert_workbook(source, resolutions=resolutions) + if converted["report"]["status"] == "blocked": + raise Blocked(converted["report"]["issues"]) + outputs = {entry["sourcePath"]: entry for entry in converted["report"]["queries"]} + output_surfaces = {s["path"]: s for s in workbook.query_surfaces(converted["workbook"])} + choices = recipe.selections(records_by_table, config["clock"]) + cases = [] + for index, query in enumerate(queries): + output_path = outputs[query["sourcePath"]]["outputPath"] + output_query = output_surfaces[output_path]["node"]["query"] + if workbook.query_hash(output_query) != query["querySha256"]: + raise Blocked([{"path": output_path, + "reason": "AH rewrite lacks a reviewed fixture mapping/oracle."}]) + query["outputPath"] = output_path + contract = recipe.contracts[query["querySha256"]] + for selection in choices: + case = { + "id": f"q{index:02d}-{selection['id']}", "sourcePath": query["sourcePath"], + "outputPath": output_path, "querySha256": query["querySha256"], + "selection": selection, "substitutions": recipe.substitutions(selection), + "expected": recipe.expected(contract, records_by_table, selection), + } + if contract.time_parameter: + case["tableTimeRanges"] = {table: { + "column": contract.time_column, "start": selection["start"], "end": selection["end"]} + for table in contract.tables} + cases.append(case) + manifest = { + "version": VERSION, "generator": "workbook_mock", "runId": run_id, + "provenance": {"kind": "generated-synthetic-operational", "sourceSha256": source_hash, + "recipeSha256": digest(json_bytes(recipe.spec)), + "implementation": implementation}, + "config": config, "seed": config["seed"], "clock": config["clock"], + "recipe": recipe.spec, "inventory": queries, "tables": tables, "cases": cases, + "ingestion": {"status": "not-ingestible", "directLogsIngestionSupported": False, + "stream": None, "payload": None, + "reason": "No exact custom input stream and transform verified for " + + ", ".join(tables) + ". Destination table fixtures are not raw connector payloads."}, + "validation": {"schema": "passed", "harnessAssembly": "checked", + "oracle": "reviewed-python-reference", "kqlExecution": "not-run", + "engineParity": "not-run", "portalParameterSerialization": "unverified"}, + "limitations": [ + "Only exact reviewed query hashes and registered table adapters are supported.", + "Time context is modeled as inclusive TimeGenerated bounds on the input table; " + "this is not a verification of the portal provider's boundary behavior.", + "Computer picker has no time binding and intentionally sees the complete fixture.", + "Computer:lable is an upstream typo retained unchanged. Its fixture substitution " + "is explicit, not proof that the portal resolves this token.", + "Expanded-all and literal All/* are separate contracts. Events has no wildcard " + "clause and returns zero for literal All/*; no source query is silently fixed.", + "Severity tiles ignore Computer. Emergency timecharts include emerg, not panic; " + "timecharts group exact timestamps, not hourly bins.", + "Synthetic severity levels are operational test coverage, not attack evidence.", + ], + } + harnesses = workbook_fixtures.harness_files(manifest, source, converted["workbook"]) + # A recipe-level diagnostic exercises hour-boundary bins independently of + # the nine panels (which group raw timestamps). It is not a new workbook panel. + diagnostic = recipe.diagnostic(records_by_table) + stub = "\n".join(dataframes.datatable_stub(table, tables[table]["schema"], + rows=tables[table]["rows"]) + for table in diagnostic["tables"]) + files = {"manifest.json": json_bytes(manifest), "diagnostic.json": json_bytes(diagnostic)} + for table, records in records_by_table.items(): + files[f"tables/{table}.json"] = json_bytes(records) + files[f"tables/{table}.schema.json"] = json_bytes(tables[table]["schema"]) + # Adjacent guard is understood by the future Azure-Sentinel ingestion tool. + files["tables/scenario.json"] = json_bytes({ + "generationStatus": "table-fixture-only", "runId": run_id, + "ingestion": manifest["ingestion"], + "validation": {"schemaValidation": "passed", "rawStreamValidation": "not-run"}, + "fixtures": {"tableRecords": sum(len(records) for records in records_by_table.values())}, + }) + for name, text in harnesses.items(): + files[f"harnesses/{name}"] = text.encode("utf-8") + for side in ("sentinel", "ah"): + files[f"harnesses/{diagnostic['id']}.{side}.kql"] = ( + "// FIXTURE DIAGNOSTIC ONLY; not a workbook panel.\n" + stub + "\n" + + diagnostic["query"] + "\n").encode("utf-8") + files["checksums.json"] = json_bytes({name: digest(content) for name, content in sorted(files.items())}) + return files + + +def generate(source: Path, out: Path, config: dict) -> dict: + if out.exists() or out.is_symlink(): + raise ValueError("Output must be a new directory; existing evidence is never overwritten.") + if out.resolve() == source.resolve() or source.resolve().is_relative_to(out.resolve()): + raise ValueError("Output must not contain or replace the source.") + before = source.read_bytes() + files = build(before, config) + if source.read_bytes() != before: + raise ValueError("Source changed during generation; no output written.") + out.mkdir(parents=True, exist_ok=False) + for name, content in files.items(): + path = out / name + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("xb") as handle: + handle.write(content) + manifest = json.loads(files["manifest.json"]) + return {"status": "generated-local", "runId": manifest["runId"], + "sourceSha256": manifest["provenance"]["sourceSha256"], + "rows": sum(len(t["rows"]) for t in manifest["tables"].values()), + "cases": len(manifest["cases"]), "harnesses": len(manifest["cases"]) * 2 + 2, + "ingestion": "not-ingestible", "kqlExecution": "not-run"} + + +def validate(source: Path, bundle: Path) -> dict: + manifest = json.loads((bundle / "manifest.json").read_bytes()) + expected = build(source.read_bytes(), manifest["config"]) + paths = {p.relative_to(bundle).as_posix() for p in bundle.rglob("*") if p.is_file()} + if paths != expected.keys(): + raise ValueError("Bundle file inventory mismatch (missing or unexpected artifacts).") + for name, content in expected.items(): + path = bundle / name + if path.is_symlink() or path.read_bytes() != content: + raise ValueError(f"Offline regeneration mismatch: {name}") + return {"status": "offline-checks-passed", "files": len(expected), + "checks": ["source/recipe/code hashes", "typed rows", "reference cells/aggregates/bins", + "parameter cases", "actual-row harness bindings", "artifact integrity"], + "kqlExecution": "not-run", "engineParity": "not-run", "ingestion": "not-ingestible"} + + +def assert_captured(bundle: Path, results_path: Path, side: str) -> dict: + manifest = json.loads((bundle / "manifest.json").read_bytes()) + captured = json.loads(results_path.read_bytes()) + expected = {case["id"]: case["expected"] for case in manifest["cases"]} + diagnostic = json.loads((bundle / "diagnostic.json").read_bytes()) + expected[diagnostic["id"]] = diagnostic["expected"] + if not isinstance(captured, dict) or captured.keys() != expected.keys(): + raise ValueError("Captured results must contain every case ID and fixture diagnostic, without extras.") + for case_id, oracle in expected.items(): + try: + workbook_fixtures.assert_result(oracle, captured[case_id]) + except (AssertionError, TypeError, KeyError) as exc: + raise ValueError( + f"Captured result assertion failed: {side}/{case_id}; " + "column, cell, multiplicity or ordering mismatch (captured values omitted)." + ) from exc + return {"status": "captured-results-match", "side": side, "cases": len(expected), + "execution": "external-capture-not-performed-by-this-tool"} diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/workbook_mock_recipes.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/workbook_mock_recipes.py new file mode 100644 index 00000000000..139e6d277ff --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/workbook_mock_recipes.py @@ -0,0 +1,292 @@ +"""Reviewed operational recipes, not a KQL constraint solver or attack generator. + +Table adapters own correlations and typed rows. Query contracts separately own +source dependencies, external time context, parameter semantics and exact oracles. +Unknown hashes require review; a known table alone never authorizes fake rows. +""" +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass +from datetime import datetime, timedelta +import json +import random +from typing import Callable + + +def instant(value: str) -> datetime: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def stamp(value: datetime) -> str: + return value.isoformat(timespec="seconds").replace("+00:00", "Z") + + +SEVERITIES = { + "emerg": (0, "Emergency"), "panic": (0, "Emergency"), + "alert": (1, "Alert"), "crit": (2, "Critical"), + "err": (3, "Error"), "error": (3, "Error"), + "warning": (4, "Warning"), "warn": (4, "Warning"), + "notice": (5, "Notice"), "info": (6, "Informational"), + "debug": (7, "Debug"), "unknown": (8, "Unknown"), +} +LABELS = list(dict.fromkeys(label for _, label in SEVERITIES.values() if label != "Unknown")) +SYSLOG_SCHEMA = { + "CollectorHostName": "string", "Computer": "string", "EventTime": "datetime", + "Facility": "string", "HostIP": "string", "HostName": "string", "MG": "string", + "ProcessID": "int", "ProcessName": "string", "SeverityLevel": "string", + "SourceSystem": "string", "SyslogMessage": "string", "TenantId": "string", + "TimeGenerated": "datetime", "Type": "string", "_ResourceId": "string", +} +SYSLOG_SPEC = { + "id": "syslog-operational-v1", + "purpose": "Synthetic operational workbook mechanics; no malicious behavior or SOC-value claim.", + "evidence": [ + "Syslog/Workbooks/LinuxMachines.json: ten exact reviewed source queries", + "data/la_schema_catalog.json and data/ah_schema_catalog.json: Syslog types", + "docs/Standard-table-mock-ingestion-onboarding.md: table rows are not a raw input contract", + ], + "hostWeights": [1, 2, 3], + "severityWeights": dict(zip(SEVERITIES, [1, 1, 2, 3, 2, 1, 4, 2, 5, 8, 3, 1])), + "processes": [["cron", "cron", 120], ["systemd", "daemon", 1], + ["sshd", "authpriv", 900], ["kernel", "kern", None]], + "windowSeconds": 21600, + "boundaryOffsetsSeconds": [-21601, -21600, -18001, -18000, -1, 0, 1], + "correlations": ["Computer -> HostName, HostIP, _ResourceId", + "Computer + ProcessName -> ProcessID", + "ProcessName -> Facility, synthetic operational message"], + "variation": "Seeded process choice and seconds within six hours; weighted hosts/severities, scale 1..10.", +} + + +def syslog_rows(seed: int, clock: str, scale: int, marker: str) -> list[dict]: + rng = random.Random(seed) + base = instant(clock) + rows = [] + for host_index, weight in enumerate(SYSLOG_SPEC["hostWeights"]): + host = f"mock-{marker}-{host_index + 1}" + resource = (f"/subscriptions/00000000-0000-0000-0000-000000000000/" + f"resourceGroups/mock-only/providers/Microsoft.Compute/virtualMachines/{host}" + if host_index == 1 else "") + + def event(severity, offset, process_index): + process, facility, pid = SYSLOG_SPEC["processes"][process_index] + timestamp = stamp(base + timedelta(seconds=offset)) + return dict(zip(SYSLOG_SCHEMA, [ + "mock-collector", host, timestamp, facility, f"192.0.2.{10 + host_index}", + host, "", None if pid is None else pid + host_index * 1000, process, + severity, "Linux", f"MOCK {marker}: synthetic {process} operational test", + "", timestamp, "Syslog", resource, + ])) + + for severity, count in SYSLOG_SPEC["severityWeights"].items(): + for _ in range(count * weight * scale): + rows.append(event(severity, -rng.randrange(1, 21600), + rng.randrange(len(SYSLOG_SPEC["processes"])))) + for offset in SYSLOG_SPEC["boundaryOffsetsSeconds"]: + # Deliberate duplicates verify count(), not just distinct output shape. + rows.extend([event("emerg", offset, 3)] * (host_index + 1)) + return rows + + +@dataclass(frozen=True) +class TableAdapter: + table: str + schema: dict[str, str] + spec: dict + generate: Callable + + +TABLE_ADAPTERS = {"Syslog": TableAdapter("Syslog", SYSLOG_SCHEMA, SYSLOG_SPEC, syslog_rows)} + + +@dataclass(frozen=True) +class QueryContract: + kind: str + tables: tuple[str, ...] = ("Syslog",) + parameter: bool = False + time_parameter: str | None = "TimeRange" + time_column: str = "TimeGenerated" + + +# Exact UTF-8 hashes include whitespace and templates. No regex source guessing. +QUERY_CONTRACTS = { + "7e4626fc678e25bfbb8010e213842a4db34ec565bf1d6b2f301d679539f0e658": + QueryContract("computers", parameter=True, time_parameter=None), + "24464f7e0e663896659523f152ad9566a9a33d479834cbc681932dcb8960f3ea": + QueryContract("severity"), + "206654d1ec09162aa64ff5d8ef0b72484cc4aea0eb58786714589e90c447ca7a": + QueryContract("emerg"), + "8661e28bd84bfe3a117cd6391d3d330aedc5d2af95534ffb8bae4cbf2371a28d": + QueryContract("crit"), + "71dad0100fcd2088e011ec965f7731eb4314df0419360377a874e24a0a319f54": + QueryContract("alert"), + "a569777af35afbe110ff64ced2df34f1d96a088fbba88e53372e56a5fc917d8a": + QueryContract("events"), + "43b873d68c1dfffa1fb72d48e3121abf844f31e98ce1982bad0af3844a344da9": + QueryContract("messages"), + "1eddbe8eae830de68f7a473ecf6e08e984430295d9436b3ca490fb1aa81eeced": + QueryContract("processes"), + "fb8370f59c577695bb940c54ca190a8d25ad09775e38b0e411996b63a996b761": + QueryContract("facilities"), + "335b9b05951ed2347c0d56c0a25bad137f6bca005500c80f4fef1f5d74d2eaed": + QueryContract("facilities"), +} + + +def selections(rows: list[dict], clock: str) -> list[dict]: + hosts = sorted({row["Computer"] for row in rows}) + base = instant(clock) + result = [] + for name, computers, severity, start, end, label in [ + ("all-expanded", hosts, LABELS, -21600, 0, "All"), + ("all-literal", ["All"], ["*"], -21600, 0, "All"), + ("single", hosts[:1], LABELS, -21600, 0, hosts[0]), + ("multiple", hosts[:2], ["Warning", "Error"], -21600, 0, "Multiple"), + ("single-severity", hosts, ["Emergency"], -21600, 0, "All"), + ("empty-computers", [], LABELS, -21600, 0, ""), + ("empty-severity", hosts, [], -21600, 0, "All"), + ("narrow-time", hosts, LABELS, -3600, 0, "All"), + ("end-boundary", hosts, LABELS, 0, 0, "All"), + ("empty-time", hosts, LABELS, 60, 120, "All"), + ]: + result.append({"id": name, "Computer": computers, "Severity": severity, + "computerLabel": label, + "start": stamp(base + timedelta(seconds=start)), + "end": stamp(base + timedelta(seconds=end))}) + return result + + +def literals(values: list[str]) -> str: + # All values originate in reviewed recipes, but still quote as KQL data. + return ",".join("'" + v.replace("\\", "\\\\").replace("'", "\\'") + "'" + for v in values) if values else "dynamic([])" + + +def substitutions(selection: dict) -> dict: + return {"Computer": literals(selection["Computer"]), + "Computer:lable": selection["computerLabel"], + "Severity": literals(selection["Severity"])} + + +def scoped_rows(rows: list[dict], selection: dict, timed: bool) -> list[dict]: + if not timed: + return rows + start, end = instant(selection["start"]), instant(selection["end"]) + return [row for row in rows if start <= instant(row["TimeGenerated"]) <= end] + + +def expected_result(contract: QueryContract, rows: list[dict], selection: dict) -> dict: + """A finite reviewed relational oracle, deliberately not a general KQL evaluator.""" + kind = contract.kind + records = scoped_rows(rows, selection, contract.time_parameter is not None) + if kind == "computers": + counts = Counter(row["Computer"] for row in records) + ordered = len(set(counts.values())) == len(counts) + return result([("Computer", "string")], + [[host] for host in sorted(counts, key=lambda h: (-counts[h], h))], + ordered=ordered) + if kind != "severity": + records = [row for row in records if row["Computer"] in selection["Computer"] + or (kind != "events" and selection["computerLabel"] == "All")] + records = [row for row in records if SEVERITIES[row["SeverityLevel"]][1] in selection["Severity"] + or (kind != "events" and "*" in selection["Severity"])] + if kind == "severity": + counts = Counter(row["SeverityLevel"] for row in records) + return result([("SeverityLevel", "string"), ("count_", "long"), ("Severity", "string")], + [[level, count, SEVERITIES[level][1]] for level, count in + sorted(counts.items(), key=lambda item: (SEVERITIES[item[0]][0], item[0]))], + order_by_severity=True) + if kind in ("emerg", "crit", "alert"): + records = [row for row in records if row["SeverityLevel"] == kind] + keys = ("Computer", "TimeGenerated") + columns = [("Computer", "string"), ("TimeGenerated", "datetime"), ("count_", "long")] + elif kind == "messages": + keys, columns = ("SyslogMessage",), [("SyslogMessage", "string"), ("count_", "long")] + elif kind == "processes": + keys, columns = ("Facility", "ProcessName"), [("Process", "string"), ("Count", "long")] + elif kind == "facilities": + keys = ("Facility", "SeverityLevel") + columns = [("Facility", "string"), ("SeverityLevel", "string"), ("count_", "long")] + elif kind == "events": + return result([ + ("TimeGenerated", "datetime"), ("Computer", "string"), ("SeverityLevel", "string"), + ("SeverityNumber", "long"), ("Facility", "string"), ("HostIP", "string"), + ("ProcessNameAndID", "string")], + [[row["TimeGenerated"], row["_ResourceId"] or row["Computer"], row["SeverityLevel"], + SEVERITIES[row["SeverityLevel"]][0], row["Facility"], row["HostIP"], + f'{row["ProcessName"]} ({row["ProcessID"] if row["ProcessID"] is not None else "-"})'] + for row in records]) + else: + raise ValueError(f"No reviewed oracle for {kind}") + counts = Counter(tuple(row[key] for key in keys) for row in records) + output = [[*key, count] for key, count in sorted(counts.items())] + if kind == "processes": + output = [[f"{process} ({facility})", count] for facility, process, count in output] + return result(columns, output) + + +def result(columns, rows, *, ordered=False, order_by_severity=False): + value = {"columns": [{"name": name, "type": typ} for name, typ in columns], + "rows": rows, "ordered": ordered} + if order_by_severity: + # Aliases share ranks; KQL does not define tie ordering. + value["sortGroups"] = {"column": "SeverityLevel", + "ranks": {key: rank for key, (rank, _) in SEVERITIES.items()}} + return value + + +def hourly_result(rows: list[dict]) -> dict: + counts = Counter((row["Computer"], stamp(instant(row["TimeGenerated"]).replace( + minute=0, second=0, microsecond=0))) for row in rows) + return result([("Computer", "string"), ("TimeGenerated", "datetime"), ("Events", "long")], + [[*key, value] for key, value in sorted(counts.items())]) + + +def parameter_blockers(parameters: dict, inventory: list[dict]) -> list[dict]: + blockers = [] + if any(not entry["parameter"] for entry in inventory): + for name, typ, all_value in (("Computer", 5, "All"), ("Severity", 2, "*")): + param = parameters.get(name, {}) + if (param.get("type") != typ or param.get("quote") != "'" + or param.get("delimiter") != "," or param.get("multiSelect") is not True + or param.get("typeSettings", {}).get("selectAllValue") != all_value): + blockers.append({"path": name, "reason": "Unreviewed multiselect parameter contract."}) + if parameters.get("TimeRange", {}).get("type") != 4: + blockers.append({"path": "TimeRange", "reason": "Missing reviewed time-range parameter."}) + try: + severity_labels = json.loads(parameters.get("Severity", {}).get("jsonData", "null")) + except (ValueError, TypeError): + severity_labels = None + if severity_labels != LABELS: + blockers.append({"path": "Severity", "reason": "Unreviewed static severity values."}) + return blockers + + +@dataclass(frozen=True) +class WorkbookRecipe: + spec: dict + contracts: dict[str, QueryContract] + selections: Callable + substitutions: Callable + expected: Callable + diagnostic: Callable + parameter_blockers: Callable + + +RECIPES = { + SYSLOG_SPEC["id"]: WorkbookRecipe( + spec=SYSLOG_SPEC, contracts=QUERY_CONTRACTS, + parameter_blockers=parameter_blockers, + selections=lambda tables, clock: selections(tables["Syslog"], clock), + substitutions=substitutions, + expected=lambda contract, tables, selection: expected_result( + contract, tables["Syslog"], selection), + diagnostic=lambda tables: { + "id": "diagnostic-hourly", "tables": ["Syslog"], + "purpose": "Fixture-only hourly distribution diagnostic; not a source workbook panel.", + "query": "Syslog | summarize Events=count() by Computer, bin(TimeGenerated, 1h)", + "expected": hourly_result(tables["Syslog"]), + }, + ), +} diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/arm_acceptance.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/arm_acceptance.py index 03d0635f2c2..3241b7122f4 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/arm_acceptance.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/arm_acceptance.py @@ -81,6 +81,7 @@ def _parse_build_validation_receipt( *, package_report_path: Path, package_identity: dict[str, Any], + arm_validation_method: str | None = None, ) -> dict[str, Any]: candidate = Path(receipt_path).expanduser() if candidate.is_symlink() or not candidate.is_file(): @@ -146,7 +147,10 @@ def field(name: str) -> str: raise ValueError( "validation receipt does not reference the repository build-and-validate evidence layout" ) - if field("Result") != "PASSED" or "**BUILD SUCCESSFUL**" not in report: + candidate_route = arm_validation_method == "lab-deployment" + expected_result = "CANDIDATE_GENERATED" if candidate_route else "PASSED" + expected_banner = "**CANDIDATE GENERATED — NOT ACCEPTED**" if candidate_route else "**BUILD SUCCESSFUL**" + if field("Result") != expected_result or expected_banner not in report: raise ValueError("repository build-and-validate receipt did not report success") if field("Validation binding result") != "PASSED": raise ValueError("repository build-and-validate producer binding did not report success") @@ -215,6 +219,24 @@ def field(name: str) -> str: raise ValueError(f"validation binding artifact is invalid JSON: {exc}") from exc if not isinstance(binding, dict) or binding.get("schemaVersion") != 1: raise ValueError("validation binding artifact has an unsupported schema") + recorded_method = binding.get("armValidationMethod", "arm-ttk") + if recorded_method not in {"arm-ttk", "lab-deployment"}: + raise ValueError("validation binding has an unsupported ARM validation method") + if recorded_method != (arm_validation_method or "arm-ttk"): + raise ValueError("validation binding ARM validation method does not match this route") + if candidate_route and ( + binding.get("armValidationStatus") != "pending" + or binding.get("packageLifecycleStatus") != "candidate-generated" + or binding.get("packageAccepted") is not False + or binding.get("workbookCapabilityVersion") != 1 + or len([row for row in validator_rows if row["validator"] == "ARM-TTK" and row["status"] == "SKIP"]) != 1 + or any(row["validator"] == "ARM-TTK" and row["status"] != "SKIP" for row in validator_rows) + ): + raise ValueError("lab validation binding must be a pending, unaccepted versioned candidate") + if arm_validation_method == "arm-ttk" and not any( + row["validator"] == "ARM-TTK" and row["status"] == "PASS" for row in validator_rows + ): + raise ValueError("the selected ARM-TTK method requires an actual passed ARM-TTK result") source_path_semantics = binding.get("sourcePathSemantics") producer_platform = binding.get("producerPlatform") if ( diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/catalog.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/catalog.py index 7fa68017895..5f6fe9f7a41 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/catalog.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/catalog.py @@ -1,6 +1,7 @@ from __future__ import annotations import re +from collections.abc import Iterable from typing import Any # Native Defender tables expose Timestamp in Advanced Hunting even when their @@ -82,7 +83,9 @@ def referenced_custom_tables(query: str) -> set[str]: ) -def query_table_references(query: str) -> list[dict[str, str | None]]: +def query_table_references( + query: str, *, known_tables: Iterable[str] = (), +) -> list[dict[str, str | None]]: """Return conservative table/alias bindings found at KQL table positions. This deliberately recognizes only common table sources. It is evidence @@ -98,7 +101,10 @@ def query_table_references(query: str) -> list[dict[str, str | None]]: for match in token_pattern.finditer(query) if not match.group(0).startswith(("//", "/*", "'", '"', "@'", '@"')) ] - known = {table.casefold(): table for table in NATIVE_XDR_TABLES | PASSTHROUGH_TABLES.keys()} + known = { + table.casefold(): table + for table in NATIVE_XDR_TABLES | PASSTHROUGH_TABLES.keys() | set(known_tables) + } results: list[dict[str, str | None]] = [] seen: set[tuple[str, str | None]] = set() for index, token in enumerate(tokens): diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/cli.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/cli.py index d1c74f48d6e..4e24bc7fa27 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/cli.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/cli.py @@ -14,8 +14,9 @@ runtime_validation_plan, validate_solution, ) -from .artifacts import migrate_reports, select_run, reset_run_selection, list_runs +from .artifacts import migrate_reports, select_run, reset_run_selection, list_runs, existing_artifact_path from .evidence import export_evidence +from .dependencies import plan_solution_dependencies from .budget import ( DEFAULT_POLICY, budget_summary, @@ -152,6 +153,22 @@ def main(argv: list[str] | None = None) -> int: description="Convert Microsoft Sentinel analytic rules into XDR Custom Detection YAML." ) subparsers = parser.add_subparsers(dest="command", required=True) + dependency_parser = subparsers.add_parser( + "dependency-plan", + help="Propose YAML dependencies offline, or verify them from approved lab metadata.", + ) + dependency_parser.add_argument("--solution", required=True) + dependency_parser.add_argument("--results", help="Exact-query-bound successful LA metadata results.") + dependency_parser.add_argument("--workspace-id", help="Explicitly approved lab workspace customer ID.") + workbook_dependencies = subparsers.add_parser( + "workbook-dependency-plan", help="Propose offline workbook gaps; import separately approved exact-query lab metadata.") + workbook_dependencies.add_argument("--solution", required=True) + workbook_dependencies.add_argument("--results") + workbook_dependencies.add_argument("--workspace-id") + workbook_review = subparsers.add_parser( + "workbook-review", help="Apply complete hash-bound accept/change/defer decisions to run-local config and receipt.") + for option in ("solution", "plan", "decisions", "plan-sha256", "actor", "reference", "output-dir"): + workbook_review.add_argument("--" + option, required=True) migration = subparsers.add_parser("migrate-reports", help="Plan a non-destructive copy of legacy reports into solution Logs.") migration.add_argument("--solution", required=True) migration.add_argument("--apply", action="store_true", help="Copy after reviewing the dry-run; originals are preserved.") @@ -174,6 +191,8 @@ def main(argv: list[str] | None = None) -> int: subparser.add_argument("--solution", required=True) if command == "convert": subparser.add_argument("--overwrite", action="store_true") + subparser.add_argument("--overwrite-workbooks", action="store_true", + help="Replace only unchanged hash-bound managed workbook drafts; never overwrite detections.") subparser.add_argument("--config") subparser.add_argument( "--rule-id", action="append", @@ -606,7 +625,7 @@ def main(argv: list[str] | None = None) -> int: ) package_v3_1.add_argument( "--target-version", - help="Explicit canonical V3.1 solution version (>=3.1.0 and <4.0.0); use with --version-bump none.", + help="Explicit canonical V3.1 solution version (>=3.2.0 and <4.0.0); use with --version-bump none.", ) package_v3_1.add_argument( "--legacy-baseline-evidence", @@ -634,7 +653,7 @@ def main(argv: list[str] | None = None) -> int: package_release.add_argument("--version-bump", choices=["patch", "minor", "major"]) package_release.add_argument( "--target-version", - help="Explicit canonical V3.1 solution version (>=3.1.0 and <4.0.0), instead of a version bump.", + help="Explicit canonical V3.1 solution version (>=3.2.0 and <4.0.0), instead of a version bump.", ) package_release.add_argument( "--expected-package-input-sha256", @@ -712,6 +731,10 @@ def main(argv: list[str] | None = None) -> int: workflow_init.add_argument("--subscription-id") workflow_init.add_argument("--workspace-resource-id") workflow_init.add_argument("--workspace-customer-id") + workflow_init.add_argument( + "--arm-validation-method", choices=["lab-deployment", "arm-ttk"], + help="New runs default to lab-deployment; resumption never changes the selected method.", + ) workflow_init.add_argument( "--confirm-configured-target", action="store_true", @@ -723,7 +746,7 @@ def main(argv: list[str] | None = None) -> int: ) workflow_init.add_argument( "--target-version", - help="Persist an explicit canonical V3.1 solution version (>=3.1.0 and <4.0.0); use with --version-bump none.", + help="Persist an explicit canonical V3.1 solution version (>=3.2.0 and <4.0.0); use with --version-bump none.", ) workflow_init.add_argument( "--workflow-profile", @@ -755,6 +778,43 @@ def main(argv: list[str] | None = None) -> int: help="Show the persisted end-to-end migration workflow state.", ) workflow_status_parser.add_argument("--solution", required=True) + workbook_commands = {} + for command in ( + "workflow-arm-candidate", "workflow-arm-plan", "workflow-arm-review-scope", + "workflow-arm-withdraw-scope", "workflow-arm-execute", "workflow-arm-reconcile", + "workflow-arm-import", + "workflow-arm-preflight-plan", "workflow-arm-preflight-review", + "workflow-arm-preflight-revoke", "workflow-arm-preflight-execute", + "workflow-workbook-contract-plan", "workflow-workbook-contract-import", + ): + workbook_commands[command] = subparsers.add_parser(command) + workbook_commands[command].add_argument("--solution", required=True) + workbook_commands["workflow-arm-candidate"].add_argument("--validation-receipt") + workbook_commands["workflow-arm-candidate"].add_argument("--toolkit-root") + for command in ("workflow-arm-plan", "workflow-workbook-contract-plan"): + workbook_commands[command].add_argument("--target", required=True, help="Local JSON with exact lab identity and optional workspaceLocation.") + workbook_commands[command].add_argument("--selected-flags", required=True, help="JSON object selecting every declared Deploy flag independently.") + for command in ("workflow-arm-review-scope", "workflow-arm-withdraw-scope", "workflow-arm-execute", "workflow-arm-reconcile", "workflow-arm-import"): + workbook_commands[command].add_argument("--actor", required=True) + workbook_commands[command].add_argument("--reference", required=True) + workbook_commands["workflow-arm-review-scope"].add_argument("--decision", required=True, choices=["approved", "declined"]) + for command in ("workflow-arm-review-scope", "workflow-arm-execute"): + workbook_commands[command].add_argument("--plan-sha256", required=True) + workbook_commands["workflow-arm-execute"].add_argument("--confirmation", required=True) + for command in ("workflow-arm-preflight-review", "workflow-arm-preflight-revoke"): + workbook_commands[command].add_argument("--actor", required=True) + workbook_commands[command].add_argument("--reference", required=True) + for command in ("workflow-arm-preflight-review", "workflow-arm-preflight-execute"): + workbook_commands[command].add_argument("--preflight-plan-sha256", required=True) + workbook_commands["workflow-arm-preflight-review"].add_argument( + "--decision", required=True, choices=["approved", "declined"], + ) + workbook_commands["workflow-arm-preflight-execute"].add_argument("--state-dir") + for command in ("workflow-arm-reconcile", "workflow-arm-import"): + workbook_commands[command].add_argument("--attempt", required=True) + workbook_commands[command].add_argument("--observation", required=True) + workbook_commands["workflow-workbook-contract-import"].add_argument("--evidence", required=True) + workbook_commands["workflow-workbook-contract-import"].add_argument("--contract-only", action="store_true") workflow_status_parser.add_argument( "--source-provider", choices=sorted(SOURCE_PROVIDERS), @@ -932,6 +992,10 @@ def main(argv: list[str] | None = None) -> int: runs_parser = subparsers.add_parser("workflow-runs", help="List run IDs without selecting or modifying state.") runs_parser.add_argument("--solution", required=True) + from .workbook_qualification_cli import ( + register_workbook_qualification_commands, dispatch_workbook_qualification, + ) + qualification_commands = register_workbook_qualification_commands(subparsers) for subparser in subparsers.choices.values(): if any(action.dest == "solution" for action in subparser._actions): subparser.add_argument("--run-id", help="Select an existing run; required when multiple runs exist.") @@ -976,6 +1040,10 @@ def main(argv: list[str] | None = None) -> int: subparsers.choices["workflow-init"].add_argument( "--new-run", action="store_true", help="Create an isolated new workflow instead of resuming." ) + for command in (*workbook_commands, *qualification_commands): + for action in subparsers.choices[command]._actions: + if action.dest == "run_id": + action.required = True args = parser.parse_args(argv) if args.command == "convert" and args.rule_id and len(args.rule_id) != 1: parser.error("--rule-id accepts exactly one GUID; repeated selectors are not supported") @@ -984,6 +1052,8 @@ def main(argv: list[str] | None = None) -> int: isolated_permission_commands = { "plan-delegated-app-permission-update", "verify-delegated-app-permission-update", + "workbook-dependency-plan", + "workbook-review", } if getattr(args, "solution", None) and args.command not in isolated_permission_commands: selection = select_run(args.solution, args.run_id) @@ -992,7 +1062,9 @@ def main(argv: list[str] | None = None) -> int: args.command == "convert" and args.rule_id ): configure_logging(getattr(args, "solution", None)) - if args.command == "coordinator-enroll": + if args.command in qualification_commands: + result = dispatch_workbook_qualification(args) + elif args.command == "coordinator-enroll": result = coordinator_enroll( args.solution, owner=args.owner, @@ -1343,17 +1415,35 @@ def main(argv: list[str] | None = None) -> int: args.solution, legacy_baseline_evidence=args.legacy_baseline_evidence, ) + elif args.command == "dependency-plan": + result = plan_solution_dependencies( + args.solution, results_path=args.results, workspace_id=args.workspace_id, + ) + elif args.command == "workbook-dependency-plan": + from .workbook_dependencies import plan_workbook_dependencies + result = plan_workbook_dependencies( + args.solution, results_path=args.results, workspace_id=args.workspace_id) + elif args.command == "workbook-review": + from .workbook_dependencies import review_workbook_dependencies + result = review_workbook_dependencies( + args.solution, plan_path=args.plan, decisions_path=args.decisions, + expected_plan_sha256=args.plan_sha256, actor=args.actor, reference=args.reference, + output_directory=args.output_dir) elif args.command == "inspect": - result = inspect_solution(args.solution) + from .workbook_workflow import inspect_content + result = inspect_content(args.solution) elif args.command == "convert": - result = convert_solution( + from .workbook_workflow import convert_content + result = convert_content( args.solution, overwrite=args.overwrite, config_path=args.config, + overwrite_workbooks=args.overwrite_workbooks, rule_id=args.rule_id[0] if args.rule_id else None, ) if args.rule_id: configure_logging(args.solution) elif args.command == "validate": - result = validate_solution(args.solution) + from .workbook_workflow import validate_content + result = validate_content(args.solution) elif args.command == "validate-advanced-hunting": try: state = workflow_status(args.solution) @@ -1424,6 +1514,47 @@ def main(argv: list[str] | None = None) -> int: provider_gap_evidence=args.provider_gap_evidence, source_provider=args.source_provider, ) + elif args.command in workbook_commands: + from . import workbook_workflow as workbooks + if args.command.startswith("workflow-arm-preflight-"): + from . import workbook_preflight_workflow as preflight + + if args.command == "workflow-arm-preflight-plan": + result = preflight.create_plan(args.solution) + elif args.command == "workflow-arm-preflight-review": + result = preflight.review( + args.solution, expected_sha256=args.preflight_plan_sha256, + actor=args.actor, reference=args.reference, approved=args.decision == "approved", + ) + elif args.command == "workflow-arm-preflight-revoke": + result = preflight.revoke(args.solution, actor=args.actor, reference=args.reference) + else: + result = preflight.execute( + args.solution, expected_sha256=args.preflight_plan_sha256, + state_dir=args.state_dir, + ) + elif args.command == "workflow-arm-candidate": + result = workbooks.create_candidate(args.solution, validation_receipt=args.validation_receipt, + toolkit_root=args.toolkit_root) + elif args.command in {"workflow-arm-plan", "workflow-workbook-contract-plan"}: + result = workbooks.plan_candidate(args.solution, + target=json.loads(Path(args.target).read_text(encoding="utf-8-sig")), + selected_flags=json.loads(Path(args.selected_flags).read_text(encoding="utf-8-sig")), + contract_only=args.command == "workflow-workbook-contract-plan") + elif args.command == "workflow-arm-review-scope": + result = workbooks.review_scope(args.solution, actor=args.actor, reference=args.reference, + approved=args.decision == "approved", expected_plan_sha256=args.plan_sha256) + elif args.command == "workflow-arm-withdraw-scope": + result = workbooks.withdraw_scope(args.solution, actor=args.actor, reference=args.reference) + elif args.command == "workflow-arm-execute": + result = workbooks.execute_candidate(args.solution, expected_plan_sha256=args.plan_sha256, + actor=args.actor, reference=args.reference, confirmation=args.confirmation) + elif args.command in {"workflow-arm-reconcile", "workflow-arm-import"}: + action = workbooks.reconcile_attempt if args.command == "workflow-arm-reconcile" else workbooks.import_arm_only + result = action(args.solution, attempt=args.attempt, + observation=args.observation, actor=args.actor, reference=args.reference) + else: + result = workbooks.import_contract(args.solution, evidence=args.evidence, contract_only=args.contract_only) elif args.command == "workflow-init": if args.new_run and args.run_id: raise ValueError("--new-run and --run-id cannot be combined") @@ -1477,6 +1608,11 @@ def main(argv: list[str] | None = None) -> int: yolo=True if args.yolo else None, campaign_id=args.campaign_id, new_run=args.new_run, + workbook_capability_version=( + 1 if args.new_run or not existing_artifact_path(args.solution, "workflow-state.json").exists() + or args.arm_validation_method is not None else None + ), + arm_validation_method=args.arm_validation_method, ) elif args.command == "workflow-budget": result = budget_summary(args.solution, args.campaign_id) @@ -1614,7 +1750,19 @@ def main(argv: list[str] | None = None) -> int: and result.get("status") != "applied" ): return 1 - if args.command == "convert" and (result["needsReview"] or result["conflicts"]): + if args.command == "convert" and ( + result["needsReview"] or result["conflicts"] or result.get("workbooks", {}).get("selectedBlocked", 0) + ): + return 1 + if args.command == "package-v3-1" and result.get("status") == "blocked": + return 1 + if args.command in {"workflow-arm-execute", "workflow-workbook-contract-import"} and result.get("accepted") is not True: + return 1 + if args.command == "workflow-arm-preflight-execute" and result.get("status") != "ready": + return 1 + if args.command == "dependency-plan" and args.results and any( + item["validation"]["status"] != "passed" for item in result["results"] + ): return 1 if ( args.command == "workflow-complete-stage" diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/converter.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/converter.py index 8947645d53f..0d5ab03d4b2 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/converter.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/converter.py @@ -24,6 +24,7 @@ query_table_references, ) from .columns import projected_columns +from .dependencies import dependency_plan, dependency_validation_errors from .content_paths import content_path, yaml_files from .parser_bindings import normalize_parser_bindings from .report import write_transformation_report @@ -1665,6 +1666,12 @@ def build_xdr_document(source: Path, solution_root: Path, config: dict[str, Any] ] status = "needsReview" if errors or blocking_review_reasons else "converted" relative_source = source.relative_to(solution_root).as_posix() + dependencies = dependency_plan( + source_query, + target_query=converted_query, + connectors=doc.get("requiredDataConnectors"), + table_mappings=config.get("tableMappings"), + ) return { "schemaVersion": SCHEMA_VERSION, "version": INITIAL_XDR_VERSION, @@ -1677,6 +1684,9 @@ def build_xdr_document(source: Path, solution_root: Path, config: dict[str, Any] else {} ), "contentProvenance": { + "requiredWorkloads": dependencies["requiredWorkloads"], + "requiredDataConnectors": dependencies["requiredDataConnectors"], + "dependencyReviewRequired": dependencies["reviewRequired"], "source": { "platform": "Microsoft Sentinel", "kind": "AnalyticsRule", @@ -1796,7 +1806,7 @@ def _existing_identity_error( def validate_document(document: dict[str, Any]) -> list[str]: with XDR_SCHEMA_PATH.open(encoding="utf-8") as handle: schema = json.load(handle) - errors = [ + errors = dependency_validation_errors(document) + [ f"schema: {error.message}" for error in Draft202012Validator(schema).iter_errors(document) ] @@ -2056,6 +2066,17 @@ def convert_solution( continue document = build_xdr_document(source, root, config) if target.exists(): + previous_provenance = existing_document.get("contentProvenance") or {} + dependencies_current = ( + (previous_provenance.get("source") or {}).get("querySha256") + == document["contentProvenance"]["source"]["querySha256"] + and previous_target_query == document["properties"]["queryCondition"]["queryText"] + and existing_document.get("requiredDataConnectors") == source_document.get("requiredDataConnectors") + ) + if dependencies_current: + for field in ("requiredWorkloads", "requiredDataConnectors", "dependencyReviewRequired"): + if field in previous_provenance: + document["contentProvenance"][field] = deepcopy(previous_provenance[field]) if isinstance(existing_document, dict) and "version" in existing_document: error = xdr_version_error(existing_document["version"]) if error: diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/dependencies.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/dependencies.py new file mode 100644 index 00000000000..d20002e1a82 --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/dependencies.py @@ -0,0 +1,369 @@ +"""Offline dependency records and interpretation of Log Analytics input metadata.""" +from __future__ import annotations + +from collections import defaultdict +from copy import deepcopy +import hashlib +import json +import re +from pathlib import Path +from typing import Any + +import yaml + +from .catalog import NATIVE_XDR_TABLES, query_table_references + +WORKLOADS = ( + "mde", "mdo", "mdi", "mda", "mdc", "entraid", "purview", + "appgovernance", "sentinel", "mtp", +) +PLATFORM_REFERENCE_INPUTS = frozenset({"Watchlist"}) + + +def telemetry_data_types(tables): + """Platform reference data is source lineage, not connector telemetry.""" + return sorted(set(tables) - PLATFORM_REFERENCE_INPUTS) + + +TABLE_WORKLOADS = { + **{table: "mde" for table in NATIVE_XDR_TABLES if table.startswith("Device")}, + **{table: "mdo" for table in ( + "EmailAttachmentInfo", "EmailEvents", "EmailPostDeliveryEvents", + "EmailUrlInfo", "UrlClickEvents", + )}, + **{table: "mdi" for table in ( + "IdentityDirectoryEvents", "IdentityLogonEvents", "IdentityQueryEvents", + )}, + **{table: "mtp" for table in ("AlertInfo", "AlertEvidence", "IdentityInfo")}, + "CloudAppEvents": "mda", + **{table: "sentinel" for table in ( + "Syslog", "SecurityAlert", "SecurityEvent", "WindowsEvent", "Event", + "AzureActivity", "CommonSecurityLog", "SigninLogs", "AuditLogs", + "OfficeActivity", "AWSCloudTrail", "AWSGuardDuty", + )}, +} + + +def data_source_tables(metadata: Any) -> list[str]: + if metadata is None: + return [] + if not isinstance(metadata, list): + raise ValueError("Log Analytics dataSources must be an array") + tables: set[str] = set() + for source in metadata: + if not isinstance(source, dict): + raise ValueError("each dataSources entry must be an object") + source_tables = source.get("tables") + if source_tables is None: + continue + if not isinstance(source_tables, list): + raise ValueError("dataSources.tables must be an array") + for entry in source_tables: + name = entry.get("name") if isinstance(entry, dict) else entry + if not isinstance(name, str) or not name.strip(): + raise ValueError("dataSources contains an invalid input table name") + tables.add(name) + return sorted(tables) + + +def resolve_data_types(query: str, metadata: Any = None) -> dict[str, Any]: + returned = data_source_tables(metadata) + if 0 < len(returned) <= 10: + return {"method": "log-analytics", "dataTypes": returned, "returnedDataTypes": returned} + tables = sorted({ + str(item["table"]) + for item in query_table_references(query, known_tables=TABLE_WORKLOADS) + }) + return { + "method": "regex", + "dataTypes": tables, + "returnedDataTypes": returned, + "fallbackReason": "more-than-10-tables" if len(returned) > 10 else "missing-data-sources", + } + + +def workload_records(tables: list[str]) -> tuple[list[dict[str, Any]], list[str]]: + grouped: dict[str, set[str]] = defaultdict(set) + unknown = [] + for table in tables: + if table in PLATFORM_REFERENCE_INPUTS: + continue + workload = TABLE_WORKLOADS.get(table) + if workload is None and table.endswith("_CL"): + workload = "sentinel" + if workload is None: + unknown.append(table) + else: + grouped[workload].add(table) + return [ + {"workload": workload, "dataTypes": sorted(data_types)} + for workload, data_types in sorted(grouped.items()) + ], sorted(unknown) + + +def dependency_plan( + source_query: str, + *, + target_query: str | None = None, + metadata: Any = None, + connectors: Any = None, + table_mappings: dict[str, str] | None = None, +) -> dict[str, Any]: + resolution = resolve_data_types(source_query, metadata) + source_types = set(resolution["dataTypes"]) + from ._workbook_core import rewrite + code = "".join(text if kind == "code" else " " * len(text) + for kind, text in rewrite._segments(source_query)) + if re.search(r"\b_GetWatchlist\s*\(", code): + source_types.add("Watchlist") + references = sorted(source_types & PLATFORM_REFERENCE_INPUTS) + resolution["dataTypes"] = telemetry_data_types(source_types) + mappings = table_mappings or {} + target_table_names = {mappings.get(table, table) for table in resolution["dataTypes"]} + if target_query is not None: + target_table_names.update(resolve_data_types(target_query)["dataTypes"]) + target_tables = telemetry_data_types(target_table_names) + workloads, unknown = workload_records(target_tables) + reasons = [] + if not source_types: + reasons.append("No source inputs resolved; provide source evidence and any telemetry connector prerequisites.") + if unknown: + reasons.append(f"Provide workload mappings for: {', '.join(unknown)}.") + required_connectors = deepcopy(connectors) if connectors is not None else [] + if isinstance(required_connectors, list): + required_connectors = [ + {**record, "dataTypes": telemetry_data_types(record["dataTypes"])} + if isinstance(record, dict) and isinstance(record.get("dataTypes"), list) else record + for record in required_connectors + if not isinstance(record, dict) or not isinstance(record.get("dataTypes"), list) + or telemetry_data_types(record["dataTypes"]) + ] + if any(item["workload"] == "sentinel" for item in workloads) and connectors is None: + reasons.append("Provide or explicitly confirm the Sentinel connector prerequisites.") + return { + **resolution, + "targetDataTypes": target_tables, + "targetDiscoveryMethod": "regex" if target_query is not None else "source-table-mappings", + "requiredWorkloads": workloads, + "requiredDataConnectors": required_connectors, + "unmappedDataTypes": unknown, + "reviewRequired": bool(reasons), + "reviewReasons": reasons, + "runtimeParityVerified": False, + **({"sourceDataTypes": sorted(source_types), "referenceInputs": references} if references else {}), + } + + +def dependency_validation_errors(document: dict[str, Any]) -> list[str]: + provenance = document.get("contentProvenance") + if not isinstance(provenance, dict): + return ["contentProvenance must contain requiredWorkloads and requiredDataConnectors"] + errors = [] + for field, identity in ( + ("requiredWorkloads", "workload"), + ("requiredDataConnectors", "connectorId"), + ): + records = provenance.get(field) + if not isinstance(records, list): + errors.append(f"contentProvenance.{field} must be an array") + continue + for index, record in enumerate(records): + prefix = f"contentProvenance.{field}[{index}]" + if not isinstance(record, dict): + errors.append(f"{prefix} must be an object") + continue + name = record.get(identity) + if not isinstance(name, str) or not name.strip(): + errors.append(f"{prefix}.{identity} must be a non-empty string") + elif identity == "workload" and name not in WORKLOADS: + errors.append(f"{prefix}.workload must be one of {', '.join(WORKLOADS)}") + data_types = record.get("dataTypes") + if not isinstance(data_types, list) or any( + not isinstance(table, str) or not table.strip() for table in data_types + ): + errors.append(f"{prefix}.dataTypes must be an array of non-empty strings") + elif any(table in PLATFORM_REFERENCE_INPUTS for table in data_types): + errors.append(f"{prefix}.dataTypes must not include platform reference input Watchlist") + if provenance.get("dependencyReviewRequired") is True: + errors.append("Dependency provenance needs review; use sentinel-xdr-dependency-helper.") + return errors + + +def live_dependency_check( + document: dict[str, Any], source_query: str, metadata: Any, + *, table_mappings: dict[str, str] | None = None, + source_connectors: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + errors = dependency_validation_errors(document) + if errors: + return {"status": "failed", "errors": errors, "method": "not-evaluated"} + plan = dependency_plan( + source_query, metadata=metadata, + target_query=((document.get("properties") or {}).get("queryCondition") or {}).get("queryText"), + connectors=(document.get("contentProvenance") or {}).get("requiredDataConnectors"), + table_mappings=table_mappings, + ) + declared = (document.get("contentProvenance") or {}).get("requiredWorkloads") or [] + declared_pairs = { + (item["workload"], table) + for item in declared if isinstance(item, dict) and isinstance(item.get("dataTypes"), list) + for table in item["dataTypes"] + if isinstance(item.get("workload"), str) and isinstance(table, str) + } + for record in plan["requiredWorkloads"]: + for table in record["dataTypes"]: + if (record["workload"], table) not in declared_pairs: + errors.append(f"YAML does not declare {record['workload']} data type {table}") + unknown = [ + table for table in plan["unmappedDataTypes"] + if not any(pair[1] == table for pair in declared_pairs) + ] + if unknown: + errors.append(f"Provide workload mappings for: {', '.join(unknown)}.") + if not plan["dataTypes"] and not plan.get("referenceInputs"): + errors.append("No data types resolved; provide the data types and required connectors.") + connector_records = (document.get("contentProvenance") or {}).get("requiredDataConnectors") or [] + connector_pairs = { + (record.get("connectorId"), table) + for record in connector_records + if isinstance(record, dict) and isinstance(record.get("dataTypes"), list) + for table in record["dataTypes"] + if isinstance(record.get("connectorId"), str) and isinstance(table, str) + } + if source_connectors is not None: + source_errors = dependency_validation_errors({"contentProvenance": { + "requiredWorkloads": [], "requiredDataConnectors": source_connectors, + }}) + if source_errors: + return {**plan, "status": "failed", "errors": errors + source_errors} + declared_connectors = {record["connectorId"] for record in connector_records} + for record in source_connectors if source_connectors is not None else []: + if record["connectorId"] not in declared_connectors: + errors.append(f"YAML does not declare connector {record['connectorId']}") + for table in record["dataTypes"]: + if (record.get("connectorId"), table) not in connector_pairs: + errors.append(f"YAML does not declare connector {record.get('connectorId')} data type {table}") + return {**plan, "status": "failed" if errors else "passed", "errors": errors} + + +def plan_solution_dependencies( + solution: str | Path, *, results_path: str | Path | None = None, + workspace_id: str | None = None, +) -> dict[str, Any]: + # Import here because conversion itself uses the pure dependency helpers. + from .converter import load_config, runtime_validation_plan + + root = Path(solution).expanduser().resolve() + rules = runtime_validation_plan(root)["rules"] + if not rules: + raise ValueError("dependency planning requires current XDR detections") + config = load_config(root / "XDR Detections") + entries = {} + if results_path is not None: + if not workspace_id: + raise ValueError("dependency metadata import requires the explicitly approved workspace ID") + raw = json.loads(Path(results_path).expanduser().read_text(encoding="utf-8")) + if not isinstance(raw, dict) or not isinstance(raw.get("results"), list): + raise ValueError("dependency results must contain a results array") + for entry in raw["results"]: + if not isinstance(entry, dict) or not isinstance(entry.get("detection"), str): + raise ValueError("each dependency result must identify a detection") + name = entry["detection"] + if name in entries: + raise ValueError(f"duplicate dependency metadata for {name}") + entries[name] = entry + if set(entries) != {item["detection"] for item in rules}: + raise ValueError("dependency metadata must match the complete current detection inventory") + results = [] + for rule in rules: + document = yaml.safe_load( + (root / "XDR Detections" / rule["detection"]).read_text(encoding="utf-8-sig") + ) + source = yaml.safe_load(Path(rule["sourceRule"]).read_text(encoding="utf-8-sig")) + entry = entries.get(rule["detection"]) + metadata = None + if entry is not None: + if ( + entry.get("status") != "passed" + or entry.get("partial") is True + or entry.get("error") + or entry.get("errorDetails") + or entry.get("failureKind") + or ( + entry.get("statusCode") is not None + and entry["statusCode"] != 200 + ) + or entry.get("workspaceId") != workspace_id + or entry.get("querySha256") != rule["sentinelQuerySha256"] + ): + raise ValueError( + f"dependency metadata failed or has stale/cross-workspace scope: {rule['detection']}" + ) + metadata = entry.get("dataSources") + proposal = dependency_plan( + rule["sentinelQuery"], metadata=metadata, + target_query=rule["advancedHuntingQuery"], + connectors=source.get("requiredDataConnectors"), + table_mappings=config.get("tableMappings"), + ) + results.append({ + **rule, + **proposal, + "workspaceId": workspace_id, + "dataSources": metadata, + "validation": live_dependency_check( + document, rule["sentinelQuery"], metadata, + table_mappings=config.get("tableMappings"), + source_connectors=source.get("requiredDataConnectors"), + ) if entry is not None else None, + }) + return {"solution": str(root), "writesYaml": False, "liveQueriesExecuted": False, "results": results} + + +def solution_package_context(solution: str | Path) -> dict[str, str]: + from .content_paths import content_path + from .packaging import _solution_data_file + + root = Path(solution).expanduser().resolve() + data = json.loads(_solution_data_file(root / "Data").read_text(encoding="utf-8-sig")) + if not isinstance(data, dict): + raise ValueError("solution Data must be an object") + metadata_path = data.get("Metadata") + if not isinstance(metadata_path, str) or not metadata_path.strip(): + raise ValueError("solution Data must explicitly reference its Metadata") + metadata = json.loads(content_path(root, metadata_path).read_text(encoding="utf-8-sig")) + if not isinstance(metadata, dict): + raise ValueError("solution Metadata must be an object") + values = [metadata.get("publisherId"), metadata.get("offerId"), data.get("Version")] + if any(not isinstance(value, str) or not value.strip() for value in values): + raise ValueError("deployment provenance requires publisherId, offerId and solution Version") + return {"packageId": f"{values[0]}.{values[1]}", "packageVersion": values[2]} + + +def deployed_provenance(document: dict[str, Any], package_context: dict[str, str]) -> dict[str, str]: + values = { + "packageId": package_context.get("packageId"), + "packageVersion": package_context.get("packageVersion"), + "contentId": (document.get("properties") or {}).get("id"), + "contentVersion": document.get("version"), + } + result: dict[str, str] = {} + for key, value in values.items(): + if not isinstance(value, str) or not value.strip(): + raise ValueError("deployed provenance requires resolved package and content identity/version") + result[key] = value + return result + + +def dependency_identity(document: dict[str, Any]) -> str: + provenance = document.get("contentProvenance") or {} + values = { + field: provenance.get(field) + for field in ("requiredWorkloads", "requiredDataConnectors") + } + values["dependencyReviewRequired"] = provenance.get("dependencyReviewRequired", False) + if "requiredWorkspaceFunctions" in provenance: + values["requiredWorkspaceFunctions"] = provenance["requiredWorkspaceFunctions"] + return hashlib.sha256(json.dumps( + values, sort_keys=True, separators=(",", ":"), ensure_ascii=True, + ).encode()).hexdigest() diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/deployment.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/deployment.py index 61ac541a86c..783da528c8a 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/deployment.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/deployment.py @@ -290,8 +290,12 @@ def _graph_request( return exc.code, parsed -def graph_detection_payload(document: dict[str, Any]) -> dict[str, Any]: - payload, _projection = graph_detection_payload_with_projection(document) +def graph_detection_payload( + document: dict[str, Any], *, package_context: dict[str, str] | None = None, +) -> dict[str, Any]: + payload, _projection = graph_detection_payload_with_projection( + document, package_context=package_context, + ) return payload @@ -323,13 +327,16 @@ def deploy_solution( files = [files_by_path[path] for path in sorted(selected_paths)] if not files: raise ValueError(f"no generated XDR Detection YAML files found under {output}") + from .dependencies import solution_package_context + + package_context = solution_package_context(root) token = _deployment_token(state_dir, tenant_id=target["tenantId"]) results: list[dict[str, Any]] = [] for path in files: try: document = yaml.safe_load(path.read_text(encoding="utf-8-sig")) or {} payload, tactic_projection = graph_detection_payload_with_projection( - document + document, package_context=package_context, ) rule_id = urllib.parse.quote(payload["id"], safe="") item_url = f"{DEPLOYMENT_ENDPOINT}/{rule_id}" @@ -354,6 +361,7 @@ def deploy_solution( error = response.get("error") if isinstance(response, dict) else None deployed_status = None tactic_projection_verified = None + provenance_verified = None if success: verify_status, verified = _graph_request("GET", item_url, token) deployed_status = ( @@ -376,10 +384,15 @@ def deploy_solution( else [] ) tactic_projection_verified = verified_tactics == projected_tactics + provenance_verified = ( + isinstance(verified, dict) + and verified.get("contentProvenance") == payload["contentProvenance"] + ) if ( verify_status != 200 or deployed_status != "disabled" or not tactic_projection_verified + or not provenance_verified ): success = False operation = "failed" @@ -387,10 +400,11 @@ def deploy_solution( "code": "DeploymentVerificationFailed", "message": ( "deployed Custom Detection did not match its disabled " - "status and projected tactic payload" + "status, projected tactic payload and content provenance" ), "statusCode": verify_status, "tacticProjectionVerified": tactic_projection_verified, + "contentProvenanceVerified": provenance_verified, } results.append( { @@ -402,6 +416,7 @@ def deploy_solution( "deployedStatus": deployed_status, "tacticProjection": tactic_projection, "tacticProjectionVerified": tactic_projection_verified, + "contentProvenanceVerified": provenance_verified, "error": error, } ) diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/graph_readiness.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/graph_readiness.py index 1928f640c00..0c8e5bc1190 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/graph_readiness.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/graph_readiness.py @@ -153,6 +153,7 @@ def project_graph_tactics( def graph_detection_payload_with_projection( document: dict[str, Any], + *, package_context: dict[str, str] | None = None, ) -> tuple[dict[str, Any], dict[str, Any]]: """Build a disabled Graph payload and return explicit projection diagnostics.""" prepared = deepcopy(document) @@ -172,12 +173,18 @@ def graph_detection_payload_with_projection( result = deepcopy(properties) result["@odata.type"] = "#microsoft.graph.security.detectionRule" result["status"] = "disabled" + if package_context is not None: + from .dependencies import deployed_provenance + + result["contentProvenance"] = deployed_provenance(prepared, package_context) return result, projection -def graph_detection_payload(document: dict[str, Any]) -> dict[str, Any]: +def graph_detection_payload( + document: dict[str, Any], *, package_context: dict[str, str] | None = None, +) -> dict[str, Any]: """Compatibility wrapper returning only the projected Graph request body.""" - payload, _ = graph_detection_payload_with_projection(document) + payload, _ = graph_detection_payload_with_projection(document, package_context=package_context) return payload diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/logs_query.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/logs_query.py index ae7c0206063..7d7eb20da32 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/logs_query.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/logs_query.py @@ -1,6 +1,7 @@ from __future__ import annotations import base64 +import json import time import uuid from datetime import date, datetime, timedelta, timezone @@ -213,6 +214,23 @@ def execute_workspace_query( ) effective_timeout = min(request_timeout_seconds, max(1, int(remaining))) attempts += 1 + metadata: dict[str, Any] = {} + + def capture_response(pipeline_response: Any) -> None: + if pipeline_response.http_response.status_code != 200: + return + body = json.loads(pipeline_response.http_response.text()) + if not isinstance(body, dict): + raise ValueError("Log Analytics response must be a JSON object") + metadata["dataSources"] = body.get("dataSources") + + def request_metadata(pipeline_request: Any) -> None: + headers = pipeline_request.http_request.headers + preference = headers.get("Prefer", "") + headers["Prefer"] = ",".join( + value for value in (preference, "include-dataSources=true") if value + ) + try: response = client.query_workspace( workspace_id, @@ -221,6 +239,8 @@ def execute_workspace_query( server_timeout=effective_timeout, connection_timeout=min(10, effective_timeout), read_timeout=effective_timeout, + raw_request_hook=request_metadata, + raw_response_hook=capture_response, ) except Exception as exc: kind, retryable = _failure_kind(exc) @@ -279,6 +299,7 @@ def execute_workspace_query( "error": _error_details(getattr(response, "partial_error", None)) if partial else None, "errorDetails": _error_details(getattr(response, "partial_error", None)) if partial else None, "partialError": _error_payload(getattr(response, "partial_error", None)) if partial else None, + "dataSources": metadata.get("dataSources"), } diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/packaging.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/packaging.py index 4be74657e39..05f82c28ba5 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/packaging.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/packaging.py @@ -53,6 +53,7 @@ "Tools/Create-Azure-Sentinel-Solution/common/templating/baseMainTemplate.json", "Tools/Create-Azure-Sentinel-Solution/common/templating/baseCreateUiDefinition.json", "Tools/Create-Azure-Sentinel-Solution/common/customDetections.ps1", + "Tools/Create-Azure-Sentinel-Solution/common/defenderWorkbooks.ps1", "Tools/Create-Azure-Sentinel-Solution/common/customerUsageAttribution.ps1", "Tools/Create-Azure-Sentinel-Solution/common/AttributionMetadata.cs", "Tools/Create-Azure-Sentinel-Solution/common/versionPolicy.ps1", @@ -76,6 +77,7 @@ "dataconnectors", "playbooks", "workbooks", + "xdr workbooks", "analytic rules", "analytics rules", "hunting queries", @@ -93,16 +95,36 @@ def validate_v31_target_version(version: str | None) -> str: if not isinstance(version, str) or not V31_TARGET_VERSION_PATTERN.fullmatch(version): raise ValueError( "target version must be canonical major.minor.patch in the V3.1 " - "range >= 3.1.0 and < 4.0.0" + "range >= 3.2.0 and < 4.0.0" ) major, minor, _patch = version.split(".") - if major != "3" or minor == "0": + if major != "3" or minor in {"0", "1"}: raise ValueError( - "target version must be in the V3.1 range >= 3.1.0 and < 4.0.0" + "target version must be in the V3.1 range >= 3.2.0 and < 4.0.0" ) return version +def validate_v31_effective_version( + source_version: str | None, version_bump: str, target_version: str | None = None, +) -> str: + """Validate the selected solution release without consuming its version action.""" + if target_version is not None: + return validate_v31_target_version(target_version) + if not isinstance(source_version, str) or not V31_TARGET_VERSION_PATTERN.fullmatch(source_version): + return validate_v31_target_version(source_version) + major, minor, patch = (int(part) for part in source_version.split(".")) + if version_bump == "patch": + patch += 1 + elif version_bump == "minor": + minor, patch = minor + 1, 0 + elif version_bump == "major": + major, minor, patch = major + 1, 0, 0 + elif version_bump != "none": + raise ValueError("version bump must be none, patch, minor, or major") + return validate_v31_target_version(f"{major}.{minor}.{patch}") + + def package_input_file_hashes(solution: str | Path) -> dict[str, str]: """Return the complete package-input file manifest, excluding generated output.""" root = Path(solution).expanduser().resolve() @@ -184,7 +206,7 @@ def _canonical_digest(value: dict[str, Any]) -> str: def _deployment_parameters( template: dict[str, Any], ui: dict[str, Any], test_parameters: dict[str, Any], - *, has_xdr: bool, contract_path: Path | None = None, + *, has_xdr: bool, has_workbooks: bool | None = None, contract_path: Path | None = None, ) -> dict[str, bool]: contract_path = contract_path or ( Path(__file__).resolve().parents[2] / "Create-Azure-Sentinel-Solution" @@ -220,6 +242,8 @@ def _deployment_parameters( raise RuntimeError(f"Invalid V3.1 content selection contract for {name}.") if has_xdr != ("DeployCustomDetection" in selected): raise RuntimeError("V3.1 Custom Detection declaration does not match the packaged XDR inputs.") + if has_workbooks is not None and has_workbooks != ("DeployDefenderWorkbook" in selected): + raise RuntimeError("V3.1 Defender workbook declaration does not match the selected workbook inputs.") return {name: value["defaultValue"] for name, value in selected.items()} @@ -748,6 +772,159 @@ def verify_template(document: dict[str, Any], label: str) -> None: } +def _verify_workbook_package( + solution: Path, files: list[Path], template: dict[str, Any], zip_path: Path, +) -> dict[str, Any]: + """Verify the full candidate install/registration projection, not provider acceptance.""" + from copy import deepcopy + from .workbooks import _content_release + + def arm_copy(value: Any) -> Any: + if isinstance(value, str): + return "[" + value if value.startswith("[") else value + if isinstance(value, dict): + return {key: arm_copy(item) for key, item in value.items()} + if isinstance(value, list): + return [arm_copy(item) for item in value] + return value + + resources = template.get("resources") or [] + if not isinstance(resources, list): + raise RuntimeError("V3.1 workbook package resources must be an array") + installs = [row for row in resources if + row.get("type") == "Microsoft.Resources/deployments" and + row.get("name", "").startswith("DefenderWorkbook-")] + registrations = [row for row in resources if + row.get("properties", {}).get("contentKind") == "DefenderWorkbook"] + packages = [row for row in resources if row.get("type") == + "Microsoft.OperationalInsights/workspaces/providers/contentPackages"] + if len(installs) != len(files) or len(registrations) != len(files): + raise RuntimeError("Defender workbook install/registration count differs from selection") + if not files: + return {"count": 0, "contractSupport": "notApplicable"} + if len(packages) != 1: + raise RuntimeError("Defender workbook packaging requires exactly one contentPackages resource") + variables = template.get("variables") or {} + package_id = variables.get("solutionId") + version = variables.get("_solutionVersion") + validate_v31_target_version(version) + if not isinstance(package_id, str) or not package_id or package_id.startswith("["): + raise RuntimeError("Defender workbook package identity must be resolved") + expected_criteria = [] + identities = [] + for path in files: + authored = _read_json(path) + metadata = authored["xdrMetadata"] + content_id = metadata["contentId"] + instance_id = metadata["instanceId"] + if _content_release(metadata["version"]) is None: + raise ValueError("Defender workbook content version must be >=3.1.0 and <4.0.0") + deployment_name = "DefenderWorkbook-" + hashlib.sha256( + f"{solution.name}/{content_id}".encode("utf-8") + ).hexdigest()[:32] + install_rows = [row for row in installs if row.get("name") == deployment_name] + registration_rows = [row for row in registrations if row.get("properties", {}).get("contentId") == content_id] + if len(install_rows) != 1 or len(registration_rows) != 1: + raise RuntimeError("Defender workbook identities are missing or duplicated") + install, registration = install_rows[0], registration_rows[0] + function_records = authored["contentProvenance"].get("requiredWorkspaceFunctions", []) + from .workbook_functions import validate_functions, function_definition, function_parameters + from .workbooks import _root + if function_records: + validate_functions(_root(solution), function_records) + parser_dependencies = [] + for record in function_records: + saved = [resource for resource in resources if + resource.get("type", "").lower() == "microsoft.operationalinsights/workspaces/savedsearches" + and resource.get("properties", {}).get("functionAlias") == record["functionAlias"]] + if len(saved) != 1: + raise RuntimeError("Required workspace function must have exactly one emitted savedSearch") + resource = saved[0] + _, definition = function_definition(_root(solution), record["sourcePath"]) + emitted_query = resource.get("properties", {}).get("query") + source_query = definition["FunctionQuery"] + if ("condition" in resource or "copy" in resource + or not isinstance(emitted_query, str) + or emitted_query not in (source_query, f"{source_query}\n") + or resource.get("properties", {}).get("functionParameters", "") != function_parameters(definition)): + raise RuntimeError("Required workspace function must be unconditional and match current source") + name = resource.get("name") + if not isinstance(name, str) or not name: + raise RuntimeError("Required savedSearch has no exact resource identity") + if name.startswith("["): + if name.startswith("[[") or not name.endswith("]"): + raise RuntimeError("Invalid required savedSearch ARM identity") + argument = name[1:-1] + else: + if len(name.split("/")) != 2 or not all(name.split("/")): + raise RuntimeError("Required savedSearch identity must include workspace and resource name") + argument = "'" + name.replace("'", "''") + "'" + dependency = f"[resourceId('Microsoft.OperationalInsights/workspaces/savedSearches', {argument})]" + if dependency not in parser_dependencies: + parser_dependencies.append(dependency) + expected_install_dependencies = [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]", + *parser_dependencies, + ] + if install.get("dependsOn") != expected_install_dependencies: + raise RuntimeError("Workbook installation must depend on exact mandatory workspace functions") + inner = install.get("properties", {}).get("template") or {} + quartet = {"packageId": package_id, "packageVersion": version, + "contentId": content_id, "contentVersion": version} + notebook = deepcopy(authored) + notebook.pop("xdrMetadata") + notebook["contentProvenance"] = quartet + expected_resource = arm_copy({ + "import": "MicrosoftSecurity", "type": "Microsoft.Security/workbooks@2026-06-01-preview", + "properties": {"id": instance_id, "name": metadata["displayName"], "content": notebook}, + }) + if ( + install.get("apiVersion") != "2025-04-01" + or install.get("condition") != "[parameters('DeployDefenderWorkbook')]" + or install.get("properties", {}).get("expressionEvaluationOptions", {}).get("scope") != "inner" + or inner.get("languageVersion") != "2.0" + or inner.get("imports") != {"MicrosoftSecurity": {"provider": "MicrosoftSecurity", "version": "1.0.0"}} + or inner.get("resources") != {"workbook": expected_resource} + ): + raise RuntimeError("Defender workbook installation template/provenance projection is incorrect") + properties = registration.get("properties") or {} + full_provenance = {**quartet, **{ + key: authored["contentProvenance"][key] + for key in ("requiredWorkloads", "requiredDataConnectors") + }, "requiredWorkspaceFunctions": function_records} + expected_name = ( + "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/'," + f"concat(parameters('workspace'),'-dwb-',uniqueString('{content_id}')))]" + ) + if ( + registration.get("type") != "Microsoft.OperationalInsights/workspaces/providers/contentTemplates" + or registration.get("apiVersion") != "2023-04-01-preview" + or registration.get("condition") != "[parameters('DeployDefenderWorkbook')]" + or registration.get("name") != expected_name + or properties.get("mainTemplate") != inner + or properties.get("contentProvenance") != arm_copy(full_provenance) + or any(properties.get(key) != value for key, value in + {"contentId": content_id, "version": version, "packageId": package_id, + "packageVersion": version, "contentKind": "DefenderWorkbook"}.items()) + or registration.get("dependsOn") != [ + f"[resourceId('Microsoft.Resources/deployments', '{deployment_name}')]", + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]", + ] + ): + raise RuntimeError("Defender workbook registration/stored template/provenance is incorrect") + expected_criteria.append({"kind": "DefenderWorkbook", "contentId": content_id, "version": version}) + identities.append({"contentId": content_id, "instanceId": instance_id, + "path": path.relative_to(solution).as_posix()}) + criteria = packages[0].get("properties", {}).get("dependencies", {}).get("criteria") or [] + if [row for row in criteria if row.get("kind") == "DefenderWorkbook"] != expected_criteria: + raise RuntimeError("Defender workbook package dependency criteria differs from selection") + with zipfile.ZipFile(zip_path) as archive: + entries = [name for name in archive.namelist() if Path(name).name == "mainTemplate.json"] + if len(entries) != 1 or archive.read(entries[0]) != (solution / "Package" / "mainTemplate.json").read_bytes(): + raise RuntimeError("Defender workbook ZIP template does not match the exact current package bytes") + return {"count": len(files), "identities": identities, "contractSupport": "unverified"} + + def package_solution_v3_1( solution: str | Path, *, @@ -758,7 +935,10 @@ def package_solution_v3_1( package_attempt_id: str | None = None, workflow_run_id: str | None = None, selection_plan: dict[str, Any] | None = None, + arm_validation_method: str = "arm-ttk", ) -> dict[str, Any]: + if arm_validation_method not in {"arm-ttk", "lab-deployment"}: + raise ValueError("ARM validation method must be arm-ttk or lab-deployment") if version_bump not in {"none", "patch", "minor", "major"}: raise ValueError("version bump must be none, patch, minor, or major") if target_version is not None: @@ -791,7 +971,6 @@ def package_solution_v3_1( selected_toolkit_root = _validate_v31_toolkit_root( toolkit_root if toolkit_root is not None else repository ) - reports = report_directory(root, create=True) data_directory = root / "Data" if not data_directory.is_dir(): raise ValueError(f"solution data folder does not exist: {data_directory}") @@ -799,7 +978,20 @@ def package_solution_v3_1( data_file = _solution_data_file(data_directory) before = _read_json(data_file) _validate_solution_package_inputs(root, before) + validate_v31_effective_version(_value(before, "Version"), version_bump, target_version) + reports = report_directory(root, create=True) xdr_files = _xdr_references(root, before) + from .workbooks import selected_workbook_files, validate_workbooks + if _value(before, "XDR Workbooks") is None: + if any((root / "XDR Workbooks").rglob("*.json")): + raise ValueError("solution Data is missing an explicit XDR Workbooks array") + workbook_files = [] + else: + workbook_files = selected_workbook_files(root) + if workbook_files: + workbook_validation = validate_workbooks(root, selected_only=True) + if workbook_validation["invalid"] or workbook_validation["valid"] != len(workbook_files): + raise ValueError("selected Defender workbooks have unresolved structural or review blockers") script = ( selected_toolkit_root / "Tools" @@ -822,6 +1014,8 @@ def package_solution_v3_1( str(data_directory), "-VersionMode", "local", + "-ArmValidationMethod", + arm_validation_method, "-VersionBump", version_bump, ] @@ -941,6 +1135,7 @@ def package_solution_v3_1( deployment_parameters = _deployment_parameters( template_document, _read_json(create_ui), _read_json(test_parameters), has_xdr=bool(xdr_files), + has_workbooks=bool(workbook_files), contract_path=( selected_toolkit_root / "Tools" / "Create-Azure-Sentinel-Solution" / "common" @@ -975,6 +1170,7 @@ def package_solution_v3_1( _verify_package_selection(root, selection_plan, template_document, zip_path) if selection_plan is not None else None ) + workbook_verification = _verify_workbook_package(root, workbook_files, template_document, zip_path) artifact_paths = { "mainTemplate": main_template, @@ -1014,6 +1210,11 @@ def package_solution_v3_1( ) result = { "status": "passed", + "armValidationMethod": arm_validation_method, + "packageLifecycleStatus": ( + "candidate-generated" if arm_validation_method == "lab-deployment" or workbook_files else "accepted" + ), + "packageAccepted": arm_validation_method == "arm-ttk" and not workbook_files, "packager": PACKAGER_NAME, "warnings": warnings, "informational": informational, @@ -1043,6 +1244,9 @@ def package_solution_v3_1( "packageArtifactsSha256": _canonical_digest(artifact_hashes), "solutionData": str(data_file), "xdrDetectionCount": len(xdr_files), + "xdrWorkbookCount": len(workbook_files), + "workbookVerification": workbook_verification, + "defenderWorkbookContractSupport": "unverified" if workbook_files else "notApplicable", "deploymentParameters": deployment_parameters, "customDetectionRegistrationSupport": "unverified" if xdr_files else "notApplicable", "selectionOutcome": selection_outcome, diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/packaging_selection.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/packaging_selection.py index d3e0f6113df..3fea045c9d1 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/packaging_selection.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/packaging_selection.py @@ -808,6 +808,12 @@ def unavailable(status: str, reason: str, report_hash: str | None = None): result[field] = value if expected_query_hash is None or actual_hash != expected_query_hash: result.update(status="stale", reason="runtime-query-hash-stale") + elif provider != "graph" and status == "passed": + from .dependencies import dependency_identity + + current_document = yaml.safe_load(detection_path.read_text(encoding="utf-8-sig")) + if entry.get("dependencyProvenanceSha256") != dependency_identity(current_document): + result.update(status="stale", reason="runtime-dependency-provenance-stale") elif status != "passed": result["reason"] = f"runtime-result-{status}" elif provider == "graph": diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/report_dashboard.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/report_dashboard.py index 16cfbc86bdd..1332a4e877d 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/report_dashboard.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/report_dashboard.py @@ -2236,6 +2236,38 @@ def _deployment_planning_html(report: dict[str, Any]) -> str: + hash_details + '' ) +def _workbooks_html(report: dict[str, Any]) -> str: + workbooks = report.get("workbooks") + if not isinstance(workbooks, dict): + return "" + excluded = {row["path"]: row["reason"] for row in workbooks.get("excluded") or []} + rows = [] + for source in workbooks.get("sources") or []: + reasons = [excluded[source["path"]]] if source["path"] in excluded else [] + reasons.extend( + item.get("message", item.get("code", "Recorded finding")) if isinstance(item, dict) else str(item) + for item in source.get("issues") or [] + ) + rows.append("" + "".join( + f"" for value in ( + source["path"], "Selected" if source["selected"] else "Excluded", + source["status"], "; ".join(reasons), + ) + ) + "") + return ( + '

Defender workbooks

' + f'

{_text(workbooks.get("inventoryCount"))} frozen sources · {_text(workbooks.get("selectedCount"))} selected. ' + 'Workbook panels do not count toward detection coverage.

' + f'

ARM method: {_text(workbooks.get("armValidationMethod"))}. ' + f'Installation/registration contract: {_text(workbooks.get("contractSupport"))}. ' + 'Query/UI Qualification is independent of ARM acceptance and detection alert parity.

' + + "".join(f"

{_text(issue)}

" for issue in workbooks.get("issues") or []) + + '
{_text(value)}
' + '' + + "".join(rows) + "
Original workbookSelectionConversionRecorded reasons
" + ) + + def render_dashboard(report: dict[str, Any]) -> str: workflow = report.get("workflow") or {} candidates = review_candidates(report) @@ -2409,6 +2441,7 @@ def render_dashboard(report: dict[str, Any]) -> str: + _deployment_planning_html(report) + _findings_chart_html(report["rules"]) + _run_html(report) + + _workbooks_html(report) + '

Current artifact overview

' f'
{len(prepared)} rules{_text(summary.get("converted"))} automated conversions' f'{_text(summary.get("needsReview"))} converter review flags{structural} local structural passes' diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/runtime.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/runtime.py index a5e271c7424..a75a9c06a0f 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/runtime.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/runtime.py @@ -20,7 +20,8 @@ portable_artifact, write_json_artifact, ) -from .converter import custom_detail_schema_errors, xdr_detection_files +from .converter import custom_detail_schema_errors, load_config, xdr_detection_files +from .dependencies import dependency_identity, live_dependency_check from .content_paths import content_path from .graph_auth import validate_delegated_graph_claims from .onboarding import ( @@ -576,6 +577,10 @@ def candidate_assessments() -> list[dict[str, Any]]: for name, expected_hash in expected_hashes.items(): entry = by_name[name] hash_matches = entry.get("querySha256") == expected_hash + if provider in SOURCE_PROVIDERS and entry.get("status") == "passed": + current_document = yaml.safe_load(paths[name].read_text(encoding="utf-8-sig")) + if entry.get("dependencyProvenanceSha256") != dependency_identity(current_document): + failures.append(f"{provider} dependency provenance for {name} is stale or unbound") source_match = ( missing_schema_whitelist_match( entry, @@ -970,6 +975,7 @@ def run_advanced_hunting_query( timeout: int = 90, retries: int = 2, _token: str | None = None, + capture_full_results: bool = False, ) -> dict[str, Any]: """Run one Graph Advanced Hunting read within this process's provider bounds. @@ -978,6 +984,8 @@ def run_advanced_hunting_query( Transient attempts share a five-minute wall-clock budget, including request durations and retry waits. The public retry default remains two. """ + if not isinstance(capture_full_results, bool): + raise ValueError("capture_full_results must be a boolean") token = _token or _advanced_hunting_token(_state_dir(state_dir)) request = urllib.request.Request( ADVANCED_HUNTING_ENDPOINT, @@ -1057,6 +1065,7 @@ def run_advanced_hunting_query( "schema": schema, "sampleRows": rows[:MAX_ADVANCED_HUNTING_SAMPLE_ROWS], "sampleRowsTruncated": len(rows) > MAX_ADVANCED_HUNTING_SAMPLE_ROWS, + **({"results": rows} if capture_full_results else {}), "error": None, "errorDetails": None, } @@ -1513,6 +1522,7 @@ def record_runtime_validation( } normalized: list[dict[str, Any]] = [] names: set[str] = set() + dependency_config = load_config(output) for entry in entries: if not isinstance(entry, dict): raise ValueError("each runtime result must be a JSON object") @@ -1530,7 +1540,13 @@ def record_runtime_validation( raise ValueError(f"invalid runtime status {status!r} for {detection}") error = entry.get("error") schema = entry.get("schema") + dependency_validation = None query_hash = entry.get("querySha256") + if status == "passed" and ( + entry.get("partial") is True or error or entry.get("failureKind") + ): + status = "failed" + error = error or "source query returned a partial or failed result" if status == "passed" and detection in expected_query_hashes: if not isinstance(query_hash, str) or not re.fullmatch(r"[0-9a-f]{64}", query_hash): status = "blocked" @@ -1544,6 +1560,21 @@ def record_runtime_validation( "Sentinel validation query hash is stale; rerun the complete Sentinel " "query family against the current source queries" ) + if status == "passed" and detection in detection_paths: + document = yaml.safe_load(detection_paths[detection].read_text(encoding="utf-8-sig")) + source_path = content_path(root, document["contentProvenance"]["source"]["path"]) + source_document = yaml.safe_load(source_path.read_text(encoding="utf-8-sig")) + try: + dependency_validation = live_dependency_check( + document, source_document["query"], entry.get("dataSources"), + table_mappings=dependency_config.get("tableMappings"), + source_connectors=source_document.get("requiredDataConnectors"), + ) + except ValueError as exc: + dependency_validation = {"status": "failed", "errors": [str(exc)]} + if dependency_validation["status"] != "passed": + status = "failed" + error = "; ".join(dependency_validation["errors"]) if status == "passed" and provider == "graph" and detection in detection_paths: document = yaml.safe_load(detection_paths[detection].read_text(encoding="utf-8-sig")) or {} details = ( @@ -1576,9 +1607,13 @@ def record_runtime_validation( "attempts", "durationMs", "schema", + "dataSources", ): if field in entry: normalized_entry[field] = entry[field] + if dependency_validation is not None: + normalized_entry["dependencyValidation"] = dependency_validation + normalized_entry["dependencyProvenanceSha256"] = dependency_identity(document) sample_rows = entry.get("sampleRows") if isinstance(sample_rows, list): normalized_entry["sampleRows"] = sample_rows[:MAX_ADVANCED_HUNTING_SAMPLE_ROWS] diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/solution_report.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/solution_report.py index 39bc05fd7b1..ee3b2313d05 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/solution_report.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/solution_report.py @@ -1432,11 +1432,23 @@ def build_solution_report( *, ingestion_reports: list[str | Path] | None = None, ) -> dict[str, Any]: - root, analytic, output = solution_paths(solution) + from . import workbook_workflow + root = Path(solution).expanduser().resolve() + workflow = _read_json(existing_artifact_path(root, "workflow-state.json")) + workbook_context = workflow.get("context") + has_workbook_state = "workbooks" in workflow or ( + isinstance(workbook_context, dict) and "workbookCapabilityVersion" in workbook_context + ) + current_workflow = workbook_workflow.current_state(root) if has_workbook_state else None + workbook_only = bool(current_workflow and workbook_workflow.enabled(current_workflow) + and not workbook_workflow.has_analytic_rules(root)) + if workbook_only: + analytic, output = root / "Analytic Rules", root / "XDR Detections" + else: + root, analytic, output = solution_paths(solution) reports = report_directory(root, create=True) manifest = _read_json(existing_artifact_path(root, "manifest.json")) - workflow = _read_json(existing_artifact_path(root, "workflow-state.json")) - runtime_consent = runtime_query_consent_status(root) + runtime_consent = {"status": "notApplicable"} if workbook_only else runtime_query_consent_status(root) manifest_by_source = {} for item in manifest.get("results") or []: reference = str(item.get("sourceRelativePath") or item.get("source") or "").replace("\\", "/") @@ -1488,7 +1500,7 @@ def build_solution_report( ] rules: list[dict[str, Any]] = [] - for source_path in analytic_rule_files(root): + for source_path in ([] if workbook_only else analytic_rule_files(root)): source = yaml.safe_load(source_path.read_text(encoding="utf-8-sig")) or {} conversion = manifest_by_source.get(source_path.relative_to(root).as_posix()) or {} detection_name = source_path.relative_to(analytic).as_posix() @@ -1979,6 +1991,8 @@ def build_solution_report( ], } json_path = reports / JSON_NAME + if current_workflow and workbook_workflow.enabled(current_workflow): + report["workbooks"] = workbook_workflow.summary(root, current_workflow) html_path = reports / HTML_NAME report["jsonReport"] = str(json_path) report["htmlReport"] = str(html_path) diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_browser.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_browser.py new file mode 100644 index 00000000000..5a78fd1b28f --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_browser.py @@ -0,0 +1,573 @@ +"""Reviewed, bounded DOM assertions; never an authentication or deployment client.""" +from __future__ import annotations + +import hashlib +import json +import os +import re +import stat +import subprocess +import time +from pathlib import Path +from typing import Any, Callable +from urllib.parse import parse_qsl, urlsplit + + +class BrowserQualificationError(ValueError): + """Sanitized failure: browser exceptions can contain credentials and URLs.""" + + +def _fail(message: str) -> None: + raise BrowserQualificationError(message) + + +def observation_digest(value: Any) -> str: + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode()).hexdigest() + + +def _keys(value: Any, required: set[str], optional: set[str] | None = None) -> None: + if not isinstance(value, dict) or not required <= value.keys() or value.keys() - required - (optional or set()): + _fail("browser configuration has missing or unsupported fields") + + +def _selector(value: Any) -> None: + if not isinstance(value, str) or not value.strip() or len(value) > 1024: + _fail("an exact reviewed DOM selector is required") + + +def _assertion(value: Any) -> None: + _keys(value, {"selector", "kind"}, {"expected", "attribute", "bounds"}) + _selector(value["selector"]) + kind = value["kind"] + if kind in {"text", "attribute", "value"}: + if not isinstance(value.get("expected"), str) or len(value["expected"]) > 4096: + _fail("DOM text, attribute and value assertions need exact expected strings") + if kind == "attribute": + if not isinstance(value.get("attribute"), str) or not re.fullmatch(r"[A-Za-z_][\w:.-]*", value["attribute"]): + _fail("attribute assertion needs an exact attribute name") + elif "attribute" in value: + _fail("attribute name is only permitted for attribute assertions") + if "bounds" in value: + _fail("bounds are only permitted for layout assertions") + elif kind in {"visible", "hidden"}: + if set(value) != {"selector", "kind"}: + _fail("visibility assertions do not accept expected values") + elif kind == "layout": + if set(value) != {"selector", "kind", "bounds"}: + _fail("layout assertion requires reviewed bounds") + _keys(value["bounds"], {"x", "y", "width", "height"}) + for axis, bounds in value["bounds"].items(): + if ( + not isinstance(bounds, list) or len(bounds) != 2 + or any(isinstance(n, bool) or not isinstance(n, (int, float)) or not -100000 <= n <= 100000 for n in bounds) + or bounds[0] > bounds[1] + or axis in {"width", "height"} and bounds[0] <= 0 + ): + _fail("layout bounds must be finite ordered ranges with positive dimensions") + else: + _fail("unsupported DOM assertion kind") + + +def _assertions(values: Any) -> None: + if not isinstance(values, list) or not 1 <= len(values) <= 50: + _fail("each browser check needs a bounded, nonempty assertion list") + for value in values: + _assertion(value) + + +def _url(value: Any, origin: str | None) -> None: + if not isinstance(value, str): + _fail("reviewed browser URL is required") + parsed = urlsplit(value) + if ( + (parsed.scheme + "://" + parsed.netloc != origin if origin else parsed.scheme != "https" or not parsed.hostname) + or parsed.username or parsed.password + or "*" in value + or any(key.casefold().replace("_", "").replace("-", "") in { + "accesstoken", "refreshtoken", "idtoken", "token", "code", "sig", + "signature", "clientsecret", "authorization", "cookie", "apikey", + } for key, _ in parse_qsl(parsed.query) + parse_qsl(parsed.fragment.split("?", 1)[-1])) + ): + _fail("browser URLs must be exact approved URLs without credentials or signatures") + + +def validate_browser_config(plan: dict[str, Any], config: dict[str, Any]) -> None: + """Validate the versioned, hash-bound contract without importing Playwright.""" + _keys(config, { + "schemaVersion", "storageStateSha256", "readiness", "sessionAssertions", + "guards", "network", "timeoutMs", "familyTimeoutSeconds", "settleMs", "viewport", + }) + if config["schemaVersion"] != "1.0.0" or not re.fullmatch(r"[0-9a-f]{64}", str(config["storageStateSha256"])): + _fail("browser configuration version or private session hash is invalid") + for key, low, high in (("timeoutMs", 100, 30000), ("familyTimeoutSeconds", 1, 600), ("settleMs", 100, 5000)): + if type(config[key]) is not int or not low <= config[key] <= high: + _fail("browser timeouts must be explicitly bounded integers") + _keys(config["viewport"], {"width", "height"}) + if any(type(value) is not int or not 320 <= value <= 4096 for value in config["viewport"].values()): + _fail("browser viewport must have bounded reviewed dimensions") + _keys(config["readiness"], {"tenantId", "contentId", "instanceId", "ready"}) + identities = { + "tenantId": plan["target"]["tenantId"], "contentId": plan["contentId"], + "instanceId": plan["instanceId"], "ready": "true", + } + selectors = {"tenantId": "tenant", "contentId": "workbook", "instanceId": "workbook", "ready": "ready"} + for key, expected in identities.items(): + assertion = config["readiness"][key] + _assertion(assertion) + if ( + assertion["kind"] not in {"text", "attribute"} + or assertion["expected"] != expected + or assertion["selector"] != plan["readinessSelectors"][selectors[key]] + ): + _fail("browser readiness must observe both exact identities, tenant, and ready state") + if config["readiness"]["contentId"] == config["readiness"]["instanceId"]: + _fail("canonical and installed workbook identities need independent observations") + _assertions(config["sessionAssertions"]) + if not any(a["kind"] in {"text", "attribute"} and a.get("expected") for a in config["sessionAssertions"]): + _fail("session readiness requires a positive visible authenticated-session assertion") + _keys(config["guards"], {"queryErrors", "prohibitedControls", "signedOut"}) + for selectors in config["guards"].values(): + if not isinstance(selectors, list) or not 1 <= len(selectors) <= 50: + _fail("reviewed query, edit and signed-out guards are mandatory") + for selector in selectors: + _selector(selector) + _keys(config["network"], {"requests", "navigationUrls"}) + network = config["network"] + origin_parts = urlsplit(plan["url"]) + origin = origin_parts.scheme + "://" + origin_parts.netloc + if not isinstance(network["navigationUrls"], list) or not 1 <= len(network["navigationUrls"]) <= 25: + _fail("exact reviewed navigation URL allowlist is required") + for url in network["navigationUrls"]: + _url(url, origin) + if plan["url"] not in network["navigationUrls"]: + _fail("initial workbook URL is not approved for navigation") + if not isinstance(network["requests"], list) or not 1 <= len(network["requests"]) <= 100: + _fail("exact reviewed background request allowlist is required") + for rule in network["requests"]: + _keys(rule, {"url", "methods", "purpose"}) + _url(rule["url"], origin if origin_parts.hostname in {"127.0.0.1", "localhost"} else None) + if not isinstance(rule["methods"], list) or not rule["methods"] or set(rule["methods"]) - {"GET", "HEAD", "POST"}: + _fail("only specifically reviewed GET, HEAD or read-query POST requests are permitted") + if rule["purpose"] not in {"asset", "read-query"} or "POST" in rule["methods"] and rule["purpose"] != "read-query": + _fail("POST requires an explicit reviewed read-query classification") + if not any(rule["url"] == plan["url"] and "GET" in rule["methods"] for rule in network["requests"]): + _fail("initial document request is not approved") + if not plan.get("backgroundActivityReview", "").strip() or plan.get("saveEditAllowed") is not False: + _fail("non-edit and background activity review are required") + if not isinstance(plan["steps"], list) or not 7 <= len(plan["steps"]) <= 50: + _fail("browser steps must be bounded and complete") + if any( + not isinstance(step, dict) or not isinstance(step.get("id"), str) + or not step["id"].strip() or len(step["id"]) > 200 + or not isinstance(step.get("kind"), str) + for step in plan["steps"] + ): + _fail("browser check identities and kinds must be explicit bounded strings") + if ( + {step.get("kind") for step in plan["steps"]} != { + "pickers", "dependent-refresh", "filters", "drill-down", "reload", "layout", "conditional-visibility", + } + or len({step.get("id") for step in plan["steps"]}) != len(plan["steps"]) + ): + _fail("browser check kinds and identities must be complete and unique") + for step in plan["steps"]: + _keys(step, {"id", "kind", "action", "assertions"}, { + "before", "selector", "value", "expectedUrl", "selectMode", "optionSelector", + }) + _assertions(step["assertions"]) + if not any(a["kind"] in {"text", "attribute", "value", "layout"} for a in step["assertions"]): + _fail("every step must assert visible panel content or layout") + if "before" in step: + _assertions(step["before"]) + action = step["action"] + if action not in {"observe", "select", "reload", "drill-down"}: + _fail("unsupported browser action") + if action in {"select", "drill-down"}: + _selector(step.get("selector")) + if action == "select": + if not isinstance(step.get("value"), str) or not step["value"]: + _fail("select action requires an exact reviewed option value") + mode = step.get("selectMode", "native") + if mode not in {"native", "option"}: + _fail("select mode must be native or an exact reviewed option locator") + if mode == "option": + _selector(step.get("optionSelector")) + elif "optionSelector" in step: + _fail("native selections do not accept an option locator") + kinds = {"value"} if mode == "native" else {"text", "attribute", "value"} + if not any(a["selector"] == step["selector"] and a["kind"] in kinds and a["expected"] == step["value"] for a in step["assertions"]): + _fail("selection must be independently observed after the action") + elif {"value", "selectMode", "optionSelector"} & step.keys(): + _fail("only select actions accept an option value") + if action == "drill-down": + if step.get("expectedUrl") not in network["navigationUrls"]: + _fail("drill-down needs an exact reviewed destination URL") + elif "expectedUrl" in step: + _fail("only drill-down accepts a destination URL") + if action not in {"select", "drill-down"} and "selector" in step: + _fail("observation and reload do not accept action selectors") + if step["kind"] in {"pickers", "filters", "dependent-refresh"} and action != "select": + _fail("picker, filter and dependent refresh checks must exercise selection") + if step["kind"] == "reload" and action != "reload" or step["kind"] == "drill-down" and action != "drill-down": + _fail("reload and drill-down checks must exercise their reviewed action") + if step["kind"] == "layout" and not any(a["kind"] == "layout" for a in step["assertions"]): + _fail("layout check requires measured visible panel bounds") + if step["kind"] == "conditional-visibility" and not any(a["kind"] in {"visible", "hidden"} for a in step["assertions"]): + _fail("conditional visibility check needs an explicit visibility assertion") + if step["kind"] == "dependent-refresh": + if not any( + before["kind"] in {"text", "attribute", "value"} + and before["selector"] != step["selector"] + and any( + after["selector"] == before["selector"] and after["kind"] == before["kind"] + and after.get("attribute") == before.get("attribute") + and after.get("expected") != before["expected"] + for after in step["assertions"] + ) + for before in step.get("before", []) + ): + _fail("dependent refresh must prove an observed panel changed, not just a picker") + + +def _private_storage(path: str | Path, expected_hash: str) -> dict[str, Any]: + if not hasattr(os, "getuid") or not hasattr(os, "O_NOFOLLOW"): + _fail("private browser session ownership verification requires a supported POSIX host") + path = Path(path).expanduser().absolute() + if any(part.is_symlink() for part in (path, *path.parents)): + _fail("private browser session must not use symbolic links") + try: + info = path.stat() + if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o077: + _fail("existing browser storage state must be owner-only and owned by this user") + if info.st_size > 10_000_000: + _fail("private browser session exceeds the bounded size") + with os.fdopen(os.open(path, os.O_RDONLY | os.O_NOFOLLOW), "rb") as stream: + opened = os.fstat(stream.fileno()) + if ( + (opened.st_dev, opened.st_ino) != (info.st_dev, info.st_ino) + or opened.st_uid != os.getuid() or opened.st_mode & 0o077 + ): + _fail("private browser session changed while it was being opened") + raw = stream.read(10_000_001) + if hashlib.sha256(raw).hexdigest() != expected_hash: + _fail("private browser session differs from the reviewed plan") + state = json.loads(raw) + if not isinstance(state, dict) or not isinstance(state.get("cookies"), list) or not isinstance(state.get("origins"), list): + _fail("existing Playwright storage state is incomplete") + if not state["cookies"] and not state["origins"]: + _fail("an existing authenticated browser session is required") + return state + except BrowserQualificationError: + raise + except (OSError, ValueError): + _fail("existing private browser session cannot be read") + + +def prepare_private_browser_evidence(solution: str | Path) -> None: + from .artifacts import report_directory + + directory = report_directory(solution, create=True) + if not hasattr(os, "getuid"): + _fail("private browser evidence verification requires a supported POSIX host") + if any(part.is_symlink() for part in (directory, *directory.parents)): + _fail("browser evidence directory must not use symbolic links") + try: + ignored = subprocess.run( + ["git", "-C", str(directory), "check-ignore", "--quiet", str(directory / "browser-evidence.json")], + capture_output=True, timeout=5, + ) + tracked = subprocess.run( + ["git", "-C", str(directory), "ls-files", "--", str(directory)], + capture_output=True, timeout=5, + ) + if ignored.returncode != 0 or tracked.returncode != 0 or tracked.stdout: + _fail("browser evidence must be Git-ignored and untracked before execution") + directory.chmod(0o700) + except (OSError, subprocess.SubprocessError): + _fail("private local browser evidence destination cannot be verified") + + +class _DOMRun: + def __init__(self, plan: dict[str, Any], guard: Callable[[], None]): + self.plan, self.config, self.guard = plan, plan["browserConfig"], guard + self.deadline = time.monotonic() + self.config["familyTimeoutSeconds"] + self.faults: set[str] = set() + self.events: list[dict[str, Any]] = [] + self.observed: dict[str, str | bool] = {} + + def check(self) -> None: + self.guard() + if time.monotonic() >= self.deadline: + _fail("browser family deadline expired") + if self.faults: + _fail("browser safety or query monitor failed: " + ", ".join(sorted(self.faults))) + + def timeout(self) -> int: + self.check() + return max(1, min(self.config["timeoutMs"], int((self.deadline - time.monotonic()) * 1000))) + + def observe(self, page: Any, assertion: dict[str, Any]) -> str | bool | dict[str, float]: + self.check() + locator = page.locator(assertion["selector"]) + kind = assertion["kind"] + locator.wait_for(state="attached", timeout=self.timeout()) + if locator.count() != 1: + _fail("reviewed DOM assertion does not resolve exactly one element") + locator.wait_for(state="hidden" if kind == "hidden" else "visible", timeout=self.timeout()) + until = time.monotonic() + self.timeout() / 1000 + while True: + if kind == "text": + actual = locator.inner_text(timeout=self.timeout()) + elif kind == "attribute": + actual = locator.get_attribute(assertion["attribute"], timeout=self.timeout()) + elif kind == "value": + actual = locator.input_value(timeout=self.timeout()) + elif kind == "layout": + actual = locator.bounding_box(timeout=self.timeout()) + else: + actual = kind == "visible" + if kind == "layout": + matched = actual is not None and all( + bounds[0] <= actual[axis] <= bounds[1] for axis, bounds in assertion["bounds"].items() + ) + else: + matched = kind not in {"text", "attribute", "value"} or actual == assertion["expected"] + if matched: + break + if time.monotonic() >= until: + _fail("observed DOM value or layout differs from the exact reviewed expectation") + page.wait_for_timeout(min(50, self.timeout())) + self.check() + return actual + + def ready(self, page: Any) -> None: + self.check() + if page.url not in self.config["network"]["navigationUrls"]: + _fail("browser navigation left the exact approved target") + for group, selectors in self.config["guards"].items(): + for selector in selectors: + matches = page.locator(selector) + count = matches.count() + if count > 100: + _fail("browser safety guard matched too many elements") + if any(matches.nth(index).is_visible() for index in range(count)): + self.faults.add(group) + self.check() + for assertion in self.config["sessionAssertions"]: + self.observe(page, assertion) + observed = { + key: self.observe(page, assertion) + for key, assertion in self.config["readiness"].items() + } + observed["ready"] = observed["ready"] == "true" + self.observed = observed + + def proof(self, page: Any, assertion: dict[str, Any]) -> dict[str, Any]: + observed = self.observe(page, assertion) + proof = { + "assertionSha256": observation_digest(assertion), + "actualSha256": observation_digest(observed), "kind": assertion["kind"], + } + if assertion["kind"] == "layout": + proof["measurement"] = observed + return proof + + def action_locator(self, page: Any, selector: str) -> Any: + locator = page.locator(selector) + locator.wait_for(state="visible", timeout=self.timeout()) + if locator.count() != 1 or locator.is_disabled(timeout=self.timeout()): + _fail("action target is missing, ambiguous, or disabled") + labels = " ".join(filter(None, ( + locator.inner_text(timeout=self.timeout()), + locator.get_attribute("aria-label", timeout=self.timeout()), + locator.get_attribute("title", timeout=self.timeout()), + locator.get_attribute("name", timeout=self.timeout()), + ))) + if re.search(r"\b(save|edit|delete|remove|publish|sign.?in|log.?in|consent|grant)\b", labels, re.I): + _fail("prohibited edit or authentication action observed") + return locator + + def attach(self, context: Any, page: Any) -> None: + rules = {(rule["url"], method) for rule in self.config["network"]["requests"] for method in rule["methods"]} + + def route_request(route: Any) -> None: + try: + self.check() + request = route.request + if (request.url, request.method) not in rules: + self.faults.add("unreviewed-request") + route.abort() + else: + route.continue_() + except Exception: + self.faults.add("request-guard") + route.abort() + + context.route("**/*", route_request) + context.route_web_socket( + "**/*", lambda socket: (self.faults.add("unreviewed-websocket"), socket.close()), + ) + context.on("page", lambda _: self.faults.add("unexpected-page")) + page.on("pageerror", lambda _: self.faults.add("javascript-error")) + page.on("console", lambda message: self.faults.add("console-error") if message.type == "error" else None) + page.on("response", lambda response: self.faults.add("query-http-error") if response.status >= 400 else None) + page.on("requestfailed", lambda _: self.faults.add("request-failed")) + page.on("websocket", lambda _: self.faults.add("unreviewed-websocket")) + page.on("download", lambda _: self.faults.add("unexpected-download")) + page.on("dialog", lambda dialog: (self.faults.add("unexpected-dialog"), dialog.dismiss())) + page.on("framenavigated", lambda frame: self.faults.add("unreviewed-navigation") if frame.url not in self.config["network"]["navigationUrls"] else None) + context.on("serviceworker", lambda _: self.faults.add("unexpected-service-worker")) + + def run(self, context: Any, page: Any) -> dict[str, Any]: + page.set_default_timeout(self.config["timeoutMs"]) + page.set_default_navigation_timeout(self.config["timeoutMs"]) + self.attach(context, page) + self.check() + page.goto(self.plan["url"], wait_until="domcontentloaded", timeout=self.timeout()) + self.ready(page) + outcomes = [] + for step in self.plan["steps"]: + self.events.append({"stepId": step["id"], "action": step["action"], "status": "running"}) + self.ready(page) + before = [self.proof(page, assertion) for assertion in step.get("before", [])] + self.check() + action = step["action"] + if action in {"select", "drill-down"}: + locator = self.action_locator(page, step["selector"]) + self.check() + if action == "select": + if step.get("selectMode", "native") == "native": + locator.select_option(value=step["value"], timeout=self.timeout()) + else: + locator.click(timeout=self.timeout(), no_wait_after=True) + self.ready(page) + option = self.action_locator(page, step["optionSelector"]) + self.check() + option.click(timeout=self.timeout(), no_wait_after=True) + else: + locator.click(timeout=self.timeout(), no_wait_after=True) + page.wait_for_url(step["expectedUrl"], timeout=self.timeout()) + if page.url != step["expectedUrl"]: + _fail("drill-down navigation did not match the reviewed URL") + elif action == "reload": + page.reload(wait_until="domcontentloaded", timeout=self.timeout()) + self.ready(page) + for assertion in step["assertions"]: + self.observe(page, assertion) + page.wait_for_timeout(min(self.config["settleMs"], self.timeout())) + self.ready(page) + assertions = [self.proof(page, assertion) for assertion in step["assertions"]] + outcomes.append({ + "id": step["id"], "kind": step["kind"], "status": "passed", + "evidence": "Exact reviewed DOM assertions observed twice after the action.", + "assertionCount": len(step["assertions"]), + "assertions": assertions, "beforeAssertions": before, + }) + self.events[-1]["status"] = "passed" + self.ready(page) + if page.url != self.plan["url"]: + _fail("final browser URL differs from the approved workbook URL") + return { + "target": self.plan["target"], "url": page.url, "readiness": self.observed, + "steps": outcomes, "queryErrors": [], "javascriptErrors": [], + "saveEditObserved": False, "backgroundActivityReview": self.plan["backgroundActivityReview"], + "producer": "playwright-dom-v1", "diagnosticEvents": self.events, + "browserWideNoWriteSandbox": False, + } + + +def capture_workbook_browser( + plan: dict[str, Any], *, storage_state: str | Path, + before_action: Callable[[], None], +) -> dict[str, Any]: + """Use only an existing private session in a fresh, nonpersistent context. + + No traces, HAR, storage-state export, raw error messages or page content are + retained. Exact request allowlists reduce risk; they are not a no-write sandbox. + """ + config = plan.get("browserConfig") + if config is None: + _fail("real browser execution requires a reviewed versioned browser configuration") + validate_browser_config(plan, config) + before_action() + state = _private_storage(storage_state, config["storageStateSha256"]) + try: + from playwright.sync_api import sync_playwright + except ImportError: + _fail("optional browser dependency is missing; install the toolkit browser extra") + engine = None + try: + with sync_playwright() as playwright: + browser = playwright.chromium.launch(headless=True, timeout=config["timeoutMs"]) + try: + context = browser.new_context( + storage_state=state, service_workers="block", accept_downloads=False, + viewport=config["viewport"], + ) + try: + page = context.new_page() + engine = _DOMRun(plan, before_action) + return engine.run(context, page) + finally: + context.close() + finally: + browser.close() + except BrowserQualificationError as exc: + exc.diagnostic_events = _failure_events(engine) + raise + except Exception: + failure = BrowserQualificationError("browser execution failed; no raw browser exception was retained") + failure.diagnostic_events = _failure_events(engine) + raise failure from None + + +def _failure_events(engine: _DOMRun | None) -> list[dict[str, Any]]: + events = [dict(event) for event in engine.events] if engine else [] + if events and events[-1]["status"] == "running": + events[-1]["status"] = "failed" + events.append({"status": "failed", "faultCodes": sorted(engine.faults) if engine else ["browser-startup"]}) + return events + + +def validate_browser_step_proofs(step: dict[str, Any], actual: dict[str, Any]) -> None: + for field, expected in (("assertions", step["assertions"]), ("beforeAssertions", step.get("before", []))): + proofs = actual.get(field) + if not isinstance(proofs, list) or len(proofs) != len(expected): + _fail("browser capture has incomplete DOM observation proofs") + for assertion, proof in zip(expected, proofs, strict=True): + if ( + not isinstance(proof, dict) or proof.get("kind") != assertion["kind"] + or proof.get("assertionSha256") != observation_digest(assertion) + ): + _fail("browser capture DOM proofs differ from reviewed assertions") + if assertion["kind"] == "layout": + observed = proof.get("measurement") + if ( + not isinstance(observed, dict) or set(observed) != {"x", "y", "width", "height"} + or any(type(n) not in (int, float) for n in observed.values()) + or any(not bounds[0] <= observed[axis] <= bounds[1] for axis, bounds in assertion["bounds"].items()) + ): + _fail("browser capture is missing reviewed visible panel measurements") + else: + observed = assertion.get("expected") if assertion["kind"] in {"text", "attribute", "value"} else assertion["kind"] == "visible" + if proof.get("actualSha256") != observation_digest(observed): + _fail("browser capture DOM observations did not match the exact expectations") + + +def execute_workbook_browser( + solution: str | Path, plan: dict[str, Any], *, + read_consent: Callable[[], dict[str, Any]], storage_state: str | Path, +) -> dict[str, Any]: + from . import workbook_qualification as qualification + + def guard() -> None: + qualification._check_consent(plan, read_consent()) + if qualification.current_workbook_ui_plan(solution, plan) != plan: + _fail("workbook browser inputs or target changed during execution") + + return qualification.execute_workbook_ui( + solution, plan, read_consent=read_consent, + browser_runner=lambda current: capture_workbook_browser( + current, storage_state=storage_state, before_action=guard, + ), + ) diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_dependencies.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_dependencies.py new file mode 100644 index 00000000000..5be6f6cb509 --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_dependencies.py @@ -0,0 +1,454 @@ +"""Frozen, offline workbook gap proposals and explicit reviewer decisions. + +Metadata is imported, never executed. Review writes only run-local config and +receipts, not source content, candidates, workflow state or cloud resources. +""" +from __future__ import annotations + +from copy import deepcopy +from datetime import datetime, timezone +import hashlib +import json +from pathlib import Path +import os +import re + +from .dependencies import dependency_plan, dependency_validation_errors, TABLE_WORKLOADS, telemetry_data_types +from ._workbook_core import workbook, mapping_catalog, migrator, rewrite +from .workbooks import _root, _path, _document, _solution_data, _sha, _write, inventory_workbooks +from .workbook_functions import candidates, validate_functions, function_definition + + +def digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), + ensure_ascii=True).encode()).hexdigest() + + +def _load(path): + path = Path(path).expanduser() + if any(p.is_symlink() for p in (path, *path.parents)): + raise ValueError("Review inputs must not use symlinks.") + def unique(pairs): + value = {} + for key, item in pairs: + if key in value: + raise ValueError(f"Duplicate JSON key: {key}") + value[key] = item + return value + return json.loads(path.read_text("utf-8"), object_pairs_hook=unique) + + +def _connectors(root, data): + alternatives = [] + references = data.get("Data Connectors", []) + if not isinstance(references, list): + raise ValueError("Solution Data Connectors must be an array.") + for relative in references: + path = _path(root, relative) + if not relative.startswith("Data Connectors/") or not path.is_file(): + raise ValueError("Connector references must identify existing Data Connectors content.") + value = json.loads(path.read_bytes()) + if not isinstance(value, dict): + raise ValueError("Connector content must be a JSON object.") + nested = value.get("properties") or value + connector_id = value.get("id") or nested.get("id") + types = nested.get("dataTypes", []) + if isinstance(types, dict): + types = list(types) + names = sorted({entry.get("name") if isinstance(entry, dict) else entry for entry in types + if isinstance(entry, str) or isinstance(entry, dict) and isinstance(entry.get("name"), str)}) + alternatives.append({ + "sourcePath": relative, "sourceSha256": _sha(path.read_bytes()), + "connectorId": connector_id, "dataTypes": telemetry_data_types(names), + **({"referenceInputs": ["Watchlist"]} if "Watchlist" in names else {}), + "title": value.get("title") or (nested.get("connectorUiConfig") or {}).get("title"), + "recommendation": None, + }) + return alternatives + + +def _binding(raw, entry): + tokens = {match.group(0)[1:-1] for match in workbook.TOKEN.finditer(raw)} + bindings = entry.get("bindings", {}) + if (not isinstance(bindings, dict) or set(bindings) != tokens + or any(not isinstance(value, str) or not value for value in bindings.values())): + raise ValueError("Metadata needs exact bindings for every raw workbook placeholder.") + rendered = workbook.TOKEN.sub(lambda match: bindings[match.group(0)[1:-1]], raw) + if workbook.TOKEN.search(rendered): + raise ValueError("Imported metadata has unresolved rendered-query placeholders.") + if entry.get("renderedQuery") != rendered or entry.get("renderedQuerySha256") != workbook.query_hash(rendered): + raise ValueError("Metadata rendered query/hash does not match exact template bindings.") + time_range = entry.get("timeRange") + if not isinstance(time_range, dict) or set(time_range) != {"start", "end"}: + raise ValueError("Metadata requires an explicit timeRange start/end.") + try: + start, end = [datetime.fromisoformat(time_range[key].replace("Z", "+00:00")) + for key in ("start", "end")] + if start.tzinfo is None or end.tzinfo is None or start >= end: + raise ValueError() + except (ValueError, TypeError, AttributeError): + raise ValueError("Metadata timeRange must be ordered timezone-aware ISO timestamps.") from None + for key, expected in (("start", start), ("end", end)): + for token, value in bindings.items(): + if token.endswith(":" + key): + match = re.fullmatch(r"\s*datetime\(\s*(['\"]?)(.*?)\1\s*\)\s*", value) + try: + actual = datetime.fromisoformat(match.group(2).replace("Z", "+00:00")) if match else None + except ValueError: + actual = None + if actual != expected: + raise ValueError("Metadata time bindings must match the exact declared timeRange.") + return rendered + + +def plan_workbook_dependencies(solution, *, results_path=None, workspace_id=None, + imported_results=None): + root = _root(solution) + inventory = inventory_workbooks(root) + data = _solution_data(root) + connectors = _connectors(root, data) + definitions = candidates(root) + imported = _load(results_path) if results_path is not None else deepcopy(imported_results) + if imported is None and workspace_id is not None: + raise ValueError("Workspace ID is only accepted for explicitly approved metadata import.") + entries = {} + if imported is not None: + if not isinstance(workspace_id, str) or not workspace_id.strip(): + raise ValueError("Metadata import requires the explicitly approved workspace ID.") + if not isinstance(imported, dict) or set(imported) != {"results"} or not isinstance(imported["results"], list): + raise ValueError("Metadata import must contain a results array.") + for entry in imported["results"]: + if not isinstance(entry, dict) or not isinstance(entry.get("workbook"), str) or not isinstance(entry.get("path"), str): + raise ValueError("Metadata must identify each workbook/query path.") + if set(entry) - {"workbook", "path", "workspaceId", "workbookSha256", "querySha256", + "status", "statusCode", "partial", "error", "errorDetails", "failureKind", + "bindings", "renderedQuery", "renderedQuerySha256", "timeRange", "dataSources"}: + raise ValueError("Unknown workbook metadata import fields.") + key = (entry["workbook"], entry["path"]) + if key in entries: + raise ValueError("Duplicate workbook query metadata.") + entries[key] = entry + expected = {(row["path"], query["path"]) for row in inventory for query in row["queries"]} + if set(entries) != expected: + raise ValueError("Metadata must cover the complete current workbook/query inventory.") + rows = [] + mappings = {name: value.get("ahTable") or name + for name, value in mapping_catalog.load_mapping_catalog()["tables"].items()} + for row in inventory: + doc = _document(_path(root, row["path"])) + helper_entries, removed, _ = workbook._scope_parameters(doc) + removed_prefixes = [path + "/" for path, parameter in helper_entries if parameter["name"] in removed] + for surface in workbook.query_surfaces(doc): + template = surface["node"]["query"] + blockers = [] + raw_valid = True + try: + raw = workbook._raw_query(template, surface["queryType"]) + except ValueError as exc: + raw = template + raw_valid = False + blockers.append(str(exc)) + provider_supported = surface["queryType"] in (0, None, workbook.AH) and ( + surface["queryType"] == workbook.AH or + (surface["resourceType"] or "").lower() in ("", workbook.LA)) + is_helper = any(surface["path"].startswith(prefix) for prefix in removed_prefixes) + if not provider_supported and not is_helper: + blockers.append("Unsupported provider requires a separately reviewed source migration.") + code = "".join(text if kind == "code" else " " * len(text) for kind, text in rewrite._segments(raw)) + if re.search(r"\b(?:workspace|app|resource|cluster|database)\s*\(", code, re.I): + blockers.append("Explicit cross-resource/scope function cannot be converted by dependency acceptance.") + aliases = sorted(workbook.opaque_functions(raw)) if provider_supported else [] + entry = entries.get((row["path"], surface["path"])) + metadata = None + if entry is not None: + if (entry.get("status") != "passed" or entry.get("partial", False) is not False + or any(entry.get(key) for key in ("error", "errorDetails", "failureKind")) + or entry.get("statusCode", 200) != 200 + or entry.get("workspaceId") != workspace_id + or entry.get("querySha256") != workbook.query_hash(template) + or entry.get("workbookSha256") != row["sha256"] + or not provider_supported or not raw_valid): + raise ValueError("Metadata has failed/partial/stale/cross-workspace scope or unsupported provider.") + _binding(raw, entry) + metadata = entry.get("dataSources") + if not isinstance(metadata, list): + raise ValueError("Metadata requires a dataSources array from Prefer: include-dataSources=true.") + # Do not accept mixed-workspace input telemetry as target-workspace evidence. + for source in metadata: + if isinstance(source, dict) and any( + source.get(key) not in (None, workspace_id) + for key in ("workspaceId", "workspace", "workspaceCustomerId") + ): + raise ValueError("Metadata dataSources contains cross-workspace input.") + proposal = dependency_plan(raw, metadata=metadata, table_mappings=mappings) + if aliases and metadata is None: + proposal["reviewReasons"].append("Underlying function tables need distinct exact-query or manual evidence.") + proposal["reviewRequired"] = True + proposals = [record for record in definitions if record.get("functionAlias") in aliases] + missing = sorted(set(aliases) - {record["functionAlias"] for record in proposals}) + reasons = list(proposal["reviewReasons"]) + if missing: + reasons.append("No current local definition for: " + ", ".join(missing)) + rows.append({ + "workbook": row["path"], "workbookSha256": row["sha256"], + "path": surface["path"], "querySha256": workbook.query_hash(template), + "sourceKql": template, "queryKql": raw, "queryType": surface["queryType"], + "resourceType": surface["resourceType"], "scopeHelper": is_helper, + "proposal": proposal, "proposalAccepted": False, + "staticConfidence": "heuristic-not-reviewed" if entry is None else "imported-not-reviewed", + "workspaceFunctions": aliases, "functionPrerequisites": proposals, + "externalSourceWarnings": ( + ["Query uses externaldata; review exact external source binding and availability separately."] + if re.search(r"\bexternaldata\s*[\[(]", code) else []), + "technicalBlockers": blockers, "gapExplanations": reasons, + "labBindingRequirements": { + "placeholders": sorted({m.group(0)[1:-1] for m in workbook.TOKEN.finditer(raw)}), + "requiredFields": ["workspaceId", "workbookSha256", "querySha256", "bindings", + "renderedQuery", "renderedQuerySha256", "timeRange", "dataSources"], + "prefer": "include-dataSources=true", + }, + "recommendedActions": ["Review raw source inputs and their telemetry/platform-reference classification.", + "Choose connector alternatives only for Sentinel telemetry; platform reference inputs require none.", + "Preserve reviewed functions; request optional parser changes separately.", + "Supply bound read-only metadata only after separate exact workspace approval."], + }) + result = { + "schemaVersion": "1.0.0", "kind": "workbook-dependency-plan", "solution": str(root), + "inventory": inventory, "solutionDataSha256": digest(data), + "workloadMap": deepcopy(TABLE_WORKLOADS), "connectorAlternatives": connectors, + "workspaceFunctionCandidates": definitions, "queries": rows, + "sourceRecommendationEvidence": { + "text": data.get("Description"), "solutionDataSha256": digest(data), + "reviewRequired": True, "titleHeuristicsUsed": False, + }, + "metadataImport": imported, "workspaceId": workspace_id, + "liveQueriesExecuted": False, "writesSource": False, "runtimeParityVerified": False, + } + return {**result, "planSha256": digest(result)} + + +def _decision(value): + if not isinstance(value, dict) or value.get("decision") not in ("accept", "change", "defer"): + raise ValueError("Every review decision must be explicit accept, change or defer.") + return value["decision"] + + +def _strings(value, field): + if (not isinstance(value, list) or any(not isinstance(item, str) or not item.strip() for item in value) + or len(value) != len(set(value))): + raise ValueError(f"{field} must be an array of unique nonempty strings.") + return value + + +def review_workbook_dependencies(solution, *, plan_path, decisions_path, expected_plan_sha256, + actor, reference, output_directory): + root = _root(solution) + plan, decisions = _load(plan_path), _load(decisions_path) + if not isinstance(actor, str) or not actor.strip() or not isinstance(reference, str) or not reference.strip(): + raise ValueError("Review requires an explicit actor and reference.") + if not isinstance(plan, dict) or plan.get("planSha256") != expected_plan_sha256: + raise ValueError("Review must bind the exact frozen plan hash.") + current = plan_workbook_dependencies(root, workspace_id=plan.get("workspaceId"), + imported_results=plan.get("metadataImport")) + if plan != current: + raise ValueError("Frozen workbook plan is malformed or stale; regenerate and review.") + if (not isinstance(decisions, dict) or set(decisions) != {"planSha256", "queries", "workbooks"} + or decisions["planSha256"] != expected_plan_sha256 + or not isinstance(decisions["queries"], list) or not isinstance(decisions["workbooks"], list)): + raise ValueError("Decisions require exact planSha256, queries and workbooks arrays.") + query_decisions = {} + for decision in decisions["queries"]: + _decision(decision) + if set(decision) - {"workbook", "path", "querySha256", "decision", "tables", "workspaceFunctions", "optionalBranches"}: + raise ValueError("Unknown query review decision fields.") + if any(not isinstance(decision.get(key), str) for key in ("workbook", "path", "querySha256")): + raise ValueError("Query review identities and hashes must be strings.") + for field in ("tables", "workspaceFunctions"): + if field in decision: + _strings(decision[field], field) + if "optionalBranches" in decision: + branches = decision["optionalBranches"] + if not isinstance(branches, list) or any( + not isinstance(branch, dict) or set(branch) != { + "functionAlias", "sourcePath", "sourceSha256", "branch", "decision"} + or any(not isinstance(value, str) or not value.strip() for value in branch.values()) + or branch["decision"] not in ("keep", "remove-request", "defer") + for branch in branches + ): + raise ValueError("Malformed optional branch decisions.") + key = (decision.get("workbook"), decision.get("path")) + if key in query_decisions: + raise ValueError("Duplicate query review decision.") + query_decisions[key] = decision + expected_queries = {(row["workbook"], row["path"]) for row in plan["queries"]} + if set(query_decisions) != expected_queries: + raise ValueError("Decisions must cover every query exactly once.") + dependency_decisions = {} + for decision in decisions["workbooks"]: + _decision(decision) + if set(decision) - {"path", "decision", "requiredWorkloads", "requiredDataConnectors", "requiredWorkspaceFunctions"}: + raise ValueError("Unknown workbook dependency decision fields.") + key = decision.get("path") + if not isinstance(key, str): + raise ValueError("Workbook review path must be a string.") + errors = dependency_validation_errors({"contentProvenance": { + field: decision.get(field, []) for field in ("requiredWorkloads", "requiredDataConnectors")}}) + if errors: + raise ValueError("; ".join(errors)) + if "requiredWorkspaceFunctions" in decision: + validate_functions(root, decision["requiredWorkspaceFunctions"]) + if key in dependency_decisions: + raise ValueError("Duplicate workbook dependency decision.") + dependency_decisions[key] = decision + if set(dependency_decisions) != {row["path"] for row in plan["inventory"]}: + raise ValueError("Decisions must cover every workbook exactly once.") + config = {"workbookResolutions": {}, "workbookDependencies": {}} + unresolved, parser_actions = [], [] + for path, decision in dependency_decisions.items(): + if decision["decision"] == "defer": + unresolved.append({"workbook": path, "reason": "Dependency review deferred."}) + settings = {"dependencyReviewRequired": True} + else: + if not {"requiredWorkloads", "requiredDataConnectors", "requiredWorkspaceFunctions"} <= set(decision): + raise ValueError("Accepted/changed dependencies require explicit workloads, connectors and functions.") + settings = {key: deepcopy(decision[key]) for key in ( + "requiredWorkloads", "requiredDataConnectors", "requiredWorkspaceFunctions")} + errors = dependency_validation_errors({"contentProvenance": settings}) + if errors: + raise ValueError("; ".join(errors)) + validate_functions(root, settings["requiredWorkspaceFunctions"]) + config["workbookDependencies"][path] = settings + for row in plan["queries"]: + decision = query_decisions[(row["workbook"], row["path"])] + if decision.get("querySha256") != row["querySha256"]: + raise ValueError("Query decision must bind the exact current source template hash.") + path = row["workbook"] + settings = config["workbookDependencies"][path] + if decision["decision"] == "defer": + settings["dependencyReviewRequired"] = True + unresolved.append({"workbook": path, "path": row["path"], "reason": "Query review deferred."}) + continue + tables = _strings(decision.get("tables"), "tables") + aliases = _strings(decision.get("workspaceFunctions", []), "workspaceFunctions") + declared_aliases = {item["functionAlias"] for item in settings.get("requiredWorkspaceFunctions", [])} + explicit_calls = workbook.opaque_functions(row["queryKql"], include_bare=False) + expected_aliases = set(row["workspaceFunctions"]) - (set(tables) - explicit_calls - declared_aliases) + if set(aliases) != expected_aliases: + raise ValueError("Reviewed workspaceFunctions must match exact source aliases.") + if not set(aliases) <= declared_aliases or set(aliases) & set(tables): + raise ValueError("Function aliases need vetted prerequisites and separate underlying tables.") + if not tables and not row["scopeHelper"]: + raise ValueError("Accepted query requires explicit underlying source tables.") + resolution = {"querySha256": row["querySha256"], "tables": tables, + "evidence": {"actor": actor, "reference": reference, "planSha256": expected_plan_sha256, + "decision": decision["decision"], "querySha256": row["querySha256"]}} + if aliases: + resolution["workspaceFunctions"] = aliases + config["workbookResolutions"].setdefault(path, {})[row["path"]] = resolution + blockers = list(row["technicalBlockers"]) + contract = migrator.convert(row["queryKql"], tables=tables, source_id=row["path"], + artifact_type="workbook", resolve=False) + if not row["scopeHelper"] and (contract["status"] != "converted" or contract["unsupported"]): + blockers.append(contract["blockedReason"] or "Shared conversion requires technical review.") + if aliases and contract["convertedKql"] != row["queryKql"]: + blockers.append("Shared conversion would modify preserved function KQL.") + branches = decision.get("optionalBranches", []) + if not isinstance(branches, list): + raise ValueError("optionalBranches must be an array.") + for branch in branches: + if (not isinstance(branch, dict) or set(branch) != { + "functionAlias", "sourcePath", "sourceSha256", "branch", "decision"} + or branch["decision"] not in ("keep", "remove-request", "defer") + or not isinstance(branch["branch"], str) or not branch["branch"].strip() + or {key: branch[key] for key in ("functionAlias", "sourcePath", "sourceSha256")} + not in settings.get("requiredWorkspaceFunctions", [])): + raise ValueError("Optional branch decisions must bind a current exact reviewed function prerequisite.") + if branch["decision"] == "remove-request": + _, definition = function_definition(root, branch["sourcePath"]) + if definition["FunctionQuery"].count(branch["branch"]) != 1: + raise ValueError("Parser-change requests require an exact unambiguous current source branch.") + parser_actions.append({**branch, "actor": actor, "reference": reference, + "branchSha256": workbook.query_hash(branch["branch"]), + "status": "requested-not-applied", "planSha256": expected_plan_sha256}) + blockers.append("Optional parser change requested, not applied or verified in current source.") + elif branch["decision"] == "defer": + blockers.append("Optional parser branch review deferred.") + for reason in blockers: + unresolved.append({"workbook": path, "path": row["path"], "reason": reason}) + if blockers: + settings["dependencyReviewRequired"] = True + catalog = mapping_catalog.load_mapping_catalog()["tables"] + for path, settings in config["workbookDependencies"].items(): + sources = {table for row in config["workbookResolutions"].get(path, {}).values() for table in row["tables"]} + targets = {catalog.get(table, {}).get("ahTable") or table for table in telemetry_data_types(sources)} + declared = {table for record in settings.get("requiredWorkloads", []) for table in record["dataTypes"]} + if not targets <= declared: + settings["dependencyReviewRequired"] = True + unresolved.append({"workbook": path, "reason": "Reviewed workloads do not cover all output tables."}) + if any(table not in sources for record in settings.get("requiredDataConnectors", []) for table in record["dataTypes"]): + raise ValueError("Reviewed connector references tables outside reviewed query sources.") + used = {alias for row in config["workbookResolutions"].get(path, {}).values() + for alias in row.get("workspaceFunctions", [])} + declared_aliases = {record["functionAlias"] for record in settings.get("requiredWorkspaceFunctions", [])} + if used != declared_aliases: + settings["dependencyReviewRequired"] = True + unresolved.append({"workbook": path, "reason": "Function prerequisite coverage is incomplete."}) + reviewed_aliases = {record["functionAlias"] for settings in config["workbookDependencies"].values() + for record in settings.get("requiredWorkspaceFunctions", [])} + reviewed_definitions = [record for record in plan["workspaceFunctionCandidates"] + if record.get("functionAlias") in reviewed_aliases] + receipt = { + "schemaVersion": "1.0.0", "planSha256": expected_plan_sha256, + "actor": actor, "reference": reference, "reviewedAt": datetime.now(timezone.utc).isoformat(), + "decisions": decisions, "configSha256": digest(config), "unresolved": unresolved, + "sourceInventory": plan["inventory"], "metadataWorkspaceId": plan["workspaceId"], + "parserChangeActions": parser_actions, + "coverageWarnings": [record["coverageWarning"] for record in reviewed_definitions + if "coverageWarning" in record], + "functionDependencyCoverage": [{ + key: record[key] for key in ( + "functionAlias", "sourcePath", "sourceSha256", "nestedFunctions", + "unresolvedBranchCandidates", "nestedDiscoveryExhaustive", "externalSourceWarnings") + } for record in reviewed_definitions], + "runtimeParityVerified": False, "writesSource": False, + "dependencyReviewRequired": bool(unresolved), + } + output = Path(output_directory).expanduser() + if any(p.is_symlink() for p in (output, *output.parents)): + raise ValueError("Review output must not use symlinks.") + output = output.resolve() + if (not output.is_relative_to(root) or output == root + or output.parts[len(root.parts)].casefold() in { + "workbooks", "xdr workbooks", "data", "parsers", "workspace functions", + "xdr detections", "analytic rules", "package", "data connectors", "hunting queries", "playbooks"}): + raise ValueError("Review output must be run-local and outside solution content directories.") + files = [(output / "conversion-config.json", config), (output / "workbook-review-receipt.json", receipt)] + if any(path.exists() or path.is_symlink() for path, _ in files): + raise ValueError("Review outputs already exist; choose a new run-local review directory.") + output.mkdir(parents=True, exist_ok=True) + created = [] + try: + for path, value in files: + # Atomic publication uses the existing safe compare-and-replace writer. + staged = path.with_name("." + path.name + ".staged") + if staged.exists() or staged.is_symlink(): + raise ValueError("Review staging file already exists.") + _write(staged, (json.dumps(value, indent=2, ensure_ascii=False) + "\n").encode(), None) + try: + if path.exists() or path.is_symlink(): + raise ValueError("Review output changed before publication.") + if any(parent.is_symlink() for parent in (output, *output.parents)): + raise ValueError("Review output changed to a symlink before publication.") + # Hard-link publication is atomic and refuses an existing target. + os.link(staged, path, follow_symlinks=False) + finally: + if staged.exists() and not staged.is_symlink(): + staged.unlink() + created.append(path) + except Exception: + for path in created: + path.unlink() + raise + return {"planSha256": expected_plan_sha256, "configPath": str(files[0][0]), + "receiptPath": str(files[1][0]), "dependencyReviewRequired": bool(unresolved), + "unresolved": unresolved, "parserChangeActions": parser_actions} diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_functions.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_functions.py new file mode 100644 index 00000000000..8115e5123a2 --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_functions.py @@ -0,0 +1,129 @@ +"""Exact, offline Sentinel workspace-function prerequisites for workbooks only.""" +from __future__ import annotations + +from copy import deepcopy +import re + +import yaml + +IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z") +FIELDS = {"functionAlias", "sourcePath", "sourceSha256"} + + +def function_parameters(document): + parameters = document.get("FunctionParams") + if parameters is None: + return "" + if not isinstance(parameters, list): + raise ValueError("Workspace function FunctionParams must be an array.") + rendered, names = [], set() + for parameter in parameters: + if (not isinstance(parameter, dict) or not isinstance(parameter.get("Name"), str) + or not IDENTIFIER.fullmatch(parameter["Name"]) + or parameter["Name"] in names or not isinstance(parameter.get("Type"), str) + or not parameter["Type"].strip()): + raise ValueError("Workspace function parameters require unique valid names and explicit types.") + names.add(parameter["Name"]) + kind = parameter["Type"] + value = f"{parameter['Name']}:{kind.split(':')[-1] if kind.startswith('table:') else kind}" + if not kind.startswith("table:") and parameter.get("Default") is not None: + default = parameter["Default"] + if isinstance(default, (dict, list)): + raise ValueError("Workspace function defaults must be scalar values.") + value += "=" + (f"'{default}'" if kind.lower() == "string" else str(default)) + rendered.append(value) + return ",".join(rendered) + + +def function_definition(root, relative): + from .workbooks import _path, _sha + + if not isinstance(relative, str) or not relative.startswith(("Parsers/", "Workspace Functions/")): + raise ValueError("Workspace function source must be Parsers/*.yaml or Workspace Functions/*.yaml.") + if re.search(r"[\x00-\x1f\x7f]", relative): + raise ValueError("Workspace function paths must not contain control characters.") + path = _path(root, relative) + if path.suffix != ".yaml" or not path.is_file(): + raise ValueError(f"Missing YAML workspace function: {relative}") + source = path.read_bytes() + try: + document = yaml.safe_load(source.decode("utf-8-sig")) + except (yaml.YAMLError, UnicodeError) as exc: + raise ValueError(f"Malformed workspace function YAML: {relative}") from exc + if not isinstance(document, dict): + raise ValueError(f"Workspace function must contain a YAML object: {relative}") + names = [document[field] for field in ("FunctionName", "FunctionAlias") if field in document] + if not names or any(not isinstance(name, str) or not IDENTIFIER.fullmatch(name) for name in names): + raise ValueError(f"Workspace function requires exact valid FunctionName/FunctionAlias: {relative}") + if not isinstance(document.get("FunctionQuery"), str) or not document["FunctionQuery"].strip(): + raise ValueError(f"Workspace function requires FunctionQuery: {relative}") + function_parameters(document) + # FunctionAlias is the saved-query alias; FunctionName is only its fallback. + alias = document.get("FunctionAlias", document.get("FunctionName")) + return {"functionAlias": alias, "sourcePath": relative, "sourceSha256": _sha(source)}, document + + +def candidates(root): + from .workbooks import _path, _sha + from ._workbook_core.workbook import opaque_functions, union_source_names + from ._workbook_core import mapping_catalog, rewrite + + result = [] + for directory in ("Parsers", "Workspace Functions"): + base = _path(root, directory) + if not base.exists(): + continue + for path in sorted(base.rglob("*")): + if path.is_symlink(): + raise ValueError("Symlink in workspace function inventory.") + if path.is_file() and path.suffix == ".yaml": + relative = path.relative_to(root).as_posix() + try: + record, document = function_definition(root, relative) + query = document["FunctionQuery"] + code = "".join(text if kind == "code" else " " * len(text) + for kind, text in rewrite._segments(query)) + known_tables = set(mapping_catalog.load_mapping_catalog()["tables"]) + branches = union_source_names(query) - known_tables - { + "externaldata", "datatable", "range", "print", "_GetWatchlist"} + result.append({**record, + "nestedFunctions": sorted(opaque_functions(document["FunctionQuery"]) - branches), + "unresolvedBranchCandidates": sorted(branches), + "nestedDiscoveryExhaustive": False, + "externalSourceWarnings": ( + ["Definition uses externaldata; review exact external source binding and availability separately."] + if re.search(r"\bexternaldata\s*[\[(]", code) else []), + "coverageWarning": "Nested/optional function branches need separate review; runtime fidelity is not verified."}) + except (ValueError, yaml.YAMLError) as exc: + result.append({"sourcePath": relative, "sourceSha256": _sha(path.read_bytes()), "error": str(exc)}) + return result + + +def validate_functions(root, records): + from .workbooks import _solution_data + + if not isinstance(records, list): + raise ValueError("requiredWorkspaceFunctions must be an array.") + data = _solution_data(root) + selected = [] + for field in ("Parsers", "Data Parsers", "Workspace Functions"): + values = data.get(field, []) + if not isinstance(values, list) or any(not isinstance(value, str) for value in values): + raise ValueError(f"Solution {field} selection must be a string array.") + selected.extend(values) + aliases, paths = set(), set() + result = [] + for record in records: + if not isinstance(record, dict) or set(record) != FIELDS: + raise ValueError("Workspace function records require exactly functionAlias, sourcePath, sourceSha256.") + current, _ = function_definition(root, record["sourcePath"]) + if current != record: + raise ValueError("Workspace function alias/hash is stale or does not match its current definition.") + if selected.count(record["sourcePath"]) != 1: + raise ValueError("Workspace function requires explicit packaged Data Parsers/Workspace Functions selection.") + if record["functionAlias"] in aliases or record["sourcePath"] in paths: + raise ValueError("Duplicate workspace function prerequisite.") + aliases.add(record["functionAlias"]) + paths.add(record["sourcePath"]) + result.append(deepcopy(record)) + return result diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_preflight.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_preflight.py new file mode 100644 index 00000000000..115b826a6a4 --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_preflight.py @@ -0,0 +1,739 @@ +"""Consent-bound candidate metadata reads, never authentication or cloud writes. + +The injected reader must honor each operation's timeout and retry limits. Read +consent permits only scoped ARM token acquisition/refresh with an already cached +identity, never sign-in, new credentials, permission grants or Graph/AH tokens. +The reader attests request/principal provenance, not access booleans. Nonempty +deny lists conservatively block; their applicability is not evaluated. Seals +bind evidence, not signatures or a substitute for a trusted reader. +""" +from __future__ import annotations + +import copy +import hashlib +import json +import re +import time +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable +from urllib.parse import quote + +from .arm_acceptance_support import _canonical_sha256 +from .content_paths import content_path +from .target_context import workspace_identity +from .workbook_validation import ( + _expected_workbook_contracts, + _FLAG_KINDS, + _resource_inventory, + _resolve, + _sealed, + _target, +) + +ARM_BASE = "https://management.azure.com" +ARM_SCOPE = ARM_BASE + "/.default" +GRAPH_BASE = "https://graph.microsoft.com" +WORKSPACE_API_VERSION = "2023-09-01" # Existing target_context workspace reads. +RESOURCE_GROUP_API_VERSION = "2021-04-01" +PERMISSIONS_API_VERSION = "2022-04-01" # dcr_readiness control-plane API. +DENY_API_VERSION = "2022-04-01" # List-for-scope atScope() includes ancestors. +CONTENT_API_VERSION = "2025-09-01" # Verified current Content Hub read/list contract. +MAX_OPERATIONS = 64 +MAX_RESPONSE_BYTES = 4 * 1024 * 1024 +MAX_TEMPLATE_BYTES = 16 * 1024 * 1024 +MAX_LIST_ITEMS = 5000 +TIMEOUT_SECONDS = 15 +_HASH = re.compile(r"^[a-f0-9]{64}$") +_REGISTRATION_TYPES = { + "microsoft.operationalinsights/workspaces/providers/contenttemplates": "contentTemplates", + "microsoft.operationalinsights/workspaces/providers/contentpackages": "contentPackages", +} +_ACCESS_FLAGS = ( + "tenantMatches", "subscriptionMatches", "workspaceMatches", "nonProduction", + "armDeploymentAllowed", "disabledContentVerified", "ownershipVerified", + "extensionResourceWriteAllowed", +) +_REDACTED_KEYS = { + "authorization", "accesstoken", "refreshtoken", "token", "secret", + "clientsecret", "password", "cookie", "setcookie", +} +_PRINCIPAL_CONTRACT = { + "source": "authenticated-arm-token", + "audiences": [ + ARM_BASE, ARM_BASE + "/", + "https://management.core.windows.net", "https://management.core.windows.net/", + ], + "currentExpiryRequired": True, "sameArmTokenPrincipalRequired": True, + "accountUserNameIsPrincipalProof": False, "graphReadProvesExtensionWriteIdentity": False, +} +_FORBIDDEN_AUTHORIZATIONS = ( + "writesAuthorized", "signInAuthorized", "interactiveSignInAuthorized", + "newCredentialsAuthorized", "permissionGrantsAuthorized", + "graphTokenAcquisitionAuthorized", "graphTokenReuseAuthorized", + "advancedHuntingCredentialReuseAuthorized", +) +_READ_FAILURES = (ValueError, RuntimeError, OSError, KeyError, TypeError) + + +def _arm_token_access(target: dict) -> dict: + return { + "scope": ARM_SCOPE, "tenantId": target["tenantId"], "subscriptionId": target["subscriptionId"], + "existingCachedIdentityOnly": True, "acquisitionAuthorized": True, "refreshAuthorized": True, + "interactiveSignInAuthorized": False, "newCredentialsAuthorized": False, + "permissionGrantsAuthorized": False, "graphTokenAcquisitionAuthorized": False, + "graphTokenReuseAuthorized": False, + "advancedHuntingCredentialReuseAuthorized": False, + } + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _guid(value: Any) -> bool: + try: + return isinstance(value, str) and str(uuid.UUID(value)) == value.lower() + except (ValueError, AttributeError): + return False + + +def _same(left: Any, right: Any) -> bool: + return isinstance(left, str) and isinstance(right, str) and left.casefold() == right.casefold() + + +def _copy(value: Any) -> Any: + return json.loads(json.dumps(value, allow_nan=False)) + + +def _sanitize(value: Any, *, depth: int = 0) -> Any: + if depth > 50: + return "[redacted: depth limit]" + if isinstance(value, dict): + return { + key: "[redacted]" if key.casefold().replace("_", "").replace("-", "") in _REDACTED_KEYS + else _sanitize(item, depth=depth + 1) + for key, item in value.items() + } + if isinstance(value, list): + return [_sanitize(item, depth=depth + 1) for item in value] + if isinstance(value, str): + value = re.sub(r"(?i)\bBearer\s+\S+", "Bearer [redacted]", value) + value = re.sub(r"\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b", "[redacted-token]", value) + return value + return value + + +def _resource_at(template: dict[str, Any], path: str) -> dict[str, Any]: + current: Any = template + for part in path.strip("/").split("/"): + current = current[int(part)] if isinstance(current, list) else current[part] + if not isinstance(current, dict): + raise ValueError("resource inventory path does not identify a resource") + return current + + +def _inspect(arm_plan: dict[str, Any]) -> dict[str, Any]: + """Re-run the actual fail-closed planner against current template bytes.""" + if not isinstance(arm_plan, dict) or not _sealed(arm_plan, "planSha256"): + raise ValueError("candidate ARM plan seal is missing or stale") + if arm_plan.get("status") != "ready" or arm_plan.get("workbookCapabilityVersion") != 1: + raise ValueError("preflight requires a versioned ready candidate ARM plan") + if not _HASH.fullmatch(str(arm_plan.get("candidateSha256", ""))): + raise ValueError("candidate binding is missing") + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}", str(arm_plan.get("runId", ""))): + raise ValueError("run identity is unsafe") + target = _target(arm_plan["target"]) + parameters, flags = arm_plan["parameters"], arm_plan["selectedFlags"] + if not isinstance(parameters, dict) or not isinstance(flags, dict): + raise ValueError("candidate parameters or content selection are malformed") + path = content_path(Path(arm_plan["solutionPath"]), arm_plan["artifactPaths"]["mainTemplate"]) + if path.stat().st_size > MAX_TEMPLATE_BYTES: + raise ValueError("candidate template exceeds bounded inspection size") + raw = path.read_bytes() + if hashlib.sha256(raw).hexdigest() != arm_plan.get("mainTemplateSha256"): + raise ValueError("current template does not match candidate") + template = json.loads(raw.decode("utf-8-sig")) + definitions = template.get("parameters", {}) + declared = {key for key in definitions if key.startswith("Deploy")} + if set(flags) != declared or declared - set(_FLAG_KINDS) or any( + type(value) is not bool or parameters.get(key) is not value + or not isinstance(definitions[key], dict) or definitions[key].get("type") != "bool" + for key, value in flags.items() + ): + raise ValueError("candidate selection cannot be reconstructed") + if parameters.get("workspace") != workspace_identity(target["workspaceResourceId"])["workspaceName"]: + raise ValueError("candidate workspace parameter differs from selected target") + inventory = _resource_inventory(template, parameters, target) + contracts = _expected_workbook_contracts(template, flags.get("DeployDefenderWorkbook") is True) + if inventory != arm_plan.get("resources") or contracts != arm_plan.get("expectedWorkbooks"): + raise ValueError("candidate inventory or workbook contract differs from current template") + if arm_plan.get("resourceCount") != len(inventory): + raise ValueError("candidate resource count is stale") + scope = arm_plan.get("scope", {}) + expected_types = sorted({item["type"] for item in inventory} | {"Microsoft.Resources/deployments"}) + if scope.get("lab") != target or scope.get("resourceTypes") != expected_types: + raise ValueError("candidate scope is not bound to current inventory") + if scope.get("contentTypes") != sorted({_FLAG_KINDS[key] for key, selected in flags.items() if selected}): + raise ValueError("candidate content scope differs from exact selections") + registrations, unsupported = [], [] + for row in inventory: + kind = row["type"].casefold() + if kind in _REGISTRATION_TYPES: + resource = _resource_at(template, row["path"]) + identities = {} + for field in ("contentId", "packageId", "contentKind"): + try: + identities[field] = _resolve(resource.get("properties", {}).get(field), template, parameters, target) + except ValueError: + identities[field] = None + try: + resolved_name = _resolve(row["name"], template, parameters, target) + except ValueError: + resolved_name = None + registrations.append({ + "family": _REGISTRATION_TYPES[kind], + "properties": resource.get("properties", {}), + "name": row["name"], "resolvedName": resolved_name, "identity": identities, + }) + elif kind not in {"microsoft.resources/deployments", "microsoft.security/workbooks"}: + unsupported.append(row["type"]) + return { + "inventory": inventory, "expectedWorkbooks": contracts, + "registrations": registrations, "unsupportedOwnershipTypes": sorted(set(unsupported)), + } + + +def _make_plan(arm_plan: dict[str, Any], inspection: dict[str, Any]) -> dict[str, Any]: + target = _target(arm_plan["target"]) + workspace = target["workspaceResourceId"] + rg = f"/subscriptions/{target['subscriptionId']}/resourceGroups/{target['resourceGroup']}" + operations = [{ + "id": "account", "kind": "azure-cli-account", "method": "LOCAL", + "subscriptionId": target["subscriptionId"], "timeoutSeconds": TIMEOUT_SECONDS, "retryCount": 0, + }] + + def get(identifier: str, kind: str, url: str) -> None: + operations.append({ + "id": identifier, "kind": kind, "method": "GET", "url": url, + "timeoutSeconds": TIMEOUT_SECONDS, "retryCount": 0, + }) + + def arm(identifier: str, resource: str, version: str, suffix: str = "") -> None: + get(identifier, "arm-get", f"{ARM_BASE}{quote(resource, safe='/')}?api-version={version}{suffix}") + + arm("workspace", workspace, WORKSPACE_API_VERSION) + arm("resource-group", rg, RESOURCE_GROUP_API_VERSION) + for identifier, scope in (("rg", rg), ("workspace", workspace)): + arm(f"permissions-{identifier}", scope + "/providers/Microsoft.Authorization/permissions", PERMISSIONS_API_VERSION) + arm(f"role-assignments-{identifier}", scope + "/providers/Microsoft.Authorization/roleAssignments", + PERMISSIONS_API_VERSION, "&$filter=atScope()") + arm(f"denies-{identifier}", scope + "/providers/Microsoft.Authorization/denyAssignments", + DENY_API_VERSION, "&$filter=atScope()") + for family in ("contentPackages", "contentTemplates"): + arm(f"registrations-{family}", workspace + "/providers/Microsoft.SecurityInsights/" + family, + CONTENT_API_VERSION) + for workbook in sorted(inspection["expectedWorkbooks"], key=lambda row: row["instanceId"]): + iid = workbook["instanceId"] + if not _guid(iid): + raise ValueError("selected workbook instance identity is malformed") + get("workbook-" + iid, "graph-get", f"{GRAPH_BASE}/beta/security/workbooks/{iid}") + if len(operations) > MAX_OPERATIONS: + raise ValueError("candidate exceeds bounded preflight operation count") + binding = {key: arm_plan[key] for key in ("runId", "candidateSha256", "mainTemplateSha256", "target")} + binding["armPlanSha256"] = arm_plan["planSha256"] + prefix = arm_plan["planSha256"] + token_access = _arm_token_access(target) + authorization = { + "readOnly": True, "tokenAcquisitionAuthorized": True, + **{key: False for key in _FORBIDDEN_AUTHORIZATIONS}, + "armTokenAccess": token_access, + } + read_scope = { + "purpose": "candidate-arm-read-preflight", "runId": arm_plan["runId"], "target": target, + "operations": operations, + "principalContract": {**_copy(_PRINCIPAL_CONTRACT), "armTokenAccess": _copy(token_access)}, + **authorization, + } + plan = { + "schemaVersion": 1, "purpose": "candidate-arm-read-preflight", + **binding, "armPlan": _copy(arm_plan), "operations": operations, + "inspectionSha256": _canonical_sha256(inspection), + **authorization, "paginationPolicy": "reject-next-link", + "readScope": read_scope, "preflightScopeSha256": _canonical_sha256(read_scope), + "maxOperations": MAX_OPERATIONS, "maxResponseBytes": MAX_RESPONSE_BYTES, + "maxListItems": MAX_LIST_ITEMS, + "artifacts": { + "plan": "arm-preflight-plan.json", "consent": "arm-preflight-consent.json", + "observations": f"arm-preflight-observations-{prefix}.json", + "result": f"arm-preflight-result-{prefix}.json", + }, + } + plan["preflightPlanSha256"] = _canonical_sha256(plan) + return plan + + +def plan_arm_preflight(arm_plan: dict) -> dict: + """Inspect local candidate bytes and seal a bounded exact metadata read list.""" + try: + arm_plan = _copy(arm_plan) + return _make_plan(arm_plan, _inspect(arm_plan)) + except (KeyError, TypeError, AttributeError, IndexError, OSError, RecursionError) as exc: + raise ValueError(f"candidate preflight plan cannot be reconstructed ({type(exc).__name__})") from exc + + +def _validate(plan: dict) -> None: + if not isinstance(plan, dict) or not _sealed(plan, "preflightPlanSha256"): + raise ValueError("preflight plan seal is missing or stale") + if plan != plan_arm_preflight(plan["armPlan"]): + raise ValueError("preflight plan does not reconstruct from current candidate") + + +def _binding(plan: dict) -> dict: + return {key: plan[key] for key in ( + "preflightPlanSha256", "preflightScopeSha256", "armPlanSha256", "candidateSha256", + "mainTemplateSha256", "runId", "target", + )} + + +def review_arm_preflight(plan: dict, actor: str, reference: str, approved: bool) -> dict: + _validate(plan) + if not isinstance(actor, str) or not actor.strip() or not isinstance(reference, str) or not reference.strip() or type(approved) is not bool: + raise ValueError("read review requires actor, reference and an explicit boolean decision") + consent = { + "schemaVersion": 1, "consentPolicyVersion": 1, **_binding(plan), + "readScope": _copy(plan["readScope"]), "reviewedPreflightPlanSha256": plan["preflightPlanSha256"], + "operations": _copy(plan["operations"]), "status": "approved" if approved else "declined", + "actor": actor.strip(), "reference": reference.strip(), "recordedAt": _now(), + "revoked": False, "readOnly": True, "tokenAcquisitionAuthorized": True, + **{key: False for key in _FORBIDDEN_AUTHORIZATIONS}, + "armTokenAccess": _copy(plan["armTokenAccess"]), + } + consent["consentSha256"] = _canonical_sha256(consent) + return consent + + +def revoke_arm_preflight(consent: dict, actor: str, reference: str) -> dict: + if not isinstance(consent, dict) or not _sealed(consent, "consentSha256"): + raise ValueError("read consent seal is missing or stale") + if not isinstance(actor, str) or not actor.strip() or not isinstance(reference, str) or not reference.strip(): + raise ValueError("revocation requires actor and reference") + result = _copy(consent) + result.pop("consentSha256") + result.update(status="revoked", revoked=True, revocationActor=actor.strip(), + revocationReference=reference.strip(), revokedAt=_now()) + result["consentSha256"] = _canonical_sha256(result) + return result + + +def _check_consent(plan: dict, consent: Any) -> dict: + reasons = [] + try: + _validate(plan) + if not isinstance(consent, dict) or not _sealed(consent, "consentSha256"): + raise ValueError("read consent is missing or stale") + if consent.get("status") != "approved" or consent.get("revoked") is not False: + raise ValueError("read consent is declined or revoked") + if any(consent.get(key) != plan[key] for key in ("preflightScopeSha256", "runId", "target")) or consent.get("readScope") != plan["readScope"]: + raise ValueError("read consent belongs to another exact metadata scope, principal contract, run or target") + if consent.get("operations") != plan["operations"]: + raise ValueError("read consent operation scope differs") + if consent.get("schemaVersion") != 1 or consent.get("consentPolicyVersion") != 1 or consent.get("readOnly") is not True or ( + consent.get("tokenAcquisitionAuthorized") is not True or consent.get("armTokenAccess") != plan["armTokenAccess"] + ) or any( + consent.get(key) is not False for key in _FORBIDDEN_AUTHORIZATIONS + ) or not consent.get("actor") or not consent.get("reference"): + raise ValueError("read-only consent boundary is invalid") + except (ValueError, KeyError, TypeError, AttributeError, OSError) as exc: + reasons.append(str(exc) if isinstance(exc, ValueError) else type(exc).__name__) + return {"status": "ready" if not reasons else "blocked", "reasons": reasons} + + +def _normalize_response(operation: dict, response: Any, elapsed: float) -> dict: + result = {"operation": _copy(operation), "operationSha256": _canonical_sha256(operation)} + if elapsed > operation["timeoutSeconds"]: + return {**result, "status": "unknown", "reason": "reader exceeded bounded timeout"} + try: + if not isinstance(response, dict) or type(response.get("statusCode")) is not int or not isinstance(response.get("body"), dict): + raise ValueError("reader returned malformed status/body") + encoded = json.dumps(response, allow_nan=False).encode() + if len(encoded) > MAX_RESPONSE_BYTES: + raise ValueError("response exceeds bounded evidence size") + sanitized = _sanitize(_copy({key: response[key] for key in ( + "statusCode", "body", "principalId", "tenantId", "provenance", + ) if key in response})) + if operation["kind"] != "azure-cli-account" and "provenance" not in sanitized: + sanitized["provenance"] = { + "requestUrl": operation["url"], "method": operation["method"], + "source": "consent-bound-reader-dispatch", + "principalId": sanitized.get("principalId"), "tenantId": sanitized.get("tenantId"), + } + if response["statusCode"] != 200: + return {**result, **sanitized, "status": "blocked" if response["statusCode"] in {401, 403} else "unknown", + "reason": f"metadata read HTTP {response['statusCode']}; not success or proven absence"} + if response["body"].get("nextLink") not in (None, "") or response["body"].get("@odata.nextLink") not in (None, ""): + raise ValueError("paginated response is incomplete; nextLink is not followed") + return {**result, **sanitized, "operation": _copy(operation), + "operationSha256": _canonical_sha256(operation), "status": "observed"} + except (ValueError, TypeError, RecursionError): + return {**result, "status": "unknown", "reason": "malformed, oversized or paginated metadata response"} + + +def execute_arm_preflight(plan: dict, *, read_consent: Callable[[], dict], reader: Callable[[dict], dict]) -> dict: + """Dispatch only exact reviewed reads; no default reader or credential creation.""" + _validate(plan) + current = copy.deepcopy(plan) + observations = {**_binding(current), "responses": [], "consentChecks": [], "readAttempted": False} + + def consent_check() -> dict: + try: + consent = read_consent() + check = {**_check_consent(current, consent), "consent": _copy(consent)} + _validate(plan) + if plan != current: + raise ValueError("preflight plan mutated during execution") + except _READ_FAILURES as exc: + check = {"status": "blocked", "reasons": [f"current read consent/plan unavailable ({type(exc).__name__})"]} + observations["consentChecks"].append(check) + return check + + for operation in current["operations"]: + if consent_check()["status"] != "ready": + break + observations["readAttempted"] = True + started = time.monotonic() + try: + response = reader(_copy(operation)) + normalized = _normalize_response(operation, response, time.monotonic() - started) + except _READ_FAILURES as exc: + normalized = {"operation": _copy(operation), "operationSha256": _canonical_sha256(operation), + "status": "unknown", "reason": f"metadata reader failed ({type(exc).__name__})"} + observations["responses"].append(normalized) + if consent_check()["status"] != "ready": + normalized.update(status="blocked", reason="read consent/plan changed after dispatch; evidence unusable") + break + if normalized["status"] != "observed": + break + result = assess_arm_preflight(current, observations) + return {**result, "observations": observations, "readAttempted": observations["readAttempted"], + "writeAttempted": False} + + +def _expiry(value: Any) -> float: + if isinstance(value, bool): + raise ValueError("invalid token expiry") + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, str): + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + raise ValueError("token expiry requires timezone") + return parsed.timestamp() + raise ValueError("invalid token expiry") + + +def _principal(account: dict) -> dict | None: + principal = account.get("principal") + if not isinstance(principal, dict) or not _guid(principal.get("objectId")) or not _guid(principal.get("tenantId")): + return None + if principal.get("source") != _PRINCIPAL_CONTRACT["source"] or principal.get("audience") not in _PRINCIPAL_CONTRACT["audiences"]: + return None + try: + if _expiry(principal.get("expiresOn")) <= time.time(): + return None + except (ValueError, OverflowError): + return None + return principal + + +def _trusted_response(row: Any, operation: dict, principal: dict | None) -> bool: + if not isinstance(row, dict) or row.get("operation") != operation or row.get("operationSha256") != _canonical_sha256(operation): + return False + normalized = _normalize_response(operation, row, 0) + if row.get("status") != "observed" or normalized.get("status") != "observed": + return False + body = row["body"] + if operation["kind"] == "azure-cli-account": + return _guid(body.get("id")) and _guid(body.get("tenantId")) and _principal(body) is not None + identifier = operation["id"] + if identifier == "workspace" and not ( + isinstance(body.get("id"), str) and isinstance(body.get("properties"), dict) + and _guid(body["properties"].get("customerId")) + ): + return False + if identifier == "resource-group" and not isinstance(body.get("id"), str): + return False + if identifier.startswith(("permissions-", "role-assignments-", "denies-", "registrations-")) and _collection(body) is None: + return False + if identifier.startswith("workbook-") and not (isinstance(body.get("id"), str) and isinstance(body.get("content"), dict)): + return False + proof = row.get("provenance") + source = "authenticated-arm-token" if operation["kind"] == "arm-get" else "authenticated-graph-token" + return bool(principal and isinstance(proof, dict) and proof.get("requestUrl") == operation["url"] + and proof.get("method") == "GET" and proof.get("source") in {source, "consent-bound-reader-dispatch"} + and _same(proof.get("principalId"), principal["objectId"]) + and _same(proof.get("tenantId"), principal["tenantId"]) + and _same(row.get("principalId"), principal["objectId"]) + and _same(row.get("tenantId"), principal["tenantId"])) + + +def _collection(body: dict | None) -> list[dict] | None: + if not isinstance(body, dict): + return None + rows = body.get("value") + if not isinstance(rows, list) or len(rows) > MAX_LIST_ITEMS or any(not isinstance(row, dict) for row in rows): + return None + return rows + + +def _unconditional(value: dict) -> bool: + return all( + value.get(key) is None or (isinstance(value.get(key), str) and value[key] == "") + for key in ("condition", "conditionVersion") + ) + + +def _permissions(rows: list[dict] | None, action: str) -> dict: + if rows is None: + return {"status": "unknown", "reason": "complete effective permissions are unavailable"} + projected = [] + for row in rows: + if not _unconditional(row): + return {"status": "unknown", "reason": "control-plane grant conditions cannot be evaluated"} + if any(not isinstance(row.get(key), list) or any(not isinstance(item, str) for item in row[key]) + for key in ("actions", "notActions")): + return {"status": "unknown", "reason": "malformed control-plane actions/notActions"} + projected.append({key: row[key] for key in ("actions", "notActions")}) + from .dcr_readiness import _action_permission + return _action_permission(projected, action) + + +def assess_arm_preflight(plan: dict, observations: dict) -> dict: + """Evaluate independently observed metadata, retaining explicit unknown blockers.""" + access = {key: False for key in _ACCESS_FLAGS} + access.update(identityId=None, tenantId=None, subscriptionId=None, workspaceResourceId=None) + checks: dict[str, dict] = {} + + def check(name: str, passed: bool, reason: str, **details: Any) -> None: + checks[name] = {"status": "verified" if passed else "blocked", "reasons": [] if passed else [reason], **details} + if name in access: + access[name] = passed + + try: + observations = _copy(observations) + except (ValueError, TypeError, RecursionError): + observations = {} + try: + _validate(plan) + inspection = _inspect(plan["armPlan"]) + check("disabledContentVerified", True, "") + except (ValueError, KeyError, TypeError, AttributeError, OSError, RecursionError) as exc: + check("disabledContentVerified", False, f"fresh candidate inspection failed ({type(exc).__name__})") + bindings = {key: plan.get(key) for key in ( + "target", "preflightPlanSha256", "preflightScopeSha256", "candidateSha256", + "armPlanSha256", "mainTemplateSha256", "runId", + )} if isinstance(plan, dict) else {"target": None, "preflightPlanSha256": None} + return {"schemaVersion": 1, **bindings, "status": "blocked", **access, "checks": checks, + "reasons": checks["disabledContentVerified"]["reasons"], "writeAttempted": False, + "providerWriteContractVerified": False} + binding_valid = isinstance(observations, dict) and all(observations.get(key) == value for key, value in _binding(plan).items()) + check("observationBinding", binding_valid, "observations are not bound to exact target/run/candidate/plan") + raw_rows = observations.get("responses") if binding_valid else None + if not isinstance(raw_rows, list) or len(raw_rows) > MAX_OPERATIONS: + raw_rows = [] + by_id = {} + for row in raw_rows: + identifier = row.get("operation", {}).get("id") if isinstance(row, dict) and isinstance(row.get("operation"), dict) else None + if identifier in by_id: + check("observationBinding", False, "duplicate operation observations") + by_id = {} + break + by_id[identifier] = row + account_row = by_id.get("account", {}) + account = account_row.get("body", {}) if _trusted_response(account_row, plan["operations"][0], None) else {} + principal = _principal(account) + check("principalVerified", principal is not None, "verified current authenticated ARM principal proof is unavailable; account.user.name is insufficient") + bodies = {} + for operation in plan["operations"]: + row = by_id.get(operation["id"]) + trusted = _trusted_response(row, operation, principal) + check("read:" + operation["id"], trusted, "missing, partial, failed or unbound authenticated metadata response") + if trusted: + bodies[operation["id"]] = row["body"] + principal_bound = principal is not None and any( + operation["kind"] == "arm-get" and operation["id"] in bodies for operation in plan["operations"] + ) + check("principalVerified", principal_bound, "authenticated ARM principal is not bound to any successful exact ARM observation") + if principal_bound: + access["identityId"], access["tenantId"] = principal["objectId"], principal["tenantId"] + complete = len(raw_rows) == len(plan["operations"]) and all( + checks["read:" + operation["id"]]["status"] == "verified" for operation in plan["operations"] + ) + check("allRequiredReads", complete, "every exact planned metadata read requires complete authenticated evidence") + consent_checks = observations.get("consentChecks") if binding_valid else None + consent_current = ( + isinstance(consent_checks, list) and len(consent_checks) == 2 * len(plan["operations"]) + and all(isinstance(row, dict) and row.get("status") == "ready" + and _check_consent(plan, row.get("consent"))["status"] == "ready" for row in consent_checks) + ) + check("readConsentVerified", consent_current, "current consent checks before and after every required read are missing or blocked") + target = plan["target"] + check("tenantMatches", bool(principal_bound and _same(principal["tenantId"], target["tenantId"]) + and _same(account.get("tenantId"), target["tenantId"])), "observed account/token tenant mismatches or is unavailable") + subscription = account.get("id") + if isinstance(subscription, str): + access["subscriptionId"] = subscription + check("subscriptionMatches", principal_bound and _same(subscription, target["subscriptionId"]), "observed ARM account subscription mismatches or is unavailable") + workspace = bodies.get("workspace", {}) + observed_workspace = workspace.get("id") + if isinstance(observed_workspace, str): + access["workspaceResourceId"] = observed_workspace + workspace_properties = workspace.get("properties") + check("workspaceMatches", _same(observed_workspace, target["workspaceResourceId"]) + and isinstance(workspace_properties, dict) and _same(workspace_properties.get("customerId"), target["workspaceCustomerId"]), + "fresh selected workspace resource/customer identity mismatches or is unavailable") + rg_id = f"/subscriptions/{target['subscriptionId']}/resourceGroups/{target['resourceGroup']}" + group = bodies.get("resource-group", {}) + tags = group.get("tags") + environment = [(key, value) for key, value in tags.items() + if key.casefold() in {"environment", "env"} and isinstance(value, str)] if isinstance(tags, dict) else [] + approved_environments = {"lab", "test", "testing", "dev", "development", "sandbox", "non-production", "nonproduction"} + lab = bool(environment and all(value.strip().casefold() in approved_environments for _, value in environment)) + check("nonProduction", lab and _same(group.get("id"), rg_id) and access["workspaceMatches"] + and access["tenantMatches"] and access["subscriptionMatches"] and consent_current, + "exact approved RG/workspace lacks consistent recognized existing non-production environment tags", + scope="selected-resource-group-and-workspace-only", tenantNonProductionVerified=False, + environmentTags=environment) + action_checks = {} + required = {"Microsoft.Resources/deployments/*": "rg"} + for resource in inspection["inventory"]: + family = _REGISTRATION_TYPES.get(resource["type"].casefold()) + if family: + required[f"Microsoft.SecurityInsights/{family}/write"] = "workspace" + for action, scope in required.items(): + action_checks[action] = _permissions(_collection(bodies.get("permissions-" + scope)), action) + deny_checks = {} + assignment_checks = {} + for scope in ("rg", "workspace"): + denies = _collection(bodies.get("denies-" + scope)) + deny_checks[scope] = { + "status": "verified" if denies == [] else "unknown", + "reason": "" if denies == [] else "deny applicability/conditions cannot be safely resolved from unavailable or nonempty assignments", + "ancestorCoverage": "atScope(): selected scope and above", + "applicabilityEvaluated": False, "policy": "require-complete-empty-ancestor-deny-list", + } + assignments = _collection(bodies.get("role-assignments-" + scope)) + selected_scope = rg_id if scope == "rg" else target["workspaceResourceId"] + unconditioned = assignments is not None and all( + isinstance(row.get("properties"), dict) + and isinstance(row["properties"].get("scope"), str) + and _guid(row["properties"].get("principalId")) + and isinstance(row["properties"].get("roleDefinitionId"), str) + and (selected_scope.casefold() == row["properties"]["scope"].casefold() + or selected_scope.casefold().startswith(row["properties"]["scope"].rstrip("/").casefold() + "/")) + and _unconditional(row["properties"]) + for row in assignments + ) + assignment_checks[scope] = { + "status": "verified" if unconditioned else "unknown", + "reason": "" if unconditioned else "role assignment conditions/applicability are unresolved; effective permissions alone cannot prove access", + } + arm_allowed = (all(row["status"] == "ready" for row in action_checks.values()) + and all(row["status"] == "verified" for row in deny_checks.values()) + and all(row["status"] == "verified" for row in assignment_checks.values()) + and not inspection["unsupportedOwnershipTypes"] + and access["tenantMatches"] and access["subscriptionMatches"] and access["workspaceMatches"]) + check("armDeploymentAllowed", arm_allowed, "effective control-plane allow/exclusion or ancestor deny evidence is insufficient", + actions=action_checks, denies=deny_checks, roleAssignments=assignment_checks, + unsupportedResourceWrites=inspection["unsupportedOwnershipTypes"]) + managed_updates, ownership_reasons = [], [] + for workbook in inspection["expectedWorkbooks"]: + installed = bodies.get("workbook-" + workbook["instanceId"]) + if not installed: + ownership_reasons.append(f"{workbook['instanceId']}: no credible existing workbook evidence; Graph 404 absence contract is unverified") + continue + content = installed.get("content") + provenance = content.get("contentProvenance") if isinstance(content, dict) else None + expected = workbook["installation"]["contentProvenance"] + if installed.get("id") != workbook["instanceId"] or not isinstance(provenance, dict) or set(provenance) != set(expected) or any( + provenance.get(key) != expected[key] for key in ("packageId", "contentId") + ) or any(not isinstance(provenance.get(key), str) or not provenance[key] for key in ("packageVersion", "contentVersion")): + ownership_reasons.append(f"{workbook['instanceId']}: wrong instance/package/canonical-content identity or unintended copy") + continue + managed_updates.append({ + "instanceId": workbook["instanceId"], "contentId": workbook["contentId"], + "contentChanged": _canonical_sha256(content) != workbook["installation"]["contentSha256"], + "observedContentSha256": _canonical_sha256(content), + "observedProvenance": provenance, "policy": "managed-same-id-upsert", + }) + for expected in inspection["registrations"]: + rows = _collection(bodies.get("registrations-" + expected["family"])) + properties = expected["properties"] + identities = expected["identity"] + identity_key = "contentId" if isinstance(identities.get("contentId"), str) else "packageId" + expected_identity = identities.get(identity_key) + if not isinstance(expected_identity, str) or expected_identity.startswith("[") or rows is None: + ownership_reasons.append(f"{expected['family']}: incomplete list or unresolved canonical registration identity") + continue + prefix = target["workspaceResourceId"] + "/providers/Microsoft.SecurityInsights/" + expected["family"] + "/" + if any(not isinstance(row.get("properties"), dict) or not isinstance(row.get("id"), str) + or not row["id"].casefold().startswith(prefix.casefold()) + or not isinstance(row["properties"].get("contentId"), str) for row in rows): + ownership_reasons.append(f"{expected['family']}: malformed or out-of-scope complete-list entry") + continue + resolved_name = expected["resolvedName"] + name_parts = resolved_name.split("/", 2) if isinstance(resolved_name, str) else [] + resolved_id = prefix + name_parts[2] if len(name_parts) == 3 and name_parts[1] == "Microsoft.SecurityInsights" else None + exact_id_matches = [row for row in rows if resolved_id is not None and _same(row["id"], resolved_id)] + if len(exact_id_matches) > 1 or any(row["properties"].get(identity_key) != expected_identity for row in exact_id_matches): + ownership_reasons.append(f"{expected['family']}: exact selected registration ID has ambiguous/foreign canonical identity") + continue + matches = [row for row in rows if isinstance(row.get("properties"), dict) + and row["properties"].get(identity_key) == expected_identity] + if len(matches) > 1: + ownership_reasons.append(f"{expected['family']}: ambiguous canonical registration matches") + continue + if not matches: + if resolved_id is None: + ownership_reasons.append(f"{expected['family']}: unresolved resource name has no complete canonical identity mapping; absence cannot be proved") + continue # Only a complete list and resolved exact ID prove registration absence. + actual = matches[0] + props = actual["properties"] + package = identities.get("packageId") or identities.get("contentId") + actual_package = props.get("packageId", props.get("contentId")) + if (actual_package != package or props.get("contentKind") != identities.get("contentKind") + or (resolved_id is not None and not _same(actual["id"], resolved_id))): + ownership_reasons.append(f"{expected['family']}: foreign/out-of-scope registration collision") + continue + managed_updates.append({ + "registrationId": actual["id"], "contentChanged": props != properties, + "dependenciesPresent": isinstance(props.get("dependencies"), dict), + "storedTemplatePresent": isinstance(props.get("mainTemplate"), dict), + "policy": "managed-same-id-upsert", + }) + if inspection["unsupportedOwnershipTypes"]: + ownership_reasons.append("ownership unsupported for non-workbook resource families: " + ", ".join(inspection["unsupportedOwnershipTypes"])) + check("ownershipVerified", not ownership_reasons and complete and access["tenantMatches"] + and access["subscriptionMatches"] and access["workspaceMatches"], + "; ".join(ownership_reasons) or "incomplete exact-target ownership metadata", + managedUpdates=managed_updates) + imported = any(resource["import"] for resource in inspection["inventory"]) + if imported: + check("extensionResourceWriteAllowed", False, + "VERIFIED documented MicrosoftSecurity ARM authentication/write contract is unavailable; GET, portal roles and known IDs do not establish imported-resource writes") + else: + checks["extensionResourceWriteAllowed"] = { + "status": "not-required", "reasons": [], "required": False, + "writeAccessVerified": False, + } + reasons = [reason for row in checks.values() for reason in row["reasons"]] + ready = all(access[key] for key in _ACCESS_FLAGS if key != "extensionResourceWriteAllowed") and ( + not imported or access["extensionResourceWriteAllowed"] + ) and checks["readConsentVerified"]["status"] == "verified" and checks["observationBinding"]["status"] == "verified" + return { + "schemaVersion": 1, **_binding(plan), "status": "ready" if ready else "blocked", + **access, "checks": checks, "reasons": reasons, "writeAttempted": False, + "observationSha256": _canonical_sha256(_sanitize(observations)), + "providerWriteContractVerified": False, + } diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_preflight_workflow.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_preflight_workflow.py new file mode 100644 index 00000000000..a2b8d7f397b --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_preflight_workflow.py @@ -0,0 +1,304 @@ +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +import time +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any, Callable +from urllib.parse import urlsplit +from uuid import UUID + +from . import workbook_workflow +from .artifacts import report_directory + + +PLAN_FILE = "arm-preflight-plan.json" +CONSENT_FILE = "arm-preflight-consent.json" +RESULT_FILE = "arm-preflight-result.json" + + +class _RejectRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, request, response, code, message, headers, new_url): + return None + + +def _open_read(request: urllib.request.Request, *, timeout: int): + return urllib.request.build_opener(_RejectRedirect()).open(request, timeout=timeout) + + +def execution_profile(state_dir: str | Path | None = None) -> str: + from .graph_client_config import configured_azure_cli_profile + from .onboarding import _state_dir + + configured = configured_azure_cli_profile(_state_dir(state_dir)) + return str(Path(configured or os.environ.get("AZURE_CONFIG_DIR") or Path.home() / ".azure").resolve()) + + +def current_plan(solution: str | Path) -> dict[str, Any]: + from .workbook_preflight import plan_arm_preflight + + recorded = workbook_workflow._read_run_json(solution, "arm-validation-plan.json") + fresh = workbook_workflow.plan_candidate( + solution, target=workbook_workflow._plan_target(recorded), + selected_flags=recorded["selectedFlags"], + ) + if fresh["planSha256"] != recorded["planSha256"]: + raise ValueError("ARM preflight requires a current candidate deployment plan") + return plan_arm_preflight(fresh) + + +def create_plan(solution: str | Path) -> dict[str, Any]: + from .workflow import _state_writer_lock + + with _state_writer_lock(solution): + plan = current_plan(solution) + path = workbook_workflow._write_run_json(solution, PLAN_FILE, plan) + return {**plan, "planPath": path.relative_to(Path(solution).resolve()).as_posix()} + + +def _reviewed_plan(solution: str | Path, expected_sha256: str) -> dict[str, Any]: + plan = workbook_workflow._read_run_json(solution, PLAN_FILE) + fresh = current_plan(solution) + if plan != fresh or fresh["preflightPlanSha256"] != expected_sha256: + raise ValueError("ARM preflight requires the exact current displayed read plan") + return fresh + + +def review( + solution: str | Path, *, expected_sha256: str, actor: str, + reference: str, approved: bool, +) -> dict[str, Any]: + from .workbook_preflight import review_arm_preflight + from .workflow import _state_writer_lock + + with _state_writer_lock(solution): + plan = _reviewed_plan(solution, expected_sha256) + consent = review_arm_preflight(plan, actor=actor, reference=reference, approved=approved) + workbook_workflow._write_run_json(solution, CONSENT_FILE, consent) + return consent + + +def revoke(solution: str | Path, *, actor: str, reference: str) -> dict[str, Any]: + from .workbook_preflight import revoke_arm_preflight + from .workflow import _state_writer_lock + + with _state_writer_lock(solution): + consent = revoke_arm_preflight( + workbook_workflow._read_run_json(solution, CONSENT_FILE), + actor=actor, reference=reference, + ) + workbook_workflow._write_run_json(solution, CONSENT_FILE, consent) + return consent + + +def default_reader( + target: dict[str, Any], *, state_dir: str | Path | None = None, + operations: list[dict[str, Any]], +) -> Callable[[dict[str, Any]], dict[str, Any]]: + """Use existing cached ARM authentication; never infer extension credentials.""" + from .arm_acceptance_support import _run_cli + from .graph_client_config import ( + azure_cli_credential, azure_cli_profile, + ) + from .graph_readiness import _token_claims + from .onboarding import _state_dir + + state_root = _state_dir(state_dir) + profile = execution_profile(state_dir) + token = None + principal: dict[str, Any] | None = None + credential = None + allowed_operations = { + json.dumps(operation, sort_keys=True, separators=(",", ":")) + for operation in operations + } + + def authenticate() -> tuple[str, dict[str, Any]]: + nonlocal token, principal, credential + from azure.core.exceptions import AzureError + + if execution_profile(state_dir) != profile: + raise ValueError("ARM authentication profile changed during preflight") + if token is None or token.expires_on <= time.time() + 30: + if credential is None: + credential = azure_cli_credential( + tenant_id=target["tenantId"], subscription=target["subscriptionId"], + state_root=state_root, + ) + if execution_profile(state_dir) != profile: + raise ValueError("ARM authentication profile changed during preflight") + try: + with azure_cli_profile(profile): + token = credential.get_token("https://management.azure.com/.default") + except AzureError as exc: + raise ValueError( + "existing ARM authentication is unavailable; no sign-in or permission change was attempted" + ) from exc + claims = _token_claims(token.token) + if ( + not isinstance(claims, dict) + or str(claims.get("tid", "")).casefold() != target["tenantId"].casefold() + or claims.get("aud") not in { + "https://management.azure.com/", "https://management.azure.com", + "https://management.core.windows.net/", "https://management.core.windows.net", + } + or not isinstance(claims.get("oid"), str) or not claims["oid"].strip() + or token.expires_on <= time.time() + 30 + ): + raise ValueError("cached ARM credential has no supported current principal/tenant binding") + try: + UUID(claims["oid"]) + except ValueError as exc: + raise ValueError("cached ARM credential principal is not an object ID") from exc + observed = { + "objectId": claims["oid"], "tenantId": claims["tid"], + "audience": claims["aud"], "expiresOn": token.expires_on, + "source": "authenticated-arm-token", + } + if principal is not None and any( + observed[key] != principal[key] for key in ("objectId", "tenantId", "audience") + ): + raise ValueError("ARM authentication principal changed during preflight") + principal = observed + return token.token, observed + + def read(operation: dict[str, Any]) -> dict[str, Any]: + if json.dumps(operation, sort_keys=True, separators=(",", ":")) not in allowed_operations: + raise ValueError("ARM preflight operation is outside the frozen approved read plan") + kind = operation.get("kind") + if kind == "graph-get": + return { + "statusCode": 0, "body": {"error": { + "code": "MicrosoftSecurityIdentityUnverified", + "message": "ARM extension read/write identity is unverified; unrelated cached Graph credentials were not used.", + }}, + } + if kind == "azure-cli-account" and operation.get("method") == "LOCAL": + if operation.get("subscriptionId") != target["subscriptionId"]: + raise ValueError("ARM preflight account operation targets another subscription") + def bounded_account_runner(*args, **kwargs): + return subprocess.run( + *args, **{**kwargs, "timeout": operation.get("timeoutSeconds", 15)}, + ) + + with azure_cli_profile(profile): + code, account, _ = _run_cli([ + "account", "show", "--subscription", target["subscriptionId"], + "--output", "json", "--only-show-errors", + ], runner=bounded_account_runner) + if code or not isinstance(account, dict): + return {"statusCode": 0, "body": {"error": {"code": "CachedAccountUnavailable"}}} + if ( + str(account.get("tenantId", "")).casefold() != target["tenantId"].casefold() + or str(account.get("id", "")).casefold() != target["subscriptionId"].casefold() + ): + return {"statusCode": 200, "body": account} + _, identity = authenticate() + return {"statusCode": 200, "body": {**account, "principal": identity}} + if kind != "arm-get" or operation.get("method") != "GET": + raise ValueError("preflight reader permits only planned local account and ARM GET operations") + url = operation.get("url") + parsed = urlsplit(url) if isinstance(url, str) else None + if ( + parsed is None or parsed.scheme != "https" or parsed.netloc != "management.azure.com" + or parsed.username or parsed.password or parsed.fragment + or not parsed.path.casefold().startswith( + f"/subscriptions/{target['subscriptionId']}/".casefold() + ) + ): + raise ValueError("ARM preflight URL is outside the selected subscription") + bearer, identity = authenticate() + request = urllib.request.Request( + url, headers={"Authorization": f"Bearer {bearer}", "Accept": "application/json"}, + method="GET", + ) + from .workbook_preflight import MAX_RESPONSE_BYTES + + try: + with _open_read(request, timeout=operation.get("timeoutSeconds", 15)) as response: + status = int(response.status) + raw = response.read(MAX_RESPONSE_BYTES + 1) + if len(raw) > MAX_RESPONSE_BYTES: + raise ValueError("ARM preflight response exceeds the bounded evidence size") + body = json.loads(raw) + except urllib.error.HTTPError as exc: + return { + "statusCode": int(exc.code), "body": {"error": {"code": "ArmReadRejected"}}, + "principalId": identity["objectId"], "tenantId": identity["tenantId"], + } + except (urllib.error.URLError, OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("ARM preflight read failed or returned malformed evidence") from exc + return { + "statusCode": status, "body": body, + "principalId": identity["objectId"], "tenantId": identity["tenantId"], + } + + return read + + +def execute( + solution: str | Path, *, expected_sha256: str, + state_dir: str | Path | None = None, + reader: Callable[[dict[str, Any]], dict[str, Any]] | None = None, +) -> dict[str, Any]: + from .workbook_preflight import execute_arm_preflight + from .workflow import _run_writer_lock + + root = Path(solution).resolve() + with _run_writer_lock(report_directory(root) / "arm-preflight-executor"): + plan = _reviewed_plan(root, expected_sha256) + + def consent() -> dict[str, Any]: + _reviewed_plan(root, expected_sha256) + return workbook_workflow._read_run_json(root, CONSENT_FILE) + + prior = report_directory(root) / RESULT_FILE + if prior.is_file(): + archive = f"arm-preflight-result.{hashlib.sha256(prior.read_bytes()).hexdigest()}.json" + workbook_workflow._write_run_json(root, archive, json.loads(prior.read_bytes())) + workbook_workflow._write_run_json(root, RESULT_FILE, { + "status": "running", "preflightPlanSha256": expected_sha256, + "writeAttempted": False, + }) + try: + result = execute_arm_preflight( + plan, read_consent=consent, + reader=reader if reader is not None else default_reader( + plan["target"], state_dir=state_dir, operations=plan["operations"], + ), + ) + except (ValueError, RuntimeError, OSError, KeyError, TypeError) as exc: + workbook_workflow._write_run_json(root, RESULT_FILE, { + "status": "failed", "preflightPlanSha256": expected_sha256, + "target": plan["target"], "writeAttempted": False, + "reasons": [f"ARM metadata preflight failed ({type(exc).__name__})"], + }) + raise + path = workbook_workflow._write_run_json(root, RESULT_FILE, result) + return {**result, "reportPath": path.relative_to(root).as_posix()} + + +def production_identity_check( + solution: str | Path, target: dict[str, Any], *, + profile: str | None = None, +) -> dict[str, Any]: + from .graph_client_config import azure_cli_profile + + if profile is not None and execution_profile() != profile: + raise ValueError("ARM execution authentication profile changed") + if profile is not None: + with azure_cli_profile(profile): + result = execute( + solution, expected_sha256=current_plan(solution)["preflightPlanSha256"], + ) + else: + result = execute( + solution, expected_sha256=current_plan(solution)["preflightPlanSha256"], + ) + if result.get("target") != target: + raise ValueError("ARM execution preflight target changed") + return result diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_qualification.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_qualification.py new file mode 100644 index 00000000000..5983e19c40b --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_qualification.py @@ -0,0 +1,623 @@ +from __future__ import annotations + +import hashlib +import json +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable +from urllib.parse import parse_qsl, urlsplit + +from .artifacts import report_directory, write_json_artifact +from .content_paths import content_path +from .workbook_workflow import enabled, selected_workbook_references, verify_inventory + + +def _sha(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _seal(value: dict[str, Any]) -> str: + return _sha(json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode()) + + +def _context(solution: str | Path) -> tuple[Path, dict[str, Any], dict[str, str]]: + from .workflow import _load + + root = Path(solution).expanduser().resolve() + _, state = _load(root) + context = state["context"] + if not enabled(state) or context.get("workflowProfile") != "qualification": + raise ValueError("workbook query/UI qualification requires an enabled Qualification run") + if context.get("targetContextLocked") is not True: + raise ValueError("workbook qualification requires the locked Qualification target") + target = { + key: context[key] + for key in ("tenantId", "subscriptionId", "workspaceResourceId", "workspaceCustomerId") + } + if any(not isinstance(value, str) or not value for value in target.values()): + raise ValueError("workbook qualification target is incomplete") + verify_inventory(root, state) + return root, state, target + + +def plan_workbook_queries( + solution: str | Path, + *, + workbook: str, + seed: int = 42, + clock: str, + scale: int = 1, + recipe: str = "syslog-operational-v1", +) -> dict[str, Any]: + from ._workbook_core.workbook_fixtures import harness_files + from ._workbook_core.workbook_mock import build, configuration + from .workbooks import validate_workbooks + + root, state, target = _context(solution) + selected = selected_workbook_references(root) + path = content_path(root, workbook) + if path not in selected: + raise ValueError("query qualification requires an explicitly selected Defender workbook") + validation = validate_workbooks(root, selected_only=True) + if validation["invalid"] or validation["valid"] != len(selected): + raise ValueError("selected workbooks have unresolved structural or review blockers") + converted = json.loads(path.read_text(encoding="utf-8-sig")) + source = content_path(root, converted["contentProvenance"]["source"]["path"]) + if _sha(source.read_bytes()) != converted["contentProvenance"]["source"]["sha256"]: + raise ValueError("workbook qualification source provenance is stale") + config = configuration( + seed, clock, scale, recipe, "run-" + _sha(state["runId"].encode())[:16] + ) + bundle = build(source.read_bytes(), config) + manifest = json.loads(bundle["manifest.json"]) + harnesses = harness_files( + manifest, json.loads(source.read_text(encoding="utf-8-sig")), converted + ) + entries = [] + for case in manifest["cases"]: + for side, suffix in (("sentinel", "sentinel"), ("advanced-hunting", "ah")): + name = f"{case['id']}.{suffix}.kql" + query = harnesses[name] + if query.encode() != bundle["harnesses/" + name]: + raise ValueError("authored workbook queries differ from the reviewed fixture oracle") + entries.append({ + "caseId": case["id"], + "side": side, + "sourcePath": case["sourcePath"], + "outputPath": case["outputPath"], + "originalQuerySha256": case["querySha256"], + "query": query, + "querySha256": _sha(query.encode()), + "expected": case["expected"], + }) + diagnostic = json.loads(bundle["diagnostic.json"])["expected"] + for side, suffix in (("sentinel", "sentinel"), ("advanced-hunting", "ah")): + query = bundle[f"harnesses/diagnostic-hourly.{suffix}.kql"].decode() + entries.append({ + "caseId": "diagnostic-hourly", + "side": side, + "sourcePath": None, + "outputPath": None, + "originalQuerySha256": None, + "query": query, + "querySha256": _sha(query.encode()), + "expected": diagnostic, + }) + plan = { + "schemaVersion": "1.0.0", + "operation": "workbook-query-qualification", + "runId": state["runId"], + "target": target, + "workbook": path.relative_to(root).as_posix(), + "contentId": converted["xdrMetadata"]["contentId"], + "sourceSha256": _sha(source.read_bytes()), + "targetSha256": _sha(path.read_bytes()), + "fixtureConfig": config, + "fixtureProvenance": manifest["provenance"], + "queries": entries, + "ingestion": "not-ingestible", + "uiStatus": "not-run", + } + plan["planSha256"] = _seal(plan) + return plan + + +def _current_plan(solution: str | Path, plan: dict[str, Any]) -> dict[str, Any]: + config = plan["fixtureConfig"] + current = plan_workbook_queries( + solution, workbook=plan["workbook"], seed=config["seed"], + clock=config["clock"], scale=config["scale"], recipe=config["recipe"], + ) + if current != plan: + raise ValueError("workbook query plan is stale or has been edited") + return current + + +def review_workbook_query_consent( + plan: dict[str, Any], *, actor: str, reference: str, approved: bool, +) -> dict[str, Any]: + if not actor.strip() or not reference.strip() or not isinstance(approved, bool): + raise ValueError("workbook query consent requires an actor, reference, and affirmative decision") + if plan.get("operation") not in {"workbook-query-qualification", "workbook-ui-qualification"}: + raise ValueError("workbook consent cannot authorize other operations") + return { + "schemaVersion": "1.0.0", + "operation": plan["operation"], + "runId": plan["runId"], + "target": plan["target"], + "planSha256": plan["planSha256"], + "approved": approved, + "withdrawn": False, + "actor": actor, + "reference": reference, + } + + +def _check_consent(plan: dict[str, Any], consent: dict[str, Any]) -> None: + if ( + consent.get("schemaVersion") != "1.0.0" + or consent.get("approved") is not True + or consent.get("withdrawn") is not False + or not str(consent.get("actor") or "").strip() + or not str(consent.get("reference") or "").strip() + or any(consent.get(key) != plan[key] for key in ( + "operation", "runId", "target", "planSha256", + )) + ): + raise ValueError("current exact workbook harness query consent is missing, stale, or withdrawn") + + +def _current_inputs(solution: str | Path, plan: dict[str, Any]) -> None: + root, state, target = _context(solution) + path = content_path(root, plan["workbook"]) + if ( + state["runId"] != plan["runId"] or target != plan["target"] + or path not in selected_workbook_references(root) + or _sha(path.read_bytes()) != plan["targetSha256"] + ): + raise ValueError("workbook qualification inputs or target changed during execution") + source = json.loads(path.read_text(encoding="utf-8-sig"))["contentProvenance"]["source"] + if _sha(content_path(root, source["path"]).read_bytes()) != plan["sourceSha256"]: + raise ValueError("workbook qualification source changed during execution") + + +def execute_workbook_queries( + solution: str | Path, + plan: dict[str, Any], + *, + read_consent: Callable[[], dict[str, Any]], + providers: dict[str, Callable[[str, dict[str, str]], dict[str, Any]]], + family_timeout_seconds: int = 1800, +) -> dict[str, Any]: + from .workflow import _run_writer_lock + + if isinstance(family_timeout_seconds, bool) or not isinstance(family_timeout_seconds, int) or not 1 <= family_timeout_seconds <= 21600: + raise ValueError("workbook query family timeout must be an integer between 1 and 21600 seconds") + with _run_writer_lock(report_directory(solution) / "workbook-qualification-executor"): + return _execute_workbook_queries( + solution, plan, read_consent=read_consent, providers=providers, + family_timeout_seconds=family_timeout_seconds, + ) + + +def _execute_workbook_queries( + solution: str | Path, plan: dict[str, Any], *, + read_consent: Callable[[], dict[str, Any]], + providers: dict[str, Callable[[str, dict[str, str]], dict[str, Any]]], + family_timeout_seconds: int, +) -> dict[str, Any]: + from ._workbook_core.workbook_fixtures import assert_result + + _current_plan(solution, plan) + _check_consent(plan, read_consent()) + if set(providers) != {"sentinel", "advanced-hunting"}: + raise ValueError("qualification requires independent Sentinel and Advanced Hunting providers") + if providers["sentinel"] is providers["advanced-hunting"]: + raise ValueError("the two qualification sides must use independent providers") + root = Path(solution).expanduser().resolve() + path = report_directory(root) / f"workbook-queries.{plan['contentId']}.json" + running = { + "schemaVersion": "1.0.0", "status": "running", "runId": plan["runId"], + "target": plan["target"], "planSha256": plan["planSha256"], "plan": plan, + "results": [], "uiStatus": "not-run", "ingestion": "not-run", + } + if path.is_file(): + prior = json.loads(path.read_text(encoding="utf-8-sig")) + archive = path.with_name(f"{path.stem}.{_sha(path.read_bytes())}.json") + write_json_artifact(root, archive, prior) + write_json_artifact(root, path, running) + deadline = time.monotonic() + family_timeout_seconds + captured = [] + for entry in plan["queries"]: + if time.monotonic() >= deadline: + write_json_artifact(root, path, { + **running, "status": "failed", "results": captured, + "error": "query-family-timeout", + }) + raise ValueError("bounded workbook query family timeout expired") + _current_inputs(solution, plan) + _check_consent(plan, read_consent()) + result = providers[entry["side"]](entry["query"], plan["target"]) + if time.monotonic() >= deadline: + write_json_artifact(root, path, { + **running, "status": "failed", "results": captured, + "error": "query-family-timeout", + }) + raise ValueError("bounded workbook query family timeout expired") + if ( + not isinstance(result, dict) + or result.get("status") != "passed" + or result.get("complete") is not True + or result.get("provider") != entry["side"] + or result.get("target") != plan["target"] + or result.get("querySha256") != entry["querySha256"] + ): + write_json_artifact(root, path, { + **running, "status": "failed", "results": captured, + "failedCase": {"caseId": entry["caseId"], "side": entry["side"]}, + "providerFailure": result, + }) + raise ValueError(f"workbook query execution failed or is incomplete: {entry['side']}/{entry['caseId']}") + try: + assert_result(entry["expected"], result["result"]) + except (AssertionError, KeyError, TypeError, ValueError) as exc: + write_json_artifact(root, path, { + **running, "status": "failed", "results": captured, + "failedCase": {"caseId": entry["caseId"], "side": entry["side"]}, + "error": "exact-result-mismatch", + }) + raise ValueError( + f"exact workbook result mismatch: {entry['side']}/{entry['caseId']}" + ) from exc + captured.append({ + "caseId": entry["caseId"], + "side": entry["side"], + "querySha256": entry["querySha256"], + "status": "passed", + "provider": result["provider"], + "result": result["result"], + }) + _current_plan(solution, plan) + final_consent = read_consent() + _check_consent(plan, final_consent) + receipt = { + "schemaVersion": "1.0.0", + "status": "passed", + "operation": plan["operation"], + "runId": plan["runId"], + "target": plan["target"], + "planSha256": plan["planSha256"], + "plan": plan, + "approval": final_consent, + "results": captured, + "completedAt": datetime.now(timezone.utc).isoformat(), + "ingestion": "not-run", + "uiStatus": "not-run", + } + write_json_artifact(root, path, receipt) + return {"status": "passed", "cases": len(captured), "reportPath": str(path), "uiStatus": "not-run"} + + +def default_query_providers( + *, state_dir: str | Path | None = None, +) -> dict[str, Callable[[str, dict[str, str]], dict[str, Any]]]: + def sentinel(query: str, target: dict[str, str]) -> dict[str, Any]: + from .logs_query import ( + azure_cli_credential, execute_workspace_query, logs_query_client, + verify_azure_cli_target, + ) + + verify_azure_cli_target(target) + client = logs_query_client(azure_cli_credential(target)) + try: + response = execute_workspace_query(client, target["workspaceCustomerId"], query) + finally: + client.close() + tables = response.get("tables") or [] + complete = response.get("status") == "passed" and response.get("partial") is False and len(tables) == 1 + return { + "status": "passed" if complete else "failed", + "complete": complete, "provider": "sentinel", "target": target, + "querySha256": _sha(query.encode()), + "result": _normalize_result(tables[0]["columns"], tables[0]["rows"]) if complete else None, + "error": response.get("error"), + } + + def advanced_hunting(query: str, target: dict[str, str]) -> dict[str, Any]: + from .onboarding import _state_dir + from .runtime import _advanced_hunting_token, run_advanced_hunting_query + + token = _advanced_hunting_token(_state_dir(state_dir), tenant_id=target["tenantId"]) + response = run_advanced_hunting_query( + query, _token=token, capture_full_results=True, + ) + complete = response.get("ok") is True and isinstance(response.get("results"), list) + return { + "status": "passed" if complete else "failed", + "complete": complete, "provider": "advanced-hunting", "target": target, + "querySha256": _sha(query.encode()), + "result": _normalize_result(response["schema"], response["results"]) if complete else None, + "error": response.get("error"), + } + + return {"sentinel": sentinel, "advanced-hunting": advanced_hunting} + + +def _normalize_result(columns: list[dict[str, str]], rows: list[Any]) -> dict[str, Any]: + aliases = { + "string": "string", "datetime": "datetime", "int64": "long", + "long": "long", "int32": "int", "int": "int", "double": "real", + "real": "real", "boolean": "bool", "bool": "bool", "dynamic": "dynamic", + } + normalized = [] + for column in columns: + name = column.get("name", column.get("Name")) + raw_type = column.get("type", column.get("Type")) + if not isinstance(name, str) or not isinstance(raw_type, str) or raw_type.lower() not in aliases: + raise ValueError("query provider returned an unsupported or malformed column schema") + normalized.append({"name": name, "type": aliases[raw_type.lower()]}) + names = [column["name"] for column in normalized] + if len(set(names)) != len(names): + raise ValueError("query provider returned duplicate columns") + values = [] + for row in rows: + if isinstance(row, dict): + if set(row) - {"@odata.type"} != set(names): + raise ValueError("query provider row keys differ from its schema") + cells = [row[name] for name in names] + elif isinstance(row, list) and len(row) == len(names): + cells = list(row) + else: + raise ValueError("query provider returned a malformed result row") + for index, column in enumerate(normalized): + if column["type"] == "datetime" and cells[index] is not None: + value = cells[index] + if isinstance(value, dict) and set(value) == {"$datetime"}: + value = value["$datetime"] + if not isinstance(value, str): + raise ValueError("query provider returned an invalid datetime cell") + timestamp = datetime.fromisoformat(value.replace("Z", "+00:00")) + if timestamp.tzinfo is None: + raise ValueError("query provider returned a timezone-less datetime cell") + cells[index] = timestamp.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + values.append(cells) + return {"columns": normalized, "rows": values} + + +def plan_workbook_ui( + solution: str | Path, *, workbook: str, url: str, + readiness_selectors: dict[str, str], + steps: list[dict[str, Any]], + background_activity_review: str, + browser_config: dict[str, Any] | None = None, +) -> dict[str, Any]: + root, state, target = _context(solution) + path = content_path(root, workbook) + if path not in selected_workbook_references(root): + raise ValueError("UI qualification requires an explicitly selected Defender workbook") + parsed = urlsplit(url) + if ( + parsed.scheme != "https" or parsed.hostname != "security.microsoft.com" + or parsed.username or parsed.password or parsed.port not in (None, 443) + or not parsed.path.strip("/") + or any(key.casefold() in { + "access_token", "token", "code", "sig", "signature", "client_secret", + } for key, _ in parse_qsl(parsed.query)) + ): + raise ValueError("workbook UI requires an exact reviewed Defender portal HTTPS URL") + if ( + set(readiness_selectors) != {"tenant", "workbook", "ready"} + or any(not value.strip() for value in readiness_selectors.values()) + or not background_activity_review.strip() + ): + raise ValueError("UI readiness selectors and background/service-worker review are required") + required = {"pickers", "dependent-refresh", "filters", "drill-down", "reload", "layout", "conditional-visibility"} + if ( + {step.get("kind") for step in steps} != required + or any(not str(step.get("id") or "").strip() for step in steps) + or len({step["id"] for step in steps}) != len(steps) + or any(step.get("action") not in {"observe", "select", "reload", "drill-down"} for step in steps) + ): + raise ValueError("UI plan must include every reviewed check and only permitted non-edit actions") + document = json.loads(path.read_text(encoding="utf-8-sig")) + plan = { + "schemaVersion": "1.0.0", + "operation": "workbook-ui-qualification", + "runId": state["runId"], + "target": target, + "workbook": path.relative_to(root).as_posix(), + "contentId": document["xdrMetadata"]["contentId"], + "instanceId": document["xdrMetadata"]["instanceId"], + "targetSha256": _sha(path.read_bytes()), + "url": url, + "readinessSelectors": readiness_selectors, + "steps": steps, + "backgroundActivityReview": background_activity_review, + "saveEditAllowed": False, + "browserWideNoWriteSandbox": False, + } + if browser_config is not None: + from .workbook_browser import validate_browser_config + + source = document["contentProvenance"]["source"] + source_hash = _sha(content_path(root, source["path"]).read_bytes()) + if source_hash != source["sha256"]: + raise ValueError("workbook UI qualification source provenance is stale") + plan["schemaVersion"] = "1.1.0" + plan["sourceSha256"] = source_hash + plan["browserConfig"] = browser_config + validate_browser_config(plan, browser_config) + plan["planSha256"] = _seal(plan) + return plan + + +def current_workbook_ui_plan(solution: str | Path, plan: dict[str, Any]) -> dict[str, Any]: + return plan_workbook_ui( + solution, workbook=plan["workbook"], url=plan["url"], + readiness_selectors=plan["readinessSelectors"], steps=plan["steps"], + background_activity_review=plan["backgroundActivityReview"], + browser_config=plan.get("browserConfig"), + ) + + +def execute_workbook_ui( + solution: str | Path, plan: dict[str, Any], *, + read_consent: Callable[[], dict[str, Any]], + browser_runner: Callable[[dict[str, Any]], dict[str, Any]], +) -> dict[str, Any]: + from .workflow import _run_writer_lock + + with _run_writer_lock(report_directory(solution) / "workbook-qualification-executor"): + return _execute_workbook_ui( + solution, plan, read_consent=read_consent, browser_runner=browser_runner, + ) + + +def _execute_workbook_ui( + solution: str | Path, plan: dict[str, Any], *, + read_consent: Callable[[], dict[str, Any]], + browser_runner: Callable[[dict[str, Any]], dict[str, Any]], +) -> dict[str, Any]: + current = current_workbook_ui_plan(solution, plan) + if current != plan: + raise ValueError("workbook UI plan is stale or has been edited") + _check_consent(plan, read_consent()) + root = Path(solution).expanduser().resolve() + if plan.get("browserConfig") is not None: + from .workbook_browser import prepare_private_browser_evidence + + prepare_private_browser_evidence(root) + path = report_directory(root) / f"workbook-ui.{plan['contentId']}.json" + running = { + "schemaVersion": "1.0.0", "status": "running", "runId": plan["runId"], + "target": plan["target"], "targetSha256": plan["targetSha256"], + "plan": plan, "planSha256": plan["planSha256"], + "execution": "approved-browser-capture", "checksComplete": False, + } + if path.is_file(): + prior = json.loads(path.read_text(encoding="utf-8-sig")) + archive = path.with_name(f"{path.stem}.{_sha(path.read_bytes())}.json") + write_json_artifact(root, archive, prior) + write_json_artifact(root, path, running) + capture = None + try: + capture = browser_runner(plan) + _check_consent(plan, read_consent()) + _validate_ui_capture(plan, capture) + if current != current_workbook_ui_plan(solution, plan): + raise ValueError("workbook UI target changed during capture") + approval = read_consent() + _check_consent(plan, approval) + except Exception as exc: + diagnostics = {} + if plan.get("browserConfig") is not None: + from .workbook_browser import BrowserQualificationError + + if isinstance(exc, BrowserQualificationError): + diagnostics["diagnosticEvents"] = getattr(exc, "diagnostic_events", []) + elif capture is not None: + diagnostics["capture"] = capture + write_json_artifact(root, path, { + **running, "status": "failed", + "error": "incomplete-or-failed-ui-capture", + **diagnostics, + }) + raise + receipt = { + "schemaVersion": "1.0.0", + "status": "passed", + "runId": plan["runId"], + "target": plan["target"], + "targetSha256": plan["targetSha256"], + "plan": plan, + "planSha256": plan["planSha256"], + "approval": approval, + "capture": capture, + "execution": "approved-browser-capture", + "queryErrors": capture["queryErrors"], + "javascriptErrors": capture["javascriptErrors"], + "checksComplete": True, + "browserWideNoWriteSandbox": False, + } + write_json_artifact(root, path, receipt) + return {"status": "passed", "reportPath": str(path), "queryQualification": "separate"} + + +def _validate_ui_capture(plan: dict[str, Any], capture: dict[str, Any]) -> None: + if not isinstance(capture, dict): + raise ValueError("workbook UI capture must be a complete result object") + readiness = capture.get("readiness") or {} + results = capture.get("steps") + if ( + capture.get("target") != plan["target"] or capture.get("url") != plan["url"] + or readiness.get("tenantId") != plan["target"]["tenantId"] + or readiness.get("contentId") != plan["contentId"] + or readiness.get("instanceId") != plan["instanceId"] + or readiness.get("ready") is not True + or capture.get("queryErrors") != [] or capture.get("javascriptErrors") != [] + or capture.get("saveEditObserved") is not False + or capture.get("backgroundActivityReview") != plan["backgroundActivityReview"] + or not isinstance(results, list) or len(results) != len(plan["steps"]) + ): + raise ValueError("workbook UI capture has incomplete readiness, errors, or prohibited actions") + for expected, actual in zip(plan["steps"], results, strict=True): + if ( + not isinstance(actual, dict) + or any(actual.get(key) != expected[key] for key in ("id", "kind")) + or actual.get("status") != "passed" + or not str(actual.get("evidence") or "").strip() + ): + raise ValueError("workbook UI capture has incomplete reviewed step evidence") + if plan.get("browserConfig") is not None: + from .workbook_browser import validate_browser_step_proofs + + validate_browser_step_proofs(expected, actual) + + +def require_current_qualification(solution: str | Path, state: dict[str, Any]) -> None: + if not enabled(state) or state["context"]["workflowProfile"] != "qualification": + return + root, current, target = _context(solution) + if current["runId"] != state["runId"]: + raise ValueError("workbook qualification run changed") + for path in selected_workbook_references(root): + document = json.loads(path.read_text(encoding="utf-8-sig")) + content_id = document["xdrMetadata"]["contentId"] + query_path = report_directory(root) / f"workbook-queries.{content_id}.json" + ui_path = report_directory(root) / f"workbook-ui.{content_id}.json" + if not query_path.is_file() or not ui_path.is_file(): + raise ValueError("selected workbook query and UI qualification evidence is missing") + query = json.loads(query_path.read_text(encoding="utf-8-sig")) + plan = _current_plan(root, query["plan"]) + if query.get("status") != "passed" or query.get("planSha256") != plan["planSha256"]: + raise ValueError("workbook query qualification evidence failed or is stale") + _check_consent(plan, query["approval"]) + from ._workbook_core.workbook_fixtures import assert_result + + results = query.get("results") or [] + if len(results) != len(plan["queries"]): + raise ValueError("workbook query evidence has incomplete case coverage") + for expected, actual in zip(plan["queries"], results, strict=True): + if any(actual.get(key) != expected[key] for key in ("caseId", "side", "querySha256")): + raise ValueError("workbook query evidence has mismatched case identities") + if actual.get("status") != "passed" or actual.get("provider") != expected["side"]: + raise ValueError("workbook query evidence has unsuccessful provider execution") + try: + assert_result(expected["expected"], actual["result"]) + except (AssertionError, KeyError, TypeError, ValueError) as exc: + raise ValueError("workbook captured result evidence does not match the exact oracle") from exc + ui = json.loads(ui_path.read_text(encoding="utf-8-sig")) + if ( + ui.get("status") != "passed" or ui.get("runId") != state["runId"] + or ui.get("target") != target or ui.get("targetSha256") != _sha(path.read_bytes()) + or ui.get("queryErrors") != [] or ui.get("javascriptErrors") != [] + or ui.get("checksComplete") is not True + or ui.get("execution") != "approved-browser-capture" + ): + raise ValueError("workbook UI qualification remains failed, stale, or unverified") + ui_plan = current_workbook_ui_plan(root, ui["plan"]) + if ui_plan != ui["plan"] or ui.get("planSha256") != ui_plan["planSha256"]: + raise ValueError("workbook UI qualification plan is stale or edited") + _check_consent(ui_plan, ui["approval"]) + _validate_ui_capture(ui_plan, ui["capture"]) diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_qualification_cli.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_qualification_cli.py new file mode 100644 index 00000000000..5cebb30bfe6 --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_qualification_cli.py @@ -0,0 +1,182 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from . import workbook_qualification as qualification +from .artifacts import report_directory, write_json_artifact +from .content_paths import content_path + + +def register_workbook_qualification_commands(subparsers) -> tuple[str, ...]: + names = ( + "workbook-query-plan", "workbook-query-review", "workbook-query-revoke", + "workbook-query-execute", "workbook-ui-plan", "workbook-ui-review", + "workbook-ui-revoke", "workbook-ui-import", "workbook-ui-execute", + ) + parsers = {} + for name in names: + parser = subparsers.add_parser(name, help="Qualification-only workbook read planning, approval, or evidence.") + parser.add_argument("--solution", required=True) + parsers[name] = parser + for name in ("workbook-query-plan", "workbook-ui-plan"): + parsers[name].add_argument("--workbook", required=True) + parsers["workbook-query-plan"].add_argument("--clock", required=True, help="Reviewed fixed UTC fixture clock.") + parsers["workbook-query-plan"].add_argument("--seed", type=int, default=42) + parsers["workbook-query-plan"].add_argument("--scale", type=int, default=1) + parsers["workbook-query-plan"].add_argument("--recipe", default="syslog-operational-v1") + parsers["workbook-ui-plan"].add_argument("--url", required=True) + parsers["workbook-ui-plan"].add_argument("--readiness-selectors", required=True, help="Local JSON with reviewed tenant/workbook/ready selectors.") + parsers["workbook-ui-plan"].add_argument("--steps", required=True, help="Local JSON array of every reviewed non-edit UI check.") + parsers["workbook-ui-plan"].add_argument("--background-activity-review", required=True) + parsers["workbook-ui-plan"].add_argument( + "--browser-config", help="Optional local JSON with exact reviewed DOM assertions, private session hash and network allowlists.", + ) + parsers["workbook-ui-execute"].add_argument("--plan-sha256", required=True) + parsers["workbook-ui-execute"].add_argument( + "--storage-state", required=True, help="Existing owner-only Playwright storage state; never authenticates or exports it.", + ) + for name in names: + if name not in ("workbook-query-plan", "workbook-ui-plan"): + parsers[name].add_argument("--plan", required=True, help="Exact selected-run plan JSON path, relative to the solution.") + for name in ("workbook-query-review", "workbook-ui-review", "workbook-ui-import"): + parsers[name].add_argument("--plan-sha256", required=True) + parsers["workbook-query-execute"].add_argument("--plan-sha256", required=True) + parsers["workbook-query-execute"].add_argument("--state-dir") + parsers["workbook-query-execute"].add_argument("--family-timeout-seconds", type=int, default=1800) + for name in ( + "workbook-query-review", "workbook-ui-review", + "workbook-query-revoke", "workbook-ui-revoke", "workbook-ui-import", + ): + parsers[name].add_argument("--actor", required=True) + parsers[name].add_argument("--reference", required=True) + for name in ("workbook-query-review", "workbook-ui-review"): + parsers[name].add_argument("--decision", required=True, choices=("approved", "declined")) + parsers["workbook-ui-import"].add_argument( + "--evidence", required=True, + help="Reviewed complete external browser capture JSON; this command does not launch a browser.", + ) + return names + + +def _family(plan: dict[str, Any]) -> str: + if plan.get("operation") == "workbook-query-qualification": + return "workbook-queries" + if plan.get("operation") == "workbook-ui-qualification": + return "workbook-ui" + raise ValueError("unsupported workbook qualification operation") + + +def _read_plan(solution: str | Path, reference: str, *, ui: bool) -> dict[str, Any]: + root, state, _ = qualification._context(solution) + path = content_path(root, reference) + if path.parent != report_directory(root): + raise ValueError("workbook qualification plans must belong to the exact selected run") + plan = json.loads(path.read_text(encoding="utf-8-sig")) + expected_operation = "workbook-ui-qualification" if ui else "workbook-query-qualification" + if plan.get("operation") != expected_operation or plan.get("runId") != state["runId"]: + raise ValueError("workbook qualification plan has the wrong operation or run") + if ui: + current = qualification.current_workbook_ui_plan(root, plan) + if current != plan: + raise ValueError("workbook UI plan is stale or edited") + else: + qualification._current_plan(root, plan) + return plan + + +def _consent_path(solution: str | Path, plan: dict[str, Any]) -> Path: + return report_directory(solution) / f"{_family(plan)}.consent.{plan['contentId']}.json" + + +def _read_consent(solution: str | Path, plan: dict[str, Any]) -> dict[str, Any]: + path = _consent_path(solution, plan) + if not path.is_file(): + raise ValueError("explicit workbook qualification consent is missing") + return json.loads(path.read_text(encoding="utf-8-sig")) + + +def dispatch_workbook_qualification(args) -> dict[str, Any]: + root = Path(args.solution).expanduser().resolve() + ui = args.command.startswith("workbook-ui-") + if args.command == "workbook-query-plan": + plan = qualification.plan_workbook_queries( + root, workbook=args.workbook, clock=args.clock, + seed=args.seed, scale=args.scale, recipe=args.recipe, + ) + elif args.command == "workbook-ui-plan": + selectors = json.loads(content_path(root, args.readiness_selectors).read_text(encoding="utf-8-sig")) + steps = json.loads(content_path(root, args.steps).read_text(encoding="utf-8-sig")) + plan = qualification.plan_workbook_ui( + root, workbook=args.workbook, url=args.url, readiness_selectors=selectors, + steps=steps, background_activity_review=args.background_activity_review, + browser_config=( + json.loads(content_path(root, args.browser_config).read_text(encoding="utf-8-sig")) + if args.browser_config else None + ), + ) + else: + plan = _read_plan(root, args.plan, ui=ui) + if args.command.endswith("-revoke"): + if not args.actor.strip() or not args.reference.strip(): + raise ValueError("withdrawal requires an actor and reference") + approval = _read_consent(root, plan) + approval.update({ + "withdrawn": True, + "withdrawal": {"actor": args.actor, "reference": args.reference}, + }) + path = _consent_path(root, plan) + write_json_artifact(root, path, approval) + return {"status": "withdrawn", "consentPath": path.relative_to(root).as_posix()} + if plan["planSha256"] != args.plan_sha256: + raise ValueError("qualification requires the exact reviewed current plan hash") + if args.command.endswith("-review"): + approval = qualification.review_workbook_query_consent( + plan, actor=args.actor, reference=args.reference, + approved=args.decision == "approved", + ) + path = _consent_path(root, plan) + write_json_artifact(root, path, approval) + return { + "status": args.decision, "planSha256": plan["planSha256"], + "consentPath": path.relative_to(root).as_posix(), + } + if args.command == "workbook-query-execute": + return qualification.execute_workbook_queries( + root, plan, read_consent=lambda: _read_consent(root, plan), + providers=qualification.default_query_providers(state_dir=args.state_dir), + family_timeout_seconds=args.family_timeout_seconds, + ) + if args.command == "workbook-ui-execute": + from .workbook_browser import execute_workbook_browser + + result = execute_workbook_browser( + root, plan, read_consent=lambda: _read_consent(root, plan), + storage_state=args.storage_state, + ) + return { + **result, "browserExecution": "playwright-dom-v1", + "armContentHubAcceptance": "separate; not proven by UI capture", + } + if args.command == "workbook-ui-import": + if not args.actor.strip() or not args.reference.strip(): + raise ValueError("external browser capture review requires an actor and reference") + proof = json.loads(content_path(root, args.evidence).read_text(encoding="utf-8-sig")) + proof["captureReview"] = {"actor": args.actor, "reference": args.reference} + result = qualification.execute_workbook_ui( + root, plan, read_consent=lambda: _read_consent(root, plan), + browser_runner=lambda _: proof, + ) + return {**result, "browserExecution": "external-reviewed-capture; not executed by this command"} + raise ValueError("unsupported workbook qualification command") + path = report_directory(root) / f"{_family(plan)}.plan.{plan['contentId']}.json" + write_json_artifact(root, path, plan) + return { + "status": "planned", "operation": plan["operation"], + "planPath": path.relative_to(root).as_posix(), "planSha256": plan["planSha256"], + "target": plan["target"], "workbook": plan["workbook"], + "queryCount": len(plan.get("queries") or []), + "uiStepCount": len(plan.get("steps") or []), + "execution": "not-run", "ingestion": "not-run", + } diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_validation.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_validation.py new file mode 100644 index 00000000000..6df5514b394 --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_validation.py @@ -0,0 +1,885 @@ +"""Versioned candidate ARM validation; no implicit cloud dispatch or legacy upgrades.""" +from __future__ import annotations + +import hashlib +import json +import re +import zipfile +import uuid +from pathlib import Path +from typing import Any, Callable, Mapping + +from .arm_acceptance_support import _canonical_sha256, _deployment_provisioning_state +from .content_paths import content_path +from .target_context import workspace_identity + +LAB_ARM_CONFIRMATION = "I approve the exact current candidate ARM deployment validation" +_HASH = re.compile(r"^[a-f0-9]{64}$") +_TARGET_KEYS = ("tenantId", "subscriptionId", "resourceGroup", "workspaceResourceId", "workspaceCustomerId") +_GATES = ("conversion", "review", "structure", "package") +_FLAG_KINDS = { + "DeployAnalyticsRule": "AnalyticsRule", "DeployPlaybook": "Playbook", + "DeployWorkbook": "Workbook", "DeployDefenderWorkbook": "DefenderWorkbook", + "DeployDataConnector": "DataConnector", "DeployCustomDetection": "CustomDetection", + "DeployHuntingQuery": "HuntingQuery", "DeployInvestigationQuery": "InvestigationQuery", + "DeployParser": "Parser", "DeployWatchlist": "Watchlist", "DeployNotebook": "Notebook", + "DeployAutomationRule": "AutomationRule", "DeploySummaryRule": "SummaryRule", +} + + +def _utc_now() -> str: + from .arm_acceptance import _utc_now as now + return now() + + +def _parse_build_validation_receipt(*args: Any, **kwargs: Any) -> dict[str, Any]: + from .arm_acceptance import _parse_build_validation_receipt as parse + return parse(*args, **kwargs) + + +def arm_validation_method(context: Mapping[str, Any]) -> str: + """Absent capability preserves legacy ARM-TTK; new contexts must persist a method.""" + if "workbookCapabilityVersion" not in context: + return "arm-ttk" + if type(context["workbookCapabilityVersion"]) is not int or context["workbookCapabilityVersion"] != 1: + raise ValueError("unsupported workbook capability version") + method = context.get("armValidationMethod") + if method not in {"arm-ttk", "lab-deployment"}: + raise ValueError("versioned context requires an explicit persisted ARM validation method") + if context.get("workflowProfile") not in {"authoring", "qualification"} or context.get("profileSelectionConfirmed") is not True: + raise ValueError("ARM validation requires an explicitly selected workflow profile") + if method == "arm-ttk" and context["workflowProfile"] != "authoring": + raise ValueError("the new Qualification route requires lab-deployment") + if context["workflowProfile"] == "qualification" and context.get("targetContextLocked") is not True: + raise ValueError("Qualification candidate validation requires its existing locked target") + return method + + +def _digest_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _sealed(value: Mapping[str, Any], field: str) -> bool: + body = dict(value) + digest = body.pop(field, None) + return isinstance(digest, str) and _HASH.fullmatch(digest) is not None and _canonical_sha256(body) == digest + + +def create_arm_candidate( + context: Mapping[str, Any], *, solution: str | Path, package_report: str | Path, + validation_receipt: str | Path, prerequisite_evidence: Mapping[str, Mapping[str, str]], +) -> dict[str, Any]: + """Verify current V3.1 artifacts and official static receipt without prior acceptance. + + The integration owner produces four Authoring gate receipts with status=passed, + packageInputSha256, and workflowRunId, after its existing conversion/review/ + structure/selection checks. Static/package gates also bind packageArtifactsSha256. + Qualification additionally requires its existing runtime/locked-target gate. + References here bind their exact current bytes. + """ + from .packaging import package_input_file_hashes, package_input_identity, validate_v31_target_version + + method = arm_validation_method(context) + if context.get("workbookCapabilityVersion") != 1: + raise ValueError("candidate validation cannot upgrade a legacy workflow") + if not isinstance(context.get("runId"), str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,95}", context["runId"]): + raise ValueError("candidate requires a safe exact workflow run identity") + source_root = Path(solution) + if source_root.is_symlink(): + raise ValueError("candidate solution cannot be a symlink") + root = source_root.resolve(strict=True) + report_path = content_path(root, package_report) + report = json.loads(report_path.read_text(encoding="utf-8-sig")) + if report.get("packager") != "V3.1" or report.get("workflowRunId") != context.get("runId") or not context.get("runId"): + raise ValueError("candidate must belong to the selected V3.1 workflow run") + if report.get("armValidationMethod") != method: + raise ValueError("candidate packager method binding is missing or mismatched") + if not report.get("packageAttemptId"): + raise ValueError("candidate package attempt identity is missing") + validate_v31_target_version(report.get("version")) + inputs = package_input_identity(root) + recorded_inputs = (report.get("packageInputs") or {}).get("after") + if ( + not isinstance(recorded_inputs, dict) + or type(recorded_inputs.get("fileCount")) is not int + or any(recorded_inputs.get(key) != value for key, value in inputs.items()) + ): + raise ValueError("candidate package inputs are stale") + current_files = package_input_file_hashes(root) + recorded_files = recorded_inputs.get("files") + manifest_sha256 = _canonical_sha256( + [{"path": path, "sha256": digest} for path, digest in current_files.items()] + ) + if ( + not isinstance(recorded_files, dict) + or recorded_files != current_files + or len(current_files) != inputs["fileCount"] + or manifest_sha256 != inputs["sha256"] + ): + raise ValueError("candidate package input file manifest is missing, incomplete or stale") + post_arm_workbook_qualification = context.get("workbookQualificationAfterArm") is True + if post_arm_workbook_qualification: + from .workbook_workflow import has_analytic_rules + + if ( + context["workflowProfile"] != "qualification" + or has_analytic_rules(root) + or report.get("xdrDetectionCount") not in (None, 0) + ): + raise ValueError("post-ARM workbook qualification cannot bypass detection runtime prerequisites") + required_gates = ( + (*_GATES, "qualification") + if context["workflowProfile"] == "qualification" and not post_arm_workbook_qualification + else _GATES + ) + if set(prerequisite_evidence) != set(required_gates): + raise ValueError("candidate requires fresh conversion, review, structure and package gate evidence") + gates = {} + for name, reference in prerequisite_evidence.items(): + path = content_path(root, reference["path"]) + if _digest_file(path) != reference.get("sha256"): + raise ValueError(f"{name} gate evidence is stale") + gate = json.loads(path.read_text(encoding="utf-8-sig")) + if gate.get("status") != "passed" or gate.get("packageInputSha256") != inputs["sha256"] or gate.get("workflowRunId") != context["runId"]: + raise ValueError(f"{name} gate did not pass for current inputs/run") + if name in {"structure", "package"} and gate.get("packageArtifactsSha256") != report.get("packageArtifactsSha256"): + raise ValueError(f"{name} gate is not bound to exact current candidate artifacts") + if name == "qualification" and ( + gate.get("qualificationRuntimeVerified") is not True or gate.get("target") != _target(context) + ): + raise ValueError("Qualification's existing runtime/target prerequisites remain mandatory") + gates[name] = dict(reference) + hashes = report.get("packageArtifactHashes") + if not isinstance(hashes, dict) or set(hashes) != {"mainTemplate", "createUiDefinition", "testParameters", "zip"}: + raise ValueError("candidate must bind every V3.1 artifact") + paths = {} + for name, digest in hashes.items(): + path = content_path(root, report[name]) + if not isinstance(digest, str) or not _HASH.fullmatch(digest) or _digest_file(path) != digest: + raise ValueError(f"candidate {name} artifact is stale") + paths[name] = path + if _canonical_sha256(hashes) != report.get("packageArtifactsSha256"): + raise ValueError("candidate artifact aggregate hash is stale") + with zipfile.ZipFile(paths["zip"]) as archive: + names = archive.namelist() + if len(names) != len(set(names)) or any( + ".." in Path(name.replace("\\", "/")).parts or Path(name).is_absolute() + or any(part.lower() in {"logs", "reports", "evidence", ".env"} or part.lower().startswith(".env.") for part in name.replace("\\", "/").split("/")) + for name in names + ): + raise ValueError("candidate ZIP contains unsafe, duplicate or operational entries") + for name in ("mainTemplate", "createUiDefinition"): + matches = [member for member in names if Path(member).name == paths[name].name] + if len(matches) != 1 or hashlib.sha256(archive.read(matches[0])).hexdigest() != hashes[name]: + raise ValueError(f"candidate ZIP {name} does not match exact artifact bytes") + package_files = [] + for path in sorted((root / "Package").rglob("*")): + if path.is_symlink(): + raise ValueError("candidate Package contains a symlink") + if path.is_file(): + package_files.append({"path": path.relative_to(root / "Package").as_posix(), "sha256": _digest_file(path)}) + package_files.sort(key=lambda item: item["path"]) + identity = { + "version": report["version"], "packageAttemptId": report["packageAttemptId"], + "workflowRunId": context["runId"], "packageInputSha256": inputs["sha256"], + "packageArtifactHashes": hashes, "recordedPackageArtifactHashes": hashes, + "packageReportSha256": _digest_file(report_path), + "packageArtifactsSha256": report["packageArtifactsSha256"], + "packageContentSha256": hashlib.sha256(json.dumps(package_files, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()).hexdigest(), + } + receipt = _parse_build_validation_receipt( + root, validation_receipt, package_report_path=report_path, + package_identity=identity, arm_validation_method=method, + ) + candidate = { + "schemaVersion": 1, "workbookCapabilityVersion": 1, "armValidationMethod": method, + "status": "candidate-generated", "accepted": False, "solutionPath": str(root), + "workflowProfile": context["workflowProfile"], + "runId": context["runId"], "packageReport": report_path.relative_to(root).as_posix(), + "validationReceipt": receipt, "prerequisiteEvidence": gates, "packageIdentity": identity, + "artifactPaths": {name: path.relative_to(root).as_posix() for name, path in paths.items()}, + "mainTemplateSha256": hashes["mainTemplate"], + } + candidate["candidateSha256"] = _canonical_sha256(candidate) + return candidate + + +def _target(target: Mapping[str, Any]) -> dict[str, str]: + if any(not isinstance(target.get(key), str) or not target[key].strip() for key in _TARGET_KEYS): + raise ValueError("selected lab requires complete tenant/subscription/workspace identity") + result = {key: target[key] for key in _TARGET_KEYS} + for key in ("tenantId", "workspaceCustomerId"): + if str(uuid.UUID(result[key])) != result[key].lower(): + raise ValueError(f"selected lab has an invalid {key}") + identity = workspace_identity(result["workspaceResourceId"]) + if any(str(identity[key]).casefold() != result[key].casefold() for key in ("subscriptionId", "resourceGroup")): + raise ValueError("selected lab subscription/resource group conflicts with workspace") + return result + + +def refresh_arm_candidate(context: Mapping[str, Any], candidate: Mapping[str, Any]) -> dict[str, Any]: + """Recompute all input, artifact, official receipt and prerequisite bindings.""" + if not _sealed(candidate, "candidateSha256"): + raise ValueError("candidate receipt is malformed") + current = create_arm_candidate( + context, solution=candidate["solutionPath"], package_report=candidate["packageReport"], + validation_receipt=candidate["validationReceipt"]["path"], + prerequisite_evidence=candidate["prerequisiteEvidence"], + ) + if current["candidateSha256"] != candidate["candidateSha256"]: + raise ValueError("candidate evidence changed") + return current + + +def _arguments(text: str) -> list[str]: + args, start, depth, quoted = [], 0, 0, False + index = 0 + while index < len(text): + char = text[index] + if char == "'": + if quoted and index + 1 < len(text) and text[index + 1] == "'": + index += 2 + continue + quoted = not quoted + elif not quoted: + if char == "(": + depth += 1 + elif char == ")": + depth -= 1 + elif char == "," and depth == 0: + args.append(text[start:index].strip()) + start = index + 1 + index += 1 + if quoted or depth != 0: + raise ValueError("unresolvable ARM expression") + return args + ([text[start:].strip()] if text[start:].strip() else []) + + +def _resolve(value: Any, template: Mapping[str, Any], parameters: Mapping[str, Any], target: Mapping[str, str], *, depth: int = 0) -> Any: + if depth > 30: + raise ValueError("recursive ARM expression cannot be resolved safely") + if not isinstance(value, str) or not value.startswith("["): + return value + if value.startswith("[["): + return value[1:] + if not value.endswith("]"): + raise ValueError("unresolvable ARM expression") + + def expression(text: str) -> Any: + if re.fullmatch(r"'(?:[^']|'')*'", text): + return text[1:-1].replace("''", "'") + if text.casefold() in {"true", "false"}: + return text.casefold() == "true" + if re.fullmatch(r"-?\d+", text): + return int(text) + fixed = {"tenant().tenantId": target["tenantId"], "subscription().subscriptionId": target["subscriptionId"], + "resourceGroup().name": target["resourceGroup"], + "resourceGroup().id": f"/subscriptions/{target['subscriptionId']}/resourceGroups/{target['resourceGroup']}"} + if text in fixed: + return fixed[text] + match = re.fullmatch(r"([A-Za-z]+)\((.*)\)", text, re.DOTALL) + if not match: + raise ValueError(f"unresolvable ARM scope/condition expression: {text}") + function = match[1].casefold() + args = [expression(arg) for arg in _arguments(match[2])] + if function == "parameters" and len(args) == 1: + if args[0] not in parameters: + definition = (template.get("parameters") or {}).get(args[0]) + if not isinstance(definition, dict) or "defaultValue" not in definition: + raise ValueError("ARM parameter is not bound") + return _resolve(definition["defaultValue"], template, parameters, target, depth=depth + 1) + return parameters[args[0]] + if function == "variables" and len(args) == 1: + variables = template.get("variables") or {} + if args[0] not in variables: + raise ValueError("ARM variable is not bound") + return _resolve(variables[args[0]], template, parameters, target, depth=depth + 1) + if function == "concat": + return "".join(str(arg) for arg in args) + if function == "format" and args and isinstance(args[0], str): + return args[0].format(*args[1:]) + if function == "equals" and len(args) == 2: + return args[0] == args[1] + if function in {"and", "or"} and args and all(type(arg) is bool for arg in args): + return all(args) if function == "and" else any(args) + if function == "not" and len(args) == 1 and type(args[0]) is bool: + return not args[0] + if function == "if" and len(args) == 3 and type(args[0]) is bool: + return args[1] if args[0] else args[2] + if function == "resourceid" and len(args) >= 2 and isinstance(args[0], str) and "/" in args[0]: + kinds = args[0].split("/") + if len(kinds) != len(args): + raise ValueError("resourceId scope arguments are ambiguous") + suffix = "/".join(part for kind, name in zip(kinds[1:], args[1:]) for part in (kind, str(name))) + return f"{fixed['resourceGroup().id']}/providers/{kinds[0]}/{suffix}" + raise ValueError(f"unresolvable ARM scope/condition function: {function}") + + return expression(value[1:-1].strip()) + + +def _resource_inventory(template: dict[str, Any], parameters: dict[str, Any], target: dict[str, str]) -> list[dict[str, Any]]: + inventory: list[dict[str, Any]] = [] + rg_scope = f"/subscriptions/{target['subscriptionId']}/resourceGroups/{target['resourceGroup']}" + tenant_scope = f"/tenants/{target['tenantId']}" + workspace = workspace_identity(target["workspaceResourceId"])["workspaceName"] + + def walk(document: dict[str, Any], values: dict[str, Any], inherited: str, path: str, depth: int = 0) -> None: + if depth > 25: + raise ValueError("nested ARM scope depth exceeded") + imports = document.get("imports") or {} + if not isinstance(imports, dict): + raise ValueError("unknown ARM imports") + for alias, imported in imports.items(): + if not isinstance(imported, dict) or imported.get("provider") != "MicrosoftSecurity" or imported.get("version") != "1.0.0": + raise ValueError("unsupported ARM extension scope") + config = imported.get("config") or {} + if not isinstance(config, dict) or set(config) - {"tenantId"}: + raise ValueError("unknown MicrosoftSecurity extension configuration") + tenant = _resolve(config.get("tenantId", target["tenantId"]), document, values, target) + if str(tenant).casefold() != target["tenantId"].casefold(): + raise ValueError("MicrosoftSecurity import crosses the approved tenant") + resources = document.get("resources", []) + if isinstance(resources, dict): + entries = list(resources.items()) + elif isinstance(resources, list): + entries = list(enumerate(resources)) + else: + raise ValueError("ARM resources must be a list or symbolic object") + for key, resource in entries: + if not isinstance(resource, dict) or "copy" in resource: + raise ValueError("resource copy/unknown inventory cannot be bounded safely") + condition = _resolve(resource.get("condition", True), document, values, target) + if type(condition) is not bool: + raise ValueError("ARM resource condition is unresolved") + if not condition: + continue + kind = resource.get("type") + if not isinstance(kind, str) or "[" in kind or not re.fullmatch(r"[A-Za-z][A-Za-z0-9.]*/[A-Za-z0-9/]+(?:@[A-Za-z0-9.-]+)?", kind): + raise ValueError("unknown ARM resource type") + kind = kind.split("@", 1)[0] + if kind.casefold().startswith(("microsoft.authorization/", "microsoft.graph/")): + raise ValueError("deployment consent excludes role/permission grants") + imported_alias = resource.get("import") + if imported_alias is not None and imported_alias not in imports: + raise ValueError("resource refers to an unknown ARM extension") + is_extension = imported_alias is not None + if kind.casefold().startswith("microsoft.security/") and not is_extension: + raise ValueError("MicrosoftSecurity resource scope requires a supported import") + scope = _resolve(resource.get("scope", tenant_scope if is_extension else inherited), document, values, target) + allowed_scope = tenant_scope if is_extension else inherited + if not isinstance(scope, str) or scope.casefold() not in {allowed_scope.casefold(), target["workspaceResourceId"].casefold()}: + raise ValueError("resource scope is unknown or crosses selected lab") + props = resource.get("properties") or {} + if not isinstance(props, dict): + raise ValueError("unknown resource properties") + if is_extension and kind.casefold() not in {"microsoft.security/workbooks", "microsoft.security/detectionrules"}: + raise ValueError("unsupported MicrosoftSecurity resource scope contract") + if kind.casefold().endswith(("/alertrules", "/detectionrules")): + if props.get("isEnabled", props.get("enabled")) is not False: + raise ValueError("detection deployment requires explicitly disabled content") + if "microsoft.operationalinsights/workspaces/" in kind.casefold(): + name = resource.get("name") + # Registration suffixes may use uniqueString; only the workspace prefix + # must be proven here, not fabricated by an approximate ARM evaluator. + prefix = re.match(r"^\[concat\(parameters\('workspace'\),\s*'(/[^']*)'", str(name)) + if not prefix: + resolved_name = _resolve(name, document, values, target) + if not isinstance(resolved_name, str) or not resolved_name.startswith(workspace + "/"): + raise ValueError("workspace-scoped resource name is not bound to selected workspace") + if scope.casefold() != rg_scope.casefold(): + raise ValueError("workspace resource cannot have a tenant/extension scope") + location = f"{path}/resources/{key}" + inventory.append({"path": location, "type": kind, "scope": scope, "name": resource.get("name"), "import": imported_alias}) + if kind.casefold() == "microsoft.resources/deployments": + if props.get("mode") != "Incremental" or "templateLink" in props: + raise ValueError("only inline Incremental nested deployments are supported") + for field, expected in (("subscriptionId", target["subscriptionId"]), ("resourceGroup", target["resourceGroup"])): + if field in resource and str(_resolve(resource[field], document, values, target)).casefold() != expected.casefold(): + raise ValueError("nested deployment crosses selected lab") + nested = props.get("template") + if not isinstance(nested, dict): + raise ValueError("nested deployment template is unresolved") + evaluation_scope = (props.get("expressionEvaluationOptions") or {}).get("scope", "outer") + if evaluation_scope not in {"inner", "outer"}: + raise ValueError("unknown nested expression scope") + child_values = {} + for name, binding in (props.get("parameters") or {}).items(): + if not isinstance(binding, dict) or set(binding) != {"value"}: + raise ValueError("nested parameter binding is unresolved") + child_values[name] = _resolve(binding["value"], document, values, target) + walk(nested, child_values if evaluation_scope == "inner" else values, scope, location + "/properties/template", depth + 1) + if "resources" in resource: + raise ValueError("relative child resources require an explicit supported scope contract") + walk(template, parameters, rg_scope, "") + return inventory + + +def _expected_workbook_contracts(template: Mapping[str, Any], selected: bool) -> list[dict[str, Any]]: + if not selected: + return [] + installations, registrations, install_templates = {}, {}, {} + + def installed_literals(value: Any) -> Any: + if isinstance(value, str) and value.startswith("[["): + return value[1:] + if isinstance(value, dict): + return {key: installed_literals(item) for key, item in value.items()} + if isinstance(value, list): + return [installed_literals(item) for item in value] + return value + + def walk(document: Mapping[str, Any]) -> None: + resources = document.get("resources") or [] + for resource in resources.values() if isinstance(resources, dict) else resources: + props = resource.get("properties") or {} + kind = str(resource.get("type") or "").split("@", 1)[0].casefold() + if kind == "microsoft.security/workbooks": + notebook = props.get("content") + if not isinstance(notebook, dict) or notebook.get("version") != "Notebook/1.0": + raise ValueError("workbook installation has no complete Notebook content") + provenance = notebook.get("contentProvenance") + quartet = {"packageId", "packageVersion", "contentId", "contentVersion"} + if not isinstance(provenance, dict) or set(provenance) != quartet: + raise ValueError("workbook installation provenance must be the exact resolved quartet") + cid = provenance["contentId"] + if cid in installations: + raise ValueError("duplicate installed workbook content identity") + installations[cid] = {"contentId": cid, "instanceId": props.get("id"), + "contentSha256": _canonical_sha256(installed_literals(notebook)), "contentProvenance": installed_literals(provenance)} + install_templates[cid] = _canonical_sha256(document) + if props.get("contentKind") == "DefenderWorkbook": + cid = props.get("contentId") + if cid in registrations: + raise ValueError("duplicate workbook registration identity") + registrations[cid] = props + if kind == "microsoft.resources/deployments" and isinstance(props.get("template"), dict): + walk(props["template"]) + walk(template) + if not installations or set(installations) != set(registrations): + raise ValueError("selected workbook installation/registration inventory is incomplete") + result = [] + for cid, installation in sorted(installations.items()): + registration = registrations[cid] + provenance = registration.get("contentProvenance") + if not isinstance(provenance, dict) or any(provenance.get(key) != value for key, value in installation["contentProvenance"].items()): + raise ValueError("workbook registration provenance mismatches installation") + iid = installation["instanceId"] + if not isinstance(iid, str) or iid == cid or str(uuid.UUID(iid)) != iid or str(uuid.UUID(cid)) != cid: + raise ValueError("installed workbook identity must be distinct") + if _canonical_sha256(registration.get("mainTemplate")) != install_templates[cid]: + raise ValueError("registered workbook installation template differs from candidate install") + quartet = installation["contentProvenance"] + if any(registration.get(field) != quartet[provenance_field] for field, provenance_field in ( + ("contentId", "contentId"), ("version", "contentVersion"), + ("packageId", "packageId"), ("packageVersion", "packageVersion"), + )): + raise ValueError("registered workbook content/package version/identity mismatches installation") + registration_contract = { + "contentId": cid, "contentKind": "DefenderWorkbook", "version": registration.get("version"), + "packageId": registration.get("packageId"), "packageVersion": registration.get("packageVersion"), + "contentProvenance": provenance, + "storedTemplateSha256": _canonical_sha256(registration.get("mainTemplate")), + } + result.append({"contentId": cid, "instanceId": iid, + "installation": installation, "registration": registration_contract, + "repeatInstallation": installation, "update": installation}) + return result + + +def _candidate_scope_plan(context: Mapping[str, Any], candidate: Mapping[str, Any], *, target: Mapping[str, Any], selected_flags: Mapping[str, bool], expected_workbooks: list[dict[str, Any]] | None = None, lab_route: bool) -> dict[str, Any]: + method = arm_validation_method(context) + if context.get("workbookCapabilityVersion") != 1 or (lab_route and method != "lab-deployment"): + raise ValueError("lab candidate route requires the explicit versioned capability") + if not _sealed(candidate, "candidateSha256") or candidate.get("status") != "candidate-generated" or candidate.get("accepted") is not False or candidate.get("armValidationMethod") != method: + raise ValueError("lab validation requires an intact unaccepted candidate") + if candidate.get("runId") != context.get("runId") or candidate.get("workflowProfile") != context["workflowProfile"]: + raise ValueError("candidate belongs to another workflow run") + lab = _target(target) + if context["workflowProfile"] == "qualification" and lab != _target(context): + raise ValueError("candidate lab differs from Qualification's locked target") + root = Path(candidate["solutionPath"]) + template_path = content_path(root, candidate["artifactPaths"]["mainTemplate"]) + if _digest_file(template_path) != candidate["mainTemplateSha256"]: + raise ValueError("candidate template changed") + template = json.loads(template_path.read_text(encoding="utf-8-sig")) + definitions = template.get("parameters") or {} + declared = {name for name in definitions if name.startswith("Deploy")} + if set(selected_flags) != declared or any(type(value) is not bool for value in selected_flags.values()) or declared - set(_FLAG_KINDS): + raise ValueError("every declared content flag requires an exact supported boolean selection") + parameters = {} + for name, definition in definitions.items(): + if not isinstance(definition, dict): + raise ValueError("malformed ARM parameter") + if name in selected_flags: + if definition.get("type") != "bool": + raise ValueError("Deploy flags must be boolean") + parameters[name] = selected_flags[name] + elif name == "workspace": + parameters[name] = workspace_identity(lab["workspaceResourceId"])["workspaceName"] + elif name == "workspace-location" and isinstance(target.get("workspaceLocation"), str) and target["workspaceLocation"]: + parameters[name] = target["workspaceLocation"] + elif "defaultValue" in definition: + continue + else: + raise ValueError(f"unbound required ARM parameter: {name}") + if not isinstance(parameters[name], (str, bool, int, float)): + raise ValueError("candidate ARM parameters must be explicit scalar values") + inventory = _resource_inventory(template, parameters, lab) + contracts = _expected_workbook_contracts(template, selected_flags.get("DeployDefenderWorkbook") is True) + if expected_workbooks is not None and expected_workbooks != contracts: + raise ValueError("expected workbook readback contract does not match candidate template") + body = { + "schemaVersion": 1, "workbookCapabilityVersion": 1, "status": "ready", + "armValidationMethod": method, "runId": candidate["runId"], + "deploymentAuthorizedRoute": lab_route, + "purpose": "lab-arm-validation" if lab_route else "workbook-contract-readback", + "candidateSha256": candidate["candidateSha256"], "target": lab, + "solutionPath": candidate["solutionPath"], + "selectedFlags": dict(selected_flags), "parameters": parameters, + "mainTemplateSha256": candidate["mainTemplateSha256"], + "packageIdentity": candidate["packageIdentity"], "resources": inventory, + "artifactPaths": candidate["artifactPaths"], "prerequisiteEvidence": candidate["prerequisiteEvidence"], + "validationReceiptFingerprint": {key: candidate["validationReceipt"].get(key) for key in ("path", "sha256", "bindingPath", "bindingSha256")}, + "scope": {"lab": lab, "operations": ["arm-incremental-deployment"], + "resourceTypes": sorted({item["type"] for item in inventory} | {"Microsoft.Resources/deployments"}), + "contentTypes": sorted({_FLAG_KINDS[name] for name, selected in selected_flags.items() if selected})}, + "writeCount": 1, "resourceCount": len(inventory), "confirmation": LAB_ARM_CONFIRMATION, + "installationVerified": False, "registrationVerified": False, + "uiVerified": False, "parityVerified": False, "accepted": False, + "expectedWorkbooks": contracts, + } + body["planSha256"] = _canonical_sha256(body) + return body + + +def plan_lab_arm_validation(context: Mapping[str, Any], candidate: Mapping[str, Any], *, target: Mapping[str, Any], selected_flags: Mapping[str, bool], expected_workbooks: list[dict[str, Any]] | None = None) -> dict[str, Any]: + """Local-only exact execution plan; selected flags never default to all-true.""" + return _candidate_scope_plan(context, candidate, target=target, selected_flags=selected_flags, + expected_workbooks=expected_workbooks, lab_route=True) + + +def plan_workbook_contract_verification(context: Mapping[str, Any], candidate: Mapping[str, Any], *, target: Mapping[str, Any], selected_flags: Mapping[str, bool]) -> dict[str, Any]: + """Read-only expected contract for either method; never a deployment plan.""" + return _candidate_scope_plan(context, candidate, target=target, selected_flags=selected_flags, lab_route=False) + + +def review_lab_arm_scope(plan: Mapping[str, Any], *, actor: str, approval_reference: str, approved: bool) -> dict[str, Any]: + if not _sealed(plan, "planSha256") or plan.get("status") != "ready" or plan.get("deploymentAuthorizedRoute") is not True or plan.get("armValidationMethod") != "lab-deployment": + raise ValueError("review requires an intact current lab scope plan") + if not actor.strip() or not approval_reference.strip() or type(approved) is not bool: + raise ValueError("scope review requires affirmative decision, actor and approval reference") + return { + "schemaVersion": 1, "consentPolicyVersion": 1, + "status": "approved" if approved else "declined", "scope": plan["scope"], + "actor": actor.strip(), "approvalReference": approval_reference.strip(), + "recordedAt": _utc_now(), "withdrawn": False, + "deploymentOnly": True, "activationAuthorized": False, "ingestionAuthorized": False, + "deletionAuthorized": False, "grantsAuthorized": False, "cleanupAuthorized": False, + "reviewedPlanSha256": plan["planSha256"], + } + + +def withdraw_lab_arm_scope(consent: Mapping[str, Any], *, actor: str, withdrawal_reference: str) -> dict[str, Any]: + if not actor.strip() or not withdrawal_reference.strip(): + raise ValueError("withdrawal requires actor and reference") + return {**consent, "status": "withdrawn", "withdrawn": True, "withdrawalActor": actor.strip(), + "withdrawalReference": withdrawal_reference.strip(), "withdrawnAt": _utc_now()} + + +def check_lab_arm_scope(plan: Mapping[str, Any], consent: Mapping[str, Any] | None) -> dict[str, Any]: + reasons = [] + if not _sealed(plan, "planSha256") or plan.get("deploymentAuthorizedRoute") is not True or plan.get("armValidationMethod") != "lab-deployment": + reasons.append("execution plan is stale or malformed") + if not isinstance(consent, Mapping) or consent.get("consentPolicyVersion") != 1 or consent.get("status") != "approved" or consent.get("withdrawn") is not False: + reasons.append("missing, declined, legacy or withdrawn scope approval") + else: + scope = consent.get("scope") + if not isinstance(scope, Mapping): + return {"status": "blocked", "reasons": ["malformed approved scope"]} + approved_lab = scope.get("lab") + if not isinstance(approved_lab, Mapping) or any( + str(approved_lab.get(key, "")).casefold() != str(plan["scope"]["lab"][key]).casefold() + for key in _TARGET_KEYS + ): + reasons.append("lab changed; new approval required") + for field in ("operations", "resourceTypes", "contentTypes"): + approved = scope.get(field) + if not isinstance(approved, list) or any(not isinstance(item, str) or not item for item in approved) or not set(plan["scope"][field]).issubset(set(approved)): + reasons.append(f"new {field}; new approval required") + if not consent.get("actor") or not consent.get("approvalReference") or consent.get("deploymentOnly") is not True or any(consent.get(key) is not False for key in ("activationAuthorized", "ingestionAuthorized", "deletionAuthorized", "grantsAuthorized", "cleanupAuthorized")): + reasons.append("approval identity or deployment-only boundary is invalid") + return {"status": "ready" if not reasons else "blocked", "reasons": reasons} + + +def execute_lab_arm_validation( + context: Mapping[str, Any], candidate: Mapping[str, Any], *, target: Mapping[str, Any], + selected_flags: Mapping[str, bool], expected_plan_sha256: str, actor: str, + approval_reference: str, confirmation: str, evidence_directory: str | Path, + refresh_candidate: Callable[[], Mapping[str, Any]], read_consent: Callable[[], Mapping[str, Any]], + identity_check: Callable[[Mapping[str, Any]], Mapping[str, Any]], + command_runner: Callable[[list[str]], tuple[int, Any, str]], + expected_workbooks: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """One reserved executor; callers supply existing auth/provider/consent readers. + + No default dispatcher exists. The command runner receives Azure CLI arguments + (without executable); the integration adapter may use the existing _run_cli. + """ + from .arm_acceptance import _error_status, _recorded_provisioning_state, _reserve_attempt, _write_template_snapshot + + plan = plan_lab_arm_validation(context, candidate, target=target, selected_flags=selected_flags, expected_workbooks=expected_workbooks) + if plan["planSha256"] != expected_plan_sha256 or confirmation != LAB_ARM_CONFIRMATION or not actor.strip() or not approval_reference.strip(): + raise ValueError("immediate exact-plan write approval is missing or stale") + consent = check_lab_arm_scope(plan, read_consent()) + if consent["status"] != "ready": + return {**consent, "writeAttempted": False, "accepted": False} + directory = Path(evidence_directory) + root = Path(candidate["solutionPath"]).resolve(strict=True) + expected_directory = root / "Logs" / "sentinel-xdr-migration" / candidate["runId"] + if directory.resolve() != expected_directory or any(parent.is_symlink() for parent in (directory, *directory.parents)): + raise ValueError("attempt evidence must remain in the exact selected run Logs directory without links") + directory.mkdir(parents=True, exist_ok=True) + identity = _canonical_sha256({"solution": str(root), "runId": candidate["runId"], "target": plan["target"], "candidateSha256": candidate["candidateSha256"], "parameters": plan["parameters"], "approvalReference": approval_reference}) + attempt_path = directory / f"arm-validation-attempt-{identity}.json" + if attempt_path.exists() or attempt_path.is_symlink(): + raise ValueError("existing durable attempt requires reconciliation before retry") + current = refresh_candidate() + current = refresh_arm_candidate(context, current) + fresh = plan_lab_arm_validation(context, current, target=target, selected_flags=selected_flags, expected_workbooks=expected_workbooks) + if fresh["planSha256"] != expected_plan_sha256: + raise ValueError("candidate evidence changed after immediate approval") + access = identity_check(plan["target"]) + if any(access.get(key) is not True for key in ("tenantMatches", "subscriptionMatches", "workspaceMatches", "nonProduction", "armDeploymentAllowed", "disabledContentVerified", "ownershipVerified")) or ( + any(item["import"] for item in plan["resources"]) and access.get("extensionResourceWriteAllowed") is not True + ): + return {"status": "blocked", "writeAttempted": False, "accepted": False, "identityCheck": dict(access)} + identity_fields = ("tenantId", "subscriptionId", "workspaceResourceId") + if not isinstance(access.get("identityId"), str) or not access["identityId"].strip() or any( + str(access.get(key, "")).casefold() != plan["target"][key].casefold() for key in identity_fields + ): + return {"status": "blocked", "writeAttempted": False, "accepted": False, "identityCheck": dict(access)} + # Re-read approval and all evidence after access checks, immediately before reservation. + current = refresh_candidate() + current = refresh_arm_candidate(context, current) + fresh = plan_lab_arm_validation(context, current, target=target, selected_flags=selected_flags, expected_workbooks=expected_workbooks) + scope = check_lab_arm_scope(fresh, read_consent()) + if fresh["planSha256"] != expected_plan_sha256 or scope["status"] != "ready": + raise ValueError("candidate/approved scope changed immediately before write") + snapshot = directory / f"arm-validation-template-{plan['mainTemplateSha256']}.json" + _write_template_snapshot(snapshot, content_path(root, current["artifactPaths"]["mainTemplate"]).read_bytes()) + report = { + "schemaVersion": 1, "status": "intent-reserved", "planSha256": plan["planSha256"], + "candidateSha256": candidate["candidateSha256"], "target": plan["target"], + "solutionPath": str(root), + "actor": actor.strip(), "approvalReference": approval_reference.strip(), + "approvedAt": _utc_now(), "writeAttempted": False, "accepted": False, + "installationVerified": False, "registrationVerified": False, + "uiVerified": False, "parityVerified": False, "retryAuthorized": False, + "workbookContractRequired": bool(plan["expectedWorkbooks"]), + "mainTemplateSha256": plan["mainTemplateSha256"], "parameters": plan["parameters"], + "packageIdentity": plan["packageIdentity"], + "scopeConsentReference": None, + "attemptPath": str(attempt_path), + } + from .workflow import _run_writer_lock + from .runtime_consent import _atomic_write_json + + guard_path = directory / "arm-validation-executor.json" + with _run_writer_lock(directory): + if guard_path.exists() or guard_path.is_symlink(): + if guard_path.is_symlink(): + raise ValueError("executor receipt cannot be a symlink") + prior_guard = json.loads(guard_path.read_text(encoding="utf-8")) + prior_path = Path(prior_guard["attemptPath"]) + if prior_path.parent != directory or prior_path.is_symlink(): + raise ValueError("executor receipt does not bind a safe attempt") + prior = json.loads(prior_path.read_text(encoding="utf-8")) + if prior.get("status") not in {"failed", "arm-accepted-contract-pending"} or not prior.get("reconciliation"): + raise ValueError("previous executor requires independent reconciliation before any changed candidate/retry") + _reserve_attempt(attempt_path, report) + _atomic_write_json(guard_path, {"attemptPath": str(attempt_path), "identity": identity}) + + def persist() -> None: + _atomic_write_json(attempt_path, report) + + deployment_name = "workbook-arm-" + identity[:32] + command = ["deployment", "group", "create", "--name", deployment_name, + "--subscription", plan["target"]["subscriptionId"], "--resource-group", plan["target"]["resourceGroup"], + "--template-file", str(snapshot), "--mode", "Incremental", "--parameters", + *(f"{name}={str(value).lower() if type(value) is bool else value}" for name, value in sorted(plan["parameters"].items())), + "--output", "json", "--only-show-errors"] + # Reservation happens before submission. A late revocation/freshness/access + # failure remains durable and cannot become an unrecorded cloud attempt. + try: + current = refresh_arm_candidate(context, refresh_candidate()) + immediate = plan_lab_arm_validation(context, current, target=target, selected_flags=selected_flags, expected_workbooks=expected_workbooks) + latest_access = identity_check(plan["target"]) + latest_consent = read_consent() + if immediate["planSha256"] != expected_plan_sha256 or check_lab_arm_scope(immediate, latest_consent)["status"] != "ready" or _digest_file(snapshot) != plan["mainTemplateSha256"]: + raise ValueError("scope/evidence changed at dispatch boundary") + access_fields = (*identity_fields, "identityId", "tenantMatches", "subscriptionMatches", "workspaceMatches", + "nonProduction", "armDeploymentAllowed", "disabledContentVerified", "ownershipVerified", + "extensionResourceWriteAllowed") + if any(latest_access.get(key) != access.get(key) for key in access_fields): + raise ValueError("execution identities/access changed at dispatch boundary") + report["scopeConsentReference"] = latest_consent["approvalReference"] + except Exception as exc: + report.update(status="blocked-before-write", error=f"{type(exc).__name__}: current approval/evidence/access check failed") + persist() + return report + report.update(status="write-intent-persisted", writeAttempted=True, deploymentName=deployment_name) + persist() + try: + code, response, error = command_runner(command) + state = _deployment_provisioning_state(response) + status = "arm-accepted-contract-pending" if code == 0 and state == "succeeded" else ("failed" if code and state == "succeeded" else _error_status(state)) + report.update(status=status, commandExitCode=code, provisioningState=_recorded_provisioning_state(response), + deploymentId=response.get("id") if isinstance(response, dict) else None, error=error or None) + except Exception as exc: + report.update(status="unknown", error=f"{type(exc).__name__}: write outcome requires reconciliation") + report["completedAt"] = _utc_now() + persist() + return report + + +def reconcile_lab_arm_attempt(attempt_path: str | Path, *, observation: Mapping[str, Any], actor: str, reference: str) -> dict[str, Any]: + """Record an exact independent deployment GET; never authorizes a retry.""" + path = Path(attempt_path) + if path.is_symlink() or not path.is_file() or not actor.strip() or not reference.strip(): + raise ValueError("reconciliation requires an existing attempt, actor and evidence reference") + record = json.loads(path.read_text(encoding="utf-8")) + expected_id = ( + f"/subscriptions/{record['target']['subscriptionId']}/resourceGroups/{record['target']['resourceGroup']}" + f"/providers/Microsoft.Resources/deployments/{record.get('deploymentName')}" + ) + if observation.get("httpMethod") != "GET" or str(observation.get("deploymentId", "")).casefold() != expected_id.casefold() or observation.get("target") != record["target"] or not isinstance(observation.get("responseSha256"), str) or not _HASH.fullmatch(observation["responseSha256"]): + raise ValueError("reconciliation is not bound to independent exact deployment readback") + outcome = str(observation.get("provisioningState") or "").casefold() + if outcome not in {"succeeded", "failed", "canceled", "cancelled"} or observation.get("status") != "passed": + raise ValueError("nonterminal/denied/unknown readback cannot reconcile an attempt") + response_path = content_path(Path(record["solutionPath"]), observation.get("responsePath")) + if not response_path.is_relative_to(Path(record["solutionPath"]) / "Logs") or _digest_file(response_path) != observation["responseSha256"]: + raise ValueError("reconciliation raw response bytes are missing or stale") + response = json.loads(response_path.read_text(encoding="utf-8-sig")) + if str(response.get("id", "")).casefold() != expected_id.casefold() or _deployment_provisioning_state(response) != outcome: + raise ValueError("reconciliation raw deployment identity/outcome mismatched") + record["reconciliation"] = {**observation, "actor": actor.strip(), "reference": reference.strip(), "recordedAt": _utc_now()} + record["status"] = "arm-accepted-contract-pending" if outcome == "succeeded" else "failed" + record["accepted"] = False + record["retryAuthorized"] = False + from .runtime_consent import _atomic_write_json + from .workflow import _run_writer_lock + with _run_writer_lock(path.parent): + current = json.loads(path.read_text(encoding="utf-8")) + if current.get("completedAt") != record.get("completedAt") or current.get("status") not in {"unknown", "nonterminal", "failed", "arm-accepted-contract-pending"}: + raise ValueError("active/changed attempt cannot be reconciled") + _atomic_write_json(path, record) + return record + + +def verify_workbook_readback(plan: Mapping[str, Any], evidence: Mapping[str, Any]) -> dict[str, Any]: + """Require independent install, registration and repeat/update observations. + + Expected workbook records are plan-bound by the integration adapter. Each + observation supplies a solution-relative raw response path/hash and distinct + exact GET operation identity (not ARM outputs or caller-asserted summaries). + """ + from .packaging import package_input_identity + + reasons = [] + if not _sealed(plan, "planSha256") or evidence.get("planSha256") != plan.get("planSha256") or evidence.get("target") != plan.get("target"): + reasons.append("readback plan/target binding is missing or stale") + try: + root = Path(plan["solutionPath"]) + identity = plan["packageIdentity"] + if package_input_identity(root)["sha256"] != identity["packageInputSha256"]: + raise ValueError("readback refers to stale candidate inputs") + for name, digest in identity["packageArtifactHashes"].items(): + if _digest_file(content_path(root, plan["artifactPaths"][name])) != digest: + raise ValueError("readback refers to stale candidate artifacts") + for reference in plan["prerequisiteEvidence"].values(): + if _digest_file(content_path(root, reference["path"])) != reference["sha256"]: + raise ValueError("readback prerequisite evidence changed") + fingerprints = plan["validationReceiptFingerprint"] + for path_key, hash_key in (("path", "sha256"), ("bindingPath", "bindingSha256")): + path_value = fingerprints.get(path_key) + if path_value is not None and _digest_file(Path(path_value)) != fingerprints.get(hash_key): + raise ValueError("readback static receipt/binding changed") + except (KeyError, TypeError, ValueError, OSError) as exc: + reasons.append(f"readback current candidate/evidence binding failed ({type(exc).__name__})") + expected = plan.get("expectedWorkbooks") + if not isinstance(expected, list) or not expected: + reasons.append("no exact expected workbook install/registration contract") + if evidence.get("armStatus") != "succeeded" or not evidence.get("deploymentId"): + reasons.append("terminal ARM acceptance evidence is missing") + observations = evidence.get("workbooks") + if not isinstance(observations, list) or not isinstance(expected, list) or len(observations) != len(expected): + reasons.append("workbook readback inventory is incomplete") + else: + seen_operations: set[str] = set() + seen_writes: set[str] = set() + for wanted, observed in zip(expected, observations): + if wanted.get("contentId") == wanted.get("instanceId") or not wanted.get("contentId") or not wanted.get("instanceId"): + reasons.append("distinct canonical content/installation identity required") + for phase in ("installation", "registration", "repeatInstallation", "update"): + result = observed.get(phase) or {} + expected_phase = wanted.get(phase) + operation = result.get("operationId") + if not isinstance(expected_phase, dict) or not expected_phase: + reasons.append(f"missing exact expected {phase} contract") + if result.get("status") != "passed" or result.get("httpMethod") != "GET" or result.get("provider") not in {"azure-resource-manager", "microsoft-security"} or not isinstance(result.get("responseSha256"), str) or not _HASH.fullmatch(result["responseSha256"]) or not isinstance(operation, str) or not operation or operation in seen_operations: + reasons.append(f"{phase} independent readback failed/denied/unknown") + continue + seen_operations.add(operation) + if phase != "registration": + write_id = result.get("writeOperationId") + if result.get("writeStatus") != "succeeded" or not isinstance(write_id, str) or not write_id or write_id in seen_writes: + reasons.append(f"{phase} requires a distinct successful bounded installation/update operation") + else: + seen_writes.add(write_id) + try: + attempt_path = content_path(Path(plan["solutionPath"]), result.get("writeAttemptPath")) + if not attempt_path.is_relative_to(Path(plan["solutionPath"]) / "Logs") or _digest_file(attempt_path) != result.get("writeAttemptSha256"): + raise ValueError("installation/update attempt bytes are stale") + attempt = json.loads(attempt_path.read_text(encoding="utf-8-sig")) + if ( + attempt.get("status") != "arm-accepted-contract-pending" + or attempt.get("target") != plan["target"] + or attempt.get("mainTemplateSha256") != plan["mainTemplateSha256"] + or attempt.get("parameters") != plan["parameters"] + or attempt.get("deploymentId") != write_id + or (attempt.get("packageIdentity") or {}).get("packageInputSha256") != plan["packageIdentity"]["packageInputSha256"] + ): + raise ValueError("installation/update operation does not bind current approved candidate") + except (KeyError, TypeError, ValueError, OSError) as exc: + reasons.append(f"{phase} successful bounded attempt could not be verified ({type(exc).__name__})") + try: + response_path = content_path(Path(plan["solutionPath"]), result.get("responsePath")) + if not response_path.is_relative_to(Path(plan["solutionPath"]) / "Logs") or _digest_file(response_path) != result["responseSha256"]: + raise ValueError("readback raw response hash/path is stale") + response = json.loads(response_path.read_text(encoding="utf-8-sig")) + props = response["properties"] + resource_id = response["id"] + if not isinstance(props, dict) or not isinstance(resource_id, str) or result.get("requestResourceId") != resource_id: + raise ValueError("readback does not match its exact GET resource identity") + if phase == "registration": + prefix = plan["target"]["workspaceResourceId"] + "/providers/Microsoft.SecurityInsights/contentTemplates/" + if not resource_id.casefold().startswith(prefix.casefold()) or result["provider"] != "azure-resource-manager": + raise ValueError("registration readback is not in selected workspace") + actual = {key: props.get(key) for key in expected_phase if key != "storedTemplateSha256"} + actual["storedTemplateSha256"] = _canonical_sha256(props.get("mainTemplate")) + else: + prefix = f"/tenants/{plan['target']['tenantId']}/providers/Microsoft.Security/workbooks/" + if resource_id.casefold() != (prefix + wanted["instanceId"]).casefold() or result["provider"] != "microsoft-security": + raise ValueError("installation readback crosses selected tenant or instance identity") + notebook = props["content"] + actual = {"contentId": notebook["contentProvenance"]["contentId"], + "instanceId": props["id"], "contentSha256": _canonical_sha256(notebook), + "contentProvenance": notebook["contentProvenance"]} + if actual != expected_phase: + raise ValueError("readback content identity/provenance mismatched") + except (KeyError, TypeError, ValueError, OSError) as exc: + reasons.append(f"{phase} raw independent readback could not verify contract ({type(exc).__name__})") + return {"status": "verified" if not reasons else "blocked", "accepted": not reasons, + "installationVerified": not reasons, "registrationVerified": not reasons, + "repeatInstallUpdateVerified": not reasons, "uiVerified": False, "parityVerified": False, "reasons": reasons} diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_workflow.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_workflow.py new file mode 100644 index 00000000000..a167984f89a --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_workflow.py @@ -0,0 +1,793 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from .artifacts import ( + existing_artifact_path, + report_directory, + resolve_artifact_reference, +) +from .content_paths import content_path + + +CAPABILITY_VERSION = 1 +ARM_VALIDATION_METHODS = ("arm-ttk", "lab-deployment") + + +def enabled(state: dict[str, Any]) -> bool: + return state.get("context", {}).get("workbookCapabilityVersion") == CAPABILITY_VERSION + + +def has_analytic_rules(solution: str | Path) -> bool: + root = Path(solution).expanduser().resolve() + from .converter import analytic_rule_files + + return ( + any((root / name).is_dir() for name in ("Analytic Rules", "Analytics Rules")) + and bool(analytic_rule_files(root)) + ) + + +def _digest(value: Any) -> str: + return hashlib.sha256( + json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + + +def _inventory(solution: Path) -> list[dict[str, str]]: + from .workbooks import inventory_workbooks + + rows = inventory_workbooks(solution) + inventory = [ + {"path": row["path"], "sha256": row["sha256"]} + for row in rows + ] + inventory.sort(key=lambda row: row["path"]) + if len({row["path"] for row in inventory}) != len(inventory): + raise ValueError("workbook inventory contains duplicate source paths") + return inventory + + +def _data(solution: Path) -> tuple[Path, dict[str, Any]]: + from .packaging import _read_json, _solution_data_file + + path = _solution_data_file(solution / "Data") + return path, _read_json(path) + + +def selected_workbook_references( + solution: str | Path, + document: dict[str, Any] | None = None, + *, + require_existing: bool = True, +) -> list[Path]: + root = Path(solution).expanduser().resolve() + from .packaging import _solution_relative_reference + + if document is None: + _, document = _data(root) + keys = [key for key in document if key.casefold() == "xdr workbooks"] + if len(keys) > 1: + raise ValueError("solution Data has duplicate XDR Workbooks selection keys") + if not keys: + return [] + references = document[keys[0]] + if isinstance(references, str): + references = json.loads(references) + if not isinstance(references, list) or any( + not isinstance(reference, str) or not reference.strip() + for reference in references + ): + raise ValueError("solution Data XDR Workbooks must be a content-reference array") + paths = [] + for reference in references: + if require_existing: + path = _solution_relative_reference(root, reference, "XDR Workbooks") + else: + normalized = reference.replace("\\", "/") + parts = normalized.split("/") + if parts[:2] == ["Solutions", root.name]: + normalized = "/".join(parts[2:]) + path = content_path(root, normalized) + paths.append(path) + if len(set(paths)) != len(paths): + raise ValueError("solution Data selects the same Defender workbook more than once") + for path in paths: + if not path.is_relative_to(root / "XDR Workbooks") or path.suffix.lower() != ".json": + raise ValueError("XDR Workbooks selections must reference authored workbook JSON") + return paths + + +def freeze_inventory(solution: str | Path) -> dict[str, Any]: + root = Path(solution).expanduser().resolve() + inventory = _inventory(root) + included = [row["path"] for row in inventory] + excluded: list[dict[str, str]] = [] + explicit_selection = False + if (root / "Data").is_dir(): + _, document = _data(root) + if any(key.casefold() == "xdr workbooks" for key in document): + explicit_selection = True + included = [] + known_sources = {row["path"] for row in inventory} + for path in selected_workbook_references(root, document): + target = json.loads(path.read_text(encoding="utf-8-sig")) + source = target.get("contentProvenance", {}).get("source", {}).get("path") + if source not in known_sources or source in included: + raise ValueError("explicit workbook selection does not match the source inventory") + expected_output = root / "XDR Workbooks" / source.removeprefix("Workbooks/") + if path != expected_output: + raise ValueError("explicit workbook selection must preserve the original relative workbook path") + included.append(source) + excluded = [ + {"path": row["path"], "reason": "Excluded by the explicit solution Data selection."} + for row in inventory + if row["path"] not in included + ] + selection = {"included": sorted(included), "excluded": excluded} + return { + "schemaVersion": "1.0.0", + "inventory": inventory, + "inventorySha256": _digest(inventory), + "selection": selection, + "selectionSha256": _digest(selection), + "selectionExplicit": explicit_selection, + } + + +def verify_inventory(solution: str | Path, state: dict[str, Any]) -> None: + if not enabled(state): + return + frozen = state["workbooks"] + current = _inventory(Path(solution).expanduser().resolve()) + if current != frozen["inventory"] or _digest(current) != frozen["inventorySha256"]: + raise ValueError( + "frozen workbook inventory changed; preserve this run and initialize a new run" + ) + selection = frozen["selection"] + if _digest(selection) != frozen["selectionSha256"]: + raise ValueError("frozen workbook selection changed; initialize a new run") + included = selection["included"] + excluded = [row["path"] for row in selection["excluded"]] + if ( + len(set(included + excluded)) != len(included + excluded) + or set(included + excluded) != {row["path"] for row in current} + or any(not row["reason"].strip() for row in selection["excluded"]) + ): + raise ValueError("workbook selection must account for the entire frozen inventory") + root = Path(solution).expanduser().resolve() + if (root / "Data").is_dir(): + _, document = _data(root) + present = any(key.casefold() == "xdr workbooks" for key in document) + if present: + expected = {root / "XDR Workbooks" / reference.removeprefix("Workbooks/") + for reference in included} + if set(selected_workbook_references( + root, document, + require_existing=state["stages"]["conversion"]["status"] == "passed", + )) != expected: + raise ValueError("solution Data workbook selection differs from the frozen run selection") + elif frozen["selectionExplicit"] or ( + current and state["stages"]["conversion"]["status"] == "passed" + ): + raise ValueError("the frozen explicit workbook selection cannot be removed") + + +def bind_conversion( + solution: str | Path, + state: dict[str, Any], +) -> dict[str, str]: + if not enabled(state): + return {} + verify_inventory(solution, state) + if not state["workbooks"]["inventory"]: + return {} + root = Path(solution).expanduser().resolve() + path = report_directory(root) / "workbooks-manifest.json" + manifest = json.loads(path.read_text(encoding="utf-8-sig")) + inventory = {row["path"]: row["sha256"] for row in state["workbooks"]["inventory"]} + results = manifest.get("results") + if ( + not isinstance(results, list) + or any(not isinstance(row, dict) for row in results) + or len(results) != len(inventory) + or {row.get("sourceRelativePath") for row in results} != set(inventory) + ): + raise ValueError("workbook conversion manifest must retain the complete frozen inventory") + for row in results: + if row.get("sourceSha256") != inventory[row["sourceRelativePath"]]: + raise ValueError("workbook conversion manifest has stale source evidence") + if row.get("outputRelativePath"): + output = content_path(root, row["outputRelativePath"]) + if ( + not output.is_relative_to(root / "XDR Workbooks") + or not output.is_file() + or hashlib.sha256(output.read_bytes()).hexdigest() != row.get("outputSha256") + ): + raise ValueError("workbook conversion manifest has stale target evidence") + if row.get("status") == "converted": + authored = json.loads(output.read_bytes()) + provenance = authored.get("contentProvenance") or {} + conversion = provenance.get("conversion") or {} + if ( + provenance.get("source", {}).get("path") != row["sourceRelativePath"] + or conversion.get("queries") != row.get("queries") + or conversion.get("reviewRequired") != row.get("reviewRequired") + or row.get("managedOutputSha256") != row.get("outputSha256") + ): + raise ValueError("workbook manifest per-query review evidence differs from the current authored bytes") + if row["sourceRelativePath"] in state["workbooks"]["selection"]["included"]: + from .workbook_functions import validate_functions + validate_functions(root, provenance.get("requiredWorkspaceFunctions", [])) + binding = { + "workbookManifest": path.relative_to(root).as_posix(), + "workbookManifestSha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "workbookSelectionSha256": _digest(state["workbooks"]["selection"]), + } + function_records = [] + for row in results: + if row["sourceRelativePath"] in state["workbooks"]["selection"]["included"] and row.get("outputRelativePath"): + authored = json.loads(content_path(root, row["outputRelativePath"]).read_bytes()) + records = authored.get("contentProvenance", {}).get("requiredWorkspaceFunctions", []) + if records: + function_records.append({"workbook": row["sourceRelativePath"], "functions": records}) + if function_records: + binding["workbookFunctionDependenciesSha256"] = _digest(function_records) + return binding + + +def validate_evidence(solution: str | Path, state: dict[str, Any]) -> dict[str, Any]: + if not enabled(state): + return {"total": 0, "valid": 0, "invalid": 0, "results": []} + root = Path(solution).expanduser().resolve() + current = bind_conversion(root, state) + recorded = state["stages"]["conversion"].get("artifacts") or {} + if any(current[key] != recorded.get(key) for key in current): + raise ValueError("workbook conversion evidence is missing, changed, or bound to another selection") + included = state["workbooks"]["selection"]["included"] + if not state["workbooks"]["inventory"]: + return {"total": 0, "valid": 0, "invalid": 0, "results": []} + manifest_path = resolve_artifact_reference(root, current["workbookManifest"]) + manifest = json.loads(manifest_path.read_text(encoding="utf-8-sig")) + selected = [ + row for row in manifest["results"] if row["sourceRelativePath"] in included + ] + if any( + row.get("status") != "converted" or row.get("reviewRequired") is not False + for row in selected + ): + raise ValueError("selected workbooks have unresolved conversion or review blockers") + expected = {content_path(root, row["outputRelativePath"]) for row in selected} + if set(selected_workbook_references(root)) != expected: + raise ValueError("solution Data workbook selection differs from the frozen reviewed selection") + from .workbooks import validate_workbooks + + result = validate_workbooks(root, selected_only=True) + if ( + result.get("total") != len(selected) + or result.get("valid") != len(selected) + or result.get("invalid") != 0 + or len(result.get("results") or []) != len(selected) + or any(row.get("valid") is not True for row in result.get("results") or []) + ): + raise ValueError("selected workbooks must individually pass current structural validation") + return result + + +def inspect_content(solution: str | Path) -> dict[str, Any]: + from .converter import inspect_solution + from .workbooks import inventory_workbooks + + root = Path(solution).expanduser().resolve() + state = current_state(root) + if not state or not enabled(state): + return inspect_solution(root) + verify_inventory(root, state) + result = inspect_solution(root) if has_analytic_rules(root) else { + "solution": str(root), "analyticRuleCount": 0, "analyticRules": [], + "existingXdrDetectionCount": 0, + } + inventory = inventory_workbooks(root) + return {**result, "workbookCount": len(inventory), + "workbookQueryCount": sum(len(row["queries"]) for row in inventory), + "workbooks": inventory, "workbookSelection": state["workbooks"]["selection"]} + + +def convert_content(solution: str | Path, *, overwrite: bool = False, + config_path: str | Path | None = None, + rule_id: str | None = None, + overwrite_workbooks: bool = False) -> dict[str, Any]: + from .converter import convert_solution, load_config + from .workbooks import convert_workbooks + + root = Path(solution).expanduser().resolve() + state = current_state(root) + if not state or not enabled(state): + if overwrite_workbooks: + raise ValueError("--overwrite-workbooks requires a workbook-capable frozen workflow run.") + return convert_solution(root, overwrite=overwrite, config_path=config_path, rule_id=rule_id) + verify_inventory(root, state) + if rule_id: + if overwrite_workbooks: + raise ValueError("--overwrite-workbooks cannot be combined with rule-scoped conversion.") + # Rule-scoped repairs must never rewrite workbook evidence or selection. + return convert_solution(root, overwrite=overwrite, config_path=config_path, rule_id=rule_id) + from .packaging import solution_data_version, validate_v31_effective_version, validate_v31_target_version + selected_version = state["context"].get("targetVersion") + bump = state["context"].get("versionBump") + if state["workbooks"]["inventory"] and (selected_version is not None or bump is not None): + selected_version = validate_v31_effective_version( + solution_data_version(root), bump or "none", selected_version, + ) + config = load_config(root / "XDR Detections", config_path) if state["workbooks"]["inventory"] else {} + if "targetVersion" in config: + validate_v31_target_version(config["targetVersion"]) + if selected_version and config.get("targetVersion") not in (None, selected_version): + raise ValueError("workbook targetVersion conflicts with the selected solution release") + result = convert_solution(root, overwrite=overwrite, config_path=config_path) if has_analytic_rules(root) else { + "solution": str(root), "total": 0, "converted": 0, "blocked": 0, "results": [], + "needsReview": 0, "conflicts": 0, + } + if not state["workbooks"]["inventory"]: + return {**result, "workbooks": {"total": 0, "converted": 0, "blocked": 0, + "selectedBlocked": 0, "results": []}, + "workbookArtifacts": {}, "workbookExcluded": []} + if selected_version is None: + current_version = solution_data_version(root) + try: + selected_version = validate_v31_effective_version(current_version, "none") + except ValueError: + # Older solution releases are not silently promoted by workbook conversion. + pass + if selected_version: + if config.get("targetVersion") not in (None, selected_version): + raise ValueError("workbook targetVersion conflicts with the selected solution release") + config["targetVersion"] = selected_version + workbook_result = convert_workbooks(root, config=config, + report_directory=report_directory(root), + overwrite=overwrite or overwrite_workbooks) + binding = bind_conversion(root, state) + manifest = json.loads(Path(workbook_result["manifestPath"]).read_bytes()) + selected_sources = set(state["workbooks"]["selection"]["included"]) + selected_rows = [row for row in manifest["results"] if row["sourceRelativePath"] in selected_sources] + workbook_result["selectedBlocked"] = sum( + row.get("status") != "converted" or row.get("reviewRequired") is not False + for row in selected_rows + ) + data_path, data = _data(root) + if not any(key.casefold() == "xdr workbooks" for key in data): + # Preserve blocked sources in the selection; validation must reject them. + data["XDR Workbooks"] = [row["outputRelativePath"] for row in selected_rows] + from .workflow import _atomic_json + _atomic_json(data_path, data) + expected = {content_path(root, row["outputRelativePath"]) for row in selected_rows} + if set(selected_workbook_references(root)) != expected: + raise ValueError("edited workbook selection conflicts with the frozen run selection") + return {**result, "workbooks": workbook_result, "workbookArtifacts": binding, + "workbookExcluded": state["workbooks"]["selection"]["excluded"]} + + +def validate_content(solution: str | Path) -> dict[str, Any]: + from .converter import validate_solution + + root = Path(solution).expanduser().resolve() + state = current_state(root) + if not state or not enabled(state): + return validate_solution(root) + verify_inventory(root, state) + result = validate_solution(root) if has_analytic_rules(root) else { + "solution": str(root), "total": 0, "valid": 0, "invalid": 0, "results": [], + } + workbook_result = validate_evidence(root, state) + return {**result, "workbooks": workbook_result, + "workbookExcluded": state["workbooks"]["selection"]["excluded"]} + + +def _context(state: dict[str, Any]) -> dict[str, Any]: + if not enabled(state) or not state.get("runId"): + raise ValueError("candidate lifecycle requires an explicitly selected workbook-capable run") + context = {**state["context"], "runId": state["runId"]} + if context.get("workspaceResourceId"): + from .target_context import workspace_identity + context["resourceGroup"] = workspace_identity(context["workspaceResourceId"])["resourceGroup"] + return context + + +def _plan_target(plan: dict[str, Any]) -> dict[str, Any]: + target = dict(plan["target"]) + if plan.get("parameters", {}).get("workspace-location"): + target["workspaceLocation"] = plan["parameters"]["workspace-location"] + return target + + +def _read_run_json(solution: str | Path, name: str) -> dict[str, Any]: + path = existing_artifact_path(solution, name) + if path.is_symlink() or not path.is_file(): + raise ValueError(f"selected run is missing {name}") + value = json.loads(path.read_text(encoding="utf-8-sig")) + if not isinstance(value, dict): + raise ValueError(f"invalid selected-run artifact: {name}") + return value + + +def _write_run_json(solution: str | Path, name: str, value: dict[str, Any]) -> Path: + from .artifacts import artifact_path + from .workflow import _atomic_json + path = artifact_path(solution, name, create_parent=True) + _atomic_json(path, value) + return path + + +def create_candidate(solution: str | Path, *, validation_receipt: str | Path | None = None, + toolkit_root: str | Path | None = None) -> dict[str, Any]: + """Produce gates only after deterministic checks, and verify the official receipt.""" + import shutil + import subprocess + import uuid + from . import workflow + from .artifacts import repository_root + from .packaging import package_input_identity, _read_json, _verify_workbook_package + from .workbook_validation import create_arm_candidate + + root = Path(solution).expanduser().resolve() + with workflow._state_writer_lock(root) as state_path: + state = workflow._load(root)[1] + context = _context(state) + verify_inventory(root, state) + validate_evidence(root, state) + if state["stages"]["conversion"]["status"] != "passed" or state["stages"]["validation"]["status"] != "passed": + raise ValueError("candidate creation requires passed conversion and content validation") + if context["workflowProfile"] == "authoring": + workflow._validate_offline_authoring_gates(root, state_path, state) + else: + if has_analytic_rules(root): + from .runtime import current_runtime_evidence + if current_runtime_evidence(root).get("passed") is not True: + raise ValueError("combined Qualification still requires all current detection runtime gates") + elif context.get("workbookQualificationAfterArm") is not True: + from .workbook_qualification import require_current_qualification + require_current_qualification(root, state) + artifacts = state["stages"]["packaging"].get("artifacts") or {} + workflow._validate_packaging_evidence(state, artifacts, root, allow_candidate=True) + report_path = resolve_artifact_reference(root, artifacts["packageReport"]) + report = _read_json(report_path) + template = _read_json(resolve_artifact_reference(root, report["mainTemplate"])) + _verify_workbook_package(root, selected_workbook_references(root), template, + resolve_artifact_reference(root, report["zip"])) + current = package_input_identity(root) + if report["packageInputs"]["after"]["sha256"] != current["sha256"]: + raise ValueError("candidate package input identity is stale") + official_report = None + if validation_receipt is None: + toolkit = Path(toolkit_root).expanduser().resolve() if toolkit_root else repository_root(root) + script = toolkit / ".script" / "local-validation" / "build-and-validate.ps1" + pwsh = shutil.which("pwsh") + if not pwsh or not script.is_file(): + raise ValueError("candidate validation requires the official build-and-validate producer and PowerShell 7") + evidence = toolkit / ".script" / "local-validation" / "Logs" / f"candidate-{state['runId']}-{uuid.uuid4().hex}" + command = [pwsh, "-NoProfile", "-File", str(script), "-SolutionName", root.name, + "-ContentRoot", str(repository_root(root)), "-EvidenceDirectory", str(evidence), + "-SkipPackaging", "-PackageReportPath", str(report_path), + "-ValidationSkip", "kql,detection-schema,non-ascii", + "-ArmValidationMethod", context["armValidationMethod"]] + completed = subprocess.run(command, cwd=toolkit, capture_output=True, text=True, check=False) + official_report = completed.stdout + if completed.returncode: + raise RuntimeError(f"official candidate static validation failed ({completed.returncode}): {completed.stdout[-4000:]}\n{completed.stderr[-1000:]}") + validation_receipt = evidence / "build-and-validate-report.md" + gate_names = ["conversion", "review", "structure", "package"] + if context["workflowProfile"] == "qualification" and context.get("workbookQualificationAfterArm") is not True: + gate_names.append("qualification") + gates = {} + for name in gate_names: + gate = { + "status": "passed", "gate": name, "workflowRunId": state["runId"], + "packageInputSha256": current["sha256"], + "checks": { + "conversion": "current full source inventory and hash-bound conversion", + "review": "selected outputs have no unresolved technical/dependency/query review findings", + "structure": "current deterministic content structure and full package workbook projection", + "package": "V3.1 current attempt artifacts, selections and ZIP", + "qualification": "Existing detection runtime prerequisites; workbook query/UI qualification remains post-ARM.", + }[name], + } + if name in {"structure", "package"}: + gate["packageArtifactsSha256"] = report["packageArtifactsSha256"] + if name == "qualification": + from .target_context import workspace_identity + gate["qualificationRuntimeVerified"] = True + gate["target"] = {key: context[key] for key in + ("tenantId", "subscriptionId", "workspaceResourceId", "workspaceCustomerId")} + gate["target"]["resourceGroup"] = workspace_identity(context["workspaceResourceId"])["resourceGroup"] + gate_path = _write_run_json(root, f"arm-candidate-gate-{name}.json", gate) + gates[name] = {"path": gate_path.relative_to(root).as_posix(), + "sha256": hashlib.sha256(gate_path.read_bytes()).hexdigest()} + candidate = create_arm_candidate(context, solution=root, package_report=report_path.relative_to(root).as_posix(), + validation_receipt=validation_receipt, prerequisite_evidence=gates) + candidate_path = _write_run_json(root, "arm-candidate.json", candidate) + state["stages"]["packaging"]["artifacts"].update({ + "armCandidate": str(candidate_path), + "armCandidateSha256": hashlib.sha256(candidate_path.read_bytes()).hexdigest(), + }) + workflow._write_unlocked(state_path, state) + return {**candidate, "candidatePath": str(candidate_path), "officialReport": official_report} + + +def plan_candidate(solution: str | Path, *, target: dict[str, Any], + selected_flags: dict[str, bool], contract_only: bool = False) -> dict[str, Any]: + from .workbook_validation import (refresh_arm_candidate, plan_lab_arm_validation, + plan_workbook_contract_verification) + state = current_state(solution) + context = _context(state or {}) + candidate = refresh_arm_candidate(context, _read_run_json(solution, "arm-candidate.json")) + plan = (plan_workbook_contract_verification if contract_only else plan_lab_arm_validation)( + context, candidate, target=target, selected_flags=selected_flags) + _write_run_json(solution, "workbook-contract-plan.json" if contract_only else "arm-validation-plan.json", plan) + return plan + + +def review_scope(solution: str | Path, *, actor: str, reference: str, approved: bool, + expected_plan_sha256: str) -> dict[str, Any]: + from .workbook_validation import review_lab_arm_scope + plan = _read_run_json(solution, "arm-validation-plan.json") + if plan["planSha256"] != expected_plan_sha256: + raise ValueError("scope review requires the exact current displayed plan hash") + current = plan_candidate(solution, target=_plan_target(plan), selected_flags=plan["selectedFlags"]) + if current["planSha256"] != plan["planSha256"]: + raise ValueError("candidate scope changed before review") + consent = review_lab_arm_scope(current, actor=actor, approval_reference=reference, approved=approved) + _write_run_json(solution, "arm-validation-consent.json", consent) + return consent + + +def withdraw_scope(solution: str | Path, *, actor: str, reference: str) -> dict[str, Any]: + from .workbook_validation import withdraw_lab_arm_scope + consent = withdraw_lab_arm_scope(_read_run_json(solution, "arm-validation-consent.json"), + actor=actor, withdrawal_reference=reference) + _write_run_json(solution, "arm-validation-consent.json", consent) + return consent + + +def execute_candidate(solution: str | Path, *, expected_plan_sha256: str, actor: str, + reference: str, confirmation: str, identity_check=None, + command_runner=None) -> dict[str, Any]: + from .workbook_validation import execute_lab_arm_validation + from .arm_acceptance_support import _run_cli + from .workbook_preflight_workflow import production_identity_check, execution_profile + + state = current_state(solution) + context = _context(state or {}) + candidate = _read_run_json(solution, "arm-candidate.json") + plan = _read_run_json(solution, "arm-validation-plan.json") + profile = execution_profile() if identity_check is None or command_runner is None else None + + def dispatch(command: list[str]): + from .graph_client_config import azure_cli_profile + + if execution_profile() != profile: + raise ValueError("ARM execution authentication profile changed before dispatch") + with azure_cli_profile(profile): + return _run_cli(command) + + result = execute_lab_arm_validation( + context, candidate, target=_plan_target(plan), selected_flags=plan["selectedFlags"], + expected_plan_sha256=expected_plan_sha256, actor=actor, approval_reference=reference, + confirmation=confirmation, evidence_directory=report_directory(solution), + refresh_candidate=lambda: _read_run_json(solution, "arm-candidate.json"), + read_consent=lambda: _read_run_json(solution, "arm-validation-consent.json"), + identity_check=identity_check or (lambda target: production_identity_check(solution, target, profile=profile)), + command_runner=command_runner or dispatch, + ) + _write_run_json(solution, "arm-validation-result.json", result) + return result + + +def reconcile_attempt(solution: str | Path, *, attempt: str | Path, + observation: str | Path, actor: str, reference: str) -> dict[str, Any]: + from .workbook_validation import reconcile_lab_arm_attempt + root = Path(solution).expanduser().resolve() + attempt_path = content_path(root, attempt) + if attempt_path.parent != report_directory(root): + raise ValueError("attempt reconciliation must use the selected run") + observation_path = content_path(root, observation) + result = reconcile_lab_arm_attempt(attempt_path, observation=json.loads(observation_path.read_bytes()), + actor=actor, reference=reference) + _write_run_json(root, "arm-validation-result.json", result) + return result + + +def import_contract(solution: str | Path, *, evidence: str | Path, + contract_only: bool = False) -> dict[str, Any]: + from .workbook_validation import verify_workbook_readback, refresh_arm_candidate + from . import workflow + + root = Path(solution).expanduser().resolve() + with workflow._state_writer_lock(root) as path: + state = workflow._load(root)[1] + context = _context(state) + candidate = refresh_arm_candidate(context, _read_run_json(root, "arm-candidate.json")) + plan_name = "workbook-contract-plan.json" if contract_only else "arm-validation-plan.json" + plan = _read_run_json(root, plan_name) + current = plan_candidate(root, target=_plan_target(plan), selected_flags=plan["selectedFlags"], contract_only=contract_only) + if current["planSha256"] != plan["planSha256"]: + raise ValueError("contract evidence refers to a stale plan") + if len(plan["expectedWorkbooks"]) != len(state["workbooks"]["selection"]["included"]): + raise ValueError("contract plan must select every included Defender workbook") + source = content_path(root, evidence) + proof = json.loads(source.read_bytes()) + result = verify_workbook_readback(plan, proof) + proof_path = _write_run_json(root, "workbook-contract-evidence.json", proof) + result_path = _write_run_json(root, "workbook-contract-result.json", result) + if result["accepted"] is not True: + return result + artifacts = state["stages"]["packaging"]["artifacts"] + artifacts.update({"workbookContractPlan": str(existing_artifact_path(root, plan_name)), + "workbookContractEvidence": str(proof_path), + "workbookContractResult": str(result_path), + "packageLifecycleStatus": "accepted"}) + state["packaging"]["currentAttempt"]["status"] = "passed" + state["packaging"]["lastConsumedAction"]["packageSucceeded"] = True + state["stages"]["packaging"].update(status="passed", message="Selected ARM route and independently verified workbook contract accepted.") + report_path = resolve_artifact_reference(root, artifacts["packageReport"]) + report = json.loads(report_path.read_bytes()) + state["packaging"]["lastSuccessful"] = { + **state["packaging"]["lastConsumedAction"], "packageReport": str(report_path), + "packageReportSha256": hashlib.sha256(report_path.read_bytes()).hexdigest(), + "packageArtifactsSha256": report["packageArtifactsSha256"], "version": report["version"], + } + state["workflowStatus"] = "inProgress" + workflow._validate_packaging_evidence(state, artifacts, root) + workflow._write_unlocked(path, state) + return {**result, "runId": candidate["runId"], "next": workflow._next_stage_name(state)} + + +def require_accepted(solution: str | Path, state: dict[str, Any]) -> None: + from .workbook_validation import (refresh_arm_candidate, verify_workbook_readback, + plan_lab_arm_validation, plan_workbook_contract_verification) + artifacts = state["stages"]["packaging"].get("artifacts") or {} + candidate = refresh_arm_candidate(_context(state), _read_run_json(solution, "arm-candidate.json")) + if not state["workbooks"]["selection"]["included"]: + attempt = _read_run_json(solution, "arm-validation-result.json") + _verify_arm_only(solution, candidate, _read_run_json(solution, "arm-validation-plan.json"), attempt) + return + for key in ("workbookContractPlan", "workbookContractEvidence", "workbookContractResult"): + if not artifacts.get(key): + raise ValueError("candidate packaging is not accepted; independent workbook contract evidence is required") + plan = json.loads(resolve_artifact_reference(solution, artifacts["workbookContractPlan"]).read_bytes()) + fresh = (plan_workbook_contract_verification if plan.get("purpose") == "workbook-contract-readback" + else plan_lab_arm_validation)(_context(state), candidate, + target=_plan_target(plan), selected_flags=plan["selectedFlags"]) + if ( + fresh["planSha256"] != plan.get("planSha256") + or len(plan.get("expectedWorkbooks") or []) != len(state["workbooks"]["selection"]["included"]) + ): + raise ValueError("accepted workbook contract plan is not current for this run and selection") + proof = json.loads(resolve_artifact_reference(solution, artifacts["workbookContractEvidence"]).read_bytes()) + recorded = json.loads(resolve_artifact_reference(solution, artifacts["workbookContractResult"]).read_bytes()) + result = verify_workbook_readback(plan, proof) + if result.get("accepted") is not True or result != recorded: + raise ValueError("workbook contract acceptance is missing, blocked or stale") + + +def _verify_arm_only(solution: str | Path, candidate: dict[str, Any], + plan: dict[str, Any], attempt: dict[str, Any]) -> None: + from .arm_acceptance_support import _deployment_provisioning_state + from .workbook_validation import plan_lab_arm_validation + + state = current_state(solution) + fresh = plan_lab_arm_validation(_context(state or {}), candidate, + target=_plan_target(plan), selected_flags=plan["selectedFlags"]) + if ( + fresh["planSha256"] != plan["planSha256"] or plan["expectedWorkbooks"] + or attempt.get("planSha256") != plan["planSha256"] + or attempt.get("candidateSha256") != candidate["candidateSha256"] + or attempt.get("status") != "arm-accepted-contract-pending" + or attempt.get("target") != plan["target"] + or attempt.get("parameters") != plan["parameters"] + or attempt.get("packageIdentity") != plan["packageIdentity"] + ): + raise ValueError("ARM-only acceptance requires the exact current terminal attempt") + observation = attempt.get("reconciliation") or {} + if not observation.get("responsePath"): + raise ValueError("independent current ARM deployment GET reconciliation is required") + response_path = content_path(Path(solution).expanduser().resolve(), observation.get("responsePath")) + if ( + observation.get("httpMethod") != "GET" or observation.get("status") != "passed" + or observation.get("target") != plan["target"] + or hashlib.sha256(response_path.read_bytes()).hexdigest() != observation.get("responseSha256") + ): + raise ValueError("independent current ARM deployment GET reconciliation is required") + response = json.loads(response_path.read_bytes()) + expected_id = ( + f"/subscriptions/{plan['target']['subscriptionId']}/resourceGroups/{plan['target']['resourceGroup']}" + f"/providers/Microsoft.Resources/deployments/{attempt.get('deploymentName')}" + ) + if ( + str(response.get("id", "")).casefold() != expected_id.casefold() + or str(observation.get("deploymentId", "")).casefold() != expected_id.casefold() + or _deployment_provisioning_state(response) != "succeeded" + ): + raise ValueError("ARM deployment identity/outcome is not independently accepted") + + +def import_arm_only(solution: str | Path, *, attempt: str | Path, observation: str | Path, + actor: str, reference: str) -> dict[str, Any]: + from . import workflow + from .workbook_validation import refresh_arm_candidate + root = Path(solution).expanduser().resolve() + with workflow._state_writer_lock(root) as path: + state = workflow._load(root)[1] + _context(state) + if state["workbooks"]["selection"]["included"]: + raise ValueError("selected Defender workbooks require independent full contract evidence, not ARM-only acceptance") + candidate = refresh_arm_candidate(_context(state), _read_run_json(root, "arm-candidate.json")) + result = reconcile_attempt(root, attempt=attempt, observation=observation, actor=actor, reference=reference) + _verify_arm_only(root, candidate, _read_run_json(root, "arm-validation-plan.json"), result) + state["packaging"]["currentAttempt"]["status"] = "passed" + state["packaging"]["lastConsumedAction"]["packageSucceeded"] = True + stage = state["stages"]["packaging"] + stage.update(status="passed", message="Selected ARM method accepted by exact independent terminal deployment evidence.") + stage["artifacts"]["packageLifecycleStatus"] = "accepted" + report_path = resolve_artifact_reference(root, stage["artifacts"]["packageReport"]) + report = json.loads(report_path.read_bytes()) + state["packaging"]["lastSuccessful"] = { + **state["packaging"]["lastConsumedAction"], "packageReport": str(report_path), + "packageReportSha256": hashlib.sha256(report_path.read_bytes()).hexdigest(), + "packageArtifactsSha256": report["packageArtifactsSha256"], "version": report["version"], + } + state["workflowStatus"] = "inProgress" + workflow._validate_packaging_evidence(state, stage["artifacts"], root) + workflow._write_unlocked(path, state) + return {"status": "accepted", "accepted": True, "workbookContractApplicable": False, + "uiVerified": False, "parityVerified": False, "next": workflow._next_stage_name(state)} + + +def current_state(solution: str | Path) -> dict[str, Any] | None: + path = existing_artifact_path(solution, "workflow-state.json") + if not path.is_file(): + return None + from .workflow import _load + + return _load(solution)[1] + + +def summary(solution: str | Path, state: dict[str, Any]) -> dict[str, Any] | None: + if not enabled(state): + return None + workbook_state = state["workbooks"] + rows = {} + manifest_path = report_directory(solution) / "workbooks-manifest.json" + issues = [] + try: + verify_inventory(solution, state) + if manifest_path.is_file(): + bind_conversion(solution, state) + rows = {row["sourceRelativePath"]: row for row in json.loads(manifest_path.read_bytes())["results"]} + except (OSError, ValueError, KeyError, TypeError) as exc: + issues.append(str(exc)) + contract_support = "unverified" if workbook_state["selection"]["included"] else "notApplicable" + if workbook_state["selection"]["included"] and state["stages"]["packaging"]["status"] == "passed": + try: + require_accepted(solution, state) + contract_support = "verified" + except (OSError, ValueError, KeyError, TypeError): + issues.append("previous workbook acceptance evidence is no longer current") + return { + "inventoryCount": len(workbook_state["inventory"]), + "selectedCount": len(workbook_state["selection"]["included"]), + "excluded": workbook_state["selection"]["excluded"], + "inventorySha256": workbook_state["inventorySha256"], + "armValidationMethod": state["context"]["armValidationMethod"], + "contractSupport": contract_support, + "issues": issues, + "sources": [{ + **source, "selected": source["path"] in workbook_state["selection"]["included"], + "status": rows.get(source["path"], {}).get("status", "not-converted"), + "issues": rows.get(source["path"], {}).get("issues", []), + } for source in workbook_state["inventory"]], + "manifest": str(manifest_path) if manifest_path.is_file() else None, + "qualificationSeparateFromArmAndDetectionParity": True, + } diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbooks.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbooks.py new file mode 100644 index 00000000000..19702d6af1e --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbooks.py @@ -0,0 +1,578 @@ +"""Offline, loss-aware solution workbook inventory, conversion and validation. + +Reviewed resolutions bind exact original query templates, not scanner output. +Local validation establishes authored structure and freshness, never runtime, +installation, registration, UI fidelity or engine parity. +""" +from __future__ import annotations + +from copy import deepcopy +import hashlib +from importlib.resources import files as package_files +import json +import os +from pathlib import Path, PurePosixPath +import re +from uuid import UUID, NAMESPACE_URL, uuid4, uuid5 + +from jsonschema import Draft202012Validator + +from . import __version__ +from .artifacts import report_directory as resolve_report_directory +from .dependencies import dependency_validation_errors, workload_records, telemetry_data_types +from ._workbook_core import workbook, mapping_catalog + +MANIFEST = "workbooks-manifest.json" +REPORT = "workbooks-validation.json" + + +def _sha(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _root(solution) -> Path: + path = Path(solution).expanduser() + if any(p.is_symlink() for p in (path, *path.parents)) or not path.is_dir(): + raise ValueError("Solution must be an existing nonsymlink directory.") + return path.resolve() + + +def _path(root: Path, relative: str) -> Path: + if not isinstance(relative, str) or "\\" in relative: + raise ValueError("Content references must use solution-relative POSIX paths.") + parts = PurePosixPath(relative) + if parts.is_absolute() or not parts.parts or any(p in ("", "..", ".") for p in relative.split("/")): + raise ValueError(f"Unsafe content reference: {relative}") + path = root + for part in parts.parts: + path = path / part + if path.is_symlink(): + raise ValueError(f"Symlink content reference: {relative}") + if not path.resolve().is_relative_to(root): + raise ValueError(f"Content path escapes solution: {relative}") + return path + + +def _files(root: Path, directory: str) -> list[Path]: + base = _path(root, directory) + if not base.exists(): + return [] + if not base.is_dir(): + raise ValueError(f"{directory} must be a directory.") + paths = [] + for path in sorted(base.rglob("*")): + if path.is_symlink(): + raise ValueError(f"Symlink in workbook inventory: {path.relative_to(root)}") + if path.is_file() and path.suffix.lower() == ".json": + paths.append(path) + folded = [p.relative_to(root).as_posix().casefold() for p in paths] + if len(set(folded)) != len(folded): + raise ValueError("Case-insensitive workbook path collision.") + return paths + + +def _document(path: Path) -> dict: + value = json.loads(path.read_bytes()) + if not isinstance(value, dict): + raise ValueError("Workbook JSON must be an object.") + if value.get("version") != "Notebook/1.0" or not isinstance(value.get("items"), list): + raise ValueError("Workbook must contain Notebook/1.0 version and items array.") + return value + + +def _guid(value) -> str | None: + if not isinstance(value, str): + return None + try: + return str(UUID(value)) + except ValueError: + return None + + +def _identity(root: Path, relative: str, document: dict) -> str: + metadata = document.get("metadata") or {} + if not isinstance(metadata, dict): + raise ValueError("Workbook metadata must be an object.") + candidates = [ + document.get("fromTemplateId"), document.get("id"), + metadata.get("contentId"), metadata.get("workbookId"), + ] + for value in candidates: + identity = _guid(value) + if identity: + return identity + return str(uuid5(NAMESPACE_URL, f"sentinel-workbook:{root.name}:{relative}")) + + +def inventory_workbooks(solution) -> list[dict]: + root = _root(solution) + result, identities = [], set() + for path in _files(root, "Workbooks"): + document = _document(path) + relative = path.relative_to(root).as_posix() + identity = _identity(root, relative, document) + if identity in identities: + raise ValueError(f"Duplicate workbook identity: {identity}") + identities.add(identity) + queries = [] + for surface in workbook.query_surfaces(document): + raw = surface["node"]["query"] + if not isinstance(raw, str): + raise ValueError(f"Nonstring workbook query at {surface['path']}") + queries.append({ + "path": surface["path"], "querySha256": workbook.query_hash(raw), + "queryType": surface["queryType"], "resourceType": surface["resourceType"], + "parameter": surface["parameter"], + }) + result.append({"path": relative, "sha256": _sha(path.read_bytes()), + "identity": identity, "queries": queries}) + return result + + +def _solution_data(root: Path) -> dict: + directory = _path(root, "Data") + files = sorted(directory.glob("Solution_*.json")) if directory.exists() else [] + if len(files) != 1: + raise ValueError("Expected exactly one Data/Solution_*.json.") + path = _path(root, files[0].relative_to(root).as_posix()) + value = json.loads(path.read_bytes()) + if not isinstance(value, dict): + raise ValueError("Solution data must be an object.") + return value + + +def selected_workbook_files(solution) -> list[Path]: + """Return only explicit XDR Workbooks references; absence is not selection.""" + root = _root(solution) + data = _solution_data(root) + references = data.get("XDR Workbooks") + if not isinstance(references, list): + raise ValueError("An explicit XDR Workbooks selection array is required.") + paths, seen = [], set() + for reference in references: + if not isinstance(reference, str) or not reference.startswith("XDR Workbooks/"): + raise ValueError("Workbook selection must reference XDR Workbooks/ JSON content.") + path = _path(root, reference) + if path.suffix.lower() != ".json" or not path.is_file(): + raise ValueError(f"Missing workbook selection: {reference}") + folded = reference.casefold() + if folded in seen: + raise ValueError(f"Duplicate workbook selection: {reference}") + seen.add(folded) + paths.append(path) + return paths + + +def _release(data: dict, config: dict) -> str | None: + from .packaging import validate_v31_target_version + value = config.get("targetVersion", data.get("Version", data.get("version"))) + try: + return validate_v31_target_version(value) + except ValueError: + return None + + +def _content_release(value: str | None) -> str | None: + if not isinstance(value, str) or not re.fullmatch(r"3\.[1-9]\d*\.(0|[1-9]\d*)", value): + return None + return value + + +def _dependencies(queries: list[dict], configured: dict | None, reviewed_sources=()) -> dict: + targets, source_targets = set(), {} + catalog = mapping_catalog.load_mapping_catalog()["tables"] + for table in telemetry_data_types(reviewed_sources): + source_targets[table] = catalog.get(table, {}).get("ahTable") or table + for query in queries: + contract = query.get("conversion") + if not contract or query["status"] == "removed-scope-helper": + continue + for table in telemetry_data_types(contract["resolvedTables"]): + target = catalog.get(table, {}).get("ahTable") or table + source_targets[table] = target + targets.add(target) + records, unknown = workload_records(sorted(targets)) + settings = configured if configured is not None else {} + if not isinstance(settings, dict): + raise ValueError("Workbook dependencies must be an object.") + records = deepcopy(settings.get("requiredWorkloads", records)) + connectors = settings.get("requiredDataConnectors") + if not isinstance(records, list) or any(not isinstance(r, dict) for r in records): + raise ValueError("Workbook workload prerequisites must be object records.") + if connectors is not None and not isinstance(connectors, list): + raise ValueError("Workbook connector prerequisites must be an array.") + reasons = [] + # Connector records must explicitly identify source tables; rewrite that + # association to output names rather than guessing from a connector ID. + output_connectors = [] + for connector in connectors if connectors is not None else []: + if not isinstance(connector, dict) or not isinstance(connector.get("dataTypes"), list): + raise ValueError("Workbook connector prerequisites require source dataTypes.") + if any(table not in source_targets for table in connector["dataTypes"]): + raise ValueError("Connector prerequisite references a table outside reviewed query sources.") + output_connectors.append({ + **deepcopy(connector), + "dataTypes": sorted({source_targets[t] for t in connector["dataTypes"]}), + }) + declared = {t for record in records for t in record.get("dataTypes", []) + if isinstance(t, str)} if all(isinstance(r.get("dataTypes"), list) for r in records) else set() + if not targets <= declared: + reasons.append("Output data types require reviewed workload mappings.") + if unknown and "requiredWorkloads" not in settings: + reasons.append("Unknown output workload: " + ", ".join(unknown)) + if any(r.get("workload") == "sentinel" for r in records) and connectors is None: + reasons.append("Explicit source-associated Sentinel connector prerequisites required.") + if settings.get("dependencyReviewRequired") is True: + reasons.append("Reviewer deferred dependencies; explicit current review is required.") + provenance = {"requiredWorkloads": records, "requiredDataConnectors": output_connectors, + "dependencyReviewRequired": bool(reasons)} + errors = dependency_validation_errors({"contentProvenance": { + **provenance, "dependencyReviewRequired": False}}) + if errors: + raise ValueError("; ".join(errors)) + return {**provenance, "reviewReasons": reasons} + + +def _write(path: Path, content: bytes, previous: bytes | None) -> None: + if any(p.is_symlink() for p in (path, *path.parents)): + raise ValueError("Refusing symlink write.") + if previous is None: + with path.open("xb") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + else: + staged = path.with_name(f".{path.name}.{uuid4().hex}.staged") + try: + with staged.open("xb") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + if (any(p.is_symlink() for p in (path, *path.parents)) + or not path.is_file() or path.read_bytes() != previous): + raise ValueError("Workbook changed during conversion; refusing overwrite.") + os.replace(staged, path) + finally: + if staged.exists(): + staged.unlink() + + +def convert_workbooks(solution, *, config=None, report_directory=None, overwrite=False) -> dict: + root = _root(solution) + inventory = inventory_workbooks(root) + settings = deepcopy(config if config is not None else {}) + if not isinstance(settings, dict): + raise ValueError("Workbook configuration must be an object.") + if "targetVersion" in settings: + from .packaging import validate_v31_target_version + validate_v31_target_version(settings["targetVersion"]) + resolutions = settings.get("workbookResolutions", {}) + dependencies = settings.get("workbookDependencies", {}) + if not isinstance(resolutions, dict) or not isinstance(dependencies, dict): + raise ValueError("Workbook resolutions and dependencies must be objects.") + sources = {row["path"] for row in inventory} + if set(resolutions) - sources or set(dependencies) - sources: + raise ValueError("Workbook configuration references missing source content.") + from .workbook_functions import validate_functions + function_dependencies = {} + for relative, configured in dependencies.items(): + if not isinstance(configured, dict): + raise ValueError("Workbook dependencies must be an object.") + function_dependencies[relative] = validate_functions( + root, configured.get("requiredWorkspaceFunctions", [])) + version = _release(_solution_data(root), settings) + output_dir = _path(root, "XDR Workbooks") + _files(root, "XDR Workbooks") + report_dir = (Path(report_directory).expanduser() if report_directory is not None + else resolve_report_directory(root, create=True)) + if any(p.is_symlink() for p in (report_dir, *report_dir.parents)): + raise ValueError("Report directory must not use symlinks.") + report_dir = report_dir.resolve() + if report_dir == root or report_dir.is_relative_to(_path(root, "Workbooks")) or report_dir.is_relative_to(output_dir): + raise ValueError("Reports must be outside workbook content directories.") + report_dir.mkdir(parents=True, exist_ok=True) + manifest_file, report_file = report_dir / MANIFEST, report_dir / REPORT + for path in (manifest_file, report_file): + if path.is_symlink(): + raise ValueError("Report artifacts must not be symlinks.") + previous_manifest = json.loads(manifest_file.read_bytes()) if manifest_file.exists() else {} + previous_results = {r["sourceRelativePath"]: r for r in previous_manifest.get("results", [])} + planned = [] + for row in inventory: + path = _path(root, row["path"]) + source_bytes = path.read_bytes() + if _sha(source_bytes) != row["sha256"]: + raise ValueError("Source workbook changed during inventory.") + source = _document(path) + query_resolutions = resolutions.get(row["path"], {}) + if not isinstance(query_resolutions, dict): + raise ValueError("Per-workbook resolutions must be an object.") + if set(query_resolutions) - {q["path"] for q in row["queries"]}: + raise ValueError("Workbook resolution references a missing query pointer.") + functions = function_dependencies.get(row["path"], []) + converted = workbook.convert_workbook( + source, resolutions=query_resolutions, + allowed_workspace_functions=[r["functionAlias"] for r in functions]) + target, report = converted["workbook"], converted["report"] + used_functions = {name for q in report["queries"] for name in q.get("workspaceFunctions", [])} + if used_functions != {r["functionAlias"] for r in functions}: + report["issues"].append({"code": "workspace-function-coverage", + "message": "Workspace prerequisites must match exact query aliases."}) + output_surfaces = {q["path"]: q for q in workbook.query_surfaces(target)} + originals = {q["path"]: q["querySha256"] for q in row["queries"]} + for query in report["queries"]: + query["sourceSha256"] = originals[query["sourcePath"]] + query["targetSha256"] = (workbook.query_hash( + output_surfaces[query["outputPath"]]["node"]["query"]) + if query.get("outputPath") else None) + reviewed_sources = set() + for pointer, resolution in query_resolutions.items(): + if (isinstance(resolution, dict) and resolution.get("querySha256") == originals[pointer] + and isinstance(resolution.get("tables"), list) and resolution.get("evidence") + and all(isinstance(table, str) and table.strip() for table in resolution["tables"])): + reviewed_sources.update(resolution["tables"]) + dependency = _dependencies(report["queries"], dependencies.get(row["path"]), reviewed_sources) + if version is None: + report["issues"].append({ + "path": "/xdrMetadata/version", "code": "release-version-required", + "message": "Select targetVersion >=3.2.0 and <4.0.0; source release is unchanged."}) + canonical = row["identity"] + instance = str(uuid5(NAMESPACE_URL, f"xdr-workbook-instance:{root.name}:{canonical}")) + target["xdrMetadata"] = { + "contentId": canonical, "instanceId": instance, "version": version, + "schemaVersion": "1.0.0", + "displayName": (source.get("metadata") or {}).get("name") or path.stem, + } + source_metadata = source.get("metadata") or {} + target["contentProvenance"] = { + "source": {"path": row["path"], "sha256": row["sha256"], + "contentId": (source.get("fromTemplateId") or source.get("id") or + source_metadata.get("contentId") or source_metadata.get("workbookId")), + "originalMetadataVersion": source_metadata.get("version")}, + "conversion": {"tool": "sentinel-xdr-migration", "version": __version__, + "reviewRequired": bool(report["issues"]), "queries": report["queries"]}, + **{key: dependency[key] for key in ( + "requiredWorkloads", "requiredDataConnectors", "dependencyReviewRequired")}, + } + if functions: + target["contentProvenance"]["requiredWorkspaceFunctions"] = functions + relative = "XDR Workbooks/" + row["path"].removeprefix("Workbooks/") + output = _path(root, relative) + content = (json.dumps(target, indent=2, ensure_ascii=False) + "\n").encode("utf-8") + old = output.read_bytes() if output.exists() else None + issues = report["issues"] + [ + {"code": "dependency-review", "message": message} + for message in dependency["reviewReasons"]] + status = "blocked" if issues else "converted" + if old is not None and old != content: + receipt = previous_results.get(row["path"]) + managed_hash = (receipt.get("managedOutputSha256", receipt.get("outputSha256")) + if receipt else None) + if not overwrite or not receipt or managed_hash != _sha(old): + status = "conflict" + issues.append({"code": "output-conflict", + "message": "Existing edited/unbound workbook retained; explicit review required."}) + result = { + "sourceRelativePath": row["path"], "outputRelativePath": relative, + "sourceSha256": row["sha256"], "outputSha256": _sha(old if status == "conflict" else content), + "status": status, "reviewRequired": bool(issues), "identity": canonical, + "managedOutputSha256": ( + previous_results.get(row["path"], {}).get( + "managedOutputSha256", previous_results.get(row["path"], {}).get("outputSha256")) + if status == "conflict" else _sha(content)), + "instanceId": instance, "issues": issues, "queries": report["queries"], + "gates": report["gates"], "validation": report["validation"], + } + planned.append((path, source_bytes, output, content, old, result)) + # Complete all conflict/malformed-input checks before touching content. + for path, source_bytes, output, content, old, result in planned: + if path.read_bytes() != source_bytes: + raise ValueError("Source changed before conversion commit.") + if result["status"] != "conflict" and old != content: + output.parent.mkdir(parents=True, exist_ok=True) + _write(output, content, old) + results = [p[-1] for p in planned] + summary = { + "total": len(results), "converted": sum(r["status"] == "converted" for r in results), + "blocked": sum(r["status"] != "converted" for r in results), "results": results, + "manifestPath": str(manifest_file), "reportPath": str(report_file), + } + manifest = {"schemaVersion": "1.0.0", "scope": "solution", "inventory": inventory, + **summary} + for output, value in ((manifest_file, manifest), (report_file, { + **summary, "validation": {"localStructureOnly": True, "runtime": "not-run", "ui": "not-run"}})): + _write(output, (json.dumps(value, indent=2) + "\n").encode(), + output.read_bytes() if output.exists() else None) + return summary + + +def validate_workbooks(solution, *, selected_only=False) -> dict: + root = _root(solution) + inventory = {r["path"]: r for r in inventory_workbooks(root)} + files = selected_workbook_files(root) if selected_only else _files(root, "XDR Workbooks") + results, identities, instances = [], set(), set() + source_paths = set() + schema = json.loads(package_files("sentinel_xdr_migration._workbook_core").joinpath( + "data/xdr-workbook.schema.json").read_text("utf-8")) + validator = Draft202012Validator(schema) + for path in files: + document = _document(path) + schema_errors = [f"{'/'.join(map(str, e.absolute_path))}: {e.message}" + for e in validator.iter_errors(document)] + if schema_errors: + provenance = document.get("contentProvenance") + if isinstance(provenance, dict) and isinstance(provenance.get("source"), dict): + source_paths.add(provenance["source"].get("path")) + results.append({"path": path.relative_to(root).as_posix(), + "sha256": _sha(path.read_bytes()), "valid": False, "errors": schema_errors}) + continue + errors = dependency_validation_errors(document) + metadata = document.get("xdrMetadata", {}) + provenance = document.get("contentProvenance", {}) + source = provenance.get("source", {}) + conversion = provenance.get("conversion", {}) + from .workbook_functions import validate_functions + function_records = provenance.get("requiredWorkspaceFunctions", []) + try: + validated_functions = validate_functions(root, function_records) + except (ValueError, TypeError) as exc: + errors.append(str(exc)) + validated_functions = [] + approved_aliases = {record["functionAlias"] for record in validated_functions} + identity, instance = _guid(metadata.get("contentId")), _guid(metadata.get("instanceId")) + if not identity or not instance or identity == instance: + errors.append("Canonical and instance identities must be distinct GUIDs.") + if identity in identities or instance in instances: + errors.append("Duplicate authored workbook identity.") + identities.add(identity) + instances.add(instance) + if metadata.get("schemaVersion") != "1.0.0" or _content_release(metadata.get("version")) is None: + errors.append("Authored metadata requires schema 1.0.0 and content release >=3.1.0,<4.0.0.") + if not isinstance(metadata.get("displayName"), str) or not metadata["displayName"].strip(): + errors.append("Workbook displayName is required.") + if any(key in provenance for key in ("packageId", "packageVersion", "contentId", "contentVersion")): + errors.append("Deployment provenance quartet must not be authored.") + row = inventory.get(source.get("path")) + if not row or row["sha256"] != source.get("sha256"): + errors.append("Source provenance is missing or stale.") + if row: + expected_instance = str(uuid5(NAMESPACE_URL, f"xdr-workbook-instance:{root.name}:{row['identity']}")) + if identity != row["identity"] or instance != expected_instance: + errors.append("Authored workbook identities do not match the stable source assignment.") + if source.get("path") in source_paths: + errors.append("Duplicate workbook source provenance.") + source_paths.add(source.get("path")) + queries = conversion.get("queries") + if conversion.get("reviewRequired") is not False: + errors.append("Conversion review is unresolved.") + output_queries = {q["path"]: q for q in workbook.query_surfaces(document)} + if not isinstance(queries, list): + errors.append("Per-query conversion provenance is required.") + else: + recorded_source, recorded_output = set(), set() + original_queries = {q["path"]: q["querySha256"] for q in row["queries"]} if row else {} + original_surfaces = {} + removed_prefixes = [] + if row: + original = _document(_path(root, row["path"])) + original_surfaces = {q["path"]: q for q in workbook.query_surfaces(original)} + parameters, removed, _ = workbook._scope_parameters(original) + removed_prefixes = [pointer + "/" for pointer, parameter in parameters + if parameter["name"] in removed] + used_aliases = set() + for query in queries: + source_pointer, output_pointer = query.get("sourcePath"), query.get("outputPath") + if source_pointer in recorded_source or query.get("sourceSha256") != original_queries.get(source_pointer): + errors.append("Original query provenance is duplicate or stale.") + recorded_source.add(source_pointer) + if query.get("status") == "removed-scope-helper": + if output_pointer is not None or query.get("targetSha256") is not None: + errors.append("Removed scope query must not have output provenance.") + if not any(source_pointer.startswith(prefix) for prefix in removed_prefixes): + errors.append("Only proven scope-helper queries may be removed.") + continue + if query.get("status") != "converted": + errors.append("Query conversion remains blocked or review-required.") + output_query = output_queries.get(output_pointer) + source_query = original_surfaces.get(source_pointer) + try: + raw_source = workbook._raw_query( + source_query["node"]["query"], source_query["queryType"]) if source_query else "" + raw_target = workbook._raw_query( + output_query["node"]["query"], output_query["queryType"]) if output_query else "" + source_aliases = workbook.opaque_functions(raw_source) if source_query else set() + target_aliases = workbook.opaque_functions(raw_target) if output_query else set() + explicit_calls = ( + (workbook.opaque_functions(raw_source, include_bare=False) if source_query else set()) + | (workbook.opaque_functions(raw_target, include_bare=False) if output_query else set()) + ) + except ValueError as exc: + errors.append(str(exc)) + source_aliases, target_aliases, explicit_calls = set(), set(), set() + contract = query.get("conversion") + if not isinstance(contract, dict): + errors.append("Shared-converter query contract must be an object.") + contract = {} + resolved_tables = (set(contract.get("resolvedTables", [])) + if isinstance(contract.get("resolvedTables"), list) else set()) + aliases = query.get("workspaceFunctions", []) + alias_set = (set(aliases) if isinstance(aliases, list) + and all(isinstance(a, str) for a in aliases) else set()) + non_function_tables = resolved_tables - explicit_calls - approved_aliases - alias_set + source_aliases -= non_function_tables + target_aliases -= non_function_tables + if (not isinstance(aliases, list) or any(not isinstance(a, str) for a in aliases) + or len(aliases) != len(set(aliases)) + or set(aliases) != source_aliases or set(aliases) != target_aliases + or not source_aliases <= approved_aliases): + errors.append("Workspace function aliases must exactly match current source, target and prerequisites.") + used_aliases.update(source_aliases) + if output_pointer in recorded_output or not output_query or query.get("targetSha256") != workbook.query_hash(output_query["node"]["query"]): + errors.append("Output query provenance is missing, duplicate or stale.") + if output_query and output_query["queryType"] != workbook.AH: + errors.append("Converted query must use Advanced Hunting.") + if (not query.get("resolutionEvidence") or contract.get("status") != "converted" + or contract.get("sourceId") != source_pointer + or not output_query + or contract.get("convertedKql") != output_query["node"]["query"]): + errors.append("Current reviewed shared-converter query evidence is required.") + if source_query: + try: + replay = workbook.migrator.convert( + workbook._raw_query(source_query["node"]["query"], source_query["queryType"]), + tables=contract.get("resolvedTables"), source_id=source_pointer, + artifact_type="workbook", resolve=False) + if replay != contract: + errors.append("Shared query contract does not match current deterministic conversion.") + except ValueError as exc: + errors.append(str(exc)) + references = sorted(set(contract.get("resolvedTables", [])) & {"Watchlist"}) + if query.get("referenceInputs", []) != references: + errors.append("Platform reference inputs must match reviewed raw source lineage.") + catalog = mapping_catalog.load_mapping_catalog()["tables"] + expected_types = {catalog.get(table, {}).get("ahTable") or table + for table in telemetry_data_types(contract.get("resolvedTables", []))} + declared_types = {table for record in provenance["requiredWorkloads"] for table in record["dataTypes"]} + if not expected_types <= declared_types: + errors.append("Workload prerequisites do not cover reviewed telemetry source tables.") + if source_aliases and ( + not contract.get("resolvedTables") + or source_aliases & set(contract.get("resolvedTables", [])) + or not source_query or not output_query + or workbook._raw_query(source_query["node"]["query"], source_query["queryType"]) + != output_query["node"]["query"] + ): + errors.append("Preserved function calls require distinct underlying table evidence and unchanged KQL.") + recorded_output.add(output_pointer) + if used_aliases != approved_aliases: + errors.append("Workspace function prerequisites must exactly cover query aliases.") + if recorded_source != set(original_queries) or recorded_output != set(output_queries): + errors.append("Query provenance does not cover complete original/output inventory.") + results.append({"path": path.relative_to(root).as_posix(), + "sha256": _sha(path.read_bytes()), "valid": not errors, "errors": errors}) + if not selected_only: + for relative in sorted(set(inventory) - source_paths): + results.append({"path": relative, "valid": False, "errors": ["Source workbook has no authored candidate."]}) + valid = sum(r["valid"] for r in results) + return {"total": len(results), "valid": valid, "invalid": len(results) - valid, + "results": results} diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workflow.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workflow.py index 7febb8b49cf..0fb673ae09d 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workflow.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workflow.py @@ -56,6 +56,7 @@ package_input_identity, package_solution_v3_1, solution_data_version, + validate_v31_effective_version, validate_v31_target_version, version_bump_was_consumed, ) @@ -64,6 +65,7 @@ verify_applied_package_selection, ) from .source_runtime_waiver import SOURCE_PROVIDERS, missing_schema_whitelist_match +from . import workbook_workflow STATE_FILE_NAME = "workflow-state.json" SCHEMA_VERSION = "1.2.0" @@ -1248,6 +1250,13 @@ def refresh_yolo_authoring(solution: str | Path) -> dict[str, Any]: if path != state_path: raise ValueError("workflow state selection changed while acquiring its writer lock") with using_run(root, state["runId"]): + if state["context"].get("workbookFullInventory"): + workbook_workflow.verify_inventory(root, state) + workbook_workflow.validate_evidence(root, state) + return {"runId": state["runId"], "status": state["workflowStatus"], + "workbookSummary": workbook_workflow.summary(root, state), + "updated": False, "workflowStatus": state["workflowStatus"], + "next": _next_stage_name(state)} state = _recover_yolo_state(root, path, state) if state["authoring"]["status"] == "conflicted": raise ValueError("conflicted YOLO runs cannot be refreshed") @@ -1562,6 +1571,10 @@ def _validate_offline_authoring_gates( ) root = solution.expanduser().resolve() + if workbook_workflow.enabled(state): + workbook_workflow.validate_evidence(root, state) + if not workbook_workflow.has_analytic_rules(root): + return inventory = _source_inventory(root) conversion_artifacts = state["stages"]["conversion"].get("artifacts") or {} expected_manifest_hash = conversion_artifacts.get("manifestSha256") @@ -2055,9 +2068,64 @@ def _complete_runtime_pending_stage( return _load(solution)[1] +def run_workbook_only_authoring(solution: str | Path) -> dict[str, Any]: + """Account workbook-only full-inventory passes without inventing rule coverage.""" + root = Path(solution).expanduser().resolve() + with _state_writer_lock(root) as path: + state = _load(root)[1] + if not workbook_workflow.enabled(state) or not state["context"].get("workbookFullInventory"): + raise ValueError("workbook-only full-inventory execution requires a new explicit YOLO run") + workbook_workflow.verify_inventory(root, state) + budget = state.get("budget") + if not budget or budget.get("status") == "stopped": + raise ValueError("workbook full-inventory execution requires an active campaign budget") + for name, dimension, action in ( + ("discovery", "workflowDiscovery", workbook_workflow.inspect_content), + ("conversion", "conversionPass", workbook_workflow.convert_content), + ("validation", "workflowValidation", workbook_workflow.validate_content), + ): + state = _load(root)[1] + if state["stages"][name]["status"] == "passed": + continue + _start_workflow_stage_unlocked(root, name, path) + try: + with measured_operation(root, budget["campaignId"], + inventory_sha256=budget["inventorySha256"], + run_id=state["runId"], dimension=dimension): + result = action(root) + if name == "conversion" and result.get("workbooks", {}).get("selectedBlocked"): + raise ValueError("workbook conversion contains unresolved review or technical blockers") + if ( + name == "validation" + and state["context"]["workflowProfile"] == "qualification" + and state["context"].get("workbookQualificationAfterArm") is not True + ): + _complete_workflow_stage_unlocked(root, name, path, status="blocked", + message="Workbook query/UI Qualification evidence is still required; no detection ingestion/parity is requested.") + break + _complete_workflow_stage_unlocked(root, name, path, status="passed", + artifacts=({"runtimeStatus": "not-run", "validationMode": + "workbook-content" if state["context"]["workflowProfile"] == "qualification" + else "offline-authoring"} + if name == "validation" else {})) + except BudgetError as exc: + _stop_budget_run(path, _load(root)[1], name, str(exc)) + break + except (OSError, ValueError, RuntimeError) as exc: + _complete_workflow_stage_unlocked(root, name, path, status="failed", message=str(exc)) + break + state = _load(root)[1] + return {"runId": state["runId"], "status": state["workflowStatus"], + "workbookSummary": workbook_workflow.summary(root, state), + "next": _next_stage_name(state)} + + def run_yolo_authoring(solution: str | Path) -> dict[str, Any]: """Run existing local discovery/conversion/validation once under the frozen YOLO contract.""" root = Path(solution).expanduser().resolve() + initial = _load(root)[1] + if initial["context"].get("workbookFullInventory") is True: + return run_workbook_only_authoring(root) with _state_writer_lock(root) as state_path: loaded_path, state = _load(root) if loaded_path != state_path: @@ -2166,7 +2234,10 @@ def run_yolo_authoring(solution: str | Path) -> dict[str, Any]: dimension="workflowDiscovery", ): discovery_dispatched = True - inspection = inspect_solution(root) + inspection = ( + workbook_workflow.inspect_content(root) + if workbook_workflow.enabled(state) else inspect_solution(root) + ) except BudgetError as exc: if discovery_dispatched: raise @@ -2218,7 +2289,10 @@ def run_yolo_authoring(solution: str | Path) -> dict[str, Any]: dimension="conversionPass", ): conversion_dispatched = True - conversion = convert_solution(root) + conversion = ( + workbook_workflow.convert_content(root) + if workbook_workflow.enabled(state) else convert_solution(root) + ) _verify_source_inventory(root, record) manifest = _read_full_conversion_manifest( root, state_path.parent, inventory @@ -2525,6 +2599,8 @@ def initialize_workflow( yolo: bool | None = None, campaign_id: str | None = None, new_run: bool = False, + workbook_capability_version: int | None = None, + arm_validation_method: str | None = None, ) -> dict[str, Any]: root = Path(solution).expanduser().resolve() if not root.is_dir(): @@ -2535,12 +2611,26 @@ def initialize_workflow( validate_v31_target_version(target_version) if version_bump not in (None, "none"): raise ValueError("target version cannot be combined with a version bump") + if target_version is not None or ( + version_bump is not None and solution_data_version(root) is not None + ): + validate_v31_effective_version( + solution_data_version(root), version_bump or "none", target_version, + ) if workflow_profile is None: raise ValueError( "workflow profile selection is required: choose authoring or qualification" ) if workflow_profile not in WORKFLOW_PROFILES: raise ValueError("workflow profile must be authoring or qualification") + if workbook_capability_version not in (None, 1): + raise ValueError("unsupported workbook capability version") + if arm_validation_method not in (None, *workbook_workflow.ARM_VALIDATION_METHODS): + raise ValueError("ARM validation method must be arm-ttk or lab-deployment") + if arm_validation_method and workbook_capability_version != 1: + raise ValueError("an explicit ARM method requires workbook capability version 1") + if workflow_profile == "qualification" and arm_validation_method == "arm-ttk": + raise ValueError("the workbook-capable Qualification route requires lab-deployment") explicit_campaign_id = campaign_id if campaign_id is not None: from .budget import _safe_campaign_id @@ -2589,7 +2679,9 @@ def initialize_workflow( _preflight_campaign_binding( root, campaign_id, - yolo=yolo is True, + yolo=yolo is True and not ( + workbook_capability_version == 1 and not workbook_workflow.has_analytic_rules(root) + ), ) run = create_run(root) with using_run(root, run.name): @@ -2599,12 +2691,14 @@ def initialize_workflow( workspace_customer_id=workspace_customer_id, version_bump=version_bump, target_version=target_version, workflow_profile=workflow_profile, yolo=yolo, campaign_id=campaign_id, + workbook_capability_version=workbook_capability_version, + arm_validation_method=arm_validation_method, ) existing_path = existing_artifact_path(root, STATE_FILE_NAME) if existing_path.exists(): with _run_writer_lock(existing_path.parent): path, state = _load(root) - if yolo is True and "authoring" not in state: + if yolo is True and "authoring" not in state and not state["context"].get("workbookFullInventory"): raise ValueError( "YOLO cannot be enabled on an existing legacy run; initialize a new run explicitly" ) @@ -2614,6 +2708,11 @@ def initialize_workflow( "--campaign-id to link it conservatively" ) context = state["context"] + if workbook_capability_version is not None and context.get("workbookCapabilityVersion") != workbook_capability_version: + raise ValueError("workbook capability cannot change on resumption; initialize a new run") + if arm_validation_method is not None and context.get("armValidationMethod") != arm_validation_method: + raise ValueError("ARM validation method cannot change on resumption; initialize a new run") + workbook_workflow.verify_inventory(root, state) if target_version is not None and context.get("versionBump") not in (None, "none"): raise ValueError( "target version cannot be combined with the workflow's selected version bump" @@ -2706,13 +2805,14 @@ def initialize_workflow( "next": _next_stage_name(state), } - frozen_inventory = _source_inventory(root) if yolo is True else None + workbook_only = workbook_capability_version == 1 and not workbook_workflow.has_analytic_rules(root) + frozen_inventory = _source_inventory(root) if yolo is True and not workbook_only else None if frozen_inventory is not None and not campaign_id: raise ValueError("new YOLO workflows require an explicit --campaign-id") preflight_inventory = _preflight_campaign_binding( root, campaign_id, - yolo=yolo is True, + yolo=yolo is True and not workbook_only, inventory=frozen_inventory, ) if yolo is True and requested_run_id(root) is None: @@ -2729,6 +2829,8 @@ def initialize_workflow( workflow_profile=workflow_profile, yolo=True, campaign_id=campaign_id, + workbook_capability_version=workbook_capability_version, + arm_validation_method=arm_validation_method, ) now = _utc_now() @@ -2753,6 +2855,16 @@ def initialize_workflow( } state["context"]["workflowProfile"] = workflow_profile state["context"]["profileSelectionConfirmed"] = True + if workbook_capability_version == 1: + state["context"]["workbookCapabilityVersion"] = 1 + state["context"]["armValidationMethod"] = arm_validation_method or "lab-deployment" + state["workbooks"] = workbook_workflow.freeze_inventory(root) + if yolo and workbook_only: + state["context"]["workbookFullInventory"] = True + if workbook_only and workflow_profile == "qualification": + state["context"]["workbookQualificationAfterArm"] = True + for name in ("mockIngestion", "alertParity"): + state["stages"][name] = _stage_result(name, required=False) if frozen_inventory is not None: record, _ = _persist_contract_snapshot( path.parent, @@ -2826,6 +2938,8 @@ def _validate_packaging_evidence( state: dict[str, Any], artifacts: dict[str, str], solution: str | Path, + *, + allow_candidate: bool = False, ) -> None: if artifacts.get("packager") != "V3.1": raise ValueError("passed packaging stage requires packager=V3.1; rebuild old packages") @@ -2910,7 +3024,9 @@ def _validate_packaging_evidence( if packaging: attempt = packaging.get("currentAttempt") or {} if ( - attempt.get("status") != "passed" + attempt.get("status") not in ( + {"passed", "blocked"} if allow_candidate and workbook_workflow.enabled(state) else {"passed"} + ) or report.get("packageAttemptId") != attempt.get("attemptId") or report.get("workflowRunId") != state.get("runId") or report.get("requestedVersionBump") != attempt.get("requestedVersionBump") @@ -2933,6 +3049,10 @@ def _validate_packaging_evidence( report_digest = hashlib.sha256(report_path.read_bytes()).hexdigest() if report_digest != attempt.get("packageReportSha256"): raise ValueError("V3.1 package report hash does not match the current workflow attempt") + if workbook_workflow.enabled(state) and not allow_candidate and ( + state["context"]["armValidationMethod"] == "lab-deployment" or report.get("xdrWorkbookCount", 0) + ): + workbook_workflow.require_accepted(solution, state) def _yolo_package_readiness( @@ -4537,6 +4657,7 @@ def plan_new_package_release( raise ValueError("expected package input identity must be a lowercase SHA-256") root = Path(solution).expanduser().resolve() + validate_v31_effective_version(solution_data_version(root), version_bump, target_version) with _state_writer_lock(root) as path: loaded_path, state = _load(root) if loaded_path != path: @@ -4744,6 +4865,9 @@ def package_workflow_v3_1( if requested_version_bump != "none": raise ValueError("target version cannot be combined with a version bump") root = Path(solution).expanduser().resolve() + validate_v31_effective_version( + solution_data_version(root), requested_version_bump, requested_target_version, + ) with _state_writer_lock(root) as path: loaded_path, state = _load(root) if loaded_path != path: @@ -4752,6 +4876,9 @@ def package_workflow_v3_1( raise ValueError("workflow packaging requires an explicitly selected run") if state["context"].get("profileSelectionConfirmed") is not True: raise ValueError("workflow packaging requires explicit profile selection") + workbook_workflow.verify_inventory(root, state) + if workbook_workflow.enabled(state): + workbook_workflow.validate_evidence(root, state) selection_evidence = verify_applied_package_selection( root, str(state["runId"]), ) @@ -5172,6 +5299,9 @@ def package_workflow_v3_1( selection_plan=( current_selection["plan"] if current_selection else None ), + **({ + "arm_validation_method": state["context"]["armValidationMethod"], + } if workbook_workflow.enabled(state) else {}), ) except BaseException: version_after_failure = solution_data_version(root) @@ -5320,6 +5450,24 @@ def package_workflow_v3_1( "selectionPlanSha256": selection_evidence["plan"]["planSha256"], "selectionOutcome": report.get("selectionOutcome"), }) + if workbook_workflow.enabled(state) and ( + state["context"]["armValidationMethod"] == "lab-deployment" + or report.get("xdrWorkbookCount", 0) > 0 + ): + if report.get("packageAccepted") is not False or report.get("packageLifecycleStatus") != "candidate-generated": + raise ValueError("versioned candidate package receipt must remain explicitly unaccepted") + stage.update({ + "status": "blocked", "completedAt": _utc_now(), + "message": "Candidate generated; official static validation, selected ARM method and independent workbook contract evidence remain required.", + "artifacts": {**artifacts, "packageLifecycleStatus": "candidate-generated"}, + "evidence": [result["packageReport"]], + }) + current_attempt["status"] = "blocked" + package_state["lastConsumedAction"]["packageSucceeded"] = False + state["workflowStatus"] = "blocked" + _write_unlocked(path, state) + return {**result, "status": "blocked", "runId": state["runId"], + "workflowStatus": "blocked", "next": "workflow-arm-candidate"} stage.update({ "status": "passed", "completedAt": _utc_now(), @@ -5558,6 +5706,7 @@ def workflow_status(solution: str | Path) -> dict[str, Any]: } return { **state, + "workbookSummary": workbook_workflow.summary(solution, state), "packageFreshness": freshness, "runtimeAssessment": { "runtimeStatus": ( @@ -5695,6 +5844,13 @@ def invalidate_runtime_completion( def _dependencies(state: dict[str, Any], stage: str) -> tuple[str, ...]: + if ( + workbook_workflow.enabled(state) + and state["context"]["workflowProfile"] == "qualification" + and state["stages"]["alertParity"]["status"] == "notRequired" + and stage == "report" + ): + return ("deployment",) if state["context"]["workflowProfile"] == "authoring": return AUTHORING_DEPENDENCIES.get(stage, ()) return QUALIFICATION_DEPENDENCIES[stage] @@ -5873,7 +6029,30 @@ def _complete_workflow_stage_unlocked( if current["status"] != "running": raise ValueError(f"workflow stage is not running: {stage}") final_artifacts = dict(artifacts or {}) - if stage == "validation" and status == "passed": + if status == "passed" and workbook_workflow.enabled(state): + workbook_workflow.verify_inventory(solution, state) + if stage == "conversion": + final_artifacts.update(workbook_workflow.bind_conversion(solution, state)) + if stage in {"validation", "packaging", "report"}: + workbook_workflow.validate_evidence(solution, state) + if stage == "report" and state["context"]["workflowProfile"] == "qualification": + from .workbook_qualification import require_current_qualification + require_current_qualification(solution, state) + if ( + stage == "validation" and status == "passed" + and workbook_workflow.enabled(state) + and not workbook_workflow.has_analytic_rules(solution) + and state["context"]["workflowProfile"] == "qualification" + ): + if state["context"].get("workbookQualificationAfterArm") is True: + final_artifacts["validationMode"] = "workbook-content" + final_artifacts["workbookQueryStatus"] = "not-run" + final_artifacts["workbookUiStatus"] = "not-run" + else: + from .workbook_qualification import require_current_qualification + require_current_qualification(solution, state) + final_artifacts["validationMode"] = "workbook-qualification" + elif stage == "validation" and status == "passed": runtime_status = final_artifacts.get("runtimeStatus") from .runtime import current_runtime_evidence @@ -5979,6 +6158,17 @@ def _complete_workflow_stage_unlocked( and state.get("packaging") ): _verify_current_package_state(Path(solution).expanduser().resolve(), state) + if ( + status == "passed" and stage == "deployment" + and state["context"].get("workbookQualificationAfterArm") is True + ): + workbook_workflow.require_accepted(solution, state) + final_artifacts["deploymentMode"] = "candidate-arm-lab-validation" + final_artifacts.update({ + key: value + for key, value in state["stages"]["packaging"]["artifacts"].items() + if key in {"armCandidate", "workbookContractPlan", "workbookContractEvidence", "workbookContractResult"} + }) current.update( { "status": status, diff --git a/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/manifest.json b/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/manifest.json new file mode 100644 index 00000000000..1fd9024f759 --- /dev/null +++ b/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/manifest.json @@ -0,0 +1,82 @@ +{ + "version": 1, + "clock": "2026-01-02T12:00:00Z", + "seed": 1729, + "tables": { + "Syslog": { + "schema": {"TimeGenerated": "datetime", "Computer": "string", "SeverityLevel": "string"}, + "rows": [ + ["2026-01-02T10:05:00Z", "alpha", "warning"], + ["2026-01-02T10:25:00Z", "alpha", "warning"], + ["2026-01-02T11:05:00Z", "alpha", "info"], + ["2026-01-02T11:20:00Z", "beta", "warning"], + ["2026-01-02T09:59:00Z", "beta", "warning"], + ["2026-01-02T12:01:00Z", "gamma", "warning"] + ] + } + }, + "cases": [ + { + "id": "all-computers-warning", + "sourcePath": "/items/1/content/items/0/content/query", + "substitutions": {"TimeRange:start": "datetime(2026-01-02T10:00:00Z)", "TimeRange:end": "datetime(2026-01-02T12:00:00Z)", "Computer": "'*'", "Severity": "'warning'"}, + "expected": { + "columns": [{"name": "Computer", "type": "string"}, {"name": "Hour", "type": "datetime"}, {"name": "Events", "type": "long"}], + "ordered": true, + "rows": [["alpha", "2026-01-02T10:00:00Z", 2], ["beta", "2026-01-02T11:00:00Z", 1]] + } + }, + { + "id": "single-computer-all-severities", + "sourcePath": "/items/1/content/items/0/content/query", + "substitutions": {"TimeRange:start": "datetime(2026-01-02T10:00:00Z)", "TimeRange:end": "datetime(2026-01-02T12:00:00Z)", "Computer": "'alpha'", "Severity": "'*'"}, + "expected": { + "columns": [{"name": "Computer", "type": "string"}, {"name": "Hour", "type": "datetime"}, {"name": "Events", "type": "long"}], + "ordered": true, + "rows": [["alpha", "2026-01-02T10:00:00Z", 2], ["alpha", "2026-01-02T11:00:00Z", 1]] + } + }, + { + "id": "empty-computer", + "sourcePath": "/items/1/content/items/0/content/query", + "substitutions": {"TimeRange:start": "datetime(2026-01-02T10:00:00Z)", "TimeRange:end": "datetime(2026-01-02T12:00:00Z)", "Computer": "'missing'", "Severity": "'*'"}, + "expected": { + "columns": [{"name": "Computer", "type": "string"}, {"name": "Hour", "type": "datetime"}, {"name": "Events", "type": "long"}], + "rows": [] + } + }, + { + "id": "multiple-computers", + "sourcePath": "/items/1/content/items/0/content/query", + "substitutions": {"TimeRange:start": "datetime(2026-01-02T10:00:00Z)", "TimeRange:end": "datetime(2026-01-02T12:00:00Z)", "Computer": "'alpha','beta'", "Severity": "'warning'"}, + "expected": { + "columns": [{"name": "Computer", "type": "string"}, {"name": "Hour", "type": "datetime"}, {"name": "Events", "type": "long"}], + "ordered": true, + "rows": [["alpha", "2026-01-02T10:00:00Z", 2], ["beta", "2026-01-02T11:00:00Z", 1]] + } + }, + { + "id": "narrow-time-range", + "sourcePath": "/items/1/content/items/0/content/query", + "substitutions": {"TimeRange:start": "datetime(2026-01-02T11:00:00Z)", "TimeRange:end": "datetime(2026-01-02T12:00:00Z)", "Computer": "'*'", "Severity": "'warning'"}, + "expected": { + "columns": [{"name": "Computer", "type": "string"}, {"name": "Hour", "type": "datetime"}, {"name": "Events", "type": "long"}], + "ordered": true, + "rows": [["beta", "2026-01-02T11:00:00Z", 1]] + } + }, + { + "id": "computer-picker", + "sourcePath": "/items/0/content/parameters/3/query", + "outputPath": "/items/0/content/parameters/1/query", + "substitutions": {}, + "expected": {"columns": [{"name": "Computer", "type": "string"}], "ordered": true, "rows": [["alpha"], ["beta"], ["gamma"]]} + }, + { + "id": "click-drilldown", + "sourcePath": "/items/1/content/items/1/content/query", + "substitutions": {"SelectedComputer": "beta"}, + "expected": {"columns": [{"name": "Computer", "type": "string"}, {"name": "SeverityLevel", "type": "string"}], "rows": [["beta", "warning"], ["beta", "warning"]]} + } + ] +} diff --git a/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/notebook.json b/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/notebook.json new file mode 100644 index 00000000000..c0b4705f23b --- /dev/null +++ b/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/notebook.json @@ -0,0 +1,62 @@ +{ + "version": "Notebook/1.0", + "context": {"ownerId": "/subscriptions/example/resourceGroups/example/providers/Microsoft.OperationalInsights/workspaces/example"}, + "fallbackResourceIds": ["{Workspace}"], + "isLocked": true, + "styleSettings": {"padding": "4px"}, + "items": [ + { + "type": 9, + "name": "filters", + "content": { + "version": "KqlParameterItem/1.0", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": ["{Workspace}"], + "parameters": [ + {"name": "Subscription", "type": 6, "queryType": 1}, + {"name": "Workspace", "type": 5, "typeSettings": {"resourceTypeFilter": {"microsoft.operationalinsights/workspaces": true}}, "crossComponentResources": ["{Subscription}"]}, + {"name": "TimeRange", "type": 4, "value": {"durationMs": 7200000}}, + {"name": "Computer", "type": 5, "query": "Syslog | summarize Events=count() by Computer | order by Events desc, Computer asc | project Computer", "queryType": 0, "multiSelect": true, "quote": "'", "delimiter": ",", "value": ["value::all"]}, + {"name": "Severity", "type": 2, "value": ["warning"]} + ] + } + }, + { + "type": 12, + "name": "group", + "content": { + "version": "NotebookGroup/1.0", + "items": [ + { + "type": 3, + "name": "counts", + "customWidth": "50", + "content": { + "queryType": 0, + "query": "Syslog\n| where TimeGenerated between ({TimeRange:start} .. {TimeRange:end})\n| where '*' in ({Computer}) or Computer in ({Computer})\n| where '*' in ({Severity}) or SeverityLevel in ({Severity})\n| summarize Events=count() by Computer, Hour=bin(TimeGenerated, 1h)\n| order by Computer asc, Hour asc", + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": ["{Workspace}"], + "timeContextFromParameter": "TimeRange", + "visualization": "table", + "exportFieldName": "Computer", + "exportParameterName": "SelectedComputer", + "gridSettings": {"rowLimit": 50} + } + }, + { + "type": 3, + "name": "drilldown", + "conditionalVisibility": {"parameterName": "SelectedComputer", "comparison": "isNotEqualTo", "value": ""}, + "content": { + "queryType": 0, + "query": "Syslog | where Computer == '{SelectedComputer}' | project Computer, SeverityLevel", + "visualization": "table" + } + } + ] + } + }, + {"type": 1, "name": "explanation", "content": {"json": "Synthetic conversion fixture, not security detection content."}} + ] +} diff --git a/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/sources/Azure Activity/Workbooks/AzureActivity.json b/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/sources/Azure Activity/Workbooks/AzureActivity.json new file mode 100644 index 00000000000..70fe31b4d24 --- /dev/null +++ b/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/sources/Azure Activity/Workbooks/AzureActivity.json @@ -0,0 +1,336 @@ +{ + "version": "Notebook/1.0", + "items": [ + { + "type": 9, + "content": { + "version": "KqlParameterItem/1.0", + "query": "", + "crossComponentResources": [], + "parameters": [ + { + "id": "52bfbd84-1639-480c-bda5-bfc87fd81832", + "version": "KqlParameterItem/1.0", + "name": "TimeRange", + "type": 4, + "isRequired": true, + "value": { + "durationMs": 604800000 + }, + "typeSettings": { + "selectableValues": [ + { + "durationMs": 300000 + }, + { + "durationMs": 900000 + }, + { + "durationMs": 1800000 + }, + { + "durationMs": 3600000 + }, + { + "durationMs": 14400000 + }, + { + "durationMs": 43200000 + }, + { + "durationMs": 86400000 + }, + { + "durationMs": 172800000 + }, + { + "durationMs": 259200000 + }, + { + "durationMs": 604800000 + }, + { + "durationMs": 1209600000 + }, + { + "durationMs": 2419200000 + }, + { + "durationMs": 2592000000 + }, + { + "durationMs": 5184000000 + }, + { + "durationMs": 7776000000 + } + ] + } + }, + { + "id": "eeb5dcf9-e898-46af-9c12-d91d97e13cd3", + "version": "KqlParameterItem/1.0", + "name": "Caller", + "type": 2, + "isRequired": true, + "multiSelect": true, + "quote": "'", + "delimiter": ",", + "query": "AzureActivity\r\n| summarize by Caller", + "value": [ + "value::all" + ], + "typeSettings": { + "additionalResourceOptions": [ + "value::all" + ], + "selectAllValue": "All" + }, + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + { + "id": "46375a76-7ae1-4d7e-9082-4191531198a9", + "version": "KqlParameterItem/1.0", + "name": "ResourceGroup", + "type": 2, + "isRequired": true, + "multiSelect": true, + "quote": "'", + "delimiter": ",", + "query": "AzureActivity\r\n| summarize by ResourceGroup", + "value": [ + "value::all" + ], + "typeSettings": { + "resourceTypeFilter": { + "microsoft.resources/resourcegroups": true + }, + "additionalResourceOptions": [ + "value::all" + ], + "selectAllValue": "All" + }, + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + } + ], + "style": "pills", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + "name": "parameters - 2" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "let data = AzureActivity\r\n| where \"{Caller:lable}\" == \"All\" or \"{Caller:lable}\" == \"All\" or Caller in ({Caller})\r\n| where \"{ResourceGroup:lable}\" == \"All\" or \"{ResourceGroup:lable}\" == \"All\" or ResourceGroup in ({ResourceGroup});\r\ndata\r\n| summarize Count = count() by ResourceGroup\r\n| join kind = fullouter (datatable(ResourceGroup:string)['Medium', 'high', 'low']) on ResourceGroup\r\n| project ResourceGroup = iff(ResourceGroup == '', ResourceGroup1, ResourceGroup), Count = iff(ResourceGroup == '', 0, Count)\r\n| join kind = inner (data\r\n | make-series Trend = count() default = 0 on TimeGenerated from {TimeRange:start} to {TimeRange:end} step {TimeRange:grain} by ResourceGroup)\r\n on ResourceGroup\r\n| project-away ResourceGroup1, TimeGenerated\r\n| extend ResourceGroups = ResourceGroup\r\n| union (\r\n data \r\n | summarize Count = count() \r\n | extend jkey = 1\r\n | join kind=inner (data\r\n | make-series Trend = count() default = 0 on TimeGenerated from {TimeRange:start} to {TimeRange:end} step {TimeRange:grain}\r\n | extend jkey = 1) on jkey\r\n | extend ResourceGroup = 'All', ResourceGroups = '*' \r\n)\r\n| order by Count desc\r\n| take 10", + "size": 4, + "exportToExcelOptions": "visible", + "title": "Top 10 active resource groups", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "tiles", + "tileSettings": { + "titleContent": { + "columnMatch": "ResourceGroup", + "formatter": 1, + "formatOptions": { + "showIcon": true + } + }, + "leftContent": { + "columnMatch": "Count", + "formatter": 12, + "formatOptions": { + "palette": "auto", + "showIcon": true + }, + "numberFormat": { + "unit": 17, + "options": { + "maximumSignificantDigits": 3, + "maximumFractionDigits": 2 + } + } + }, + "secondaryContent": { + "columnMatch": "Trend", + "formatter": 9, + "formatOptions": { + "palette": "blueOrange", + "showIcon": true + } + }, + "showBorder": false + } + }, + "name": "query - 3" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureActivity\r\n| where \"{Caller:lable}\" == \"All\" or Caller in ({Caller})\r\n| where \"{ResourceGroup:lable}\" == \"All\" or ResourceGroup in ({ResourceGroup})\r\n| summarize deletions = countif(OperationNameValue hassuffix \"delete\"), creations = countif(OperationNameValue hassuffix \"write\"), updates = countif(OperationNameValue hassuffix \"write\"), Activities = count(OperationNameValue) by bin_at(TimeGenerated, 1h, now())\r\n", + "size": 0, + "exportToExcelOptions": "visible", + "title": "Activities over time", + "color": "gray", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "linechart", + "graphSettings": { + "type": 0 + } + }, + "name": "query - 1" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureActivity\r\n| where \"{Caller:lable}\" == \"All\" or Caller in ({Caller})\r\n| where \"{ResourceGroup:lable}\" == \"All\" or ResourceGroup in ({ResourceGroup})\r\n| summarize deletions = countif(OperationNameValue hassuffix \"Delete\"), creations = countif(OperationNameValue hassuffix \"write\"), updates = countif(OperationNameValue hassuffix \"write\"), Activities = count() by Caller\r\n", + "size": 1, + "exportToExcelOptions": "visible", + "title": "Caller activities", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "gridSettings": { + "formatters": [ + { + "columnMatch": "Caller", + "formatter": 0, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "deletions", + "formatter": 4, + "formatOptions": { + "showIcon": true, + "aggregation": "Count" + } + }, + { + "columnMatch": "creations", + "formatter": 4, + "formatOptions": { + "palette": "purple", + "showIcon": true, + "aggregation": "Count" + } + }, + { + "columnMatch": "updates", + "formatter": 4, + "formatOptions": { + "palette": "gray", + "showIcon": true, + "aggregation": "Count" + } + }, + { + "columnMatch": "Activities", + "formatter": 4, + "formatOptions": { + "palette": "greenDark", + "linkTarget": "GenericDetails", + "linkIsContextBlade": true, + "showIcon": true, + "aggregation": "Count", + "workbookContext": { + "componentIdSource": "workbook", + "resourceIdsSource": "workbook", + "templateIdSource": "static", + "templateId": "https://go.microsoft.com/fwlink/?linkid=874159&resourceId=%2Fsubscriptions%2F44e4eff8-1fcb-4a22-a7d6-992ac7286382%2FresourceGroups%2FSOC&featureName=Workbooks&itemId=%2Fsubscriptions%2F44e4eff8-1fcb-4a22-a7d6-992ac7286382%2Fresourcegroups%2Fsoc%2Fproviders%2Fmicrosoft.insights%2Fworkbooks%2F4c195aec-747f-40bb-addb-934acb3ec646&name=CiscoASA&func=NavigateToPortalFeature&type=workbook", + "typeSource": "workbook", + "gallerySource": "workbook" + } + } + } + ], + "sortBy": [ + { + "itemKey": "$gen_bar_updates_3", + "sortOrder": 2 + } + ], + "labelSettings": [] + } + }, + "name": "query - 1" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureActivity \r\n| where \"{Caller:lable}\" == \"All\" or Caller in ({Caller})\r\n| where \"{ResourceGroup:lable}\" == \"All\" or ResourceGroup in ({ResourceGroup})\r\n| summarize Informational = countif(Level == \"Informational\"), Warning = countif(Level == \"Warning\"), Error = countif(Level == \"Error\") by bin_at(TimeGenerated, 1h, now())\r\n", + "size": 0, + "exportToExcelOptions": "visible", + "title": "Activities by log level over time", + "color": "redBright", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "scatterchart", + "tileSettings": { + "showBorder": false + }, + "graphSettings": { + "type": 2, + "topContent": { + "columnMatch": "Error", + "formatter": 12, + "formatOptions": { + "showIcon": true + } + }, + "hivesContent": { + "columnMatch": "TimeGenerated", + "formatter": 1, + "formatOptions": { + "showIcon": true + } + }, + "nodeIdField": "Error", + "sourceIdField": "Error", + "targetIdField": "Error", + "nodeSize": null, + "staticNodeSize": 100, + "colorSettings": null, + "groupByField": "TimeGenerated", + "hivesMargin": 5 + } + }, + "name": "query - 4" + } + ], + "styleSettings": {}, + "fromTemplateId": "sentinel-AzureActivity", + "$schema": "https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json" +} \ No newline at end of file diff --git a/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/sources/Azure Activity/Workbooks/AzureServiceHealthWorkbook.json b/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/sources/Azure Activity/Workbooks/AzureServiceHealthWorkbook.json new file mode 100644 index 00000000000..26b634d90cf --- /dev/null +++ b/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/sources/Azure Activity/Workbooks/AzureServiceHealthWorkbook.json @@ -0,0 +1,977 @@ +{ + "version": "Notebook/1.0", + "items": [ + { + "type": 1, + "content": { + "json": "## Azure Activity Logs for Azure Service Health Analysis Workbook" + }, + "name": "text - 1" + }, + { + "type": 9, + "content": { + "version": "KqlParameterItem/1.0", + "crossComponentResources": [ + "{Subscription}" + ], + "parameters": [ + { + "id": "76e0f423-6828-47f3-aa86-522604bbbc6f", + "version": "KqlParameterItem/1.0", + "name": "Subscription", + "type": 6, + "description": "Azure Subscription where is the Log Analytics with Activity Logs informaiton", + "isRequired": true, + "query": "summarize by subscriptionId\r\n| project value = strcat(\"/subscriptions/\",subscriptionId), label = subscriptionId\r\n| order by value ", + "crossComponentResources": [ + "value::all" + ], + "typeSettings": { + "additionalResourceOptions": [], + "showDefault": false + }, + "timeContext": { + "durationMs": 86400000 + }, + "queryType": 1, + "resourceType": "microsoft.resourcegraph/resources", + "value": "/subscriptions/fcb7d51b-418f-45a9-8418-5c0f17e343c8" + }, + { + "id": "ea100a5b-3c79-4761-8235-6a7989c83da5", + "version": "KqlParameterItem/1.0", + "name": "LAworkspace", + "label": "Log Analytics workspace", + "type": 5, + "description": "Log Analytics workspace where the Activity Logs information is sent", + "isRequired": true, + "query": "where type == 'microsoft.operationalinsights/workspaces'\r\n| project id\r\n| order by id asc", + "crossComponentResources": [ + "{Subscription}" + ], + "typeSettings": { + "additionalResourceOptions": [], + "showDefault": false + }, + "timeContext": { + "durationMs": 86400000 + }, + "queryType": 1, + "resourceType": "microsoft.resourcegraph/resources", + "value": "" + }, + { + "id": "6b112ffe-a3a2-4b9e-a17a-c4ea5af739d7", + "version": "KqlParameterItem/1.0", + "name": "TimeRange", + "type": 4, + "typeSettings": { + "selectableValues": [ + { + "durationMs": 3600000 + }, + { + "durationMs": 86400000 + }, + { + "durationMs": 259200000 + }, + { + "durationMs": 604800000 + }, + { + "durationMs": 1209600000 + }, + { + "durationMs": 2592000000 + } + ], + "allowCustom": true + }, + "timeContext": { + "durationMs": 86400000 + }, + "value": { + "durationMs": 2592000000 + } + } + ], + "style": "pills", + "queryType": 1, + "resourceType": "microsoft.resourcegraph/resources" + }, + "name": "parameters - 0" + }, + { + "type": 9, + "content": { + "version": "KqlParameterItem/1.0", + "crossComponentResources": [ + "{LAworkspace}" + ], + "parameters": [ + { + "id": "2fb175cb-a2b6-4dd3-9473-886a8db4cd9c", + "version": "KqlParameterItem/1.0", + "name": "TrackingID_P", + "label": "Tracking ID", + "type": 2, + "isRequired": true, + "multiSelect": true, + "quote": "'", + "delimiter": ",", + "query": "AzureActivity\r\n| where CategoryValue == \"ServiceHealth\"\r\n| extend TrackingId = tostring(Properties_d.trackingId)\r\n| distinct TrackingId\r\n| order by TrackingId asc", + "crossComponentResources": [ + "{LAworkspace}" + ], + "typeSettings": { + "additionalResourceOptions": [ + "value::all" + ], + "selectAllValue": "All", + "showDefault": false + }, + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "value": [ + "value::all" + ] + } + ], + "style": "pills", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + "name": "parameters - 3" + }, + { + "type": 12, + "content": { + "version": "NotebookGroup/1.0", + "groupType": "editable", + "items": [ + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureActivity\r\n| where CategoryValue == \"ServiceHealth\" \r\n| summarize TimeGenerated=arg_max(TimeGenerated,*) by SubscriptionId\r\n| extend SubscriptionId == strcat('/subscriptions/', SubscriptionId)", + "size": 1, + "showAnalytics": true, + "title": "Subscriptions Impacted", + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": [ + "{LAworkspace}" + ], + "visualization": "tiles", + "gridSettings": { + "formatters": [ + { + "columnMatch": "TimeGenerated", + "formatter": 0, + "formatOptions": { + "customColumnWidthSetting": "22ch" + } + }, + { + "columnMatch": "TrackingId", + "formatter": 7, + "formatOptions": { + "linkTarget": "GenericDetails", + "linkIsContextBlade": true, + "customColumnWidthSetting": "14ch" + } + }, + { + "columnMatch": "Status", + "formatter": 18, + "formatOptions": { + "thresholdsOptions": "icons", + "thresholdsGrid": [ + { + "operator": "==", + "thresholdValue": "Active", + "representation": "2", + "text": "{0}{1}" + }, + { + "operator": "==", + "thresholdValue": "Resolved", + "representation": "Resolved", + "text": "{0}{1}" + }, + { + "operator": "Default", + "thresholdValue": null, + "representation": "more", + "text": "{0}{1}" + } + ], + "customColumnWidthSetting": "12ch" + } + }, + { + "columnMatch": "Impact Start Time", + "formatter": 5 + }, + { + "columnMatch": "Level", + "formatter": 18, + "formatOptions": { + "thresholdsOptions": "icons", + "thresholdsGrid": [ + { + "operator": "==", + "thresholdValue": "Information", + "representation": "info", + "text": "{0}{1}" + }, + { + "operator": "==", + "thresholdValue": "Warning", + "representation": "warning", + "text": "{0}{1}" + }, + { + "operator": "==", + "thresholdValue": "Error", + "representation": "error", + "text": "{0}{1}" + }, + { + "operator": "Default", + "thresholdValue": null, + "representation": "Unknown", + "text": "{0}{1}" + } + ] + } + }, + { + "columnMatch": "IncidentType", + "formatter": 18, + "formatOptions": { + "thresholdsOptions": "icons", + "thresholdsGrid": [ + { + "operator": "==", + "thresholdValue": "Incident", + "representation": "error", + "text": "{0}{1}" + }, + { + "operator": "==", + "thresholdValue": "Maintenance", + "representation": "Tools", + "text": "{0}{1}" + }, + { + "operator": "==", + "thresholdValue": "ActionRequired", + "representation": "Clock", + "text": "{0}{1}" + }, + { + "operator": "==", + "thresholdValue": "Informational", + "representation": "1", + "text": "{0}{1}" + }, + { + "operator": "Default", + "thresholdValue": null, + "representation": "more", + "text": "{0}{1}" + } + ] + } + }, + { + "columnMatch": "ImpactedSubscriptions", + "formatter": 5 + }, + { + "columnMatch": "# Subs.", + "formatter": 8, + "formatOptions": { + "min": -1, + "palette": "red" + } + }, + { + "columnMatch": "ImpactedRegions", + "formatter": 5 + }, + { + "columnMatch": "# Regions", + "formatter": 8, + "formatOptions": { + "min": -1, + "palette": "magenta" + } + }, + { + "columnMatch": "ImpactMitigationTime", + "formatter": 5 + }, + { + "columnMatch": "ResolutionTime (hours)", + "formatter": 4, + "formatOptions": { + "min": -1, + "palette": "orange", + "customColumnWidthSetting": "26ch" + } + } + ] + }, + "sortBy": [], + "tileSettings": { + "titleContent": { + "columnMatch": "SubscriptionId", + "formatter": 13, + "formatOptions": { + "linkTarget": null, + "showIcon": true + } + }, + "showBorder": false + } + }, + "customWidth": "30", + "name": "query - 0 - Copy", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureActivity\r\n| where CategoryValue == \"ServiceHealth\"\r\n| extend TrackingId = tostring(Properties_d.trackingId)\r\n| summarize arg_max(TimeGenerated,*) by TrackingId\r\n| where TrackingId in ({TrackingID_P:value}) or '{TrackingID_P:label}'=='All'\r\n| extend ImpactedRegions = tostring(Properties_d.region)\r\n| summarize count() by ImpactedRegions", + "size": 1, + "title": "Impacted Regions", + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "tiles", + "tileSettings": { + "titleContent": { + "columnMatch": "ImpactedRegions", + "formatter": 1 + }, + "leftContent": { + "columnMatch": "count_", + "formatter": 12, + "formatOptions": { + "palette": "auto" + }, + "numberFormat": { + "unit": 17, + "options": { + "maximumSignificantDigits": 3, + "maximumFractionDigits": 2 + } + } + }, + "showBorder": false + } + }, + "customWidth": "35", + "name": "query - 4", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureActivity\r\n| where CategoryValue == \"ServiceHealth\"\r\n| extend TrackingId = tostring(Properties_d.trackingId)\r\n| summarize arg_max(TimeGenerated,*) by TrackingId\r\n| where TrackingId in ({TrackingID_P:value}) or '{TrackingID_P:label}'=='All'\r\n| extend IncidentType= tostring(Properties_d.incidentType)\r\n| summarize count() by IncidentType", + "size": 1, + "title": "Type of Incidents", + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "piechart", + "tileSettings": { + "titleContent": { + "columnMatch": "IncidentType", + "formatter": 1 + }, + "leftContent": { + "columnMatch": "count_", + "formatter": 12, + "formatOptions": { + "palette": "auto" + }, + "numberFormat": { + "unit": 17, + "options": { + "maximumSignificantDigits": 3, + "maximumFractionDigits": 2 + } + } + }, + "showBorder": false + }, + "chartSettings": { + "seriesLabelSettings": [ + { + "seriesName": "Maintenance", + "color": "gray" + }, + { + "seriesName": "Incident", + "color": "yellowDark" + }, + { + "seriesName": "ActionRequired", + "color": "redBright" + } + ] + } + }, + "customWidth": "35", + "name": "query - 4 - Copy", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureActivity\r\n| where CategoryValue == \"ServiceHealth\"\r\n| extend TrackingId = tostring(Properties_d.trackingId)\r\n| summarize arg_max(TimeGenerated,*) by TrackingId\r\n| where TrackingId in ({TrackingID_P:value}) or '{TrackingID_P:label}'=='All'\r\n| extend Status = tostring(Properties_d.activityStatusValue)\r\n| summarize count() by Status", + "size": 0, + "title": "Service Health on Status", + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "piechart", + "tileSettings": { + "titleContent": { + "columnMatch": "IncidentType", + "formatter": 1 + }, + "leftContent": { + "columnMatch": "count_", + "formatter": 12, + "formatOptions": { + "palette": "auto" + }, + "numberFormat": { + "unit": 17, + "options": { + "maximumSignificantDigits": 3, + "maximumFractionDigits": 2 + } + } + }, + "showBorder": false + }, + "chartSettings": { + "seriesLabelSettings": [ + { + "seriesName": "Active", + "color": "red" + }, + { + "seriesName": "Resolved", + "color": "green" + } + ] + } + }, + "customWidth": "45", + "name": "query - 4 - Copy - Copy", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureActivity\r\n| where CategoryValue == \"ServiceHealth\"\r\n| extend TrackingId = tostring(Properties_d.trackingId)\r\n| summarize arg_max(TimeGenerated,*) by TrackingId\r\n| where TrackingId in ({TrackingID_P:value}) or '{TrackingID_P:label}'=='All'\r\n| extend Service = tostring(Properties_d.service)\r\n| summarize count() by Service", + "size": 0, + "title": "Service Health Across Services", + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "categoricalbar", + "tileSettings": { + "titleContent": { + "columnMatch": "IncidentType", + "formatter": 1 + }, + "leftContent": { + "columnMatch": "count_", + "formatter": 12, + "formatOptions": { + "palette": "auto" + }, + "numberFormat": { + "unit": 17, + "options": { + "maximumSignificantDigits": 3, + "maximumFractionDigits": 2 + } + } + }, + "showBorder": false + }, + "chartSettings": { + "seriesLabelSettings": [ + { + "seriesName": "Active", + "color": "red" + }, + { + "seriesName": "Resolved", + "color": "green" + } + ] + } + }, + "customWidth": "55", + "name": "query - 4 - Copy - Copy - Copy", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureActivity\r\n| where CategoryValue == \"ServiceHealth\"\r\n| extend TrackingId = tostring(Properties_d.trackingId)\r\n| summarize arg_max(TimeGenerated,*) by TrackingId\r\n| where TrackingId in ({TrackingID_P:value}) or '{TrackingID_P:label}'=='All'\r\n| extend Service = tostring(Properties_d.service)\r\n| extend ImpactMitigationTime = tostring(Properties_d.impactMitigationTime)\r\n| extend ImpactStartTime = tostring(Properties_d.impactStartTime)\r\n| extend [\"ResolutionTime\"]=datetime_diff('hour',todatetime(ImpactMitigationTime),todatetime(ImpactStartTime))\r\n| extend ImpactedRegions = todynamic(parse_json(tostring(parse_json(tostring(parse_json(Properties).impactedServices))[0].ImpactedRegions)))\r\n| mv-expand ImpactedRegions\r\n| project Service,ImpactedRegions.RegionName, [\"ResolutionTime\"], _ResourceId", + "size": 0, + "title": "Resolution Time", + "timeContextFromParameter": "TimeRange", + "showExportToExcel": true, + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "table", + "gridSettings": { + "formatters": [ + { + "columnMatch": "$gen_group", + "formatter": 13, + "formatOptions": { + "linkTarget": null, + "showIcon": true + } + }, + { + "columnMatch": "ResolutionTime", + "formatter": 8, + "formatOptions": { + "palette": "redDark" + }, + "numberFormat": { + "unit": 25, + "options": { + "style": "decimal", + "useGrouping": false + } + } + }, + { + "columnMatch": "_ResourceId", + "formatter": 5 + } + ], + "filter": true, + "hierarchySettings": { + "treeType": 1, + "groupBy": [ + "_ResourceId" + ], + "expandTopLevel": true + } + }, + "tileSettings": { + "titleContent": { + "columnMatch": "IncidentType", + "formatter": 1 + }, + "leftContent": { + "columnMatch": "count_", + "formatter": 12, + "formatOptions": { + "palette": "auto" + }, + "numberFormat": { + "unit": 17, + "options": { + "maximumSignificantDigits": 3, + "maximumFractionDigits": 2 + } + } + }, + "showBorder": false + }, + "chartSettings": { + "seriesLabelSettings": [ + { + "seriesName": "Active", + "color": "red" + }, + { + "seriesName": "Resolved", + "color": "green" + } + ] + } + }, + "customWidth": "70", + "name": "query - 4 - Copy - Copy - Copy - Copy", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureActivity\r\n| where CategoryValue == \"ServiceHealth\"\r\n| extend TrackingId = tostring(Properties_d.trackingId)\r\n| summarize arg_max(TimeGenerated,*) by TrackingId\r\n| where TrackingId in ({TrackingID_P:value}) or '{TrackingID_P:label}'=='All'\r\n| extend Stage = tostring(parse_json(Properties).stage)\r\n| summarize count() by Stage", + "size": 0, + "title": "Current Stage", + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "tiles", + "tileSettings": { + "titleContent": { + "columnMatch": "Stage", + "formatter": 1 + }, + "leftContent": { + "columnMatch": "count_", + "formatter": 12, + "formatOptions": { + "palette": "auto" + }, + "numberFormat": { + "unit": 17, + "options": { + "maximumSignificantDigits": 3, + "maximumFractionDigits": 2 + } + } + }, + "showBorder": false + }, + "chartSettings": { + "seriesLabelSettings": [ + { + "seriesName": "Active", + "color": "red" + }, + { + "seriesName": "Resolved", + "color": "green" + } + ] + } + }, + "customWidth": "30", + "name": "query - 4 - Copy - Copy - Copy - Copy - Copy", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureActivity\r\n| where CategoryValue == \"ServiceHealth\"\r\n| extend TrackingId = tostring(Properties_d.trackingId)\r\n| summarize arg_max(TimeGenerated,) by TrackingId\r\n| where TrackingId in ({TrackingID_P:value}) or '{TrackingID_P:label}'=='All'\r\n//| where Level in ('Warning','Error','Information')\r\n| extend IncidentType= tostring(Properties_d.incidentType)\r\n//| where IncidentType in ('Incident','Maintenance','ActionRequired')\r\n| extend ImpactMitigationTime = tostring(Properties_d.impactMitigationTime)\r\n| extend ImpactStartTime = tostring(Properties_d.impactStartTime)\r\n| extend Status = tostring(Properties_d.activityStatusValue)\r\n| extend Service = tostring(Properties_d.service)\r\n| extend Title = tostring(Properties_d.defaultLanguageTitle)\r\n| extend ImpactedRegions = tostring(parse_json(tostring(parse_json(tostring(parse_json(Properties).impactedServices))[0].ImpactedRegions)))\r\n| summarize ImpactedSubscriptions=make_set(SubscriptionId), TimeGenerated=arg_max(TimeGenerated,) by Level, ImpactedRegions, Service, TrackingId, Status\r\n| project TimeGenerated\r\n , TrackingId\r\n , Title\r\n , Status\r\n , [\"Impact Start Time\"]=todatetime(ImpactStartTime)\r\n , Level\r\n , IncidentType\r\n , ImpactedSubscriptions\r\n , Service\r\n , [\"# Subs.\"]=array_length(ImpactedSubscriptions)\r\n , ImpactedRegions\r\n , [\"# Regions\"]=array_length(parse_json(ImpactedRegions))\r\n , todatetime(ImpactMitigationTime)\r\n , [\"ResolutionTime (hours)\"]=datetime_diff('hour',todatetime(ImpactMitigationTime),todatetime(ImpactStartTime))\r\n| order by TimeGenerated ", + "size": 0, + "showAnalytics": true, + "title": "Analysis of incident", + "timeContextFromParameter": "TimeRange", + "showExportToExcel": true, + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": [ + "{LAworkspace}" + ], + "gridSettings": { + "formatters": [ + { + "columnMatch": "TimeGenerated", + "formatter": 0, + "formatOptions": { + "customColumnWidthSetting": "22ch" + } + }, + { + "columnMatch": "TrackingId", + "formatter": 7, + "formatOptions": { + "linkTarget": "GenericDetails", + "linkIsContextBlade": true, + "customColumnWidthSetting": "14ch" + } + }, + { + "columnMatch": "Status", + "formatter": 18, + "formatOptions": { + "thresholdsOptions": "icons", + "thresholdsGrid": [ + { + "operator": "==", + "thresholdValue": "Active", + "representation": "2", + "text": "{0}{1}" + }, + { + "operator": "==", + "thresholdValue": "Resolved", + "representation": "Resolved", + "text": "{0}{1}" + }, + { + "operator": "Default", + "thresholdValue": null, + "representation": "more", + "text": "{0}{1}" + } + ], + "customColumnWidthSetting": "12ch" + } + }, + { + "columnMatch": "Impact Start Time", + "formatter": 5 + }, + { + "columnMatch": "Level", + "formatter": 18, + "formatOptions": { + "thresholdsOptions": "icons", + "thresholdsGrid": [ + { + "operator": "==", + "thresholdValue": "Information", + "representation": "info", + "text": "{0}{1}" + }, + { + "operator": "==", + "thresholdValue": "Warning", + "representation": "warning", + "text": "{0}{1}" + }, + { + "operator": "==", + "thresholdValue": "Error", + "representation": "error", + "text": "{0}{1}" + }, + { + "operator": "Default", + "thresholdValue": null, + "representation": "Unknown", + "text": "{0}{1}" + } + ] + } + }, + { + "columnMatch": "IncidentType", + "formatter": 18, + "formatOptions": { + "thresholdsOptions": "icons", + "thresholdsGrid": [ + { + "operator": "==", + "thresholdValue": "Incident", + "representation": "error", + "text": "{0}{1}" + }, + { + "operator": "==", + "thresholdValue": "Maintenance", + "representation": "Tools", + "text": "{0}{1}" + }, + { + "operator": "==", + "thresholdValue": "ActionRequired", + "representation": "Clock", + "text": "{0}{1}" + }, + { + "operator": "==", + "thresholdValue": "Informational", + "representation": "1", + "text": "{0}{1}" + }, + { + "operator": "Default", + "thresholdValue": null, + "representation": "more", + "text": "{0}{1}" + } + ] + } + }, + { + "columnMatch": "ImpactedSubscriptions", + "formatter": 5 + }, + { + "columnMatch": "# Subs.", + "formatter": 8, + "formatOptions": { + "min": -1, + "palette": "red" + } + }, + { + "columnMatch": "ImpactedRegions", + "formatter": 5 + }, + { + "columnMatch": "# Regions", + "formatter": 8, + "formatOptions": { + "min": -1, + "palette": "magenta" + } + }, + { + "columnMatch": "ImpactMitigationTime", + "formatter": 5 + }, + { + "columnMatch": "ResolutionTime (hours)", + "formatter": 4, + "formatOptions": { + "min": -1, + "palette": "orange", + "customColumnWidthSetting": "26ch" + } + } + ], + "filter": true, + "sortBy": [ + { + "itemKey": "$gen_link_TrackingId_1", + "sortOrder": 1 + } + ] + }, + "sortBy": [ + { + "itemKey": "$gen_link_TrackingId_1", + "sortOrder": 1 + } + ] + }, + "name": "query - 0", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureActivity\r\n| where CategoryValue == \"ServiceHealth\"\r\n| extend TrackingId = tostring(Properties_d.trackingId)\r\n| where TrackingId in ({TrackingID_P:value}) or '{TrackingID_P:label}'=='All'\r\n| summarize ImpactedSubscriptions=make_set(SubscriptionId) by TrackingId\r\n| project ImpactedSubscriptions, TrackingId\r\n| mv-expand ImpactedSubscriptions\r\n| project Subscription=strcat('/subscriptions/',ImpactedSubscriptions), TrackingId\r\n| extend Info=strcat('https://app.azure.com/h/',TrackingId)\r\n| order by TrackingId asc", + "size": 0, + "showAnalytics": true, + "title": "Affected subscriptions", + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": [ + "{LAworkspace}" + ], + "gridSettings": { + "formatters": [ + { + "columnMatch": "TrackingId", + "formatter": 1, + "formatOptions": { + "linkColumn": "Info", + "linkTarget": "Url" + } + }, + { + "columnMatch": "Info", + "formatter": 5, + "formatOptions": { + "linkTarget": "Url", + "linkLabel": "Shareable link" + } + } + ], + "filter": true + }, + "sortBy": [] + }, + "customWidth": "50", + "name": "query - 2", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "AzureActivity\r\n| where CategoryValue == \"ServiceHealth\"\r\n| extend TrackingId = tostring(Properties_d.trackingId)\r\n| where TrackingId in ({TrackingID_P:value}) or '{TrackingID_P:label}'=='All'\r\n| extend ImpactedRegions = tostring(parse_json(tostring(parse_json(tostring(parse_json(Properties).impactedServices))[0].ImpactedRegions)))\r\n| project todynamic(ImpactedRegions), TrackingId\r\n| mv-expand ImpactedRegions\r\n| project Region=strcat(tostring(ImpactedRegions.RegionName),' (',tostring(ImpactedRegions.RegionId),')'),TrackingId\r\n| distinct Region, TrackingId\r\n| extend Info=strcat('https://app.azure.com/h/', TrackingId)\r\n| order by TrackingId asc\r\n", + "size": 0, + "showAnalytics": true, + "title": "Affected regions", + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": [ + "{LAworkspace}" + ], + "gridSettings": { + "formatters": [ + { + "columnMatch": "TrackingId", + "formatter": 1, + "formatOptions": { + "linkColumn": "Info", + "linkTarget": "Url" + } + }, + { + "columnMatch": "Info", + "formatter": 5 + } + ], + "filter": true + } + }, + "customWidth": "50", + "name": "query - 1", + "styleSettings": { + "showBorder": true + } + } + ] + }, + "name": "GroupActivityLogs01", + "styleSettings": { + "showBorder": true + } + } + ], + "fallbackResourceIds": [], + "fromTemplateId": "sentinel-AzureServiceHealthWorkbook", + "$schema": "https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json" +} diff --git a/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/sources/Syslog/Workbooks/LinuxMachines.json b/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/sources/Syslog/Workbooks/LinuxMachines.json new file mode 100644 index 00000000000..d7492a31129 --- /dev/null +++ b/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/sources/Syslog/Workbooks/LinuxMachines.json @@ -0,0 +1,392 @@ +{ + "version": "Notebook/1.0", + "items": [ + { + "type": 1, + "content": { + "json": "## Linux Machines" + }, + "name": "text - 0" + }, + { + "type": 9, + "content": { + "version": "KqlParameterItem/1.0", + "query": "", + "crossComponentResources": [], + "parameters": [ + { + "id": "1025a43d-241c-4e40-95dc-c9eb9c789bc5", + "version": "KqlParameterItem/1.0", + "name": "TimeRange", + "type": 4, + "isRequired": true, + "value": { + "durationMs": 1209600000 + }, + "typeSettings": { + "selectableValues": [ + { + "durationMs": 300000 + }, + { + "durationMs": 900000 + }, + { + "durationMs": 1800000 + }, + { + "durationMs": 3600000 + }, + { + "durationMs": 14400000 + }, + { + "durationMs": 43200000 + }, + { + "durationMs": 86400000 + }, + { + "durationMs": 172800000 + }, + { + "durationMs": 259200000 + }, + { + "durationMs": 604800000 + }, + { + "durationMs": 1209600000 + }, + { + "durationMs": 2419200000 + }, + { + "durationMs": 2592000000 + }, + { + "durationMs": 5184000000 + }, + { + "durationMs": 7776000000 + } + ], + "allowCustom": true + } + }, + { + "id": "bc241870-7874-4927-8c74-d17e747522b1", + "version": "KqlParameterItem/1.0", + "name": "Computer", + "type": 5, + "isRequired": true, + "multiSelect": true, + "quote": "'", + "delimiter": ",", + "query": "Syslog\r\n| summarize syslogEventsCount = count() by Computer\r\n| sort by syslogEventsCount desc\r\n| project Computer\r\n", + "value": [ + "value::all" + ], + "typeSettings": { + "additionalResourceOptions": [ + "value::all" + ], + "selectAllValue": "All" + }, + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + { + "id": "e073f36e-2fb5-421d-9099-217205b247f5", + "version": "KqlParameterItem/1.0", + "name": "Severity", + "type": 2, + "isRequired": true, + "multiSelect": true, + "quote": "'", + "delimiter": ",", + "value": [ + "value::all" + ], + "typeSettings": { + "additionalResourceOptions": [ + "value::all" + ], + "selectAllValue": "*" + }, + "jsonData": "[\"Emergency\", \"Alert\", \"Critical\", \"Error\", \"Warning\", \"Notice\", \"Informational\", \"Debug\"]", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange" + } + ], + "style": "pills", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + "name": "parameters - 1" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "Syslog\r\n| summarize count() by SeverityLevel\r\n| extend severityNumber = iif(SeverityLevel == 'emerg' or SeverityLevel == 'panic', 0, iif(SeverityLevel == 'alert', 1, iif(SeverityLevel == 'crit', 2, iif(SeverityLevel == 'err' or SeverityLevel == 'error', 3, iif(SeverityLevel == 'warning' or SeverityLevel == 'warn', 4, iif(SeverityLevel == 'notice', 5, iif(SeverityLevel == 'info', 6, iif(SeverityLevel == 'debug', 7, 8))))))))\r\n| sort by severityNumber asc\r\n| extend Severity = iif(SeverityLevel == 'emerg' or SeverityLevel == 'panic', 'Emergency', iif(SeverityLevel == 'alert', 'Alert', iif(SeverityLevel == 'crit', 'Critical', iif(SeverityLevel == 'err' or SeverityLevel == 'error', 'Error', iif(SeverityLevel == 'warning' or SeverityLevel == 'warn', 'Warning', iif(SeverityLevel == 'notice', 'Notice', iif(SeverityLevel == 'info', 'Informational', iif(SeverityLevel == 'debug', 'Debug', 'Unknown'))))))))\r\n| project-away severityNumber\r\n| where \"*\" in ({Severity}) or Severity in ({Severity})\r\n", + "size": 4, + "exportToExcelOptions": "visible", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "tiles", + "tileSettings": { + "titleContent": { + "columnMatch": "SeverityLevel", + "formatter": 1, + "formatOptions": { + "showIcon": true + } + }, + "leftContent": { + "columnMatch": "count_", + "formatter": 12, + "formatOptions": { + "palette": "hotCold", + "showIcon": true + }, + "numberFormat": { + "unit": 17, + "options": { + "style": "decimal", + "maximumFractionDigits": 2, + "maximumSignificantDigits": 3 + } + } + }, + "showBorder": false, + "sortOrderField": 2 + } + }, + "name": "query - 2" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "Syslog\r\n| where \"{Computer:lable}\" == \"All\" or Computer in ({Computer})\r\n| extend Severity = iif(SeverityLevel == 'emerg' or SeverityLevel == 'panic', 'Emergency', iif(SeverityLevel == 'alert', 'Alert', iif(SeverityLevel == 'crit', 'Critical', iif(SeverityLevel == 'err' or SeverityLevel == 'error', 'Error', iif(SeverityLevel == 'warning' or SeverityLevel == 'warn', 'Warning', iif(SeverityLevel == 'notice', 'Notice', iif(SeverityLevel == 'info', 'Informational', iif(SeverityLevel == 'debug', 'Debug', 'Unknown'))))))))\r\n| where \"*\" in ({Severity}) or Severity in ({Severity})\r\n| where SeverityLevel in (\"emerg\")\r\n| summarize count() by Computer, TimeGenerated\r\n", + "size": 1, + "exportToExcelOptions": "visible", + "title": "\"Emergency\" level events, by computer", + "noDataMessage": "No emergency events within the defined scope", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "timechart" + }, + "customWidth": "33", + "name": "query - 2 - Copy" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "Syslog\r\n| where \"{Computer:lable}\" == \"All\" or Computer in ({Computer})\r\n| extend Severity = iif(SeverityLevel == 'emerg' or SeverityLevel == 'panic', 'Emergency', iif(SeverityLevel == 'alert', 'Alert', iif(SeverityLevel == 'crit', 'Critical', iif(SeverityLevel == 'err' or SeverityLevel == 'error', 'Error', iif(SeverityLevel == 'warning' or SeverityLevel == 'warn', 'Warning', iif(SeverityLevel == 'notice', 'Notice', iif(SeverityLevel == 'info', 'Informational', iif(SeverityLevel == 'debug', 'Debug', 'Unknown'))))))))\r\n| where \"*\" in ({Severity}) or Severity in ({Severity})\r\n| where SeverityLevel in (\"crit\")\r\n| summarize count() by Computer, TimeGenerated\r\n", + "size": 1, + "exportToExcelOptions": "visible", + "title": "\"Critical\" level events, by computer", + "noDataMessage": "No critical events within the defined scope", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "timechart" + }, + "customWidth": "33", + "name": "query - 2 - Copy" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "Syslog\r\n| where \"{Computer:lable}\" == \"All\" or Computer in ({Computer})\r\n| extend Severity = iif(SeverityLevel == 'emerg' or SeverityLevel == 'panic', 'Emergency', iif(SeverityLevel == 'alert', 'Alert', iif(SeverityLevel == 'crit', 'Critical', iif(SeverityLevel == 'err' or SeverityLevel == 'error', 'Error', iif(SeverityLevel == 'warning' or SeverityLevel == 'warn', 'Warning', iif(SeverityLevel == 'notice', 'Notice', iif(SeverityLevel == 'info', 'Informational', iif(SeverityLevel == 'debug', 'Debug', 'Unknown'))))))))\r\n| where \"*\" in ({Severity}) or Severity in ({Severity})\r\n| where SeverityLevel in (\"alert\")\r\n| summarize count() by Computer, TimeGenerated\r\n", + "size": 1, + "exportToExcelOptions": "visible", + "title": "\"Alert\" level events, by computer", + "noDataMessage": "No alert events within the defined scope", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "timechart" + }, + "customWidth": "33", + "name": "query - 2 - Copy - Copy" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "Syslog\r\n| where Computer in ({Computer})\r\n| extend Severity = iif(SeverityLevel == 'emerg' or SeverityLevel == 'panic', 'Emergency', iif(SeverityLevel == 'alert', 'Alert', iif(SeverityLevel == 'crit', 'Critical', iif(SeverityLevel == 'err' or SeverityLevel == 'error', 'Error', iif(SeverityLevel == 'warning' or SeverityLevel == 'warn', 'Warning', iif(SeverityLevel == 'notice', 'Notice', iif(SeverityLevel == 'info', 'Informational', iif(SeverityLevel == 'debug', 'Debug', 'Unknown'))))))))\r\n| extend SeverityNumber = iif(SeverityLevel == 'emerg' or SeverityLevel == 'panic', 0, iif(SeverityLevel == 'alert', 1, iif(SeverityLevel == 'crit', 2, iif(SeverityLevel == 'err' or SeverityLevel == 'error', 3, iif(SeverityLevel == 'warning' or SeverityLevel == 'warn', 4, iif(SeverityLevel == 'notice', 5, iif(SeverityLevel == 'info', 6, iif(SeverityLevel == 'debug', 7, 8))))))))\r\n| where Severity in ({Severity})\r\n|extend Computer = iif(isempty(_ResourceId), Computer, _ResourceId)\r\n| project TimeGenerated, Computer, SeverityLevel, SeverityNumber, Facility, HostIP, ProcessNameAndID = strcat(ProcessName, ' (', iff(isempty(ProcessID), \"-\", tostring(ProcessID)), ')') \r\n", + "size": 0, + "exportToExcelOptions": "visible", + "title": "Events", + "noDataMessage": "No events", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "gridSettings": { + "formatters": [ + { + "columnMatch": "TimeGenerated", + "formatter": 0, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "Computer", + "formatter": 0, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "SeverityLevel", + "formatter": 0, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "SeverityNumber", + "formatter": 8, + "formatOptions": { + "min": 7, + "max": 0, + "palette": "redDark", + "showIcon": true + } + }, + { + "columnMatch": "Facility", + "formatter": 0, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "HostIP", + "formatter": 0, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "ProcessNameAndID", + "formatter": 0, + "formatOptions": { + "showIcon": true + } + } + ], + "labelSettings": [] + } + }, + "name": "query - 2 - Copy - Copy - Copy" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "Syslog\r\n| where \"{Computer:lable}\" == \"All\" or Computer in ({Computer})\r\n| extend Severity = iif(SeverityLevel == 'emerg' or SeverityLevel == 'panic', 'Emergency', iif(SeverityLevel == 'alert', 'Alert', iif(SeverityLevel == 'crit', 'Critical', iif(SeverityLevel == 'err' or SeverityLevel == 'error', 'Error', iif(SeverityLevel == 'warning' or SeverityLevel == 'warn', 'Warning', iif(SeverityLevel == 'notice', 'Notice', iif(SeverityLevel == 'info', 'Informational', iif(SeverityLevel == 'debug', 'Debug', 'Unknown'))))))))\r\n| where \"*\" in ({Severity}) or Severity in ({Severity})\r\n| summarize count() by SyslogMessage\r\n", + "size": 0, + "exportToExcelOptions": "visible", + "title": "Syslog messages of events", + "noDataMessage": "No messages", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "piechart" + }, + "customWidth": "50", + "name": "query - 7" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "Syslog\r\n| where \"{Computer:lable}\" == \"All\" or Computer in ({Computer})\r\n| extend Severity = iif(SeverityLevel == 'emerg' or SeverityLevel == 'panic', 'Emergency', iif(SeverityLevel == 'alert', 'Alert', iif(SeverityLevel == 'crit', 'Critical', iif(SeverityLevel == 'err' or SeverityLevel == 'error', 'Error', iif(SeverityLevel == 'warning' or SeverityLevel == 'warn', 'Warning', iif(SeverityLevel == 'notice', 'Notice', iif(SeverityLevel == 'info', 'Informational', iif(SeverityLevel == 'debug', 'Debug', 'Unknown'))))))))\r\n| where \"*\" in ({Severity}) or Severity in ({Severity})\r\n| summarize count() by Facility, ProcessName\r\n| project Process = strcat(ProcessName, ' (', Facility, ')'), Count = count_ \r\n", + "size": 0, + "exportToExcelOptions": "visible", + "title": "Process names of events", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "piechart" + }, + "customWidth": "50", + "name": "query - 11" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "Syslog\r\n| where \"{Computer:lable}\" == \"All\" or Computer in ({Computer})\r\n| extend Severity = iif(SeverityLevel == 'emerg' or SeverityLevel == 'panic', 'Emergency', iif(SeverityLevel == 'alert', 'Alert', iif(SeverityLevel == 'crit', 'Critical', iif(SeverityLevel == 'err' or SeverityLevel == 'error', 'Error', iif(SeverityLevel == 'warning' or SeverityLevel == 'warn', 'Warning', iif(SeverityLevel == 'notice', 'Notice', iif(SeverityLevel == 'info', 'Informational', iif(SeverityLevel == 'debug', 'Debug', 'Unknown'))))))))\r\n| where \"*\" in ({Severity}) or Severity in ({Severity})\r\n| summarize count() by Facility, SeverityLevel\r\n", + "size": 0, + "exportToExcelOptions": "visible", + "title": "Event distribution, by facility", + "noDataMessage": "No events", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "piechart" + }, + "customWidth": "50", + "name": "query - 7 - Copy - Copy - Copy" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "Syslog\r\n| where \"{Computer:lable}\" == \"All\" or Computer in ({Computer})\r\n| extend Severity = iif(SeverityLevel == 'emerg' or SeverityLevel == 'panic', 'Emergency', iif(SeverityLevel == 'alert', 'Alert', iif(SeverityLevel == 'crit', 'Critical', iif(SeverityLevel == 'err' or SeverityLevel == 'error', 'Error', iif(SeverityLevel == 'warning' or SeverityLevel == 'warn', 'Warning', iif(SeverityLevel == 'notice', 'Notice', iif(SeverityLevel == 'info', 'Informational', iif(SeverityLevel == 'debug', 'Debug', 'Unknown'))))))))\r\n| where \"*\" in ({Severity}) or Severity in ({Severity})\r\n| summarize count() by Facility, SeverityLevel", + "size": 0, + "exportToExcelOptions": "visible", + "title": "Severity levels, by facility", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "categoricalbar" + }, + "customWidth": "50", + "name": "query - 11 - Copy" + } + ], + "styleSettings": {}, + "fromTemplateId": "sentinel-LinuxMachines", + "$schema": "https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json" +} \ No newline at end of file diff --git a/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/sources/Syslog/Workbooks/SyslogConnectorsOverviewWorkbook.json b/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/sources/Syslog/Workbooks/SyslogConnectorsOverviewWorkbook.json new file mode 100644 index 00000000000..aa77c877996 --- /dev/null +++ b/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/sources/Syslog/Workbooks/SyslogConnectorsOverviewWorkbook.json @@ -0,0 +1,268 @@ +{ + "version": "Notebook/1.0", + "items": [ + { + "type": 1, + "content": { + "json": "# Syslog Ingestion Overview\n---\n\nThis Workbook gives an overview of ingestion of logs in the Syslog table" + }, + "name": "text - 2" + }, + { + "type": 9, + "content": { + "version": "KqlParameterItem/1.0", + "crossComponentResources": [ + "{Workspace}" + ], + "parameters": [ + { + "id": "37fd3af4-5d65-41e2-81ae-0720f281eb04", + "version": "KqlParameterItem/1.0", + "name": "Subscription", + "type": 6, + "isRequired": true, + "isGlobal": true, + "multiSelect": true, + "quote": "'", + "delimiter": ",", + "typeSettings": { + "additionalResourceOptions": [ + "value::all" + ], + "includeAll": true, + "showDefault": false + }, + "timeContext": { + "durationMs": 86400000 + }, + "value": [] + }, + { + "id": "0407e413-4562-4a87-b660-72facb921954", + "version": "KqlParameterItem/1.0", + "name": "TimeRange", + "type": 4, + "isRequired": true, + "isGlobal": true, + "typeSettings": { + "selectableValues": [ + { + "durationMs": 300000 + }, + { + "durationMs": 900000 + }, + { + "durationMs": 1800000 + }, + { + "durationMs": 3600000 + }, + { + "durationMs": 14400000 + }, + { + "durationMs": 43200000 + }, + { + "durationMs": 86400000 + }, + { + "durationMs": 172800000 + }, + { + "durationMs": 259200000 + }, + { + "durationMs": 604800000 + }, + { + "durationMs": 1209600000 + }, + { + "durationMs": 2419200000 + }, + { + "durationMs": 2592000000 + }, + { + "durationMs": 5184000000 + }, + { + "durationMs": 7776000000 + } + ] + }, + "timeContext": { + "durationMs": 86400000 + }, + "value": { + "durationMs": 7776000000 + } + }, + { + "id": "f0722820-a155-4a53-a8e3-1483213a5d67", + "version": "KqlParameterItem/1.0", + "name": "Workspace", + "label": "Sentinel Workspace", + "type": 5, + "isRequired": true, + "isGlobal": true, + "multiSelect": true, + "quote": "'", + "delimiter": ",", + "query": "where type =~ 'microsoft.operationalinsights/workspaces'\r\n| project id", + "crossComponentResources": [ + "{Subscription}" + ], + "typeSettings": { + "resourceTypeFilter": { + "microsoft.operationalinsights/workspaces": true + }, + "additionalResourceOptions": [], + "showDefault": false + }, + "timeContext": { + "durationMs": 2592000000 + }, + "queryType": 1, + "resourceType": "microsoft.resourcegraph/resources" + } + ], + "style": "pills", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + "name": "parameters - 0" + }, + { + "type": 1, + "content": { + "json": "## Log Ingestion Overview\r\n##### Select the Device Product to render the time chart\r\n---" + }, + "name": "text - 3" + }, + { + "type": 12, + "content": { + "version": "NotebookGroup/1.0", + "groupType": "editable", + "items": [ + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "SyslogConnectorsOverallStatus({TimeRange:start}, {TimeRange:end})", + "size": 0, + "timeContextFromParameter": "TimeRange", + "exportFieldName": "DeviceProduct", + "exportParameterName": "DeviceProduct", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": [ + "{Workspace}" + ], + "gridSettings": { + "formatters": [ + { + "columnMatch": "ConnectionStatus", + "formatter": 18, + "formatOptions": { + "thresholdsOptions": "icons", + "thresholdsGrid": [ + { + "operator": "==", + "thresholdValue": "Connected", + "representation": "success", + "text": "{0}{1}" + }, + { + "operator": "!=", + "thresholdValue": "Connected", + "representation": "error", + "text": "{0}{1}" + }, + { + "operator": "Default", + "thresholdValue": null, + "representation": "success", + "text": "{0}{1}" + } + ] + }, + "numberFormat": { + "unit": 0, + "options": { + "style": "decimal" + } + } + }, + { + "columnMatch": "OutofBox", + "formatter": 18, + "formatOptions": { + "thresholdsOptions": "icons", + "thresholdsGrid": [ + { + "operator": "==", + "thresholdValue": "Available", + "representation": "Available", + "text": "{0}{1}" + }, + { + "operator": "!=", + "thresholdValue": "Yes", + "representation": "Unavailable", + "text": "{0}{1}" + }, + { + "operator": "Default", + "thresholdValue": null, + "representation": "success", + "text": "{0}{1}" + } + ] + } + } + ], + "labelSettings": [ + { + "columnId": "DeviceProduct", + "label": "Device Product" + } + ] + }, + "sortBy": [] + }, + "customWidth": "50", + "name": "query - 1 - Copy" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "SyslogConnectorsEventVolumebyDeviceProduct({TimeRange:start},{TimeRange:end},\"{DeviceProduct}\")\r\n| render timechart ", + "size": 0, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": [ + "{Workspace}" + ], + "visualization": "timechart", + "chartSettings": { + "showMetrics": false + } + }, + "customWidth": "50", + "name": "query - 2" + } + ] + }, + "name": "IngestedLogs" + } + ], + "fromTemplateId": "sentinel-SyslogOverview", + "$schema": "https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json" +} diff --git a/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/sources/Windows Security Events/Workbooks/EventAnalyzer.json b/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/sources/Windows Security Events/Workbooks/EventAnalyzer.json new file mode 100644 index 00000000000..1ba22b65830 --- /dev/null +++ b/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/sources/Windows Security Events/Workbooks/EventAnalyzer.json @@ -0,0 +1,1076 @@ +{ + "version": "Notebook/1.0", + "items": [ + { + "type": 1, + "content": { + "json": "## Event Analyzer\n---\n\nEvent Analyzer is a visualizing workbook to explore and audit Windows Event Log and explore all events details and attributes for viewing, analyzing and monitoring events recorded in Microsoft Windows event logs. The analyzer speeds up the analysis of event logs (security, application, system, setup, directory service, DNS and others)." + }, + "name": "text - 2" + }, + { + "type": 9, + "content": { + "version": "KqlParameterItem/1.0", + "crossComponentResources": [ + "value::all" + ], + "parameters": [ + { + "id": "6bdfb4ab-a58b-4035-bfca-4b4bdb67aa60", + "version": "KqlParameterItem/1.0", + "name": "DefaultWorkspace", + "type": 5, + "isRequired": true, + "value": "/subscriptions//resourcegroups//providers/microsoft.operationalinsights/workspaces/", + "isHiddenWhenLocked": true, + "typeSettings": { + "resourceTypeFilter": { + "microsoft.operationalinsights/workspaces": true + }, + "additionalResourceOptions": [] + }, + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "resourceType": "microsoft.insights/components" + }, + { + "id": "66c96524-06de-4fb2-be22-3314ff7f96b0", + "version": "KqlParameterItem/1.0", + "name": "ContextFree", + "type": 1, + "query": "{\"version\":\"1.0.0\",\"content\":\"\\\"{DefaultWorkspace}\\\"\",\"transformers\":null}", + "isHiddenWhenLocked": true, + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 8 + }, + { + "id": "9ecea5c4-8730-4503-a052-95a2418ddd70", + "version": "KqlParameterItem/1.0", + "name": "Selection", + "type": 1, + "query": "where type =~ 'microsoft.operationalinsights/workspaces'\r\n| extend match = strcat(\"'\", id, \"'\") =~ \"{DefaultWorkspace:value}\"\r\n| order by match desc, name asc\r\n| take 1\r\n| project value = tostring(pack('sub', subscriptionId, 'rg', resourceGroup, 'ws', id))", + "crossComponentResources": [ + "value::all" + ], + "isHiddenWhenLocked": true, + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 1, + "resourceType": "microsoft.resourcegraph/resources" + } + ], + "style": "above", + "queryType": 1, + "resourceType": "microsoft.resourcegraph/resources" + }, + "conditionalVisibility": { + "parameterName": "_", + "comparison": "isEqualTo", + "value": "_" + }, + "name": "parameters - 3" + }, + { + "type": 9, + "content": { + "version": "KqlParameterItem/1.0", + "crossComponentResources": [ + "{Subscriptions}" + ], + "parameters": [ + { + "id": "401c9381-d3bc-4594-be03-c322c0c6a135", + "version": "KqlParameterItem/1.0", + "name": "Subscriptions", + "type": 6, + "isRequired": true, + "multiSelect": true, + "quote": "'", + "delimiter": ",", + "query": "summarize by subscriptionId\r\n| project value = strcat('/subscriptions/', subscriptionId), label = subscriptionId, selected = iff(subscriptionId =~ todynamic('{Selection}').sub, true, false)", + "crossComponentResources": [ + "value::all" + ], + "typeSettings": { + "additionalResourceOptions": [ + "value::all" + ] + }, + "queryType": 1, + "resourceType": "microsoft.resourcegraph/resources" + }, + { + "id": "8f5a2b0e-71bf-49f4-b570-c3d84c7cc7f6", + "version": "KqlParameterItem/1.0", + "name": "Workspaces", + "type": 5, + "isRequired": true, + "multiSelect": true, + "quote": "'", + "delimiter": ",", + "typeSettings": { + "additionalResourceOptions": [ + "value::all" + ], + "selectAllValue": "*", + "showDefault": false + } + }, + { + "id": "bcb5eb6b-b6a8-47c4-95a2-87753cb10ea8", + "version": "KqlParameterItem/1.0", + "name": "TimeRange", + "type": 4, + "value": { + "durationMs": 86400000 + }, + "typeSettings": { + "selectableValues": [ + { + "durationMs": 300000 + }, + { + "durationMs": 900000 + }, + { + "durationMs": 1800000 + }, + { + "durationMs": 3600000 + }, + { + "durationMs": 14400000 + }, + { + "durationMs": 43200000 + }, + { + "durationMs": 86400000 + }, + { + "durationMs": 172800000 + }, + { + "durationMs": 259200000 + }, + { + "durationMs": 604800000 + }, + { + "durationMs": 1209600000 + }, + { + "durationMs": 2419200000 + }, + { + "durationMs": 2592000000 + }, + { + "durationMs": 5184000000 + }, + { + "durationMs": 7776000000 + } + ] + }, + "resourceType": "microsoft.insights/components" + }, + { + "id": "1c1b8f51-b434-468a-b256-29826c15d9e1", + "version": "KqlParameterItem/1.0", + "name": "TestConnectivity", + "type": 1, + "query": "InsightsMetrics\r\n| where TimeGenerated {TimeRange}\r\n| take 1", + "isHiddenWhenLocked": true, + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + } + ], + "style": "above", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + "name": "parameters - 2" + }, + { + "type": 11, + "content": { + "version": "LinkItem/1.0", + "style": "tabs", + "links": [ + { + "cellValue": "Tab", + "linkTarget": "parameter", + "linkLabel": "Audit File Share", + "subTarget": "1", + "preText": "", + "style": "link" + }, + { + "cellValue": "Tab", + "linkTarget": "parameter", + "linkLabel": "Audit File System", + "subTarget": "2", + "style": "link" + }, + { + "cellValue": "Tab", + "linkTarget": "parameter", + "linkLabel": "Audit Filtering Platform Connection", + "subTarget": "3", + "style": "link" + }, + { + "cellValue": "Tab", + "linkTarget": "parameter", + "linkLabel": "Audit Handle Manipulation", + "subTarget": "4", + "style": "link" + }, + { + "cellValue": "Tab", + "linkTarget": "parameter", + "linkLabel": "Audit Kernel Object", + "subTarget": "5", + "style": "link" + }, + { + "cellValue": "Tab", + "linkTarget": "parameter", + "linkLabel": "Audit Other Object Access Events", + "subTarget": "6", + "style": "link" + }, + { + "cellValue": "Tab", + "linkTarget": "parameter", + "linkLabel": "Audit Registry", + "subTarget": "7", + "style": "link" + }, + { + "cellValue": "Tab", + "linkTarget": "parameter", + "linkLabel": "Audit Removable Storage", + "subTarget": "8", + "style": "link" + }, + { + "cellValue": "Tab", + "linkTarget": "parameter", + "linkLabel": "Audit SAM", + "subTarget": "9", + "style": "link" + }, + { + "cellValue": "Tab", + "linkTarget": "parameter", + "linkLabel": "Audit Policy Change", + "subTarget": "10", + "style": "link" + }, + { + "cellValue": "Tab", + "linkTarget": "parameter", + "linkLabel": "Audit Sensitive Privilege Use", + "subTarget": "11", + "style": "link" + } + ] + }, + "name": "links - 4" + }, + { + "type": 12, + "content": { + "version": "NotebookGroup/1.0", + "groupType": "editable", + "loadType": "always", + "items": [ + { + "type": 1, + "content": { + "json": "Events:\r\n- A network share object was accessed\r\n- A network share object was added\r\n- A network share object was modified\r\n- A network share object was deleted\r\n- SPN check for SMB/SMB2 failed" + }, + "name": "text - 1", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "SecurityEvent\r\n| where EventID in (5140, 5142, 5143, 5144, 5168)\r\n| project TimeGenerated, Account , Computer , EventData , EventID , Activity ", + "size": 1, + "showAnnotations": true, + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "timeBrushParameterName": "TimeBrush", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "timechart", + "tileSettings": { + "showBorder": false, + "titleContent": { + "columnMatch": "Account", + "formatter": 1 + }, + "leftContent": { + "columnMatch": "Task", + "formatter": 12, + "formatOptions": { + "palette": "auto" + }, + "numberFormat": { + "unit": 17, + "options": { + "maximumSignificantDigits": 3, + "maximumFractionDigits": 2 + } + } + } + }, + "graphSettings": { + "type": 0, + "topContent": { + "columnMatch": "Account", + "formatter": 1 + }, + "centerContent": { + "columnMatch": "Task", + "formatter": 1, + "numberFormat": { + "unit": 17, + "options": { + "maximumSignificantDigits": 3, + "maximumFractionDigits": 2 + } + } + } + }, + "chartSettings": { + "xAxis": "TimeGenerated", + "yAxis": [ + "EventID" + ], + "showLegend": true, + "xSettings": {}, + "ySettings": {} + } + }, + "name": "query - 2" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "SecurityEvent\r\n| where EventID in (5140, 5142, 5143, 5144, 5168)\r\n| project TimeGenerated, Account , Computer , EventData , EventID , Activity ", + "size": 0, + "timeContext": { + "durationMs": 86400000 + }, + "timeContextFromParameter": "TimeBrush", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "table", + "gridSettings": { + "filter": true + } + }, + "showPin": false, + "name": "query - 0" + } + ] + }, + "conditionalVisibility": { + "parameterName": "Tab", + "comparison": "isEqualTo", + "value": "1" + }, + "name": "AuditFileShare" + }, + { + "type": 12, + "content": { + "version": "NotebookGroup/1.0", + "groupType": "editable", + "loadType": "always", + "items": [ + { + "type": 1, + "content": { + "json": "Events: \r\n- A handle to an object was requested \r\n- The handle to an object was closed \r\n- An object was deleted- An attempt was made to access an object\r\n- An attempt was made to create a hard link- The state of a transaction has changed \r\n- A file was virtualized\r\n- Permissions on an object were changed" + }, + "name": "text - 1", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "SecurityEvent\r\n| where EventID in (4656, 4658, 4660, 4663, 4664, 4985, 5051, 4670)\r\n| project TimeGenerated, Account , AccountType , Computer , EventSourceName , Channel , Task , EventData , EventID , Activity ", + "size": 1, + "showAnnotations": true, + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "timeBrushParameterName": "TimeBrush", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "timechart", + "chartSettings": { + "showLegend": true, + "xSettings": {}, + "ySettings": {} + } + }, + "name": "query - 2" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "SecurityEvent\r\n| where EventID in (4656, 4658, 4660, 4663, 4664, 4985, 5051, 4670)\r\n| project TimeGenerated, Account , Computer , EventData , EventID , Activity ", + "size": 0, + "timeContext": { + "durationMs": 86400000 + }, + "timeContextFromParameter": "TimeBrush", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "gridSettings": { + "filter": true + } + }, + "name": "query - 0" + } + ] + }, + "conditionalVisibility": { + "parameterName": "Tab", + "comparison": "isEqualTo", + "value": "2" + }, + "name": "AuditFileSystem", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 12, + "content": { + "version": "NotebookGroup/1.0", + "groupType": "editable", + "loadType": "always", + "items": [ + { + "type": 1, + "content": { + "json": "Events:\r\n - The handle to an object was closed\r\n - An attempt was made to duplicate a handle to an object" + }, + "name": "text - 0", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "SecurityEvent\r\n| where EventID in (4658, 4690)\r\n| project TimeGenerated, Account , AccountType , Computer , EventSourceName , Channel , Task , EventData , EventID , Activity ", + "size": 1, + "showAnnotations": true, + "timeContext": { + "durationMs": 2592000000 + }, + "timeBrushParameterName": "TimeBrush", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "timechart" + }, + "name": "query - 2" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "SecurityEvent\r\n| where EventID in (4658, 4690)\r\n| project TimeGenerated, Account , Computer , EventData , EventID , Activity ", + "size": 0, + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeBrush", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "gridSettings": { + "filter": true + } + }, + "name": "query - 1" + } + ] + }, + "conditionalVisibility": { + "parameterName": "Tab", + "comparison": "isEqualTo", + "value": "4" + }, + "name": "AuditHandleManipulation", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 12, + "content": { + "version": "NotebookGroup/1.0", + "groupType": "editable", + "loadType": "always", + "items": [ + { + "type": 1, + "content": { + "json": "Events:\r\n- The Windows Firewall Service blocked an application from accepting incoming connections on the network\r\n- The Windows Filtering Platform blocked a packet\r\n- A more restrictive Windows Filtering Platform filter has blocked a packet\r\n- The Windows Filtering Platform has permitted an application or service to listen on a port for incoming connections\r\n- The Windows Filtering Platform has blocked an application or service from listening on a port for incoming connections\r\n- The Windows Filtering Platform has permitted a connection\r\n- The Windows Filtering Platform has blocked a connection\r\n- The Windows Filtering Platform has permitted a bind to a local port\r\n- The Windows Filtering Platform has blocked a bind to a local port" + }, + "name": "text - 0", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "SecurityEvent\r\n| where EventID in (5031, 5150, 5151, 5154, 5155, 5156, 5157, 5158, 5159)\r\n| project TimeGenerated, Account , AccountType , Computer , EventSourceName , Channel , Task , EventData , EventID , Activity", + "size": 1, + "showAnnotations": true, + "timeContext": { + "durationMs": 2592000000 + }, + "timeBrushParameterName": "TimeBrush", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "timechart" + }, + "name": "query - 2" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "SecurityEvent\r\n| where EventID in (5031, 5150, 5151, 5154, 5155, 5156, 5157, 5158, 5159)\r\n| project TimeGenerated, Account , Computer , EventData , EventID , Activity", + "size": 0, + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeBrush", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "gridSettings": { + "filter": true + } + }, + "name": "query - 1" + } + ] + }, + "conditionalVisibility": { + "parameterName": "Tab", + "comparison": "isEqualTo", + "value": "3" + }, + "name": "AuditFilterPlatformConnection", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 12, + "content": { + "version": "NotebookGroup/1.0", + "groupType": "editable", + "loadType": "always", + "items": [ + { + "type": 1, + "content": { + "json": "Events:\r\n- A handle to an object was requested\r\n- The handle to an object was closed\r\n- An object was deleted\r\n- An attempt was made to access an object" + }, + "name": "text - 0", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "SecurityEvent\r\n| where EventID in (4656, 4658, 4660, 4663)\r\n| project TimeGenerated, Account , AccountType , Computer , EventSourceName , Channel , Task , EventData , EventID , Activity ", + "size": 1, + "showAnnotations": true, + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "timeBrushParameterName": "TimeBrush", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "timechart" + }, + "name": "query - 2" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "SecurityEvent\r\n| where EventID in (4656, 4658, 4660, 4663)\r\n| project TimeGenerated, Account , Computer , EventData , EventID , Activity ", + "size": 0, + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeBrush", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "gridSettings": { + "filter": true + } + }, + "name": "query - 1" + } + ] + }, + "conditionalVisibility": { + "parameterName": "Tab", + "comparison": "isEqualTo", + "value": "5" + }, + "name": "AuditKernelObject", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 12, + "content": { + "version": "NotebookGroup/1.0", + "groupType": "editable", + "loadType": "always", + "items": [ + { + "type": 1, + "content": { + "json": "Events:\r\n- An application attempted to access a blocked ordinal through the TBS\r\n- Indirect access to an object was requested\r\n- The Windows Filtering Platform has detected a DoS attack and entered a defensive mode; packets associated with this attack will be discarded\r\n- The DoS attack has subsided and normal processing is being resumed\r\n- A scheduled task was created\r\n- A scheduled task was deleted\r\n- A scheduled task was enabled\r\n- A scheduled task was disabled\r\n- A scheduled task was updated\r\n- An object in the COM+ Catalog was modified\r\n- An object was deleted from the COM+ Catalog\r\n- An object was added to the COM+ Catalog" + }, + "name": "text - 0", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "SecurityEvent\r\n| where EventID in (4671, 4691, 5148, 5149, 4698, 4699, 4700, 4701, 4702, 5888, 5889, 5890)\r\n| project TimeGenerated, Account , AccountType , Computer , EventSourceName , Channel , Task , EventData , EventID , Activity \r\n| top 1000 by TimeGenerated desc", + "size": 1, + "showAnnotations": true, + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "timeBrushParameterName": "TImeBrush", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "timechart", + "tileSettings": { + "showBorder": false, + "titleContent": { + "columnMatch": "Account", + "formatter": 1 + }, + "leftContent": { + "columnMatch": "EventID", + "formatter": 12, + "formatOptions": { + "palette": "auto" + }, + "numberFormat": { + "unit": 17, + "options": { + "maximumSignificantDigits": 3, + "maximumFractionDigits": 2 + } + } + } + } + }, + "name": "query - 2" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "SecurityEvent\r\n| where EventID in (4671, 4691, 5148, 5149, 4698, 4699, 4700, 4701, 4702, 5888, 5889, 5890)\r\n| project TimeGenerated, Account , Computer , EventData , EventID , Activity ", + "size": 0, + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TImeBrush", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "gridSettings": { + "filter": true + } + }, + "name": "query - 1" + } + ] + }, + "conditionalVisibility": { + "parameterName": "Tab", + "comparison": "isEqualTo", + "value": "6" + }, + "name": "AuditOtherObject", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 12, + "content": { + "version": "NotebookGroup/1.0", + "groupType": "editable", + "items": [ + { + "type": 1, + "content": { + "json": "Events:\r\n- An attempt was made to access an object\r\n- A handle to an object was requested\r\n- The handle to an object was closed\r\n- An object was deleted\r\n- A registry value was modified\r\n- A registry key was virtualized\r\n- Permissions on an object were changed" + }, + "name": "text - 0", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "SecurityEvent\r\n| where EventID in (4663, 4656, 4658, 4660, 4657, 5039, 4670)\r\n| project TimeGenerated, Account , AccountType , Computer , EventSourceName , Channel , Task , EventData , EventID , Activity \r\n| top 1000 by TimeGenerated desc", + "size": 1, + "showAnnotations": true, + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "timeBrushParameterName": "TimeBrush", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "timechart" + }, + "name": "query - 2" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "SecurityEvent\r\n| where EventID in (4663, 4656, 4658, 4660, 4657, 5039, 4670)\r\n| project TimeGenerated, Account , Computer , EventData , EventID , Activity ", + "size": 0, + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeBrush", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "gridSettings": { + "filter": true + } + }, + "name": "query - 1" + } + ] + }, + "conditionalVisibility": { + "parameterName": "Tab", + "comparison": "isEqualTo", + "value": "7" + }, + "name": "AuditRegistry", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 12, + "content": { + "version": "NotebookGroup/1.0", + "groupType": "editable", + "loadType": "always", + "items": [ + { + "type": 1, + "content": { + "json": "Events:\r\n- A handle to an object was requested\r\n- The handle to an object was closed\r\n- An attempt was made to access an object" + }, + "name": "text - 0", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "SecurityEvent\r\n| where EventID in (4656, 4658, 4663)\r\n| project TimeGenerated, Account , AccountType , Computer , EventSourceName , Channel , Task , EventData , EventID , Activity \r\n| top 1000 by TimeGenerated desc", + "size": 4, + "showAnnotations": true, + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "timeBrushParameterName": "TimeBrush", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "timechart" + }, + "name": "query - 2" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "SecurityEvent\r\n| where EventID in (4656, 4658, 4663)\r\n| project TimeGenerated, Account , Computer , EventData , EventID , Activity ", + "size": 0, + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeBrush", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "gridSettings": { + "filter": true + } + }, + "name": "query - 1" + } + ] + }, + "conditionalVisibility": { + "parameterName": "Tab", + "comparison": "isEqualTo", + "value": "8" + }, + "name": "AuditRemovable", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 12, + "content": { + "version": "NotebookGroup/1.0", + "groupType": "editable", + "loadType": "always", + "items": [ + { + "type": 1, + "content": { + "json": "Events:\r\n- A handle to an object was requested" + }, + "name": "text - 0", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "SecurityEvent\r\n| where EventID == 4661\r\n| project TimeGenerated, Account , AccountType , Computer , EventSourceName , Channel , Task , EventData , EventID , Activity ", + "size": 1, + "showAnnotations": true, + "timeContext": { + "durationMs": 2592000000 + }, + "timeBrushParameterName": "TimeBrush", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "timechart" + }, + "name": "query - 2" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "SecurityEvent\r\n| where EventID == 4661\r\n| project TimeGenerated, Account , Computer , EventData , EventID , Activity ", + "size": 0, + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeBrush", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "gridSettings": { + "filter": true + } + }, + "name": "query - 1" + } + ] + }, + "conditionalVisibility": { + "parameterName": "Tab", + "comparison": "isEqualTo", + "value": "9" + }, + "name": "AuditSAM", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 12, + "content": { + "version": "NotebookGroup/1.0", + "groupType": "editable", + "loadType": "always", + "items": [ + { + "type": 1, + "content": { + "json": "Events:\r\n- The audit policy (SACL) on an object was changed\r\n- System audit policy was changed\r\n- Auditing settings on object were changed\r\n- The Per-user audit policy table was created\r\n- The CrashOnAuditFail value has changed\r\n- Auditing settings on object were changed\r\n- Special Groups Logon table modified\r\n- Per User Audit Policy was changed\r\n- An attempt was made to register a security event source\r\n- An attempt was made to unregister a security event source" + }, + "name": "text - 0" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "SecurityEvent\r\n| where EventID in (4715, 4719, 4817, 4902, 4906, 4907, 4908, 4912, 4904, 4905)\r\n| project TimeGenerated, Account , AccountType , Computer , EventSourceName , Channel , Task , EventData , EventID , Activity \r\n| top 1000 by TimeGenerated desc", + "size": 1, + "showAnnotations": true, + "timeContext": { + "durationMs": 2592000000 + }, + "timeBrushParameterName": "TimeBrush", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "timechart" + }, + "name": "query - 2" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "SecurityEvent\r\n| where EventID in (4715, 4719, 4817, 4902, 4906, 4907, 4908, 4912, 4904, 4905)\r\n| project TimeGenerated, Account , Computer , EventData , EventID , Activity ", + "size": 0, + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeBrush", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "gridSettings": { + "filter": true + } + }, + "name": "query - 1" + } + ] + }, + "conditionalVisibility": { + "parameterName": "Tab", + "comparison": "isEqualTo", + "value": "10" + }, + "name": "AuditPolicyChange", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 12, + "content": { + "version": "NotebookGroup/1.0", + "groupType": "editable", + "loadType": "always", + "items": [ + { + "type": 1, + "content": { + "json": "Events:\r\n- A privileged service was called\r\n- An operation was attempted on a privileged object\r\n- The state of a transaction has changed" + }, + "name": "text - 0", + "styleSettings": { + "showBorder": true + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "SecurityEvent\r\n| where EventID in (4673, 4674, 4985)\r\n| project TimeGenerated, Account , Computer , EventData , EventID , Activity ", + "size": 1, + "showAnnotations": true, + "timeContext": { + "durationMs": 2592000000 + }, + "timeBrushParameterName": "TimeBrush", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "timechart" + }, + "name": "query - 2" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "SecurityEvent\r\n| where EventID in (4673, 4674, 4985)\r\n| project Account , Computer , EventData , EventID , Activity ", + "size": 1, + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeBrush", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "gridSettings": { + "filter": true + } + }, + "name": "query - 1" + } + ] + }, + "conditionalVisibility": { + "parameterName": "Tab", + "comparison": "isEqualTo", + "value": "11" + }, + "name": "AuditSensitive", + "styleSettings": { + "showBorder": true + } + } + ], + "styleSettings": { + "progressStyle": "loader" + }, + "fromTemplateId": "sentinel-EventAnalyzerWorkbook", + "$schema": "https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json" +} \ No newline at end of file diff --git a/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/sources/Windows Security Events/Workbooks/IdentityAndAccess.json b/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/sources/Windows Security Events/Workbooks/IdentityAndAccess.json new file mode 100644 index 00000000000..0399cc8950a --- /dev/null +++ b/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/sources/Windows Security Events/Workbooks/IdentityAndAccess.json @@ -0,0 +1,617 @@ +{ + "version": "Notebook/1.0", + "items": [ + { + "type": 1, + "content": { + "json": "" + }, + "name": "text - 1" + }, + { + "type": 9, + "content": { + "version": "KqlParameterItem/1.0", + "query": "", + "crossComponentResources": [], + "parameters": [ + { + "id": "80e332f7-8176-461f-b27a-0a52242fe6c9", + "version": "KqlParameterItem/1.0", + "name": "TimeRange", + "type": 4, + "isRequired": true, + "value": { + "durationMs": 86400000 + }, + "typeSettings": { + "selectableValues": [ + { + "durationMs": 300000 + }, + { + "durationMs": 900000 + }, + { + "durationMs": 1800000 + }, + { + "durationMs": 3600000 + }, + { + "durationMs": 14400000 + }, + { + "durationMs": 43200000 + }, + { + "durationMs": 86400000 + }, + { + "durationMs": 172800000 + }, + { + "durationMs": 259200000 + }, + { + "durationMs": 604800000 + }, + { + "durationMs": 1209600000 + }, + { + "durationMs": 2419200000 + }, + { + "durationMs": 2592000000 + }, + { + "durationMs": 5184000000 + }, + { + "durationMs": 7776000000 + } + ], + "allowCustom": true + } + }, + { + "id": "5a93ede8-361d-4cc6-93f8-967dfc355143", + "version": "KqlParameterItem/1.0", + "name": "Activity", + "type": 2, + "isRequired": true, + "multiSelect": true, + "quote": "'", + "delimiter": ",", + "query": "SecurityEvent\r\n| summarize Count = count() by Activity\r\n| order by Count desc, Activity asc\r\n| project Value = Activity, Label = strcat(Activity, ' - ', Count)", + "value": [ + "value::all" + ], + "typeSettings": { + "additionalResourceOptions": [ + "value::all" + ], + "selectAllValue": "All" + }, + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + } + ], + "style": "pills", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + "name": "parameters - 1" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "let data = SecurityEvent\r\n| where \"{Activity:lable}\" == \"All\" or Activity in ({Activity});\r\ndata\r\n| summarize Count = count() by Activity\r\n| join kind = fullouter (datatable(Activity:string)['Medium', 'high', 'low']) on Activity\r\n| project Activity = iff(Activity == '', Activity1, Activity), Count = iff(Activity == '', 0, Count)\r\n| join kind = inner (data\r\n | make-series Trend = count() default = 0 on TimeGenerated from {TimeRange:start} to {TimeRange:end} step {TimeRange:grain} by Activity)\r\n on Activity\r\n| project-away Activity1, TimeGenerated\r\n| extend Activitys = Activity\r\n| union (\r\n data \r\n | summarize Count = count() \r\n | extend jkey = 1\r\n | join kind=inner (data\r\n | make-series Trend = count() default = 0 on TimeGenerated from {TimeRange:start} to {TimeRange:end} step {TimeRange:grain}\r\n | extend jkey = 1) on jkey\r\n | extend Activity = 'All', Activitys = '*' \r\n)\r\n| order by Count desc\r\n| take 10", + "size": 4, + "exportFieldName": "Activity", + "exportParameterName": "ActivityPiker", + "exportDefaultValue": "All", + "exportToExcelOptions": "visible", + "title": "Top 10 activities - click to filter by activity", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "tiles", + "tileSettings": { + "titleContent": { + "columnMatch": "Activity", + "formatter": 1, + "formatOptions": { + "showIcon": true + } + }, + "leftContent": { + "columnMatch": "Count", + "formatter": 12, + "formatOptions": { + "palette": "auto", + "showIcon": true + }, + "numberFormat": { + "unit": 17, + "options": { + "maximumSignificantDigits": 3, + "maximumFractionDigits": 2 + } + } + }, + "secondaryContent": { + "columnMatch": "Trend", + "formatter": 9, + "formatOptions": { + "palette": "lightBlue", + "showIcon": true + } + }, + "showBorder": false + } + }, + "name": "query - 2" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "let data = SecurityEvent\r\n| where \"{Activity:lable}\" == \"All\" or Activity in ({Activity})\r\n| where Activity == '{ActivityPiker}' or '{ActivityPiker}' == \"All\" and AccountType == 'User'\r\n| extend Name = extract(@'^(.*\\\\)?([^@]*)(@.*)?$', 2, tolower(Account));\r\nlet appData = data\r\n| summarize TotalCount = count() by Name\r\n| join kind=inner (data\r\n | make-series Trend = count() default = 0 on TimeGenerated in range({TimeRange:start}, {TimeRange:end}, {TimeRange:grain}) by Name\r\n | project-away TimeGenerated) on Name\r\n| order by TotalCount desc, Name asc\r\n| project Name, TotalCount, Trend\r\n| serialize Id = row_number();\r\ndata\r\n| summarize TotalCount = count() by Activity , Name\r\n| join kind=inner (data\r\n | make-series Trend = count() default = 0 on TimeGenerated in range({TimeRange:start}, {TimeRange:end}, {TimeRange:grain}) by Name, Activity\r\n | project-away TimeGenerated) on Name, Activity\r\n| order by TotalCount desc, Name asc\r\n| project Name, Activity, TotalCount, Trend\r\n| serialize Id = row_number(1000000)\r\n| join kind=inner (appData) on Name\r\n| project Id, Name = Activity, Type = 'Activity', ['Activity Count'] = TotalCount, Trend, ParentId = Id1\r\n| union (appData \r\n | project Id, Name = Name, Type = 'Computer', ['Activity Count'] = TotalCount, Trend )\r\n| order by ['Activity Count'] desc, Name asc", + "size": 0, + "exportParameterName": "Userinfo", + "exportDefaultValue": "{ \"Name\":\"\", \"Type\":\"*\"}", + "exportToExcelOptions": "visible", + "title": "User activities", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "gridSettings": { + "formatters": [ + { + "columnMatch": "Id", + "formatter": 5, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "Name", + "formatter": 0, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "Type", + "formatter": 0, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "Activity Count", + "formatter": 8, + "formatOptions": { + "min": 0, + "palette": "blueGreen", + "showIcon": true + } + }, + { + "columnMatch": "Trend", + "formatter": 9, + "formatOptions": { + "palette": "greenDark", + "showIcon": true + } + }, + { + "columnMatch": "IpAddress", + "formatter": 5, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "ParentId", + "formatter": 5, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "Account", + "formatter": 0, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "Names Count", + "formatter": 0, + "formatOptions": { + "showIcon": true + } + } + ], + "filter": true, + "hierarchySettings": { + "idColumn": "Id", + "parentColumn": "ParentId", + "treeType": 0, + "expanderColumn": "Name" + }, + "labelSettings": [] + } + }, + "customWidth": "50", + "name": "query - 3" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "//let Users = dynamic({Userinfo});\r\nlet data = SecurityEvent\r\n| where \"{Activity:lable}\" == \"All\" or Activity in ({Activity})\r\n| where Activity == '{ActivityPiker}' or '{ActivityPiker}' == \"All\" and AccountType == 'Machine'\r\n| extend user = extract(@'^(.*\\\\)?([^@]*)(@.*)?$', 2, tolower(Account))\r\n| where dynamic({Userinfo}).Type == '*' or (dynamic({Userinfo}).Type == 'Computer' and user == dynamic({Userinfo}).Name);\r\nlet appData = data\r\n| summarize TotalCount = count() by Computer\r\n| join kind=inner (data\r\n | make-series Trend = count() default = 0 on TimeGenerated in range({TimeRange:start}, {TimeRange:end}, {TimeRange:grain}) by Computer\r\n | project-away TimeGenerated) on Computer\r\n| order by TotalCount desc, Computer asc\r\n| project Computer, TotalCount, Trend\r\n| serialize Id = row_number();\r\ndata\r\n| summarize TotalCount = count() by Activity , Computer\r\n| join kind=inner (data\r\n | make-series Trend = count() default = 0 on TimeGenerated in range({TimeRange:start}, {TimeRange:end}, {TimeRange:grain}) by Computer, Activity\r\n | project-away TimeGenerated) on Computer, Activity\r\n| order by TotalCount desc, Computer asc\r\n| project Computer, Activity, TotalCount, Trend\r\n| serialize Id = row_number(1000000)\r\n| join kind=inner (appData) on Computer\r\n| project Id, Name = Activity, Type = 'Activity', ['Activity Count'] = TotalCount, Trend, ParentId = Id1\r\n| union (appData \r\n | project Id, Name = Computer, Type = 'Computer', ['Activity Count'] = TotalCount, Trend )\r\n| order by ['Activity Count'] desc, Name asc", + "size": 0, + "exportFieldName": "", + "exportParameterName": "MachineInfo", + "exportDefaultValue": "{ \"Name\":\"\", \"Type\":\"*\"}", + "exportToExcelOptions": "visible", + "title": "Machine activities", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "gridSettings": { + "formatters": [ + { + "columnMatch": "Id", + "formatter": 5, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "Name", + "formatter": 0, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "Type", + "formatter": 0, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "Activity Count", + "formatter": 8, + "formatOptions": { + "min": 0, + "palette": "blue", + "showIcon": true + } + }, + { + "columnMatch": "Trend", + "formatter": 9, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "ParentId", + "formatter": 5, + "formatOptions": { + "showIcon": true + } + } + ], + "filter": true, + "hierarchySettings": { + "idColumn": "Id", + "parentColumn": "ParentId", + "treeType": 0, + "expanderColumn": "Name" + }, + "labelSettings": [] + } + }, + "customWidth": "50", + "name": "query - 4" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "let Users = dynamic({Userinfo});\r\nlet Machines = dynamic({MachineInfo});\r\nSecurityEvent\r\n| where \"{Activity:lable}\" == \"All\" or Activity in ({Activity})\r\n| where Activity == '{ActivityPiker}' or '{ActivityPiker}' == \"All\"\r\n| extend Name = extract(@'^(.*\\\\)?([^@]*)(@.*)?$', 2, tolower(Account))\r\n| where (Users.Type == '*' or (Users.Type == 'Computer' and Name == Users.Name)) and (Machines.Type == '*' or (Machines.Type == 'Computer' and Computer == Machines.Name))\r\n| extend Status = iif(Activity contains \"success\",\"✔️ Success\" , iif(Activity contains \"faile\", \"❌Failed\", \"none\"))\r\n| summarize Count = count() by Name, Computer,AccountType, Activity, IpAddress , Process, CallerProcessId, CallerProcessName, CommandLine, FilePath, IpPort,GroupMembership, Status, bin(TimeGenerated, 1d), Detalis=\"more details\"\r\n| project Name, Computer,AccountType, Activity, IpAddress, CallerProcessId, CallerProcessName, CommandLine, FilePath, IpPort, Process, GroupMembership, Status, TimeGenerated, Count, Detalis\r\n| order by Count\r\n", + "size": 0, + "exportParameterName": "Info", + "exportDefaultValue": "\"*\"", + "exportToExcelOptions": "visible", + "title": "Full details", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "gridSettings": { + "formatters": [ + { + "columnMatch": "Name", + "formatter": 0, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "Computer", + "formatter": 0, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "AccountType", + "formatter": 0, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "Activity", + "formatter": 0, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "IpAddress", + "formatter": 5, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "CallerProcessId", + "formatter": 5, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "CallerProcessName", + "formatter": 5, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "CommandLine", + "formatter": 5, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "FilePath", + "formatter": 5, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "IpPort", + "formatter": 5, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "Process", + "formatter": 0, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "GroupMembership", + "formatter": 5, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "Status", + "formatter": 0, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "TimeGenerated", + "formatter": 6, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "Count", + "formatter": 8, + "formatOptions": { + "min": 0, + "palette": "greenBlue", + "showIcon": true + } + }, + { + "columnMatch": "Detalis", + "formatter": 7, + "formatOptions": { + "linkTarget": "GenericDetails", + "linkIsContextBlade": true, + "showIcon": true + } + }, + { + "columnMatch": "NumberOfProcess", + "formatter": 0, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "detalis", + "formatter": 7, + "formatOptions": { + "linkTarget": "GenericDetails", + "linkIsContextBlade": true, + "showIcon": true + } + }, + { + "columnMatch": "count_", + "formatter": 0, + "formatOptions": { + "showIcon": true + } + } + ], + "rowLimit": 1000, + "filter": true, + "labelSettings": [] + } + }, + "name": "query - 5" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "let Users = dynamic({Userinfo});\r\nlet Info = dynamic({Info});\r\nlet Machines = dynamic({MachineInfo});\r\nSecurityEvent\r\n| where Info == \"*\" or Process == Info.Process\r\n| where Process != \"\" and Process != \"-\"\r\n| where \"{Activity:lable}\" == \"All\" or Activity in ({Activity})\r\n| where Activity == '{ActivityPiker}' or '{ActivityPiker}' == \"All\"\r\n| extend Name = extract(@'^(.*\\\\)?([^@]*)(@.*)?$', 2, tolower(Account))\r\n| where (Users.Type == '*' or (Users.Type == 'Computer' and Name == Users.Name)) and (Machines.Type == '*' or (Machines.Type == 'Computer' and Computer == Machines.Name))\r\n| summarize Count = count() by Process, bin(TimeGenerated, 1h)", + "size": 0, + "exportToExcelOptions": "visible", + "title": "Processes", + "color": "lightBlue", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "areachart" + }, + "customWidth": "30", + "name": "query - 6" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "let Users = dynamic({Userinfo});\r\nlet Info = dynamic({Info});\r\nlet Machines = dynamic({MachineInfo});\r\nSecurityEvent\r\n| where Process != \"\" and Process != \"-\"\r\n| where \"{Activity:lable}\" == \"All\" or Activity in ({Activity})\r\n| where Activity == '{ActivityPiker}' or '{ActivityPiker}' == \"All\"\r\n| where Info == \"*\" or Process == Info.Process\r\n| extend user = extract(@'^(.*\\\\)?([^@]*)(@.*)?$', 2, tolower(Account))\r\n| where (Users.Type == '*' or (Users.Type == 'Computer' and user == Users.Name)) and (Machines.Type == '*' or (Machines.Type == 'Computer' and Computer == Machines.Name))\r\n| summarize Count = count() by Process, Activity, user, Computer, bin(TimeGenerated, 1h)\r\n| order by TimeGenerated, Count", + "size": 0, + "exportToExcelOptions": "visible", + "title": "Processes details", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "gridSettings": { + "filter": true, + "labelSettings": [] + } + }, + "customWidth": "70", + "name": "query - 8" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "let Users = dynamic({Userinfo});\r\nlet Info = dynamic({Info});\r\nlet Machines = dynamic({MachineInfo});\r\nSecurityEvent\r\n| where LogonTypeName != \"\"\r\n| where \"{Activity:lable}\" == \"All\" or Activity in ({Activity})\r\n| where Activity == '{ActivityPiker}' or '{ActivityPiker}' == \"All\"\r\n| where Info == \"*\" or Activity == Info.Activity\r\n| extend user = extract(@'^(.*\\\\)?([^@]*)(@.*)?$', 2, tolower(Account))\r\n| where (Users.Type == '*' or (Users.Type == 'Computer' and user == Users.Name)) and (Machines.Type == '*' or (Machines.Type == 'Computer' and Computer == Machines.Name))\r\n| summarize Count = count() by LogonTypeName = strcat( \"Logon type: \", LogonTypeName), Activity, user\r\n| order by Count", + "size": 0, + "exportToExcelOptions": "visible", + "title": "Logon activity types", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "gridSettings": { + "formatters": [ + { + "columnMatch": "LogonTypeName", + "formatter": 0, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "Activity", + "formatter": 0, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "user", + "formatter": 0, + "formatOptions": { + "showIcon": true + } + }, + { + "columnMatch": "Count", + "formatter": 8, + "formatOptions": { + "min": 0, + "palette": "green", + "showIcon": true, + "aggregation": "Sum" + } + }, + { + "columnMatch": "$gen_group", + "formatter": 0, + "formatOptions": { + "showIcon": true + } + } + ], + "filter": true, + "hierarchySettings": { + "treeType": 1, + "groupBy": [ + "LogonTypeName", + "Activity" + ], + "expandTopLevel": false + }, + "labelSettings": [] + } + }, + "customWidth": "70", + "name": "query - 7" + } + ], + "styleSettings": {}, + "fromTemplateId": "sentinel-IdentityAndAccess", + "$schema": "https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json" +} \ No newline at end of file diff --git a/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/upstream-cases.json b/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/upstream-cases.json new file mode 100644 index 00000000000..c10cd14db8b --- /dev/null +++ b/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/upstream-cases.json @@ -0,0 +1,56 @@ +[ + { + "path": "Syslog/Workbooks/LinuxMachines.json", + "tables": ["Syslog"], + "sourceQueries": 10, + "outputQueries": 10, + "status": "needs-validation", + "removed": [], + "retained": ["Computer", "TimeRange", "Severity"] + }, + { + "path": "Azure Activity/Workbooks/AzureServiceHealthWorkbook.json", + "tables": ["AzureActivity"], + "sourceQueries": 13, + "outputQueries": 11, + "status": "needs-validation", + "removed": ["Subscription", "LAworkspace"], + "retained": ["TrackingID_P", "TimeRange"] + }, + { + "path": "Windows Security Events/Workbooks/EventAnalyzer.json", + "tables": ["SecurityEvent", "InsightsMetrics"], + "sourceQueries": 26, + "outputQueries": 23, + "status": "needs-validation", + "removed": ["DefaultWorkspace", "ContextFree", "Selection", "Subscriptions", "Workspaces"], + "retained": ["TimeRange", "TestConnectivity"] + }, + { + "path": "Windows Security Events/Workbooks/IdentityAndAccess.json", + "tables": ["SecurityEvent"], + "sourceQueries": 8, + "outputQueries": 8, + "status": "needs-validation", + "removed": [], + "retained": ["TimeRange"] + }, + { + "path": "Azure Activity/Workbooks/AzureActivity.json", + "tables": ["AzureActivity"], + "sourceQueries": 6, + "outputQueries": 6, + "status": "needs-validation", + "removed": [], + "retained": ["TimeRange", "Caller", "ResourceGroup"] + }, + { + "path": "Syslog/Workbooks/SyslogConnectorsOverviewWorkbook.json", + "tables": ["Syslog"], + "sourceQueries": 3, + "outputQueries": 3, + "status": "blocked", + "removed": [], + "retained": ["TimeRange"] + } +] diff --git a/Tools/SentinelToXDRMigration/tests/test_connector_metadata.py b/Tools/SentinelToXDRMigration/tests/test_connector_metadata.py index c6867d60e4b..b9c47b74c9b 100644 --- a/Tools/SentinelToXDRMigration/tests/test_connector_metadata.py +++ b/Tools/SentinelToXDRMigration/tests/test_connector_metadata.py @@ -152,9 +152,32 @@ class PackagedConnectorMetadataTests(unittest.TestCase): write_metadata = identity.PackagedIdentityTests.write_metadata fixture = identity.PackagedIdentityTests.fixture - def test_authoring_metadata_does_not_leak_into_install_or_registration(self): + def test_packager_rejects_missing_unknown_or_unreviewed_dependencies(self): + entry, path, document = self.fixture() + for mutation in ("missing", "unknown", "unreviewed"): + candidate = copy.deepcopy(document) + if mutation == "missing": + del candidate["contentProvenance"]["requiredWorkloads"] + elif mutation == "unknown": + candidate["contentProvenance"]["requiredWorkloads"] = [ + {"workload": "defender", "dataTypes": ["DeviceEvents"]}, + ] + else: + candidate["contentProvenance"]["dependencyReviewRequired"] = True + path.write_text("---\n" + json.dumps(candidate)) + result = self.package(entry, False) + self.assertNotEqual(0, result.returncode, mutation) + self.assertTrue(any(word in result.stdout + result.stderr for word in ( + "contentProvenance.requiredWorkloads", "workload 'defender'", "unresolved dependency", + )), result.stdout + result.stderr) + + def test_dependency_records_are_kept_in_registration_not_install(self): entry, path, document = self.fixture() document["requiredDataConnectors"] = CONNECTORS + document["contentProvenance"]["requiredDataConnectors"] = CONNECTORS + document["contentProvenance"]["requiredWorkloads"] = [ + {"workload": "mde", "dataTypes": ["DeviceEvents"]}, + ] path.write_text("---\n" + json.dumps(document)) before = path.read_bytes() result = self.package(entry, False) @@ -164,9 +187,19 @@ def test_authoring_metadata_does_not_leak_into_install_or_registration(self): and r.get("condition") == "[parameters('DeployCustomDetection')]") registration = next(r for r in template["resources"] if r.get("properties", {}).get("contentKind") == "CustomDetection") - for wrapper in (install, registration): - self.assertNotIn("requiredDataConnectors", json.dumps(wrapper)) - self.assertNotIn("OtherConnector", json.dumps(wrapper)) + deployed = install["properties"]["template"]["resources"]["detectionRule"]["properties"]["contentProvenance"] + stored = registration["properties"]["mainTemplate"]["resources"]["detectionRule"]["properties"]["contentProvenance"] + full = registration["properties"]["contentProvenance"] + self.assertEqual({ + "packageId": "testpublisher.test-sentinel", "packageVersion": "3.2.0", + "contentId": registration["properties"]["contentId"], "contentVersion": "3.1.0", + }, deployed) + self.assertEqual(deployed, stored) + self.assertEqual(deployed, {key: full[key] for key in deployed}) + self.assertEqual(CONNECTORS, full["requiredDataConnectors"]) + self.assertEqual(document["contentProvenance"]["requiredWorkloads"], full["requiredWorkloads"]) + self.assertNotIn("requiredDataConnectors", json.dumps(install)) + self.assertNotIn("OtherConnector", json.dumps(install)) self.assertEqual(before, path.read_bytes()) self.assertEqual(CONNECTORS, yaml.safe_load(path.read_text())["requiredDataConnectors"]) diff --git a/Tools/SentinelToXDRMigration/tests/test_content_deployment_parameters.py b/Tools/SentinelToXDRMigration/tests/test_content_deployment_parameters.py index e5def24ba15..d92bbcfc69e 100644 --- a/Tools/SentinelToXDRMigration/tests/test_content_deployment_parameters.py +++ b/Tools/SentinelToXDRMigration/tests/test_content_deployment_parameters.py @@ -176,9 +176,15 @@ def test_vocabulary_subsets_and_unclassified_resources_fail_specifically(self): self.assertEqual(0, result.returncode, result.stderr) template = json.loads(result.stdout)["template"] self.assertEqual(set(CONTRACT), set(template["parameters"])) - self.assertNotIn("DeployDefenderWorkbook", template["parameters"]) + self.assertIs(False, template["parameters"]["DeployDefenderWorkbook"]["defaultValue"]) + workbook = next( + resource for resource in template["resources"] + if resource["name"] == "DefenderWorkbook" + ) + self.assertEqual( + "[parameters('DeployDefenderWorkbook')]", workbook["condition"] + ) for resource in ( - registration("DefenderWorkbook", "unsupported-workbook"), registration("FutureKind", "future-kind"), {"type": "Microsoft.Resources/deployments", "name": "unowned-deployment", "properties": {}}, ): @@ -229,7 +235,7 @@ def test_real_entry_points_defaults_registration_and_legacy_isolation(self): self.assertEqual("disabled", body["resources"]["detectionRule"]["properties"]["status"]) self.assertNotIn("Deploy", json.dumps(body)) self.assertIn(attribution.AttributionTests.marker(attribution.TRACKING_ID), template["resources"]) - with zipfile.ZipFile(package / "3.1.0.zip") as archive: + with zipfile.ZipFile(package / "3.2.0.zip") as archive: self.assertEqual(template, json.loads(archive.read("mainTemplate.json"))) self.assertEqual(ui, json.loads(archive.read("createUiDefinition.json"))) original["XDR Detections"] = [] @@ -499,6 +505,12 @@ def test_first_authored_tactic_only_in_both_arm_copies_without_source_mutation(s ) expected["id"] = f"[if(true(), '{expected['id']}', '{expected['id']}')]" expected["detectionAction"]["alertTemplate"]["tactics"] = [authored[0]] + expected["contentProvenance"] = { + "packageId": "testpublisher.test-sentinel", + "packageVersion": "3.2.0", + "contentId": "xdr-" + document["contentProvenance"]["source"]["id"], + "contentVersion": document["version"], + } for body in bodies: self.assertEqual(expected, body["resources"]["detectionRule"]["properties"]) self.assertEqual(original, {p: p.read_bytes() for p in original}) diff --git a/Tools/SentinelToXDRMigration/tests/test_converter.py b/Tools/SentinelToXDRMigration/tests/test_converter.py index 13e8ec4dc3b..a7a0d7a7550 100644 --- a/Tools/SentinelToXDRMigration/tests/test_converter.py +++ b/Tools/SentinelToXDRMigration/tests/test_converter.py @@ -384,6 +384,9 @@ def test_rewrites_skip_strings_and_comments(self) -> None: def test_passthrough_table_preserves_timegenerated(self) -> None: path = self.solution / "Analytic Rules" / "SampleRule.yaml" document = yaml.safe_load(path.read_text(encoding="utf-8")) + document["requiredDataConnectors"] = [ + {"connectorId": "AzureActivity", "dataTypes": ["AzureActivity"]}, + ] document["query"] = """\ AzureActivity | where TimeGenerated > ago(1h) diff --git a/Tools/SentinelToXDRMigration/tests/test_customer_usage_attribution.py b/Tools/SentinelToXDRMigration/tests/test_customer_usage_attribution.py index b9771a1e749..21763e9dc17 100644 --- a/Tools/SentinelToXDRMigration/tests/test_customer_usage_attribution.py +++ b/Tools/SentinelToXDRMigration/tests/test_customer_usage_attribution.py @@ -45,7 +45,7 @@ def write_metadata(self, tracking_id=ABSENT) -> None: value = { "publisherId": "testpublisher", "offerId": "test-sentinel", - "version": "3.1.0", + "version": "3.2.0", "firstPublishDate": "2021-01-01", "providers": ["Microsoft"], "categories": {"domains": ["Security"]}, @@ -150,7 +150,7 @@ def prepare_integration(self) -> Path: "schemaVersion": "1.0.0", "version": "3.1.0", "kind": "CustomDetection", "resourceType": "Microsoft.Security/detectionRules", "apiVersion": "2025-06-01", - "contentProvenance": {"source": { + "contentProvenance": {"requiredWorkloads": [], "requiredDataConnectors": [], "source": { "id": source_id, "version": "1.0.0", "platform": "Microsoft Sentinel", "kind": "AnalyticsRule", }}, @@ -167,7 +167,7 @@ def prepare_integration(self) -> Path: path.write_text("---\n" + json.dumps(document), encoding="utf-8") self.data.write_text(json.dumps({ "Name": "Sample", "Author": "Test", "Description": "Test package", - "Version": "3.1.0", "TemplateSpec": True, + "Version": "3.2.0", "TemplateSpec": True, "Analytic Rules": ["Analytic Rules/Test.yaml"], "XDR Detections": ["XDR Detections/Test.yaml"], "Metadata": "SolutionMetadata.json", @@ -178,7 +178,7 @@ def prepare_integration(self) -> Path: def package( self, entry: Path, pipeline: bool, *, version_bump: str = "none", version_mode: str = "local", - calculated_version: str = "3.1.0", + calculated_version: str = "3.2.0", skip_lookup: bool = False, ) -> subprocess.CompletedProcess: extra = " -SkipAttributionLookup" if skip_lookup else "" @@ -467,7 +467,7 @@ def test_lookup_local_and_pipeline_use_authoritative_metadata_and_keep_cached_id result = self.package(entry, False, version_bump="patch") self.assertEqual(0, result.returncode, result.stdout + result.stderr) metadata = json.loads(self.metadata.read_text()) - self.assertEqual("3.1.1", metadata["version"]) + self.assertEqual("3.2.1", metadata["version"]) self.assertEqual(TRACKING_ID, metadata["trackingId"]) def test_failed_packaging_prerequisite_does_not_persist_discovery(self) -> None: @@ -511,10 +511,10 @@ def test_local_and_positional_pipeline_preserve_nested_xdr_and_authoritative_id( if r.get("properties", {}).get("contentKind") == "AnalyticsRule") self.assertEqual("[parameters('DeployAnalyticsRule')]", analytic["condition"]) self.assertEqual("3.0.0", analytic["properties"]["contentSchemaVersion"]) - self.assertEqual("3.1.0", template["variables"]["_solutionVersion"]) + self.assertEqual("3.2.0", template["variables"]["_solutionVersion"]) self.assertNotIn("trackingId", json.dumps(template)) self.assertNotIn("pid-stale-data", json.dumps(template)) - with zipfile.ZipFile(package / "3.1.0.zip") as archive: + with zipfile.ZipFile(package / "3.2.0.zip") as archive: self.assertEqual(template, json.loads(archive.read("mainTemplate.json").decode("utf-8-sig"))) self.assertEqual(before, (self.data.read_bytes(), self.metadata.read_bytes())) diff --git a/Tools/SentinelToXDRMigration/tests/test_defender_workbooks_packaging.py b/Tools/SentinelToXDRMigration/tests/test_defender_workbooks_packaging.py new file mode 100644 index 00000000000..3c376deee47 --- /dev/null +++ b/Tools/SentinelToXDRMigration/tests/test_defender_workbooks_packaging.py @@ -0,0 +1,883 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE in the repository root. +from __future__ import annotations + +import copy +import hashlib +import json +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest +import uuid +import zipfile + + +ROOT = Path(__file__).resolve().parents[3] +COMMON = ROOT / "Tools/Create-Azure-Sentinel-Solution/common/commonFunctions.ps1" + + +def literal(value: str | Path) -> str: + return "'" + str(value).replace("'", "''") + "'" + + +@unittest.skipUnless(shutil.which("pwsh"), "PowerShell 7 is required") +class DefenderWorkbookPackagingTests(unittest.TestCase): + def setUp(self) -> None: + self.scratch = tempfile.TemporaryDirectory(prefix="wb-packaging-", dir=Path.cwd()) + self.root = Path(self.scratch.name) + self.solution = self.root / "Solutions" / "Sample" + self.source = self.solution / "Workbooks" / "nested" / "source.json" + self.source.parent.mkdir(parents=True) + self.source.write_text(json.dumps({ + "version": "Notebook/1.0", + "items": [ + {"type": 3, "content": {"query": "SourceTable | take 1"}}, + {"query": "scope"}, + ], + }), encoding="utf-8") + self.notebook = { + "version": "Notebook/1.0", + "items": [ + {"type": 3, "content": {"query": "DeviceEvents | take 1"}}, + {"type": 1, "content": {"json": "[parameters('authored-literal')]"}}, + ], + "fallbackResourceIds": ["[[already-bracketed]", "literal"], + "xdrMetadata": { + "schemaVersion": "1.0.0", + "contentId": str(uuid.uuid5(uuid.NAMESPACE_URL, "fixture/workbook/content")), + "instanceId": str(uuid.uuid5(uuid.NAMESPACE_URL, "fixture/workbook/instance")), + "version": "3.1.0", + "displayName": "Synthetic workbook", + }, + "contentProvenance": { + "source": { + "path": "Workbooks/nested/source.json", + "sha256": hashlib.sha256(self.source.read_bytes()).hexdigest(), + "originalMetadataVersion": "1.2.0", + }, + "conversion": { + "tool": "sentinel-xdr-migration", + "version": "0.1.0", + "reviewRequired": False, + "queries": [{ + "sourcePath": "/items/0/content/query", + "outputPath": "/items/0/content/query", + "sourceSha256": hashlib.sha256(b"SourceTable | take 1").hexdigest(), + "targetSha256": hashlib.sha256(b"DeviceEvents | take 1").hexdigest(), + "status": "converted", + }], + }, + "requiredWorkloads": [{"workload": "mde", "dataTypes": ["DeviceEvents"]}], + "requiredDataConnectors": [{"connectorId": "SyntheticConnector", "dataTypes": ["DeviceEvents"]}], + "dependencyReviewRequired": False, + }, + } + self.reference = "XDR Workbooks/nested/workbook.json" + self.target = self.solution / self.reference + self.target.parent.mkdir(parents=True) + self.write() + self.data = {"Version": "3.2.7", "XDR Workbooks": [self.reference]} + self.native = { + "type": "Microsoft.Insights/workbooks", "name": "synthetic-native", + "properties": {"serializedData": '{"version":"Notebook/1.0","items":[]}'}, + "condition": "[equals('native','native')]", + } + self.template = { + "parameters": {}, + "variables": {"solutionId": "synthetic.publisher", "_solutionVersion": "3.2.7"}, + "resources": [{ + "type": "Microsoft.OperationalInsights/workspaces/providers/contentPackages", + "properties": {"contentKind": "Solution", "descriptionHtml": "", "icon": "", "dependencies": { + "criteria": [{"kind": "Workbook", "contentId": "native", "version": "1.0.0"}], + }}, + }, copy.deepcopy(self.native)], + } + self.ui = {"parameters": { + "outputs": {}, "steps": [], "config": {"basics": { + "description": "{{SolutionDescription}}\n\n**Workbooks:** 1\n\n{{Logo}}", + }}, + }} + + def tearDown(self) -> None: + self.scratch.cleanup() + + def write(self) -> None: + self.target.write_text(json.dumps(self.notebook), encoding="utf-8") + + def invoke(self, *, success: bool = True, package: bool = False, unchanged: bool = False) -> dict: + payload = self.root / "input.json" + payload.write_text(json.dumps({ + "data": self.data, "template": self.template, "ui": self.ui, + }), encoding="utf-8") + emit = ( + f"$count=Add-XdrDefenderWorkbooksToSolution -SolutionName Sample -ContentToImport $inputData.data " + f"-Template $template -RepositoryRoot {literal(self.root)}; " + "Add-SolutionContentDeploymentParameters -Template $template -UiDefinition $ui; " + "$global:baseCreateUiDefinition=$ui; Update-DefenderWorkbookDescriptionCount -Count $count; " + ) + if package: + metadata = self.solution / "SolutionMetadata.json" + metadata.write_text("{}", encoding="utf-8") + emit = ( + f"function git {{ $global:LASTEXITCODE=0; return {literal(self.root)} }}; " + "$global:baseMainTemplate=$template; $global:baseCreateUiDefinition=$ui; " + "$template | Add-Member -NotePropertyName metadata -NotePropertyValue " + "([pscustomobject]@{author='';comments='{{SolutionName}}'}); " + "$contentResourceDetails=[pscustomobject]@{apiVersion='3.0.0'}; " + "GeneratePackage -solutionName Sample -contentToImport $inputData.data -IncludeXdrDetections $true " + f"-SolutionMetadataPath {literal(metadata)} -SkipAttributionLookup; " + "$count=@($template.resources | Where-Object {$_.properties.contentKind -eq 'DefenderWorkbook'}).Count; " + ) + command = ( + "$ErrorActionPreference='Stop'; " + f". {literal(COMMON)}; " + f"$inputData=Get-Content -Raw -LiteralPath {literal(payload)} | ConvertFrom-Json; " + "$template=$inputData.template; $ui=$inputData.ui; " + "Register-SolutionResourceKind -Resources @($template.resources | Where-Object " + "{$_.type -eq 'Microsoft.OperationalInsights/workspaces/savedSearches'}) -Kind Parser; " + + ( + "$before=$template | ConvertTo-Json -Depth 100 -Compress; try {" + emit + + "} catch {if (($template | ConvertTo-Json -Depth 100 -Compress) -cne $before) " + "{throw 'TEMPLATE_MUTATED_BEFORE_FAILURE'}; throw}; " + if unchanged else emit + ) + + "Write-Output '__RESULT__'; " + "[pscustomobject]@{count=$count;template=$template;ui=$ui} | ConvertTo-Json -Depth 100 -Compress" + ) + result = subprocess.run( + ["pwsh", "-NoLogo", "-NoProfile", "-NonInteractive", "-Command", command], + capture_output=True, text=True, check=False, + ) + if not success: + self.assertNotEqual(0, result.returncode, result.stdout) + self.assertNotIn("__RESULT__", result.stdout) + self.assertNotIn("TEMPLATE_MUTATED_BEFORE_FAILURE", result.stderr) + return {"error": result.stderr} + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("__RESULT__", result.stdout) + return json.loads(result.stdout.split("__RESULT__", 1)[1].strip()) + + def add_function( + self, alias: str = "ReviewedParser", *, folder: str = "Parsers", + name: str = "[variables('parserObject1')._parserName1]", + ) -> dict: + path = self.solution / folder / f"{alias}.yaml" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + f"FunctionName: {alias}\nFunctionAlias: {alias}\n" + "FunctionParams:\n - Name: limit\n Type: int\n Default: 1\n" + "FunctionQuery: |\n DeviceEvents | take limit\n", encoding="utf-8", + ) + record = { + "functionAlias": alias, "sourcePath": path.relative_to(self.solution).as_posix(), + "sourceSha256": hashlib.sha256(path.read_bytes()).hexdigest(), + } + self.notebook["contentProvenance"].setdefault("requiredWorkspaceFunctions", []).append(record) + self.notebook["contentProvenance"]["conversion"]["queries"][0].setdefault( + "workspaceFunctions", [], + ).append(alias) + self.data.setdefault("Parsers", []).append(record["sourcePath"]) + self.template["variables"]["parserObject1"] = { + "_parserName1": "[concat(parameters('workspace'),'/','different-arm-name')]", + } + resource = { + "type": "Microsoft.OperationalInsights/workspaces/savedSearches", + "name": name, + "properties": { + "functionAlias": alias, "query": "DeviceEvents | take limit\n", + "functionParameters": "limit:int=1", "eTag": "*", + }, + } + self.template["resources"].append(resource) + self.write() + return resource + + def test_required_parser_dependencies_use_emitted_names_and_preserve_provenance(self) -> None: + self.add_function() + self.add_function("SecondParser", folder="Workspace Functions", name="lab/arm-name-not-alias") + source = self.source.read_bytes() + authored = self.target.read_bytes() + result = self.invoke() + resources = result["template"]["resources"] + install = next(r for r in resources if r["type"] == "Microsoft.Resources/deployments") + self.assertEqual([ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), " + "'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]", + "[resourceId('Microsoft.OperationalInsights/workspaces/savedSearches', variables('parserObject1')._parserName1)]", + "[resourceId('Microsoft.OperationalInsights/workspaces/savedSearches', 'lab/arm-name-not-alias')]", + ], install["dependsOn"]) + registration = next(r for r in resources if r["type"].endswith("/contentTemplates")) + self.assertEqual( + self.notebook["contentProvenance"]["requiredWorkspaceFunctions"], + registration["properties"]["contentProvenance"]["requiredWorkspaceFunctions"], + ) + installed = install["properties"]["template"]["resources"]["workbook"]["properties"]["content"] + self.assertEqual({"packageId", "packageVersion", "contentId", "contentVersion"}, set(installed["contentProvenance"])) + self.assertEqual(authored, self.target.read_bytes()) + self.assertEqual(source, self.source.read_bytes()) + self.assertNotIn("DeployParser", result["template"]["parameters"]) + self.assertNotIn("DeployParser", result["ui"]["parameters"]["outputs"]) + for resource in resources: + if resource["type"].endswith("/savedSearches"): + self.assertNotIn("condition", resource) + self.assertEqual("*", resource["properties"]["eTag"]) + + def test_function_dependency_ids_deduplicate_and_contract_references_work(self) -> None: + self.add_function() + self.add_function("SecondParser") + query = self.notebook["contentProvenance"]["conversion"]["queries"][0] + query["contract"] = {"workspaceFunctions": query.pop("workspaceFunctions")} + self.write() + result = self.invoke() + install = next(r for r in result["template"]["resources"] if r["type"] == "Microsoft.Resources/deployments") + self.assertEqual(2, len(install["dependsOn"])) + + def test_full_package_requires_parsers_and_linked_registrations_but_not_unrelated_parsers(self) -> None: + self.add_function() + metadata = { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "name": "lab/Microsoft.SecurityInsights/required-parser", + "properties": { + "kind": "Parser", "contentId": "[variables('parserObject1').parserContentId1]", + }, + } + registration = { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "name": "lab/Microsoft.SecurityInsights/required-registration", + "properties": { + "contentKind": "Parser", "contentId": "[variables('parserObject1').parserContentId1]", + }, + } + self.template["variables"]["parserObject1"]["parserContentId1"] = "ReviewedParser-Parser" + self.template["resources"] += [ + metadata, registration, { + "type": "Microsoft.OperationalInsights/workspaces/savedSearches", + "name": "lab/unrelated", + "properties": {"functionAlias": "UnrelatedParser", "query": "DeviceEvents"}, + }, + ] + original = self.target.read_bytes() + self.invoke(package=True) + package = json.loads((self.solution / "Package/mainTemplate.json").read_text(encoding="utf-8-sig")) + self.assertIn("DeployParser", package["parameters"]) + for resource in package["resources"][2:5]: + self.assertNotIn("condition", resource) + self.assertEqual("[parameters('DeployParser')]", package["resources"][5]["condition"]) + installation = next(r for r in package["resources"] if r["type"] == "Microsoft.Resources/deployments") + self.assertIn( + "[resourceId('Microsoft.OperationalInsights/workspaces/savedSearches', variables('parserObject1')._parserName1)]", + installation["dependsOn"], + ) + self.assertEqual(original, self.target.read_bytes()) + self.template["resources"][3]["condition"] = False + self.invoke(success=False, unchanged=True) + + def test_invalid_parser_provenance_and_selections_fail_without_mutation(self) -> None: + self.add_function() + baseline = copy.deepcopy(self.notebook) + data = copy.deepcopy(self.data) + for mutation in ( + "missing", "empty", "null", "extra", "duplicate", "bad-alias", "bad-path", + "uppercase-hash", "stale-hash", "not-selected", "duplicate-selection", + "missing-yaml", "wrong-yaml-alias", "conflicting-yaml-alias", + ): + with self.subTest(mutation=mutation): + self.notebook = copy.deepcopy(baseline) + self.data = copy.deepcopy(data) + provenance = self.notebook["contentProvenance"] + record = provenance["requiredWorkspaceFunctions"][0] + path = self.solution / record["sourcePath"] + original = path.read_bytes() + if mutation == "missing": + provenance.pop("requiredWorkspaceFunctions") + elif mutation == "empty": + provenance["requiredWorkspaceFunctions"] = [] + elif mutation == "null": + provenance["requiredWorkspaceFunctions"] = None + elif mutation == "extra": + record["unreviewed"] = True + elif mutation == "duplicate": + provenance["requiredWorkspaceFunctions"].append(copy.deepcopy(record)) + elif mutation == "bad-alias": + record["functionAlias"] = "bad-alias" + elif mutation == "bad-path": + record["sourcePath"] = "Parsers/../Parsers/ReviewedParser.yaml" + elif mutation == "uppercase-hash": + record["sourceSha256"] = record["sourceSha256"].upper() + elif mutation == "stale-hash": + record["sourceSha256"] = "0" * 64 + elif mutation == "not-selected": + self.data["Parsers"] = [] + elif mutation == "duplicate-selection": + self.data["Parsers"] *= 2 + elif mutation == "missing-yaml": + path.unlink() + else: + path.write_bytes( + original.replace(b"ReviewedParser", b"OtherParser") + if mutation == "wrong-yaml-alias" + else original.replace(b"FunctionAlias: ReviewedParser", b"FunctionAlias: OtherParser") + ) + record["sourceSha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + self.write() + self.invoke(success=False, unchanged=True) + path.write_bytes(original) + + def test_effective_yaml_alias_and_all_selection_fields(self) -> None: + self.add_function() + record = self.notebook["contentProvenance"]["requiredWorkspaceFunctions"][0] + path = self.solution / record["sourcePath"] + original = path.read_bytes() + for field in ("Parsers", "Data Parsers", "Workspace Functions"): + with self.subTest(field=field): + for selection in ("Parsers", "Data Parsers", "Workspace Functions"): + self.data.pop(selection, None) + self.data[field] = [record["sourcePath"]] + path.write_bytes(original.replace(b"FunctionName: ReviewedParser", b"FunctionName: InternalParserName")) + record["sourceSha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + self.write() + self.assertEqual(1, self.invoke()["count"]) + path.write_bytes(original.replace(b"FunctionAlias: ReviewedParser\n", b"")) + record["sourceSha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + self.write() + self.assertEqual(1, self.invoke()["count"]) + + def test_parser_resource_failures_are_atomic_across_workbooks(self) -> None: + self.add_function() + other = self.target.with_name("ordinary.json") + ordinary = copy.deepcopy(self.notebook) + ordinary["contentProvenance"].pop("requiredWorkspaceFunctions") + ordinary["contentProvenance"]["conversion"]["queries"][0].pop("workspaceFunctions") + for field in ("contentId", "instanceId"): + ordinary["xdrMetadata"][field] = str(uuid.uuid5(uuid.NAMESPACE_URL, f"ordinary/{field}")) + other.write_text(json.dumps(ordinary), encoding="utf-8") + self.data["XDR Workbooks"].insert(0, other.relative_to(self.solution).as_posix()) + template = copy.deepcopy(self.template) + for mutation in ( + "missing", "duplicate", "conditional", "conditional-true", "copy", + "wrong-alias", "wrong-query", "wrong-parameters", "stored-only", "nested-only", + "bad-name", + ): + with self.subTest(mutation=mutation): + self.template = copy.deepcopy(template) + resource = self.template["resources"][-1] + if mutation == "missing": + self.template["resources"].pop() + elif mutation == "duplicate": + self.template["resources"].append(copy.deepcopy(resource)) + elif mutation in ("conditional", "conditional-true"): + resource["condition"] = True if mutation.endswith("true") else "[parameters('DeployParser')]" + elif mutation == "copy": + resource["copy"] = {"name": "parserLoop", "count": 1} + elif mutation == "wrong-alias": + resource["properties"]["functionAlias"] = "reviewedparser" + elif mutation == "wrong-query": + resource["properties"]["query"] = "DeviceEvents | take 2" + elif mutation == "wrong-parameters": + resource["properties"]["functionParameters"] = "" + elif mutation == "bad-name": + resource["name"] = "alias-only" + else: + self.template["resources"].pop() + self.template["resources"].append({ + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates" + if mutation == "stored-only" else "Microsoft.Resources/deployments", + "properties": {"mainTemplate" if mutation == "stored-only" else "template": { + "resources": [resource], + }}, + }) + self.invoke(success=False, unchanged=True) + + def test_implicit_function_calls_cannot_omit_reviewed_prerequisites(self) -> None: + source = self.source.read_bytes() + baseline = copy.deepcopy(self.notebook) + for query in ( + "HiddenParser() | take 1", + "(HiddenParser()) | count", + "SourceTable | invoke HiddenParser()", + "DeviceEvents | join (HiddenParser()) on DeviceId", + "DeviceEvents | lookup kind=leftouter (HiddenParser()) on DeviceId", + "union DeviceEvents, (HiddenParser())", + "let Data = HiddenParser(); Data", + "let x = (HiddenParser()); x", + "print toscalar(HiddenParser() | summarize count())", + "print HiddenParser()", + "print tostring(HiddenParser())", + "DeviceEvents | extend x = HiddenParser()", + "DeviceEvents | where DeviceName == HiddenParser()", + "DeviceEvents | where DeviceName in (HiddenParser())", + "print (HiddenParser())", + "find in (HiddenParser()) where * has 'x'", + "search in (HiddenParser()) 'x'", + "find in (HiddenParser) where * has 'x'", + "search in (HiddenParser) 'x'", + "HiddenParser | count", + "(HiddenParser) | count", + "let x = HiddenParser; x", + "let x = (HiddenParser); x", + "let x = materialize(HiddenParser | take 1); x", + "let x = materialize(HiddenParser); x", + "DeviceEvents | where DeviceName in (HiddenParser | project DeviceName)", + "DeviceEvents | where DeviceName in ((HiddenParser) | project DeviceName)", + "DeviceEvents | where DeviceName in ( ( HiddenParser ) | project DeviceName)", + "['HiddenParser']() | take 1", + '["HiddenParser"]() | take 1', + "[\n'HiddenParser'\n]() | take 1", + "print ['HiddenParser']()", + "['HiddenParser'] | take 10", + "union ['HiddenParser']", + "union kind=outer ['HiddenParser']", + "union isfuzzy=true ['HiddenParser']", + "DeviceEvents | invoke ['HiddenParser']()", + "DeviceEvents | invoke ['HiddenParser']", + "table('HiddenParser') | take 10", + "table('HiddenParser', 'all') | take 10", + "table('HiddenParser', 'hotcache') | take 10", + "table('HiddenParser', 'default') | take 10", + "totable('HiddenParser') | take 10", + "totable('HiddenParser', 'all') | take 10", + "table(strcat('Hidden', 'Parser')) | take 10", + "totable(strcat('Hidden', 'Parser')) | take 10", + "let local=(){HiddenParser()}; local()", + "let local=(){HiddenParser}; local()", + "let local=view(){HiddenParser}; local()", + "view() { HiddenParser }", + "DeviceEvents | partition by DeviceId { HiddenParser }", + "DeviceEvents | fork (HiddenParser)", + "DeviceEvents | fork (['HiddenParser'])", + "DeviceEvents | where DeviceName has ['HiddenParser']()", + "DeviceEvents | where DeviceName !has ['HiddenParser']()", + "DeviceEvents | where DeviceName contains ['HiddenParser']()", + "DeviceEvents | where DeviceName !contains ['HiddenParser']()", + "DeviceEvents | where DeviceName startswith ['HiddenParser']()", + "DeviceEvents | where DeviceName endswith ['HiddenParser']()", + "DeviceEvents | where DeviceName has_cs ['HiddenParser']()", + "DeviceEvents | where DeviceName has['HiddenParser']()", + "DeviceEvents | where DeviceName !has['HiddenParser']()", + "DeviceEvents | where DeviceName has['HiddenParser'] ()", + "DeviceEvents | where DeviceName has['HiddenParser']\n()", + "DeviceEvents | where DeviceName contains['HiddenParser']()", + "DeviceEvents | where DeviceName !contains['HiddenParser']()", + "DeviceEvents | where DeviceName startswith['HiddenParser']()", + "DeviceEvents | where DeviceName endswith['HiddenParser']()", + "DeviceEvents | where DeviceName has_cs['HiddenParser']()", + "DeviceEvents | distinct ['HiddenParser']()", + "DeviceEvents | distinct['HiddenParser']()", + "DeviceEvents | mv-expand ['HiddenParser']()", + "DeviceEvents | mv-expand['HiddenParser']()", + "print['HiddenParser']()", + "union['HiddenParser']", + "union kind=outer['HiddenParser']", + "union isfuzzy=true['HiddenParser']", + "DeviceEvents | invoke['HiddenParser']()", + "DeviceEvents | invoke['HiddenParser']", + "print bag['HiddenParser']()", + "DeviceEvents | join hint.strategy=broadcast (HiddenParser) on DeviceId", + "DeviceEvents | join kind=inner hint.strategy=broadcast (HiddenParser) on DeviceId", + "DeviceEvents | join hint.strategy=shuffle (['HiddenParser']) on DeviceId", + "Evil_CL | take 1", + "union Evil_CL", + "table('Evil_CL') | take 1", + "/* HiddenParser() */ | take 1", + "{QueryParameter} | take 1", + "{QueryParameter}() | take 1", + "({QueryParameter} | project DeviceName)", + "DeviceEvents | where DeviceName in ({QueryParameter} | project DeviceName)", + "{Hidden Func}() | take 1", + "({Hidden Func} | project DeviceName)", + "table({QueryParameter}) | take 1", + ): + for side in ("source", "target"): + with self.subTest(query=query, side=side): + self.notebook = copy.deepcopy(baseline) + self.source.write_bytes(source) + record = self.notebook["contentProvenance"]["conversion"]["queries"][0] + if side == "source": + document = json.loads(source) + document["items"][0]["content"]["query"] = query + self.source.write_text(json.dumps(document), encoding="utf-8") + self.notebook["contentProvenance"]["source"]["sha256"] = hashlib.sha256(self.source.read_bytes()).hexdigest() + record["sourceSha256"] = hashlib.sha256(query.encode()).hexdigest() + else: + self.notebook["items"][0]["content"]["query"] = query + record["targetSha256"] = hashlib.sha256(query.encode()).hexdigest() + self.write() + self.invoke(success=False, unchanged=True) + self.notebook = copy.deepcopy(baseline) + self.source.write_bytes(source) + wrapped = json.dumps({"__advancedHuntingSettings": True, "query": "HiddenParser()", "selectedWorkspaceIds": []}) + self.notebook["items"][0]["content"].update({"query": wrapped, "queryType": 18}) + self.notebook["contentProvenance"]["conversion"]["queries"][0]["targetSha256"] = hashlib.sha256(wrapped.encode()).hexdigest() + self.write() + self.invoke(success=False, unchanged=True) + self.notebook = copy.deepcopy(baseline) + self.source.write_bytes(source) + self.add_function() + self.notebook["items"][0]["content"]["query"] = "ReviewedParser() | take 1" + self.notebook["contentProvenance"]["conversion"]["queries"][0]["targetSha256"] = hashlib.sha256( + b"ReviewedParser() | take 1", + ).hexdigest() + self.write() + self.assertEqual(1, self.invoke()["count"]) + + def test_ordinary_workbooks_allow_absent_or_empty_functions_and_ignore_literals(self) -> None: + for functions in (None, []): + with self.subTest(functions=functions): + if functions is not None: + self.notebook["contentProvenance"]["requiredWorkspaceFunctions"] = functions + self.write() + result = self.invoke() + registration = result["template"]["resources"][-1] + self.assertEqual([], registration["properties"]["contentProvenance"]["requiredWorkspaceFunctions"]) + self.assertEqual(1, len(result["template"]["resources"][-2]["dependsOn"])) + for query in ( + "print text='HiddenParser()' // HiddenParser()\n", + "print text=@'HiddenParser()' // /* HiddenParser() */\n", + "let Local = () { DeviceEvents }; Local()", + "let LocalTable = DeviceEvents; LocalTable | count", + "DeviceEvents | where DeviceName in ({QueryParameter}) | take 1", + "let Watchlist = _GetWatchlist('sample'); Watchlist", + "_GetWatchlist('sample') | take 1", + "let Window = ago(1h); DeviceEvents | where Timestamp > Window", + "DeviceEvents | where (array_length(AdditionalFields) > 0)", + "DeviceEvents | extend ['Activity Count'] = 1 | project ['Activity Count'], [\"ResolutionTime\"] = 2 | order by ['Activity Count'] desc", + "DeviceEvents | extend v = todynamic(AdditionalFields)['nested'] | where isnotempty(['Activity Count'])", + "DeviceEvents | where tostring(AdditionalFields['Leak']) != '' and isnotempty(AdditionalFields['Leak']['nested'])", + "let LocalView = view() { DeviceEvents }; LocalView()", + "table('DeviceEvents') | take 1", + "table('DeviceEvents', 'all') | take 1", + "totable('DeviceEvents') | take 1", + "externaldata(Value:string)['https://example.invalid/reference.csv'] with (format='csv')", + ): + with self.subTest(query=query): + self.notebook["items"][0]["content"]["query"] = query + self.notebook["contentProvenance"]["conversion"]["queries"][0]["targetSha256"] = hashlib.sha256(query.encode()).hexdigest() + self.write() + self.assertEqual(1, self.invoke()["count"]) + + def test_decoy_functions_cannot_satisfy_hint_flags_or_unsupported_table_indirection(self) -> None: + source = self.source.read_bytes() + decoys = ("hint", "broadcast", "__unsupported_table_indirection__", "__unsupported_escaped_call_context__") + for idx, alias in enumerate(decoys, start=1): + self.add_function(alias=alias, name=f"synthetic-parser-{idx}") + baseline = copy.deepcopy(self.notebook) + for query in ( + "DeviceEvents | join hint.strategy=broadcast (HiddenParser) on DeviceId", + "DeviceEvents | join kind=inner hint.strategy=broadcast (HiddenParser) on DeviceId", + "DeviceEvents | join hint.strategy=shuffle (['HiddenParser']) on DeviceId", + "table(strcat('Hidden', 'Parser')) | take 1", + "print bag['HiddenParser']()", + "DeviceEvents | where isnotempty(todynamic(AdditionalFields)['HiddenParser']())", + ): + with self.subTest(query=query): + self.notebook = copy.deepcopy(baseline) + self.source.write_bytes(source) + record = self.notebook["contentProvenance"]["conversion"]["queries"][0] + self.notebook["items"][0]["content"]["query"] = query + record["targetSha256"] = hashlib.sha256(query.encode()).hexdigest() + self.write() + self.invoke(success=False, unchanged=True) + + def test_watchlist_reference_data_cannot_be_a_telemetry_dependency(self) -> None: + baseline = copy.deepcopy(self.notebook) + for field in ("requiredWorkloads", "requiredDataConnectors"): + for data_type in ("Watchlist", "watchlist"): + with self.subTest(field=field, data_type=data_type): + self.notebook = copy.deepcopy(baseline) + self.notebook["contentProvenance"][field][0]["dataTypes"].append(data_type) + self.write() + self.invoke(success=False, unchanged=True) + self.notebook = copy.deepcopy(baseline) + query = "let Reference = _GetWatchlist('sample'); DeviceEvents | take 1" + self.notebook["items"][0]["content"]["query"] = query + provenance = self.notebook["contentProvenance"]["conversion"]["queries"][0] + provenance["targetSha256"] = hashlib.sha256(query.encode()).hexdigest() + provenance["referenceInputs"] = ["Watchlist"] + self.write() + self.assertEqual(1, self.invoke()["count"]) + + def test_three_records_identity_registration_projection_and_source_preservation(self) -> None: + authored = {self.source: self.source.read_bytes(), self.target: self.target.read_bytes()} + for index in (1, 2): + document = copy.deepcopy(self.notebook) + for field in ("contentId", "instanceId"): + document["xdrMetadata"][field] = str(uuid.uuid5(uuid.NAMESPACE_URL, f"fixture/{index}/{field}")) + path = self.target.with_name(f"workbook-{index}.json") + path.write_text(json.dumps(document), encoding="utf-8") + self.data["XDR Workbooks"].append(path.relative_to(self.solution).as_posix()) + authored[path] = path.read_bytes() + result = self.invoke() + self.assertEqual(3, result["count"]) + resources = result["template"]["resources"] + installs = [r for r in resources if r["type"] == "Microsoft.Resources/deployments"] + registrations = [r for r in resources if r["type"].endswith("/contentTemplates")] + self.assertEqual(3, len(installs)) + self.assertEqual(3, len(registrations)) + self.assertEqual(3, len({r["name"] for r in installs})) + criteria = resources[0]["properties"]["dependencies"]["criteria"] + self.assertEqual(3, len([c for c in criteria if c["kind"] == "DefenderWorkbook"])) + self.assertEqual("Workbook", criteria[0]["kind"]) + for installation, registration in zip(installs, registrations): + inner = installation["properties"]["template"] + stored = registration["properties"]["mainTemplate"] + self.assertEqual(inner, stored) + self.assertEqual("inner", installation["properties"]["expressionEvaluationOptions"]["scope"]) + self.assertEqual("2.0", inner["languageVersion"]) + self.assertEqual({"provider": "MicrosoftSecurity", "version": "1.0.0"}, inner["imports"]["MicrosoftSecurity"]) + self.assertEqual("Microsoft.Security/workbooks@2026-06-01-preview", inner["resources"]["workbook"]["type"]) + props = inner["resources"]["workbook"]["properties"] + self.assertIsInstance(props["content"], dict) + self.assertEqual("Notebook/1.0", props["content"]["version"]) + self.assertNotIn("xdrMetadata", props["content"]) + provenance = props["content"]["contentProvenance"] + self.assertEqual({"packageId", "packageVersion", "contentId", "contentVersion"}, set(provenance)) + self.assertEqual("3.2.7", provenance["contentVersion"]) + self.assertNotEqual(props["id"], provenance["contentId"]) + for field in ("packageId", "packageVersion", "contentId"): + self.assertEqual(provenance[field], registration["properties"][field]) + self.assertEqual(provenance["contentVersion"], registration["properties"]["version"]) + full = registration["properties"]["contentProvenance"] + self.assertEqual(self.notebook["contentProvenance"]["requiredWorkloads"], full["requiredWorkloads"]) + self.assertEqual(self.notebook["contentProvenance"]["requiredDataConnectors"], full["requiredDataConnectors"]) + self.assertNotIn("source", full) + self.assertNotIn("variables(", json.dumps(provenance)) + self.assertEqual("[parameters('DeployDefenderWorkbook')]", installation["condition"]) + self.assertEqual(installation["condition"], registration["condition"]) + self.assertEqual("DefenderWorkbook", registration["properties"]["contentKind"]) + self.assertEqual("2023-04-01-preview", registration["apiVersion"]) + self.assertIn(installation["name"], registration["dependsOn"][0]) + for path, original in authored.items(): + self.assertEqual(original, path.read_bytes()) + self.assertIn("**XDR Workbooks:** 3", result["ui"]["parameters"]["config"]["basics"]["description"]) + + def test_flags_ui_test_parameter_defaults_and_native_route_are_independent(self) -> None: + result = self.invoke() + parameters = result["template"]["parameters"] + self.assertTrue(parameters["DeployWorkbook"]["defaultValue"]) + self.assertFalse(parameters["DeployDefenderWorkbook"]["defaultValue"]) + self.assertNotIn("DeployCustomDetection", parameters) + outputs = result["ui"]["parameters"]["outputs"] + self.assertEqual("[steps('contentSelection').DeployDefenderWorkbook]", outputs["DeployDefenderWorkbook"]) + elements = result["ui"]["parameters"]["steps"][0]["elements"] + self.assertFalse(next(e for e in elements if e["name"] == "DeployDefenderWorkbook")["defaultValue"]) + native = result["template"]["resources"][1] + condition = native.pop("condition") + original = copy.deepcopy(self.native) + original.pop("condition") + self.assertEqual(original, native) + self.assertEqual("[and(parameters('DeployWorkbook'), equals('native','native'))]", condition) + self.assertNotIn("DeployWorkbook", json.dumps(result["template"]["resources"][2:])) + + def test_generate_package_writes_independent_test_parameters_and_zip(self) -> None: + original = self.target.read_bytes() + result = self.invoke(package=True) + package = self.solution / "Package" + parameters = json.loads((package / "testParameters.json").read_text(encoding="utf-8-sig")) + self.assertEqual(result["template"]["parameters"], parameters) + self.assertTrue(parameters["DeployWorkbook"]["defaultValue"]) + self.assertFalse(parameters["DeployDefenderWorkbook"]["defaultValue"]) + with zipfile.ZipFile(package / "3.2.7.zip") as archive: + template = json.loads(archive.read("mainTemplate.json")) + ui = json.loads(archive.read("createUiDefinition.json")) + self.assertEqual(4, len(template["resources"])) + self.assertEqual("[steps('contentSelection').DeployDefenderWorkbook]", ui["parameters"]["outputs"]["DeployDefenderWorkbook"]) + self.assertEqual(original, self.target.read_bytes()) + self.assertIn("**XDR Workbooks:** 1", ui["parameters"]["config"]["basics"]["description"]) + + def test_empty_selection_never_scans_or_emits_flag(self) -> None: + self.data["XDR Workbooks"] = [] + result = self.invoke() + self.assertEqual(0, result["count"]) + self.assertNotIn("DeployDefenderWorkbook", result["template"]["parameters"]) + self.assertEqual(2, len(result["template"]["resources"])) + + def test_existing_files_require_selection_and_selection_must_be_array(self) -> None: + for value in (None, "[]", {}, [7], [None]): + with self.subTest(value=value): + if value is None: + self.data.pop("XDR Workbooks", None) + else: + self.data["XDR Workbooks"] = value + self.invoke(success=False) + + def test_no_workbook_directory_does_not_add_selection(self) -> None: + shutil.rmtree(self.solution / "XDR Workbooks") + self.data.pop("XDR Workbooks") + result = self.invoke() + self.assertEqual(0, result["count"]) + self.assertNotIn("DeployDefenderWorkbook", result["template"]["parameters"]) + + def test_version_bounds_and_package_boundary(self) -> None: + for version in ("3.0.9", "4.0.0", "Notebook/1.0", "3.1", "03.1.0", "3.01.0"): + with self.subTest(version=version): + self.notebook["xdrMetadata"]["version"] = version + self.write() + self.invoke(success=False) + self.notebook["xdrMetadata"]["version"] = "3.1.0" + self.write() + self.template["variables"]["_solutionVersion"] = "4.0.0" + self.invoke(success=False) + self.template["variables"]["_solutionVersion"] = "3.2.6" + self.invoke(success=False) + + def test_review_missing_arrays_unknown_workload_and_stale_source_block(self) -> None: + baseline = copy.deepcopy(self.notebook) + mutations = [ + ("conversion", "reviewRequired", True), ("conversion", "reviewRequired", None), + ("conversion", "tool", ""), ("conversion", "version", None), + (None, "dependencyReviewRequired", True), (None, "dependencyReviewRequired", None), + (None, "requiredWorkloads", None), (None, "requiredDataConnectors", "[]"), + (None, "requiredWorkloads", [{"workload": "unknown", "dataTypes": []}]), + (None, "requiredDataConnectors", [{"connectorId": "Connector", "dataTypes": [7]}]), + ("source", "sha256", "a" * 64), ("source", "path", "../outside.json"), + ] + for section, field, value in mutations: + with self.subTest(field=field, value=value): + self.notebook = copy.deepcopy(baseline) + container = self.notebook["contentProvenance"] + if section: + container = container[section] + container[field] = value + self.write() + self.invoke(success=False) + + def test_selection_paths_duplicates_and_symlinks_fail_closed(self) -> None: + for selection in ( + [self.reference, self.reference], + [self.reference, f"Solutions/Sample/{self.reference}"], + ["../outside.json"], ["Workbooks/nested/source.json"], + ["/absolute.json"], ["XDR Workbooks/../Workbooks/nested/source.json"], + ): + with self.subTest(selection=selection): + self.data["XDR Workbooks"] = selection + self.invoke(success=False) + link = self.target.with_name("link.json") + link.symlink_to(self.target) + self.data["XDR Workbooks"] = [link.relative_to(self.solution).as_posix()] + self.invoke(success=False) + link.unlink() + source_link = self.source.with_name("link.json") + source_link.symlink_to(self.source) + self.notebook["contentProvenance"]["source"]["path"] = "Workbooks/nested/link.json" + self.write() + self.data["XDR Workbooks"] = [self.reference] + self.invoke(success=False) + + def test_colliding_authored_identities_are_not_rewritten(self) -> None: + self.notebook["xdrMetadata"]["instanceId"] = self.notebook["xdrMetadata"]["contentId"] + self.write() + self.invoke(success=False) + self.notebook["xdrMetadata"]["instanceId"] = str(uuid.uuid5(uuid.NAMESPACE_URL, "fixture/other")) + self.write() + other = self.target.with_name("other.json") + other.write_bytes(self.target.read_bytes()) + self.data["XDR Workbooks"].append(other.relative_to(self.solution).as_posix()) + self.invoke(success=False) + + def test_arm_looking_strings_escape_only_deployment_copies(self) -> None: + self.notebook["xdrMetadata"]["displayName"] = "[authored title]" + self.notebook["items"].append({"type": 1, "content": {"json": ["[literal]", None, True, 42]}}) + self.write() + original = self.target.read_bytes() + result = self.invoke() + installation = result["template"]["resources"][2] + properties = installation["properties"]["template"]["resources"]["workbook"]["properties"] + self.assertEqual("[[authored title]", properties["name"]) + self.assertEqual("[[parameters('authored-literal')]", properties["content"]["items"][1]["content"]["json"]) + self.assertEqual("[[[already-bracketed]", properties["content"]["fallbackResourceIds"][0]) + self.assertEqual(["[[literal]", None, True, 42], properties["content"]["items"][2]["content"]["json"]) + self.assertEqual(original, self.target.read_bytes()) + self.assertEqual("[[authored title]", result["template"]["resources"][3]["properties"]["displayName"]) + + def test_query_current_hash_blockers_and_pruned_scope_record(self) -> None: + baseline = copy.deepcopy(self.notebook) + for mutation in ("stale", "blocked", "missing-pointer", "missing-array"): + with self.subTest(mutation=mutation): + self.notebook = copy.deepcopy(baseline) + conversion = self.notebook["contentProvenance"]["conversion"] + if mutation == "missing-array": + conversion.pop("queries") + elif mutation == "stale": + self.notebook["items"][0]["content"]["query"] += " " + elif mutation == "blocked": + conversion["queries"][0]["status"] = "needs-review" + else: + conversion["queries"][0]["outputPath"] = "/items/99/query" + self.write() + self.invoke(success=False) + self.notebook = copy.deepcopy(baseline) + self.notebook["contentProvenance"]["conversion"]["queries"].append({ + "sourcePath": "/items/1/query", "sourceSha256": hashlib.sha256(b"scope").hexdigest(), + "outputPath": None, "targetSha256": None, "status": "removed-scope-helper", + }) + self.write() + self.assertEqual(1, self.invoke()["count"]) + + def test_no_query_notebook_keeps_complete_layout(self) -> None: + self.notebook["items"] = [{"type": 1, "name": "text-only", "content": {"json": "fixture text"}}] + self.notebook["contentProvenance"]["conversion"]["queries"] = [] + self.notebook["contentProvenance"]["requiredWorkloads"] = [] + self.notebook["contentProvenance"]["requiredDataConnectors"] = [] + self.source.write_text(json.dumps({"version": "Notebook/1.0", "items": []}), encoding="utf-8") + self.notebook["contentProvenance"]["source"]["sha256"] = hashlib.sha256(self.source.read_bytes()).hexdigest() + self.write() + result = self.invoke() + content = result["template"]["resources"][2]["properties"]["template"]["resources"]["workbook"]["properties"]["content"] + self.assertEqual(self.notebook["items"], content["items"]) + + def test_nested_notebook_layout_is_not_serialized_to_truncated_text(self) -> None: + nested = {"leaf": "[literal]"} + for _ in range(60): + nested = {"child": nested} + self.notebook["items"][1]["content"]["layout"] = nested + self.write() + result = self.invoke(package=True) + packaged = json.loads((self.solution / "Package/mainTemplate.json").read_text(encoding="utf-8-sig")) + content = packaged["resources"][2]["properties"]["template"]["resources"]["workbook"]["properties"]["content"] + value = content["items"][1]["content"]["layout"] + for _ in range(60): + self.assertIsInstance(value, dict) + value = value["child"] + self.assertEqual({"leaf": "[[literal]"}, value) + self.assertEqual(1, result["count"]) + for _ in range(30): + nested = {"child": nested} + self.notebook["items"][1]["content"]["layout"] = nested + self.write() + self.invoke(success=False) + + def test_v3_entrypoint_method_forwarding_preserves_legacy_default(self) -> None: + toolkit = self.root / "Tools/Create-Azure-Sentinel-Solution" + (toolkit / "V3").mkdir(parents=True) + (toolkit / "common").mkdir() + (toolkit / "common/createSolutionLocal.ps1").write_text( + "param($SolutionDataFolderPath,$VersionMode,$VersionBump,$TargetVersion," + "$EntryPointName,$DefaultInputPath,$ToolkitRootPath,$IncludeXdrDetections," + "$EnforceV31Versions,[switch]$SkipAttributionLookup," + "[ValidateSet('arm-ttk','lab-deployment')]$ArmValidationMethod='arm-ttk')\n" + "[pscustomobject]@{method=$ArmValidationMethod;entry=$EntryPointName;" + "xdr=$IncludeXdrDetections} | ConvertTo-Json -Compress\n", + encoding="utf-8", + ) + for entry, method in (("createSolutionV3_1.ps1", None), ("createSolutionV3_1.ps1", "lab-deployment"), + ("createSolutionV3.ps1", "lab-deployment")): + with self.subTest(entry=entry, method=method): + destination = toolkit / "V3" / entry + shutil.copy2(ROOT / "Tools/Create-Azure-Sentinel-Solution/V3" / entry, destination) + arguments = ["pwsh", "-NoLogo", "-NoProfile", "-NonInteractive", "-File", + str(destination), "-SolutionDataFolderPath", "fixture"] + if method: + arguments += ["-ArmValidationMethod", method] + result = subprocess.run(arguments, capture_output=True, text=True, check=False) + self.assertEqual(0, result.returncode, result.stderr) + record = json.loads(result.stdout) + self.assertEqual(method or "arm-ttk", record["method"]) + self.assertEqual(entry == "createSolutionV3_1.ps1", record["xdr"] is True) + legacy = (ROOT / "Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3.ps1").read_text() + self.assertIn('RunArmTtkOnPackage -solutionName $solutionName -isPipelineRun $false', legacy) + + +if __name__ == "__main__": + unittest.main() diff --git a/Tools/SentinelToXDRMigration/tests/test_dependency_provenance.py b/Tools/SentinelToXDRMigration/tests/test_dependency_provenance.py new file mode 100644 index 00000000000..33b1d876eef --- /dev/null +++ b/Tools/SentinelToXDRMigration/tests/test_dependency_provenance.py @@ -0,0 +1,338 @@ +from __future__ import annotations + +import copy +import contextlib +import hashlib +import io +import json +import tempfile +import time +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +import yaml +from azure.core.credentials import AccessToken +from azure.core.pipeline.transport import HttpResponse, HttpTransport +from azure.monitor.query import LogsQueryClient + +from sentinel_xdr_migration.converter import build_xdr_document, convert_solution, validate_document +from sentinel_xdr_migration import cli +from sentinel_xdr_migration.dependencies import ( + dependency_plan, live_dependency_check, plan_solution_dependencies, + resolve_data_types, solution_package_context, +) +from sentinel_xdr_migration.graph_readiness import graph_detection_payload +from sentinel_xdr_migration.logs_query import execute_workspace_query +from sentinel_xdr_migration.runtime import current_runtime_evidence, record_runtime_validation +from test_converter import RULE +from test_logs_query import WORKSPACE, logs_table, success + + +class DependencyProvenanceTests(unittest.TestCase): + def setUp(self): + self.fixture = tempfile.TemporaryDirectory() + self.addCleanup(self.fixture.cleanup) + self.root = Path(self.fixture.name) + self.source = self.root / "Analytic Rules/Rule.yaml" + self.source.parent.mkdir() + self.source.write_text(RULE) + + def document(self): + return build_xdr_document(self.source, self.root, {}) + + def test_conversion_is_offline_and_separates_dependencies_from_deployment_keys(self): + with mock.patch("sentinel_xdr_migration.logs_query.execute_workspace_query") as query: + document = self.document() + query.assert_not_called() + provenance = document["contentProvenance"] + self.assertEqual([{"workload": "mde", "dataTypes": ["DeviceEvents"]}], provenance["requiredWorkloads"]) + self.assertEqual([], provenance["requiredDataConnectors"]) + for key in ("packageId", "packageVersion", "contentId", "contentVersion"): + self.assertNotIn(key, provenance) + self.assertEqual([], validate_document(document)) + + def test_yaml_requires_dependency_fields_and_supported_workload_records(self): + document = self.document() + for key in ("requiredWorkloads", "requiredDataConnectors"): + candidate = copy.deepcopy(document) + del candidate["contentProvenance"][key] + self.assertTrue(validate_document(candidate)) + for record in ( + "mde", {}, {"workload": "defender", "dataTypes": []}, + {"workload": "mde", "dataTypes": [""]}, + {"workload": "mde", "dataTypes": "DeviceEvents"}, + ): + candidate = copy.deepcopy(document) + candidate["contentProvenance"]["requiredWorkloads"] = [record] + self.assertTrue(validate_document(candidate), record) + document["contentProvenance"]["dependencyReviewRequired"] = True + self.assertTrue(any("needs review" in error for error in validate_document(document))) + + def test_metadata_threshold_counts_distinct_input_tables(self): + for count in (1, 10): + tables = [f"Table{index}" for index in range(count)] + result = resolve_data_types("DeviceEvents | take 1", [ + {"tables": tables + tables}, + ]) + self.assertEqual("log-analytics", result["method"]) + self.assertEqual(sorted(tables), result["dataTypes"]) + result = resolve_data_types("DeviceEvents | take 1", [ + {"tables": [{"name": f"Table{index}"} for index in range(11)]}, + ]) + self.assertEqual("regex", result["method"]) + self.assertEqual("more-than-10-tables", result["fallbackReason"]) + self.assertEqual(["DeviceEvents"], result["dataTypes"]) + self.assertEqual(11, len(result["returnedDataTypes"])) + + def test_missing_metadata_falls_back_but_malformed_metadata_is_an_error(self): + for metadata in (None, [], [{"resourceId": "/workspace/example"}], [{"tables": None}]): + result = resolve_data_types("DeviceInfo | take 1", metadata) + self.assertEqual("regex", result["method"]) + self.assertEqual(["DeviceInfo"], result["dataTypes"]) + for metadata in ({}, [None], [{"tables": [""]}], [{"tables": "DeviceInfo"}]): + with self.assertRaises(ValueError): + resolve_data_types("DeviceInfo | take 1", metadata) + result = dependency_plan("SavedFunction() | take 1") + self.assertEqual([], result["dataTypes"]) + self.assertEqual([], result["requiredWorkloads"]) + self.assertEqual([], result["requiredDataConnectors"]) + self.assertTrue(result["reviewRequired"]) + + def test_shared_security_alert_does_not_invent_defender_or_connector_requirements(self): + result = dependency_plan("SecurityAlert | take 1") + self.assertEqual([{"workload": "sentinel", "dataTypes": ["SecurityAlert"]}], result["requiredWorkloads"]) + self.assertEqual([], result["requiredDataConnectors"]) + self.assertTrue(result["reviewRequired"]) + + def test_live_validation_compares_workloads_and_source_connector_declarations(self): + document = self.document() + metadata = [{"tables": ["DeviceEvents"]}] + self.assertEqual("passed", live_dependency_check( + document, "DeviceEvents | take 1", metadata, + )["status"]) + connectors = [{"connectorId": "MicrosoftThreatProtection", "dataTypes": ["DeviceEvents"]}] + result = live_dependency_check( + document, "DeviceEvents | take 1", metadata, source_connectors=connectors, + ) + self.assertEqual("failed", result["status"]) + self.assertIn("connector MicrosoftThreatProtection", "; ".join(result["errors"])) + document["contentProvenance"]["requiredDataConnectors"] = connectors + self.assertEqual("passed", live_dependency_check( + document, "DeviceEvents | take 1", metadata, source_connectors=connectors, + )["status"]) + document["contentProvenance"]["requiredWorkloads"] = [] + self.assertEqual("failed", live_dependency_check( + document, "DeviceEvents | take 1", metadata, + )["status"]) + + def test_live_validation_covers_tables_added_to_the_converted_query(self): + document = self.document() + document["properties"]["queryCondition"]["queryText"] = ( + "DeviceEvents | join kind=inner (DeviceInfo) on DeviceId" + ) + result = live_dependency_check( + document, "DeviceEvents", [{"tables": ["DeviceEvents"]}], + ) + self.assertEqual("failed", result["status"]) + self.assertTrue(any("DeviceInfo" in error for error in result["errors"])) + + def test_reviewed_unknown_table_mapping_can_be_declared_without_private_catalog(self): + document = self.document() + document["properties"]["queryCondition"]["queryText"] = "ReviewedIdentityTable | take 1" + document["contentProvenance"]["requiredWorkloads"] = [ + {"workload": "entraid", "dataTypes": ["ReviewedIdentityTable"]}, + ] + result = live_dependency_check( + document, "ReviewedIdentityTable | take 1", + [{"tables": ["ReviewedIdentityTable"]}], + ) + self.assertEqual("passed", result["status"]) + + def test_live_validation_checks_connector_identity_even_without_data_types(self): + document = self.document() + result = live_dependency_check( + document, "DeviceEvents", [{"tables": ["DeviceEvents"]}], + source_connectors=[{"connectorId": "OtherConnector", "dataTypes": []}], + ) + self.assertEqual("failed", result["status"]) + self.assertTrue(any("OtherConnector" in error for error in result["errors"])) + + def test_live_validation_reports_malformed_source_connectors(self): + document = self.document() + for connectors in (False, 1, {}, [{"connectorId": 1, "dataTypes": []}]): + result = live_dependency_check( + document, "DeviceEvents", [{"tables": ["DeviceEvents"]}], + source_connectors=connectors, + ) + self.assertEqual("failed", result["status"]) + + def test_import_rejects_failed_partial_stale_and_cross_workspace_metadata(self): + convert_solution(self.root) + query = yaml.safe_load(self.source.read_text())["query"] + entry = { + "detection": "Rule.yaml", "status": "passed", "workspaceId": WORKSPACE, + "querySha256": hashlib.sha256(query.encode()).hexdigest(), + "dataSources": [{"tables": ["DeviceEvents"]}], + } + results = self.root / "results.json" + results.write_text(json.dumps({"results": [entry]})) + proposed = plan_solution_dependencies(self.root, results_path=results, workspace_id=WORKSPACE) + self.assertEqual("passed", proposed["results"][0]["validation"]["status"]) + for updates in ( + {"status": "failed"}, {"partial": True}, {"error": "permission denied"}, + {"querySha256": "stale"}, {"workspaceId": "another"}, + ): + results.write_text(json.dumps({"results": [{**entry, **updates}]})) + with self.assertRaises(ValueError): + plan_solution_dependencies(self.root, results_path=results, workspace_id=WORKSPACE) + + def test_graph_projection_has_only_four_resolved_keys_and_preserves_yaml(self): + document = self.document() + document["contentProvenance"]["requiredDataConnectors"] = [ + {"connectorId": "Test", "dataTypes": ["DeviceEvents"]}, + ] + before = copy.deepcopy(document) + payload = graph_detection_payload(document, package_context={ + "packageId": "publisher.offer", "packageVersion": "3.1.2", + }) + self.assertEqual({ + "packageId": "publisher.offer", "packageVersion": "3.1.2", + "contentId": payload["id"], "contentVersion": document["version"], + }, payload["contentProvenance"]) + self.assertEqual(before, document) + self.assertNotIn("requiredWorkloads", payload["contentProvenance"]) + self.assertEqual("disabled", payload["status"]) + + def test_deployment_context_uses_real_solution_metadata(self): + (self.root / "Data").mkdir() + (self.root / "Data/Solution_Test.json").write_text(json.dumps({ + "Version": "3.1.7", "Metadata": "SolutionMetadata.json", + })) + (self.root / "SolutionMetadata.json").write_text(json.dumps({ + "publisherId": "publisher", "offerId": "offer", + })) + self.assertEqual({"packageId": "publisher.offer", "packageVersion": "3.1.7"}, + solution_package_context(self.root)) + + def test_sdk_requests_and_captures_input_metadata_not_output_tables(self): + metadata = [{"tables": ["DeviceEvents"]}] + + class Client: + def query_workspace(inner_self, workspace, query, **kwargs): + request = SimpleNamespace(http_request=SimpleNamespace(headers={"Prefer": "wait=90"})) + kwargs["raw_request_hook"](request) + self.assertEqual("wait=90,include-dataSources=true", request.http_request.headers["Prefer"]) + kwargs["raw_response_hook"](SimpleNamespace( + http_response=SimpleNamespace( + status_code=200, text=lambda: json.dumps({"dataSources": metadata}), + ), + )) + return success(logs_table("PrimaryResult", ["Count"], [["0"]])) + + result = execute_workspace_query(Client(), WORKSPACE, "DeviceEvents | count") + self.assertEqual(metadata, result["dataSources"]) + self.assertEqual("PrimaryResult", result["tables"][0]["name"]) + + def test_real_sdk_pipeline_preserves_wait_and_requests_input_metadata(self): + body = { + "dataSources": [{"tables": ["DeviceEvents"]}], + "tables": [{"name": "PrimaryResult", "columns": [{"name": "Count", "type": "long"}], "rows": []}], + } + sent = [] + + class Response(HttpResponse): + def __init__(inner_self, request): + super().__init__(request, None) + inner_self.status_code = 200 + inner_self.headers = {"Content-Type": "application/json"} + inner_self.content_type = "application/json" + + def body(inner_self): + return json.dumps(body).encode() + + @property + def content(inner_self): + return inner_self.body() + + def json(inner_self): + return json.loads(inner_self.body()) + + class Transport(HttpTransport): + def open(inner_self): + pass + + def close(inner_self): + pass + + def __exit__(inner_self, *args): + inner_self.close() + + def send(inner_self, request, **kwargs): + sent.append(request) + return Response(request) + + credential = SimpleNamespace(get_token=lambda *args, **kwargs: AccessToken("mock", int(time.time()) + 3600)) + with LogsQueryClient(credential, transport=Transport()) as client: + result = execute_workspace_query(client, WORKSPACE, "DeviceEvents | count") + self.assertIn("include-dataSources=true", sent[0].headers["Prefer"]) + self.assertIn("wait=90", sent[0].headers["Prefer"]) + self.assertEqual(body["dataSources"], result["dataSources"]) + + def test_reconversion_preserves_reviewed_dependencies_only_for_unchanged_queries(self): + convert_solution(self.root) + path = self.root / "XDR Detections/Rule.yaml" + document = yaml.safe_load(path.read_text()) + reviewed = [{"workload": "mde", "dataTypes": ["DeviceEvents", "DeviceInfo"]}] + document["contentProvenance"]["requiredWorkloads"] = reviewed + path.write_text(yaml.safe_dump(document, sort_keys=False)) + convert_solution(self.root, overwrite=True) + self.assertEqual(reviewed, yaml.safe_load(path.read_text())["contentProvenance"]["requiredWorkloads"]) + source = yaml.safe_load(self.source.read_text()) + source["query"] += "\n| take 1" + self.source.write_text(yaml.safe_dump(source)) + convert_solution(self.root, overwrite=True) + self.assertNotEqual(reviewed, yaml.safe_load(path.read_text())["contentProvenance"]["requiredWorkloads"]) + + def test_helper_cli_reports_failed_dependency_comparison_with_nonzero_exit(self): + convert_solution(self.root) + path = self.root / "XDR Detections/Rule.yaml" + document = yaml.safe_load(path.read_text()) + document["contentProvenance"]["requiredWorkloads"] = [] + path.write_text(yaml.safe_dump(document)) + results = self.root / "metadata.json" + results.write_text(json.dumps({"results": [{ + "detection": "Rule.yaml", "status": "passed", "workspaceId": WORKSPACE, + "querySha256": document["contentProvenance"]["source"]["querySha256"], + "dataSources": [{"tables": ["DeviceEvents"]}], + }]})) + output = io.StringIO() + with contextlib.redirect_stdout(output): + code = cli.main([ + "dependency-plan", "--solution", str(self.root), + "--results", str(results), "--workspace-id", WORKSPACE, + ]) + self.assertEqual(1, code) + self.assertEqual("failed", json.loads(output.getvalue())["results"][0]["validation"]["status"]) + + def test_dependency_edits_invalidate_source_validation_without_changing_query(self): + convert_solution(self.root) + path = self.root / "XDR Detections/Rule.yaml" + document = yaml.safe_load(path.read_text()) + with mock.patch("sentinel_xdr_migration.runtime._runtime_consent_for_import", return_value=None): + report = record_runtime_validation(self.root, provider="triage-mcp", results_data=[{ + "detection": "Rule.yaml", "status": "passed", "statusCode": 200, + "querySha256": document["contentProvenance"]["source"]["querySha256"], + "dataSources": [{"tables": ["DeviceEvents"]}], + }]) + self.assertEqual(1, report["valid"]) + document["contentProvenance"]["requiredWorkloads"][0]["dataTypes"].append("DeviceInfo") + path.write_text(yaml.safe_dump(document)) + evidence = current_runtime_evidence(self.root) + self.assertFalse(evidence["passed"]) + self.assertTrue(any("dependency provenance" in issue for issue in evidence["issues"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/Tools/SentinelToXDRMigration/tests/test_deployment.py b/Tools/SentinelToXDRMigration/tests/test_deployment.py index 2114e305e8f..cb322a34124 100644 --- a/Tools/SentinelToXDRMigration/tests/test_deployment.py +++ b/Tools/SentinelToXDRMigration/tests/test_deployment.py @@ -47,6 +47,8 @@ def document(rule_id: str = RULE_ID) -> dict: "resourceType": "Microsoft.Security/detectionRules", "apiVersion": "2026-06-01-preview", "contentProvenance": { + "requiredWorkloads": [{"workload": "mde", "dataTypes": ["DeviceInfo"]}], + "requiredDataConnectors": [], "source": { "platform": "Microsoft Sentinel", "kind": "AnalyticsRule", @@ -90,6 +92,19 @@ def document(rule_id: str = RULE_ID) -> dict: class DeploymentTests(unittest.TestCase): + def write_package_context(self, root: Path) -> dict: + (root / "Data").mkdir() + (root / "Data/Solution_Test.json").write_text(json.dumps({ + "Version": "3.1.0", "Metadata": "SolutionMetadata.json", + })) + (root / "SolutionMetadata.json").write_text(json.dumps({ + "publisherId": "testpublisher", "offerId": "test-sentinel", + })) + return { + "packageId": "testpublisher.test-sentinel", "packageVersion": "3.1.0", + "contentId": f"xdr-{RULE_ID}", "contentVersion": "3.1.0", + } + def test_custom_detection_token_accepts_scope_supersets_but_not_invalid_claims( self, ) -> None: @@ -901,6 +916,12 @@ def test_deploy_creates_missing_rule( yaml.safe_dump(document()), encoding="utf-8" ) + provenance = self.write_package_context(root) + request.side_effect = [ + (404, {"error": {"code": "NotFound"}}), + (201, {"id": RULE_ID}), + (200, {"id": RULE_ID, "status": "disabled", "contentProvenance": provenance}), + ] result = deploy_solution(root) self.assertEqual(result["succeeded"], 1) @@ -947,6 +968,7 @@ def test_deploy_fails_if_graph_readback_does_not_match_projection( yaml.safe_dump(value), encoding="utf-8" ) + self.write_package_context(root) result = deploy_solution(root) self.assertEqual(result["succeeded"], 0) @@ -981,6 +1003,12 @@ def test_deploy_updates_existing_rule( yaml.safe_dump(document()), encoding="utf-8" ) + provenance = self.write_package_context(root) + request.side_effect = [ + (200, {"id": RULE_ID}), + (200, {"id": RULE_ID}), + (200, {"id": RULE_ID, "status": "disabled", "contentProvenance": provenance}), + ] result = deploy_solution(root) self.assertEqual(result["succeeded"], 1) @@ -990,6 +1018,26 @@ def test_deploy_updates_existing_rule( update = request.call_args_list[1].args[3] self.assertNotIn("id", update) + def test_deployment_requires_matching_provenance_readback(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + output = root / "XDR Detections" + output.mkdir() + (output / "rule.yaml").write_text(yaml.safe_dump(document())) + self.write_package_context(root) + with ( + mock.patch("sentinel_xdr_migration.deployment.require_locked_target", return_value=TARGET), + mock.patch("sentinel_xdr_migration.deployment._deployment_token", return_value="token"), + mock.patch("sentinel_xdr_migration.deployment._graph_request", side_effect=[ + (404, {}), (201, {}), (200, {"status": "disabled"}), + ]), + ): + result = deploy_solution(root) + self.assertEqual(0, result["succeeded"]) + self.assertFalse(result["results"][0]["contentProvenanceVerified"]) + self.assertTrue(result["results"][0]["tacticProjectionVerified"]) + self.assertEqual("DeploymentVerificationFailed", result["results"][0]["error"]["code"]) + if __name__ == "__main__": unittest.main() diff --git a/Tools/SentinelToXDRMigration/tests/test_deployment_planning.py b/Tools/SentinelToXDRMigration/tests/test_deployment_planning.py index dd01f0d72b4..6d8b1e15706 100644 --- a/Tools/SentinelToXDRMigration/tests/test_deployment_planning.py +++ b/Tools/SentinelToXDRMigration/tests/test_deployment_planning.py @@ -73,6 +73,8 @@ def _candidate(rule: dict, *, multi_tactic: bool = False, review: bool = False, "resourceType": "Microsoft.Security/detectionRules", "apiVersion": "2026-06-01-preview", "contentProvenance": { + "requiredWorkloads": [{"workload": "mde", "dataTypes": ["DeviceEvents"]}], + "requiredDataConnectors": [], "source": { "platform": "Microsoft Sentinel", "kind": "AnalyticsRule", diff --git a/Tools/SentinelToXDRMigration/tests/test_graph_cli_scope.py b/Tools/SentinelToXDRMigration/tests/test_graph_cli_scope.py index e31649fe18c..09fca8bf1f3 100644 --- a/Tools/SentinelToXDRMigration/tests/test_graph_cli_scope.py +++ b/Tools/SentinelToXDRMigration/tests/test_graph_cli_scope.py @@ -56,6 +56,8 @@ def detection_document() -> dict: "resourceType": "Microsoft.Security/detectionRules", "apiVersion": "2026-06-01-preview", "contentProvenance": { + "requiredWorkloads": [{"workload": "mde", "dataTypes": ["DeviceInfo"]}], + "requiredDataConnectors": [], "source": { "platform": "Microsoft Sentinel", "kind": "AnalyticsRule", diff --git a/Tools/SentinelToXDRMigration/tests/test_logs_query.py b/Tools/SentinelToXDRMigration/tests/test_logs_query.py index e81cd18641d..6879de128ee 100644 --- a/Tools/SentinelToXDRMigration/tests/test_logs_query.py +++ b/Tools/SentinelToXDRMigration/tests/test_logs_query.py @@ -475,6 +475,11 @@ def setUp(self): yaml.safe_dump({ "kind": "CustomDetection", "contentProvenance": { + "requiredWorkloads": [{ + "workload": "mde" if relative == "Rule.yaml" else "sentinel", + "dataTypes": ["DeviceEvents" if relative == "Rule.yaml" else "SecurityEvent"], + }], + "requiredDataConnectors": [], "source": { "path": f"Analytic Rules/{relative}", "querySha256": hashlib.sha256(query.encode()).hexdigest(), diff --git a/Tools/SentinelToXDRMigration/tests/test_nested_content.py b/Tools/SentinelToXDRMigration/tests/test_nested_content.py index 2782d221ef3..e556e3fa68f 100644 --- a/Tools/SentinelToXDRMigration/tests/test_nested_content.py +++ b/Tools/SentinelToXDRMigration/tests/test_nested_content.py @@ -248,6 +248,13 @@ def test_parity_uses_relative_paths_and_rejects_ambiguous_stems(self) -> None: def test_mocked_deployment_covers_all_nested_ars_and_cds(self) -> None: convert_solution(self.solution) + (self.solution / "Data").mkdir(exist_ok=True) + (self.solution / "Data/Solution_Nested.json").write_text(json.dumps({ + "Version": "3.1.0", "Metadata": "SolutionMetadata.json", + })) + (self.solution / "SolutionMetadata.json").write_text(json.dumps({ + "publisherId": "testpublisher", "offerId": "nested", + })) with ( mock.patch("sentinel_xdr_migration.analytic_deployment.require_locked_target", return_value={"workspaceResourceId": "test"}), mock.patch("sentinel_xdr_migration.analytic_deployment._arm_token", return_value="test"), @@ -256,10 +263,17 @@ def test_mocked_deployment_covers_all_nested_ars_and_cds(self) -> None: result = deploy_analytic_rules(self.solution) self.assertEqual(len(self.names), result["succeeded"]) self.assertEqual(len(self.names), arm.call_count) + stored = {} + + def graph_request(method, url, token, payload=None): + if method == "PATCH": + stored[url] = payload + return 200, {"status": "disabled", **stored.get(url, {})} + with ( mock.patch("sentinel_xdr_migration.deployment.require_locked_target", return_value={"tenantId": "test"}), mock.patch("sentinel_xdr_migration.deployment._deployment_token", return_value="test"), - mock.patch("sentinel_xdr_migration.deployment._graph_request", return_value=(200, {"status": "disabled"})), + mock.patch("sentinel_xdr_migration.deployment._graph_request", side_effect=graph_request), ): result = deploy_solution(self.solution) self.assertEqual(len(self.names), result["succeeded"]) diff --git a/Tools/SentinelToXDRMigration/tests/test_packaging.py b/Tools/SentinelToXDRMigration/tests/test_packaging.py index 99588416a04..0a49f0b0caa 100644 --- a/Tools/SentinelToXDRMigration/tests/test_packaging.py +++ b/Tools/SentinelToXDRMigration/tests/test_packaging.py @@ -52,7 +52,7 @@ def setUp(self) -> None: (data / "Solution_Sample.json").write_text( json.dumps({ "Name": "Sample", - "Version": "1.2.3", + "Version": "3.2.3", "XDR Detections": ["XDR Detections/Detection.yaml"], }), encoding="utf-8", @@ -77,7 +77,7 @@ def setUp(self) -> None: }]}], }, })) - with zipfile.ZipFile(package / "1.2.3.zip", "w") as archive: + with zipfile.ZipFile(package / "3.2.3.zip", "w") as archive: archive.writestr("mainTemplate.json", "{}") archive.writestr("createUiDefinition.json", "{}") @@ -121,12 +121,14 @@ def test_v3_1_refuses_to_omit_an_applied_selection_plan(self) -> None: run.assert_not_called() def test_v31_target_version_boundaries_and_conflicting_bump(self) -> None: - for version in ("3.1.0", "3.1.2", "3.999.999"): + for version in ("3.2.0", "3.2.1", "3.999.999"): with self.subTest(version=version): self.assertEqual(version, validate_v31_target_version(version)) for version in ( "3.0.999", + "3.1.999", "4.0.0", + "4.1.0", "3.1", "3.1.0.1", "3.1.0-preview", @@ -142,9 +144,24 @@ def test_v31_target_version_boundaries_and_conflicting_bump(self) -> None: package_solution_v3_1( self.solution, version_bump="patch", - target_version="3.1.2", + target_version="3.2.1", ) + def test_below_floor_effective_release_rejects_before_reports_or_subprocess(self) -> None: + data = self.solution / "Data/Solution_Sample.json" + document = json.loads(data.read_text()) + document["Version"] = "3.1.999" + data.write_text(json.dumps(document)) + before = {str(path): path.read_bytes() for path in self.solution.rglob("*") if path.is_file()} + with mock.patch("sentinel_xdr_migration.packaging.subprocess.run") as run: + for bump in ("none", "patch"): + with self.subTest(bump=bump), self.assertRaisesRegex(ValueError, ">= 3.2.0"): + package_solution_v3_1(self.solution, version_bump=bump) + run.assert_not_called() + self.assertEqual(before, { + str(path): path.read_bytes() for path in self.solution.rglob("*") if path.is_file() + }) + def test_v31_target_version_is_forwarded_to_official_packager(self) -> None: with mock.patch( "sentinel_xdr_migration.packaging.subprocess.run", @@ -156,11 +173,11 @@ def test_v31_target_version_is_forwarded_to_official_packager(self) -> None: package_solution_v3_1( self.solution, version_bump="none", - target_version="3.1.2", + target_version="3.2.1", ) command = run.call_args.args[0] self.assertIn("-TargetVersion", command) - self.assertEqual("3.1.2", command[command.index("-TargetVersion") + 1]) + self.assertEqual("3.2.1", command[command.index("-TargetVersion") + 1]) def test_template_and_zip_match_partial_selection_exclusions(self) -> None: data_path = self.solution / "Data" / "Solution_Sample.json" @@ -733,14 +750,14 @@ def test_cli_runs_real_packager_for_same_and_cross_worktree_roots(self) -> None: (data / "Solution_Sample.json").write_text(json.dumps({ "Name": "Sample", "Author": "Packaging Smoke Test - test@example.invalid", - "Version": "3.1.0", + "Version": "3.2.0", "Metadata": "SolutionMetadata.json", "Description": "Synthetic V3.1 packaging smoke test.", "XDR Detections": [], }), encoding="utf-8") (solution / "SolutionMetadata.json").write_text(json.dumps({ "name": "Sample", - "version": "3.1.0", + "version": "3.2.0", "trackingId": "pid-cross-worktree-smoke-test", "support": { "name": "Synthetic Test", @@ -910,7 +927,7 @@ def test_rejects_unsafe_general_content_paths_before_powershell(self) -> None: run.assert_not_called() which.assert_not_called() - with zipfile.ZipFile(self.solution / "Package" / "1.2.3.zip") as archive: + with zipfile.ZipFile(self.solution / "Package" / "3.2.3.zip") as archive: self.assertNotIn( b"PATH_TRAVERSAL_FIXTURE_ONLY", b"".join(archive.read(name) for name in archive.namelist()), @@ -942,7 +959,7 @@ def test_rejects_path_like_solution_name_and_metadata_before_powershell(self) -> run.assert_not_called() which.assert_not_called() - with zipfile.ZipFile(self.solution / "Package" / "1.2.3.zip") as archive: + with zipfile.ZipFile(self.solution / "Package" / "3.2.3.zip") as archive: self.assertNotIn( b"METADATA_FIXTURE_ONLY", b"".join(archive.read(name) for name in archive.namelist()), @@ -1070,7 +1087,7 @@ def test_validates_json_encoded_and_malformed_content_arrays(self) -> None: run.assert_not_called() which.assert_not_called() - with zipfile.ZipFile(self.solution / "Package" / "1.2.3.zip") as archive: + with zipfile.ZipFile(self.solution / "Package" / "3.2.3.zip") as archive: self.assertNotIn( b"ENCODED_ESCAPE_FIXTURE_ONLY", b"".join(archive.read(name) for name in archive.namelist()), @@ -1258,7 +1275,7 @@ def test_powershell_contained_reader_rejects_escape_and_accepts_approved_forms(s @mock.patch("sentinel_xdr_migration.packaging.subprocess.run") def test_rejects_evidence_in_marketplace_zip(self, run, which) -> None: run.return_value = CompletedProcess([], 0, "Starting Package Creation using V3.1 tool", "") - with zipfile.ZipFile(self.solution / "Package" / "1.2.3.zip", "a") as archive: + with zipfile.ZipFile(self.solution / "Package" / "3.2.3.zip", "a") as archive: archive.writestr("eViDeNcE/run/evidence.json", "{}") with self.assertRaisesRegex(RuntimeError, "Evidence"): package_solution_v3_1(self.solution, version_bump="none") @@ -1267,7 +1284,7 @@ def test_rejects_evidence_in_marketplace_zip(self, run, which) -> None: @mock.patch("sentinel_xdr_migration.packaging.subprocess.run") def test_rejects_runtime_logs_in_marketplace_zip(self, run, which) -> None: run.return_value = CompletedProcess([], 0, "Starting Package Creation using V3.1 tool", "") - with zipfile.ZipFile(self.solution / "Package" / "1.2.3.zip", "a") as archive: + with zipfile.ZipFile(self.solution / "Package" / "3.2.3.zip", "a") as archive: archive.writestr("Logs/sentinel-xdr-migration/workflow-state.json", "{}") with self.assertRaisesRegex(RuntimeError, "must not contain Logs"): package_solution_v3_1(self.solution, version_bump="none") @@ -1276,7 +1293,7 @@ def test_rejects_runtime_logs_in_marketplace_zip(self, run, which) -> None: @mock.patch("sentinel_xdr_migration.packaging.subprocess.run") def test_rejects_local_env_in_marketplace_zip(self, run, which) -> None: run.return_value = CompletedProcess([], 0, "Starting Package Creation using V3.1 tool", "") - with zipfile.ZipFile(self.solution / "Package" / "1.2.3.zip", "a") as archive: + with zipfile.ZipFile(self.solution / "Package" / "3.2.3.zip", "a") as archive: archive.writestr("configuration/.env", "PRIVATE_ENV_SENTINEL") with self.assertRaisesRegex(RuntimeError, "must not contain local .env"): package_solution_v3_1(self.solution, version_bump="none") diff --git a/Tools/SentinelToXDRMigration/tests/test_packaging_selection.py b/Tools/SentinelToXDRMigration/tests/test_packaging_selection.py index c1ed847d67f..1ecb1ce5954 100644 --- a/Tools/SentinelToXDRMigration/tests/test_packaging_selection.py +++ b/Tools/SentinelToXDRMigration/tests/test_packaging_selection.py @@ -8,10 +8,12 @@ import sys import tempfile import unittest +import yaml from pathlib import Path from unittest import mock from sentinel_xdr_migration.artifacts import report_directory, using_run, write_json_artifact +from sentinel_xdr_migration.dependencies import dependency_identity from sentinel_xdr_migration.packaging_selection import ( PLAN_NAME, _build_package_selection_plan, @@ -143,6 +145,10 @@ def entries(provider: str) -> list[dict]: } if provider == "graph": item["schema"] = [{"name": "Column", "type": "string"}] + else: + item["dependencyProvenanceSha256"] = dependency_identity( + yaml.safe_load(detection_paths[name].read_text()) + ) result.append(item) return result diff --git a/Tools/SentinelToXDRMigration/tests/test_parser_bindings.py b/Tools/SentinelToXDRMigration/tests/test_parser_bindings.py index 277d5301ab2..5274e0c2fa9 100644 --- a/Tools/SentinelToXDRMigration/tests/test_parser_bindings.py +++ b/Tools/SentinelToXDRMigration/tests/test_parser_bindings.py @@ -121,7 +121,7 @@ def test_actual_shared_packager_normalizes_both_faces_without_editing_yaml(self) expected = [] template = { "parameters": {}, - "variables": {}, + "variables": {"solutionId": "testpublisher.test-sentinel", "_solutionVersion": "3.1.0"}, "resources": [{ "type": "Microsoft.OperationalInsights/workspaces/providers/contentPackages", "properties": { @@ -149,12 +149,23 @@ def test_actual_shared_packager_normalizes_both_faces_without_editing_yaml(self) document["contentProvenance"]["conversion"].update( status="converted", reviewRequired=False, errors=[], reviewReasons=[] ) + document["contentProvenance"]["requiredWorkloads"] = [ + {"workload": "sentinel", "dataTypes": ["CommonSecurityLog"]}, + ] + document["contentProvenance"]["requiredDataConnectors"] = [ + {"connectorId": "CEF", "dataTypes": ["CommonSecurityLog"]}, + ] + document["contentProvenance"]["dependencyReviewRequired"] = False path = output / f"Detection{index}.yaml" path.write_text(yaml.safe_dump(document), encoding="utf-8") paths.append(path) properties = copy.deepcopy(document["properties"]) packaged_id = f"xdr-{source_id}" properties["id"] = f"[if(true(), '{packaged_id}', '{packaged_id}')]" + properties["contentProvenance"] = { + "packageId": "testpublisher.test-sentinel", "packageVersion": "3.1.0", + "contentId": packaged_id, "contentVersion": document["version"], + } for mappings in properties["detectionAction"]["alertTemplate"]["entityMappings"].values(): for mapping in mappings: if mapping.get("id"): diff --git a/Tools/SentinelToXDRMigration/tests/test_runtime_consent.py b/Tools/SentinelToXDRMigration/tests/test_runtime_consent.py index 7438f814cb0..0cbc2fb9763 100644 --- a/Tools/SentinelToXDRMigration/tests/test_runtime_consent.py +++ b/Tools/SentinelToXDRMigration/tests/test_runtime_consent.py @@ -996,6 +996,8 @@ def _prepare_sdk_detection(self): "kind": "CustomDetection", "id": "xdr-11111111-2222-3333-4444-555555555555", "contentProvenance": { + "requiredWorkloads": [{"workload": "sentinel", "dataTypes": ["SecurityEvent"]}], + "requiredDataConnectors": [], "source": { "path": "Analytic Rules/Rule.yaml", "querySha256": hashlib.sha256(query.encode()).hexdigest(), diff --git a/Tools/SentinelToXDRMigration/tests/test_runtime_fallback.py b/Tools/SentinelToXDRMigration/tests/test_runtime_fallback.py index 6e47a6f94f7..4b9cc156735 100644 --- a/Tools/SentinelToXDRMigration/tests/test_runtime_fallback.py +++ b/Tools/SentinelToXDRMigration/tests/test_runtime_fallback.py @@ -90,7 +90,11 @@ def setUp(self): self.query_hash = hashlib.sha256(source_query.encode()).hexdigest() (output / "Rule.yaml").write_text(yaml.safe_dump({ "kind": "CustomDetection", - "contentProvenance": {"source": {"path": "Analytic Rules/Rule.yaml"}}, + "contentProvenance": { + "requiredWorkloads": [{"workload": "mde", "dataTypes": ["DeviceEvents"]}], + "requiredDataConnectors": [], + "source": {"path": "Analytic Rules/Rule.yaml"}, + }, "properties": { "queryCondition": {"queryText": "DeviceEvents | where Timestamp > ago(4h)"}, "detectionAction": {"alertTemplate": {}}, diff --git a/Tools/SentinelToXDRMigration/tests/test_solution_report.py b/Tools/SentinelToXDRMigration/tests/test_solution_report.py index 6a5a4b357f1..ca835e59d0c 100644 --- a/Tools/SentinelToXDRMigration/tests/test_solution_report.py +++ b/Tools/SentinelToXDRMigration/tests/test_solution_report.py @@ -12,6 +12,7 @@ import yaml from sentinel_xdr_migration.converter import convert_solution +from sentinel_xdr_migration.dependencies import dependency_identity from sentinel_xdr_migration.artifacts import existing_artifact_path, report_directory from sentinel_xdr_migration import deployment_planning from sentinel_xdr_migration.report_dashboard import _deployment_planning_html @@ -674,6 +675,9 @@ def test_source_waiver_is_visible_without_hiding_failure_or_blocked_validation(s "detection": path.name, "status": "passed", "rowCount": 1, + "dependencyProvenanceSha256": dependency_identity(yaml.safe_load( + (root / "XDR Detections" / path.name).read_text(encoding="utf-8") + )), "querySha256": hashlib.sha256( rule["query"].encode("utf-8") ).hexdigest(), diff --git a/Tools/SentinelToXDRMigration/tests/test_source_runtime_waiver.py b/Tools/SentinelToXDRMigration/tests/test_source_runtime_waiver.py index abb1a133767..cf915f25169 100644 --- a/Tools/SentinelToXDRMigration/tests/test_source_runtime_waiver.py +++ b/Tools/SentinelToXDRMigration/tests/test_source_runtime_waiver.py @@ -16,6 +16,7 @@ using_run, ) from sentinel_xdr_migration.report_dashboard import _flow_stage +from sentinel_xdr_migration.dependencies import dependency_identity from sentinel_xdr_migration.runtime import ( current_runtime_evidence, record_runtime_validation, @@ -154,6 +155,7 @@ def _create_solution( "failureKind": None, "querySha256": _sha256(query), "rowCount": 0, + "dependencyProvenanceSha256": dependency_identity(detection_doc), } result["detection"] = name_on_disk if source_overrides and index in source_overrides: diff --git a/Tools/SentinelToXDRMigration/tests/test_supplemental_custom_details.py b/Tools/SentinelToXDRMigration/tests/test_supplemental_custom_details.py index ab671e2fe7b..cfbd1faf6a0 100644 --- a/Tools/SentinelToXDRMigration/tests/test_supplemental_custom_details.py +++ b/Tools/SentinelToXDRMigration/tests/test_supplemental_custom_details.py @@ -229,6 +229,13 @@ def test_sftp_aggregate_bindings_are_warnings_and_survive_validation_and_reports self.rule["entityMappings"] = [entity("IP", Address="SourceIP")] document = self.build() + document["contentProvenance"]["requiredWorkloads"] = [ + {"workload": "sentinel", "dataTypes": ["SftpFileEvents", "SftpProcessEvents"]}, + ] + document["contentProvenance"]["requiredDataConnectors"] = [ + {"connectorId": "TestSftpConnector", "dataTypes": ["SftpFileEvents", "SftpProcessEvents"]}, + ] + document["contentProvenance"]["dependencyReviewRequired"] = False conversion = self.conversion(document) alert = self.alert(document) warning = "static analysis cannot establish its output" @@ -257,6 +264,7 @@ def test_sftp_aggregate_bindings_are_warnings_and_survive_validation_and_reports conversion_result = convert_solution(self.solution) self.assertEqual(conversion_result["converted"], 1) + (self.solution / "XDR Detections" / self.source.name).write_text(yaml.safe_dump(document)) validation = validate_solution(self.solution) self.assertEqual(validation["valid"], 1) self.assertTrue(any(warning in item for item in validation["results"][0]["warnings"])) diff --git a/Tools/SentinelToXDRMigration/tests/test_version_policy.py b/Tools/SentinelToXDRMigration/tests/test_version_policy.py index b8054bd2832..8bffd77559a 100644 --- a/Tools/SentinelToXDRMigration/tests/test_version_policy.py +++ b/Tools/SentinelToXDRMigration/tests/test_version_policy.py @@ -2,9 +2,78 @@ import json import shutil +import tempfile import unittest +from pathlib import Path +from unittest import mock import test_customer_usage_attribution as attribution +from sentinel_xdr_migration.packaging import validate_v31_effective_version, validate_v31_target_version +from sentinel_xdr_migration.workflow import initialize_workflow, plan_new_package_release, package_workflow_v3_1 + + +class PythonSolutionVersionPolicyTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory(dir=Path.cwd()) + self.addCleanup(self.directory.cleanup) + self.solution = Path(self.directory.name) / "Sample" + (self.solution / "Data").mkdir(parents=True) + self.data = self.solution / "Data/Solution_Sample.json" + self.data.write_text(json.dumps({"Name": "Sample", "Version": "3.1.999"})) + + def snapshot(self): + return {str(path): path.read_bytes() for path in self.solution.rglob("*") if path.is_file()} + + def test_exact_solution_target_boundaries(self): + for version in ("3.2.0", "3.2.1", "3.999.999"): + self.assertEqual(version, validate_v31_target_version(version)) + for version in ("3.1.999", "4.0.0", "4.1.0", "3.2", "3.2.0.0", + "03.2.0", "3.02.0", "3.2.00", "3.2.0-preview"): + with self.subTest(version=version), self.assertRaises(ValueError): + validate_v31_target_version(version) + + def test_effective_solution_selection_does_not_promote(self): + for bump in ("none", "patch", "major"): + with self.subTest(bump=bump), self.assertRaises(ValueError): + validate_v31_effective_version("3.1.999", bump) + self.assertEqual("3.2.0", validate_v31_effective_version("3.1.999", "minor")) + self.assertEqual("3.2.1", validate_v31_effective_version("1.0.0", "none", "3.2.1")) + + def test_authoring_rejects_selected_old_release_without_mutation(self): + before = self.snapshot() + for bump in ("none", "patch"): + with self.subTest(bump=bump), self.assertRaisesRegex(ValueError, ">= 3.2.0"): + initialize_workflow(self.solution, workflow_profile="authoring", version_bump=bump) + self.assertEqual(before, self.snapshot()) + with self.assertRaises(ValueError): + initialize_workflow(self.solution, workflow_profile="authoring", target_version="3.1.999") + self.assertEqual(before, self.snapshot()) + + def test_old_source_discovery_and_explicit_upgrade_are_legal(self): + state = initialize_workflow(self.solution, workflow_profile="authoring") + self.assertEqual("pending", state["stages"]["discovery"]["status"]) + for options in ({"target_version": "3.2.1"}, {"version_bump": "minor"}): + with self.subTest(options=options): + state = initialize_workflow( + self.solution, workflow_profile="authoring", new_run=True, **options, + ) + self.assertEqual("pending", state["stages"]["discovery"]["status"]) + self.assertEqual("3.1.999", json.loads(self.data.read_text())["Version"]) + + def test_release_planning_and_wrapper_preflight_reject_below_floor_before_state_write(self): + initialize_workflow(self.solution, workflow_profile="authoring") + before = self.snapshot() + for bump in ("patch", "major"): + with self.subTest(bump=bump), self.assertRaisesRegex(ValueError, ">= 3.2.0"): + plan_new_package_release( + self.solution, version_bump=bump, expected_package_input_sha256="a" * 64, + ) + self.assertEqual(before, self.snapshot()) + with mock.patch("sentinel_xdr_migration.workflow.package_solution_v3_1") as package: + with self.assertRaisesRegex(ValueError, ">= 3.2.0"): + package_workflow_v3_1(self.solution, requested_version_bump="patch") + package.assert_not_called() + self.assertEqual(before, self.snapshot()) @unittest.skipUnless(shutil.which("pwsh"), "PowerShell 7 is required") @@ -15,7 +84,7 @@ class VersionPolicyTests(unittest.TestCase): package = attribution.AttributionTests.package write_metadata = attribution.AttributionTests.write_metadata - def fixture(self, version="3.1.0", *, xdr=True): + def fixture(self, version="3.2.0", *, xdr=True): entry = self.prepare_integration() self.write_metadata(attribution.TRACKING_ID) self.update_data(Version=version) @@ -54,8 +123,9 @@ def assert_rejected_unchanged(self, entry, *, pipeline=False, **kwargs): def test_solution_boundaries_without_xdr_local_and_pipeline(self): entry = self.fixture(xdr=False) - for version, accepted in (("3.0.99", False), ("3.1.0", True), - ("3.9.99", True), ("4.0.0", False)): + for version, accepted in (("3.0.99", False), ("3.1.999", False), + ("3.2.0", True), ("3.2.1", True), + ("3.999.999", True), ("4.0.0", False), ("4.1.0", False)): for pipeline in (False, True): with self.subTest(version=version, pipeline=pipeline): self.update_data(Version=version) @@ -117,9 +187,11 @@ def test_analytic_versions_keep_legacy_floor(self): def test_bumps_persist_only_accepted_effective_versions(self): entry = self.fixture("3.1.0") - for bump, expected in (("none", "3.1.0"), ("patch", "3.1.1"), ("minor", "3.2.0")): + for bump, current, expected in (("none", "3.2.0", "3.2.0"), + ("patch", "3.2.0", "3.2.1"), + ("minor", "3.1.0", "3.2.0")): with self.subTest(bump=bump): - self.update_data(Version="3.1.0") + self.update_data(Version=current) result = self.package(entry, False, version_bump=bump) self.assertEqual(0, result.returncode, result.stdout + result.stderr) self.assertEqual(expected, json.loads(self.data.read_text())["Version"]) @@ -131,7 +203,10 @@ def test_bumps_persist_only_accepted_effective_versions(self): else: result = self.package(entry, False, version_bump=bump) self.assertEqual(0, result.returncode, result.stdout + result.stderr) - self.update_data(Version="3.1.0") + for bump in ("none", "patch"): + self.update_data(Version="3.1.999") + self.assert_rejected_unchanged(entry, version_bump=bump) + self.update_data(Version="3.2.0") self.update_detection("1.0.0") self.assert_rejected_unchanged(entry, version_bump="patch") @@ -146,11 +221,11 @@ def package_target(target): f"-TargetVersion {attribution.ps_literal(target)}" ) - result = package_target("3.1.2") + result = package_target("3.2.1") self.assertEqual(0, result.returncode, result.stdout + result.stderr) - self.assertEqual("3.1.2", json.loads(self.data.read_text())["Version"]) - self.assertEqual("3.1.2", json.loads(self.metadata.read_text())["version"]) - self.assertTrue((self.solution / "Package" / "3.1.2.zip").is_file()) + self.assertEqual("3.2.1", json.loads(self.data.read_text())["Version"]) + self.assertEqual("3.2.1", json.loads(self.metadata.read_text())["version"]) + self.assertTrue((self.solution / "Package" / "3.2.1.zip").is_file()) result = package_target("3.999.999") self.assertEqual(0, result.returncode, result.stdout + result.stderr) @@ -158,7 +233,7 @@ def package_target(target): self.assertEqual("3.999.999", json.loads(self.metadata.read_text())["version"]) self.assertTrue((self.solution / "Package" / "3.999.999.zip").is_file()) - for version in ("3.0.999", "4.0.0", "3.1", "3.1.0-preview", "03.1.0"): + for version in ("3.0.999", "3.1.999", "4.0.0", "3.2", "3.2.0-preview", "03.2.0", "3.02.0", "3.2.00"): with self.subTest(version=version): before = self.snapshot() rejected = package_target(version) @@ -168,7 +243,8 @@ def package_target(target): def test_pipeline_mismatch_and_missing_solution_default_fail(self): entry = self.fixture(xdr=False) - self.assert_rejected_unchanged(entry, pipeline=True, calculated_version="3.2.0") + self.assert_rejected_unchanged(entry, pipeline=True, calculated_version="3.2.1") + self.assert_rejected_unchanged(entry, pipeline=True, calculated_version="3.1.999") document = json.loads(self.data.read_text()) document.pop("Version") self.data.write_text(json.dumps(document)) @@ -215,13 +291,13 @@ def test_catalog_result_is_guarded_without_source_mutation(self): entry = self.fixture(xdr=False) catalog = self.repository / ".script/package-automation/catalogAPI.ps1" catalog.parent.mkdir(parents=True) - for version in ("3.0.99", "3.1.0", "4.0.0"): + for version in ("3.0.99", "3.1.999", "3.2.0", "4.0.0"): # Isolate only the external catalog response, not any packaging or guard logic. catalog.write_text( "function GetCatalogDetails { return @{} }\n" f"function GetPackageVersion {{ return '{version}' }}\n" ) - if version != "3.1.0": + if version != "3.2.0": self.assert_rejected_unchanged(entry, version_mode="catalog") else: before = self.data.read_bytes(), self.metadata.read_bytes() @@ -244,7 +320,7 @@ def test_legacy_context_is_unrestricted_and_checks_cannot_be_swallowed(self): def test_numeric_bounds_do_not_apply_floor_to_other_content_kinds(self): policy = attribution.ps_literal(attribution.PACKAGER / "common/versionPolicy.ps1") - kinds = ("Parser", "Workbook", "Playbook", "DataConnector", "LogicAppsCustomConnector", + kinds = ("AnalyticsRule", "HuntingQuery", "Parser", "Workbook", "Playbook", "DataConnector", "LogicAppsCustomConnector", "AzureFunction", "SummaryRule", "Watchlist", "CCF DataConnector/Connections") for kind in kinds: with self.subTest(kind=kind): diff --git a/Tools/SentinelToXDRMigration/tests/test_workbook_arm_validation.py b/Tools/SentinelToXDRMigration/tests/test_workbook_arm_validation.py new file mode 100644 index 00000000000..4acf43c86c2 --- /dev/null +++ b/Tools/SentinelToXDRMigration/tests/test_workbook_arm_validation.py @@ -0,0 +1,660 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import shutil +import subprocess +import uuid +import zipfile +from pathlib import Path +from unittest import mock + +import pytest + +from sentinel_xdr_migration import workbook_validation as validation +from sentinel_xdr_migration.arm_acceptance_support import _canonical_sha256 +from sentinel_xdr_migration.packaging import package_input_file_hashes, package_input_identity + +ROOT = Path(__file__).resolve().parents[3] +TARGET = { + "tenantId": "11111111-2222-3333-4444-555555555555", + "subscriptionId": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + "resourceGroup": "fixture-rg", + "workspaceResourceId": "/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/fixture-rg/providers/Microsoft.OperationalInsights/workspaces/fixture-ws", + "workspaceCustomerId": "99999999-8888-7777-6666-555555555555", +} +CONTEXT = {"runId": "fixture-run", "workflowProfile": "authoring", "profileSelectionConfirmed": True, + "workbookCapabilityVersion": 1, "armValidationMethod": "lab-deployment"} +FLAGS = {"DeployWorkbook": False, "DeployDefenderWorkbook": True} +ACCESS = {key: True for key in ("tenantMatches", "subscriptionMatches", "workspaceMatches", "nonProduction", + "armDeploymentAllowed", "extensionResourceWriteAllowed", "disabledContentVerified", "ownershipVerified")} +ACCESS.update(identityId="fixture-principal", **{key: TARGET[key] for key in ("tenantId", "subscriptionId", "workspaceResourceId")}) + + +def workbook_template(): + provenance = {"packageId": "publisher.fixture", "packageVersion": "3.2.1", + "contentId": "aaaaaaaa-1111-2222-3333-444444444444", "contentVersion": "3.2.1"} + notebook = {"version": "Notebook/1.0", "items": [], "contentProvenance": provenance} + inner = { + "languageVersion": "2.0", "imports": {"MicrosoftSecurity": {"provider": "MicrosoftSecurity", "version": "1.0.0"}}, + "resources": {"workbook": {"import": "MicrosoftSecurity", "type": "Microsoft.Security/workbooks@2026-06-01-preview", + "properties": {"id": "bbbbbbbb-1111-2222-3333-444444444444", "name": "Fixture", "content": notebook}}}, + } + return { + "parameters": {"workspace": {"type": "string"}, **{key: {"type": "bool", "defaultValue": False} for key in FLAGS}}, + "resources": [ + {"type": "Microsoft.Resources/deployments", "name": "fixture-install", "condition": "[parameters('DeployDefenderWorkbook')]", + "properties": {"mode": "Incremental", "expressionEvaluationOptions": {"scope": "inner"}, "template": inner}}, + {"type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-dwb-',uniqueString('fixture')))]", + "condition": "[parameters('DeployDefenderWorkbook')]", + "properties": {"contentKind": "DefenderWorkbook", "contentId": provenance["contentId"], "version": "3.2.1", + "packageId": provenance["packageId"], "packageVersion": "3.2.1", "mainTemplate": copy.deepcopy(inner), + "contentProvenance": {**provenance, "requiredWorkloads": [], "requiredDataConnectors": []}}}, + ], + } + + +@pytest.fixture +def case(): + directory = ROOT / ".script/local-validation/Logs" / ("workbook-tests-" + uuid.uuid4().hex) + solution = directory / "Solutions/Fixture" + package = solution / "Package" + logs = solution / "Logs" + package.mkdir(parents=True) + logs.mkdir() + (solution / "input.json").write_text("{}") + template = workbook_template() + (package / "mainTemplate.json").write_text(json.dumps(template)) + (package / "createUiDefinition.json").write_text("{}") + (package / "testParameters.json").write_text("{}") + with zipfile.ZipFile(package / "3.2.1.zip", "w") as archive: + archive.write(package / "mainTemplate.json", "mainTemplate.json") + archive.write(package / "createUiDefinition.json", "createUiDefinition.json") + paths = {"mainTemplate": "Package/mainTemplate.json", "createUiDefinition": "Package/createUiDefinition.json", + "testParameters": "Package/testParameters.json", "zip": "Package/3.2.1.zip"} + hashes = {name: hashlib.sha256((solution / path).read_bytes()).hexdigest() for name, path in paths.items()} + identity = package_input_identity(solution) + report = {"packager": "V3.1", "workflowRunId": CONTEXT["runId"], "packageAttemptId": "fixture-attempt", "version": "3.2.1", + "armValidationMethod": "lab-deployment", "packageInputs": {"after": {**identity, "files": package_input_file_hashes(solution)}}, + "packageArtifactHashes": hashes, "packageArtifactsSha256": _canonical_sha256(hashes), **paths} + report_path = logs / "packaging.v3_1.json" + report_path.write_text(json.dumps(report)) + gates = {} + for gate in ("conversion", "review", "structure", "package"): + path = logs / f"{gate}.json" + path.write_text(json.dumps({"status": "passed", "workflowRunId": CONTEXT["runId"], "packageInputSha256": identity["sha256"], + "packageArtifactsSha256": report["packageArtifactsSha256"]})) + gates[gate] = {"path": f"Logs/{gate}.json", "sha256": hashlib.sha256(path.read_bytes()).hexdigest()} + receipt = logs / "build-and-validate-report.md" + receipt.write_text("mocked official producer receipt") + receipt_binding = {"path": str(receipt), "sha256": hashlib.sha256(receipt.read_bytes()).hexdigest()} + # Official receipt parsing is independently tested below against its full + # producer layout. These lifecycle fixtures isolate artifacts and dispatch. + with mock.patch.object(validation, "_parse_build_validation_receipt", return_value=receipt_binding): + candidate = validation.create_arm_candidate( + CONTEXT, solution=solution, package_report="Logs/packaging.v3_1.json", + validation_receipt=receipt, prerequisite_evidence=gates, + ) + yield {"solution": solution, "candidate": candidate, "report": report, "gates": gates, "receipt": receipt} + shutil.rmtree(directory) + + +def plan(case, *, flags=None, target=None): + return validation.plan_lab_arm_validation(CONTEXT, case["candidate"], target=target or TARGET, selected_flags=FLAGS if flags is None else flags) + + +def consent_for(plan): + return validation.review_lab_arm_scope(plan, actor="fixture-reviewer", approval_reference="scope-approval", approved=True) + + +def execute(case, *, runner=None, consent_reader=None, access=None, refresh=None, **kwargs): + current_plan = plan(case) + consent = consent_for(current_plan) + return validation.execute_lab_arm_validation( + CONTEXT, case["candidate"], target=TARGET, selected_flags=FLAGS, + expected_plan_sha256=current_plan["planSha256"], actor="fixture-reviewer", approval_reference="immediate-approval", + confirmation=validation.LAB_ARM_CONFIRMATION, evidence_directory=case["solution"] / "Logs/sentinel-xdr-migration/fixture-run", + refresh_candidate=refresh or (lambda: case["candidate"]), read_consent=consent_reader or (lambda: consent), + identity_check=lambda _target: ACCESS if access is None else access, + command_runner=runner or (lambda _command: (0, {"id": "fixture-deployment", "properties": {"provisioningState": "Succeeded"}}, "")), + **kwargs, + ) + + +def test_methods_legacy_and_versioned_boundaries(): + assert validation.arm_validation_method({"armValidationMethod": "lab-deployment"}) == "arm-ttk" + assert validation.arm_validation_method(CONTEXT) == "lab-deployment" + assert validation.arm_validation_method({**CONTEXT, "armValidationMethod": "arm-ttk"}) == "arm-ttk" + for change in ({"workbookCapabilityVersion": True}, {"workbookCapabilityVersion": 2}, + {"armValidationMethod": None}, {"profileSelectionConfirmed": False}, + {"workflowProfile": "qualification", "armValidationMethod": "arm-ttk"}): + with pytest.raises(ValueError): + validation.arm_validation_method({**CONTEXT, **change}) + + +def test_qualification_retains_locked_target_and_runtime_gate(case): + qualification = {**CONTEXT, **TARGET, "workflowProfile": "qualification", "targetContextLocked": True} + assert validation.arm_validation_method(qualification) == "lab-deployment" + with pytest.raises(ValueError, match="locked target"): + validation.arm_validation_method({**qualification, "targetContextLocked": False}) + with pytest.raises(ValueError, match="another workflow run"): + validation.plan_lab_arm_validation(qualification, case["candidate"], target=TARGET, selected_flags=FLAGS) + with pytest.raises(ValueError, match="gate evidence"): + validation.create_arm_candidate(qualification, solution=case["solution"], package_report="Logs/packaging.v3_1.json", + validation_receipt=case["receipt"], prerequisite_evidence=case["gates"]) + path = case["solution"] / "Logs/qualification.json" + path.write_text(json.dumps({"status": "passed", "packageInputSha256": case["candidate"]["packageIdentity"]["packageInputSha256"], + "workflowRunId": CONTEXT["runId"], "qualificationRuntimeVerified": False, "target": TARGET})) + evidence = {**case["gates"], "qualification": {"path": "Logs/qualification.json", "sha256": hashlib.sha256(path.read_bytes()).hexdigest()}} + with pytest.raises(ValueError, match="existing runtime/target"): + validation.create_arm_candidate(qualification, solution=case["solution"], package_report="Logs/packaging.v3_1.json", + validation_receipt=case["receipt"], prerequisite_evidence=evidence) + payload = json.loads(path.read_text()) + payload["qualificationRuntimeVerified"] = True + path.write_text(json.dumps(payload)) + evidence["qualification"]["sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + candidate = validation.create_arm_candidate(qualification, solution=case["solution"], package_report="Logs/packaging.v3_1.json", + validation_receipt=case["receipt"], prerequisite_evidence=evidence) + assert validation.plan_lab_arm_validation(qualification, candidate, target=TARGET, selected_flags=FLAGS)["status"] == "ready" + retargeted = {**TARGET, "tenantId": "cccccccc-bbbb-cccc-dddd-eeeeeeeeeeee"} + with pytest.raises(ValueError, match="locked target"): + validation.plan_lab_arm_validation(qualification, candidate, target=retargeted, selected_flags=FLAGS) + + +def test_candidate_is_not_accepted_and_binds_every_artifact(case): + candidate = case["candidate"] + assert candidate["accepted"] is False + assert candidate["status"] == "candidate-generated" + assert set(candidate["packageIdentity"]["packageArtifactHashes"]) == {"mainTemplate", "createUiDefinition", "testParameters", "zip"} + assert validation.refresh_arm_candidate(CONTEXT, candidate) == candidate + + +def test_workbook_only_qualification_candidate_precedes_query_and_ui_proof(case): + context = { + **CONTEXT, **TARGET, "workflowProfile": "qualification", + "targetContextLocked": True, "workbookQualificationAfterArm": True, + } + candidate = validation.create_arm_candidate( + context, solution=case["solution"], package_report="Logs/packaging.v3_1.json", + validation_receipt=case["receipt"], prerequisite_evidence=case["gates"], + ) + assert candidate["accepted"] is False + assert validation.plan_lab_arm_validation( + context, candidate, target=TARGET, selected_flags=FLAGS, + )["status"] == "ready" + assert validation.refresh_arm_candidate(context, candidate) == candidate + case["report"]["xdrDetectionCount"] = 1 + (case["solution"] / "Logs/packaging.v3_1.json").write_text(json.dumps(case["report"])) + with pytest.raises(ValueError, match="cannot bypass detection runtime"): + validation.create_arm_candidate( + context, solution=case["solution"], package_report="Logs/packaging.v3_1.json", + validation_receipt=case["receipt"], prerequisite_evidence=case["gates"], + ) + + +def test_production_input_manifest_is_verified_without_rejecting_additive_fields(case): + recorded = case["report"]["packageInputs"]["after"] + assert set(recorded) == {"sha256", "fileCount", "files"} + assert recorded["files"] == package_input_file_hashes(case["solution"]) + case["report"]["packageInputs"]["after"]["captureStatus"] = "complete" + (case["solution"] / "Logs/packaging.v3_1.json").write_text(json.dumps(case["report"])) + candidate = validation.create_arm_candidate( + CONTEXT, solution=case["solution"], package_report="Logs/packaging.v3_1.json", + validation_receipt=case["receipt"], prerequisite_evidence=case["gates"], + ) + assert candidate["packageIdentity"]["packageInputSha256"] == recorded["sha256"] + + +@pytest.mark.parametrize("corruption", ["missing", "null", "list", "wrong-hash", "missing-file", "extra-file"]) +def test_production_input_manifest_must_be_exact_even_when_aggregate_is_current(case, corruption): + recorded = case["report"]["packageInputs"]["after"] + if corruption == "missing": + recorded.pop("files") + elif corruption == "null": + recorded["files"] = None + elif corruption == "list": + recorded["files"] = [] + elif corruption == "wrong-hash": + recorded["files"]["input.json"] = "b" * 64 + elif corruption == "missing-file": + recorded["files"].pop("input.json") + else: + recorded["files"]["unexpected.json"] = "c" * 64 + (case["solution"] / "Logs/packaging.v3_1.json").write_text(json.dumps(case["report"])) + with pytest.raises(ValueError, match="input file manifest"): + validation.create_arm_candidate( + CONTEXT, solution=case["solution"], package_report="Logs/packaging.v3_1.json", + validation_receipt=case["receipt"], prerequisite_evidence=case["gates"], + ) + + +@pytest.mark.parametrize("item", ["input.json", "Package/mainTemplate.json", "Package/3.2.1.zip", "Logs/review.json", "Logs/packaging.v3_1.json"]) +def test_stale_input_artifact_gate_or_report_blocks_dispatch(case, item): + path = case["solution"] / item + path.write_bytes(path.read_bytes() + b"\n") + runner = mock.Mock() + with pytest.raises(ValueError): + execute(case, runner=runner) + runner.assert_not_called() + + +@pytest.mark.parametrize("gate", ["conversion", "review", "structure", "package"]) +def test_failed_or_missing_gate_is_not_candidate(case, gate): + case["gates"].pop(gate) + with pytest.raises(ValueError, match="gate evidence"): + validation.create_arm_candidate(CONTEXT, solution=case["solution"], package_report="Logs/packaging.v3_1.json", + validation_receipt=case["receipt"], prerequisite_evidence=case["gates"]) + + +def test_selected_flags_are_exact_and_tenant_extension_scope_is_explicit(case): + current = plan(case) + assert current["parameters"]["DeployWorkbook"] is False + assert current["parameters"]["DeployDefenderWorkbook"] is True + assert current["scope"]["contentTypes"] == ["DefenderWorkbook"] + assert "Microsoft.Security/workbooks" in current["scope"]["resourceTypes"] + assert any(item["scope"] == f"/tenants/{TARGET['tenantId']}" for item in current["resources"]) + for flags in ({"DeployDefenderWorkbook": True}, {**FLAGS, "DeployUnsupported": True}, {**FLAGS, "DeployWorkbook": "false"}): + with pytest.raises(ValueError): + plan(case, flags=flags) + empty = plan(case, flags={name: False for name in FLAGS}) + assert empty["resources"] == [] and empty["expectedWorkbooks"] == [] + + +@pytest.mark.parametrize("change", ["scope", "subscription", "link", "extension", "extension-tenant", "copy", "unknown-condition", "grant"]) +def test_nested_or_unresolved_scope_fails_closed(case, change): + template = workbook_template() + deployment = template["resources"][0] + inner = deployment["properties"]["template"] + if change == "scope": + deployment["scope"] = "/subscriptions/other/resourceGroups/other" + elif change == "subscription": + deployment["subscriptionId"] = "other" + elif change == "link": + deployment["properties"]["templateLink"] = {"uri": "https://example.invalid/template"} + elif change == "extension": + inner["imports"]["MicrosoftSecurity"]["provider"] = "Unknown" + elif change == "extension-tenant": + inner["imports"]["MicrosoftSecurity"]["config"] = {"tenantId": "other"} + elif change == "copy": + deployment["copy"] = {"count": 2} + elif change == "unknown-condition": + deployment["condition"] = "[unreviewed('something')]" + else: + deployment["type"] = "Microsoft.Authorization/roleAssignments" + with pytest.raises(ValueError): + validation._resource_inventory(template, {"workspace": "fixture-ws", **FLAGS}, TARGET) + + +def test_scope_reuses_types_but_not_new_lab_types_or_revocation(case): + current = plan(case) + consent = consent_for(current) + changed = copy.deepcopy(current) + changed["candidateSha256"] = "b" * 64 + changed["resourceCount"] += 4 + changed.pop("planSha256") + changed["planSha256"] = _canonical_sha256(changed) + assert validation.check_lab_arm_scope(changed, consent)["status"] == "ready" + for field, value in (("contentTypes", "Workbook"), ("resourceTypes", "Microsoft.Compute/virtualMachines"), ("operations", "deletion")): + extra = copy.deepcopy(changed) + extra["scope"][field].append(value) + extra.pop("planSha256") + extra["planSha256"] = _canonical_sha256(extra) + assert validation.check_lab_arm_scope(extra, consent)["status"] == "blocked" + new_lab = copy.deepcopy(changed) + new_lab["scope"]["lab"]["tenantId"] = "other" + new_lab.pop("planSha256") + new_lab["planSha256"] = _canonical_sha256(new_lab) + assert validation.check_lab_arm_scope(new_lab, consent)["status"] == "blocked" + assert validation.check_lab_arm_scope(current, None)["status"] == "blocked" + revoked = validation.withdraw_lab_arm_scope(consent, actor="fixture-reviewer", withdrawal_reference="withdrawn") + assert validation.check_lab_arm_scope(current, revoked)["status"] == "blocked" + + +def test_execution_one_executor_and_no_automatic_content_flags(case): + commands = [] + def runner(command): + commands.append(command) + paths = list((case["solution"] / "Logs/sentinel-xdr-migration/fixture-run").glob("arm-validation-attempt-*.json")) + assert len(paths) == 1 + assert json.loads(paths[0].read_text())["status"] == "write-intent-persisted" + return 0, {"id": "fixture-deployment", "properties": {"provisioningState": "Succeeded"}}, "" + report = execute(case, runner=runner) + assert report["status"] == "arm-accepted-contract-pending" and report["accepted"] is False + assert "DeployWorkbook=false" in commands[0] and "DeployDefenderWorkbook=true" in commands[0] + assert len(commands) == 1 + with pytest.raises(ValueError, match="reconciliation"): + execute(case, runner=runner) + assert len(commands) == 1 + + +@pytest.mark.parametrize("state,code,expected", [("Failed", 1, "failed"), ("Running", 0, "nonterminal"), + (None, 124, "unknown"), ("Succeeded", 1, "failed")]) +def test_failed_unknown_and_nonterminal_are_durable(case, state, code, expected): + report = execute(case, runner=lambda _: (code, {"properties": {"provisioningState": state}}, "fixture error")) + assert report["status"] == expected and report["accepted"] is False + assert json.loads(Path(report["attemptPath"]).read_text())["status"] == expected + + +def test_exception_is_unknown_and_missing_permissions_block_without_write(case): + runner = mock.Mock(side_effect=TimeoutError()) + assert execute(case, access={**ACCESS, "extensionResourceWriteAllowed": False}, runner=runner)["writeAttempted"] is False + runner.assert_not_called() + report = execute(case, runner=runner) + assert report["status"] == "unknown" and report["retryAuthorized"] is False + + +def test_revocation_at_dispatch_boundary_is_durable_and_never_dispatches(case): + consent = consent_for(plan(case)) + count = 0 + def read(): + nonlocal count + count += 1 + return consent if count < 3 else validation.withdraw_lab_arm_scope(consent, actor="fixture-reviewer", withdrawal_reference="stop") + runner = mock.Mock() + report = execute(case, consent_reader=read, runner=runner) + assert report["status"] == "blocked-before-write" and report["writeAttempted"] is False + runner.assert_not_called() + + +def readback(plan): + wanted = plan["expectedWorkbooks"][0] + observed = {} + for phase in ("installation", "registration", "repeatInstallation", "update"): + template = workbook_template() + if phase == "registration": + resource_id = TARGET["workspaceResourceId"] + "/providers/Microsoft.SecurityInsights/contentTemplates/fixture-registration" + props = template["resources"][1]["properties"] + else: + resource_id = f"/tenants/{TARGET['tenantId']}/providers/Microsoft.Security/workbooks/{wanted['instanceId']}" + props = template["resources"][0]["properties"]["template"]["resources"]["workbook"]["properties"] + response_path = Path(plan["solutionPath"]) / "Logs" / f"{phase}-readback.json" + response_path.write_text(json.dumps({"id": resource_id, "properties": props})) + write_attempt_path = Path(plan["solutionPath"]) / "Logs" / f"{phase}-attempt.json" + write_attempt_path.write_text(json.dumps({ + "status": "arm-accepted-contract-pending", "target": TARGET, + "mainTemplateSha256": plan["mainTemplateSha256"], "parameters": plan["parameters"], + "candidateSha256": plan["candidateSha256"], "packageIdentity": plan["packageIdentity"], + "deploymentId": f"fixture-{phase}-write", + })) + observed[phase] = {"status": "passed", "httpMethod": "GET", + "provider": "microsoft-security" if phase != "registration" else "azure-resource-manager", + "operationId": f"fixture-{phase}", "responseSha256": hashlib.sha256(response_path.read_bytes()).hexdigest(), + "responsePath": f"Logs/{phase}-readback.json", "requestResourceId": resource_id, + "writeOperationId": f"fixture-{phase}-write", "writeStatus": "succeeded", + "writeAttemptPath": f"Logs/{phase}-attempt.json", "writeAttemptSha256": hashlib.sha256(write_attempt_path.read_bytes()).hexdigest()} + return {"planSha256": plan["planSha256"], "target": TARGET, "armStatus": "succeeded", + "deploymentId": "fixture-deployment", "workbooks": [observed]} + + +def test_contract_requires_independent_registration_provenance_and_repeat_update(case): + current = plan(case) + evidence = readback(current) + result = validation.verify_workbook_readback(current, evidence) + assert result["status"] == "verified" and result["accepted"] is True + assert result["uiVerified"] is False and result["parityVerified"] is False + for phase in ("installation", "registration", "repeatInstallation", "update"): + broken = copy.deepcopy(evidence) + broken["workbooks"][0][phase]["status"] = "denied" + assert validation.verify_workbook_readback(current, broken)["accepted"] is False + stale = copy.deepcopy(evidence) + path = case["solution"] / stale["workbooks"][0]["registration"]["responsePath"] + payload = json.loads(path.read_text()) + payload["properties"]["contentProvenance"]["packageId"] = "unexpected" + path.write_text(json.dumps(payload)) + stale["workbooks"][0]["registration"]["responseSha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + assert validation.verify_workbook_readback(current, stale)["status"] == "blocked" + + +def test_arm_success_alone_never_verifies_workbook_contract(case): + current = plan(case) + result = validation.verify_workbook_readback(current, {"planSha256": current["planSha256"], "target": TARGET, "armStatus": "succeeded"}) + assert result["accepted"] is False + + +def test_powershell_method_route_replaces_only_native_arm_branch(): + script = ROOT / ".script/local-validation/build-and-validate.ps1" + source = script.read_text() + section = source.split("# ARM-TTK Validator (local", 1)[1].split("# STEP 4:", 1)[0] + # Isolate only the official ARM routing block; no dependency installation, + # authentication or other live/network validator is invoked in these tests. + branch = section[section.index(' if ($ArmValidationMethod -eq "lab-deployment")'):section.rfind("\n}")] + command = ( + "$ErrorActionPreference='Stop';$ArmValidationMethod='lab-deployment';" + "$overallSuccess=$true;$reportEntries=[System.Collections.ArrayList]::new();" + "function Test-Path { throw 'unexpected filesystem/dependency route' };" + + branch + + ";if($reportEntries.Count -ne 1 -or $reportEntries[0].Name -ne 'ARM-TTK' -or $reportEntries[0].Status -ne 'skipped' -or -not $overallSuccess){exit 1};'Lab ARM routing passed'" + ) + result = subprocess.run(["pwsh", "-NoProfile", "-Command", command], text=True, capture_output=True) + assert result.returncode == 0, result.stderr + assert "Lab ARM routing passed" in result.stdout + assert "STEP 4: CI PowerShell Validators" in source + assert "Complete-ValidationEvidenceBinding" in source + assert 'packageAccepted = $false' in source + assert "**CANDIDATE GENERATED — NOT ACCEPTED**" in source + + +def test_arm_ttk_metadata_finding_is_not_filtered_and_actual_resource_id_still_fails(case): + (case["solution"] / "Package/createUiDefinition.json").unlink() + runner = ROOT / ".script/local-validation/run-arm-ttk.ps1" + command = """ +function Get-Module { return @{Name='arm-ttk'} } +function Import-Module {} +function Test-AzTemplate { + $errors=@( + [pscustomobject]@{Exception=[Exception]::new('Property "packageId" must use resourceId() at contentProvenance.packageId')}, + [pscustomobject]@{Exception=[Exception]::new('Property "resourceId" contains a hardcoded actual resource ID')} + ) + @([pscustomobject]@{Name='IDs Should Be Derived From ResourceIDs';Errors=$errors}, + [pscustomobject]@{Name='Summary';Errors=@();Summary=[pscustomobject]@{Pass=0;Fail=2}}) +} +""" + f"& '{runner}' -SolutionPath '{case['solution']}' -ToolkitRoot '{ROOT}'" + result = subprocess.run(["pwsh", "-NoProfile", "-Command", command], text=True, capture_output=True) + assert result.returncode == 1, result.stderr + assert "Fail=2" in result.stdout + # No unsupported broad skip, identifier mutation, or external-module patch. + assert "packageId" not in runner.read_text() + + +def test_official_receipt_method_binding_distinguishes_candidates_and_legacy(): + import test_arm_acceptance as fixtures + from sentinel_xdr_migration import arm_acceptance + + harness = fixtures.ArmTemplateAcceptanceTests() + created = harness._write_build_validation_receipt( + rows=["| TS | JSON Syntax | PASS | fixture |", "| .NET | ARM-TTK | SKIP | replaced |"], + result="CANDIDATE_GENERATED", + ) + toolkit, receipt, report, identity, package_files = created + try: + binding_path = receipt.parent / "build-and-validate-binding.json" + old_hash = hashlib.sha256(binding_path.read_bytes()).hexdigest() + binding = json.loads(binding_path.read_text()) + binding.update(validationResult="PASSED", armValidationMethod="lab-deployment", armValidationStatus="pending", + packageLifecycleStatus="candidate-generated", packageAccepted=False, workbookCapabilityVersion=1) + binding_path.write_text(json.dumps(binding)) + receipt.write_text(receipt.read_text().replace("**BUILD SUCCESSFUL**", "**CANDIDATE GENERATED — NOT ACCEPTED**").replace("**BUILD FAILED**", "**CANDIDATE GENERATED — NOT ACCEPTED**").replace("Validation binding result: CANDIDATE_GENERATED", "Validation binding result: PASSED").replace(old_hash, hashlib.sha256(binding_path.read_bytes()).hexdigest())) + result = arm_acceptance._parse_build_validation_receipt( + fixtures.FIXTURE_ROOT, receipt, package_report_path=report, package_identity=identity, + arm_validation_method="lab-deployment", + ) + assert result["binding"]["packageAccepted"] is False + with pytest.raises(ValueError): + arm_acceptance._parse_build_validation_receipt( + fixtures.FIXTURE_ROOT, receipt, package_report_path=report, package_identity=identity, + ) + with pytest.raises(ValueError): + arm_acceptance._parse_build_validation_receipt( + fixtures.FIXTURE_ROOT, receipt, package_report_path=report, package_identity=identity, arm_validation_method="arm-ttk", + ) + finally: + harness._cleanup_build_validation_fixture(toolkit, report, package_files) + + +def test_actual_arm_ttk_pass_is_required_for_explicit_new_alternative(): + import test_arm_acceptance as fixtures + from sentinel_xdr_migration import arm_acceptance + + harness = fixtures.ArmTemplateAcceptanceTests() + created = harness._write_build_validation_receipt( + rows=["| TS | JSON Syntax | PASS | fixture |", "| .NET | ARM-TTK | PASS | actual runner |"], + summary="Total: 2 validators | 2 passed | 0 failed | 0 skipped", + ) + toolkit, receipt, report, identity, package_files = created + try: + assert arm_acceptance._parse_build_validation_receipt( + fixtures.FIXTURE_ROOT, receipt, package_report_path=report, package_identity=identity, arm_validation_method="arm-ttk", + )["status"] == "PASSED" + finally: + harness._cleanup_build_validation_fixture(toolkit, report, package_files) + created = harness._write_build_validation_receipt() + toolkit, receipt, report, identity, package_files = created + try: + with pytest.raises(ValueError, match="actual passed ARM-TTK"): + arm_acceptance._parse_build_validation_receipt( + fixtures.FIXTURE_ROOT, receipt, package_report_path=report, package_identity=identity, arm_validation_method="arm-ttk", + ) + finally: + harness._cleanup_build_validation_fixture(toolkit, report, package_files) + + +def test_raw_readback_hash_identity_and_arm_output_reuse_block(case): + current = plan(case) + evidence = readback(current) + for field, value in (("responseSha256", "b" * 64), ("requestResourceId", "/tenants/other/workbooks/other"), + ("httpMethod", "PUT"), ("provider", "arm-deployment-output"), + ("operationId", "fixture-registration")): + broken = copy.deepcopy(evidence) + broken["workbooks"][0]["installation"][field] = value + assert validation.verify_workbook_readback(current, broken)["status"] == "blocked" + broken = copy.deepcopy(evidence) + broken["workbooks"][0]["update"]["writeOperationId"] = "fixture-installation-write" + assert validation.verify_workbook_readback(current, broken)["status"] == "blocked" + + +def test_unreconciled_attempt_blocks_changed_candidates_and_execution_references(case): + execute(case, runner=lambda _: (124, None, "timeout")) + case["candidate"]["packageIdentity"]["packageAttemptId"] = "changed-attempt" + case["candidate"].pop("candidateSha256") + case["candidate"]["candidateSha256"] = _canonical_sha256(case["candidate"]) + runner = mock.Mock() + with mock.patch.object(validation, "refresh_arm_candidate", return_value=case["candidate"]): + with pytest.raises(ValueError, match="executor requires independent reconciliation"): + execute(case, runner=runner) + runner.assert_not_called() + + +def test_reconcile_requires_independent_exact_terminal_get_and_retains_history(case): + first = execute(case, runner=lambda _: (124, None, "timeout")) + resource_id = ( + f"/subscriptions/{TARGET['subscriptionId']}/resourceGroups/{TARGET['resourceGroup']}" + f"/providers/Microsoft.Resources/deployments/{first['deploymentName']}" + ) + response_path = case["solution"] / "Logs/deployment-get.json" + response_path.write_text(json.dumps({"id": resource_id, "properties": {"provisioningState": "Failed"}})) + observation = {"status": "passed", "httpMethod": "GET", "target": TARGET, "deploymentId": resource_id, + "provisioningState": "Failed", "responsePath": "Logs/deployment-get.json", + "responseSha256": hashlib.sha256(response_path.read_bytes()).hexdigest()} + for change in ({"httpMethod": "PUT"}, {"provisioningState": "Running"}, {"deploymentId": "other"}, {"status": "denied"}): + with pytest.raises(ValueError): + validation.reconcile_lab_arm_attempt(first["attemptPath"], observation={**observation, **change}, actor="reviewer", reference="reconciliation") + reconciled = validation.reconcile_lab_arm_attempt(first["attemptPath"], observation=observation, actor="reviewer", reference="reconciliation") + assert reconciled["status"] == "failed" and reconciled["accepted"] is False and reconciled["retryAuthorized"] is False + assert reconciled["commandExitCode"] == 124 + with pytest.raises(ValueError): + execute(case) + # A new explicit immediate approval reference is a new attempt, after + # independent terminal reconciliation; the old receipt is never overwritten. + current_plan = plan(case) + consent = consent_for(current_plan) + next_result = validation.execute_lab_arm_validation( + CONTEXT, case["candidate"], target=TARGET, selected_flags=FLAGS, + expected_plan_sha256=current_plan["planSha256"], actor="reviewer", approval_reference="new-immediate-approval", + confirmation=validation.LAB_ARM_CONFIRMATION, + evidence_directory=case["solution"] / "Logs/sentinel-xdr-migration/fixture-run", + refresh_candidate=lambda: case["candidate"], read_consent=lambda: consent, identity_check=lambda _: ACCESS, + command_runner=lambda _: (0, {"properties": {"provisioningState": "Succeeded"}}, ""), + ) + assert next_result["status"] == "arm-accepted-contract-pending" + assert next_result["attemptPath"] != first["attemptPath"] + assert json.loads(Path(first["attemptPath"]).read_text())["status"] == "failed" + + +def test_same_type_changed_hash_count_uses_old_scope_only_with_fresh_evidence(case): + original = plan(case) + consent = consent_for(original) + root = case["solution"] + template_path = root / "Package/mainTemplate.json" + template = json.loads(template_path.read_text()) + template["resources"].append({ + "type": "Microsoft.Resources/deployments", "name": "another-fixture-install", + "properties": {"mode": "Incremental", "template": {"resources": []}}, + }) + template_path.write_text(json.dumps(template)) + with zipfile.ZipFile(root / "Package/3.2.1.zip", "w") as archive: + archive.write(template_path, "mainTemplate.json") + archive.write(root / "Package/createUiDefinition.json", "createUiDefinition.json") + report = case["report"] + report["packageArtifactHashes"] = {name: hashlib.sha256((root / report[name]).read_bytes()).hexdigest() + for name in report["packageArtifactHashes"]} + report["packageArtifactsSha256"] = _canonical_sha256(report["packageArtifactHashes"]) + (root / "Logs/packaging.v3_1.json").write_text(json.dumps(report)) + with pytest.raises(ValueError, match="exact current candidate artifacts"): + validation.create_arm_candidate(CONTEXT, solution=root, package_report="Logs/packaging.v3_1.json", + validation_receipt=case["receipt"], prerequisite_evidence=case["gates"]) + for name in ("structure", "package"): + path = root / case["gates"][name]["path"] + payload = json.loads(path.read_text()) + payload["packageArtifactsSha256"] = report["packageArtifactsSha256"] + path.write_text(json.dumps(payload)) + case["gates"][name]["sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + case["candidate"] = validation.create_arm_candidate( + CONTEXT, solution=root, package_report="Logs/packaging.v3_1.json", + validation_receipt=case["receipt"], prerequisite_evidence=case["gates"], + ) + current = plan(case) + assert original["candidateSha256"] != current["candidateSha256"] + assert current["resourceCount"] == original["resourceCount"] + 1 + assert validation.check_lab_arm_scope(current, consent)["status"] == "ready" + assert execute(case, consent_reader=lambda: consent)["status"] == "arm-accepted-contract-pending" + + +def test_powershell_native_arm_route_still_calls_original_validator(): + source = (ROOT / ".script/local-validation/build-and-validate.ps1").read_text() + section = source.split("# ARM-TTK Validator (local", 1)[1].split("# STEP 4:", 1)[0] + branch = section[section.index(' if ($ArmValidationMethod -eq "lab-deployment")'):section.rfind("\n}")] + command = ( + "$ErrorActionPreference='Stop';$ArmValidationMethod='arm-ttk';$overallSuccess=$true;" + "$reportEntries=[System.Collections.ArrayList]::new();$saveProcessOutput={param($name,$output)'fixture-log'};" + f"$toolkitRoot='{ROOT}';$solutionPath='{ROOT}';$packageDir='{ROOT}';" + "function Test-Path {return $true};function pwsh {$global:LASTEXITCODE=0;'ARM-TTK-SUMMARY: Pass=1 Fail=0 Total=1'};" + + branch + + ";if($reportEntries.Count -ne 1 -or $reportEntries[0].Name -ne 'ARM-TTK' -or $reportEntries[0].Status -ne 'passed' -or -not $overallSuccess){exit 1};'Native ARM routing passed'" + ) + result = subprocess.run(["pwsh", "-NoProfile", "-Command", command], text=True, capture_output=True) + assert result.returncode == 0, result.stderr + result.stdout + assert "Native ARM routing passed" in result.stdout + + +def test_arm_ttk_contract_plan_is_read_only_and_cannot_authorize_lab_write(case): + context = {**CONTEXT, "armValidationMethod": "arm-ttk"} + candidate = copy.deepcopy(case["candidate"]) + candidate["armValidationMethod"] = "arm-ttk" + candidate.pop("candidateSha256") + candidate["candidateSha256"] = _canonical_sha256(candidate) + current = validation.plan_workbook_contract_verification(context, candidate, target=TARGET, selected_flags=FLAGS) + assert current["deploymentAuthorizedRoute"] is False and current["purpose"] == "workbook-contract-readback" + with pytest.raises(ValueError): + consent_for(current) + with pytest.raises(ValueError): + validation.plan_lab_arm_validation(context, candidate, target=TARGET, selected_flags=FLAGS) + assert validation.verify_workbook_readback(current, readback(current))["status"] == "verified" + + +def test_readback_becomes_stale_after_current_candidate_inputs_change(case): + current = plan(case) + evidence = readback(current) + (case["solution"] / "input.json").write_text('{"changed":true}') + assert validation.verify_workbook_readback(current, evidence)["status"] == "blocked" diff --git a/Tools/SentinelToXDRMigration/tests/test_workbook_browser.py b/Tools/SentinelToXDRMigration/tests/test_workbook_browser.py new file mode 100644 index 00000000000..c577b8ce8dc --- /dev/null +++ b/Tools/SentinelToXDRMigration/tests/test_workbook_browser.py @@ -0,0 +1,653 @@ +from __future__ import annotations + +from copy import deepcopy +import hashlib +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +from pathlib import Path +import shutil +import sys +import threading +from types import ModuleType +from unittest import mock +from uuid import uuid4 + +import pytest + +from sentinel_xdr_migration import workbook_browser as browser +from sentinel_xdr_migration import workbook_qualification as qualification +from sentinel_xdr_migration.artifacts import report_directory +from test_workbook_qualification import qualified_source, ui_plan, consent, cli_run + + +def text(selector, expected): + return {"selector": selector, "kind": "text", "expected": expected} + + +def attr(selector, attribute, expected): + return {"selector": selector, "kind": "attribute", "attribute": attribute, "expected": expected} + + +def value(selector, expected): + return {"selector": selector, "kind": "value", "expected": expected} + + +def reviewed_browser(plan): + panel = text("#panel", "ok") + steps = [ + {"id": "pickers", "kind": "pickers", "action": "select", "selector": "#picker", "value": "one", + "assertions": [value("#picker", "one"), panel]}, + {"id": "dependent-refresh", "kind": "dependent-refresh", "action": "select", + "selector": "#picker", "value": "two", "before": [text("#dependent", "before")], + "assertions": [value("#picker", "two"), text("#dependent", "after")]}, + {"id": "filters", "kind": "filters", "action": "select", "selector": "#filter", "value": "one", + "assertions": [value("#filter", "one"), panel]}, + {"id": "drill-down", "kind": "drill-down", "action": "drill-down", "selector": "#details", + "expectedUrl": plan["url"], "assertions": [text("#drill", "opened")]}, + {"id": "reload", "kind": "reload", "action": "reload", "assertions": [text("#drill", "closed"), panel]}, + {"id": "layout", "kind": "layout", "action": "observe", "assertions": [{ + "selector": "#panel", "kind": "layout", + "bounds": {"x": [0, 100], "y": [0, 300], "width": [50, 1000], "height": [5, 100]}, + }]}, + {"id": "conditional-visibility", "kind": "conditional-visibility", "action": "observe", + "assertions": [{"selector": "#conditional", "kind": "hidden"}, panel]}, + ] + config = { + "schemaVersion": "1.0.0", "storageStateSha256": "0" * 64, + "readiness": { + "tenantId": attr("#tenant", "data-tenant", plan["target"]["tenantId"]), + "contentId": attr("#workbook", "data-content", plan["contentId"]), + "instanceId": attr("#workbook", "data-instance", plan["instanceId"]), + "ready": attr("#ready", "data-ready", "true"), + }, + "sessionAssertions": [text("#session", "signed in")], + "guards": {"queryErrors": ["#query-error"], "prohibitedControls": ["#edit", "#save"], "signedOut": ["#login"]}, + "network": { + "requests": [{"url": plan["url"], "methods": ["GET"], "purpose": "asset"}], + "navigationUrls": [plan["url"]], + }, + "timeoutMs": 100, "familyTimeoutSeconds": 30, "settleMs": 100, + "viewport": {"width": 1024, "height": 768}, + } + return steps, config + + +def standalone_plan(): + plan = { + "target": {"tenantId": "tenant-observed"}, "contentId": "canonical-observed", + "instanceId": "installed-observed", "url": "https://security.microsoft.com/workbooks/fixture", + "readinessSelectors": {"tenant": "#tenant", "workbook": "#workbook", "ready": "#ready"}, + "backgroundActivityReview": "Explicit exact background request review.", "saveEditAllowed": False, + } + plan["steps"], plan["browserConfig"] = reviewed_browser(plan) + return plan + + +class Locator: + def __init__(self, page, selector): + self.page, self.selector = page, selector + + def count(self): + return 1 if self.selector in self.page.nodes else 0 + + def nth(self, index): + return self + + def wait_for(self, **kwargs): + if not self.count() or kwargs["state"] == "visible" and not self.is_visible(): + raise RuntimeError("synthetic missing selector; sensitive detail must not escape") + + def is_visible(self): + return self.count() and self.page.nodes[self.selector].get("visible", True) + + def is_disabled(self, **kwargs): + return not self.page.nodes[self.selector].get("enabled", True) + + def inner_text(self, **kwargs): + return self.page.nodes[self.selector].get("text", "") + + def get_attribute(self, name, **kwargs): + return self.page.nodes[self.selector].get(name) + + def input_value(self, **kwargs): + return self.page.nodes[self.selector].get("value", "") + + def bounding_box(self, **kwargs): + return {"x": 8, "y": 50, "width": 400, "height": 20} + + def select_option(self, value, **kwargs): + self.page.actions.append("select") + self.page.nodes[self.selector]["value"] = value + if self.selector == "#picker" and value == "two": + self.page.nodes["#dependent"]["text"] = "after" + + def click(self, **kwargs): + self.page.actions.append("click") + if self.selector == "#custom-filter": + self.page.nodes["#custom-option"]["visible"] = True + elif self.selector == "#custom-option": + self.page.nodes["#custom-filter"]["data-value"] = "one" + self.page.nodes["#custom-option"]["visible"] = False + else: + self.page.nodes["#drill"]["text"] = "opened" + + +class Page: + def __init__(self, plan): + self.url, self.actions, self.handlers = plan["url"], [], {} + self.nodes = { + "#tenant": {"data-tenant": plan["target"]["tenantId"]}, + "#workbook": {"data-content": plan["contentId"], "data-instance": plan["instanceId"]}, + "#ready": {"data-ready": "true"}, "#session": {"text": "signed in"}, + "#panel": {"text": "ok"}, "#picker": {"text": "Picker", "value": "initial"}, + "#filter": {"text": "Filter", "value": "initial"}, + "#dependent": {"text": "before"}, "#details": {"text": "Details"}, + "#drill": {"text": "closed"}, "#conditional": {"visible": False}, + } + + def locator(self, selector): + return Locator(self, selector) + + def set_default_timeout(self, timeout): + pass + + def set_default_navigation_timeout(self, timeout): + pass + + def on(self, name, handler): + self.handlers[name] = handler + + def goto(self, url, **kwargs): + self.actions.append("goto") + self.url = url + + def reload(self, **kwargs): + self.actions.append("reload") + self.nodes["#drill"]["text"] = "closed" + + def wait_for_url(self, url, **kwargs): + assert self.url == url + + def wait_for_timeout(self, ms): + pass + + +@pytest.fixture +def scratch(): + directory = Path.cwd() / f".workbook-browser-tests-{uuid4().hex}" + directory.mkdir(mode=0o700) + try: + yield directory + finally: + shutil.rmtree(directory) + + +def test_real_dom_engine_exercises_every_check_without_copying_identity(): + plan = standalone_plan() + browser.validate_browser_config(plan, plan["browserConfig"]) + guard = mock.Mock() + page = Page(plan) + capture = browser._DOMRun(plan, guard).run(mock.Mock(), page) + assert len(capture["steps"]) == 7 + assert page.actions == ["goto", "select", "select", "select", "click", "reload"] + assert capture["readiness"] == { + "tenantId": "tenant-observed", "contentId": "canonical-observed", + "instanceId": "installed-observed", "ready": True, + } + assert capture["browserWideNoWriteSandbox"] is False + assert guard.call_count > 20 + assert "cookies" not in json.dumps(capture) + + +def custom_filter(plan): + plan["steps"][2].update({ + "selector": "#custom-filter", "selectMode": "option", "optionSelector": "#custom-option", + "assertions": [attr("#custom-filter", "data-value", "one"), text("#panel", "ok")], + }) + + +def test_custom_dropdown_uses_only_reviewed_option_and_observed_value(): + plan = standalone_plan() + custom_filter(plan) + browser.validate_browser_config(plan, plan["browserConfig"]) + page = Page(plan) + page.nodes.update({ + "#custom-filter": {"text": "Filter", "data-value": "initial"}, + "#custom-option": {"text": "One", "visible": False}, + }) + capture = browser._DOMRun(plan, lambda: None).run(mock.Mock(), page) + assert len(capture["steps"]) == 7 + assert page.actions == ["goto", "select", "select", "click", "click", "click", "reload"] + del plan["steps"][2]["optionSelector"] + with pytest.raises(ValueError, match="selector"): + browser.validate_browser_config(plan, plan["browserConfig"]) + + +@pytest.mark.parametrize("field", ["tenantId", "contentId", "instanceId", "ready"]) +def test_unobserved_or_wrong_identity_never_passes(field): + plan = standalone_plan() + page = Page(plan) + assertion = plan["browserConfig"]["readiness"][field] + page.nodes[assertion["selector"]][assertion["attribute"]] = "wrong" + with pytest.raises(ValueError, match="observed DOM"): + browser._DOMRun(plan, lambda: None).run(mock.Mock(), page) + assert page.actions == ["goto"] + + +@pytest.mark.parametrize("fault", [ + "missing-panel", "query-error", "edit", "signed-out", "wrong-session", + "ambiguous", "disabled", "unsafe-label", "wrong-layout", "deadline", +]) +def test_dom_safety_failures_block_actions(fault): + plan = standalone_plan() + page = Page(plan) + engine = browser._DOMRun(plan, lambda: None) + if fault == "missing-panel": + del page.nodes["#panel"] + elif fault in {"query-error", "edit", "signed-out"}: + page.nodes[{"query-error": "#query-error", "edit": "#edit", "signed-out": "#login"}[fault]] = {} + elif fault == "wrong-session": + page.nodes["#session"]["text"] = "signed out" + elif fault == "ambiguous": + page.locator = lambda _: mock.Mock(count=lambda: 2, wait_for=lambda **kw: None, nth=lambda i: mock.Mock(is_visible=lambda: False)) + elif fault == "disabled": + page.nodes["#picker"]["enabled"] = False + elif fault == "unsafe-label": + page.nodes["#picker"]["aria-label"] = "Save workbook" + elif fault == "wrong-layout": + plan["steps"][5]["assertions"][0]["bounds"]["width"] = [1, 2] + else: + engine.deadline = 0 + with pytest.raises((ValueError, RuntimeError)): + engine.run(mock.Mock(), page) + if fault in {"query-error", "edit", "signed-out", "wrong-session", "disabled", "unsafe-label", "deadline"}: + assert "select" not in page.actions + + +@pytest.mark.parametrize("event", [ + "pageerror", "console", "response", "requestfailed", "websocket", "download", "dialog", "framenavigated", +]) +def test_error_events_are_fail_closed_and_sanitized(event): + plan = standalone_plan() + page, context = Page(plan), mock.Mock() + engine = browser._DOMRun(plan, lambda: None) + engine.attach(context, page) + message = mock.Mock(type="error", status=500, url="https://attacker.invalid/token=secret") + page.handlers[event](message) + with pytest.raises(ValueError) as failure: + engine.check() + assert "secret" not in str(failure.value) + assert "attacker" not in str(failure.value) + + +def test_network_and_popups_are_rejected_without_retaining_headers_or_urls(): + plan = standalone_plan() + page, context = Page(plan), mock.Mock() + engine = browser._DOMRun(plan, lambda: None) + engine.attach(context, page) + route = mock.Mock(request=mock.Mock(url=plan["url"], method="GET")) + handler = context.route.call_args.args[1] + handler(route) + route.continue_.assert_called_once() + route = mock.Mock(request=mock.Mock(url="https://unreviewed.invalid/?token=secret", method="POST")) + handler(route) + route.abort.assert_called_once() + route.continue_.assert_not_called() + with pytest.raises(ValueError, match="unreviewed-request"): + engine.check() + assert "secret" not in json.dumps(engine.events) + for event in ("page", "serviceworker"): + callback = next(call.args[1] for call in context.on.call_args_list if call.args[0] == event) + callback(mock.Mock()) + socket = mock.Mock() + context.route_web_socket.call_args.args[1](socket) + socket.close.assert_called_once() + assert {"unexpected-page", "unexpected-service-worker", "unreviewed-websocket"} <= engine.faults + + +@pytest.mark.parametrize("mutation", [ + "missing-readiness", "wrong-identity", "same-identities", "empty-session", "negative-session", + "empty-guards", "wildcard-url", "write-method", "post-asset", "unapproved-navigation", + "arbitrary-code", "empty-assertions", "incomplete-refresh", "unobserved-selection", + "unexercised-reload", "missing-layout", "missing-visibility", "unbounded", "unknown-action", +]) +def test_configuration_requires_complete_exact_reviewed_contract(mutation): + plan = standalone_plan() + config = plan["browserConfig"] + if mutation == "missing-readiness": + del config["readiness"]["instanceId"] + elif mutation == "wrong-identity": + config["readiness"]["tenantId"]["expected"] = "other" + elif mutation == "same-identities": + config["readiness"]["instanceId"] = config["readiness"]["contentId"] + elif mutation == "empty-session": + config["sessionAssertions"] = [] + elif mutation == "negative-session": + config["sessionAssertions"] = [{"selector": "#login", "kind": "hidden"}] + elif mutation == "empty-guards": + config["guards"]["queryErrors"] = [] + elif mutation == "wildcard-url": + config["network"]["navigationUrls"] = ["https://*.microsoft.com/*"] + elif mutation == "write-method": + config["network"]["requests"][0]["methods"] = ["PUT"] + elif mutation == "post-asset": + config["network"]["requests"][0]["methods"] = ["POST"] + elif mutation == "unapproved-navigation": + plan["steps"][3]["expectedUrl"] = "https://security.microsoft.com/other" + elif mutation == "arbitrary-code": + config["eval"] = "window.secret" + elif mutation == "empty-assertions": + plan["steps"][0]["assertions"] = [] + elif mutation == "incomplete-refresh": + plan["steps"][1]["before"] = [] + elif mutation == "unobserved-selection": + plan["steps"][0]["assertions"] = [text("#panel", "ok")] + elif mutation == "unexercised-reload": + plan["steps"][4]["action"] = "observe" + elif mutation == "missing-layout": + plan["steps"][5]["assertions"] = [text("#panel", "ok")] + elif mutation == "missing-visibility": + plan["steps"][6]["assertions"] = [text("#panel", "ok")] + elif mutation == "unbounded": + config["familyTimeoutSeconds"] = 999999 + else: + plan["steps"][5]["action"] = "execute-python" + with pytest.raises(ValueError): + browser.validate_browser_config(plan, config) + + +def test_private_storage_is_hash_bound_owner_only_and_never_exported(scratch): + path = scratch / "auth.json" + raw = json.dumps({"cookies": [{"value": "PRIVATE"}], "origins": []}).encode() + path.write_bytes(raw) + digest = hashlib.sha256(raw).hexdigest() + path.chmod(0o600) + assert browser._private_storage(path, digest)["cookies"][0]["value"] == "PRIVATE" + for mode, expected in ((0o644, digest), (0o600, "0" * 64)): + path.chmod(mode) + with pytest.raises(ValueError) as failure: + browser._private_storage(path, expected) + assert "PRIVATE" not in str(failure.value) + link = scratch / "linked.json" + link.symlink_to(path) + with pytest.raises(ValueError, match="symbolic"): + browser._private_storage(link, digest) + path.write_text('{"cookies":[],"origins":[]}') + with pytest.raises(ValueError, match="authenticated"): + browser._private_storage(path, hashlib.sha256(path.read_bytes()).hexdigest()) + + +def test_producer_closes_fresh_context_and_sanitizes_playwright_failures(scratch): + plan = standalone_plan() + path = scratch / "auth.json" + path.write_text('{"cookies":[{"name":"private","value":"PRIVATE"}],"origins":[]}') + path.chmod(0o600) + plan["browserConfig"]["storageStateSha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + playwright = mock.MagicMock() + context = playwright.chromium.launch.return_value.new_context.return_value + page = Page(plan) + context.new_page.return_value = page + entry = mock.MagicMock() + entry.__enter__.return_value = playwright + api = ModuleType("playwright.sync_api") + api.sync_playwright = mock.Mock(return_value=entry) + package = ModuleType("playwright") + with mock.patch.dict(sys.modules, {"playwright": package, "playwright.sync_api": api}): + result = browser.capture_workbook_browser(plan, storage_state=path, before_action=lambda: None) + assert result["producer"] == "playwright-dom-v1" + context.close.assert_called_once() + playwright.chromium.launch.return_value.close.assert_called_once() + assert playwright.chromium.launch.return_value.new_context.call_args.kwargs["service_workers"] == "block" + page.goto = mock.Mock(side_effect=RuntimeError("Authorization: PRIVATE")) + with mock.patch.dict(sys.modules, {"playwright": package, "playwright.sync_api": api}): + with pytest.raises(ValueError) as failure: + browser.capture_workbook_browser(plan, storage_state=path, before_action=lambda: None) + assert "PRIVATE" not in str(failure.value) + assert failure.value.diagnostic_events[-1]["status"] == "failed" + assert "PRIVATE" not in json.dumps(failure.value.diagnostic_events) + + +def test_optional_dependency_missing_is_a_clean_failure(scratch): + plan = standalone_plan() + path = scratch / "auth.json" + path.write_text('{"cookies":[{"value":"PRIVATE"}],"origins":[]}') + path.chmod(0o600) + plan["browserConfig"]["storageStateSha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + original = __import__ + + def import_without_browser(name, *args, **kwargs): + if name.startswith("playwright"): + raise ImportError("not installed") + return original(name, *args, **kwargs) + + with mock.patch("builtins.__import__", side_effect=import_without_browser): + with pytest.raises(ValueError, match="optional browser dependency"): + browser.capture_workbook_browser(plan, storage_state=path, before_action=lambda: None) + + +@pytest.mark.parametrize("change", ["withdrawn", "input"]) +def test_latest_guard_is_checked_between_browser_actions(change): + plan, approval = standalone_plan(), {"approved": True} + page = Page(plan) + changed = False + original = Locator.select_option + + def guarded(): + if not approval["approved"]: + raise ValueError("withdrawn") + if changed: + raise ValueError("changed input") + + def select(locator, *args, **kwargs): + nonlocal changed + original(locator, *args, **kwargs) + if change == "withdrawn": + approval["approved"] = False + else: + changed = True + + with mock.patch.object(Locator, "select_option", select): + with pytest.raises(ValueError): + browser._DOMRun(plan, guarded).run(mock.Mock(), page) + assert page.actions == ["goto", "select"] + + +def test_assertions_must_remain_true_after_settling(): + plan = standalone_plan() + page = Page(plan) + page.wait_for_timeout = lambda _: page.nodes["#panel"].update(text="query failed") + with pytest.raises(ValueError, match="observed DOM"): + browser._DOMRun(plan, lambda: None).run(mock.Mock(), page) + + +@pytest.mark.parametrize("change", ["missing", "identity", "value", "layout"]) +def test_versioned_capture_cannot_omit_or_forge_assertion_proofs(change): + plan = standalone_plan() + capture = browser._DOMRun(plan, lambda: None).run(mock.Mock(), Page(plan)) + if change == "layout": + index = 5 + capture["steps"][index]["assertions"][0]["measurement"]["width"] = 200000 + else: + index = 0 + if change == "missing": + capture["steps"][index].pop("assertions") + elif change == "identity": + capture["steps"][index]["assertions"][0]["assertionSha256"] = "0" * 64 + else: + capture["steps"][index]["assertions"][0]["actualSha256"] = "0" * 64 + with pytest.raises(ValueError): + browser.validate_browser_step_proofs(plan["steps"][index], capture["steps"][index]) + + +def test_evidence_must_be_ignored_untracked_and_private(qualified_source): + root, _ = qualified_source + with mock.patch.object(browser.subprocess, "run", return_value=mock.Mock(returncode=1, stdout=b"")): + with pytest.raises(ValueError, match="Git-ignored"): + browser.prepare_private_browser_evidence(root) + with mock.patch.object(browser.subprocess, "run", side_effect=[ + mock.Mock(returncode=0), mock.Mock(returncode=0, stdout=b"tracked evidence"), + ]): + with pytest.raises(ValueError, match="untracked"): + browser.prepare_private_browser_evidence(root) + browser.prepare_private_browser_evidence(root) + assert report_directory(root).stat().st_mode & 0o077 == 0 + + +def test_no_import_time_browser_dependency_and_legacy_plan_cannot_dispatch(): + plan = standalone_plan() + del plan["browserConfig"] + with mock.patch("builtins.__import__", side_effect=AssertionError("must not import")): + with pytest.raises(ValueError, match="reviewed versioned"): + browser.capture_workbook_browser(plan, storage_state="missing", before_action=lambda: None) + + +def test_browser_plan_is_versioned_and_bound_to_source_config_and_expected_values(qualified_source): + root, _ = qualified_source + legacy = ui_plan(root) + steps, config = reviewed_browser(legacy) + plan = qualification.plan_workbook_ui( + root, workbook=legacy["workbook"], url=legacy["url"], readiness_selectors=legacy["readinessSelectors"], + steps=steps, background_activity_review=legacy["backgroundActivityReview"], browser_config=config, + ) + assert legacy["schemaVersion"] == "1.0.0" + assert "browserConfig" not in legacy + assert plan["schemaVersion"] == "1.1.0" and len(plan["sourceSha256"]) == 64 + assert qualification.current_workbook_ui_plan(root, plan) == plan + changed = deepcopy(plan) + changed["browserConfig"]["viewport"]["width"] += 1 + assert qualification.current_workbook_ui_plan(root, changed)["planSha256"] != plan["planSha256"] + called = mock.Mock() + with pytest.raises(ValueError, match="stale"): + qualification.execute_workbook_ui(root, changed, read_consent=lambda: consent(plan), browser_runner=called) + called.assert_not_called() + + +@pytest.mark.parametrize("change", ["withdrawn", "source", "target", "revoked-after-capture"]) +def test_execution_rechecks_live_consent_and_inputs_and_persists_failure(qualified_source, change): + root, _ = qualified_source + legacy = ui_plan(root) + steps, config = reviewed_browser(legacy) + plan = qualification.plan_workbook_ui( + root, workbook=legacy["workbook"], url=legacy["url"], readiness_selectors=legacy["readinessSelectors"], + steps=steps, background_activity_review=legacy["backgroundActivityReview"], browser_config=config, + ) + approval = consent(plan) + + def capture(current, *, before_action, **kwargs): + page = Page(current) + engine = browser._DOMRun(current, before_action) + if change == "withdrawn": + approval["withdrawn"] = True + elif change == "source": + source = root / "Workbooks/LinuxMachines.json" + source.write_bytes(source.read_bytes() + b"\n") + elif change == "target": + output = root / current["workbook"] + output.write_bytes(output.read_bytes() + b"\n") + result = engine.run(mock.Mock(), page) + if change == "revoked-after-capture": + approval["withdrawn"] = True + return result + + with mock.patch.object(browser, "capture_workbook_browser", side_effect=capture): + with pytest.raises(ValueError): + browser.execute_workbook_browser(root, plan, read_consent=lambda: approval, storage_state="unused") + report = report_directory(root) / f"workbook-ui.{plan['contentId']}.json" + assert json.loads(report.read_bytes())["status"] == "failed" + assert report_directory(root).stat().st_mode & 0o077 == 0 + + +def test_execute_cli_uses_current_hash_and_explicit_run_and_preserves_import(qualified_source): + root, state = qualified_source + legacy = ui_plan(root) + steps, config = reviewed_browser(legacy) + paths = {} + for name, content in (("selectors", legacy["readinessSelectors"]), ("steps", steps), ("browser", config)): + path = report_directory(root) / f"reviewed-{name}.json" + path.write_text(json.dumps(content)) + paths[name] = path.relative_to(root).as_posix() + common = ["--solution", str(root), "--run-id", state["runId"]] + code, planned, error = cli_run([ + "workbook-ui-plan", *common, "--workbook", legacy["workbook"], "--url", legacy["url"], + "--readiness-selectors", paths["selectors"], "--steps", paths["steps"], "--browser-config", paths["browser"], + "--background-activity-review", legacy["backgroundActivityReview"], + ]) + assert code == 0, error + binding = ["--plan", planned["planPath"], "--plan-sha256", planned["planSha256"]] + code, _, error = cli_run([ + "workbook-ui-review", *common, *binding, "--actor", "reviewer", + "--reference", "explicit-browser-read-approval", "--decision", "approved", + ]) + assert code == 0, error + + def capture(current, *, before_action, **kwargs): + return browser._DOMRun(current, before_action).run(mock.Mock(), Page(current)) + + with mock.patch.object(browser, "capture_workbook_browser", side_effect=capture): + code, result, error = cli_run(["workbook-ui-execute", *common, *binding, "--storage-state", "unused"]) + assert code == 0, error + assert result["browserExecution"] == "playwright-dom-v1" + assert result["queryQualification"] == "separate" + code, _, error = cli_run([ + "workbook-ui-execute", *common, "--plan", planned["planPath"], "--plan-sha256", "0" * 64, + "--storage-state", "unused", + ]) + assert code == 2 and "exact reviewed" in error + with pytest.raises(SystemExit): + cli_run(["workbook-ui-execute", "--solution", str(root), *binding, "--storage-state", "unused"]) + + +def test_local_loopback_browser_smoke(): + api = pytest.importorskip("playwright.sync_api") + with api.sync_playwright() as playwright: + if not Path(playwright.chromium.executable_path).is_file(): + pytest.skip("local Chromium is not installed") + plan = standalone_plan() + html = b""" +
Tenant
+
Workbook
+
Ready
signed in
+ + + + +
before
+
closed
ok
+""" + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header("Content-Type", "text/html") + self.end_headers() + self.wfile.write(html) + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + plan["url"] = f"http://127.0.0.1:{server.server_port}/" + plan["steps"], plan["browserConfig"] = reviewed_browser(plan) + custom_filter(plan) + plan["browserConfig"]["timeoutMs"] = 3000 + browser.validate_browser_config(plan, plan["browserConfig"]) + instance = playwright.chromium.launch(headless=True) + try: + context = instance.new_context(service_workers="block", accept_downloads=False, viewport=plan["browserConfig"]["viewport"]) + try: + result = browser._DOMRun(plan, lambda: None).run(context, context.new_page()) + finally: + context.close() + finally: + instance.close() + assert len(result["steps"]) == 7 + assert result["readiness"]["instanceId"] == "installed-observed" + assert result["queryErrors"] == result["javascriptErrors"] == [] + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) diff --git a/Tools/SentinelToXDRMigration/tests/test_workbook_dependencies.py b/Tools/SentinelToXDRMigration/tests/test_workbook_dependencies.py new file mode 100644 index 00000000000..882e6983033 --- /dev/null +++ b/Tools/SentinelToXDRMigration/tests/test_workbook_dependencies.py @@ -0,0 +1,741 @@ +from copy import deepcopy +import json + +import pytest + +from sentinel_xdr_migration import cli, workbooks +from sentinel_xdr_migration import workbook_dependencies as dependencies +from sentinel_xdr_migration.workbook_functions import candidates +from sentinel_xdr_migration._workbook_core import workbook +from test_workbooks import solution, config, convert, authored + +SOURCE = "Workbooks/nested/fixture.json" +POINTER = "/items/0/content/query" + + +def write_query(solution, query): + path = solution / SOURCE + value = json.loads(path.read_bytes()) + value["items"][0]["content"]["query"] = query + path.write_text(json.dumps(value)) + + +def function(solution, alias="ReviewedFunction"): + path = solution / "Parsers/nested/function.yaml" + path.parent.mkdir(parents=True) + path.write_text(f"FunctionName: {alias}\nFunctionAlias: {alias}\nFunctionQuery: Syslog | count\n") + data_path = solution / "Data/Solution_Fixture.json" + data = json.loads(data_path.read_bytes()) + data["Parsers"] = ["Parsers/nested/function.yaml"] + data_path.write_text(json.dumps(data)) + return {key: candidates(solution)[0][key] for key in ("functionAlias", "sourcePath", "sourceSha256")} + + +def decisions(plan, *, decision="accept", tables=None, functions=None): + return { + "planSha256": plan["planSha256"], + "queries": [{"workbook": row["workbook"], "path": row["path"], + "querySha256": row["querySha256"], "decision": decision, + "tables": tables or ["DeviceEvents"], + "workspaceFunctions": [a for a in row["workspaceFunctions"] + if a not in (tables or ["DeviceEvents"])]} + for row in plan["queries"]], + "workbooks": [{"path": row["path"], "decision": decision, + "requiredWorkloads": [{"workload": "sentinel" if tables == ["Syslog"] else "mde", + "dataTypes": tables or ["DeviceEvents"]}], + "requiredDataConnectors": ([{"connectorId": "ReviewedConnector", "dataTypes": tables}] + if tables == ["Syslog"] else []), + "requiredWorkspaceFunctions": functions or []} + for row in plan["inventory"]], + } + + +def review(solution, plan, value, output="Logs/review-1"): + folder = solution / "Logs/inputs" + folder.mkdir(parents=True, exist_ok=True) + plan_path, decisions_path = folder / "plan.json", folder / "decisions.json" + plan_path.write_text(json.dumps(plan)) + decisions_path.write_text(json.dumps(value)) + return dependencies.review_workbook_dependencies( + solution, plan_path=plan_path, decisions_path=decisions_path, + expected_plan_sha256=plan["planSha256"], actor="test-reviewer", + reference="approval-42", output_directory=solution / output) + + +def metadata(plan): + return {"results": [ + {"workbook": row["workbook"], "path": row["path"], "workspaceId": "approved-workspace", + "workbookSha256": row["workbookSha256"], "querySha256": row["querySha256"], + "status": "passed", "statusCode": 200, "partial": False, + "bindings": {}, "renderedQuery": row["queryKql"], + "renderedQuerySha256": workbook.query_hash(row["queryKql"]), + "timeRange": {"start": "2026-01-01T00:00:00Z", "end": "2026-01-02T00:00:00Z"}, + "dataSources": [{"tables": ["DeviceEvents"]}]} + for row in plan["queries"]]} + + +def test_plan_is_offline_hash_bound_and_never_accepts_heuristics(solution): + before = {str(path): path.read_bytes() for path in solution.rglob("*") if path.is_file()} + plan = dependencies.plan_workbook_dependencies(solution) + assert plan["queries"][0]["sourceKql"].startswith("DeviceEvents") + assert plan["queries"][0]["proposal"]["dataTypes"] == ["DeviceEvents"] + assert plan["queries"][0]["proposalAccepted"] is False + assert plan["queries"][0]["staticConfidence"] == "heuristic-not-reviewed" + assert plan["liveQueriesExecuted"] is False + assert plan["planSha256"] == dependencies.digest({k: v for k, v in plan.items() if k != "planSha256"}) + assert before == {str(path): path.read_bytes() for path in solution.rglob("*") if path.is_file()} + + +def test_saved_function_lineage_static_proposal_remains_unresolved(solution): + record = function(solution) + write_query(solution, "let DeviceEvents = Syslog; ReviewedFunction() | count") + plan = dependencies.plan_workbook_dependencies(solution) + row = plan["queries"][0] + assert row["workspaceFunctions"] == [record["functionAlias"]] + assert row["proposalAccepted"] is False + assert row["proposal"]["reviewRequired"] is True + assert "Underlying function tables" in " ".join(row["proposal"]["reviewReasons"]) + assert row["functionPrerequisites"][0]["coverageWarning"] + + +@pytest.mark.parametrize("decision", ["accept", "change"]) +def test_review_writes_supported_config_receipt_not_source_and_converts(solution, decision): + before = (solution / SOURCE).read_bytes() + plan = dependencies.plan_workbook_dependencies(solution) + receipt = review(solution, plan, decisions(plan, decision=decision)) + settings = json.loads((solution / "Logs/review-1/conversion-config.json").read_bytes()) + assert receipt["dependencyReviewRequired"] is False + assert convert(solution, settings)["converted"] == 1 + assert workbooks.validate_workbooks(solution)["valid"] == 1 + assert (solution / SOURCE).read_bytes() == before + assert json.loads((solution / "Logs/review-1/workbook-review-receipt.json").read_bytes())["actor"] == "test-reviewer" + + +def test_defer_cannot_clear_dependency_review(solution): + plan = dependencies.plan_workbook_dependencies(solution) + receipt = review(solution, plan, decisions(plan, decision="defer")) + settings = json.loads((solution / "Logs/review-1/conversion-config.json").read_bytes()) + assert receipt["dependencyReviewRequired"] is True + assert settings["workbookDependencies"][SOURCE]["dependencyReviewRequired"] is True + assert convert(solution, settings)["blocked"] == 1 + + +@pytest.mark.parametrize("failure", ["hash", "query", "unknown", "incomplete", "stale-source", "stale-parser", "malformed", "unknown-key"]) +def test_review_rejects_stale_malformed_incomplete_before_output_mutation(solution, failure): + record = function(solution) + plan = dependencies.plan_workbook_dependencies(solution) + value = decisions(plan) + if failure == "hash": + value["planSha256"] = "0" * 64 + elif failure == "query": + value["queries"][0]["querySha256"] = "0" * 64 + elif failure == "unknown": + value["queries"][0]["path"] = "/unknown" + elif failure == "incomplete": + value["queries"] = [] + elif failure == "stale-source": + write_query(solution, "DeviceEvents | take 1") + elif failure == "stale-parser": + with (solution / record["sourcePath"]).open("a") as handle: + handle.write("# source changed\n") + elif failure == "malformed": + value["workbooks"][0]["requiredWorkloads"] = "mde" + else: + value["queries"][0]["typo"] = True + with pytest.raises(ValueError): + review(solution, plan, value) + assert not (solution / "Logs/review-1").exists() + assert not authored(solution).exists() + + +def test_technical_gaps_survive_reviewer_accept(solution): + write_query(solution, "workspace('elsewhere').DeviceEvents | count") + plan = dependencies.plan_workbook_dependencies(solution) + receipt = review(solution, plan, decisions(plan)) + assert receipt["dependencyReviewRequired"] is True + settings = json.loads((solution / "Logs/review-1/conversion-config.json").read_bytes()) + assert convert(solution, settings)["blocked"] == 1 + + +def test_reviewed_workspace_function_preserves_calls_and_shared_contract(solution): + record = function(solution) + raw = "ReviewedFunction({TimeRange:start}, {TimeRange:end}) | count" + write_query(solution, raw) + assert convert(solution, config(solution, ["Syslog"]))["blocked"] == 1 + plan = dependencies.plan_workbook_dependencies(solution) + review(solution, plan, decisions(plan, tables=["Syslog"], functions=[record])) + settings = json.loads((solution / "Logs/review-1/conversion-config.json").read_bytes()) + assert convert(solution, settings, overwrite=True)["converted"] == 1 + document = json.loads(authored(solution).read_bytes()) + provenance = document["contentProvenance"] + assert document["items"][0]["content"]["query"] == raw + assert provenance["requiredWorkspaceFunctions"] == [record] + assert provenance["conversion"]["queries"][0]["workspaceFunctions"] == ["ReviewedFunction"] + assert provenance["conversion"]["queries"][0]["conversion"]["resolvedTables"] == ["Syslog"] + assert workbooks.validate_workbooks(solution)["valid"] == 1 + # Omission cannot forge an ordinary table-only query. + del provenance["requiredWorkspaceFunctions"] + del provenance["conversion"]["queries"][0]["workspaceFunctions"] + authored(solution).write_text(json.dumps(document)) + assert workbooks.validate_workbooks(solution)["invalid"] == 1 + + +@pytest.mark.parametrize("failure", ["stale", "missing-selection", "alias", "extra", "unsafe", "malformed", "duplicate-selection"]) +def test_function_prerequisite_failures_block_before_candidate_writes(solution, failure): + record = function(solution) + write_query(solution, "ReviewedFunction() | count") + settings = config(solution, ["Syslog"]) + settings["workbookResolutions"][SOURCE][POINTER]["workspaceFunctions"] = ["ReviewedFunction"] + settings["workbookDependencies"] = {SOURCE: { + "requiredWorkspaceFunctions": [deepcopy(record)], + "requiredWorkloads": [{"workload": "sentinel", "dataTypes": ["Syslog"]}], + "requiredDataConnectors": []}} + selected = settings["workbookDependencies"][SOURCE]["requiredWorkspaceFunctions"][0] + if failure == "stale": + selected["sourceSha256"] = "0" * 64 + elif failure == "alias": + selected["functionAlias"] = "OtherFunction" + elif failure in ("missing-selection", "duplicate-selection"): + data_path = solution / "Data/Solution_Fixture.json" + data = json.loads(data_path.read_bytes()) + data["Parsers"] = [] if failure == "missing-selection" else [record["sourcePath"]] * 2 + data_path.write_text(json.dumps(data)) + elif failure == "extra": + settings["workbookResolutions"][SOURCE][POINTER]["workspaceFunctions"].append("Unexpected") + assert convert(solution, settings)["blocked"] == 1 + return + elif failure == "unsafe": + selected["sourcePath"] = "Parsers/../Data/Solution_Fixture.json" + else: + selected["extra"] = True + with pytest.raises(ValueError): + convert(solution, settings) + assert not authored(solution).exists() + assert not (solution / "Logs").exists() + + +def test_optional_branch_removal_is_separate_requested_action_not_rewrite(solution): + record = function(solution) + write_query(solution, "ReviewedFunction()") + plan = dependencies.plan_workbook_dependencies(solution) + value = decisions(plan, tables=["Syslog"], functions=[record]) + value["queries"][0]["optionalBranches"] = [{**record, "branch": "Syslog | count", "decision": "remove-request"}] + before = (solution / record["sourcePath"]).read_bytes() + result = review(solution, plan, value) + assert result["parserChangeActions"][0]["status"] == "requested-not-applied" + assert result["dependencyReviewRequired"] is True + assert (solution / record["sourcePath"]).read_bytes() == before + + +def test_import_exact_metadata_and_binding_requirements(solution): + write_query(solution, "DeviceEvents | where Timestamp >= {TimeRange:start}") + plan = dependencies.plan_workbook_dependencies(solution) + imported = metadata(plan) + entry = imported["results"][0] + entry["bindings"] = {"TimeRange:start": "datetime(2026-01-01T00:00:00Z)"} + entry["renderedQuery"] = plan["queries"][0]["queryKql"].replace("{TimeRange:start}", entry["bindings"]["TimeRange:start"]) + entry["renderedQuerySha256"] = workbook.query_hash(entry["renderedQuery"]) + current = dependencies.plan_workbook_dependencies(solution, workspace_id="approved-workspace", imported_results=imported) + assert current["queries"][0]["proposal"]["method"] == "log-analytics" + assert current["queries"][0]["proposalAccepted"] is False + assert current["liveQueriesExecuted"] is False + + +@pytest.mark.parametrize("failure", ["workspace", "source", "query", "error", "partial", "status", "malformed", + "incomplete", "cross-input", "binding", "rendered-hash", "time"]) +def test_import_rejects_failed_stale_partial_cross_workspace_unbound_metadata(solution, failure): + write_query(solution, "DeviceEvents | where Timestamp >= {TimeRange:start}") + plan = dependencies.plan_workbook_dependencies(solution) + imported = metadata(plan) + entry = imported["results"][0] + entry["bindings"] = {"TimeRange:start": "datetime(2026-01-01T00:00:00Z)"} + entry["renderedQuery"] = entry["renderedQuery"].replace("{TimeRange:start}", entry["bindings"]["TimeRange:start"]) + entry["renderedQuerySha256"] = workbook.query_hash(entry["renderedQuery"]) + changes = { + "workspace": ("workspaceId", "another-workspace"), "source": ("workbookSha256", "0" * 64), + "query": ("querySha256", "0" * 64), "error": ("error", "failed"), + "partial": ("partial", True), "status": ("statusCode", 403), + "malformed": ("dataSources", "Syslog"), "binding": ("bindings", {}), + "rendered-hash": ("renderedQuerySha256", "0" * 64), + "time": ("timeRange", {"start": "2026-01-03", "end": "2026-01-01"}), + } + if failure == "incomplete": + imported["results"] = [] + elif failure == "cross-input": + entry["dataSources"][0]["workspaceId"] = "another-workspace" + else: + key, value = changes[failure] + entry[key] = value + with pytest.raises(ValueError): + dependencies.plan_workbook_dependencies(solution, workspace_id="approved-workspace", imported_results=imported) + assert not (solution / "Logs").exists() + + +def test_cli_pure_json_plan_and_review(solution, capsys): + assert cli.main(["workbook-dependency-plan", "--solution", str(solution)]) == 0 + plan = json.loads(capsys.readouterr().out) + assert not (solution / "Logs").exists() + folder = solution / "Logs/inputs" + folder.mkdir(parents=True) + (folder / "plan.json").write_text(json.dumps(plan)) + (folder / "decisions.json").write_text(json.dumps(decisions(plan))) + assert cli.main([ + "workbook-review", "--solution", str(solution), "--plan", str(folder / "plan.json"), + "--decisions", str(folder / "decisions.json"), "--plan-sha256", plan["planSha256"], + "--actor", "reviewer", "--reference", "ticket", "--output-dir", str(solution / "Logs/review"), + ]) == 0 + assert json.loads(capsys.readouterr().out)["configPath"].endswith("conversion-config.json") + + +def test_review_symlink_output_and_existing_receipts_are_protected(solution): + plan = dependencies.plan_workbook_dependencies(solution) + target = solution / "linked" + target.symlink_to(solution / "Data", target_is_directory=True) + with pytest.raises(ValueError, match="symlink"): + review(solution, plan, decisions(plan), output="linked/review") + review(solution, plan, decisions(plan)) + before = (solution / "Logs/review-1/workbook-review-receipt.json").read_bytes() + with pytest.raises(ValueError, match="already exist"): + review(solution, plan, decisions(plan)) + assert (solution / "Logs/review-1/workbook-review-receipt.json").read_bytes() == before + + +@pytest.mark.parametrize("query", ["_GetWatchlist('fixture') | count", + "Syslog | invoke _GetWatchlist('fixture')"]) +def test_watchlist_is_builtin_not_workspace_installation_prerequisite(solution, query): + write_query(solution, query) + plan = dependencies.plan_workbook_dependencies(solution) + assert plan["queries"][0]["workspaceFunctions"] == [] + assert not any("No current local definition" in text for text in plan["queries"][0]["gapExplanations"]) + settings = config(solution, ["Syslog"]) + settings["workbookDependencies"] = {SOURCE: { + "requiredWorkloads": [{"workload": "sentinel", "dataTypes": ["Syslog"]}], + "requiredDataConnectors": []}} + assert convert(solution, settings)["converted"] == 1 + assert json.loads(authored(solution).read_bytes())["items"][0]["content"]["query"] == query + + +def test_watchlist_inside_saved_definition_is_not_missing_function(solution): + record = function(solution) + path = solution / record["sourcePath"] + path.write_text("FunctionName: ReviewedFunction\nFunctionQuery: _GetWatchlist('fixture') | count\n") + assert candidates(solution)[0]["nestedFunctions"] == [] + + +@pytest.mark.parametrize("query,aliases", [ + ("union ReviewedFunction('x','y'), OtherFunction() | count", {"ReviewedFunction", "OtherFunction"}), + ("union kind=outer (ReviewedFunction()), (OtherFunction()) | count", {"ReviewedFunction", "OtherFunction"}), + ("DeviceEvents | join (ReviewedFunction()) on Computer", {"ReviewedFunction"}), + ("let local=(){ReviewedFunction()}; local()", {"ReviewedFunction"}), + ("let ids=dynamic([1,2]); DeviceEvents | summarize max(tolong(EventCount))", set()), + ("print x='FakeFunction()' // FakeFunction()\n", set()), +]) +def test_alias_inventory_is_tabular_comment_safe_and_complete(query, aliases): + assert workbook.opaque_functions(query) == aliases + + +def test_later_union_operand_cannot_be_forged_away(solution): + record = function(solution) + write_query(solution, "union ReviewedFunction(), UnreviewedFunction() | count") + settings = config(solution, ["Syslog"]) + settings["workbookResolutions"][SOURCE][POINTER]["workspaceFunctions"] = ["ReviewedFunction"] + settings["workbookDependencies"] = {SOURCE: { + "requiredWorkspaceFunctions": [record], + "requiredWorkloads": [{"workload": "sentinel", "dataTypes": ["Syslog"]}], + "requiredDataConnectors": []}} + assert convert(solution, settings)["blocked"] == 1 + + +@pytest.mark.parametrize("query", ["ReviewedFunction() | count", "_GetWatchlist('fixture') | count"]) +def test_explicit_reviewed_watchlist_lineage_is_passthrough_without_catalog_mutation(solution, query): + record = function(solution) if query.startswith("ReviewedFunction") else None + write_query(solution, query) + plan = dependencies.plan_workbook_dependencies(solution) + from sentinel_xdr_migration._workbook_core import mapping_catalog + catalog_before = deepcopy(mapping_catalog.load_mapping_catalog()) + value = decisions(plan, tables=["Syslog"], functions=[record] if record else []) + value["queries"][0]["tables"] = ["Syslog", "Watchlist"] + value["workbooks"][0]["requiredWorkloads"][0]["dataTypes"] = ["Syslog"] + result = review(solution, plan, value) + assert result["dependencyReviewRequired"] is False + settings = json.loads((solution / "Logs/review-1/conversion-config.json").read_bytes()) + conversion = convert(solution, settings) + assert conversion["converted"] == 1 + document = json.loads(authored(solution).read_bytes()) + contract = document["contentProvenance"]["conversion"]["queries"][0]["conversion"] + assert contract["resolvedTables"] == ["Syslog", "Watchlist"] + assert document["contentProvenance"]["conversion"]["queries"][0]["referenceInputs"] == ["Watchlist"] + assert document["items"][0]["content"]["query"] == query + assert workbooks.validate_workbooks(solution)["valid"] == 1 + assert "blank-ah-selection-table-visibility" in { + gate["code"] for gate in conversion["results"][0]["gates"]} + assert catalog_before == mapping_catalog.load_mapping_catalog() + + +@pytest.mark.parametrize("mapping", [{"DeviceEvents": "DeviceEvents"}, {"Watchlist": "Syslog"}, + {"Unknown": "Unknown"}, ["Watchlist"]]) +def test_reviewed_mapping_cannot_override_catalog_or_source_scope(solution, mapping): + plan = dependencies.plan_workbook_dependencies(solution) + value = decisions(plan) + value["queries"][0]["tableMappings"] = mapping + with pytest.raises(ValueError): + review(solution, plan, value) + assert not (solution / "Logs/review-1").exists() + + +def test_raw_watchlist_metadata_is_lineage_not_telemetry_prerequisite(solution): + from sentinel_xdr_migration.dependencies import dependency_plan, workload_records + raw = [{"tables": ["Syslog", "Watchlist"]}] + proposal = dependency_plan("Syslog | join _GetWatchlist('fixture') on Computer", metadata=raw) + assert proposal["returnedDataTypes"] == ["Syslog", "Watchlist"] + assert proposal["sourceDataTypes"] == ["Syslog", "Watchlist"] + assert proposal["referenceInputs"] == ["Watchlist"] + assert proposal["dataTypes"] == proposal["targetDataTypes"] == ["Syslog"] + assert proposal["requiredWorkloads"] == [{"workload": "sentinel", "dataTypes": ["Syslog"]}] + assert workload_records(["Watchlist", "UnknownSource"]) == ([], ["UnknownSource"]) + + +def test_watchlist_only_query_requires_no_telemetry_workload_or_connector(solution): + write_query(solution, "_GetWatchlist('fixture') | count") + plan = dependencies.plan_workbook_dependencies(solution) + proposal = plan["queries"][0]["proposal"] + assert proposal["referenceInputs"] == ["Watchlist"] + assert proposal["dataTypes"] == proposal["targetDataTypes"] == [] + assert proposal["requiredWorkloads"] == proposal["requiredDataConnectors"] == [] + value = decisions(plan, tables=["Watchlist"]) + value["workbooks"][0]["requiredWorkloads"] = [] + result = review(solution, plan, value) + assert result["dependencyReviewRequired"] is False + settings = json.loads((solution / "Logs/review-1/conversion-config.json").read_bytes()) + assert convert(solution, settings)["converted"] == 1 + assert workbooks.validate_workbooks(solution)["valid"] == 1 + provenance = json.loads(authored(solution).read_bytes())["contentProvenance"] + assert provenance["requiredWorkloads"] == provenance["requiredDataConnectors"] == [] + assert "requiredWorkspaceFunctions" not in provenance + + +@pytest.mark.parametrize("field", ["requiredWorkloads", "requiredDataConnectors"]) +def test_watchlist_cannot_be_declared_as_connector_telemetry(solution, field): + plan = dependencies.plan_workbook_dependencies(solution) + value = decisions(plan) + record = ({"workload": "sentinel", "dataTypes": ["Watchlist"]} + if field == "requiredWorkloads" else {"connectorId": "Watchlist", "dataTypes": ["Watchlist"]}) + value["workbooks"][0][field] = [record] + with pytest.raises(ValueError, match="platform reference"): + review(solution, plan, value) + assert not (solution / "Logs/review-1").exists() + + +def test_other_unknown_tables_are_not_hidden_by_platform_reference_classification(solution): + write_query(solution, "UnknownSource | count") + plan = dependencies.plan_workbook_dependencies(solution) + value = decisions(plan, tables=["UnknownSource"]) + value["workbooks"][0]["requiredWorkloads"] = [{"workload": "sentinel", "dataTypes": ["UnknownSource"]}] + result = review(solution, plan, value) + assert result["dependencyReviewRequired"] is True + settings = json.loads((solution / "Logs/review-1/conversion-config.json").read_bytes()) + assert convert(solution, settings)["blocked"] == 1 + + +def test_metadata_time_binding_cannot_misstate_declared_range(solution): + write_query(solution, "DeviceEvents | where Timestamp >= {TimeRange:start}") + plan = dependencies.plan_workbook_dependencies(solution) + imported = metadata(plan) + entry = imported["results"][0] + entry["bindings"] = {"TimeRange:start": "datetime(2025-01-01T00:00:00Z)"} + entry["renderedQuery"] = entry["renderedQuery"].replace("{TimeRange:start}", entry["bindings"]["TimeRange:start"]) + entry["renderedQuerySha256"] = workbook.query_hash(entry["renderedQuery"]) + with pytest.raises(ValueError, match="time bindings"): + dependencies.plan_workbook_dependencies(solution, workspace_id="approved-workspace", imported_results=imported) + + +def test_vendored_component_manifest_hashes_are_current(): + import hashlib + from importlib.resources import files + root = files("sentinel_xdr_migration._workbook_core") + manifest = json.loads(root.joinpath("import-manifest.json").read_text()) + for item in manifest["components"]: + assert hashlib.sha256(root.joinpath(item["component"]).read_bytes()).hexdigest() == item["sha256"] + + +def test_connector_alternatives_are_not_cumulative_or_title_recommendations(solution): + folder = solution / "Data Connectors" + folder.mkdir() + for identifier, title in (("Legacy", "Modern Agent"), ("Modern", "Older connector")): + (folder / f"{identifier}.json").write_text(json.dumps({ + "id": identifier, "title": title, "dataTypes": [{"name": "Syslog"}]})) + data_path = solution / "Data/Solution_Fixture.json" + data = json.loads(data_path.read_bytes()) + data["Data Connectors"] = ["Data Connectors/Legacy.json", "Data Connectors/Modern.json"] + data["Description"] = "Reviewed source recommendation: use Modern; legacy agent is deprecated." + data_path.write_text(json.dumps(data)) + write_query(solution, "Syslog | count") + plan = dependencies.plan_workbook_dependencies(solution) + assert {record["connectorId"] for record in plan["connectorAlternatives"]} == {"Legacy", "Modern"} + assert all(record["recommendation"] is None for record in plan["connectorAlternatives"]) + assert plan["sourceRecommendationEvidence"]["reviewRequired"] is True + assert plan["sourceRecommendationEvidence"]["titleHeuristicsUsed"] is False + value = decisions(plan, tables=["Syslog"]) + value["workbooks"][0]["requiredDataConnectors"] = [{"connectorId": "Modern", "dataTypes": ["Syslog"]}] + result = review(solution, plan, value) + assert result["dependencyReviewRequired"] is False + settings = json.loads((solution / "Logs/review-1/conversion-config.json").read_bytes()) + assert settings["workbookDependencies"][SOURCE]["requiredDataConnectors"] == [ + {"connectorId": "Modern", "dataTypes": ["Syslog"]}] + + +@pytest.mark.parametrize("text", ["[]", "FunctionName: bad-name\nFunctionQuery: Syslog", + "FunctionAlias: 42\nFunctionQuery: Syslog", + "FunctionName: ValidName\nFunctionQuery: ''", + "FunctionName: ValidName\nFunctionQuery: Syslog\nFunctionParams: malformed"]) +def test_malformed_workspace_function_definition_is_not_approved(solution, text): + record = function(solution) + path = solution / record["sourcePath"] + path.write_text(text) + settings = config(solution) + record["sourceSha256"] = workbooks._sha(path.read_bytes()) + settings["workbookDependencies"] = {SOURCE: {"requiredWorkspaceFunctions": [record]}} + with pytest.raises(ValueError): + convert(solution, settings) + assert not (solution / "Logs").exists() + assert not authored(solution).exists() + + +def test_externaldata_is_source_operator_not_workspace_function(solution): + record = function(solution) + path = solution / record["sourcePath"] + path.write_text("FunctionName: ReviewedFunction\nFunctionQuery: externaldata(value:string)['https://example.invalid/reviewed']\n") + current = candidates(solution)[0] + assert current["nestedFunctions"] == [] + assert current["externalSourceWarnings"] + assert current["nestedDiscoveryExhaustive"] is False + assert workbook.opaque_functions("externaldata(value:string)['https://example.invalid']") == set() + + +def test_static_nested_union_discovery_reports_unresolved_bare_branches_not_installed_claims(solution): + record = function(solution) + path = solution / record["sourcePath"] + path.write_text("FunctionName: ReviewedFunction\nFunctionQuery: union isfuzzy=true CiscoISEEvent, OtherVendorParser, Syslog\n") + current = candidates(solution)[0] + assert current["nestedFunctions"] == [] + assert current["unresolvedBranchCandidates"] == ["CiscoISEEvent", "OtherVendorParser"] + assert current["nestedDiscoveryExhaustive"] is False + + +@pytest.mark.parametrize("query", [ + "(ReviewedFunction()) | count", + "let x = (ReviewedFunction()); x", + "print toscalar(ReviewedFunction() | summarize count())", + "print ReviewedFunction()", + "print tostring(ReviewedFunction())", + "Syslog | extend x = ReviewedFunction()", + "Syslog | where Computer == ReviewedFunction()", + "Syslog | where Computer in (ReviewedFunction())", + "print (ReviewedFunction())", + "Syslog | where Computer in ((ReviewedFunction() | project Computer))", + "find in (ReviewedFunction()) where * has 'x'", + "search in (ReviewedFunction()) 'x'", + "find in (ReviewedFunction) where * has 'x'", + "search in (ReviewedFunction) 'x'", + "ReviewedFunction | count", + "(ReviewedFunction) | count", + "let x = ReviewedFunction; x", + "let x = (ReviewedFunction); x", + "let x = materialize(ReviewedFunction | take 1); x", + "let x = materialize(ReviewedFunction); x", + "Syslog | where Computer in (ReviewedFunction | project Computer)", + "Syslog | where Computer in ((ReviewedFunction) | project Computer)", + "Syslog | where Computer in ( ( ReviewedFunction ) | project Computer)", + "['ReviewedFunction']() | count", + '["ReviewedFunction"]() | count', + "[\n'ReviewedFunction'\n]() | count", + "print ['ReviewedFunction']()", + "['ReviewedFunction'] | take 10", + "union ['ReviewedFunction']", + "union kind=outer ['ReviewedFunction']", + "union isfuzzy=true ['ReviewedFunction']", + "Syslog | invoke ['ReviewedFunction']()", + "Syslog | invoke ['ReviewedFunction']", + "table('ReviewedFunction') | take 10", + "table('ReviewedFunction', 'all') | take 10", + "table('ReviewedFunction', 'hotcache') | take 10", + "table('ReviewedFunction', 'default') | take 10", + "totable('ReviewedFunction') | take 10", + "totable('ReviewedFunction', 'all') | take 10", + "let local=(){ReviewedFunction()}; local()", + "let local=(){ReviewedFunction}; local()", + "let local=view(){ReviewedFunction}; local()", + "view() { ReviewedFunction }", + "Syslog | partition by Computer { ReviewedFunction }", + "Syslog | fork (ReviewedFunction)", + "Syslog | fork (['ReviewedFunction'])", + "Syslog | where Computer has ['ReviewedFunction']()", + "Syslog | where Computer !has ['ReviewedFunction']()", + "Syslog | where Computer contains ['ReviewedFunction']()", + "Syslog | where Computer !contains ['ReviewedFunction']()", + "Syslog | where Computer startswith ['ReviewedFunction']()", + "Syslog | where Computer endswith ['ReviewedFunction']()", + "Syslog | where Computer has_cs ['ReviewedFunction']()", + "Syslog | where Computer has['ReviewedFunction']()", + "Syslog | where Computer !has['ReviewedFunction']()", + "Syslog | where Computer has['ReviewedFunction'] ()", + "Syslog | where Computer has['ReviewedFunction']\n()", + "Syslog | where Computer contains['ReviewedFunction']()", + "Syslog | where Computer !contains['ReviewedFunction']()", + "Syslog | where Computer startswith['ReviewedFunction']()", + "Syslog | where Computer endswith['ReviewedFunction']()", + "Syslog | where Computer has_cs['ReviewedFunction']()", + "Syslog | distinct ['ReviewedFunction']()", + "Syslog | distinct['ReviewedFunction']()", + "Syslog | mv-expand ['ReviewedFunction']()", + "Syslog | mv-expand['ReviewedFunction']()", + "print['ReviewedFunction']()", + "union['ReviewedFunction']", + "union kind=outer['ReviewedFunction']", + "union isfuzzy=true['ReviewedFunction']", + "Syslog | invoke['ReviewedFunction']()", + "Syslog | invoke['ReviewedFunction']", + "Syslog | join hint.strategy=broadcast (ReviewedFunction) on Computer", + "Syslog | join kind=inner hint.strategy=broadcast (ReviewedFunction) on Computer", + "Syslog | join hint.strategy=shuffle (['ReviewedFunction']) on Computer", + "Syslog | lookup kind=leftouter hint.strategy=broadcast (ReviewedFunction) on Computer", + "let local=(){Syslog | where Computer in (ReviewedFunction())}; local()", + "DeviceEvents | lookup kind=leftouter (ReviewedFunction()) on DeviceId", + "DeviceEvents | lookup kind=leftouter (ReviewedFunction) on DeviceId", + "DeviceEvents | join kind = inner (ReviewedFunction) on DeviceId", + "/* not a KQL comment */ ReviewedFunction() | count", + "/* ReviewedFunction() */ | count", +]) +def test_saved_function_scanner_catches_bypass_forms_and_enforces_review_and_validation(solution, query): + assert workbook.opaque_functions(query) == {"ReviewedFunction"} + record = function(solution) + write_query(solution, query) + plan = dependencies.plan_workbook_dependencies(solution) + assert plan["queries"][0]["workspaceFunctions"] == ["ReviewedFunction"] + bad_decisions = decisions(plan, tables=["Syslog", "DeviceEvents"], functions=[]) + bad_decisions["workbooks"][0]["requiredWorkspaceFunctions"] = [] + with pytest.raises(ValueError, match="Function aliases need vetted prerequisites"): + review(solution, plan, bad_decisions) + good_decisions = decisions(plan, tables=["Syslog", "DeviceEvents"], functions=["ReviewedFunction"]) + good_decisions["workbooks"][0]["requiredWorkspaceFunctions"] = [record] + good_decisions["workbooks"][0]["requiredWorkloads"] = [ + {"workload": "sentinel", "dataTypes": ["Syslog"]}, + {"workload": "mde", "dataTypes": ["DeviceEvents"]}, + ] + good_decisions["workbooks"][0]["requiredDataConnectors"] = [ + {"connectorId": "FixtureConnector", "dataTypes": ["Syslog", "DeviceEvents"]}, + ] + review_result = review(solution, plan, good_decisions) + assert review_result["dependencyReviewRequired"] is False + settings = json.loads(sorted((solution / "Logs").glob("review-*/conversion-config.json"))[-1].read_bytes()) + assert convert(solution, settings)["converted"] == 1 + assert workbooks.validate_workbooks(solution)["valid"] == 1 + doc = json.loads(authored(solution).read_bytes()) + doc["contentProvenance"].pop("requiredWorkspaceFunctions", None) + doc["contentProvenance"]["conversion"]["queries"][0].pop("workspaceFunctions", None) + authored(solution).write_text(json.dumps(doc)) + validation = workbooks.validate_workbooks(solution) + assert validation["valid"] == 0 + assert any("workspace function" in err.lower() for err in validation["results"][0]["errors"]) + + +@pytest.mark.parametrize("query,token", [ + ("{Fn} | take 10", "{Fn}"), + ("{Fn}() | take 10", "{Fn}"), + ("({Fn} | project Computer)", "{Fn}"), + ("Syslog | where Computer in ({Fn} | project Computer)", "{Fn}"), + ("{Hidden Func}() | take 10", "{Hidden Func}"), + ("({Hidden Func} | project Computer)", "{Hidden Func}"), + ("let x = {Fn}; x | take 10", "{Fn}"), + ("let x = materialize({Fn} | take 1); x", "{Fn}"), + ("table({Fn}) | take 10", "{Fn}"), + ("table(strcat('Le', 'ak')) | take 10", "{__unsupported_table_indirection__}"), + ("totable(strcat('Le', 'ak')) | take 10", "{__unsupported_table_indirection__}"), + ("Syslog | join ({Fn}) on Computer", "{Fn}"), + ("print {Fn}()", "{Fn}"), +]) +def test_parameter_built_sources_and_call_targets_fail_closed(solution, query, token): + assert token in workbook.opaque_functions(query) + write_query(solution, query) + plan = dependencies.plan_workbook_dependencies(solution) + assert token in plan["queries"][0]["workspaceFunctions"] + bad_decisions = decisions(plan, tables=["Syslog"], functions=[]) + bad_decisions["workbooks"][0]["requiredWorkspaceFunctions"] = [] + with pytest.raises(ValueError): + review(solution, plan, bad_decisions) + res = workbook.convert_workbook(json.loads((solution / SOURCE).read_bytes()), tables=["Syslog"]) + assert res["report"]["status"] == "blocked" + + +@pytest.mark.parametrize("query", [ + "Evil_CL | take 1", + "union Evil_CL", + "table('Evil_CL') | take 1", +]) +def test_cl_suffix_requires_explicit_reviewed_table_or_function_evidence(solution, query): + assert workbook.opaque_functions(query) == {"Evil_CL"} + write_query(solution, query) + plan = dependencies.plan_workbook_dependencies(solution) + assert plan["queries"][0]["workspaceFunctions"] == ["Evil_CL"] + bad_decisions = decisions(plan, tables=["Syslog"], functions=[]) + bad_decisions["workbooks"][0]["requiredWorkspaceFunctions"] = [] + with pytest.raises(ValueError, match="Reviewed workspaceFunctions must match exact source aliases|Function aliases need vetted prerequisites"): + review(solution, plan, bad_decisions) + res_unreviewed = workbook.convert_workbook(json.loads((solution / SOURCE).read_bytes()), tables=["Syslog"]) + assert res_unreviewed["report"]["status"] == "blocked" + # When Evil_CL is explicitly supplied as a reviewed table, bare table usage is allowed + res_reviewed_table = workbook.convert_workbook(json.loads((solution / SOURCE).read_bytes()), tables=["Evil_CL"]) + assert res_reviewed_table["report"]["queries"][0].get("workspaceFunctions", []) == [] + + +def test_decoy_functions_cannot_satisfy_hint_flags_or_unsupported_table_indirection(solution): + function(solution) + decoys = ("hint", "broadcast", "__unsupported_table_indirection__", "__unsupported_escaped_call_context__") + for alias in decoys: + p = solution / "Parsers" / "nested" / f"{alias}.yaml" + p.write_text(f"FunctionName: {alias}\nFunctionAlias: {alias}\nFunctionQuery: Syslog | take 1\n") + data_file = solution / "Data" / "Solution_Fixture.json" + data = json.loads(data_file.read_bytes()) + data["Parsers"] = ["Parsers/nested/function.yaml"] + [f"Parsers/nested/{a}.yaml" for a in decoys] + data_file.write_text(json.dumps(data, indent=2)) + by_alias = {c["functionAlias"]: {k: c[k] for k in ("functionAlias", "sourcePath", "sourceSha256")} + for c in candidates(solution)} + decoy_records = [by_alias[a] for a in decoys] + for query in ( + "Syslog | join hint.strategy=broadcast (ReviewedFunction) on Computer", + "Syslog | join kind=inner hint.strategy=broadcast (ReviewedFunction) on Computer", + "Syslog | join hint.strategy=shuffle (['ReviewedFunction']) on Computer", + "table(strcat('Reviewed', 'Function')) | take 1", + "print bag['ReviewedFunction']()", + "Syslog | where isnotempty(todynamic(AdditionalFields)['ReviewedFunction']())", + ): + write_query(solution, query) + plan = dependencies.plan_workbook_dependencies(solution) + bad_decisions = decisions(plan, tables=["Syslog"], functions=list(decoys)) + for q_dec in bad_decisions["queries"]: + q_dec["workspaceFunctions"] = list(decoys) + bad_decisions["workbooks"][0]["requiredWorkspaceFunctions"] = decoy_records + with pytest.raises(ValueError): + review(solution, plan, bad_decisions) + res = workbook.convert_workbook( + json.loads((solution / SOURCE).read_bytes()), + tables=["Syslog"], + allowed_workspace_functions=list(decoys), + ) + assert res["report"]["status"] == "blocked" + + +@pytest.mark.parametrize("query", [ + "print bag['Leak']", + "Syslog | extend v = todynamic(AdditionalFields)['Leak'] | where isnotempty(AdditionalFields['Leak']) and tostring(AdditionalFields['Leak']['nested']) != ''", + "Syslog | extend ['Activity Count'] = 1 | project ['Activity Count'], [\"ResolutionTime\"] = 2 | order by ['Activity Count'] desc", + "let LocalView = view() { Syslog }; LocalView()", + "Syslog | partition by Computer { Syslog | count }", + "Syslog | fork (Syslog | count)", + "table('Syslog') | take 1", + "table('Syslog', 'all') | take 1", + "totable('Syslog') | take 1", +]) +def test_benign_member_indexing_and_catalog_table_controls(query): + assert workbook.opaque_functions(query) == set() + diff --git a/Tools/SentinelToXDRMigration/tests/test_workbook_preflight.py b/Tools/SentinelToXDRMigration/tests/test_workbook_preflight.py new file mode 100644 index 00000000000..5ed598b7fa7 --- /dev/null +++ b/Tools/SentinelToXDRMigration/tests/test_workbook_preflight.py @@ -0,0 +1,783 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import shutil +import subprocess +import sys +import time +import uuid +from pathlib import Path + +import pytest + +from sentinel_xdr_migration import workbook_preflight as preflight +from sentinel_xdr_migration import workbook_validation as validation +from sentinel_xdr_migration.arm_acceptance_support import _canonical_sha256 + +ROOT = Path(__file__).resolve().parents[3] +TARGET = { + "tenantId": "11111111-2222-3333-4444-555555555555", + "subscriptionId": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + "resourceGroup": "fixture-rg", + "workspaceResourceId": "/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/fixture-rg/providers/Microsoft.OperationalInsights/workspaces/fixture-ws", + "workspaceCustomerId": "99999999-8888-7777-6666-555555555555", +} +PRINCIPAL = "cccccccc-1111-2222-3333-444444444444" +CONTENT = "aaaaaaaa-1111-2222-3333-444444444444" +INSTANCE = "bbbbbbbb-1111-2222-3333-444444444444" +CONTEXT = { + "runId": "preflight-fixture", "workflowProfile": "authoring", "profileSelectionConfirmed": True, + "workbookCapabilityVersion": 1, "armValidationMethod": "lab-deployment", +} + + +def seal(value, field): + value.pop(field, None) + value[field] = _canonical_sha256(value) + return value + + +def template(): + provenance = {"packageId": "publisher.fixture", "packageVersion": "3.1.1", + "contentId": CONTENT, "contentVersion": "3.1.1"} + notebook = {"version": "Notebook/1.0", "items": [], "contentProvenance": provenance} + inner = { + "languageVersion": "2.0", + "imports": {"MicrosoftSecurity": {"provider": "MicrosoftSecurity", "version": "1.0.0"}}, + "resources": {"workbook": { + "type": "Microsoft.Security/workbooks@2026-06-01-preview", "import": "MicrosoftSecurity", + "properties": {"id": INSTANCE, "name": "Fixture", "content": notebook}, + }}, + } + return { + "parameters": {"workspace": {"type": "string"}, "DeployDefenderWorkbook": {"type": "bool", "defaultValue": False}}, + "resources": [ + {"type": "Microsoft.Resources/deployments", "name": "install", + "condition": "[parameters('DeployDefenderWorkbook')]", + "properties": {"mode": "Incremental", "template": inner}}, + {"type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "name": "[concat(parameters('workspace'), '/Microsoft.SecurityInsights/fixture')]", + "properties": {"contentId": CONTENT, "contentKind": "DefenderWorkbook", "version": "3.1.1", + "packageId": "publisher.fixture", "packageVersion": "3.1.1", + "mainTemplate": copy.deepcopy(inner), "contentProvenance": provenance}}, + {"type": "Microsoft.OperationalInsights/workspaces/providers/contentPackages", + "name": "[concat(parameters('workspace'), '/Microsoft.SecurityInsights/publisher.fixture')]", + "properties": {"contentId": "publisher.fixture", "contentKind": "Solution", + "dependencies": {"criteria": [{"kind": "DefenderWorkbook", "contentId": CONTENT}]} }}, + ], + } + + +@pytest.fixture +def case(): + directory = ROOT / ".script/local-validation/Logs" / ("preflight-tests-" + uuid.uuid4().hex) + directory.mkdir(parents=True) + path = directory / "mainTemplate.json" + path.write_text(json.dumps(template()), encoding="utf-8") + candidate = seal({ + "status": "candidate-generated", "accepted": False, "armValidationMethod": "lab-deployment", + "runId": CONTEXT["runId"], "workflowProfile": "authoring", "solutionPath": str(directory), + "artifactPaths": {"mainTemplate": "mainTemplate.json"}, "packageIdentity": {"packageId": "publisher.fixture"}, + "prerequisiteEvidence": {}, "validationReceipt": {}, + "mainTemplateSha256": hashlib.sha256(path.read_bytes()).hexdigest(), + }, "candidateSha256") + arm = validation.plan_lab_arm_validation(CONTEXT, candidate, target=TARGET, + selected_flags={"DeployDefenderWorkbook": True}) + plan = preflight.plan_arm_preflight(arm) + consent = preflight.review_arm_preflight(plan, "fixture-reviewer", "metadata-approval", True) + try: + yield {"plan": plan, "arm": arm, "path": path, "consent": consent} + finally: + shutil.rmtree(directory) + + +def response_for(operation, case): + identifier = operation["id"] + if identifier == "account": + body = {"id": TARGET["subscriptionId"], "tenantId": TARGET["tenantId"], "user": {"name": "fixture@example.invalid"}, + "principal": {"objectId": PRINCIPAL, "tenantId": TARGET["tenantId"], + "audience": preflight.ARM_BASE, "expiresOn": time.time() + 3600, + "source": "authenticated-arm-token"}} + elif identifier == "workspace": + body = {"id": TARGET["workspaceResourceId"], "properties": {"customerId": TARGET["workspaceCustomerId"]}} + elif identifier == "resource-group": + body = {"id": TARGET["workspaceResourceId"].split("/providers/")[0], "tags": {"Environment": "sandbox"}} + elif identifier.startswith("permissions-"): + body = {"value": [{"actions": ["*"], "notActions": [], "dataActions": [], "notDataActions": []}]} + elif identifier.startswith(("denies-", "role-assignments-")): + body = {"value": []} + elif identifier.startswith("registrations-"): + family = identifier.split("-", 1)[1] + resource = next(row for row in template()["resources"] if row["type"].endswith("/" + family)) + name = "fixture" if family == "contentTemplates" else "publisher.fixture" + body = {"value": [{"id": TARGET["workspaceResourceId"] + "/providers/Microsoft.SecurityInsights/" + family + "/" + name, + "properties": copy.deepcopy(resource["properties"])}]} + elif identifier == "workbook-" + INSTANCE: + body = {"id": INSTANCE, "name": "Fixture", "content": copy.deepcopy( + template()["resources"][0]["properties"]["template"]["resources"]["workbook"]["properties"]["content"])} + else: + raise AssertionError(identifier) + result = {"statusCode": 200, "body": body} + if operation["kind"] != "azure-cli-account": + result.update(principalId=PRINCIPAL, tenantId=TARGET["tenantId"], provenance={ + "requestUrl": operation["url"], "method": "GET", + "source": "authenticated-arm-token" if operation["kind"] == "arm-get" else "authenticated-graph-token", + "principalId": PRINCIPAL, "tenantId": TARGET["tenantId"], + }) + return result + + +def execute(case, change=None, consent_reader=None): + calls = [] + + def reader(operation): + calls.append(copy.deepcopy(operation)) + result = response_for(operation, case) + if change: + change(operation, result) + return result + + result = preflight.execute_arm_preflight(case["plan"], + read_consent=consent_reader or (lambda: case["consent"]), reader=reader) + return result, calls + + +def refresh_arm(case, current): + case["path"].write_text(json.dumps(current)) + arm = copy.deepcopy(case["arm"]) + arm["mainTemplateSha256"] = hashlib.sha256(case["path"].read_bytes()).hexdigest() + arm["candidateSha256"] = _canonical_sha256({"updatedTemplate": arm["mainTemplateSha256"]}) + arm["resources"] = validation._resource_inventory(current, arm["parameters"], arm["target"]) + arm["expectedWorkbooks"] = validation._expected_workbook_contracts(current, True) + arm["resourceCount"] = len(arm["resources"]) + arm["scope"]["resourceTypes"] = sorted({row["type"] for row in arm["resources"]} | {"Microsoft.Resources/deployments"}) + return seal(arm, "planSha256") + + +def test_deterministic_bounded_exact_read_plan_and_bindings(case): + plan = case["plan"] + assert plan == preflight.plan_arm_preflight(case["arm"]) + assert plan["armPlanSha256"] == case["arm"]["planSha256"] + assert plan["candidateSha256"] == case["arm"]["candidateSha256"] + assert plan["mainTemplateSha256"] == case["arm"]["mainTemplateSha256"] + assert plan["preflightScopeSha256"] == _canonical_sha256(plan["readScope"]) + assert plan["operations"][0]["subscriptionId"] == TARGET["subscriptionId"] + assert len(plan["operations"]) == 12 + for operation in plan["operations"][1:]: + assert operation["method"] == "GET" + assert operation["url"].startswith((preflight.ARM_BASE + "/", preflight.GRAPH_BASE + "/")) + assert operation["timeoutSeconds"] == 15 and operation["retryCount"] == 0 + assert next(row for row in plan["operations"] if row["kind"] == "graph-get")["url"] == ( + preflight.GRAPH_BASE + "/beta/security/workbooks/" + INSTANCE) + assert all(Path(name).name == name for name in plan["artifacts"].values()) + assert plan["writesAuthorized"] is False + assert plan["tokenAcquisitionAuthorized"] is True + assert plan["signInAuthorized"] is False + assert all("atScope()" in operation["url"] for operation in plan["operations"] + if operation["id"].startswith(("denies-", "role-assignments-"))) + + +def test_verified_facts_but_imported_provider_write_contract_stays_blocked(case): + result, calls = execute(case) + assert calls == case["plan"]["operations"] + assert result["status"] == "blocked" + for flag in preflight._ACCESS_FLAGS: + assert result[flag] is (flag != "extensionResourceWriteAllowed") + assert result["identityId"] == PRINCIPAL + assert result["providerWriteContractVerified"] is False + assert result["writeAttempted"] is False + assert "VERIFIED documented MicrosoftSecurity" in result["checks"]["extensionResourceWriteAllowed"]["reasons"][0] + assert result["checks"]["nonProduction"]["tenantNonProductionVerified"] is False + assert result["checks"]["armDeploymentAllowed"]["actions"]["Microsoft.SecurityInsights/contentTemplates/write"]["status"] == "ready" + assert result["observationSha256"] + + +def test_parent_minimal_trusted_reader_protocol_and_unverified_graph(case): + def change(operation, response): + response.pop("provenance", None) + if operation["kind"] == "graph-get": + response.update(statusCode=0, body={"error": {"code": "MicrosoftSecurityIdentityUnverified"}}) + result, calls = execute(case, change) + assert len(calls) == len(case["plan"]["operations"]) + assert result["tenantMatches"] is True + assert result["armDeploymentAllowed"] is True + assert result["ownershipVerified"] is False + assert result["extensionResourceWriteAllowed"] is False + assert result["target"] == TARGET + assert result["preflightPlanSha256"] == case["plan"]["preflightPlanSha256"] + assert result["observations"]["responses"][1]["provenance"]["source"] == "consent-bound-reader-dispatch" + + +def test_read_consent_explicitly_limits_arm_token_acquisition_and_refresh(case): + plan, consent = case["plan"], case["consent"] + token_access = { + "scope": "https://management.azure.com/.default", + "tenantId": TARGET["tenantId"], "subscriptionId": TARGET["subscriptionId"], + "existingCachedIdentityOnly": True, "acquisitionAuthorized": True, "refreshAuthorized": True, + "interactiveSignInAuthorized": False, "newCredentialsAuthorized": False, + "permissionGrantsAuthorized": False, "graphTokenAcquisitionAuthorized": False, + "graphTokenReuseAuthorized": False, + "advancedHuntingCredentialReuseAuthorized": False, + } + for record in (plan, plan["readScope"], consent, consent["readScope"]): + assert record["tokenAcquisitionAuthorized"] is True + assert record["armTokenAccess"] == token_access + assert all(record[key] is False for key in preflight._FORBIDDEN_AUTHORIZATIONS) + assert plan["readScope"]["principalContract"]["armTokenAccess"] == token_access + assert consent["preflightScopeSha256"] == _canonical_sha256(consent["readScope"]) + + +@pytest.mark.parametrize("field", [ + "signInAuthorized", "interactiveSignInAuthorized", "newCredentialsAuthorized", + "permissionGrantsAuthorized", "graphTokenAcquisitionAuthorized", + "graphTokenReuseAuthorized", + "advancedHuntingCredentialReuseAuthorized", "existingCachedIdentityOnly", "scope", "tenantId", +]) +def test_resealed_consent_cannot_authorize_sign_in_credentials_graph_or_ah_reuse(case, field): + consent = copy.deepcopy(case["consent"]) + if field == "existingCachedIdentityOnly": + consent["armTokenAccess"][field] = False + elif field == "scope": + consent["armTokenAccess"][field] = "https://graph.microsoft.com/.default" + elif field == "tenantId": + consent["armTokenAccess"][field] = str(uuid.uuid4()) + else: + consent[field] = True + seal(consent, "consentSha256") + result, calls = execute(case, consent_reader=lambda: consent) + assert calls == [] + assert result["readAttempted"] is False + + +def test_old_no_acquisition_consent_requires_renewed_explicit_token_scope_review(case): + consent = copy.deepcopy(case["consent"]) + consent["tokenAcquisitionAuthorized"] = False + consent.pop("armTokenAccess") + seal(consent, "consentSha256") + result, calls = execute(case, consent_reader=lambda: consent) + assert calls == [] + assert result["checks"]["readConsentVerified"]["status"] == "blocked" + + +@pytest.mark.parametrize("decision", ["missing", "declined", "revoked", "wrong-target", "tampered", "wrong-plan"]) +def test_no_current_exact_approval_means_zero_reader_calls(case, decision): + consent = copy.deepcopy(case["consent"]) + if decision == "missing": + consent = None + elif decision == "declined": + consent = preflight.review_arm_preflight(case["plan"], "reviewer", "declined", False) + elif decision == "revoked": + consent = preflight.revoke_arm_preflight(consent, "reviewer", "revoked") + elif decision == "wrong-target": + consent["target"]["tenantId"] = str(uuid.uuid4()) + seal(consent, "consentSha256") + elif decision == "wrong-plan": + consent["preflightScopeSha256"] = "0" * 64 + seal(consent, "consentSha256") + else: + consent["operations"][1]["method"] = "POST" + result, calls = execute(case, consent_reader=lambda: consent) + assert calls == [] + assert result["readAttempted"] is False + assert result["status"] == "blocked" + assert result["ownershipVerified"] is False + + +def test_revoke_during_read_invalidates_evidence_and_stops(case): + calls = [] + consent = case["consent"] + + def reader(operation): + nonlocal consent + calls.append(operation) + consent = preflight.revoke_arm_preflight(consent, "reviewer", "mid-read") + return response_for(operation, case) + + result = preflight.execute_arm_preflight(case["plan"], read_consent=lambda: consent, reader=reader) + assert len(calls) == 1 + assert result["observations"]["responses"][0]["status"] == "blocked" + assert result["checks"]["readConsentVerified"]["status"] == "blocked" + + +def test_same_read_scope_reuses_consent_for_fresh_same_id_candidate(case): + current = template() + inner = current["resources"][0]["properties"]["template"] + inner["resources"]["workbook"]["properties"]["content"]["items"] = [{"shipped": "new-version"}] + current["resources"][1]["properties"]["mainTemplate"] = copy.deepcopy(inner) + arm = refresh_arm(case, current) + old_plan = case["plan"] + case["plan"] = preflight.plan_arm_preflight(arm) + assert case["plan"]["preflightPlanSha256"] != old_plan["preflightPlanSha256"] + assert case["plan"]["candidateSha256"] != old_plan["candidateSha256"] + assert case["plan"]["mainTemplateSha256"] != old_plan["mainTemplateSha256"] + assert case["plan"]["preflightScopeSha256"] == old_plan["preflightScopeSha256"] + result, calls = execute(case) + assert calls == case["plan"]["operations"] + assert result["checks"]["readConsentVerified"]["status"] == "verified" + assert result["ownershipVerified"] is True + assert result["preflightPlanSha256"] == case["plan"]["preflightPlanSha256"] + + +def test_new_instance_url_requires_renewed_exact_metadata_scope_consent(case): + current = template() + inner = current["resources"][0]["properties"]["template"] + inner["resources"]["workbook"]["properties"]["id"] = str(uuid.uuid4()) + current["resources"][1]["properties"]["mainTemplate"] = copy.deepcopy(inner) + arm = refresh_arm(case, current) + old_scope = case["plan"]["preflightScopeSha256"] + case["plan"] = preflight.plan_arm_preflight(arm) + assert case["plan"]["preflightScopeSha256"] != old_scope + result, calls = execute(case) + assert calls == [] + assert result["readAttempted"] is False + + +@pytest.mark.parametrize("status", [403, 429, 404, 500]) +def test_failed_reads_are_not_empty_success(case, status): + def change(operation, response): + if operation["id"] == "workspace": + response.update(statusCode=status, body={"error": {"message": "Bearer synthetic-token"}}) + result, calls = execute(case, change) + assert len(calls) == 2 + assert result["workspaceMatches"] is False + assert result["armDeploymentAllowed"] is False + assert result["observations"]["responses"][-1]["body"]["error"]["message"] == "Bearer [redacted]" + + +@pytest.mark.parametrize("bad", ["nextLink", "@odata.nextLink", "empty-body", "non-object", "wrong-status-type"]) +def test_malformed_and_partial_responses_remain_unknown(case, bad): + def change(operation, response): + if operation["id"] != "workspace": + return + if bad in {"nextLink", "@odata.nextLink"}: + response["body"][bad] = "https://unreviewed.example.invalid/next" + elif bad == "empty-body": + response["body"] = {} + elif bad == "non-object": + response["body"] = [] + else: + response["statusCode"] = True + result, _ = execute(case, change) + assert result["workspaceMatches"] is False + assert result["status"] == "blocked" + + +def test_timeout_reader_error_is_sanitized_and_bounded(case): + calls = [] + + def reader(operation): + calls.append(operation) + raise TimeoutError("Bearer do-not-persist-this") + result = preflight.execute_arm_preflight(case["plan"], read_consent=lambda: case["consent"], reader=reader) + assert len(calls) == 1 + assert "TimeoutError" in result["observations"]["responses"][0]["reason"] + assert "do-not-persist" not in json.dumps(result) + + +@pytest.mark.parametrize("failure", [ValueError, RuntimeError, OSError, KeyError, TypeError]) +@pytest.mark.parametrize("phase", ["consent", "reader"]) +def test_supported_provider_and_consent_failures_are_sanitized_blockers(case, failure, phase): + calls = [] + + def unavailable(): + raise failure("Bearer forbidden-sensitive-message") + + def reader(operation): + calls.append(operation) + if phase == "reader": + unavailable() + return response_for(operation, case) + result = preflight.execute_arm_preflight(case["plan"], + read_consent=unavailable if phase == "consent" else lambda: case["consent"], reader=reader) + assert len(calls) == (0 if phase == "consent" else 1) + assert result["status"] == "blocked" + assert "forbidden-sensitive-message" not in json.dumps(result) + + +@pytest.mark.parametrize("failure", [AssertionError, AttributeError, IndexError, MemoryError]) +@pytest.mark.parametrize("phase", ["consent", "reader"]) +def test_unexpected_programmer_or_system_exceptions_propagate(case, failure, phase): + def unexpected(*args): + raise failure("unexpected implementation failure") + with pytest.raises(failure): + preflight.execute_arm_preflight(case["plan"], + read_consent=unexpected if phase == "consent" else lambda: case["consent"], + reader=unexpected if phase == "reader" else lambda operation: response_for(operation, case)) + + +def test_reader_overrun_is_rejected_without_retry(case, monkeypatch): + times = iter([0.0, 16.0]) + monkeypatch.setattr(preflight.time, "monotonic", lambda: next(times)) + result, calls = execute(case) + assert len(calls) == 1 + assert result["observations"]["responses"][0]["reason"] == "reader exceeded bounded timeout" + assert result["status"] == "blocked" + + +def test_mutation_of_caller_plan_during_read_stops_dispatch(case): + calls = [] + + def reader(operation): + calls.append(operation) + case["plan"]["operations"][1]["method"] = "POST" + return response_for(operation, case) + result = preflight.execute_arm_preflight(case["plan"], read_consent=lambda: case["consent"], reader=reader) + assert len(calls) == 1 + assert result["observations"]["responses"][0]["status"] == "blocked" + + +@pytest.mark.parametrize("bad", ["account-name-only", "wrong-principal", "wrong-request-proof", "expired", "wrong-audience", "wrong-tenant"]) +def test_identity_requires_observed_authenticated_arm_principal(case, bad): + def change(operation, response): + if bad == "account-name-only" and operation["id"] == "account": + del response["body"]["principal"] + elif bad == "expired" and operation["id"] == "account": + response["body"]["principal"]["expiresOn"] = time.time() - 1 + elif bad == "wrong-audience" and operation["id"] == "account": + response["body"]["principal"]["audience"] = preflight.GRAPH_BASE + elif bad == "wrong-tenant" and operation["id"] == "account": + response["body"]["principal"]["tenantId"] = str(uuid.uuid4()) + elif bad == "wrong-principal" and operation["kind"] != "azure-cli-account": + response["principalId"] = str(uuid.uuid4()) + elif bad == "wrong-request-proof" and operation["kind"] != "azure-cli-account": + response["provenance"]["requestUrl"] = "https://evil.example.invalid" + result, _ = execute(case, change) + assert result["identityId"] is None + assert result["tenantMatches"] is False + assert result["armDeploymentAllowed"] is False + + +@pytest.mark.parametrize("bad", ["data-actions-only", "not-actions", "permission-condition", "assignment-condition", "deny"]) +def test_control_plane_permissions_exclusions_conditions_and_denies(case, bad): + def change(operation, response): + if operation["id"].startswith("permissions-"): + if bad == "data-actions-only": + response["body"]["value"] = [{"actions": [], "notActions": [], "dataActions": ["*"]}] + elif bad == "not-actions": + response["body"]["value"][0]["notActions"] = ["Microsoft.Resources/deployments/*", "Microsoft.SecurityInsights/*"] + elif bad == "permission-condition": + response["body"]["value"].append({"actions": ["*"], "notActions": [], "condition": "unknown"}) + if operation["id"].startswith("role-assignments-") and bad == "assignment-condition": + response["body"]["value"] = [{"properties": { + "scope": TARGET["workspaceResourceId"].split("/providers/")[0], "principalId": PRINCIPAL, + "roleDefinitionId": "/providers/Microsoft.Authorization/roleDefinitions/" + str(uuid.uuid4()), + "condition": "unknown", "conditionVersion": "2.0", + }}] + if operation["id"].startswith("denies-") and bad == "deny": + response["body"]["value"] = [{"properties": {"permissions": [{"actions": ["*"]}]}}] + result, _ = execute(case, change) + assert result["armDeploymentAllowed"] is False + + +def test_not_actions_are_per_grant_not_global_denials(case): + def change(operation, response): + if operation["id"].startswith("permissions-"): + response["body"]["value"] = [ + {"actions": ["*"], "notActions": ["Microsoft.SecurityInsights/*"]}, + {"actions": ["Microsoft.SecurityInsights/*"], "notActions": []}, + ] + result, _ = execute(case, change) + assert result["armDeploymentAllowed"] is True + + +@pytest.mark.parametrize("conditions", [ + {"condition": None, "conditionVersion": None}, + {"condition": "", "conditionVersion": ""}, + {"condition": None}, + {"conditionVersion": ""}, +]) +def test_nullable_rest_condition_fields_preserve_unconditional_grants(case, conditions): + rg = TARGET["workspaceResourceId"].split("/providers/")[0] + + def change(operation, response): + if operation["id"].startswith("permissions-"): + response["body"]["value"][0].update(conditions) + elif operation["id"].startswith("role-assignments-"): + assignment_id = rg + "/providers/Microsoft.Authorization/roleAssignments/" + str(uuid.uuid4()) + response["body"]["value"] = [{ + "id": assignment_id, "type": "Microsoft.Authorization/roleAssignments", + "properties": { + "scope": rg, "principalId": PRINCIPAL, "principalType": "User", + "roleDefinitionId": "/subscriptions/" + TARGET["subscriptionId"] + + "/providers/Microsoft.Authorization/roleDefinitions/" + str(uuid.uuid4()), + **conditions, + }, + }] + result, _ = execute(case, change) + assert result["armDeploymentAllowed"] is True + assert all(row["status"] == "verified" for row in result["checks"]["armDeploymentAllowed"]["roleAssignments"].values()) + + +@pytest.mark.parametrize("conditions", [ + {"condition": "", "conditionVersion": "2.0"}, + {"condition": "@Resource[unknown]"}, + {"condition": []}, {"condition": False}, {"conditionVersion": 2}, +]) +@pytest.mark.parametrize("family", ["permissions-", "role-assignments-"]) +def test_real_or_malformed_nullable_conditions_still_block(case, conditions, family): + def change(operation, response): + if not operation["id"].startswith(family): + return + if family == "permissions-": + response["body"]["value"][0].update(conditions) + else: + response["body"]["value"] = [{"properties": { + "scope": TARGET["workspaceResourceId"].split("/providers/")[0], + "principalId": PRINCIPAL, + "roleDefinitionId": "/providers/Microsoft.Authorization/roleDefinitions/" + str(uuid.uuid4()), + **conditions, + }}] + result, _ = execute(case, change) + assert result["armDeploymentAllowed"] is False + + +def test_nonempty_denies_are_conservatively_blocked_without_applicability_claim(case): + def change(operation, response): + if operation["id"].startswith("denies-"): + response["body"]["value"] = [{"properties": { + "scope": TARGET["workspaceResourceId"], + "doNotApplyToChildScopes": True, + "principals": [{"id": PRINCIPAL, "type": "User"}], + "excludePrincipals": [{"id": PRINCIPAL, "type": "User"}], + "permissions": [{"actions": ["*"], "notActions": []}], + }}] + result, _ = execute(case, change) + assert result["armDeploymentAllowed"] is False + for check in result["checks"]["armDeploymentAllowed"]["denies"].values(): + assert check["status"] == "unknown" + assert check["applicabilityEvaluated"] is False + assert check["policy"] == "require-complete-empty-ancestor-deny-list" + + +def test_absent_imports_have_no_extension_blocker_in_ready_result(case): + current = template() + current["resources"] = [current["resources"][2]] + case["path"].write_text(json.dumps(current)) + arm = copy.deepcopy(case["arm"]) + arm["mainTemplateSha256"] = hashlib.sha256(case["path"].read_bytes()).hexdigest() + arm["candidateSha256"] = _canonical_sha256({"metadataOnly": arm["mainTemplateSha256"]}) + arm["selectedFlags"]["DeployDefenderWorkbook"] = False + arm["parameters"]["DeployDefenderWorkbook"] = False + arm["resources"] = validation._resource_inventory(current, arm["parameters"], arm["target"]) + arm["expectedWorkbooks"] = [] + arm["resourceCount"] = len(arm["resources"]) + arm["scope"]["contentTypes"] = [] + arm["scope"]["resourceTypes"] = sorted({row["type"] for row in arm["resources"]} | {"Microsoft.Resources/deployments"}) + seal(arm, "planSha256") + case["plan"] = preflight.plan_arm_preflight(arm) + case["consent"] = preflight.review_arm_preflight(case["plan"], "reviewer", "metadata-only", True) + result, _ = execute(case) + assert result["status"] == "ready" + assert result["extensionResourceWriteAllowed"] is False + assert result["checks"]["extensionResourceWriteAllowed"]["status"] == "not-required" + assert result["checks"]["extensionResourceWriteAllowed"]["writeAccessVerified"] is False + assert result["reasons"] == [] + + +@pytest.mark.parametrize("bad", ["wrong-id", "foreign-package", "wrong-content", "404", "ambiguous-registration", "foreign-registration", "wrong-registration-content", "malformed-registration"]) +def test_exact_managed_identity_ownership_boundaries(case, bad): + def change(operation, response): + if operation["kind"] == "graph-get": + if bad == "wrong-id": + response["body"]["id"] = str(uuid.uuid4()) + elif bad == "foreign-package": + response["body"]["content"]["contentProvenance"]["packageId"] = "foreign.publisher" + elif bad == "wrong-content": + response["body"]["content"]["contentProvenance"]["contentId"] = str(uuid.uuid4()) + elif bad == "404": + response.update(statusCode=404, body={}) + if operation["id"] == "registrations-contentTemplates": + if bad == "ambiguous-registration": + response["body"]["value"].append(copy.deepcopy(response["body"]["value"][0])) + elif bad == "foreign-registration": + response["body"]["value"][0]["properties"]["packageId"] = "foreign.publisher" + elif bad == "wrong-registration-content": + response["body"]["value"][0]["properties"]["contentId"] = str(uuid.uuid4()) + elif bad == "malformed-registration": + response["body"]["value"] = [{}] + result, _ = execute(case, change) + assert result["ownershipVerified"] is False + + +def test_intended_same_id_upsert_allows_old_or_edited_content_and_surfaces_change(case): + def change(operation, response): + if operation["kind"] == "graph-get": + response["body"]["content"]["items"] = [{"edited": True}] + response["body"]["content"]["contentProvenance"].update(packageVersion="1.0.0", contentVersion="1.0.0") + if operation["id"] == "registrations-contentTemplates": + response["body"]["value"][0]["properties"].update(version="1.0.0", mainTemplate={"old": "edited"}) + result, _ = execute(case, change) + assert result["ownershipVerified"] is True + updates = result["checks"]["ownershipVerified"]["managedUpdates"] + assert any(row.get("instanceId") == INSTANCE and row["contentChanged"] for row in updates) + assert all(row["policy"] == "managed-same-id-upsert" for row in updates) + assert result["extensionResourceWriteAllowed"] is False + + +@pytest.mark.parametrize("tag", [None, {"environment": "production"}, {"arbitrary": "lab"}, {"Environment": "dev", "env": "production"}]) +def test_only_existing_recognized_exact_scope_environment_tag_is_nonproduction(case, tag): + def change(operation, response): + if operation["id"] == "resource-group": + response["body"]["tags"] = tag + result, _ = execute(case, change) + assert result["nonProduction"] is False + + +def test_resealed_operation_forgery_and_template_staleness_block_dispatch(case): + forged = copy.deepcopy(case["plan"]) + forged["operations"][1]["url"] = "https://evil.example.invalid" + seal(forged, "preflightPlanSha256") + with pytest.raises(ValueError, match="reconstruct"): + preflight.review_arm_preflight(forged, "reviewer", "no", True) + current = json.loads(case["path"].read_text()) + current["resources"][0]["properties"]["template"]["resources"]["workbook"]["properties"]["name"] = "Changed" + case["path"].write_text(json.dumps(current)) + with pytest.raises(ValueError, match="current template"): + preflight.execute_arm_preflight(case["plan"], read_consent=lambda: case["consent"], reader=lambda _: pytest.fail("dispatch")) + result = preflight.assess_arm_preflight(case["plan"], {}) + assert result["disabledContentVerified"] is False + + +def test_unique_string_name_uses_complete_canonical_registration_not_approximation(case): + current = template() + current["resources"][1]["name"] = "[concat(parameters('workspace'), '/Microsoft.SecurityInsights/', uniqueString('fixture'))]" + case["path"].write_text(json.dumps(current)) + arm = copy.deepcopy(case["arm"]) + arm["mainTemplateSha256"] = hashlib.sha256(case["path"].read_bytes()).hexdigest() + arm["resources"] = validation._resource_inventory(current, arm["parameters"], arm["target"]) + seal(arm, "planSha256") + case["plan"] = preflight.plan_arm_preflight(arm) + case["consent"] = preflight.review_arm_preflight(case["plan"], "reviewer", "unique-name", True) + result, calls = execute(case) + assert result["ownershipVerified"] is True + assert all("uniqueString" not in operation.get("url", "") for operation in calls) + + +def test_other_resource_families_are_not_promised_ready(case): + current = template() + current["resources"].append({ + "type": "Microsoft.OperationalInsights/workspaces/providers/alertRules", + "name": "[concat(parameters('workspace'), '/Microsoft.SecurityInsights/disabled-rule')]", + "properties": {"enabled": False}, + }) + case["path"].write_text(json.dumps(current)) + arm = copy.deepcopy(case["arm"]) + arm["mainTemplateSha256"] = hashlib.sha256(case["path"].read_bytes()).hexdigest() + arm["resources"] = validation._resource_inventory(current, arm["parameters"], arm["target"]) + arm["resourceCount"] = len(arm["resources"]) + arm["scope"]["resourceTypes"] = sorted({row["type"] for row in arm["resources"]} | {"Microsoft.Resources/deployments"}) + seal(arm, "planSha256") + case["plan"] = preflight.plan_arm_preflight(arm) + case["consent"] = preflight.review_arm_preflight(case["plan"], "reviewer", "non-workbook", True) + result, _ = execute(case) + assert result["disabledContentVerified"] is True + assert result["ownershipVerified"] is False + assert result["armDeploymentAllowed"] is False + assert "non-workbook resource families" in result["checks"]["ownershipVerified"]["reasons"][0] + + +def test_forged_disabled_inventory_is_not_evidence(case): + current = template() + current["resources"][0]["properties"]["template"]["resources"]["workbook"]["type"] = "Microsoft.Security/detectionRules@2026-06-01-preview" + current["resources"][0]["properties"]["template"]["resources"]["workbook"]["properties"]["isEnabled"] = True + case["path"].write_text(json.dumps(current)) + arm = copy.deepcopy(case["arm"]) + arm["mainTemplateSha256"] = hashlib.sha256(case["path"].read_bytes()).hexdigest() + arm["disabledContentVerified"] = True + seal(arm, "planSha256") + with pytest.raises(ValueError, match="disabled content"): + preflight.plan_arm_preflight(arm) + + +def test_assessment_rejects_copied_flags_instead_of_observations(case): + result = preflight.assess_arm_preflight(case["plan"], { + **{key: True for key in preflight._ACCESS_FLAGS}, + "identityId": PRINCIPAL, "target": TARGET, + }) + assert result["identityId"] is None + assert result["ownershipVerified"] is False + assert result["extensionResourceWriteAllowed"] is False + assert result["status"] == "blocked" + + +def test_observation_target_or_candidate_change_and_duplicate_responses_block(case): + result, _ = execute(case) + observations = copy.deepcopy(result["observations"]) + observations["candidateSha256"] = "0" * 64 + assert preflight.assess_arm_preflight(case["plan"], observations)["ownershipVerified"] is False + observations = copy.deepcopy(result["observations"]) + observations["responses"].append(copy.deepcopy(observations["responses"][0])) + assert preflight.assess_arm_preflight(case["plan"], observations)["ownershipVerified"] is False + + +def test_response_size_bound_and_reader_cannot_assert_access(case, monkeypatch): + monkeypatch.setattr(preflight, "MAX_RESPONSE_BYTES", 400) + case["plan"] = preflight.plan_arm_preflight(case["arm"]) + case["consent"] = preflight.review_arm_preflight(case["plan"], "reviewer", "bounded", True) + def change(operation, response): + response["extensionResourceWriteAllowed"] = True + response["body"]["accessToken"] = "never-persist" + response["body"]["oversized"] = "x" * 500 + result, calls = execute(case, change) + assert len(calls) == 1 + assert result["extensionResourceWriteAllowed"] is False + assert "never-persist" not in json.dumps(result) + + +def test_operation_and_collection_bounds_fail_closed(case, monkeypatch): + monkeypatch.setattr(preflight, "MAX_OPERATIONS", 2) + with pytest.raises(ValueError, match="operation count"): + preflight.plan_arm_preflight(case["arm"]) + monkeypatch.setattr(preflight, "MAX_OPERATIONS", 64) + monkeypatch.setattr(preflight, "MAX_LIST_ITEMS", 0) + case["plan"] = preflight.plan_arm_preflight(case["arm"]) + case["consent"] = preflight.review_arm_preflight(case["plan"], "reviewer", "list-bounds", True) + result, _ = execute(case) + assert result["armDeploymentAllowed"] is False + assert result["ownershipVerified"] is False + + +def test_secret_fields_are_redacted_and_extra_reader_flags_ignored(case): + def change(operation, response): + response["extensionResourceWriteAllowed"] = True + response["accessToken"] = "secret-top-level" + response["body"]["accessToken"] = "secret-body" + response["body"]["client_secret"] = "secret-client" + result, _ = execute(case, change) + assert "secret-top-level" not in json.dumps(result) + assert "secret-body" not in json.dumps(result) + assert "secret-client" not in json.dumps(result) + assert result["extensionResourceWriteAllowed"] is False + + +def test_provenance_and_consent_checks_cannot_be_replaced_with_asserted_flags(case): + result, _ = execute(case) + observations = copy.deepcopy(result["observations"]) + for row in observations["responses"]: + row.pop("provenance", None) + assessed = preflight.assess_arm_preflight(case["plan"], observations) + assert assessed["identityId"] is None + observations = copy.deepcopy(result["observations"]) + observations["consentChecks"] = [{"status": "ready", "reasons": []}] * len(observations["consentChecks"]) + assessed = preflight.assess_arm_preflight(case["plan"], observations) + assert assessed["nonProduction"] is False + assert assessed["checks"]["readConsentVerified"]["status"] == "blocked" + + +def test_module_import_does_not_authenticate_or_dispatch_network(): + script = """ +import platform, socket, subprocess, urllib.request +platform.platform = lambda: "offline-fixture" +def forbidden(*args, **kwargs): + raise AssertionError("import attempted authentication/network/CLI") +socket.socket.connect = forbidden +urllib.request.urlopen = forbidden +subprocess.run = forbidden +import azure.identity +azure.identity.DefaultAzureCredential = forbidden +import sentinel_xdr_migration.workbook_preflight +""" + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, timeout=15) + assert result.returncode == 0, result.stderr diff --git a/Tools/SentinelToXDRMigration/tests/test_workbook_preflight_workflow.py b/Tools/SentinelToXDRMigration/tests/test_workbook_preflight_workflow.py new file mode 100644 index 00000000000..6d1fee13b43 --- /dev/null +++ b/Tools/SentinelToXDRMigration/tests/test_workbook_preflight_workflow.py @@ -0,0 +1,218 @@ +from __future__ import annotations + +import base64 +import io +import json +import time +from types import SimpleNamespace +from unittest import mock + +import pytest + +from sentinel_xdr_migration import cli +from sentinel_xdr_migration import workbook_preflight_workflow as preflight +from test_workbook_arm_validation import TARGET + + +PRINCIPAL = "12345678-1111-2222-3333-123456789abc" +ACCOUNT = {"id": TARGET["subscriptionId"], "tenantId": TARGET["tenantId"]} +ACCOUNT_OPERATION = { + "id": "account", "kind": "azure-cli-account", "method": "LOCAL", + "subscriptionId": TARGET["subscriptionId"], +} +WORKSPACE_OPERATION = { + "id": "workspace", "kind": "arm-get", "method": "GET", + "url": f"https://management.azure.com{TARGET['workspaceResourceId']}?api-version=2023-09-01", +} +GRAPH_OPERATION = { + "id": "workbook", "kind": "graph-get", "method": "GET", + "url": "https://graph.microsoft.com/beta/security/workbooks/12345678-1111-2222-3333-123456789abc", +} + + +def token(**overrides): + claims = { + "oid": PRINCIPAL, "tid": TARGET["tenantId"], + "aud": "https://management.azure.com/", **overrides, + } + encoded = base64.urlsafe_b64encode(json.dumps(claims).encode()).decode().rstrip("=") + return SimpleNamespace(token=f"e30.{encoded}.fixture-only", expires_on=time.time() + 3600) + + +@pytest.fixture +def provider(): + credential = mock.Mock() + credential.get_token.return_value = token() + response = mock.MagicMock() + response.__enter__.return_value = response + response.status = 200 + response.read.return_value = json.dumps({"id": TARGET["workspaceResourceId"]}).encode() + with mock.patch("sentinel_xdr_migration.graph_client_config.azure_cli_credential", return_value=credential), \ + mock.patch.object(preflight, "execution_profile", return_value="/fixture/private-profile"), \ + mock.patch("sentinel_xdr_migration.arm_acceptance_support._run_cli", return_value=(0, ACCOUNT, "")) as account, \ + mock.patch.object(preflight, "_open_read", return_value=response) as network: + read = preflight.default_reader( + TARGET, operations=[ACCOUNT_OPERATION, WORKSPACE_OPERATION, GRAPH_OPERATION], + ) + yield read, credential, account, network + + +def test_reader_creation_is_offline_and_uses_one_observed_arm_principal(provider): + read, credential, account, network = provider + credential.get_token.assert_not_called() + account.assert_not_called() + network.assert_not_called() + observed = read(ACCOUNT_OPERATION) + assert observed["body"]["principal"]["objectId"] == PRINCIPAL + assert "accessToken" not in json.dumps(observed) + assert "fixture-only" not in json.dumps(observed) + result = read(WORKSPACE_OPERATION) + assert result["principalId"] == PRINCIPAL + assert result["tenantId"] == TARGET["tenantId"] + assert network.call_args.args[0].get_method() == "GET" + assert network.call_args.args[0].get_header("Authorization") == f"Bearer {credential.get_token.return_value.token}" + credential.get_token.assert_called_once_with("https://management.azure.com/.default") + + +def test_reader_does_not_infer_extension_auth_from_arm_or_hunting_credentials(provider): + read, credential, account, network = provider + result = read(GRAPH_OPERATION) + assert result["statusCode"] == 0 + assert result["body"]["error"]["code"] == "MicrosoftSecurityIdentityUnverified" + credential.get_token.assert_not_called() + account.assert_not_called() + network.assert_not_called() + + +def test_account_reader_overrides_general_deployment_cli_timeout(provider): + read, _, account, _ = provider + read(ACCOUNT_OPERATION) + runner = account.call_args.kwargs["runner"] + with mock.patch.object(preflight.subprocess, "run") as process: + runner(["az", "account", "show"], timeout=1800, capture_output=True) + assert process.call_args.kwargs["timeout"] == 15 + + +@pytest.mark.parametrize("changes", [ + {"method": "POST"}, {"method": "DELETE"}, + {"url": "https://other.example/subscriptions/fixture"}, + {"url": "https://management.azure.com/subscriptions/other/resourceGroups/test"}, +]) +def test_unplanned_operations_stop_before_authentication_or_network(provider, changes): + read, credential, account, network = provider + with pytest.raises(ValueError, match="frozen approved read plan"): + read({**WORKSPACE_OPERATION, **changes}) + credential.get_token.assert_not_called() + account.assert_not_called() + network.assert_not_called() + + +@pytest.mark.parametrize("claims", [ + {"tid": "aaaaaaaa-1111-2222-3333-444444444444"}, + {"aud": "https://graph.microsoft.com"}, + {"oid": "not-an-object-id"}, +]) +def test_wrong_token_bindings_never_dispatch(provider, claims): + read, credential, _, network = provider + credential.get_token.return_value = token(**claims) + with pytest.raises(ValueError, match="principal|tenant binding"): + read(ACCOUNT_OPERATION) + network.assert_not_called() + + +def test_account_target_mismatch_never_acquires_a_token(provider): + read, credential, account, network = provider + account.return_value = (0, {**ACCOUNT, "tenantId": "wrong"}, "") + result = read(ACCOUNT_OPERATION) + assert result["body"]["tenantId"] == "wrong" + credential.get_token.assert_not_called() + network.assert_not_called() + + +def test_expired_access_token_is_rejected_before_arm_dispatch(provider): + read, credential, _, network = provider + credential.get_token.return_value.expires_on = time.time() - 60 + with pytest.raises(ValueError, match="current principal/tenant"): + read(WORKSPACE_OPERATION) + network.assert_not_called() + + +def test_changed_cached_principal_blocks_before_arm_read(provider): + read, credential, _, network = provider + read(ACCOUNT_OPERATION) + credential.get_token.return_value.expires_on = time.time() - 1 + credential.get_token.return_value = token(oid="aaaaaaaa-1111-2222-3333-444444444444") + with pytest.raises(ValueError, match="principal changed"): + read(WORKSPACE_OPERATION) + network.assert_not_called() + + +def test_http_errors_do_not_export_tokens_or_provider_error_body(provider): + read, _, _, network = provider + network.side_effect = preflight.urllib.error.HTTPError( + WORKSPACE_OPERATION["url"], 403, "Bearer fixture-private", {}, io.BytesIO(b"private"), + ) + result = read(WORKSPACE_OPERATION) + assert result["statusCode"] == 403 + assert result["body"]["error"]["code"] == "ArmReadRejected" + assert "fixture-private" not in json.dumps(result) + + +def test_sdk_auth_failure_is_explicit_and_sanitized(provider): + from azure.core.exceptions import ClientAuthenticationError + + read, credential, _, network = provider + credential.get_token.side_effect = ClientAuthenticationError("Bearer fixture-private") + with pytest.raises(ValueError, match="no sign-in") as exc: + read(ACCOUNT_OPERATION) + assert "fixture-private" not in str(exc.value) + network.assert_not_called() + + +def test_profile_change_blocks_before_auth_or_dispatch(provider): + read, credential, _, network = provider + with mock.patch.object(preflight, "execution_profile", return_value="/different-profile"): + with pytest.raises(ValueError, match="profile changed"): + read(WORKSPACE_OPERATION) + credential.get_token.assert_not_called() + network.assert_not_called() + + +def test_oversized_response_is_rejected_before_json_parsing(provider): + from sentinel_xdr_migration.workbook_preflight import MAX_RESPONSE_BYTES + + read, _, _, network = provider + response = network.return_value.__enter__.return_value + response.read.return_value = b"x" * (MAX_RESPONSE_BYTES + 1) + with pytest.raises(ValueError, match="bounded evidence size"): + read(WORKSPACE_OPERATION) + response.read.assert_called_once_with(MAX_RESPONSE_BYTES + 1) + + +def test_redirects_are_not_followed_by_authenticated_reader(): + request = preflight.urllib.request.Request( + WORKSPACE_OPERATION["url"], headers={"Authorization": "Bearer fixture-only"}, + ) + handler = preflight._RejectRedirect() + assert handler.redirect_request( + request, None, 302, "Found", {}, "https://other.example/collect", + ) is None + with mock.patch.object(preflight.urllib.request, "build_opener") as build: + preflight._open_read(request, timeout=15) + assert isinstance(build.call_args.args[0], preflight._RejectRedirect) + build.return_value.open.assert_called_once_with(request, timeout=15) + + +@pytest.mark.parametrize("name,arguments", [ + ("workflow-arm-preflight-plan", []), + ("workflow-arm-preflight-review", [ + "--preflight-plan-sha256", "a" * 64, "--actor", "fixture", + "--reference", "review", "--decision", "approved", + ]), + ("workflow-arm-preflight-execute", ["--preflight-plan-sha256", "a" * 64]), + ("workflow-arm-preflight-revoke", ["--actor", "fixture", "--reference", "revoke"]), +]) +def test_preflight_cli_requires_explicit_run_without_execution(name, arguments): + with pytest.raises(SystemExit) as exc: + cli.main([name, "--solution", "fixture", *arguments]) + assert exc.value.code == 2 diff --git a/Tools/SentinelToXDRMigration/tests/test_workbook_qualification.py b/Tools/SentinelToXDRMigration/tests/test_workbook_qualification.py new file mode 100644 index 00000000000..4bb014528c6 --- /dev/null +++ b/Tools/SentinelToXDRMigration/tests/test_workbook_qualification.py @@ -0,0 +1,416 @@ +from __future__ import annotations + +from copy import deepcopy +import hashlib +import io +import json +from pathlib import Path +import shutil +from unittest import mock +from uuid import uuid4 + +import pytest +import yaml + +from sentinel_xdr_migration import workbook_qualification as qualification +from sentinel_xdr_migration import workbook_workflow, workflow, runtime, cli +from sentinel_xdr_migration._workbook_core.workbook_mock import inventory +from sentinel_xdr_migration.artifacts import artifact_path, report_directory, using_run +from test_workbook_arm_validation import TARGET + + +@pytest.fixture +def qualified_source(): + scratch = Path.cwd() / f".workbook-qualification-tests-{uuid4().hex}" + root = scratch / "Solutions" / "Fixture" + (root / "Workbooks").mkdir(parents=True) + (root / "Data").mkdir() + (scratch / ".gitignore").write_text("Solutions/**/Logs/\n") + original = Path(__file__).parent / "fixtures/workbooks/sources/Syslog/Workbooks/LinuxMachines.json" + source = root / "Workbooks/LinuxMachines.json" + source.write_bytes(original.read_bytes()) + (root / "Data/Solution_Fixture.json").write_text(json.dumps({ + "Name": "Fixture", "Version": "3.2.3", + "Workbooks": ["Workbooks/LinuxMachines.json"], + })) + state = workflow.initialize_workflow( + root, workflow_profile="qualification", workbook_capability_version=1, + tenant_id=TARGET["tenantId"], subscription_id=TARGET["subscriptionId"], + workspace_resource_id=TARGET["workspaceResourceId"], + workspace_customer_id=TARGET["workspaceCustomerId"], + ) + try: + with using_run(root, state["runId"]): + resolutions = { + row["sourcePath"]: { + "querySha256": row["querySha256"], "tables": row["tables"], + "evidence": "Reviewed complete public synthetic fixture query contract.", + } + for row in inventory(json.loads(source.read_bytes())) + } + config = artifact_path(root, "migration-config.yaml", create_parent=True) + config.write_text(yaml.safe_dump({ + "targetVersion": "3.2.3", + "workbookResolutions": {"Workbooks/LinuxMachines.json": resolutions}, + "workbookDependencies": {"Workbooks/LinuxMachines.json": { + "requiredWorkloads": [{"workload": "sentinel", "dataTypes": ["Syslog"]}], + "requiredDataConnectors": [{"connectorId": "FixtureSyslog", "dataTypes": ["Syslog"]}], + }}, + })) + workflow.start_workflow_stage(root, "discovery") + workflow.complete_workflow_stage(root, "discovery", status="passed") + workflow.start_workflow_stage(root, "conversion") + result = workbook_workflow.convert_content(root) + assert result["workbooks"]["selectedBlocked"] == 0 + workflow.complete_workflow_stage(root, "conversion", status="passed") + yield root, state + finally: + shutil.rmtree(scratch) + + +def query_plan(root): + return qualification.plan_workbook_queries( + root, workbook="XDR Workbooks/LinuxMachines.json", clock="2026-10-07T12:00:00Z", + ) + + +def consent(plan): + return qualification.review_workbook_query_consent( + plan, actor="fixture-reviewer", reference="explicit-fixture-approval", approved=True, + ) + + +def providers(plan): + def provider(side): + expected = { + row["querySha256"]: row["expected"] + for row in plan["queries"] if row["side"] == side + } + + def execute(query, target): + digest = hashlib.sha256(query.encode()).hexdigest() + result = expected[digest] + return { + "status": "passed", "complete": True, "provider": side, + "target": target, "querySha256": digest, + "result": {"columns": deepcopy(result["columns"]), "rows": deepcopy(result["rows"])}, + } + + return execute + + return {side: provider(side) for side in ("sentinel", "advanced-hunting")} + + +def ui_plan(root): + kinds = ("pickers", "dependent-refresh", "filters", "drill-down", "reload", "layout", "conditional-visibility") + return qualification.plan_workbook_ui( + root, workbook="XDR Workbooks/LinuxMachines.json", + url="https://security.microsoft.com/workbooks/fixture", + readiness_selectors={"tenant": "#tenant", "workbook": "#workbook", "ready": "#ready"}, + steps=[{"id": kind, "kind": kind, "action": "observe"} for kind in kinds], + background_activity_review="Explicit synthetic service-worker and background-request review.", + ) + + +def browser(plan): + return { + "target": plan["target"], "url": plan["url"], + "readiness": { + "tenantId": plan["target"]["tenantId"], "contentId": plan["contentId"], + "instanceId": plan["instanceId"], "ready": True, + }, + "steps": [{"id": step["id"], "kind": step["kind"], "status": "passed", "evidence": "Synthetic browser fixture observation."} for step in plan["steps"]], + "queryErrors": [], "javascriptErrors": [], "saveEditObserved": False, + "backgroundActivityReview": plan["backgroundActivityReview"], + } + + +def test_plan_binds_all_exact_parameter_cases_and_independent_diagnostics(qualified_source): + root, state = qualified_source + plan = query_plan(root) + assert len(plan["queries"]) == 202 + for side in ("sentinel", "advanced-hunting"): + assert len({row["caseId"] for row in plan["queries"] if row["side"] == side}) == 101 + assert plan["runId"] == state["runId"] + assert plan["ingestion"] == "not-ingestible" + assert query_plan(root) == plan + assert plan["targetSha256"] == hashlib.sha256((root / plan["workbook"]).read_bytes()).hexdigest() + + +def test_workbook_only_content_validation_does_not_require_installed_ui(qualified_source): + root, state = qualified_source + assert state["context"]["workbookQualificationAfterArm"] is True + workbook_workflow.validate_content(root) + workflow.start_workflow_stage(root, "validation") + workflow.complete_workflow_stage(root, "validation", status="passed") + current = workflow._load(root)[1] + artifacts = current["stages"]["validation"]["artifacts"] + assert artifacts["validationMode"] == "workbook-content" + assert artifacts["workbookQueryStatus"] == artifacts["workbookUiStatus"] == "not-run" + with pytest.raises(ValueError): + qualification.require_current_qualification(root, current) + current["stages"]["report"]["status"] = "running" + state_path = workflow._load(root)[0] + workflow._write(state_path, current) + with pytest.raises(ValueError, match="query and UI qualification evidence is missing"): + workflow.complete_workflow_stage(root, "report", status="passed") + + +def test_complete_exact_execution_and_ui_are_separate(qualified_source): + root, state = qualified_source + plan = query_plan(root) + approval = consent(plan) + result = qualification.execute_workbook_queries( + root, plan, read_consent=lambda: approval, providers=providers(plan), + ) + assert result["cases"] == 202 + assert result["uiStatus"] == "not-run" + with pytest.raises(ValueError, match="UI qualification evidence is missing"): + qualification.require_current_qualification(root, workflow._load(root)[1]) + portal = ui_plan(root) + qualification.execute_workbook_ui(root, portal, read_consent=lambda: consent(portal), browser_runner=browser) + qualification.require_current_qualification(root, workflow._load(root)[1]) + + +@pytest.mark.parametrize("change", ["declined", "withdrawn", "hash", "target"]) +def test_query_consent_blocks_before_provider_dispatch(qualified_source, change): + root, _ = qualified_source + plan = query_plan(root) + approval = consent(plan) + if change == "declined": + approval["approved"] = False + elif change == "withdrawn": + approval["withdrawn"] = True + elif change == "hash": + approval["planSha256"] = "0" * 64 + else: + approval["target"] = {**approval["target"], "tenantId": str(uuid4())} + provider = mock.Mock() + with pytest.raises(ValueError, match="consent"): + qualification.execute_workbook_queries( + root, plan, read_consent=lambda: approval, + providers={"sentinel": provider, "advanced-hunting": mock.Mock()}, + ) + provider.assert_not_called() + + +def test_edited_plan_and_target_cannot_dispatch(qualified_source): + root, _ = qualified_source + plan = query_plan(root) + approval = consent(plan) + plan["queries"][0]["query"] += "\n| take 0" + provider = mock.Mock() + with pytest.raises(ValueError, match="stale or has been edited"): + qualification.execute_workbook_queries( + root, plan, read_consent=lambda: approval, + providers={"sentinel": provider, "advanced-hunting": mock.Mock()}, + ) + provider.assert_not_called() + + +@pytest.mark.parametrize("fault", ["partial", "wrong-query", "wrong-target", "cells"]) +def test_provider_errors_and_exact_result_mismatch_persist_failure(qualified_source, fault): + root, _ = qualified_source + plan = query_plan(root) + callbacks = providers(plan) + original = callbacks["sentinel"] + + def faulty(query, target): + result = original(query, target) + if fault == "partial": + result["complete"] = False + elif fault == "wrong-query": + result["querySha256"] = "0" * 64 + elif fault == "wrong-target": + result["target"] = {**target, "tenantId": str(uuid4())} + else: + result["result"]["rows"] = [] + return result + + callbacks["sentinel"] = faulty + with pytest.raises(ValueError): + qualification.execute_workbook_queries(root, plan, read_consent=lambda: consent(plan), providers=callbacks) + receipt = json.loads((report_directory(root) / f"workbook-queries.{plan['contentId']}.json").read_bytes()) + assert receipt["status"] == "failed" + assert receipt["results"] == [] + + +@pytest.mark.parametrize("fault", ["save", "javascript", "missing-step", "tenant", "instance", "ui-query-consent"]) +def test_ui_errors_incomplete_steps_and_wrong_consent_are_not_qualification(qualified_source, fault): + root, _ = qualified_source + plan = ui_plan(root) + approval = consent(query_plan(root)) if fault == "ui-query-consent" else consent(plan) + + def faulty(current): + capture = browser(current) + if fault == "save": + capture["saveEditObserved"] = True + elif fault == "javascript": + capture["javascriptErrors"] = ["Synthetic error"] + elif fault == "missing-step": + capture["steps"].pop() + elif fault == "tenant": + capture["readiness"]["tenantId"] = str(uuid4()) + elif fault == "instance": + capture["readiness"]["instanceId"] = str(uuid4()) + return capture + + with pytest.raises(ValueError): + qualification.execute_workbook_ui(root, plan, read_consent=lambda: approval, browser_runner=faulty) + + +def test_query_schema_normalization_preserves_exact_values(): + result = qualification._normalize_result( + [{"name": "TimeGenerated", "type": "DateTime"}, {"name": "Count", "type": "Int64"}], + [{"TimeGenerated": "2026-10-07T12:00:00+00:00", "Count": 2}], + ) + assert result == { + "columns": [{"name": "TimeGenerated", "type": "datetime"}, {"name": "Count", "type": "long"}], + "rows": [["2026-10-07T12:00:00Z", 2]], + } + with pytest.raises(ValueError, match="timezone-less"): + qualification._normalize_result([{"name": "Time", "type": "datetime"}], [["2026-10-07T12:00:00"]]) + with pytest.raises(ValueError, match="unsupported"): + qualification._normalize_result([{"name": "Count", "type": "Unknown"}], [[2]]) + + +def test_graph_full_capture_is_opt_in_and_never_uses_truncated_sample(): + rows = [{"Count": index} for index in range(runtime.MAX_ADVANCED_HUNTING_SAMPLE_ROWS + 1)] + body = {"schema": [{"name": "Count", "type": "Int64"}], "results": rows} + response = mock.MagicMock() + response.__enter__.return_value = response + response.status = 200 + response.read.return_value = json.dumps(body).encode() + with mock.patch.object(runtime.urllib.request, "urlopen", return_value=response): + old = runtime.run_advanced_hunting_query("datatable(Count:long)[1]", _token="fixture-only") + complete = runtime.run_advanced_hunting_query( + "datatable(Count:long)[1]", _token="fixture-only", capture_full_results=True, + ) + assert "results" not in old + assert old["sampleRowsTruncated"] is True + assert complete["results"] == rows + with pytest.raises(ValueError, match="boolean"): + runtime.run_advanced_hunting_query("query", capture_full_results="true") + + +def cli_run(arguments): + stdout, stderr = io.StringIO(), io.StringIO() + with mock.patch("sys.stdout", stdout), mock.patch("sys.stderr", stderr): + code = cli.main(arguments) + return code, json.loads(stdout.getvalue()) if stdout.getvalue() else None, stderr.getvalue() + + +def test_qualification_cli_exact_plan_review_execute_and_withdrawal(qualified_source): + root, state = qualified_source + common = ["--solution", str(root), "--run-id", state["runId"]] + code, planned, error = cli_run([ + "workbook-query-plan", *common, + "--workbook", "XDR Workbooks/LinuxMachines.json", "--clock", "2026-10-07T12:00:00Z", + ]) + assert code == 0, error + assert planned["queryCount"] == 202 + binding = [ + "--plan", planned["planPath"], "--plan-sha256", planned["planSha256"], + ] + code, _, error = cli_run([ + "workbook-query-review", *common, *binding, + "--actor", "fixture-reviewer", "--reference", "explicit-fixture-approval", "--decision", "approved", + ]) + assert code == 0, error + with mock.patch.object(qualification, "default_query_providers", return_value=providers(query_plan(root))): + code, executed, error = cli_run(["workbook-query-execute", *common, *binding]) + assert code == 0, error + assert executed["cases"] == 202 + code, revoked, error = cli_run([ + "workbook-query-revoke", *common, "--plan", planned["planPath"], + "--actor", "fixture-reviewer", "--reference", "explicit-withdrawal", + ]) + assert code == 0, error + assert revoked["status"] == "withdrawn" + with mock.patch.object(qualification, "default_query_providers", return_value=providers(query_plan(root))): + code, _, error = cli_run(["workbook-query-execute", *common, *binding]) + assert code == 2 + assert "withdrawn" in error + + +def test_qualification_cli_requires_exact_run_and_declines_authoring(qualified_source): + root, state = qualified_source + with pytest.raises(SystemExit) as error: + cli.main(["workbook-query-plan", "--solution", str(root), + "--workbook", "XDR Workbooks/LinuxMachines.json", "--clock", "2026-10-07T12:00:00Z"]) + assert error.value.code == 2 + state_path, current = workflow._load(root) + current["context"]["workflowProfile"] = "authoring" + current["context"].pop("workbookQualificationAfterArm") + workflow._write(state_path, current) + code, _, error = cli_run([ + "workbook-query-plan", "--solution", str(root), "--run-id", state["runId"], + "--workbook", "XDR Workbooks/LinuxMachines.json", "--clock", "2026-10-07T12:00:00Z", + ]) + assert code == 2 + assert "Qualification run" in error + + +def test_ui_cli_import_and_failed_retry_preserve_current_failure(qualified_source): + root, state = qualified_source + plan = ui_plan(root) + selectors_path = artifact_path(root, "ui-selectors.json", create_parent=True) + steps_path = artifact_path(root, "ui-steps.json", create_parent=True) + capture_path = artifact_path(root, "ui-capture.json", create_parent=True) + selectors_path.write_text(json.dumps(plan["readinessSelectors"])) + steps_path.write_text(json.dumps(plan["steps"])) + capture_path.write_text(json.dumps(browser(plan))) + common = ["--solution", str(root), "--run-id", state["runId"]] + code, planned, error = cli_run([ + "workbook-ui-plan", *common, "--workbook", plan["workbook"], "--url", plan["url"], + "--readiness-selectors", selectors_path.relative_to(root).as_posix(), + "--steps", steps_path.relative_to(root).as_posix(), + "--background-activity-review", plan["backgroundActivityReview"], + ]) + assert code == 0, error + binding = ["--plan", planned["planPath"], "--plan-sha256", planned["planSha256"]] + reviewer = ["--actor", "fixture-reviewer", "--reference", "explicit-ui-approval"] + code, _, error = cli_run(["workbook-ui-review", *common, *binding, *reviewer, "--decision", "approved"]) + assert code == 0, error + command = ["workbook-ui-import", *common, *binding, *reviewer, + "--evidence", capture_path.relative_to(root).as_posix()] + code, imported, error = cli_run(command) + assert code == 0, error + assert "not executed by this command" in imported["browserExecution"] + bad_capture = browser(plan) + bad_capture["javascriptErrors"] = ["Synthetic failed retry"] + capture_path.write_text(json.dumps(bad_capture)) + code, _, error = cli_run(command) + assert code == 2 + assert "errors" in error + receipt = json.loads(Path(imported["reportPath"]).read_bytes()) + assert receipt["status"] == "failed" + assert list(report_directory(root).glob(f"workbook-ui.{plan['contentId']}.*.json")) + + +def test_provider_overruns_family_deadline_cannot_pass(qualified_source): + root, _ = qualified_source + plan = query_plan(root) + with mock.patch.object(qualification.time, "monotonic", side_effect=(0, 0, 2)): + with pytest.raises(ValueError, match="timeout"): + qualification.execute_workbook_queries( + root, plan, read_consent=lambda: consent(plan), + providers=providers(plan), family_timeout_seconds=1, + ) + receipt = json.loads((report_directory(root) / f"workbook-queries.{plan['contentId']}.json").read_bytes()) + assert receipt["status"] == "failed" + assert receipt["error"] == "query-family-timeout" + + +def test_ui_final_consent_withdrawal_persists_failed_receipt(qualified_source): + root, _ = qualified_source + plan = ui_plan(root) + approval = consent(plan) + withdrawn = {**approval, "withdrawn": True} + with pytest.raises(ValueError, match="withdrawn"): + qualification.execute_workbook_ui( + root, plan, read_consent=mock.Mock(side_effect=[approval, approval, withdrawn]), + browser_runner=browser, + ) + receipt = json.loads((report_directory(root) / f"workbook-ui.{plan['contentId']}.json").read_bytes()) + assert receipt["status"] == "failed" diff --git a/Tools/SentinelToXDRMigration/tests/test_workbook_workflow_integration.py b/Tools/SentinelToXDRMigration/tests/test_workbook_workflow_integration.py new file mode 100644 index 00000000000..f0672ca21ef --- /dev/null +++ b/Tools/SentinelToXDRMigration/tests/test_workbook_workflow_integration.py @@ -0,0 +1,790 @@ +from __future__ import annotations + +from copy import deepcopy +import hashlib +import io +import json +from pathlib import Path +import shutil +import subprocess +from unittest import mock +from uuid import uuid4 +import zipfile + +import pytest +import yaml + +from sentinel_xdr_migration import cli, packaging, workflow, workbook_workflow as integration +from sentinel_xdr_migration import workbook_validation as validation +from sentinel_xdr_migration.artifacts import artifact_path, report_directory, using_run +from sentinel_xdr_migration._workbook_core import workbook +from test_converter import RULE +from test_workbook_arm_validation import TARGET, ACCESS + +ROOT = Path(__file__).resolve().parents[3] +REAL_SUBPROCESS_RUN = subprocess.run + + +@pytest.fixture +def solution(): + scratch = Path.cwd() / (".workbook-integration-tests-" + uuid4().hex) + root = scratch / "Solutions" / "Fixture" + (root / "Workbooks/nested").mkdir(parents=True) + (root / "Data").mkdir() + (root / "Data/Solution_Fixture.json").write_text(json.dumps({ + "Name": "Fixture", "Version": "3.2.2", "Workbooks": ["Workbooks/nested/source.json"], + })) + (root / "Workbooks/nested/source.json").write_text(json.dumps({ + "version": "Notebook/1.0", "fromTemplateId": str(uuid4()), + "metadata": {"name": "Fixture", "version": "1.0.0"}, + "items": [{"type": 3, "content": { + "query": "DeviceEvents | summarize count() by ActionType", + "queryType": 0, "resourceType": workbook.LA, + }}], + })) + try: + yield root + finally: + shutil.rmtree(scratch) + + +def initialize(root, **kwargs): + return workflow.initialize_workflow(root, workflow_profile="authoring", + workbook_capability_version=1, version_bump="none", **kwargs) + + +def config(root): + source = root / "Workbooks/nested/source.json" + if not source.is_file(): + return + query = json.loads(source.read_bytes())["items"][0]["content"]["query"] + path = artifact_path(root, "migration-config.yaml", create_parent=True) + path.write_text(yaml.safe_dump({"workbookResolutions": { + "Workbooks/nested/source.json": {"/items/0/content/query": { + "querySha256": hashlib.sha256(query.encode()).hexdigest(), "tables": ["DeviceEvents"], + "evidence": "Reviewed exact complete source query in offline fixture", + }}, + }})) + + +def content_pass(root): + config(root) + workflow.start_workflow_stage(root, "discovery") + inspection = integration.inspect_content(root) + workflow.complete_workflow_stage(root, "discovery", status="passed") + workflow.start_workflow_stage(root, "conversion") + result = integration.convert_content(root) + artifacts = {} + manifest = report_directory(root) / "manifest.json" + if manifest.is_file(): + artifacts = {"manifest": str(manifest), "manifestSha256": hashlib.sha256(manifest.read_bytes()).hexdigest()} + workflow.complete_workflow_stage(root, "conversion", status="passed", artifacts=artifacts) + integration.validate_content(root) + workflow.start_workflow_stage(root, "validation") + workflow.complete_workflow_stage(root, "validation", status="passed", + artifacts={"validationMode": "offline-authoring", "runtimeStatus": "not-run"}) + return inspection, result + + +def cli_run(arguments): + stdout, stderr = io.StringIO(), io.StringIO() + with mock.patch("sys.stdout", stdout), mock.patch("sys.stderr", stderr): + code = cli.main(arguments) + return code, json.loads(stdout.getvalue()) if stdout.getvalue() else None, stderr.getvalue() + + +def test_new_cli_defaults_and_resume_do_not_upgrade_library_runs(solution): + legacy = workflow.initialize_workflow(solution, workflow_profile="authoring") + assert "workbookCapabilityVersion" not in legacy["context"] + code, resumed, error = cli_run(["workflow-init", "--solution", str(solution), "--workflow-profile", "authoring"]) + assert code == 0, error + assert "workbooks" not in resumed + code, _, error = cli_run(["workflow-init", "--solution", str(solution), "--workflow-profile", "authoring", + "--arm-validation-method", "lab-deployment"]) + assert code == 2 + assert "new run" in error + code, current, error = cli_run(["workflow-init", "--solution", str(solution), "--workflow-profile", "authoring", "--new-run"]) + assert code == 0, error + assert current["context"]["workbookCapabilityVersion"] == 1 + assert current["context"]["armValidationMethod"] == "lab-deployment" + with using_run(solution, current["runId"]): + assert initialize(solution)["resumed"] is True + with pytest.raises(ValueError, match="new run"): + initialize(solution, arm_validation_method="arm-ttk") + + +def test_explicit_authoring_arm_ttk_is_frozen(solution): + state = initialize(solution, arm_validation_method="arm-ttk") + assert state["context"]["armValidationMethod"] == "arm-ttk" + assert initialize(solution)["context"]["armValidationMethod"] == "arm-ttk" + with pytest.raises(ValueError, match="new run"): + initialize(solution, arm_validation_method="lab-deployment") + + +@pytest.mark.parametrize("change", ["bytes", "add", "remove"]) +def test_frozen_recursive_sources_reject_drift(solution, change): + state = initialize(solution) + assert state["workbooks"]["inventory"][0]["path"] == "Workbooks/nested/source.json" + source = solution / "Workbooks/nested/source.json" + if change == "bytes": + source.write_bytes(source.read_bytes() + b"\n") + elif change == "remove": + source.unlink() + else: + document = json.loads(source.read_bytes()) + document["fromTemplateId"] = str(uuid4()) + (source.parent / "new.json").write_text(json.dumps(document)) + for action in (lambda: initialize(solution), lambda: integration.inspect_content(solution), + lambda: integration.convert_content(solution)): + with pytest.raises(ValueError, match="inventory changed"): + action() + + +def test_zero_source_inventory_is_frozen_without_fake_detection_manifest(solution): + (solution / "Workbooks/nested/source.json").unlink() + state = initialize(solution) + assert state["workbooks"]["inventory"] == [] + inspection, result = content_pass(solution) + assert inspection["workbookCount"] == inspection["analyticRuleCount"] == 0 + assert result["workbooks"]["total"] == result["total"] == 0 + assert not (report_directory(solution) / "manifest.json").exists() + (solution / "Workbooks/nested/new.json").write_text('{"version":"Notebook/1.0","items":[]}') + with pytest.raises(ValueError, match="inventory changed"): + initialize(solution) + + +def test_workbook_only_offline_authoring_is_first_class(solution): + state = initialize(solution) + inspection, conversion = content_pass(solution) + assert inspection["workbookCount"] == inspection["workbookQueryCount"] == 1 + assert inspection["analyticRuleCount"] == 0 + assert conversion["workbooks"]["converted"] == 1 + assert not (solution / "Analytic Rules").exists() + assert not (report_directory(solution) / "manifest.json").exists() + current = workflow.workflow_status(solution) + assert current["stages"]["validation"]["status"] == "passed" + assert current["workbookSummary"]["selectedCount"] == 1 + assert state["workbooks"]["selection"]["included"] == ["Workbooks/nested/source.json"] + assert current["stages"]["conversion"]["artifacts"]["workbookManifestSha256"] + assert current["runtimeAssessment"]["runtimeStatus"] == "not-run" + + +def test_reports_do_not_ignore_invalid_versioned_workbook_state(solution): + from sentinel_xdr_migration.solution_report import build_solution_report + + initialize(solution) + state_path, state = workflow._load(solution) + del state["workbooks"] + state_path.write_text(json.dumps(state)) + with pytest.raises(ValueError, match="workbooks.*required property"): + build_solution_report(solution) + + +def test_explicit_empty_selection_remains_empty_and_exclusions_visible(solution): + data = solution / "Data/Solution_Fixture.json" + document = json.loads(data.read_bytes()) + document["XDR Workbooks"] = [] + data.write_text(json.dumps(document)) + state = initialize(solution) + assert state["workbooks"]["selection"]["included"] == [] + assert state["workbooks"]["selection"]["excluded"][0]["reason"] + inspection, result = content_pass(solution) + assert json.loads(data.read_bytes())["XDR Workbooks"] == [] + assert result["workbookExcluded"] == state["workbooks"]["selection"]["excluded"] + assert integration.validate_content(solution)["workbooks"]["total"] == 0 + assert inspection["workbookCount"] == 1 + + +def test_blocked_default_sources_are_not_silently_removed(solution): + initialize(solution) + workflow.start_workflow_stage(solution, "discovery") + workflow.complete_workflow_stage(solution, "discovery", status="passed") + workflow.start_workflow_stage(solution, "conversion") + converted = integration.convert_content(solution) + assert converted["workbooks"]["selectedBlocked"] == 1 + assert json.loads((solution / "Data/Solution_Fixture.json").read_bytes())["XDR Workbooks"] == ["XDR Workbooks/nested/source.json"] + workflow.complete_workflow_stage(solution, "conversion", status="passed") + with pytest.raises(ValueError, match="review blockers"): + integration.validate_content(solution) + workflow.start_workflow_stage(solution, "validation") + with pytest.raises(ValueError, match="review blockers"): + workflow.complete_workflow_stage(solution, "validation", status="passed", + artifacts={"runtimeStatus": "not-run", "validationMode": "offline-authoring"}) + assert workflow.workflow_status(solution)["stages"]["validation"]["status"] != "passed" + + +@pytest.mark.parametrize("change", ["output", "manifest", "queryEvidence", "selection"]) +def test_exact_manifest_outputs_and_query_review_bindings(solution, change): + initialize(solution) + content_pass(solution) + if change == "output": + output = solution / "XDR Workbooks/nested/source.json" + output.write_bytes(output.read_bytes() + b"\n") + elif change in {"manifest", "queryEvidence"}: + manifest = report_directory(solution) / "workbooks-manifest.json" + document = json.loads(manifest.read_bytes()) + if change == "queryEvidence": + document["results"][0]["queries"][0]["resolutionEvidence"] = {"evidence": "unbound changed evidence"} + else: + document["added"] = True + manifest.write_text(json.dumps(document)) + else: + data = solution / "Data/Solution_Fixture.json" + document = json.loads(data.read_bytes()) + document["XDR Workbooks"] = [] + data.write_text(json.dumps(document)) + with pytest.raises(ValueError): + integration.validate_content(solution) + + +def test_frozen_selection_rejects_state_edits_and_edited_existing_outputs(solution): + state = initialize(solution) + state["workbooks"]["selection"]["included"] = [] + with pytest.raises(ValueError, match="selection changed"): + integration.verify_inventory(solution, state) + content_pass(solution) + output = solution / "XDR Workbooks/nested/source.json" + output.write_bytes(output.read_bytes() + b"\n") + before = output.read_bytes() + result = integration.convert_content(solution, overwrite=True) + assert result["workbooks"]["selectedBlocked"] == 1 + assert result["workbooks"]["results"][0]["status"] == "conflict" + assert output.read_bytes() == before + + +def test_selected_release_not_notebook_format_and_no_silent_pre31_promotion(solution): + data = solution / "Data/Solution_Fixture.json" + document = json.loads(data.read_bytes()) + document["Version"] = "3.0.8" + data.write_text(json.dumps(document)) + with pytest.raises(ValueError, match=">= 3.2.0"): + initialize(solution) + workflow.initialize_workflow( + solution, workflow_profile="authoring", workbook_capability_version=1, + ) + config(solution) + result = integration.convert_content(solution) + assert result["workbooks"]["selectedBlocked"] == 1 + assert json.loads(data.read_bytes())["Version"] == "3.0.8" + next_state = initialize(solution, new_run=True, target_version="3.2.9") + with using_run(solution, next_state["runId"]): + config(solution) + # A previously generated unbound draft cannot be overwritten by a new run. + (solution / "XDR Workbooks/nested/source.json").unlink() + result = integration.convert_content(solution) + authored = json.loads((solution / "XDR Workbooks/nested/source.json").read_bytes()) + assert result["workbooks"]["selectedBlocked"] == 0 + assert authored["version"] == "Notebook/1.0" + assert authored["xdrMetadata"]["version"] == "3.2.9" + assert json.loads(data.read_bytes())["Version"] == "3.0.8" + + +def test_combined_solution_keeps_detection_gates_and_inventory_separate(solution): + rules = solution / "Analytic Rules" + rules.mkdir() + (rules / "rule.yaml").write_text(RULE) + initialize(solution) + inspection, result = content_pass(solution) + assert inspection["analyticRuleCount"] == inspection["workbookCount"] == 1 + assert result["total"] == result["workbooks"]["total"] == 1 + current = workflow.workflow_status(solution) + artifacts = current["stages"]["conversion"]["artifacts"] + assert artifacts["manifestSha256"] != artifacts["workbookManifestSha256"] + (report_directory(solution) / "manifest.json").unlink() + with pytest.raises(ValueError): + workflow._validate_offline_authoring_gates(solution, Path(current["statePath"]), workflow._load(solution)[1]) + + +def test_family_overwrite_replaces_managed_workbook_not_detection_conflicts(solution): + rules = solution / "Analytic Rules" + rules.mkdir() + (rules / "rule.yaml").write_text(RULE) + initialize(solution) + config(solution) + first = integration.convert_content(solution) + assert first["workbooks"]["converted"] == 1 + detection = next((solution / "XDR Detections").glob("*.yaml")) + detection.write_text(detection.read_text() + "\n# Independent authored change\n") + before = detection.read_bytes() + settings_path = artifact_path(solution, "migration-config.yaml") + settings = yaml.safe_load(settings_path.read_bytes()) + settings["workbookResolutions"]["Workbooks/nested/source.json"]["/items/0/content/query"]["evidence"] += " updated" + settings_path.write_text(yaml.safe_dump(settings)) + code, result, error = cli_run(["convert", "--solution", str(solution), + "--config", str(settings_path), "--overwrite-workbooks"]) + assert code == 1, error # Detection conflict is still a real gate. + assert result["conflicts"] == 1 + assert result["workbooks"]["converted"] == 1 + assert detection.read_bytes() == before + output = solution / "XDR Workbooks/nested/source.json" + output.write_text(output.read_text() + "\n") + workbook_before = output.read_bytes() + settings["workbookResolutions"]["Workbooks/nested/source.json"]["/items/0/content/query"]["evidence"] += " again" + settings_path.write_text(yaml.safe_dump(settings)) + result = integration.convert_content(solution, config_path=settings_path, overwrite_workbooks=True) + assert result["workbooks"]["results"][0]["status"] == "conflict" + assert output.read_bytes() == workbook_before + assert detection.read_bytes() == before + + +def test_family_overwrite_rejects_unfrozen_legacy_run(solution): + workflow.initialize_workflow(solution, workflow_profile="authoring") + with pytest.raises(ValueError, match="workbook-capable"): + integration.convert_content(solution, overwrite_workbooks=True) + + +def test_workspace_function_fingerprint_and_stale_parser_block_workflow(solution): + from sentinel_xdr_migration.workbook_functions import candidates + parser = solution / "Parsers/source.yaml" + parser.parent.mkdir() + parser.write_text("FunctionName: ReviewedSource\nFunctionQuery: DeviceEvents | count\n") + data_path = solution / "Data/Solution_Fixture.json" + data = json.loads(data_path.read_bytes()) + data["Parsers"] = ["Parsers/source.yaml"] + data_path.write_text(json.dumps(data)) + source = solution / "Workbooks/nested/source.json" + document = json.loads(source.read_bytes()) + document["items"][0]["content"]["query"] = "ReviewedSource() | count" + source.write_text(json.dumps(document)) + state = initialize(solution) + record = {key: candidates(solution)[0][key] for key in ("functionAlias", "sourcePath", "sourceSha256")} + settings = {"workbookResolutions": {"Workbooks/nested/source.json": { + "/items/0/content/query": {"querySha256": workbook.query_hash("ReviewedSource() | count"), + "tables": ["DeviceEvents"], "workspaceFunctions": ["ReviewedSource"], + "evidence": "Reviewed exact fixture"}}}, + "workbookDependencies": {"Workbooks/nested/source.json": {"requiredWorkspaceFunctions": [record]}}} + settings_path = artifact_path(solution, "migration-config.yaml", create_parent=True) + settings_path.write_text(yaml.safe_dump(settings)) + workflow.start_workflow_stage(solution, "discovery") + integration.inspect_content(solution) + workflow.complete_workflow_stage(solution, "discovery", status="passed") + workflow.start_workflow_stage(solution, "conversion") + result = integration.convert_content(solution, config_path=settings_path) + assert result["workbooks"]["converted"] == 1 + assert len(result["workbookArtifacts"]["workbookFunctionDependenciesSha256"]) == 64 + workflow.complete_workflow_stage(solution, "conversion", status="passed") + state = workflow._load(solution)[1] + assert integration.validate_evidence(solution, state)["valid"] == 1 + parser.write_text(parser.read_text() + "# changed\n") + with pytest.raises(ValueError, match="stale"): + integration.bind_conversion(solution, state) + + +def test_workbook_only_yolo_uses_budget_without_invented_rule_guids(solution): + state = initialize(solution, yolo=True, campaign_id="workbook-fixture") + with using_run(solution, state["runId"]): + config(solution) + result = workflow.run_yolo_authoring(solution) + assert result["status"] == "inProgress" + current = workflow._load(solution)[1] + assert "authoring" not in current + assert current["context"]["workbookFullInventory"] is True + assert current["stages"]["validation"]["status"] == "passed" + assert result["workbookSummary"]["selectedCount"] == 1 + assert "coverage" not in result + assert initialize(solution, yolo=True, campaign_id="workbook-fixture")["resumed"] is True + + +def test_schema_explicit_capability_cannot_upgrade_legacy_semantics(solution): + state = workflow.initialize_workflow(solution, workflow_profile="authoring") + for addition in ({"armValidationMethod": "lab-deployment"}, + {"workbookCapabilityVersion": 1}): + invalid = deepcopy(state) + invalid.pop("statePath", None) + invalid.pop("resumed", None) + invalid.pop("next", None) + invalid["context"].update(addition) + with pytest.raises(ValueError, match="invalid workflow state"): + workflow._validate(invalid) + + +@pytest.fixture +def emitted(): + from test_defender_workbooks_packaging import DefenderWorkbookPackagingTests + harness = DefenderWorkbookPackagingTests() + harness.setUp() + try: + harness.notebook["xdrMetadata"]["version"] = harness.data["Version"] + harness.write() + (harness.solution / "Data").mkdir() + (harness.solution / "Data/Solution_Sample.json").write_text(json.dumps({"Name": "Sample", **harness.data})) + result = harness.invoke(package=True) + yield harness, result + finally: + harness.tearDown() + + +def test_python_verifies_full_install_registration_stored_template_and_zip(emitted): + harness, result = emitted + package = harness.solution / "Package" + template = json.loads((package / "mainTemplate.json").read_bytes()) + receipt = packaging._verify_workbook_package(harness.solution, [harness.target], + template, package / "3.2.7.zip") + assert receipt["count"] == 1 + assert receipt["contractSupport"] == "unverified" + assert template["parameters"]["DeployWorkbook"]["defaultValue"] is True + assert template["parameters"]["DeployDefenderWorkbook"]["defaultValue"] is False + + +@pytest.mark.parametrize("change", [None, "clip-newline-eof", "missing-dependency", "conditional-parser", "parameter-drift", "stale-source"]) +def test_python_rechecks_required_workspace_function_package_contract(emitted, change): + harness, _ = emitted + harness.add_function() + if change == "clip-newline-eof": + path = harness.solution / "Parsers/ReviewedParser.yaml" + path.write_text(path.read_text().rstrip("\n")) + record = harness.notebook["contentProvenance"]["requiredWorkspaceFunctions"][0] + record["sourceSha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + harness.write() + (harness.solution / "Data/Solution_Sample.json").write_text(json.dumps({"Name": "Sample", **harness.data})) + harness.invoke(package=True) + package = harness.solution / "Package" + template = json.loads((package / "mainTemplate.json").read_bytes()) + installation = next(row for row in template["resources"] if row["type"] == "Microsoft.Resources/deployments") + parser = next(row for row in template["resources"] if row["type"].endswith("/savedSearches")) + if change == "missing-dependency": + installation["dependsOn"] = installation["dependsOn"][:1] + elif change == "conditional-parser": + parser["condition"] = True + elif change == "parameter-drift": + parser["properties"]["functionParameters"] = "limit:int=2" + elif change == "stale-source": + path = harness.solution / "Parsers/ReviewedParser.yaml" + path.write_text(path.read_text() + "# changed\n") + if change not in (None, "clip-newline-eof"): + with pytest.raises((ValueError, RuntimeError)): + packaging._verify_workbook_package(harness.solution, [harness.target], template, package / "3.2.7.zip") + else: + assert packaging._verify_workbook_package( + harness.solution, [harness.target], template, package / "3.2.7.zip")["count"] == 1 + + +@pytest.mark.parametrize("change", ["provenance", "registration", "storedTemplate", "install", "dependency", "zip"]) +def test_python_rejects_wrong_workbook_package_projection(emitted, change): + harness, result = emitted + package = harness.solution / "Package" + template = json.loads((package / "mainTemplate.json").read_bytes()) + registration = next(row for row in template["resources"] if row.get("properties", {}).get("contentKind") == "DefenderWorkbook") + install = next(row for row in template["resources"] if row.get("name", "").startswith("DefenderWorkbook-")) + if change == "provenance": + registration["properties"]["contentProvenance"]["packageVersion"] = "3.2.8" + elif change == "registration": + registration["condition"] = "[parameters('DeployWorkbook')]" + elif change == "storedTemplate": + registration["properties"]["mainTemplate"]["languageVersion"] = "1.0" + elif change == "install": + install["properties"]["template"]["resources"]["workbook"]["properties"]["id"] = harness.notebook["xdrMetadata"]["contentId"] + elif change == "dependency": + template["resources"][0]["properties"]["dependencies"]["criteria"] = [] + else: + with zipfile.ZipFile(package / "3.2.7.zip", "w") as archive: + archive.writestr("mainTemplate.json", "{}") + with pytest.raises(RuntimeError): + packaging._verify_workbook_package(harness.solution, [harness.target], template, package / "3.2.7.zip") + + +def test_python_lab_method_forwarding_is_candidate_not_accepted(emitted): + harness, result = emitted + # The PowerShell emitter's synthetic source is not a converter fixture; + # its standalone tests already verify source/query freshness. + with mock.patch.object(packaging, "_validate_v31_toolkit_root", return_value=ROOT), \ + mock.patch("sentinel_xdr_migration.workbooks.validate_workbooks", return_value={"valid": 1, "invalid": 0}), \ + mock.patch.object(packaging.subprocess, "run", return_value=subprocess.CompletedProcess( + [], 0, "Starting Package Creation using V3.1 tool", "")) as run: + receipt = packaging.package_solution_v3_1(harness.solution, version_bump="none", + toolkit_root=ROOT, arm_validation_method="lab-deployment") + command = run.call_args.args[0] + assert command[command.index("-ArmValidationMethod") + 1] == "lab-deployment" + assert receipt["packageAccepted"] is False + assert receipt["packageLifecycleStatus"] == "candidate-generated" + assert receipt["xdrWorkbookCount"] == 1 + assert receipt["deploymentParameters"]["DeployDefenderWorkbook"] is False + + +def test_missing_receipts_and_unaccepted_candidates_cannot_pass_packaging(solution): + initialize(solution) + content_pass(solution) + workflow.start_workflow_stage(solution, "packaging") + with pytest.raises(ValueError, match="packageReport"): + workflow.complete_workflow_stage(solution, "packaging", status="passed", artifacts={"packager": "V3.1"}) + with pytest.raises(ValueError): + integration.require_accepted(solution, workflow._load(solution)[1]) + + +def test_local_lifecycle_commands_require_explicit_run_and_no_implicit_write_consent(solution): + with pytest.raises(SystemExit) as exc: + cli.main(["workflow-arm-plan", "--solution", str(solution), "--target", "lab.json", "--selected-flags", "flags.json"]) + assert exc.value.code == 2 + + +def emit_current_package(root): + from test_defender_workbooks_packaging import literal + data_path = root / "Data/Solution_Fixture.json" + data = json.loads(data_path.read_bytes()) + payload = report_directory(root) / "package-fixture-input.json" + payload.write_text(json.dumps({ + "data": data, + "template": { + "parameters": {"workspace": {"type": "string"}}, + "variables": {"solutionId": "fixture.publisher", "_solutionId": "[variables('solutionId')]", + "_solutionVersion": data["Version"]}, + "resources": [{ + "type": "Microsoft.OperationalInsights/workspaces/providers/contentPackages", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_solutionId'))]", + "properties": {"contentKind": "Solution", "dependencies": {"criteria": []}}, + }], + }, + "ui": {"parameters": {"outputs": {}, "steps": []}}, + })) + common = ROOT / "Tools/Create-Azure-Sentinel-Solution/common/commonFunctions.ps1" + command = ( + "$ErrorActionPreference='Stop'; " + f". {literal(common)}; " + f"$i=Get-Content -Raw -LiteralPath {literal(payload)} | ConvertFrom-Json; " + f"$count=Add-XdrDefenderWorkbooksToSolution -SolutionName Fixture -ContentToImport $i.data " + f"-Template $i.template -RepositoryRoot {literal(root.parent.parent)}; " + "Add-SolutionContentDeploymentParameters -Template $i.template -UiDefinition $i.ui; " + "Write-Output '__RESULT__'; $i | ConvertTo-Json -Depth 100 -Compress" + ) + completed = REAL_SUBPROCESS_RUN(["pwsh", "-NoProfile", "-NonInteractive", "-Command", command], + capture_output=True, text=True, check=False) + assert completed.returncode == 0, completed.stderr + document = json.loads(completed.stdout.split("__RESULT__", 1)[1]) + package = root / "Package" + package.mkdir(exist_ok=True) + (package / "mainTemplate.json").write_text(json.dumps(document["template"])) + (package / "createUiDefinition.json").write_text(json.dumps(document["ui"])) + (package / "testParameters.json").write_text(json.dumps(document["template"]["parameters"])) + with zipfile.ZipFile(package / f"{data['Version']}.zip", "w") as archive: + for name in ("mainTemplate.json", "createUiDefinition.json"): + archive.write(package / name, name) + + +def package_candidate(root, bump="none"): + def mocked_packager(command, **kwargs): + selected_bump = command[command.index("-VersionBump") + 1] + data_path = root / "Data/Solution_Fixture.json" + data = json.loads(data_path.read_bytes()) + if selected_bump == "patch": + version = list(map(int, data["Version"].split("."))) + version[2] += 1 + data["Version"] = ".".join(map(str, version)) + data_path.write_text(json.dumps(data)) + emit_current_package(root) + return subprocess.CompletedProcess(command, 0, "Starting Package Creation using V3.1 tool", "") + with mock.patch.object(packaging.subprocess, "run", side_effect=mocked_packager): + return workflow.package_workflow_v3_1(root, requested_version_bump=bump, toolkit_root=ROOT) + + +@pytest.fixture +def ready_candidate(solution): + initialize(solution) + content_pass(solution) + package = package_candidate(solution) + receipt = report_directory(solution) / "build-and-validate-report.md" + receipt.write_text("Explicit mocked official receipt; parser producer tests run separately") + binding = {"path": str(receipt), "sha256": hashlib.sha256(receipt.read_bytes()).hexdigest()} + with mock.patch.object(validation, "_parse_build_validation_receipt", return_value=binding): + candidate = integration.create_candidate(solution, validation_receipt=receipt) + yield solution, package, candidate + + +def test_candidate_preserves_draft_and_exact_four_real_gate_receipts(ready_candidate): + root, package, candidate = ready_candidate + assert package["status"] == "blocked" + assert candidate["accepted"] is False + assert candidate["status"] == "candidate-generated" + assert set(candidate["prerequisiteEvidence"]) == {"conversion", "review", "structure", "package"} + for name, reference in candidate["prerequisiteEvidence"].items(): + receipt_path = root / reference["path"] + receipt = json.loads(receipt_path.read_bytes()) + assert receipt["status"] == "passed" + assert receipt["checks"] + assert receipt["packageInputSha256"] == packaging.package_input_identity(root)["sha256"] + assert hashlib.sha256(receipt_path.read_bytes()).hexdigest() == reference["sha256"] + if name in {"structure", "package"}: + assert receipt["packageArtifactsSha256"] == package["packageArtifactsSha256"] + state = workflow._load(root)[1] + assert state["stages"]["packaging"]["status"] == "blocked" + assert state["packaging"]["lastSuccessful"] is None + assert (root / package["mainTemplate"]).is_file() + + +def test_actual_official_receipt_required_before_candidate_creation(solution): + initialize(solution) + content_pass(solution) + package_candidate(solution) + receipt = report_directory(solution) / "build-and-validate-report.md" + receipt.write_text("[REPORT_START]\nResult: CANDIDATE_GENERATED\n[REPORT_END]\n") + with pytest.raises(ValueError, match="repository build-and-validate"): + integration.create_candidate(solution, validation_receipt=receipt) + assert not (report_directory(solution) / "arm-candidate.json").exists() + assert workflow._load(solution)[1]["packaging"]["lastSuccessful"] is None + + +def test_official_producer_method_is_forwarded_without_fabricated_receipt(solution): + initialize(solution) + content_pass(solution) + package_candidate(solution) + with mock.patch.object(subprocess, "run", + return_value=subprocess.CompletedProcess([], 1, "actual validator failure", "")) as producer: + with pytest.raises(RuntimeError, match="actual validator failure"): + integration.create_candidate(solution, toolkit_root=ROOT) + command = producer.call_args.args[0] + assert command[command.index("-ArmValidationMethod") + 1] == "lab-deployment" + assert "-SkipPackaging" in command + assert command[command.index("-ValidationSkip") + 1] == "kql,detection-schema,non-ascii" + assert not (report_directory(solution) / "arm-candidate.json").exists() + + +def test_scope_review_withdrawal_and_immediate_exact_approval_are_independent(ready_candidate): + root, package, candidate = ready_candidate + plan = integration.plan_candidate(root, target=TARGET, selected_flags={"DeployDefenderWorkbook": True}) + with pytest.raises(ValueError, match="displayed plan hash"): + integration.review_scope(root, actor="owner", reference="scope-review", approved=True, expected_plan_sha256="0" * 64) + consent = integration.review_scope(root, actor="owner", reference="scope-review", approved=True, + expected_plan_sha256=plan["planSha256"]) + assert consent["activationAuthorized"] is False + with mock.patch("sentinel_xdr_migration.workbook_preflight_workflow.production_identity_check") as identity, \ + mock.patch("sentinel_xdr_migration.arm_acceptance_support._run_cli") as runner: + with pytest.raises(ValueError, match="immediate exact-plan"): + integration.execute_candidate(root, expected_plan_sha256=plan["planSha256"], actor="owner", + reference="execute-review", confirmation="not the approval") + identity.assert_not_called() + runner.assert_not_called() + withdrawn = integration.withdraw_scope(root, actor="owner", reference="withdraw") + assert withdrawn["status"] == "withdrawn" + with mock.patch("sentinel_xdr_migration.workbook_preflight_workflow.production_identity_check") as identity: + result = integration.execute_candidate(root, expected_plan_sha256=plan["planSha256"], actor="owner", + reference="execute-review", confirmation=validation.LAB_ARM_CONFIRMATION) + identity.assert_not_called() + assert result["status"] == "blocked" + assert result["writeAttempted"] is False + + +def test_write_consent_does_not_implicitly_authorize_preflight_reads(ready_candidate): + from sentinel_xdr_migration import workbook_preflight_workflow as preflight + + root, _, _ = ready_candidate + plan = integration.plan_candidate(root, target=TARGET, selected_flags={"DeployDefenderWorkbook": True}) + integration.review_scope( + root, actor="owner", reference="write-scope", approved=True, + expected_plan_sha256=plan["planSha256"], + ) + preflight.create_plan(root) + with mock.patch("sentinel_xdr_migration.arm_acceptance_support._run_cli") as runner, \ + mock.patch("sentinel_xdr_migration.graph_client_config.azure_cli_credential") as credential: + result = integration.execute_candidate( + root, expected_plan_sha256=plan["planSha256"], actor="owner", + reference="immediate-write", confirmation=validation.LAB_ARM_CONFIRMATION, + ) + assert result["status"] == "blocked" + assert result["writeAttempted"] is False + assert result["identityCheck"]["status"] == "blocked" + runner.assert_not_called() + credential.assert_not_called() + + +def test_preflight_review_execute_revoke_and_archived_receipts(ready_candidate): + from sentinel_xdr_migration import workbook_preflight_workflow as preflight + + root, _, _ = ready_candidate + integration.plan_candidate(root, target=TARGET, selected_flags={"DeployDefenderWorkbook": True}) + displayed = preflight.create_plan(root) + digest = displayed["preflightPlanSha256"] + with pytest.raises(ValueError, match="exact current displayed"): + preflight.review(root, expected_sha256="0" * 64, actor="owner", reference="stale", approved=True) + consent = preflight.review( + root, expected_sha256=digest, actor="owner", reference="read-review", approved=True, + ) + assert consent["writesAuthorized"] is False + reader = mock.Mock(return_value={"statusCode": 403, "body": {}}) + result = preflight.execute(root, expected_sha256=digest, reader=reader) + assert result["status"] == "blocked" + assert result["writeAttempted"] is False + assert reader.call_count == 1 + first = (report_directory(root) / preflight.RESULT_FILE).read_bytes() + withdrawn = preflight.revoke(root, actor="owner", reference="withdraw-reads") + assert withdrawn["status"] == "revoked" + reader.reset_mock() + result = preflight.execute(root, expected_sha256=digest, reader=reader) + assert result["status"] == "blocked" + reader.assert_not_called() + archives = list(report_directory(root).glob("arm-preflight-result.*.json")) + assert len(archives) == 1 + assert archives[0].read_bytes() == first + + +def test_preflight_known_failure_replaces_current_receipt_without_private_details(ready_candidate): + from sentinel_xdr_migration import workbook_preflight_workflow as preflight + + root, _, _ = ready_candidate + integration.plan_candidate(root, target=TARGET, selected_flags={"DeployDefenderWorkbook": True}) + displayed = preflight.create_plan(root) + with mock.patch.object(preflight, "default_reader", side_effect=ValueError("fixture-private-detail")): + with pytest.raises(ValueError, match="fixture-private-detail"): + preflight.execute(root, expected_sha256=displayed["preflightPlanSha256"]) + receipt = json.loads((report_directory(root) / preflight.RESULT_FILE).read_bytes()) + assert receipt["status"] == "failed" + assert receipt["writeAttempted"] is False + assert "fixture-private-detail" not in json.dumps(receipt) + + +def test_preflight_cli_reports_blocked_as_nonzero(ready_candidate, capsys): + root, _, _ = ready_candidate + run_id = workflow._load(root)[1]["runId"] + with mock.patch( + "sentinel_xdr_migration.workbook_preflight_workflow.execute", + return_value={"status": "blocked", "writeAttempted": False}, + ): + code = cli.main([ + "workflow-arm-preflight-execute", "--solution", str(root), + "--run-id", run_id, "--preflight-plan-sha256", "a" * 64, + ]) + assert code != 0 + assert json.loads(capsys.readouterr().out)["status"] == "blocked" + + +def test_selected_flags_only_dispatch_and_arm_success_stays_unaccepted(ready_candidate): + root, package, candidate = ready_candidate + plan = integration.plan_candidate(root, target=TARGET, selected_flags={"DeployDefenderWorkbook": True}) + integration.review_scope(root, actor="owner", reference="scope-review", approved=True, expected_plan_sha256=plan["planSha256"]) + runner = mock.Mock(return_value=(0, {"id": "fixture-deployment", "properties": {"provisioningState": "Succeeded"}}, "")) + result = integration.execute_candidate(root, expected_plan_sha256=plan["planSha256"], actor="owner", + reference="exact-execute", confirmation=validation.LAB_ARM_CONFIRMATION, + identity_check=lambda target: ACCESS, command_runner=runner) + assert result["status"] == "arm-accepted-contract-pending" + assert result["accepted"] is False + assert "DeployDefenderWorkbook=true" in runner.call_args.args[0] + assert not any(flag.startswith("DeployWorkbook=") for flag in runner.call_args.args[0]) + assert workflow._load(root)[1]["packaging"]["lastSuccessful"] is None + with pytest.raises(ValueError, match="independent workbook contract evidence"): + integration.require_accepted(root, workflow._load(root)[1]) + + +def test_consumed_candidate_version_action_retries_without_second_bump(solution): + workflow.initialize_workflow(solution, workflow_profile="authoring", workbook_capability_version=1, version_bump="patch") + content_pass(solution) + first = package_candidate(solution, "patch") + assert first["version"] == "3.2.3" + assert first["packageAccepted"] is False + with pytest.raises(ValueError, match="consumed"): + package_candidate(solution, "patch") + second = package_candidate(solution, "none") + assert second["version"] == first["version"] + assert second["effectiveVersionBump"] == "none" + current = workflow._load(solution)[1] + assert current["stages"]["packaging"]["attempts"] == 2 + assert current["packaging"]["lastSuccessful"] is None + + +def test_workbook_report_keeps_excluded_inventory_outside_rule_coverage(solution): + from sentinel_xdr_migration.solution_report import build_solution_report + initialize(solution) + content_pass(solution) + report = build_solution_report(solution) + assert report["summary"]["rules"] == 0 + assert report["rules"] == [] + assert report["workbooks"]["inventoryCount"] == 1 + assert report["workbooks"]["sources"][0]["selected"] is True diff --git a/Tools/SentinelToXDRMigration/tests/test_workbooks.py b/Tools/SentinelToXDRMigration/tests/test_workbooks.py new file mode 100644 index 00000000000..e677ca8d6b5 --- /dev/null +++ b/Tools/SentinelToXDRMigration/tests/test_workbooks.py @@ -0,0 +1,397 @@ +from __future__ import annotations + +from copy import deepcopy +import hashlib +import json +from pathlib import Path +import shutil +from uuid import uuid4 + +import jsonschema +import pytest + +from sentinel_xdr_migration import workbooks +from sentinel_xdr_migration._workbook_core import workbook, mapping_catalog, migrate + +FIXTURES = Path(__file__).parent / "fixtures/workbooks" +SCHEMA = Path(__file__).parents[1] / "schema/xdr-workbook.schema.json" + + +@pytest.fixture +def solution(): + root = Path.cwd() / f".workbooks-api-test-{uuid4()}" / "FixtureSolution" + (root / "Workbooks/nested").mkdir(parents=True) + (root / "Data").mkdir() + (root / "Data/Solution_Fixture.json").write_text(json.dumps({"Version": "3.2.2"})) + notebook = { + "version": "Notebook/1.0", "fromTemplateId": str(uuid4()), + "items": [{"type": 3, "customWidth": "50", "content": { + "query": "DeviceEvents | summarize count() by ActionType", "queryType": 0, + "resourceType": workbook.LA, + }}], + } + (root / "Workbooks/nested/fixture.json").write_text(json.dumps(notebook)) + try: + yield root + finally: + shutil.rmtree(root.parent) + + +def config(solution, tables=None, evidence="reviewed exact fixture source"): + path = solution / "Workbooks/nested/fixture.json" + raw = json.loads(path.read_text())["items"][0]["content"]["query"] + return {"workbookResolutions": {"Workbooks/nested/fixture.json": { + "/items/0/content/query": {"querySha256": workbook.query_hash(raw), + "tables": tables or ["DeviceEvents"], "evidence": evidence}}}} + + +def convert(solution, settings=None, overwrite=False): + return workbooks.convert_workbooks( + solution, config=settings if settings is not None else config(solution), + report_directory=solution / "Logs/run", overwrite=overwrite) + + +def authored(solution): + return solution / "XDR Workbooks/nested/fixture.json" + + +@pytest.mark.parametrize("case", json.loads((FIXTURES / "upstream-cases.json").read_text()), + ids=lambda case: Path(case["path"]).stem) +def test_six_public_source_contracts(case): + path = FIXTURES / "sources" / case["path"] + before = path.read_bytes() + source = json.loads(before) + result = workbook.convert_workbook(source, tables=case["tables"]) + output, report = result["workbook"], result["report"] + assert len(workbook.query_surfaces(source)) == case["sourceQueries"] + assert len(workbook.query_surfaces(output)) == case["outputQueries"] + assert report["status"] == case["status"] + assert {p["name"] for p in report["removedParameters"]} == set(case["removed"]) + retained = {node["name"] for pointer, node in workbook.walk(output) + if "/parameters/" in pointer and node.get("name")} + assert set(case["retained"]) <= retained + def items(document): + return [(p, n["type"], n.get("name")) for p, n in workbook.walk(document) + if "/items/" in p and "type" in n and "content" in n] + assert items(output) == items(source) + if case["status"] != "blocked": + assert all(s["node"]["queryType"] == workbook.AH for s in workbook.query_surfaces(output)) + assert workbook.convert_workbook(output, tables=case["tables"])["workbook"] == output + else: + assert sum(i["code"] == "saved-function" for i in report["issues"]) == 2 + assert path.read_bytes() == before + + +@pytest.mark.parametrize("case", json.loads((FIXTURES / "upstream-cases.json").read_text()), + ids=lambda case: Path(case["path"]).stem) +def test_six_public_source_api_provenance_and_query_remaps(solution, case): + shutil.rmtree(solution / "Workbooks") + relative = "Workbooks/" + Path(case["path"]).name + source = solution / relative + source.parent.mkdir() + source.write_bytes((FIXTURES / "sources" / case["path"]).read_bytes()) + before = source.read_bytes() + source_document = json.loads(before) + settings = { + "workbookResolutions": {relative: { + q["path"]: {"querySha256": workbook.query_hash(q["node"]["query"]), + "tables": case["tables"], "evidence": "Reviewed public-source contract"} + for q in workbook.query_surfaces(source_document)}}, + "workbookDependencies": {relative: { + "requiredWorkloads": [{"workload": "sentinel", "dataTypes": case["tables"]}], + "requiredDataConnectors": [{"connectorId": "ReviewedFixtureConnector", + "dataTypes": case["tables"]}]}}, + } + result = convert(solution, settings) + assert result["total"] == 1 + assert result["converted"] == (case["status"] != "blocked") + output = solution / result["results"][0]["outputRelativePath"] + document = json.loads(output.read_bytes()) + assert len(document["contentProvenance"]["conversion"]["queries"]) == case["sourceQueries"] + assert all(q["sourceSha256"] for q in document["contentProvenance"]["conversion"]["queries"]) + assert source.read_bytes() == before + if case["status"] != "blocked": + jsonschema.validate(document, json.loads(SCHEMA.read_text())) + assert workbooks.validate_workbooks(solution)["valid"] == 1 + else: + assert workbooks.validate_workbooks(solution)["invalid"] == 1 + + +def test_packaged_schema_is_exact_canonical_schema(): + from importlib.resources import files + assert files("sentinel_xdr_migration._workbook_core").joinpath( + "data/xdr-workbook.schema.json").read_bytes() == SCHEMA.read_bytes() + + +def test_inventory_provenance_and_stable_identity(solution): + source = solution / "Workbooks/nested/fixture.json" + before = source.read_bytes() + row = workbooks.inventory_workbooks(solution)[0] + assert row["path"] == "Workbooks/nested/fixture.json" + assert row["sha256"] == hashlib.sha256(before).hexdigest() + first = convert(solution) + assert first["converted"] == 1 and first["blocked"] == 0 + content = json.loads(authored(solution).read_text()) + jsonschema.validate(content, json.loads(SCHEMA.read_text())) + metadata = content["xdrMetadata"] + assert metadata["contentId"] == row["identity"] + assert metadata["contentId"] != metadata["instanceId"] + assert metadata["version"] == "3.2.2" + assert content["version"] == "Notebook/1.0" + assert content["items"][0]["customWidth"] == "50" + assert content["contentProvenance"]["requiredWorkloads"] == [ + {"workload": "mde", "dataTypes": ["DeviceEvents"]}] + assert workbooks.validate_workbooks(solution)["valid"] == 1 + output = authored(solution).read_bytes() + assert convert(solution) == first + assert authored(solution).read_bytes() == output + assert source.read_bytes() == before + assert not (solution / "Logs/run/manifest.json").exists() + + +def test_pre31_release_is_explicit_blocker_or_selected_target(solution): + data = solution / "Data/Solution_Fixture.json" + data.write_text('{"Version":"3.0.9"}') + blocked = convert(solution) + assert blocked["blocked"] == 1 + assert any(i["code"] == "release-version-required" for i in blocked["results"][0]["issues"]) + settings = config(solution) + settings["targetVersion"] = "3.2.5" + assert convert(solution, settings, overwrite=True)["converted"] == 1 + assert json.loads(authored(solution).read_text())["xdrMetadata"]["version"] == "3.2.5" + assert json.loads(data.read_text())["Version"] == "3.0.9" + + +def test_independent_authored_workbook_keeps_31_content_floor(solution): + assert convert(solution)["converted"] == 1 + document = json.loads(authored(solution).read_text()) + document["xdrMetadata"]["version"] = "3.1.0" + authored(solution).write_text(json.dumps(document)) + assert workbooks.validate_workbooks(solution)["invalid"] == 0 + + +@pytest.mark.parametrize("version", ["3.0.9", "3.1.999", "4.0.0", "3.2", "03.2.0", "3.2.0-preview"]) +def test_unsupported_target_versions_reject_without_mutation(solution, version): + settings = config(solution) + settings["targetVersion"] = version + before = {p.relative_to(solution): p.read_bytes() for p in solution.rglob("*") if p.is_file()} + with pytest.raises(ValueError, match="target version"): + convert(solution, settings) + assert before == {p.relative_to(solution): p.read_bytes() for p in solution.rglob("*") if p.is_file()} + assert not (solution / "XDR Workbooks").exists() + assert not (solution / "Logs").exists() + + +def test_missing_reviewed_resolution_retains_candidate_and_report(solution): + result = convert(solution, {}) + assert result["blocked"] == 1 + assert json.loads(authored(solution).read_text())["items"] == json.loads( + (solution / "Workbooks/nested/fixture.json").read_text())["items"] + assert workbooks.validate_workbooks(solution)["invalid"] == 1 + assert Path(result["manifestPath"]).is_file() and Path(result["reportPath"]).is_file() + + +def test_raw_wrapper_resolution_hash_and_stale_evidence(solution): + path = solution / "Workbooks/nested/fixture.json" + document = json.loads(path.read_text()) + node = document["items"][0]["content"] + original = node["query"] + node.update(queryType=workbook.AH, query=json.dumps({ + "__advancedHuntingSettings": True, "query": original, "selectedWorkspaceIds": ["synthetic"]})) + path.write_text(json.dumps(document)) + settings = config(solution) + assert convert(solution, settings)["converted"] == 1 + query = json.loads(authored(solution).read_text())["contentProvenance"]["conversion"]["queries"][0] + assert query["sourceSha256"] == workbook.query_hash(node["query"]) + assert query["targetSha256"] == workbook.query_hash(original) + settings["workbookResolutions"]["Workbooks/nested/fixture.json"]["/items/0/content/query"]["querySha256"] = workbook.query_hash(original) + assert convert(solution, settings, overwrite=True)["blocked"] == 1 + + +def test_edited_output_never_silently_overwritten(solution): + convert(solution) + path = authored(solution) + path.write_text(path.read_text() + "\n") + edited = path.read_bytes() + result = convert(solution, overwrite=True) + assert result["results"][0]["status"] == "conflict" + assert path.read_bytes() == edited + assert convert(solution, overwrite=True)["results"][0]["status"] == "conflict" + + +def test_changed_source_requires_current_resolution_and_explicit_overwrite(solution): + convert(solution) + path = solution / "Workbooks/nested/fixture.json" + old_config = config(solution) + document = json.loads(path.read_text()) + document["items"][0]["content"]["query"] += " | take 1" + path.write_text(json.dumps(document)) + assert convert(solution, old_config)["results"][0]["status"] == "conflict" + assert convert(solution, old_config, overwrite=True)["blocked"] == 1 + assert convert(solution, overwrite=True)["converted"] == 1 + + +def test_connector_associations_are_explicit(solution): + path = solution / "Workbooks/nested/fixture.json" + document = json.loads(path.read_text()) + document["items"][0]["content"]["query"] = "Syslog | count" + path.write_text(json.dumps(document)) + settings = config(solution, ["Syslog"]) + assert convert(solution, settings)["blocked"] == 1 + settings["workbookDependencies"] = {"Workbooks/nested/fixture.json": { + "requiredDataConnectors": [{"connectorId": "Syslog", "dataTypes": ["Syslog"]}]}} + assert convert(solution, settings, overwrite=True)["converted"] == 1 + assert workbooks.validate_workbooks(solution)["valid"] == 1 + settings["workbookDependencies"]["Workbooks/nested/fixture.json"]["requiredDataConnectors"][0]["dataTypes"] = ["SecurityEvent"] + with pytest.raises(ValueError, match="outside reviewed"): + convert(solution, settings, overwrite=True) + + +def test_selected_empty_missing_duplicate_and_unsafe_references(solution): + convert(solution) + data = solution / "Data/Solution_Fixture.json" + for references in ([], ["XDR Workbooks/nested/fixture.json"]): + data.write_text(json.dumps({"Version": "3.2.2", "XDR Workbooks": references})) + assert workbooks.validate_workbooks(solution, selected_only=True)["total"] == len(references) + for references in ( + ["XDR Workbooks/missing.json"], ["XDR Workbooks/../Workbooks/nested/fixture.json"], + ["XDR Workbooks/nested/fixture.json"] * 2, ["/absolute.json"], + ): + data.write_text(json.dumps({"Version": "3.2.2", "XDR Workbooks": references})) + with pytest.raises(ValueError): + workbooks.selected_workbook_files(solution) + data.write_text('{"Version":"3.2.2"}') + with pytest.raises(ValueError, match="explicit"): + workbooks.validate_workbooks(solution, selected_only=True) + + +def test_inventory_duplicate_identity_and_symlinks_are_rejected(solution): + path = solution / "Workbooks/nested/fixture.json" + duplicate = path.with_name("duplicate.json") + duplicate.write_bytes(path.read_bytes()) + with pytest.raises(ValueError, match="Duplicate workbook identity"): + workbooks.inventory_workbooks(solution) + duplicate.unlink() + duplicate.symlink_to(path) + with pytest.raises(ValueError, match="Symlink"): + workbooks.inventory_workbooks(solution) + + +def test_output_symlink_parent_and_reports_in_content_are_rejected(solution): + (solution / "XDR Workbooks").symlink_to(solution / "Workbooks") + with pytest.raises(ValueError, match="Symlink"): + convert(solution) + (solution / "XDR Workbooks").unlink() + with pytest.raises(ValueError, match="outside"): + workbooks.convert_workbooks(solution, report_directory=solution / "Workbooks") + + +@pytest.mark.parametrize("content", ["{}", "[]", "not json", '{"version":"Notebook/1.0","items":{}}']) +def test_malformed_source_rejected_before_output(solution, content): + (solution / "Workbooks/nested/fixture.json").write_text(content) + with pytest.raises((ValueError, json.JSONDecodeError)): + convert(solution, {}) + assert not (solution / "XDR Workbooks").exists() + + +def test_stale_source_and_output_query_validation(solution): + convert(solution) + path = authored(solution) + document = json.loads(path.read_text()) + document["items"][0]["content"]["query"] += " | take 2" + path.write_text(json.dumps(document)) + assert workbooks.validate_workbooks(solution)["invalid"] == 1 + source = solution / "Workbooks/nested/fixture.json" + source.write_bytes(source.read_bytes() + b"\n") + assert any("stale" in error for error in workbooks.validate_workbooks(solution)["results"][0]["errors"]) + + +def test_no_source_or_target_queries_is_valid(solution): + path = solution / "Workbooks/nested/fixture.json" + path.write_text('{"version":"Notebook/1.0","items":[{"type":1,"content":{"json":"text"}}]}') + assert convert(solution, {})["converted"] == 1 + assert workbooks.validate_workbooks(solution)["valid"] == 1 + + +def test_static_catalog_load_errors_are_not_silent(solution, monkeypatch): + monkeypatch.setattr(migrate, "_load", lambda path: (_ for _ in ()).throw(ValueError("bad schema"))) + with pytest.raises(ValueError, match="bad schema"): + convert(solution) + assert not authored(solution).exists() + + +def test_live_resolution_is_explicitly_unsupported(): + from sentinel_xdr_migration._workbook_core import migrator + with pytest.raises(ValueError, match="Live"): + migrator.convert("DeviceEvents", resolve=True) + + +def test_fallback_identities_are_stable_without_authoritative_template(solution): + source = solution / "Workbooks/nested/fixture.json" + document = json.loads(source.read_bytes()) + del document["fromTemplateId"] + source.write_text(json.dumps(document)) + first = convert(solution) + source.write_bytes(source.read_bytes() + b"\n") + second = convert(solution, overwrite=True) + assert first["results"][0]["identity"] == second["results"][0]["identity"] + assert first["results"][0]["instanceId"] == second["results"][0]["instanceId"] + + +def test_validation_rejects_identity_forgery_and_panel_omission(solution): + convert(solution) + path = authored(solution) + document = json.loads(path.read_bytes()) + document["xdrMetadata"]["contentId"] = str(uuid4()) + path.write_text(json.dumps(document)) + assert workbooks.validate_workbooks(solution)["invalid"] == 1 + document["xdrMetadata"]["contentId"] = workbooks.inventory_workbooks(solution)[0]["identity"] + document["items"] = [] + query = document["contentProvenance"]["conversion"]["queries"][0] + query.update(status="removed-scope-helper", outputPath=None, targetSha256=None) + path.write_text(json.dumps(document)) + result = workbooks.validate_workbooks(solution) + assert result["invalid"] == 1 + assert any("proven scope" in e for e in result["results"][0]["errors"]) + + +def test_late_overwrite_edit_is_detected_and_staging_is_cleaned(solution, monkeypatch): + convert(solution) + path = authored(solution) + original = path.read_bytes() + real_fsync = workbooks.os.fsync + def edit_before_commit(fd): + real_fsync(fd) + path.write_bytes(original + b"\n") + monkeypatch.setattr(workbooks.os, "fsync", edit_before_commit) + source = solution / "Workbooks/nested/fixture.json" + source.write_bytes(source.read_bytes() + b"\n") + with pytest.raises(ValueError, match="changed"): + convert(solution, overwrite=True) + assert path.read_bytes() == original + b"\n" + assert not list(path.parent.glob("*.staged")) + + +@pytest.mark.parametrize("settings", [[], False, "invalid"]) +def test_malformed_configuration_not_treated_as_empty(solution, settings): + with pytest.raises(ValueError, match="configuration"): + convert(solution, settings) + + +def test_missing_authored_candidates_remain_validation_failures(solution): + result = workbooks.validate_workbooks(solution) + assert result["total"] == 1 and result["invalid"] == 1 + + +def test_runtime_ui_never_implied_and_offline_calls_only(solution, monkeypatch): + import socket + import subprocess + def forbidden(*args, **kwargs): + raise AssertionError("Unexpected runtime operation") + monkeypatch.setattr(socket, "socket", forbidden) + monkeypatch.setattr(subprocess, "run", forbidden) + result = convert(solution) + assert result["results"][0]["validation"] == { + "localStructureOnly": True, "runtime": "not-run", "ui": "not-run"} + assert result["results"][0]["gates"] + assert workbooks.validate_workbooks(solution)["valid"] == 1 diff --git a/Tools/SentinelToXDRMigration/tests/test_workbooks_adapter.py b/Tools/SentinelToXDRMigration/tests/test_workbooks_adapter.py new file mode 100644 index 00000000000..28190bb525a --- /dev/null +++ b/Tools/SentinelToXDRMigration/tests/test_workbooks_adapter.py @@ -0,0 +1,197 @@ +from __future__ import annotations +import copy +from collections import Counter +from datetime import datetime, timedelta, timezone +import json +from pathlib import Path +import shutil +import subprocess +from uuid import uuid4 +import pytest +from sentinel_xdr_migration._workbook_core import workbook, workbook_fixtures +from sentinel_xdr_migration._workbook_core import workbook_mock as mock, workbook_mock_recipes as recipes + +FIXTURES = Path(__file__).parent / "fixtures/workbooks" + +@pytest.fixture +def source(): + return json.loads((FIXTURES / "notebook.json").read_text()) + + +@pytest.fixture +def scratch(): + # Repository-local scratch only; no system temp directories. + path = Path.cwd() / f".workbook-test-{uuid4()}" + path.mkdir() + try: + yield path + finally: + shutil.rmtree(path) + + +def converted(source): + return workbook.convert_workbook(source, tables=["Syslog"]) + + +def test_every_query_layout_templates_drilldown_and_idempotence(source): + source["items"][1]["content"]["items"][0]["content"]["gridSettings"]["customMetadata"] = { + "resourceType": workbook.LA, "resourceIds": ["literal-business-link"]} + original = copy.deepcopy(source) + result = converted(source) + output = result["workbook"] + assert source == original + assert len(workbook.query_surfaces(output)) == 3 + assert {x["name"] for x in result["report"]["removedParameters"]} == {"Subscription", "Workspace"} + assert output["items"][1]["content"]["items"][0]["customWidth"] == "50" + assert output["items"][1]["content"]["items"][1]["conditionalVisibility"] == \ + source["items"][1]["content"]["items"][1]["conditionalVisibility"] + assert output["styleSettings"] == source["styleSettings"] + assert output["items"][1]["content"]["items"][0]["content"]["gridSettings"] == \ + source["items"][1]["content"]["items"][0]["content"]["gridSettings"] + assert "ownerId" not in output["context"] + assert "fallbackResourceIds" not in output + for s, old in zip(workbook.query_surfaces(output), workbook.query_surfaces(source)): + assert s["node"]["queryType"] == "advancedHunting" + assert s["node"]["query"] == old["node"]["query"] + assert not (workbook.SCOPE_KEYS & s["node"].keys()) + assert "resourceType" not in s["node"] + assert result["report"]["queries"][0]["sourcePath"] == "/items/0/content/parameters/3/query" + assert result["report"]["queries"][0]["outputPath"] == "/items/0/content/parameters/1/query" + assert converted(output)["workbook"] == output + assert result["report"]["status"] == "needs-validation" + assert {g["code"] for g in result["report"]["gates"]} == { + "ah-query-backed-parameters", "blank-ah-selection-table-visibility"} + + +def test_scope_helper_chain_and_business_resourcegroup(source): + params = source["items"][0]["content"]["parameters"] + params.extend([ + {"name": "DefaultWorkspace", "type": 5, + "typeSettings": {"resourceTypeFilter": {workbook.LA: True}}}, + {"name": "ContextFree", "type": 1, "queryType": 8, + "isHiddenWhenLocked": True, "query": '{"content":"{DefaultWorkspace}"}'}, + {"name": "Selection", "type": 1, "queryType": 1, + "query": "where type == 'microsoft.operationalinsights/workspaces' | extend w='{DefaultWorkspace}'"}, + {"name": "Subscriptions", "type": 6, "queryType": 1, + "query": "print sub='{Selection}'"}, + {"name": "Workspaces", "type": 5, "crossComponentResources": ["{Subscriptions}"]}, + {"name": "ResourceGroup", "type": 2, "queryType": 0, + "query": "AzureActivity | summarize by ResourceGroup", + "typeSettings": {"resourceTypeFilter": {"microsoft.resources/resourcegroups": True}}}, + ]) + source["fallbackResourceIds"] = ["{Workspaces}"] + result = workbook.convert_workbook(source, tables=["Syslog", "AzureActivity"]) + removed = {p["name"] for p in result["report"]["removedParameters"]} + assert removed == {"Subscription", "Workspace", "DefaultWorkspace", "ContextFree", + "Selection", "Subscriptions", "Workspaces"} + assert "ResourceGroup" in {p["name"] for p in result["workbook"]["items"][0]["content"]["parameters"]} + assert not result["report"]["issues"] + + +def test_exported_workspace_is_not_deleted(source): + source["items"][1]["content"]["items"][0]["content"]["exportParameterName"] = "Workspace" + result = converted(source) + assert "Workspace" not in {p["name"] for p in result["report"]["removedParameters"]} + assert any(i["code"] == "scope-used-by-content" for i in result["report"]["issues"]) + + +@pytest.mark.parametrize("query", [ + "workspace('example').Syslog | count", + "let f=(){workspace('example').Syslog}; f()", + "cluster('example').database('example').Syslog", + "SyslogConnectorsOverallStatus({TimeRange:start}, {TimeRange:end})", +]) +def test_explicit_scope_blocks_without_dropping_panel(source, query): + node = source["items"][1]["content"]["items"][0]["content"] + node["query"] = query + result = converted(source) + assert result["report"]["status"] == "blocked" + assert result["workbook"]["items"][1]["content"]["items"][0]["content"] == node + assert len(result["workbook"]["items"]) == len(source["items"]) + + +def test_ah_wrapper_and_unknown_settings(source): + node = source["items"][1]["content"]["items"][0]["content"] + query = node["query"] + node.update(queryType="advancedHunting", query=json.dumps({ + "__advancedHuntingSettings": True, "query": query, "selectedWorkspaceIds": ["synthetic-guid"] + })) + result = converted(source) + assert result["workbook"]["items"][1]["content"]["items"][0]["content"]["query"] == query + assert "selectedWorkspaceIds" not in json.dumps(result["workbook"]) + node["query"] = '{"__advancedHuntingSettings":true,"query":"Syslog","unknown":1}' + assert any(i["code"] == "query-format" for i in converted(source)["report"]["issues"]) + + +def test_unresolved_unknown_provider_and_unknown_mapping_preserved(source): + result = workbook.convert_workbook(source) + assert result["report"]["status"] == "blocked" + assert result["workbook"]["items"][1] == source["items"][1] + unknown = workbook.convert_workbook(source, tables=["UnknownCustomerTable"]) + assert unknown["report"]["status"] == "blocked" + node = source["items"][1]["content"]["items"][0]["content"] + node["queryType"] = 1 + assert any(i["code"] == "unsupported-provider" for i in converted(source)["report"]["issues"]) + + +def test_hash_bound_resolution(source): + resolutions = { + s["path"]: {"querySha256": workbook.query_hash(s["node"]["query"]), + "tables": ["Syslog"], "evidence": "sanitized reviewed fixture"} + for s in workbook.query_surfaces(source) + } + result = workbook.convert_workbook(source, resolutions=resolutions) + assert result["report"]["status"] == "needs-validation" + source["items"][1]["content"]["items"][0]["content"]["query"] += " | take 1" + assert any(i["code"] == "stale-resolution" for i in + workbook.convert_workbook(source, resolutions=resolutions)["report"]["issues"]) + + +def test_real_rows_emitted_and_fixture_oracle(source, scratch): + manifest = json.loads((FIXTURES / "manifest.json").read_text()) + result = workbook_fixtures.emit(manifest, source, converted(source)["workbook"], scratch) + assert result["execution"] == "not-run" + assert len(result["files"]) == 14 + text = (scratch / "all-computers-warning.ah.kql").read_text() + assert '"alpha", "warning"' in text and "datetime(2026-01-02T09:59:00Z)" in text + assert "{Computer}" not in text + rows = manifest["tables"]["Syslog"]["rows"] + # Independent fixture arithmetic verifies the oracle, not remote KQL execution. + for case in manifest["cases"][:5]: + counts = {} + selected = case["substitutions"] + start, end = [ + datetime.fromisoformat(selected[f"TimeRange:{key}"][9:-1].replace("Z", "+00:00")) + for key in ("start", "end") + ] + for timestamp, computer, severity in rows: + when = datetime.fromisoformat(timestamp.replace("Z", "+00:00")) + if not start <= when <= end: + continue + if selected["Computer"] != "'*'" and repr(computer) not in selected["Computer"].split(","): + continue + if selected["Severity"] != "'*'" and selected["Severity"] != repr(severity): + continue + key = computer, when.replace(minute=0, second=0).isoformat().replace("+00:00", "Z") + counts[key] = counts.get(key, 0) + 1 + actual = {"columns": case["expected"]["columns"], + "rows": [[*key, count] for key, count in sorted(counts.items())]} + workbook_fixtures.assert_result(case["expected"], actual) + + +def test_exact_assertion_rejects_same_count_wrong_values(): + expected = {"columns": [{"name": "Events", "type": "long"}], "rows": [[3]]} + with pytest.raises(AssertionError, match="Cell"): + workbook_fixtures.assert_result(expected, {"columns": expected["columns"], "rows": [[9]]}) + with pytest.raises(AssertionError, match="Column"): + workbook_fixtures.assert_result(expected, {"columns": [{"name": "Other", "type": "long"}], "rows": [[3]]}) + + +def test_fixture_clock_and_json_literal_are_not_neutralized(): + query = 'print t=now ( ), x=dynamic({"key":1}), label="{Computer:label}"' + result = workbook_fixtures.substitute(query, {"Computer:label": "alpha"}, + "2026-01-02T12:00:00Z") + assert 'dynamic({"key":1})' in result + assert "datetime(2026-01-02T12:00:00Z)" in result + with pytest.raises(ValueError, match="relative time"): + workbook_fixtures.substitute("print now(-1h)", {}, "2026-01-02T12:00:00Z") diff --git a/Tools/SentinelToXDRMigration/tests/test_workbooks_mock.py b/Tools/SentinelToXDRMigration/tests/test_workbooks_mock.py new file mode 100644 index 00000000000..fa854e31731 --- /dev/null +++ b/Tools/SentinelToXDRMigration/tests/test_workbooks_mock.py @@ -0,0 +1,331 @@ +from __future__ import annotations +import copy +from collections import Counter +from datetime import datetime, timedelta, timezone +import json +from pathlib import Path +import shutil +import subprocess +from uuid import uuid4 +import pytest +from sentinel_xdr_migration._workbook_core import workbook, workbook_fixtures +from sentinel_xdr_migration._workbook_core import workbook_mock as mock, workbook_mock_recipes as recipes + +FIXTURES = Path(__file__).parent / "fixtures/workbooks" + +PICKER = ("Syslog\r\n| summarize syslogEventsCount = count() by Computer\r\n" + "| sort by syslogEventsCount desc\r\n| project Computer\r\n") + + +@pytest.fixture +def scratch(): + path = Path.cwd() / f".workbook-mock-test-{uuid4()}" + path.mkdir() + try: + yield path + finally: + shutil.rmtree(path) + + +@pytest.fixture +def config(): + return mock.configuration(42, mock.DEFAULT_CLOCK, 1, "syslog-operational-v1") + + +@pytest.fixture +def source(scratch): + path = scratch / "source.json" + path.write_bytes(mock.json_bytes({ + "version": "Notebook/1.0", "items": [{"type": 9, "content": { + "parameters": [{"name": "Computer", "query": PICKER, "queryType": 0}] + }}], + })) + return path + + +@pytest.fixture +def real_source(): + return FIXTURES / "sources/Syslog/Workbooks/LinuxMachines.json" + + +def test_deterministic_generation_actual_rows_and_source_unchanged(source, scratch, config): + before = source.read_bytes() + first, second = scratch / "one", scratch / "two" + report = mock.generate(source, first, config) + mock.generate(source, second, config) + assert report["rows"] == 240 + assert report["cases"] == 10 + assert report["harnesses"] == 22 + assert source.read_bytes() == before + one = {p.relative_to(first): p.read_bytes() for p in first.rglob("*") if p.is_file()} + two = {p.relative_to(second): p.read_bytes() for p in second.rglob("*") if p.is_file()} + assert one == two + assert mock.validate(source, first)["status"] == "offline-checks-passed" + manifest = json.loads(one[Path("manifest.json")]) + table = manifest["tables"]["Syslog"] + records = json.loads(one[Path("tables/Syslog.json")]) + assert records == [dict(zip(table["schema"], row)) for row in table["rows"]] + stub = mock.dataframes.datatable_stub("Syslog", table["schema"], rows=table["rows"]) + for case in manifest["cases"]: + for side in ("sentinel", "ah"): + text = one[Path(f"harnesses/{case['id']}.{side}.kql")].decode() + assert stub in text + assert PICKER in text + assert not case.get("tableTimeRanges") + assert manifest["ingestion"]["payload"] is None + assert not manifest["ingestion"]["directLogsIngestionSupported"] + guard = json.loads(one[Path("tables/scenario.json")]) + assert guard["generationStatus"] == "table-fixture-only" + assert not any(name.name in {"mock.json", "malicious.json", "benign.json"} for name in one) + + +def test_seed_scale_clock_and_shared_correlations(config): + rows = recipes.syslog_rows(42, config["clock"], 1, "unit") + assert rows == recipes.syslog_rows(42, config["clock"], 1, "unit") + assert rows != recipes.syslog_rows(43, config["clock"], 1, "unit") + assert len(recipes.syslog_rows(42, config["clock"], 2, "unit")) == 438 + shifted = recipes.syslog_rows(42, "2026-10-08T12:00:00Z", 1, "unit") + for a, b in zip(rows, shifted): + assert recipes.instant(b["TimeGenerated"]) - recipes.instant(a["TimeGenerated"]) == timedelta(days=1) + assert a["EventTime"] == a["TimeGenerated"] + assert Counter(row["Computer"] for row in rows) == { + "mock-unit-1": 40, "mock-unit-2": 80, "mock-unit-3": 120} + for host in {row["Computer"] for row in rows}: + group = [row for row in rows if row["Computer"] == host] + assert len({(row["HostIP"], row["_ResourceId"], row["HostName"]) for row in group}) == 1 + for process in {row["ProcessName"] for row in group}: + assert len({row["ProcessID"] for row in group if row["ProcessName"] == process}) == 1 + assert {row["SeverityLevel"] for row in rows} == set(recipes.SEVERITIES) + assert {row["Facility"] for row in rows} == {"cron", "daemon", "authpriv", "kern"} + + +@pytest.mark.parametrize("column,value", [ + ("ProcessID", True), ("ProcessID", "1"), ("ProcessID", 2**31), + ("Computer", None), ("Computer", 9), ("TimeGenerated", "tomorrow"), + ("TimeGenerated", "2026-01-01T12:00:00"), ("TimeGenerated", "2026-01-01T12:00:00+01:00"), +]) +def test_strict_destination_types(column, value): + schema = recipes.SYSLOG_SCHEMA + record = recipes.syslog_rows(42, mock.DEFAULT_CLOCK, 1, "unit")[0] + record[column] = value + with pytest.raises(ValueError, match="invalid"): + mock.validate_rows("Syslog", schema, [[record[c] for c in schema]] * 2) + + +def test_schema_drift_and_missing_schema_fail_closed(monkeypatch): + monkeypatch.setattr(mock.dataframes, "schema_for", lambda *args: None) + with pytest.raises(ValueError, match="schema"): + mock.validate_rows("Syslog", recipes.SYSLOG_SCHEMA, []) + + +def test_precise_aggregates_cells_and_time_bins(): + rows = recipes.syslog_rows(42, mock.DEFAULT_CLOCK, 1, "unit") + choices = {choice["id"]: choice for choice in recipes.selections(rows, mock.DEFAULT_CLOCK)} + severity = recipes.expected_result(recipes.QueryContract("severity"), rows, choices["all-literal"]) + assert {row[0]: row[1] for row in severity["rows"]} == { + "emerg": 36, "panic": 6, "alert": 12, "crit": 18, "err": 12, "error": 6, + "warning": 24, "warn": 12, "notice": 30, "info": 48, "debug": 18, "unknown": 6, + } + boundary = recipes.expected_result(recipes.QueryContract("emerg"), rows, choices["end-boundary"]) + assert boundary["rows"] == [[f"mock-unit-{i}", mock.DEFAULT_CLOCK, i] for i in range(1, 4)] + picker = recipes.expected_result(recipes.QueryContract("computers", parameter=True, time_parameter=None), + rows, choices["empty-time"]) + assert picker["ordered"] + assert picker["rows"] == [["mock-unit-3"], ["mock-unit-2"], ["mock-unit-1"]] + empty = recipes.expected_result(recipes.QueryContract("events"), rows, choices["all-literal"]) + assert empty["rows"] == [] + events = recipes.expected_result(recipes.QueryContract("events"), rows, choices["end-boundary"]) + assert len(events["rows"]) == 6 + assert events["columns"][3] == {"name": "SeverityNumber", "type": "long"} + assert [mock.DEFAULT_CLOCK, "mock-unit-1", "emerg", 0, "kern", "192.0.2.10", "kernel (-)"] in events["rows"] + assert "/subscriptions/00000000-" in events["rows"][1][1] + # Independent epoch-floor oracle, not the implementation's replace() binning. + expected_bins = Counter() + for row in rows: + timestamp = datetime.fromisoformat(row["TimeGenerated"].replace("Z", "+00:00")).timestamp() + hour = datetime.fromtimestamp((int(timestamp) // 3600) * 3600, timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + expected_bins[row["Computer"], hour] += 1 + actual_bins = recipes.hourly_result(rows) + assert actual_bins["rows"] == [[*key, count] for key, count in sorted(expected_bins.items())] + for i in range(1, 4): + assert [f"mock-unit-{i}", "2026-10-07T05:00:00Z", i] in actual_bins["rows"] + assert [f"mock-unit-{i}", "2026-10-07T12:00:00Z", i * 2] in actual_bins["rows"] + wrong_bins = copy.deepcopy(actual_bins) + wrong_bins["rows"][0][1] = "2026-10-07T04:00:00Z" + with pytest.raises(AssertionError, match="time-bin"): + workbook_fixtures.assert_result(actual_bins, wrong_bins) + + +def test_independent_multiple_selection_message_process_and_facility_aggregates(): + rows = recipes.syslog_rows(42, mock.DEFAULT_CLOCK, 1, "unit") + selection = next(c for c in recipes.selections(rows, mock.DEFAULT_CLOCK) if c["id"] == "multiple") + # Warning/Error records are generated strictly inside the six-hour window. + filtered = [r for r in rows if r["Computer"] in ("mock-unit-1", "mock-unit-2") + and r["SeverityLevel"] in ("warning", "warn", "error", "err")] + assert len(filtered) == 27 + for kind, keys in (("messages", ("SyslogMessage",)), ("facilities", ("Facility", "SeverityLevel")), + ("processes", ("Facility", "ProcessName"))): + counts = Counter(tuple(r[k] for k in keys) for r in filtered) + expected = [[*key, count] for key, count in sorted(counts.items())] + if kind == "processes": + expected = [[f"{process} ({facility})", count] for facility, process, count in expected] + actual = recipes.expected_result(recipes.QueryContract(kind), rows, selection) + assert actual["rows"] == expected + assert sum(row[-1] for row in actual["rows"]) == 27 + + +def test_assertions_reject_same_shape_wrong_values_duplicates_and_sort(): + rows = recipes.syslog_rows(42, mock.DEFAULT_CLOCK, 1, "unit") + selection = recipes.selections(rows, mock.DEFAULT_CLOCK)[1] + expected = recipes.expected_result(recipes.QueryContract("severity"), rows, selection) + reversed_rows = {"columns": expected["columns"], "rows": list(reversed(expected["rows"]))} + with pytest.raises(AssertionError, match="sort-group"): + workbook_fixtures.assert_result(expected, reversed_rows) + actual = copy.deepcopy(expected) + actual["rows"][0][1] += 1 + with pytest.raises(AssertionError, match="Cell"): + workbook_fixtures.assert_result(expected, actual) + actual = copy.deepcopy(expected) + actual["rows"][0] = actual["rows"][1] + with pytest.raises(AssertionError, match="Cell"): + workbook_fixtures.assert_result(expected, actual) + expected["rows"][0], expected["rows"][1] = expected["rows"][1], expected["rows"][0] + workbook_fixtures.assert_result(expected, expected) # ties may be permuted + + +@pytest.mark.parametrize("changed", [ + {"seed": -1}, {"seed": True}, {"seed": 2**32}, {"scale": 0}, {"scale": 11}, + {"clock": "now"}, {"clock": "2026-13-01T12:00:00Z"}, + {"clock": "2026-01-01T12:00:00+00:00"}, {"recipe": "generic"}, + {"runId": "../escape"}, {"runId": "x';take 1"}, +]) +def test_invalid_configuration(config, changed): + config.update(changed) + with pytest.raises(ValueError): + mock.configuration(config["seed"], config["clock"], config["scale"], config["recipe"], config["runId"]) + + +@pytest.mark.parametrize("query", [ + "UnknownTable | count", "Syslog | count", "SyslogConnectorsOverallStatus()", + "Syslog | join (Heartbeat) on Computer", PICKER + " ", +]) +def test_unsupported_queries_block_without_writes(source, scratch, config, query): + doc = json.loads(source.read_bytes()) + doc["items"][0]["content"]["parameters"][0]["query"] = query + source.write_bytes(mock.json_bytes(doc)) + with pytest.raises(mock.Blocked, match="reviewed mock contract"): + mock.generate(source, scratch / "blocked", config) + assert not (scratch / "blocked").exists() + + +def test_inherited_unreviewed_time_context_is_not_silently_ignored(source, config): + doc = json.loads(source.read_bytes()) + doc["timeContextFromParameter"] = "UnreviewedInheritedWindow" + with pytest.raises(mock.Blocked): + mock.build(mock.json_bytes(doc), config) + + +def test_artifact_tampering_and_collision(source, scratch, config): + out = scratch / "bundle" + mock.generate(source, out, config) + before = (out / "manifest.json").read_bytes() + with pytest.raises(ValueError, match="new directory"): + mock.generate(source, out, config) + assert (out / "manifest.json").read_bytes() == before + harness = next((out / "harnesses").glob("*.kql")) + harness.write_text(harness.read_text().replace('datatable(', 'datatable(Bogus:string, ', 1)) + with pytest.raises(ValueError, match="regeneration mismatch"): + mock.validate(source, out) + + +def test_source_and_oracle_drift_detected(source, scratch, config): + out = scratch / "bundle" + mock.generate(source, out, config) + source.write_bytes(source.read_bytes() + b"\n") + with pytest.raises(ValueError, match="regeneration mismatch"): + mock.validate(source, out) + source.write_bytes(source.read_bytes()[:-1]) + manifest_path = out / "manifest.json" + manifest = json.loads(manifest_path.read_bytes()) + manifest["cases"][0]["expected"]["rows"][0][0] = "WRONG" + manifest_path.write_bytes(mock.json_bytes(manifest)) + with pytest.raises(ValueError, match="regeneration mismatch"): + mock.validate(source, out) + + +def test_real_linux_all_panels_and_all_selection_cases(real_source, scratch, config): + before = real_source.read_bytes() + out = scratch / "linux" + report = mock.generate(real_source, out, config) + assert report["cases"] == 100 + assert report["harnesses"] == 202 + assert mock.validate(real_source, out)["status"] == "offline-checks-passed" + manifest = json.loads((out / "manifest.json").read_bytes()) + assert len(manifest["inventory"]) == 10 + assert sum(item["parameter"] for item in manifest["inventory"]) == 1 + for case in manifest["cases"]: + if case["sourcePath"].endswith("/parameters/1/query"): + assert len(case["expected"]["rows"]) == 3 + assert "tableTimeRanges" not in case + else: + text = (out / f"harnesses/{case['id']}.ah.kql").read_text() + assert "| where TimeGenerated between (datetime(" in text + assert text.index("| where TimeGenerated between") < text.index("\nSyslog") + assert "{Computer" not in text and "{Severity}" not in text + if case["sourcePath"] == "/items/6/content/query": + if case["selection"]["id"] in ("all-literal", "empty-computers", "empty-severity", "empty-time"): + assert case["expected"]["rows"] == [] + if case["selection"]["id"] == "all-expanded": + assert len(case["expected"]["rows"]) == 222 + if case["selection"]["id"] == "end-boundary": + assert len(case["expected"]["rows"]) == 6 + assert real_source.read_bytes() == before + + +def test_real_linux_unreviewed_context_blocked(real_source, config): + source = json.loads(real_source.read_bytes()) + source["items"][3]["content"]["timeContextFromParameter"] = "OtherTime" + with pytest.raises(mock.Blocked): + mock.build(mock.json_bytes(source), config) + source = json.loads(real_source.read_bytes()) + source["items"][3]["content"]["timeContext"] = {"start": "2020-01-01"} + with pytest.raises(mock.Blocked): + mock.build(mock.json_bytes(source), config) + source = json.loads(real_source.read_bytes()) + source["items"][1]["content"]["parameters"][2]["jsonData"] = '["invented severity"]' + with pytest.raises(mock.Blocked): + mock.build(mock.json_bytes(source), config) + source = json.loads(real_source.read_bytes()) + source["items"][1]["content"]["parameters"][1]["typeSettings"]["selectAllValue"] = "*" + with pytest.raises(mock.Blocked): + mock.build(mock.json_bytes(source), config) + + +def test_no_network_or_environment_writes(source, scratch, config, monkeypatch): + import socket + import urllib.request + + def forbidden(*args, **kwargs): + raise AssertionError("Offline generator attempted a live operation") + + monkeypatch.setattr(socket, "socket", forbidden) + monkeypatch.setattr(urllib.request, "urlopen", forbidden) + monkeypatch.setattr(subprocess, "run", forbidden) + env_files = list(scratch.rglob(".env")) + before = {path: mock.digest(path.read_bytes()) for path in env_files} + out = scratch / "offline" + mock.generate(source, out, config) + mock.validate(source, out) + assert before == {path: mock.digest(path.read_bytes()) for path in env_files} + + +def test_missing_cases_and_unexpected_files_fail(source, scratch, config): + out = scratch / "bundle" + mock.generate(source, out, config) + path = scratch / "capture.json" + path.write_text("{}") + with pytest.raises(ValueError, match="every case"): + mock.assert_captured(out, path, "sentinel") + (out / "unexpected.json").write_text("{}") + with pytest.raises(ValueError, match="file inventory"): + mock.validate(source, out) diff --git a/Tools/SentinelToXDRMigration/tests/test_workflow.py b/Tools/SentinelToXDRMigration/tests/test_workflow.py index 5db13d79cb9..bd89f574744 100644 --- a/Tools/SentinelToXDRMigration/tests/test_workflow.py +++ b/Tools/SentinelToXDRMigration/tests/test_workflow.py @@ -10,6 +10,7 @@ from unittest import mock import yaml +from sentinel_xdr_migration.dependencies import dependency_identity from sentinel_xdr_migration.workflow import ( complete_workflow_stage, @@ -209,7 +210,8 @@ def _mock_v3_1_patch_package(self, solution: Path, **kwargs: str) -> dict: result["packageInputs"]["before"] = before return result - def _prepare_workflow_for_package(self, version_bump: str = "patch") -> str: + def _prepare_workflow_for_package(self, version_bump: str | None = "patch") -> str: + self._ensure_solution_release() initialized = initialize_workflow( self.solution, version_bump=version_bump, @@ -224,11 +226,13 @@ def _prepare_offline_authoring_validation( self, *, campaign_id: str | None = "test-campaign", + version_bump: str | None = "none", ) -> tuple[dict, Path]: + self._ensure_solution_release() self._create_source_rules() initialized = initialize_workflow( self.solution, - version_bump="none", + version_bump=version_bump, workflow_profile="authoring", campaign_id=campaign_id, ) @@ -326,9 +330,17 @@ def _create_runtime_evidence(self) -> None: "status": "passed", "rowCount": 0, "querySha256": query_hash, + **({"dependencyProvenanceSha256": dependency_identity(target)} + if provider == "triage-mcp" else {}), }], }), encoding="utf-8") + def _ensure_solution_release(self) -> None: + data = self.solution / "Data/Solution_Sample.json" + if not data.exists(): + data.parent.mkdir(exist_ok=True) + data.write_text(json.dumps({"Name": "Sample", "Version": "3.2.0"}), encoding="utf-8") + def _create_source_rules(self, count: int = 1) -> list[Path]: paths = [] for index in range(count): @@ -582,6 +594,7 @@ def test_packaging_cannot_pass_without_v3_1_evidence(self) -> None: ) def test_partial_qualification_progresses_without_fabricating_full_validation(self) -> None: + self._ensure_solution_release() initialized = initialize_workflow( self.solution, version_bump="patch", @@ -663,6 +676,7 @@ def test_partial_qualification_progresses_without_fabricating_full_validation(se self.assertEqual("passed", completed_state["stages"]["alertParity"]["status"]) def test_blocked_validation_cannot_package_without_applied_selection(self) -> None: + self._ensure_solution_release() self._create_source_rules() initialize_workflow(self.solution, workflow_profile="authoring") for stage in ("discovery", "conversion"): @@ -781,6 +795,7 @@ def test_changed_package_inputs_reject_none_and_require_a_new_bump(self) -> None ) def test_initial_none_action_can_plan_first_partial_release(self) -> None: + self._ensure_solution_release() initialized = initialize_workflow( self.solution, version_bump="none", @@ -874,6 +889,7 @@ def _assert_initial_none_action_can_plan_first_complete_release( self, rule_count: int, ) -> None: + self._ensure_solution_release() initialized = initialize_workflow( self.solution, version_bump="none", @@ -960,6 +976,7 @@ def test_initial_none_action_plans_first_complete_15_rule_release(self) -> None: self._assert_initial_none_action_can_plan_first_complete_release(15) def test_first_release_requires_current_applied_complete_selection_and_exact_hash(self) -> None: + self._ensure_solution_release() initialized = initialize_workflow( self.solution, version_bump="none", @@ -1065,7 +1082,7 @@ def test_offline_authoring_validation_and_first_patch_release(self) -> None: def test_same_release_notes_refresh_rebuilds_only_approved_notes_row(self) -> None: data = self.solution / "Data" / "Solution_Sample.json" data.parent.mkdir(parents=True) - data.write_text(json.dumps({"Name": "Sample", "Version": "3.1.2"}), encoding="utf-8") + data.write_text(json.dumps({"Name": "Sample", "Version": "3.2.2"}), encoding="utf-8") aligned_header = ( "| **Version** | **Date Modified (DD-MM-YYYY)** | " "**Change History** |" @@ -1129,7 +1146,7 @@ def test_same_release_notes_refresh_rebuilds_only_approved_notes_row(self) -> No ])) planned = workflow_status(self.solution)["packaging"]["pendingReleaseAction"] expected_row = ( - "| 3.1.2 | 12-10-2026 | Refreshed the V3.1 hybrid package for three disabled " + "| 3.2.2 | 12-10-2026 | Refreshed the V3.1 hybrid package for three disabled " "Defender XDR Custom Detections with reviewed Host mappings and source-rule " "provenance. |" ) @@ -1159,7 +1176,7 @@ def test_same_release_notes_refresh_rebuilds_only_approved_notes_row(self) -> No ) package.assert_called_once() self.assertEqual("none", refreshed["effectiveVersionBump"]) - self.assertEqual("3.1.2", version_before) + self.assertEqual("3.2.2", version_before) self.assertEqual(version_before, json.loads(data.read_text(encoding="utf-8"))["Version"]) self.assertEqual(first["version"], refreshed["version"]) self.assertEqual(inserted_notes, notes.read_bytes()) @@ -1222,7 +1239,7 @@ def test_same_release_notes_refresh_rebuilds_only_approved_notes_row(self) -> No def test_same_release_notes_refresh_rejects_non_notes_changes_and_wrong_row(self) -> None: data = self.solution / "Data" / "Solution_Sample.json" data.parent.mkdir(parents=True) - data.write_text(json.dumps({"Name": "Sample", "Version": "1.0.0"}), encoding="utf-8") + data.write_text(json.dumps({"Name": "Sample", "Version": "3.2.0"}), encoding="utf-8") notes = self.solution / "ReleaseNotes.md" notes.write_text( "| **Version** | **Date Modified (DD-MM-YYYY)** | **Change History** |\n" @@ -1263,7 +1280,7 @@ def test_same_release_notes_refresh_rejects_non_notes_changes_and_wrong_row(self notes.write_text( notes.read_text(encoding="utf-8").replace( "|---|---|---|\n", - "|---|---|---|\n| 1.0.0 | 12-10-2026 | Different description |\n", + "|---|---|---|\n| 3.2.0 | 12-10-2026 | Different description |\n", 1, ), encoding="utf-8", @@ -1277,7 +1294,7 @@ def test_same_release_notes_refresh_rejects_non_notes_changes_and_wrong_row(self def test_same_release_notes_refresh_repeat_build_without_campaign(self) -> None: data = self.solution / "Data" / "Solution_Sample.json" data.parent.mkdir(parents=True) - data.write_text(json.dumps({"Name": "Sample", "Version": "3.1.2"}), encoding="utf-8") + data.write_text(json.dumps({"Name": "Sample", "Version": "3.2.2"}), encoding="utf-8") notes = self.solution / "ReleaseNotes.md" notes.write_text( "| **Version** | **Date Modified (DD-MM-YYYY)** | **Change History** |\n" @@ -1319,7 +1336,7 @@ def test_same_release_notes_refresh_repeat_build_without_campaign(self) -> None: notes.write_text( notes.read_text(encoding="utf-8").replace( "|---|---|---|\n", - "|---|---|---|\n| 3.1.2 | 12-10-2026 | Approved same-release note |\n", + "|---|---|---|\n| 3.2.2 | 12-10-2026 | Approved same-release note |\n", 1, ), encoding="utf-8", @@ -1338,7 +1355,7 @@ def test_same_release_notes_refresh_repeat_build_without_campaign(self) -> None: def test_same_version_content_rebuild_archives_inputs_and_builds_without_bumping(self) -> None: data = self.solution / "Data" / "Solution_Sample.json" data.parent.mkdir(parents=True) - data.write_text(json.dumps({"Name": "Sample", "Version": "3.1.1"}), encoding="utf-8") + data.write_text(json.dumps({"Name": "Sample", "Version": "3.2.1"}), encoding="utf-8") readme = self.solution / "README.md" readme.write_text("approved package content before the first build\n", encoding="utf-8") initialized, _ = self._prepare_offline_authoring_validation() @@ -1366,7 +1383,7 @@ def test_same_version_content_rebuild_archives_inputs_and_builds_without_bumping self.solution, requested_version_bump="patch", ) - self.assertEqual("3.1.2", first["version"]) + self.assertEqual("3.2.2", first["version"]) original_input = readme.read_bytes() readme.write_text("explicitly approved non-notes content fix\n", encoding="utf-8") approved_input = readme.read_bytes() @@ -1377,14 +1394,14 @@ def test_same_version_content_rebuild_archives_inputs_and_builds_without_bumping with self.assertRaisesRegex(ValueError, "artifact is stale: zip"): plan_same_version_content_rebuild( self.solution, - target_version="3.1.2", + target_version="3.2.2", changed_inputs=["README.md"], expected_package_input_sha256=approved_hash, ) prior_zip.write_bytes(prior_zip_bytes) planned = plan_same_version_content_rebuild( self.solution, - target_version="3.1.2", + target_version="3.2.2", changed_inputs=["README.md"], expected_package_input_sha256=approved_hash, ) @@ -1392,7 +1409,7 @@ def test_same_version_content_rebuild_archives_inputs_and_builds_without_bumping action = state["packaging"]["pendingReleaseAction"] self.assertEqual("same-version-content-rebuild", action["actionType"]) self.assertEqual("none", action["versionBump"]) - self.assertEqual("3.1.2", action["targetVersion"]) + self.assertEqual("3.2.2", action["targetVersion"]) self.assertEqual(approved_hash, action["packageInputSha256"]) self.assertEqual( hashlib.sha256(approved_input).hexdigest(), @@ -1429,8 +1446,8 @@ def test_same_version_content_rebuild_archives_inputs_and_builds_without_bumping ) package.assert_called_once() self.assertEqual("none", rebuilt["effectiveVersionBump"]) - self.assertEqual("3.1.2", rebuilt["version"]) - self.assertEqual("3.1.2", json.loads(data.read_text(encoding="utf-8"))["Version"]) + self.assertEqual("3.2.2", rebuilt["version"]) + self.assertEqual("3.2.2", json.loads(data.read_text(encoding="utf-8"))["Version"]) self.assertEqual(approved_input, readme.read_bytes()) state = workflow_status(self.solution) self.assertEqual("consumed", state["packaging"]["releaseActions"][-1]["status"]) @@ -1447,7 +1464,7 @@ def test_same_version_content_rebuild_archives_inputs_and_builds_without_bumping requested_version_bump="none", ) repeat_package.assert_called_once() - self.assertEqual("3.1.2", repeated["version"]) + self.assertEqual("3.2.2", repeated["version"]) self.assertNotEqual( first_rebuild_attempt, workflow_status(self.solution)["packaging"]["lastConsumedAction"]["attemptId"], @@ -1457,7 +1474,7 @@ def test_same_version_content_rebuild_archives_inputs_and_builds_without_bumping def test_same_version_content_rebuild_retry_and_scope_guards(self) -> None: data = self.solution / "Data" / "Solution_Sample.json" data.parent.mkdir(parents=True) - data.write_text(json.dumps({"Name": "Sample", "Version": "3.1.1"}), encoding="utf-8") + data.write_text(json.dumps({"Name": "Sample", "Version": "3.2.1"}), encoding="utf-8") readme = self.solution / "README.md" readme.write_text("before\n", encoding="utf-8") release_notes = self.solution / "ReleaseNotes.md" @@ -1489,27 +1506,27 @@ def test_same_version_content_rebuild_retry_and_scope_guards(self) -> None: with self.assertRaisesRegex(ValueError, "exactly match the current solution version"): plan_same_version_content_rebuild( self.solution, - target_version="3.1.3", + target_version="3.2.3", changed_inputs=["README.md"], expected_package_input_sha256=expected_hash, ) with self.assertRaisesRegex(ValueError, "current input identity"): plan_same_version_content_rebuild( self.solution, - target_version="3.1.2", + target_version="3.2.2", changed_inputs=["README.md"], expected_package_input_sha256="0" * 64, ) with self.assertRaisesRegex(ValueError, "ReleaseNotes"): plan_same_version_content_rebuild( self.solution, - target_version="3.1.2", + target_version="3.2.2", changed_inputs=["ReleaseNotes.md"], expected_package_input_sha256=expected_hash, ) plan_same_version_content_rebuild( self.solution, - target_version="3.1.2", + target_version="3.2.2", changed_inputs=["README.md"], expected_package_input_sha256=expected_hash, ) @@ -1537,7 +1554,7 @@ def mutate_notes_after_v31(solution: Path, **kwargs: str) -> dict: failed_state["packaging"]["pendingReleaseAction"]["packageInputSha256"], ) self.assertEqual("failed", failed_state["packaging"]["currentAttempt"]["status"]) - self.assertEqual("3.1.2", json.loads(data.read_text(encoding="utf-8"))["Version"]) + self.assertEqual("3.2.2", json.loads(data.read_text(encoding="utf-8"))["Version"]) with mock.patch.object( workflow_module, "package_solution_v3_1", @@ -1547,13 +1564,13 @@ def mutate_notes_after_v31(solution: Path, **kwargs: str) -> dict: self.solution, requested_version_bump="none", ) - self.assertEqual("3.1.2", retried["version"]) + self.assertEqual("3.2.2", retried["version"]) self.assertEqual("consumed", workflow_status(self.solution)["packaging"]["releaseActions"][-1]["status"]) def test_same_version_rebuild_requires_exact_receipt_to_current_path_diff(self) -> None: data = self.solution / "Data" / "Solution_Sample.json" data.parent.mkdir(parents=True) - data.write_text(json.dumps({"Name": "Sample", "Version": "3.1.1"}), encoding="utf-8") + data.write_text(json.dumps({"Name": "Sample", "Version": "3.2.1"}), encoding="utf-8") contents = { "README.md": b"before\n", "ReleaseNotes.md": b"historical notes\n", @@ -1595,7 +1612,7 @@ def test_same_version_rebuild_requires_exact_receipt_to_current_path_diff(self) with self.assertRaisesRegex(ValueError, "does not permit any ReleaseNotes change"): plan_same_version_content_rebuild( self.solution, - target_version="3.1.2", + target_version="3.2.2", changed_inputs=["README.md"], expected_package_input_sha256=current_hash, ) @@ -1605,7 +1622,7 @@ def test_same_version_rebuild_requires_exact_receipt_to_current_path_diff(self) with self.assertRaisesRegex(ValueError, "scope must exactly match"): plan_same_version_content_rebuild( self.solution, - target_version="3.1.2", + target_version="3.2.2", changed_inputs=["README.md"], expected_package_input_sha256=current_hash, ) @@ -1617,14 +1634,14 @@ def test_same_version_rebuild_requires_exact_receipt_to_current_path_diff(self) with self.assertRaisesRegex(ValueError, "scope must exactly match"): plan_same_version_content_rebuild( self.solution, - target_version="3.1.2", + target_version="3.2.2", changed_inputs=["README.md", "Unchanged.txt"], expected_package_input_sha256=current_hash, ) planned = plan_same_version_content_rebuild( self.solution, - target_version="3.1.2", + target_version="3.2.2", changed_inputs=["README.md"], expected_package_input_sha256=current_hash, ) @@ -1639,7 +1656,7 @@ def test_same_version_rebuild_requires_exact_receipt_to_current_path_diff(self) def test_same_version_rebuild_reconstructs_receipt_bound_legacy_baseline(self) -> None: data = self.solution / "Data" / "Solution_Sample.json" data.parent.mkdir(parents=True) - data.write_text(json.dumps({"Name": "Sample", "Version": "3.1.1"}), encoding="utf-8") + data.write_text(json.dumps({"Name": "Sample", "Version": "3.2.1"}), encoding="utf-8") readme = self.solution / "README.md" readme.write_bytes(b"legacy baseline\n") initialized, _ = self._prepare_offline_authoring_validation() @@ -1702,7 +1719,7 @@ def legacy_package(solution: Path, **kwargs: str) -> dict: with self.assertRaisesRegex(ValueError, "provide exact receipt-bound"): plan_same_version_content_rebuild( self.solution, - target_version="3.1.2", + target_version="3.2.2", changed_inputs=["README.md"], expected_package_input_sha256=current_identity, ) @@ -1715,7 +1732,7 @@ def legacy_package(solution: Path, **kwargs: str) -> dict: with self.assertRaisesRegex(ValueError, "not bound to the successful receipt"): plan_same_version_content_rebuild( self.solution, - target_version="3.1.2", + target_version="3.2.2", changed_inputs=["README.md"], expected_package_input_sha256=current_identity, legacy_baseline_evidence=( @@ -1724,7 +1741,7 @@ def legacy_package(solution: Path, **kwargs: str) -> dict: ) planned = plan_same_version_content_rebuild( self.solution, - target_version="3.1.2", + target_version="3.2.2", changed_inputs=["README.md"], expected_package_input_sha256=current_identity, legacy_baseline_evidence=evidence_path.as_posix(), @@ -1760,7 +1777,7 @@ def legacy_package(solution: Path, **kwargs: str) -> dict: self.solution, requested_version_bump="none", ) - self.assertEqual("3.1.2", rebuilt["version"]) + self.assertEqual("3.2.2", rebuilt["version"]) self.assertEqual("consumed", workflow_status(self.solution)["packaging"][ "releaseActions" ][-1]["status"]) @@ -1780,7 +1797,7 @@ def legacy_package(solution: Path, **kwargs: str) -> dict: def test_failed_v31_candidate_can_be_repaired_and_retried_at_same_version(self) -> None: data = self.solution / "Data" / "Solution_Sample.json" data.parent.mkdir(parents=True) - data.write_text(json.dumps({"Name": "Sample", "Version": "3.1.1"}), encoding="utf-8") + data.write_text(json.dumps({"Name": "Sample", "Version": "3.2.1"}), encoding="utf-8") readme = self.solution / "README.md" readme.write_text("candidate input\n", encoding="utf-8") initialized, _ = self._prepare_offline_authoring_validation() @@ -1802,9 +1819,9 @@ def test_failed_v31_candidate_can_be_repaired_and_retried_at_same_version(self) def failed_initial_build(solution: Path, **kwargs: str) -> dict: value = json.loads(data.read_text(encoding="utf-8")) - value["Version"] = "3.1.2" + value["Version"] = "3.2.2" data.write_text(json.dumps(value), encoding="utf-8") - artifacts = self._packaging_artifacts("patch", "3.1.2") + artifacts = self._packaging_artifacts("patch", "3.2.2") Path(artifacts["packageReport"]).unlink() raise RuntimeError("synthetic ARM-TTK failure after candidate generation") @@ -1829,13 +1846,13 @@ def failed_initial_build(solution: Path, **kwargs: str) -> dict: / "Logs" / "sentinel-xdr-migration" / initialized["runId"] - / "failed-v31-3.1.2-artifacts" + / "failed-v31-3.2.2-artifacts" ) for artifact in ( "mainTemplate.json", "createUiDefinition.json", "testParameters.json", - "3.1.2.zip", + "3.2.2.zip", ): source = self.solution / "Package" / artifact target = snapshot / "Package" / artifact @@ -1853,7 +1870,7 @@ def failed_initial_build(solution: Path, **kwargs: str) -> dict: ("Package/mainTemplate.json", "mainTemplate.json"), ("Package/createUiDefinition.json", "createUiDefinition.json"), ("Package/testParameters.json", "testParameters.json"), - ("Package/3.1.2.zip", "3.1.2.zip"), + ("Package/3.2.2.zip", "3.2.2.zip"), ): candidate = self.solution / relative generated[relative] = { @@ -1868,7 +1885,7 @@ def failed_initial_build(solution: Path, **kwargs: str) -> dict: "shellStatus": "completed", "cliExitCode": 2, "v31ChildExitCode": 1, - "currentVersion": "3.1.2", + "currentVersion": "3.2.2", "bumpConsumed": True, "lastConsumedAction": failed_package["lastConsumedAction"], "currentPackageInput": { @@ -1877,7 +1894,7 @@ def failed_initial_build(solution: Path, **kwargs: str) -> dict: }, "generatedArtifacts": generated, "packageReportExists": False, - "failedArtifactSnapshot": "failed-v31-3.1.2-artifacts", + "failedArtifactSnapshot": "failed-v31-3.2.2-artifacts", } handoff_path.write_text(json.dumps(handoff), encoding="utf-8") @@ -1908,7 +1925,7 @@ def failed_initial_build(solution: Path, **kwargs: str) -> dict: with self.assertRaisesRegex(ValueError, "target must exactly match"): plan_same_version_content_rebuild( self.solution, - target_version="3.1.3", + target_version="3.2.3", changed_inputs=["README.md"], expected_package_input_sha256=current_identity, failed_candidate_evidence=handoff_path.relative_to(self.solution), @@ -1919,7 +1936,7 @@ def failed_initial_build(solution: Path, **kwargs: str) -> dict: with self.assertRaisesRegex(ValueError, "must be a JSON object"): plan_same_version_content_rebuild( self.solution, - target_version="3.1.2", + target_version="3.2.2", changed_inputs=["README.md"], expected_package_input_sha256=current_identity, failed_candidate_evidence=handoff_path.relative_to(self.solution), @@ -1931,7 +1948,7 @@ def failed_initial_build(solution: Path, **kwargs: str) -> dict: with self.assertRaisesRegex(ValueError, "matching consumed, failed V3.1 attempt"): plan_same_version_content_rebuild( self.solution, - target_version="3.1.2", + target_version="3.2.2", changed_inputs=["README.md"], expected_package_input_sha256=current_identity, failed_candidate_evidence=handoff_path.relative_to(self.solution), @@ -1940,7 +1957,7 @@ def failed_initial_build(solution: Path, **kwargs: str) -> dict: handoff_path.write_bytes(original_handoff) planned = plan_same_version_content_rebuild( self.solution, - target_version="3.1.2", + target_version="3.2.2", changed_inputs=["README.md"], expected_package_input_sha256=current_identity, failed_candidate_evidence=handoff_path.relative_to(self.solution), @@ -1984,7 +2001,7 @@ def failed_initial_build(solution: Path, **kwargs: str) -> dict: ): with self.assertRaisesRegex(RuntimeError, "retry still fails"): package_workflow_v3_1(self.solution, requested_version_bump="none") - self.assertEqual("3.1.2", workflow_module.solution_data_version(self.solution)) + self.assertEqual("3.2.2", workflow_module.solution_data_version(self.solution)) self.assertEqual( "same-version-failed-candidate-rebuild", workflow_status(self.solution)["packaging"]["pendingReleaseAction"]["actionType"], @@ -1996,8 +2013,8 @@ def failed_initial_build(solution: Path, **kwargs: str) -> dict: side_effect=self._mock_v3_1_package, ): rebuilt = package_workflow_v3_1(self.solution, requested_version_bump="none") - self.assertEqual("3.1.2", rebuilt["version"]) - self.assertEqual("3.1.2", workflow_module.solution_data_version(self.solution)) + self.assertEqual("3.2.2", rebuilt["version"]) + self.assertEqual("3.2.2", workflow_module.solution_data_version(self.solution)) rebuilt_state = workflow_status(self.solution) self.assertIsNone(rebuilt_state["packaging"]["pendingReleaseAction"]) self.assertTrue(rebuilt_state["packaging"]["lastSuccessful"]["packageSucceeded"]) @@ -2041,7 +2058,7 @@ def test_same_version_content_rebuild_is_authoring_only_and_version_bound(self) with self.assertRaisesRegex(ValueError, "only for Authoring"): plan_same_version_content_rebuild( self.solution, - target_version="3.1.2", + target_version="3.2.2", changed_inputs=["README.md"], expected_package_input_sha256="a" * 64, ) @@ -2066,7 +2083,7 @@ def test_same_release_notes_refresh_is_authoring_only_and_does_not_reconsume_pat def test_same_release_notes_refresh_requires_fresh_package_receipt(self) -> None: data = self.solution / "Data" / "Solution_Sample.json" data.parent.mkdir(parents=True) - data.write_text(json.dumps({"Name": "Sample", "Version": "1.0.0"}), encoding="utf-8") + data.write_text(json.dumps({"Name": "Sample", "Version": "3.2.0"}), encoding="utf-8") notes = self.solution / "ReleaseNotes.md" notes.write_text( "| **Version** | **Date Modified (DD-MM-YYYY)** | **Change History** |\n" @@ -2111,7 +2128,7 @@ def test_same_release_notes_refresh_requires_fresh_package_receipt(self) -> None def test_release_notes_refresh_parses_aligned_header_and_rejects_malformed_history(self) -> None: data = self.solution / "Data" / "Solution_Sample.json" data.parent.mkdir(parents=True) - data.write_text(json.dumps({"Name": "Sample", "Version": "3.1.2"}), encoding="utf-8") + data.write_text(json.dumps({"Name": "Sample", "Version": "3.2.2"}), encoding="utf-8") notes = self.solution / "ReleaseNotes.md" aligned_header = ( "| **Version** | **Date Modified (DD-MM-YYYY)** | " @@ -2542,7 +2559,7 @@ def test_consumed_initial_release_cannot_be_replanned_and_retries_with_none(self data_path = self.solution / "Data" / "Solution_Sample.json" data_path.parent.mkdir(parents=True) data_path.write_text( - json.dumps({"Name": "Sample", "Version": "1.2.3"}), + json.dumps({"Name": "Sample", "Version": "3.2.3"}), encoding="utf-8", ) initialized = initialize_workflow( @@ -2578,7 +2595,7 @@ def test_consumed_initial_release_cannot_be_replanned_and_retries_with_none(self def fail_after_consuming_initial_action(solution: Path, **_kwargs: str) -> dict: data = json.loads(data_path.read_text(encoding="utf-8")) - data["Version"] = "1.2.4" + data["Version"] = "3.2.4" data_path.write_text(json.dumps(data), encoding="utf-8") raise RuntimeError("synthetic first-build failure after version update") @@ -2692,7 +2709,7 @@ def test_stale_package_status_is_read_only_and_new_release_preserves_history(sel def test_failed_new_release_consumption_retries_without_second_bump(self) -> None: data = self.solution / "Data" / "Solution_Sample.json" data.parent.mkdir() - data.write_text(json.dumps({"Name": "Sample", "Version": "1.2.3"}), encoding="utf-8") + data.write_text(json.dumps({"Name": "Sample", "Version": "3.2.3"}), encoding="utf-8") run_id = self._prepare_workflow_for_package("none") with using_run(self.solution, run_id), mock.patch.object( workflow_module, @@ -2713,7 +2730,7 @@ def test_failed_new_release_consumption_retries_without_second_bump(self) -> Non def fail_after_consuming_release(solution: Path, **kwargs: str) -> dict: document = json.loads(data.read_text(encoding="utf-8")) - document["Version"] = "1.2.4" + document["Version"] = "3.2.4" data.write_text(json.dumps(document), encoding="utf-8") raise RuntimeError("synthetic failed release after version update") @@ -2741,12 +2758,12 @@ def fail_after_consuming_release(solution: Path, **kwargs: str) -> dict: def test_failed_packager_consuming_bump_allows_only_hash_matching_none_retry(self) -> None: data = self.solution / "Data" / "Solution_Sample.json" data.parent.mkdir() - data.write_text(json.dumps({"Name": "Sample", "Version": "1.2.3"}), encoding="utf-8") + data.write_text(json.dumps({"Name": "Sample", "Version": "3.2.3"}), encoding="utf-8") run_id = self._prepare_workflow_for_package("patch") def fail_after_bumping(solution: Path, **kwargs: str) -> dict: document = json.loads(data.read_text(encoding="utf-8")) - document["Version"] = "1.2.4" + document["Version"] = "3.2.4" data.write_text(json.dumps(document), encoding="utf-8") raise RuntimeError("synthetic package failure after version update") @@ -2778,13 +2795,13 @@ def test_explicit_target_version_survives_resume_and_rebuilds_without_reapplying initialize_workflow( self.solution, version_bump="patch", - target_version="3.1.2", + target_version="3.2.2", workflow_profile="authoring", ) initialized = initialize_workflow( self.solution, version_bump="none", - target_version="3.1.2", + target_version="3.2.2", workflow_profile="authoring", campaign_id="target-version-campaign", ) @@ -2795,12 +2812,12 @@ def test_explicit_target_version_survives_resume_and_rebuilds_without_reapplying resumed = initialize_workflow( self.solution, version_bump="none", - target_version="3.1.2", + target_version="3.2.2", workflow_profile="authoring", campaign_id="target-version-campaign", ) self.assertTrue(resumed["resumed"]) - self.assertEqual("3.1.2", resumed["context"]["targetVersion"]) + self.assertEqual("3.2.2", resumed["context"]["targetVersion"]) with using_run(self.solution, run_id), mock.patch.object( workflow_module, @@ -2810,40 +2827,40 @@ def test_explicit_target_version_survives_resume_and_rebuilds_without_reapplying built = package_workflow_v3_1( self.solution, requested_version_bump="none", - requested_target_version="3.1.2", + requested_target_version="3.2.2", ) - self.assertEqual("3.1.2", json.loads(data.read_text())["Version"]) - self.assertEqual("3.1.2", built["version"]) - self.assertEqual("3.1.2", workflow_status(self.solution)["packaging"]["lastConsumedAction"]["effectiveTargetVersion"]) + self.assertEqual("3.2.2", json.loads(data.read_text())["Version"]) + self.assertEqual("3.2.2", built["version"]) + self.assertEqual("3.2.2", workflow_status(self.solution)["packaging"]["lastConsumedAction"]["effectiveTargetVersion"]) package.assert_called_once() - self.assertEqual("3.1.2", package.call_args.kwargs["target_version"]) + self.assertEqual("3.2.2", package.call_args.kwargs["target_version"]) resumed = initialize_workflow( self.solution, version_bump="none", - target_version="3.1.2", + target_version="3.2.2", workflow_profile="authoring", campaign_id="target-version-campaign", ) - self.assertEqual("3.1.2", resumed["context"]["targetVersion"]) + self.assertEqual("3.2.2", resumed["context"]["targetVersion"]) rebuilt = package_workflow_v3_1( self.solution, requested_version_bump="none", ) - self.assertEqual("3.1.2", rebuilt["version"]) + self.assertEqual("3.2.2", rebuilt["version"]) self.assertIsNone(package.call_args.kwargs["target_version"]) - self.assertEqual("3.1.2", json.loads(data.read_text())["Version"]) + self.assertEqual("3.2.2", json.loads(data.read_text())["Version"]) with self.assertRaisesRegex(ValueError, "version action was already consumed"): package_workflow_v3_1( self.solution, requested_version_bump="none", - requested_target_version="3.1.3", + requested_target_version="3.2.3", ) def test_target_release_plan_binds_exact_version_and_current_inputs(self) -> None: data = self.solution / "Data" / "Solution_Sample.json" data.parent.mkdir() - data.write_text(json.dumps({"Name": "Sample", "Version": "3.1.0"}), encoding="utf-8") + data.write_text(json.dumps({"Name": "Sample", "Version": "3.2.0"}), encoding="utf-8") run_id = self._prepare_workflow_for_package("none") with using_run(self.solution, run_id), mock.patch.object( workflow_module, @@ -2857,41 +2874,41 @@ def test_target_release_plan_binds_exact_version_and_current_inputs(self) -> Non original = changed_input.read_bytes() planned = plan_new_package_release( self.solution, - target_version="3.1.2", + target_version="3.2.2", expected_package_input_sha256=package_input_identity(self.solution)["sha256"], ) - self.assertEqual("3.1.2", planned["targetVersion"]) + self.assertEqual("3.2.2", planned["targetVersion"]) self.assertEqual("target-version", workflow_status(self.solution)["packaging"]["pendingReleaseAction"]["actionType"]) with self.assertRaisesRegex(ValueError, "exactly one version bump or target version"): plan_new_package_release( self.solution, version_bump="patch", - target_version="3.1.3", + target_version="3.2.3", expected_package_input_sha256=planned["packageInputSha256"], ) with self.assertRaisesRegex(ValueError, "differs from the explicitly selected"): package_workflow_v3_1( self.solution, requested_version_bump="none", - requested_target_version="3.1.3", + requested_target_version="3.2.3", ) changed_input.write_text("kind: CustomDetection\n# input changed\n", encoding="utf-8") with self.assertRaisesRegex(ValueError, "differs from the explicitly selected"): package_workflow_v3_1( self.solution, requested_version_bump="none", - requested_target_version="3.1.2", + requested_target_version="3.2.2", ) changed_input.write_bytes(original) built = package_workflow_v3_1( self.solution, requested_version_bump="none", - requested_target_version="3.1.2", + requested_target_version="3.2.2", ) - self.assertEqual("3.1.2", built["version"]) + self.assertEqual("3.2.2", built["version"]) state = workflow_status(self.solution)["packaging"] self.assertEqual("consumed", state["releaseActions"][-1]["status"]) - self.assertEqual("3.1.2", state["releaseActions"][-1]["targetVersion"]) + self.assertEqual("3.2.2", state["releaseActions"][-1]["targetVersion"]) def test_target_version_remains_compatible_with_campaign_and_notes_refresh(self) -> None: data = self.solution / "Data" / "Solution_Sample.json" @@ -2906,11 +2923,11 @@ def test_target_version_remains_compatible_with_campaign_and_notes_refresh(self) + "| 3.1.0 | 01-09-2026 | Older release. |\n", encoding="utf-8", ) - initialized, _ = self._prepare_offline_authoring_validation() + initialized, _ = self._prepare_offline_authoring_validation(version_bump=None) initialized = initialize_workflow( self.solution, version_bump="none", - target_version="3.1.2", + target_version="3.2.2", workflow_profile="authoring", campaign_id="test-campaign", ) @@ -2927,7 +2944,7 @@ def test_target_version_remains_compatible_with_campaign_and_notes_refresh(self) ) plan_new_package_release( self.solution, - target_version="3.1.2", + target_version="3.2.2", expected_package_input_sha256=package_input_identity(self.solution)["sha256"], ) with mock.patch.object( @@ -2938,9 +2955,9 @@ def test_target_version_remains_compatible_with_campaign_and_notes_refresh(self) first = package_workflow_v3_1( self.solution, requested_version_bump="none", - requested_target_version="3.1.2", + requested_target_version="3.2.2", ) - self.assertEqual("3.1.2", first["version"]) + self.assertEqual("3.2.2", first["version"]) base_hash = package_input_identity(self.solution)["sha256"] refresh = plan_release_notes_refresh( self.solution, @@ -2948,7 +2965,7 @@ def test_target_version_remains_compatible_with_campaign_and_notes_refresh(self) change_history="Approved target release notes.", expected_package_input_sha256=base_hash, ) - row = "| 3.1.2 | 12-10-2026 | Approved target release notes. |\n" + row = "| 3.2.2 | 12-10-2026 | Approved target release notes. |\n" notes.write_text(notes.read_text(encoding="utf-8").replace( separator, separator + row, 1 ), encoding="utf-8") @@ -2966,7 +2983,7 @@ def test_target_version_remains_compatible_with_campaign_and_notes_refresh(self) requested_version_bump="none", ) self.assertEqual(first["version"], refreshed["version"]) - self.assertEqual("3.1.2", json.loads(data.read_text())["Version"]) + self.assertEqual("3.2.2", json.loads(data.read_text())["Version"]) state = workflow_status(self.solution)["packaging"] self.assertEqual("consumed", state["releaseActions"][-1]["status"]) @@ -2977,7 +2994,7 @@ def test_failed_target_version_write_retries_with_none_without_second_change(sel initialized = initialize_workflow( self.solution, version_bump="none", - target_version="3.1.2", + target_version="3.2.2", workflow_profile="authoring", campaign_id="target-version-retry-campaign", ) @@ -3001,12 +3018,12 @@ def fail_after_target_write(solution: Path, **kwargs: str) -> dict: package_workflow_v3_1( self.solution, requested_version_bump="none", - requested_target_version="3.1.2", + requested_target_version="3.2.2", ) consumed = workflow_status(self.solution)["packaging"]["lastConsumedAction"] self.assertTrue(consumed["bumpConsumed"]) self.assertTrue(consumed["targetConsumed"]) - self.assertEqual("3.1.2", consumed["effectiveTargetVersion"]) + self.assertEqual("3.2.2", consumed["effectiveTargetVersion"]) self.assertFalse(consumed["packageSucceeded"]) with using_run(self.solution, run_id), mock.patch.object( @@ -3015,10 +3032,10 @@ def fail_after_target_write(solution: Path, **kwargs: str) -> dict: side_effect=self._mock_v3_1_package, ) as package: retry = package_workflow_v3_1(self.solution, requested_version_bump="none") - self.assertEqual("3.1.2", retry["version"]) + self.assertEqual("3.2.2", retry["version"]) package.assert_called_once() self.assertIsNone(package.call_args.kwargs["target_version"]) - self.assertEqual("3.1.2", json.loads(data.read_text())["Version"]) + self.assertEqual("3.2.2", json.loads(data.read_text())["Version"]) def test_stale_package_artifact_blocks_next_workflow_stage(self) -> None: run_id = self._prepare_workflow_for_package("none") @@ -3134,13 +3151,13 @@ def test_target_version_cli_options_route_to_workflow_apis(self) -> None: "package-v3-1", "--solution", str(self.solution), "--version-bump", "none", - "--target-version", "3.1.2", + "--target-version", "3.2.2", ]) self.assertEqual(0, result) workflow_package.assert_called_once_with( str(self.solution), requested_version_bump="none", - requested_target_version="3.1.2", + requested_target_version="3.2.2", toolkit_root=None, legacy_baseline_evidence=None, ) @@ -3152,20 +3169,20 @@ def test_target_version_cli_options_route_to_workflow_apis(self) -> None: with mock.patch.object(cli_module, "configure_logging"), mock.patch.object( cli_module, "plan_new_package_release", - return_value={"targetVersion": "3.1.2", "packageInputSha256": expected_hash}, + return_value={"targetVersion": "3.2.2", "packageInputSha256": expected_hash}, ) as release_plan: result = cli_module.main([ "workflow-plan-package-release", "--solution", str(self.solution), "--run-id", initialized["runId"], - "--target-version", "3.1.2", + "--target-version", "3.2.2", "--expected-package-input-sha256", expected_hash, ]) self.assertEqual(0, result) release_plan.assert_called_once_with( str(self.solution), version_bump=None, - target_version="3.1.2", + target_version="3.2.2", expected_package_input_sha256=expected_hash, ) @@ -3178,13 +3195,13 @@ def test_same_version_rebuild_cli_requires_run_and_exact_input_scope(self) -> No with mock.patch.object(cli_module, "configure_logging"), mock.patch.object( cli_module, "plan_same_version_content_rebuild", - return_value={"version": "3.1.2", "packageInputSha256": expected_hash}, + return_value={"version": "3.2.2", "packageInputSha256": expected_hash}, ) as plan: result = cli_module.main([ "workflow-plan-same-version-content-rebuild", "--solution", str(self.solution), "--run-id", initialized["runId"], - "--target-version", "3.1.2", + "--target-version", "3.2.2", "--expected-package-input-sha256", expected_hash, "--changed-input", "XDR Detections\\Rule.yaml", "--changed-input", "Data\\Solution_Sample.json", @@ -3197,7 +3214,7 @@ def test_same_version_rebuild_cli_requires_run_and_exact_input_scope(self) -> No self.assertEqual(0, result) plan.assert_called_once_with( str(self.solution), - target_version="3.1.2", + target_version="3.2.2", changed_inputs=[ "XDR Detections\\Rule.yaml", "Data\\Solution_Sample.json", diff --git a/Tools/SentinelToXDRMigration/tests/test_xdr_description_counts.py b/Tools/SentinelToXDRMigration/tests/test_xdr_description_counts.py index 57b2953c850..f6132c82d1e 100644 --- a/Tools/SentinelToXDRMigration/tests/test_xdr_description_counts.py +++ b/Tools/SentinelToXDRMigration/tests/test_xdr_description_counts.py @@ -144,6 +144,9 @@ def test_zero_and_legacy_descriptions_are_unchanged(self): result = self.package(entry, pipeline) self.assertEqual(0, result.returncode, result.stdout + result.stderr) self.assertEqual(baseline, self.assert_summary(expected)[:2]) + data = json.loads(self.data.read_text()) + data["Version"] = "3.1.0" + self.data.write_text(json.dumps(data)) for relative, pipeline in ( ("V3/createSolutionV3.ps1", False), ("pipeline/createSolutionV4.ps1", True),