From abfba3ae5d7510101aec386f694e8a57694807a2 Mon Sep 17 00:00:00 2001 From: kingwil <5839248+kingwil@users.noreply.github.com> Date: Wed, 14 Oct 2026 21:12:40 +0100 Subject: [PATCH 1/2] Fix reviewed Defender workbook conversion and additive packaging Support conservative Type labels, scalar Granularity, union withsource bindings, native MDO tables, source-free literal queries and repository-relative connector references. Verify removed scope helpers during prerequisite scanning and preserve independent Sentinel workbook versions. Include regression tests and documentation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../V3/README-V3_1.md | 7 + .../common/defenderWorkbooks.ps1 | 187 +++++++++- .../common/versionPolicy.ps1 | 7 +- Tools/SentinelToXDRMigration/README.md | 44 +++ .../_workbook_core/data/mapping_catalog.json | 26 +- .../_workbook_core/import-manifest.json | 8 +- .../_workbook_core/migrator.py | 10 +- .../_workbook_core/rewrite.py | 326 +++++++++++++++++- .../_workbook_core/workbook.py | 35 +- .../sentinel_xdr_migration/dependencies.py | 11 +- .../workbook_dependencies.py | 9 +- .../sentinel_xdr_migration/workbooks.py | 3 + .../test_defender_workbooks_packaging.py | 281 ++++++++++++++- .../tests/test_version_policy.py | 89 ++++- .../tests/test_workbook_dependencies.py | 324 ++++++++++++++++- .../tests/test_workbooks.py | 116 ++++++- 16 files changed, 1430 insertions(+), 53 deletions(-) diff --git a/Tools/Create-Azure-Sentinel-Solution/V3/README-V3_1.md b/Tools/Create-Azure-Sentinel-Solution/V3/README-V3_1.md index 652bbab2faa..3519173ee78 100644 --- a/Tools/Create-Azure-Sentinel-Solution/V3/README-V3_1.md +++ b/Tools/Create-Azure-Sentinel-Solution/V3/README-V3_1.md @@ -247,8 +247,15 @@ V3.1 applies these hard guards in local, catalog, and positional pipeline modes: |---|---| | Effective solution/package version, even without XDR content | `>= 3.2.0` and `< 4.0.0` | | Effective XDR Custom Detection content version | `>= 3.1.0` and `< 4.0.0` | +| Effective Defender Workbook `xdrMetadata.version` | Canonical `major.minor.patch` in `[3.1.0, 4.0.0)` | +| Ordinary Sentinel Workbook metadata content version | Independent numeric version; no solution-major ceiling (for example, `4.0.0` is valid with solution `3.2.1`) | | Every other packaged content-item release version | `< 4.0.0`; existing `1.x`/`2.x` versions remain valid | +Existing Sentinel Workbook content and metadata versions are retained unchanged; +adding XDR content does not require downgrading them to match the solution major. +This distinction does not relax solution, Custom Detection, or Defender Workbook +release-version checks. + The guard reports the content type/path, effective version, and required range. It never promotes versions automatically. Explicit local bumps are evaluated before output: for example, `major` from `3.2.0` selects `4.0.0` and is rejected. diff --git a/Tools/Create-Azure-Sentinel-Solution/common/defenderWorkbooks.ps1 b/Tools/Create-Azure-Sentinel-Solution/common/defenderWorkbooks.ps1 index 95348739488..666ec7ed657 100644 --- a/Tools/Create-Azure-Sentinel-Solution/common/defenderWorkbooks.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/common/defenderWorkbooks.ps1 @@ -45,7 +45,7 @@ function Assert-DefenderWorkbookReleaseVersion { } function Assert-DefenderWorkbookDocument { - param([psobject]$Document, [string]$SolutionRoot, [string]$SolutionName, [string]$ContentPath) + param([psobject]$Document, [string]$SolutionRoot, [string]$SolutionName, [string]$ContentPath, [switch]$PassThruSource) if ($Document -isnot [pscustomobject] -or $Document.version -cne 'Notebook/1.0' -or $Document.items -isnot [array]) { @@ -132,6 +132,7 @@ function Assert-DefenderWorkbookDocument { throw "Defender Workbook '$ContentPath' target query hash is stale." } } + Assert-DefenderWorkbookRemovedScopeQueries -Source $sourceNotebook -Document $Document $workloads = @('mde', 'mdo', 'mdi', 'mda', 'mdc', 'entraid', 'purview', 'appgovernance', 'sentinel', 'mtp') foreach ($field in @('requiredWorkloads', 'requiredDataConnectors')) { $value = $provenance.PSObject.Properties[$field] @@ -161,6 +162,7 @@ function Assert-DefenderWorkbookDocument { } } } + if ($PassThruSource) { return $sourceNotebook } } function Assert-DefenderWorkbookSerializationDepth { @@ -205,11 +207,144 @@ function Get-DefenderWorkbookDeploymentName { return 'DefenderWorkbook-' + $hash.Substring(0, 32) } +function Get-DefenderWorkbookNodes { + param([AllowNull()][object]$Value, [string]$Path = '', [object]$QueryType = 0, [string]$ResourceType = '') + if ($Value -is [pscustomobject]) { + if ($null -ne $Value.PSObject.Properties['queryType']) { $QueryType = $Value.queryType } + if ($Value.resourceType) { $ResourceType = $Value.resourceType } + [pscustomobject]@{ Path = $Path; Node = $Value; QueryType = $QueryType; ResourceType = $ResourceType } + foreach ($property in $Value.PSObject.Properties) { + $key = $property.Name.Replace('~', '~0').Replace('/', '~1') + Get-DefenderWorkbookNodes -Value $property.Value -Path "$Path/$key" -QueryType $QueryType -ResourceType $ResourceType + } + } + elseif ($Value -is [array]) { + for ($i = 0; $i -lt $Value.Count; $i++) { + Get-DefenderWorkbookNodes -Value $Value[$i] -Path "$Path/$i" -QueryType $QueryType -ResourceType $ResourceType + } + } +} + +function Get-DefenderWorkbookParameterReferences { + param([AllowNull()][object]$Value, [switch]$SkipScope, [switch]$SkipParameters) + if ($Value -is [string]) { + foreach ($match in [regex]::Matches($Value, '\{([A-Za-z_][\w .-]*)(?::[^{}]*)?\}')) { $match.Groups[1].Value } + } + elseif ($Value -is [pscustomobject]) { + foreach ($property in $Value.PSObject.Properties) { + if ($SkipParameters -and $property.Name -ceq 'parameters' -and $property.Value -is [array]) { continue } + if ($SkipScope -and $property.Name -cin @( + 'crossComponentResources', 'fallbackResourceIds', 'resourceIds', 'workspaceId', + 'workspaceResourceId', 'selectedWorkspaceIds', 'resourceContextFromParameter' + )) { continue } + if ($property.Name -cin @('parameterName', 'exportParameterName', 'timeContextFromParameter', 'resourceContextFromParameter') -and + $property.Value -is [string]) { $property.Value } + if ($property.Name -ceq 'linkTarget' -and $property.Value -ceq 'parameter') { $Value.cellValue } + Get-DefenderWorkbookParameterReferences -Value $property.Value -SkipScope:$SkipScope -SkipParameters:$SkipParameters + } + } + elseif ($Value -is [array]) { + foreach ($item in $Value) { + Get-DefenderWorkbookParameterReferences -Value $item -SkipScope:$SkipScope -SkipParameters:$SkipParameters + } + } +} + +function Assert-DefenderWorkbookRemovedScopeQueries { + param([psobject]$Source, [psobject]$Document) + $removedQueries = @($Document.contentProvenance.conversion.queries | Where-Object status -CEQ 'removed-scope-helper') + if ($removedQueries.Count -eq 0) { return } + # Mirror the reviewed Python _scope_parameters dependency closure; a status alone is not proof. + $nodes = @(Get-DefenderWorkbookNodes -Value $Source) + $entries = @($nodes | Where-Object { $_.Path -cmatch '/parameters/[0-9]+$' -and $_.Node.name }) + $params = [System.Collections.Generic.Dictionary[string,object]]::new([System.StringComparer]::Ordinal) + $eligible = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + $candidates = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + $live = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + $bindings = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($entry in $nodes) { + foreach ($property in $entry.Node.PSObject.Properties) { + if ($property.Name -cin @( + 'crossComponentResources', 'fallbackResourceIds', 'resourceIds', 'workspaceId', + 'workspaceResourceId', 'selectedWorkspaceIds', 'resourceContextFromParameter' + )) { + foreach ($name in @(Get-DefenderWorkbookParameterReferences -Value $property.Value)) { [void]$bindings.Add($name) } + if ($property.Name -ceq 'resourceContextFromParameter' -and $property.Value -is [string]) { + [void]$bindings.Add($property.Value) + } + } + } + } + foreach ($entry in $entries) { + $p = $entry.Node + if ($p.name -isnot [string] -or $params.ContainsKey($p.name)) { throw 'Duplicate or invalid scope parameter names.' } + $params.Add($p.name, $entry) + $supported = -not $p.query -or $entry.QueryType -cin @(0, $null, 'advancedHunting') + if (-not ($p.query -and $supported) -and $p.type -ne 4) { + [void]$eligible.Add($p.name) + $settings = $p.typeSettings | ConvertTo-Json -Depth 100 -Compress + if ($p.type -eq 6 -or + ($p.type -eq 5 -and ($bindings.Contains($p.name) -or $entry.ResourceType -ieq 'microsoft.operationalinsights/workspaces')) -or + $settings -imatch 'microsoft\.operationalinsights/workspaces' -or + ($p.queryType -eq 1 -and $p.query -imatch 'microsoft\.operationalinsights/workspaces')) { + [void]$candidates.Add($p.name) + } + } + $entry | Add-Member -NotePropertyName AllDependencies -NotePropertyValue @(Get-DefenderWorkbookParameterReferences -Value $p) + $entry | Add-Member -NotePropertyName Dependencies -NotePropertyValue @(Get-DefenderWorkbookParameterReferences -Value $p -SkipScope:$supported) + } + do { + $before = $candidates.Count + foreach ($name in @($candidates)) { + foreach ($dep in $params[$name].AllDependencies) { + if ($eligible.Contains($dep)) { [void]$candidates.Add($dep) } + } + } + foreach ($entry in $entries) { + if ($eligible.Contains($entry.Node.name) -and $entry.Node.isHiddenWhenLocked -and + @($entry.AllDependencies | Where-Object { $candidates.Contains($_) }).Count -gt 0) { + [void]$candidates.Add($entry.Node.name) + } + } + } while ($before -ne $candidates.Count) + foreach ($name in $params.Keys) { if (-not $candidates.Contains($name)) { [void]$live.Add($name) } } + foreach ($name in @(Get-DefenderWorkbookParameterReferences -Value $Source -SkipScope -SkipParameters)) { + if ($params.ContainsKey($name)) { [void]$live.Add($name) } + } + do { + $before = $live.Count + foreach ($name in @($live)) { + $deps = if ($candidates.Contains($name)) { $params[$name].AllDependencies } else { $params[$name].Dependencies } + foreach ($dep in $deps) { if ($params.ContainsKey($dep)) { [void]$live.Add($dep) } } + } + } while ($before -ne $live.Count) + $targetNodes = @(Get-DefenderWorkbookNodes -Value $Document.items -Path '/items') + foreach ($query in $removedQueries) { + if (-not $query.sourcePath.EndsWith('/query', [System.StringComparison]::Ordinal)) { + throw 'Removed scope provenance must identify a query surface.' + } + $helper = @($entries | Where-Object { + $query.sourcePath.StartsWith($_.Path + '/', [System.StringComparison]::Ordinal) -and + $candidates.Contains($_.Node.name) -and -not $live.Contains($_.Node.name) + }) + if ($helper.Count -ne 1) { throw 'Only proven scope-helper queries may be removed.' } + $name = $helper[0].Node.name + $sourceQuery = Get-DefenderWorkbookJsonPointer -Document $Source -Pointer $query.sourcePath + if (@($targetNodes | Where-Object { + ($_.Path -cmatch '/parameters/[0-9]+$' -and $_.Node.name -ceq $name) -or + ($_.Node.query -is [string] -and $_.Node.query -ceq $sourceQuery) + }).Count -gt 0 -or + $name -cin @(Get-DefenderWorkbookParameterReferences -Value $Document.items)) { + throw 'Removed scope helper is still present or referenced in the target workbook.' + } + } +} + function Get-DefenderWorkbookKnownCatalogTables { if ($null -eq $script:DefenderWorkbookKnownCatalogTables) { $set = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) - $catalogDir = Join-Path $PSScriptRoot '../../SentinelToXDRMigration/sentinel_xdr_migration/catalog' - foreach ($file in @('xdr_only_tables.json', 'table_mappings.json')) { + $catalogDir = Join-Path $PSScriptRoot '../../SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data' + foreach ($file in @('mapping_catalog.json')) { $full = Join-Path $catalogDir $file if (Test-Path -LiteralPath $full -PathType Leaf) { $json = Get-Content -LiteralPath $full -Raw | ConvertFrom-Json @@ -229,11 +364,16 @@ function Get-DefenderWorkbookImplicitFunctions { param( [AllowNull()][object]$Value, [AllowNull()][System.Collections.Generic.HashSet[string]]$AllowedBareTables = $null, - [AllowNull()][System.Collections.Generic.HashSet[string]]$KnownFunctionAliases = $null + [AllowNull()][System.Collections.Generic.HashSet[string]]$KnownFunctionAliases = $null, + [AllowNull()][System.Collections.Generic.HashSet[string]]$ExcludedQueryPaths = $null, + [string]$Path = '' ) if ($Value -is [pscustomobject]) { foreach ($property in $Value.PSObject.Properties) { + $key = $property.Name.Replace('~', '~0').Replace('/', '~1') + $childPath = "$Path/$key" if ($property.Name -ceq 'query' -and $property.Value -is [string]) { + if ($null -ne $ExcludedQueryPaths -and $ExcludedQueryPaths.Contains($childPath)) { continue } $queryText = $property.Value if ($Value.queryType -eq 18 -and $queryText.TrimStart().StartsWith('{')) { try { $settings = $queryText | ConvertFrom-Json -ErrorAction Stop } @@ -262,6 +402,14 @@ function Get-DefenderWorkbookImplicitFunctions { } return $lit } + function Test-KqlUnionWithsourceBinding([string]$text, [int]$start) { + $prefix = [regex]::Replace($text.Substring(0, $start), $strPat, { + param($m) ' ' * $m.Length + }) + $segmentStart = [Math]::Max($prefix.LastIndexOf('|'), $prefix.LastIndexOf(';')) + return [regex]::IsMatch($prefix.Substring($segmentStart + 1), + '(?i)\bunion\b(?:\s+[A-Za-z_][\w.]*\s*=\s*[\w.*-]+)*\s+withsource\s*=\s*$') + } $protectedSpans = [System.Collections.Generic.List[pscustomobject]]::new() foreach ($m in [regex]::Matches($noComments, $bracketPat)) { $protectedSpans.Add([pscustomobject]@{Start=$m.Index; End=($m.Index + $m.Length)}) @@ -375,6 +523,7 @@ function Get-DefenderWorkbookImplicitFunctions { } } foreach ($m in [regex]::Matches($noComments, $tableAnyCallPat)) { + if (Test-KqlUnionWithsourceBinding -text $noComments -start $m.Index) { continue } $inStr = $false foreach ($sm in $strMatches) { if ($sm.Index -le $m.Index -and ($sm.Index + $sm.Length) -gt $m.Index) { @@ -428,6 +577,10 @@ function Get-DefenderWorkbookImplicitFunctions { } } foreach ($match in [regex]::Matches($codeWithTokens, "($tokPat)\s*\(|(?i)(?:^|[;{]|/\*|\*/|\bmaterialize\s*\(|\btoscalar\s*\(|\b(?:join|lookup|union)\b(?:\s+[A-Za-z_][\w.]*\s*=\s*[\w.*-]+)*)(?:\s*\()*\s*($tokPat)(?:\s*\))*\s*(?:\*/\s*)?(?=\|)|\(\s*($tokPat)\s*(?:\*/\s*)?(?=\|)|\binvoke\s+($tokPat)|\blet\s+[A-Za-z_]\w*\s*=(?:\s*\()*\s*($tokPat)(?:\s*\))*\s*(?=[;|(])|\blet\s+[A-Za-z_]\w*\s*=\s*(?:view\s*)?\([^)]*\)\s*\{(?:\s*\()*\s*($tokPat)(?:\s*\))*\s*(?=[;|(\}])|\bview\s*\([^)]*\)\s*\{(?:\s*\()*\s*($tokPat)(?:\s*\))*\s*(?=[;|(\}])|\bpartition\b(?:\s+hint\.[\w.]+\s*=\s*[\w.*-]+)*\s+by\s+[A-Za-z_]\w*\s*\{(?:\s*\()*\s*($tokPat)(?:\s*\))*\s*(?=[;|(\}])|\bfork\b(?:\s+[A-Za-z_]\w*\s*=)?\s*\((?:\s*\()*\s*($tokPat)(?:\s*\))*\s*(?=[|)\(])|\b(?:toscalar|materialize|table|totable)\s*\((?:\s*\()*\s*($tokPat)(?:\s*\))*\s*(?=[|)\(,])")) { + $scalarGranularity = [regex]::Match($codeWithTokens.Substring($match.Index), "\A(?i)let\s+gran\s*=\s*($tokPat)\s*;") + if ($scalarGranularity.Success -and $scalarGranularity.Groups[1].Value -ieq '{Granularity}' -and + @($match.Groups | Where-Object { $_.Success -and $_.Value -ceq $scalarGranularity.Groups[1].Value }).Count -gt 0 -and + [regex]::IsMatch($codeWithTokens, '(?is)\bmake-series\b.*?\bstep\s+gran\b|\bbin\s*\([^)]*,\s*gran\s*\)')) { continue } foreach ($group in $match.Groups | Select-Object -Skip 1) { if ($group.Success -and $group.Value -match '[A-Za-z0-9_]') { $calls.Add($group.Value) } } @@ -528,14 +681,16 @@ function Get-DefenderWorkbookImplicitFunctions { } else { Get-DefenderWorkbookImplicitFunctions -Value $property.Value ` - -AllowedBareTables $AllowedBareTables -KnownFunctionAliases $KnownFunctionAliases + -AllowedBareTables $AllowedBareTables -KnownFunctionAliases $KnownFunctionAliases ` + -ExcludedQueryPaths $ExcludedQueryPaths -Path $childPath } } } elseif ($Value -is [array]) { - foreach ($item in $Value) { - Get-DefenderWorkbookImplicitFunctions -Value $item ` - -AllowedBareTables $AllowedBareTables -KnownFunctionAliases $KnownFunctionAliases + for ($i = 0; $i -lt $Value.Count; $i++) { + Get-DefenderWorkbookImplicitFunctions -Value $Value[$i] ` + -AllowedBareTables $AllowedBareTables -KnownFunctionAliases $KnownFunctionAliases ` + -ExcludedQueryPaths $ExcludedQueryPaths -Path "$Path/$i" } } } @@ -546,6 +701,8 @@ function Get-DefenderWorkbookParserDependencies { [string]$SolutionRoot, [string]$SolutionName ) + $sourceNotebook = Assert-DefenderWorkbookDocument -Document $Document -SolutionRoot $SolutionRoot ` + -SolutionName $SolutionName -ContentPath 'prerequisite scan' -PassThruSource $provenance = $Document.contentProvenance $declared = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) $property = $provenance.PSObject.Properties['requiredWorkspaceFunctions'] @@ -633,17 +790,14 @@ function Get-DefenderWorkbookParserDependencies { } } } - $source = Resolve-SolutionContentPath -SolutionRoot $SolutionRoot -SolutionName $SolutionName ` - -ContentPath $provenance.source.path -RequireFile - $sourceNotebook = Get-Content -LiteralPath $source.FullPath -Raw | ConvertFrom-Json $sourceAllowedBareTables = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) foreach ($item in $allowedBareTables) { [void]$sourceAllowedBareTables.Add($item) } - $catalogPath = Join-Path $PSScriptRoot '../../SentinelToXDRMigration/sentinel_xdr_migration/catalog/table_mappings.json' + $catalogPath = Join-Path $PSScriptRoot '../../SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/mapping_catalog.json' if (Test-Path -LiteralPath $catalogPath -PathType Leaf) { $catalogJson = Get-Content -LiteralPath $catalogPath -Raw | ConvertFrom-Json if ($null -ne $catalogJson.tables) { foreach ($prop in $catalogJson.tables.PSObject.Properties) { - $targetTable = $prop.Value.target_table + $targetTable = $prop.Value.ahTable if ($targetTable -is [string] -and $allowedBareTables.Contains($targetTable) -and -not $knownFunctionAliases.Contains($prop.Name)) { [void]$sourceAllowedBareTables.Add($prop.Name) @@ -654,8 +808,13 @@ function Get-DefenderWorkbookParserDependencies { if (-not $knownFunctionAliases.Contains('SourceTable')) { [void]$sourceAllowedBareTables.Add('SourceTable') } + $excludedQueryPaths = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($query in $provenance.conversion.queries) { + if ($query.status -ceq 'removed-scope-helper') { [void]$excludedQueryPaths.Add($query.sourcePath) } + } foreach ($alias in @(Get-DefenderWorkbookImplicitFunctions -Value $sourceNotebook.items ` - -AllowedBareTables $sourceAllowedBareTables -KnownFunctionAliases $knownFunctionAliases) + + -AllowedBareTables $sourceAllowedBareTables -KnownFunctionAliases $knownFunctionAliases ` + -ExcludedQueryPaths $excludedQueryPaths -Path '/items') + @(Get-DefenderWorkbookImplicitFunctions -Value $Document.items ` -AllowedBareTables $allowedBareTables -KnownFunctionAliases $knownFunctionAliases)) { if (-not $referenced.Contains($alias)) { diff --git a/Tools/Create-Azure-Sentinel-Solution/common/versionPolicy.ps1 b/Tools/Create-Azure-Sentinel-Solution/common/versionPolicy.ps1 index 6ad02a10252..704787b84b3 100644 --- a/Tools/Create-Azure-Sentinel-Solution/common/versionPolicy.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/common/versionPolicy.ps1 @@ -10,7 +10,7 @@ function Assert-V31ReleaseVersion { if (-not $v31VersionPolicy) { return } $minimum = if ($ContentKind -eq 'Solution') { [version]'3.2.0.0' } elseif ($ContentKind -eq 'CustomDetection') { [version]'3.1.0.0' } else { [version]'0.0.0.0' } - $range = if ($ContentKind -eq 'Solution') { '>= 3.2.0 and < 4.0.0' } elseif ($ContentKind -eq 'CustomDetection') { '>= 3.1.0 and < 4.0.0' } else { '< 4.0.0' } + $range = if ($ContentKind -eq 'Solution') { '>= 3.2.0 and < 4.0.0' } elseif ($ContentKind -eq 'CustomDetection') { '>= 3.1.0 and < 4.0.0' } elseif ($ContentKind -eq 'Workbook') { 'an independent numeric content version' } else { '< 4.0.0' } $parsed = $null $valid = $Version -cmatch '\A[0-9]+\.[0-9]+(?:\.[0-9]+){0,2}\z' -and [version]::TryParse($Version, [ref]$parsed) @@ -19,7 +19,10 @@ function Assert-V31ReleaseVersion { } if ($valid) { $normalized = [version]::new($parsed.Major, $parsed.Minor, [Math]::Max(0, $parsed.Build), [Math]::Max(0, $parsed.Revision)) - $valid = $normalized -ge $minimum -and $normalized -lt [version]'4.0.0.0' + # Sentinel Workbook metadata versions predate and are independent of solution release bounds. + # Defender Workbooks use their separate Assert-DefenderWorkbookReleaseVersion guard. + $valid = $normalized -ge $minimum -and + ($ContentKind -eq 'Workbook' -or $normalized -lt [version]'4.0.0.0') } if (-not $valid) { $message = "V3.1 version policy: $ContentKind '$ContentPath' has effective release version '$Version'; required $range (numeric release version). No version is automatically promoted." diff --git a/Tools/SentinelToXDRMigration/README.md b/Tools/SentinelToXDRMigration/README.md index e01d9c868d7..810dbf5653b 100644 --- a/Tools/SentinelToXDRMigration/README.md +++ b/Tools/SentinelToXDRMigration/README.md @@ -117,6 +117,31 @@ resolution, and unresolved review findings remain blockers. The converter record source items in `workbooks-manifest.json`, separate from detection `manifest.json`. Queries removed as proven scope-only helpers remain in provenance. +The KQL rewriter treats `Type` conservatively: literal `Type` labels explicitly +projected by a query are retained, and a bare `Type` grouping in a one-line +`let Alias = NativeTable | summarize ... by ...` can be preserved by adding +that table's name as a `Type` value before aggregation. Other `Type` references +remain review findings; they are not replaced with `ActionType` or removed. + +Workbook dependency review also recognizes source-free literal `datatable` +queries limited to literal cells, direct `project` expressions, and `order by`. +Complete `range Column from Start to Stop step Step` queries are also recognized +when all three operands are signed 64-bit integer literals or finite real +literals (decimal/exponent notation), the step is nonzero, and its direction +matches the endpoints (equal endpoints are allowed). Range keywords are lowercase; +the column must be a plain ASCII identifier. No range functions, parameters, +arithmetic, casts, comments, additional statements, or pipeline/union tails are +accepted by this exception. The original query is preserved unchanged. +These literal queries require an explicit hash-bound review decision but no +telemetry table, workload, or connector prerequisite; other table-free KQL remains +unresolved. V3.1 prerequisite scanning already treats the range operator as built-in +and continues discovering function calls and telemetry branches in larger queries. +This static exception does not verify runtime visibility or parity. The identity +mappings for `MessageEvents`, `MessageUrlInfo`, `MessagePostDeliveryEvents`, and +`FileMaliciousContentInfo` are grounded in their Microsoft Learn Advanced +Hunting table references; they do not assert runtime visibility or add a schema +where none is catalogued. + The explicit solution Data `XDR Workbooks` array selects deployable content. An empty array selects none; packaging never repopulates it by scanning files. V3.1 emits workbook installation and Content Hub registration together under @@ -135,6 +160,16 @@ alternatives, and function candidates. Optional `--results --workspace-id ` imports approved lab evidence; planning and import never execute queries or authenticate. +Connector selections in solution Data accept `Data Connectors/` and, +when the solution directory is directly under `Solutions`, the exact +`Solutions//Data Connectors/` repository-relative +form. Plans return solution-relative POSIX `sourcePath` values for both forms; +source Data selections are never rewritten. References must name existing files +within the current solution's `Data Connectors` directory. Other solutions, +absolute paths, backslashes, empty/dot/traversal segments, and symlinks are +rejected. Review regenerates the same canonical plan and checks connector hashes, +while retaining the hash of the original solution Data. + Apply explicit decisions with `workbook-review --solution --run-id --plan --decisions --plan-sha256 --actor --reference --output-dir `. @@ -197,6 +232,15 @@ resources block packaging. Full workbook registration retains function prerequis the installed notebook retains only the identity/version quartet. ARM ordering does not establish AH propagation, query correctness, or provider/UI acceptance. +The V3.1 prerequisite scanner excludes a source query only when current source +file/query hashes and its JSON pointer prove it belongs to a removable scope +parameter, and the target no longer contains or references that helper. A +`removed-scope-helper` status alone is insufficient; retained helpers and +non-helper `resources` queries still block. Source and target scanning use the +shared workbook mapping catalog and reviewed scalar `Granularity` and +`union withsource=Table (print ...)` contracts. Real saved aliases and dynamic +`table()`/`totable()` sources still require review or remain blockers. + **Provider contract unverified:** generated templates and successful local tests are candidate evidence, not accepted installation or publication evidence. Independent, approved lab installation, registration/provenance readback, repeat diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/mapping_catalog.json b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/mapping_catalog.json index d93db7b9cb8..88e2ae9fefe 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/mapping_catalog.json +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/mapping_catalog.json @@ -714,7 +714,7 @@ "dropColumns": [], "applyGlobalDropColumns": false, "discriminator": null, - "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + "notes": "Identity mapping for the Microsoft Defender XDR Advanced Hunting table documented at https://learn.microsoft.com/en-us/defender-xdr/advanced-hunting-filemaliciouscontentinfo-table; visibility and runtime semantics remain unverified." }, "GCPDNS": { "ahTable": "GCPDNS", @@ -813,6 +813,30 @@ "recommendAhTable": "GraphAPIAuditEvents", "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." }, + "MessageEvents": { + "ahTable": "MessageEvents", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "notes": "Identity mapping for the Microsoft Defender XDR Advanced Hunting table documented at https://learn.microsoft.com/en-us/defender-xdr/advanced-hunting-messageevents-table; visibility and runtime semantics remain unverified." + }, + "MessagePostDeliveryEvents": { + "ahTable": "MessagePostDeliveryEvents", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "notes": "Identity mapping for the Microsoft Defender XDR Advanced Hunting table documented at https://learn.microsoft.com/en-us/defender-xdr/advanced-hunting-messagepostdeliveryevents-table; visibility and runtime semantics remain unverified." + }, + "MessageUrlInfo": { + "ahTable": "MessageUrlInfo", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "discriminator": null, + "notes": "Identity mapping for the Microsoft Defender XDR Advanced Hunting table documented at https://learn.microsoft.com/en-us/defender-xdr/advanced-hunting-messageurlinfo-table; visibility and runtime semantics remain unverified." + }, "NTANetAnalytics": { "ahTable": "NTANetAnalytics", "tier": "1to1", diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/import-manifest.json b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/import-manifest.json index dbf99bf19cd..7c664b9d023 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/import-manifest.json +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/import-manifest.json @@ -27,7 +27,7 @@ }, { "component": "data/mapping_catalog.json", - "sha256": "5a76b154eb5a077219e900392eefce12f8a2e8d18c49e723441c1c4c9d3660c7", + "sha256": "6e81e0029e971302354e607e3f103b25aae282629205f8bb44637d45cae7d159", "originalSha256": "6a3c2c8a164923ec461208f33919518e36e34242106a3a8826650c8fcde60067" }, { @@ -55,7 +55,7 @@ }, { "component": "migrator.py", - "sha256": "694a75d820016928c127eccb51b933dae8eb5dbd95b7b9840a74dfe3ddb90c82", + "sha256": "6172c2b8ecb0069b5702403b62b254f68ce5e24a06ca5b9bec35c436e06c3a93", "originalSha256": "4f7d9d0c126d283a1c92e7429e89c01085b08c239e7dce66b8953482b0a5e92f" }, { @@ -65,7 +65,7 @@ }, { "component": "rewrite.py", - "sha256": "a77ff6aaeca6c919d0ed002085f877514a47fd670b68d6d1ac2179b45640c8c2", + "sha256": "9b53e3be9877c178d0b1aa8c255ae96da451f30b956e71f4b7e30b08bafc8ab0", "originalSha256": "0c4138edf041221f0bdfc4ee00e4417d4c9d505ff8ffec7ec29d62409ef8732a" }, { @@ -75,7 +75,7 @@ }, { "component": "workbook.py", - "sha256": "238701e7fec987ec6e7bb9ae89eaf98647c09e03802db9aac889a1a5772f28ac", + "sha256": "d23ddd2ada78e4b9da717cf9a79e58d96f31a9c5f84d25727a46129cfda020b7", "originalSha256": "eb931b23333994a56e94d3c8975c7663db258b9b8032b0ff171b9e456e1b3942" }, { diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/migrator.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/migrator.py index b7df50f9a3c..1d0917efc67 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/migrator.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/migrator.py @@ -60,13 +60,19 @@ def convert(kql: str, tables: Optional[List[str]] = None, raise ValueError("Live resolution is not supported by the offline converter.") res = rewrite.rewrite(kql, tables) - status = _status_of(tables, res) + source_free_literal = ( + not tables and rewrite.is_source_free_literal_query(kql) + ) + if source_free_literal: + res["deterministic"] = True + res["overallTier"] = "1to1" + status = "converted" if source_free_literal else _status_of(tables, res) tier = res["overallTier"] mapping_tier = tier if tier in _CONTRACT_TIERS else "blocked" unsupported = [{"construct": f["name"], "kind": f["kind"], "reason": f["reason"]} for f in res["flags"]] - if tables is None: + if tables is None and not source_free_literal: unsupported.append({ "construct": "data-sources", "kind": "unresolved", "reason": resolve_error or "Data sources could not be resolved " diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/rewrite.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/rewrite.py index d3bb107599c..75d648ea9ab 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/rewrite.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/rewrite.py @@ -30,6 +30,7 @@ import argparse import json +import math import re from pathlib import Path from typing import Dict, List, Optional, Tuple @@ -125,6 +126,154 @@ def _find_identifiers(kql: str, names: set) -> set: return found +def _type_column_rewrite(kql: str, eligible_tables: set[str]) -> Tuple[str, bool, List[str]]: + """Accept only generated Type labels and direct-table summarize groupings. + + Advanced Hunting does not expose the Log Analytics `Type` column. A literal + projected as Type is already a query-defined label; a direct table summarize + can preserve the same grouping by extending that table with its own name. + All other Type references remain review findings. + """ + mask = _code_mask(kql) + masked = "".join(char if is_code else " " for char, is_code in zip(kql, mask)) + type_positions = [m.start() for m in re.finditer(r"\bType\b", masked)] + if not type_positions: + return kql, True, [] + + producers: set[int] = set() + outputs: set[int] = set() + insertions: List[Tuple[int, str, str]] = [] + + # Only recognize a one-line let binding whose input is a reviewed native + # table and whose summarize groups by the bare Type key. + line_start = 0 + for line in kql.splitlines(keepends=True): + line_end = line_start + len(line) + code_line = masked[line_start:line_end].rstrip("\r\n") + summary = re.match( + r"\s*let\s+[A-Za-z_]\w*\s*=\s*([A-Za-z_]\w*)\s*\|\s*" + r"summarize\b(.*?)\bby\b(.*?);\s*$", + code_line, re.IGNORECASE, + ) + extended_summary = re.match( + r"\s*let\s+[A-Za-z_]\w*\s*=\s*([A-Za-z_]\w*)\s*\|\s*" + r"extend\s+Type\s*=\s*\s*\|\s*summarize\b(.*?)\bby\b(.*?);\s*$", + code_line, re.IGNORECASE, + ) + is_extended = extended_summary is not None + if is_extended: + table_name = extended_summary.group(1) + summary_match = extended_summary + raw_line = kql[line_start:line_start + len(code_line)] + label = re.match( + r"\s*let\s+[A-Za-z_]\w*\s*=\s*[A-Za-z_]\w*\s*\|\s*" + r"extend\s+Type\s*=\s*(?:\"([^\"]*)\"|'([^']*)')\s*\|\s*summarize\b", + raw_line, re.IGNORECASE, + ) + literal_label = next((value for value in label.groups() if value is not None), None) if label else None + if literal_label is None or literal_label.lower() != table_name.lower(): + summary_match = None + else: + table_name = summary.group(1) if summary else None + summary_match = summary + if summary_match and table_name in eligible_tables: + key_group = 3 + key_start = line_start + summary_match.start(key_group) + key_end = line_start + summary_match.end(key_group) + key_mask = masked[key_start:key_end] + type_keys = set() + for start, end in _split_top_level_ranges(key_mask): + item = key_mask[start:end].strip() + if item.lower() == "type": + type_token = re.search(r"\bType\b", item, re.IGNORECASE) + if type_token: + type_keys.add(key_start + start + len(key_mask[start:end]) + - len(key_mask[start:end].lstrip()) + + type_token.start()) + for pos in type_positions: + if pos in type_keys: + producers.add(pos) + if not is_extended: + insertions.append(( + line_start + summary_match.end(1), + f' | extend Type = "{table_name}"', + table_name, + )) + if is_extended: + assignment = re.search(r"\bType\b", code_line[:summary_match.start(2)]) + if assignment: + producers.add(line_start + assignment.start()) + line_start = line_end + + # Split a masked expression on commas outside parentheses and retain ranges. + # This is also used for projection items so Type embedded in an expression + # (for example, a predicate or function argument) is never waved through. + for pos in type_positions: + if pos in producers: + continue + start = kql.rfind("\n", 0, pos) + 1 + end = kql.find("\n", pos) + if end < 0: + end = len(kql) + code_line = masked[start:end] + rel = pos - start + pipe = code_line.rfind("|", 0, rel) + if pipe < 0: + continue + clause = code_line[pipe + 1:] + projection = re.match(r"\s*project(?:-reorder)?(?=\s|$)(.*)$", clause, re.IGNORECASE) + if not projection: + continue + items_start = pipe + 1 + projection.start(1) + next_pipe = code_line.find("|", items_start) + items_end = next_pipe if next_pipe >= 0 else len(code_line) + item_region = code_line[items_start:items_end] + item_ranges = _split_top_level_ranges(item_region) + for item_start, item_end in item_ranges: + absolute_start = start + items_start + item_start + absolute_end = start + items_start + item_end + if absolute_start <= pos < absolute_end: + item = code_line[items_start + item_start:items_start + item_end].strip() + raw_item = kql[absolute_start:absolute_end].strip() + if item.lower() == "type": + after_projection = masked[start + items_end:] + if (re.search(r"\bunion\b", masked[:pos], re.IGNORECASE) + and not after_projection.strip(" \t\r\n;")): + outputs.add(pos) + elif (item.lower().startswith("type") + and re.fullmatch( + r"Type\s*=\s*(?:\"(?:\\.|[^\"\\])*\"|'(?:\\.|[^'\\])*')", + raw_item, re.IGNORECASE, + )): + producers.add(pos) + break + + if not producers or any(pos not in producers and pos not in outputs + for pos in type_positions): + return kql, False, [] + + converted = kql + for offset, insertion, _ in sorted(insertions, reverse=True): + converted = converted[:offset] + insertion + converted[offset:] + return converted, True, [table for _, _, table in insertions] + + +def _split_top_level_ranges(text: str) -> List[Tuple[int, int]]: + """Return comma-delimited ranges outside parentheses.""" + parts: List[Tuple[int, int]] = [] + depth, start = 0, 0 + for index, char in enumerate(text): + if char == "(": + depth += 1 + elif char == ")": + depth -= 1 + elif char == "," and depth == 0: + parts.append((start, index)) + start = index + 1 + parts.append((start, len(text))) + return parts + + def _code_mask(kql: str) -> List[bool]: """Boolean mask, True where the char is in a code segment (not string/comment).""" mask: List[bool] = [] @@ -133,6 +282,152 @@ def _code_mask(kql: str) -> List[bool]: return mask +def is_source_free_literal_range(kql: str) -> bool: + """Recognize a complete numeric range operator, never expressions or tails.""" + number = r"-?[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?" + match = re.fullmatch( + rf"\s*range\s+[A-Za-z_][A-Za-z0-9_]*\s+from\s+({number})" + rf"\s+to\s+({number})\s+step\s+({number})\s*", + kql, + ) + if not match: + return False + values = [] + for literal in match.groups(): + try: + if re.fullmatch(r"-?[0-9]+", literal): + value = int(literal) + if not -(2**63) <= value < 2**63: + return False + else: + value = float(literal) + if not math.isfinite(value): + return False + except ValueError: + return False + values.append(value) + start, stop, step = values + return step != 0 and (start == stop or (stop > start) == (step > 0)) + + +def is_source_free_literal_query(kql: str) -> bool: + """Shared empty-lineage exception for independently verified literal inputs.""" + return is_source_free_literal_range(kql) or is_source_free_literal_datatable(kql) + + +def is_source_free_literal_datatable(kql: str) -> bool: + """Recognize only literal datatable panels with direct projections/sorting.""" + match = re.fullmatch( + r"\s*datatable\s*\(([^()]*)\)\s*\[(.*?)\]\s*(.*?)\s*", + kql, flags=re.IGNORECASE | re.DOTALL, + ) + if not match: + return False + + field = r"(?:[A-Za-z_]\w*|\[[^\]\r\n]+\])" + schema = [] + for item in match.group(1).split(","): + column = re.fullmatch( + rf"\s*([A-Za-z_]\w*)\s*:\s*(int|long|real|string|bool)\s*", + item, flags=re.IGNORECASE, + ) + if not column: + return False + schema.append((column.group(1), column.group(2).lower())) + if not schema or len({name.casefold() for name, _ in schema}) != len(schema): + return False + available_columns = {name.casefold() for name, _ in schema} + + literal = re.compile( + r'\s*(?:(?P"(?:[^"]|"")*"|\'(?:[^\']|\'\')*\')' + r"|(?Ptrue|false)" + r"|(?P-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)" + r"|(?Pnull))\s*", + flags=re.IGNORECASE, + ) + values = [] + value_text = match.group(2) + position = 0 + while position < len(value_text): + value_match = literal.match(value_text, position) + if not value_match: + return False + values.append(next( + (key, value) for key, value in value_match.groupdict().items() if value is not None + )) + position = value_match.end() + if position < len(value_text): + if value_text[position] != ",": + return False + position += 1 + if position == len(value_text): + return False + if len(values) % len(schema): + return False + for index, (kind, value) in enumerate(values): + expected = schema[index % len(schema)][1] + if kind != "null" and not ( + expected == "string" and kind == "string" + or expected == "bool" and kind == "boolean" + or expected in ("int", "long") and kind == "number" + and re.fullmatch(r"-?\d+", value) is not None + or expected == "real" and kind == "number" + ): + return False + + tail = match.group(3).strip() + if not tail: + return True + clauses = re.split(r"\s*\|\s*", tail) + if clauses[0] != "" or len(clauses) > 3: + return False + clauses = clauses[1:] + if not clauses: + return False + seen_order = False + seen_project = False + + def column_name(expression): + value = expression.strip() + if value.startswith("[") and value.endswith("]"): + value = value[1:-1].strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + value = value[1:-1] + return value.casefold() + + for clause in clauses: + project = re.fullmatch(r"project\s+(.+)", clause, flags=re.IGNORECASE) + order = re.fullmatch(r"order\s+by\s+(.+)", clause, flags=re.IGNORECASE) + if project and not seen_order and not seen_project: + projected_columns = set() + for expression in project.group(1).split(","): + projection = re.fullmatch( + rf"\s*(?:(?P{field})\s*=\s*)?(?P{field})\s*", + expression, + ) + if not projection: + return False + source = column_name(projection.group("source")) + if source not in available_columns: + return False + alias = projection.group("alias") + projected_columns.add(column_name(alias) if alias else source) + available_columns = projected_columns + seen_project = True + elif order and not seen_order: + seen_order = True + for expression in order.group(1).split(","): + ordering = re.fullmatch( + rf"\s*(?P{field})(?:\s+(?:asc|desc))?\s*", + expression, flags=re.IGNORECASE, + ) + if not ordering or column_name(ordering.group("column")) not in available_columns: + return False + else: + return False + return True + + def _is_word(kql: str, start: int, end: int) -> bool: """True if kql[start:end] is bounded by non-identifier chars (whole word).""" before = kql[start - 1] if start > 0 else "" @@ -304,9 +599,22 @@ def rewrite(kql: str, tables: Optional[List[str]], if entry.get("removedColumns"): removed_cols[t] = list(entry["removedColumns"]) + eligible_type_tables = { + t for t in tables + if t in tbl_map and tbl_map[t].get("tier") == "1to1" + and tbl_map[t].get("ahTable", t) == t + } + type_is_la_only = "Type" in drop_cols + source_has_type = type_is_la_only and bool(_find_identifiers(kql, {"Type"})) + if type_is_la_only: + converted, type_supported, synthesized_types = _type_column_rewrite( + kql, eligible_type_tables) + else: + converted, type_supported, synthesized_types = kql, True, [] + # Apply table + column renames in one whole-identifier pass. rename_map = {**col_renames, **table_renames} - converted, counts = _apply_renames(kql, rename_map) + converted, counts = _apply_renames(converted, rename_map) # Drop redundant duplicate group-by keys (AH rejects duplicate output columns). converted, dup_keys = _dedupe_summarize_keys(converted) @@ -330,6 +638,12 @@ def rewrite(kql: str, tables: Optional[List[str]], "count": 1, "source": "deterministic", "reason": "Duplicate summarize group-by key removed; " "AH rejects duplicate output columns"}) + for table in synthesized_types: + diff.append({ + "kind": "synthesize-type-column", "before": "Type", + "after": f'Type = "{table}"', "count": 1, "source": "deterministic", + "reason": "Preserve a direct table summarize grouping with its native table label", + }) diff.extend(cast_diffs) diff.extend(int_diffs) @@ -364,10 +678,16 @@ def rewrite(kql: str, tables: Optional[List[str]], diff.sort(key=lambda d: (d["kind"], d["before"])) # Flag references to LA-only columns that don't exist in AH (never delete). - for col in sorted(_find_identifiers(kql, drop_cols)): + la_only_references = _find_identifiers(kql, drop_cols - {"Type"}) + if source_has_type and not type_supported: + la_only_references.add("Type") + for col in sorted(la_only_references): flags.append({"kind": "la-only-column", "name": col, "reason": "Log-Analytics-only column not present in AH; " - "query references it and needs manual handling."}) + "query references it and needs manual handling." + if col != "Type" else + "Type is not a recognized query-defined label or a direct-table " + "summarize key; review its source semantics."}) overall = "1to1" if tables: diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/workbook.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/workbook.py index 74d879a6839..de65f8ccd41 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/workbook.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/workbook.py @@ -275,6 +275,17 @@ def _unquote_kql_string(literal: str) -> str: return literal +def _is_union_withsource_output_binding(code: str, start: int) -> bool: + prefix = _QUOTED_LITERAL_RE.sub(lambda m: " " * len(m.group(0)), code[:start]) + segment_start = max(prefix.rfind("|"), prefix.rfind(";")) + return re.search( + r"\bunion\b(?:\s+[A-Za-z_][\w.]*\s*=\s*[\w.*-]+)*" + r"\s+withsource\s*=\s*$", + prefix[segment_start + 1:], + re.IGNORECASE, + ) is not None + + def _union_operands(code): # Union operands are separated at the union's own depth, not by commas # inside function arguments. Include later operands, not only the first. @@ -361,6 +372,8 @@ def _quoted_calls_and_table_sources(query: str) -> tuple[set[str], set[str]]: for m in _TABLE_ANY_CALL_RE.finditer(no_comments): if any(s <= m.start() < e for s, e in outer_string_spans): continue + if _is_union_withsource_output_binding(no_comments, m.start()): + continue if not any(s == m.start() for s, _ in valid_spans): calls.add("{__unsupported_table_indirection__}") @@ -483,7 +496,24 @@ def _parameter_source_or_call_tokens(query: str) -> set[str]: code, flags=re.IGNORECASE, ): - tokens.update(g for g in match.groups() if g) + captured = [g for g in match.groups() if g] + scalar_granularity = False + let_match = re.match( + rf"\blet\s+gran\s*=\s*({tok_pat})\s*;", + code[match.start():], + flags=re.IGNORECASE, + ) + if let_match and let_match.group(1) in captured: + token = let_match.group(1) + scalar_granularity = ( + token.casefold() == "{granularity}" + and bool(re.search( + r"\bmake-series\b.*?\bstep\s+gran\b|\bbin\s*\([^)]*,\s*gran\s*\)", + code, flags=re.IGNORECASE | re.DOTALL, + )) + ) + if not scalar_granularity: + tokens.update(captured) for m in re.finditer(rf"(\(\s*(?:\(\s*)*)({tok_pat})((?:\s*\))*)\s*(?:\*/\s*)?(?=\|)", code): if m.group(1).count("(") > m.group(3).count(")"): tokens.add(m.group(2)) @@ -656,7 +686,8 @@ def issue(path, code, message): if surface["parameter"]: gates.add("ah-query-backed-parameters") # The mapping catalog's resolve-to-self does not establish blank-selection visibility. - gates.add("blank-ah-selection-table-visibility") + if contract["resolvedTables"]: + gates.add("blank-ah-selection-table-visibility") if opaque: gates.add("sentinel-workspace-function-availability-and-branch-coverage") if re.search(r"\bexternaldata\s*[\[(]", code): diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/dependencies.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/dependencies.py index ac176fe24fc..dbea438e543 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/dependencies.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/dependencies.py @@ -29,7 +29,8 @@ def telemetry_data_types(tables): **{table: "mde" for table in NATIVE_XDR_TABLES if table.startswith("Device")}, **{table: "mdo" for table in ( "EmailAttachmentInfo", "EmailEvents", "EmailPostDeliveryEvents", - "EmailUrlInfo", "UrlClickEvents", + "EmailUrlInfo", "FileMaliciousContentInfo", "MessageEvents", + "MessagePostDeliveryEvents", "MessageUrlInfo", "UrlClickEvents", )}, **{table: "mdi" for table in ( "IdentityDirectoryEvents", "IdentityLogonEvents", "IdentityQueryEvents", @@ -125,9 +126,10 @@ def dependency_plan( connectors: Any = None, table_mappings: dict[str, str] | None = None, ) -> dict[str, Any]: - resolution = resolve_data_types(source_query, metadata) - source_types = set(resolution["dataTypes"]) from ._workbook_core import rewrite + source_free_literal = rewrite.is_source_free_literal_query(source_query) + resolution = resolve_data_types(source_query, None if source_free_literal else metadata) + source_types = set(resolution["dataTypes"]) code = "".join(text if kind == "code" else " " * len(text) for kind, text in rewrite._segments(source_query)) if re.search(r"\b_GetWatchlist\s*\(", code): @@ -149,7 +151,8 @@ def dependency_plan( target_tables = telemetry_data_types(target_table_names) workloads, unknown = workload_records(target_tables) reasons = [] - if not source_types and not (builtin_asim_parsers and isinstance(connectors, list)): + if (not source_types and not source_free_literal + and not (builtin_asim_parsers and isinstance(connectors, list))): reasons.append("No source inputs resolved; provide source evidence and any telemetry connector prerequisites.") if unknown: reasons.append(f"Provide workload mappings for: {', '.join(unknown)}.") diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_dependencies.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_dependencies.py index 5be6f6cb509..77a24a35ee7 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_dependencies.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_dependencies.py @@ -44,6 +44,10 @@ def _connectors(root, data): if not isinstance(references, list): raise ValueError("Solution Data Connectors must be an array.") for relative in references: + prefix = f"Solutions/{root.name}/" + if (isinstance(relative, str) and root.parent.name == "Solutions" + and relative.startswith(prefix)): + relative = relative[len(prefix):] path = _path(root, relative) if not relative.startswith("Data Connectors/") or not path.is_file(): raise ValueError("Connector references must identify existing Data Connectors content.") @@ -58,7 +62,7 @@ def _connectors(root, data): names = sorted({entry.get("name") if isinstance(entry, dict) else entry for entry in types if isinstance(entry, str) or isinstance(entry, dict) and isinstance(entry.get("name"), str)}) alternatives.append({ - "sourcePath": relative, "sourceSha256": _sha(path.read_bytes()), + "sourcePath": path.relative_to(root).as_posix(), "sourceSha256": _sha(path.read_bytes()), "connectorId": connector_id, "dataTypes": telemetry_data_types(names), **({"referenceInputs": ["Watchlist"]} if "Watchlist" in names else {}), "title": value.get("title") or (nested.get("connectorUiConfig") or {}).get("title"), @@ -337,7 +341,8 @@ def review_workbook_dependencies(solution, *, plan_path, decisions_path, expecte raise ValueError("Reviewed workspaceFunctions must match exact source aliases.") if not set(aliases) <= declared_aliases or set(aliases) & set(tables): raise ValueError("Function aliases need vetted prerequisites and separate underlying tables.") - if not tables and not row["scopeHelper"]: + source_free_literal = rewrite.is_source_free_literal_query(row["queryKql"]) + if not tables and not row["scopeHelper"] and not source_free_literal: raise ValueError("Accepted query requires explicit underlying source tables.") resolution = {"querySha256": row["querySha256"], "tables": tables, "evidence": {"actor": actor, "reference": reference, "planSha256": expected_plan_sha256, diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbooks.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbooks.py index 19702d6af1e..48fc887f8f1 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbooks.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbooks.py @@ -515,6 +515,9 @@ def validate_workbooks(solution, *, selected_only=False) -> dict: contract = {} resolved_tables = (set(contract.get("resolvedTables", [])) if isinstance(contract.get("resolvedTables"), list) else set()) + if (not resolved_tables and + not workbook.rewrite.is_source_free_literal_query(raw_source)): + errors.append("Empty source-table lineage is allowed only for a verified literal datatable or numeric range query.") aliases = query.get("workspaceFunctions", []) alias_set = (set(aliases) if isinstance(aliases, list) and all(isinstance(a, str) for a in aliases) else set()) diff --git a/Tools/SentinelToXDRMigration/tests/test_defender_workbooks_packaging.py b/Tools/SentinelToXDRMigration/tests/test_defender_workbooks_packaging.py index ac3e5caed36..b6f1582ec44 100644 --- a/Tools/SentinelToXDRMigration/tests/test_defender_workbooks_packaging.py +++ b/Tools/SentinelToXDRMigration/tests/test_defender_workbooks_packaging.py @@ -206,6 +206,252 @@ def add_function( self.write() return resource + def invoke_prerequisites(self, *, success: bool = True) -> str: + self.write() + payload = self.root / "prerequisites.json" + payload.write_text(json.dumps({"data": self.data, "template": self.template}), encoding="utf-8") + result = subprocess.run([ + "pwsh", "-NoLogo", "-NoProfile", "-NonInteractive", "-Command", + "$ErrorActionPreference='Stop'; " + f". {literal(COMMON)}; " + f"$inputData=Get-Content -Raw -LiteralPath {literal(payload)} | ConvertFrom-Json; " + f"$doc=Get-Content -Raw -LiteralPath {literal(self.target)} | ConvertFrom-Json; " + "Get-DefenderWorkbookParserDependencies -Document $doc -ContentToImport $inputData.data " + f"-Template $inputData.template -SolutionRoot {literal(self.solution)} -SolutionName Sample; " + "Write-Output '__PREREQUISITES_PASSED__'", + ], capture_output=True, text=True, check=False) + self.assertEqual(success, result.returncode == 0, result.stderr) + self.assertEqual(success, "__PREREQUISITES_PASSED__" in result.stdout) + return result.stderr + + def scope_fixture(self) -> None: + original = json.loads(self.source.read_bytes()) + original["items"][1] = {"content": { + "resourceType": "microsoft.operationalinsights/workspaces", + "parameters": [{ + "name": "Workspace", "type": 5, "queryType": 1, + "query": "resources | where type =~ 'microsoft.operationalinsights/workspaces'", + }], + }} + self.source.write_text(json.dumps(original), encoding="utf-8") + provenance = self.notebook["contentProvenance"] + provenance["source"]["sha256"] = hashlib.sha256(self.source.read_bytes()).hexdigest() + provenance["conversion"]["queries"].append({ + "sourcePath": "/items/1/content/parameters/0/query", + "sourceSha256": hashlib.sha256(original["items"][1]["content"]["parameters"][0]["query"].encode()).hexdigest(), + "outputPath": None, "targetSha256": None, "status": "removed-scope-helper", + }) + + def test_prerequisite_scan_excludes_only_proven_removed_scope_helpers(self) -> None: + self.scope_fixture() + source_before = self.source.read_bytes() + self.invoke_prerequisites() + self.assertEqual(source_before, self.source.read_bytes()) + target_before = self.target.read_bytes() + self.invoke_prerequisites() + self.assertEqual(target_before, self.target.read_bytes()) + baseline = copy.deepcopy(self.notebook) + for mutation in ( + "source-hash", "query-hash", "pointer", "status", "output", + "retained-parameter", "retained-query", "retained-reference", + ): + with self.subTest(mutation=mutation): + self.notebook = copy.deepcopy(baseline) + provenance = self.notebook["contentProvenance"] + query = provenance["conversion"]["queries"][-1] + if mutation == "source-hash": + provenance["source"]["sha256"] = "0" * 64 + elif mutation == "query-hash": + query["sourceSha256"] = "0" * 64 + elif mutation == "pointer": + query["sourcePath"] = "/items/99/query" + elif mutation == "status": + query["status"] = "blocked" + elif mutation == "output": + query["outputPath"] = "/items/0/content/query" + elif mutation == "retained-parameter": + self.notebook["items"].append({"content": {"parameters": [{"name": "Workspace", "type": 5}]}}) + elif mutation == "retained-query": + self.notebook["items"].append({"query": json.loads(source_before)["items"][1]["content"]["parameters"][0]["query"]}) + else: + self.notebook["items"].append({"content": {"json": "{Workspace}"}}) + self.invoke_prerequisites(success=False) + + def test_prerequisite_scope_proof_rejects_nonhelpers_and_live_dependencies(self) -> None: + self.scope_fixture() + baseline_source = json.loads(self.source.read_bytes()) + baseline = copy.deepcopy(self.notebook) + for mutation in ("ordinary-query", "kql-parameter", "time-parameter", "live-text", "live-parameter", "duplicate-name"): + with self.subTest(mutation=mutation): + source = copy.deepcopy(baseline_source) + self.notebook = copy.deepcopy(baseline) + param = source["items"][1]["content"]["parameters"][0] + if mutation == "ordinary-query": + source["items"][1] = {"query": param["query"]} + self.notebook["contentProvenance"]["conversion"]["queries"][-1]["sourcePath"] = "/items/1/query" + elif mutation == "kql-parameter": + param["queryType"] = 0 + elif mutation == "time-parameter": + param["type"] = 4 + elif mutation == "live-text": + source["items"].append({"content": {"json": "{Workspace}"}}) + elif mutation == "live-parameter": + source["items"][1]["content"]["parameters"].append({ + "name": "Filter", "type": 1, "value": "{Workspace}", + }) + else: + source["items"][1]["content"]["parameters"].append(copy.deepcopy(param)) + self.source.write_text(json.dumps(source), encoding="utf-8") + self.notebook["contentProvenance"]["source"]["sha256"] = hashlib.sha256(self.source.read_bytes()).hexdigest() + self.invoke_prerequisites(success=False) + + def test_prerequisite_scope_dependency_closure_matches_python(self) -> None: + from sentinel_xdr_migration._workbook_core import workbook + self.scope_fixture() + source = json.loads(self.source.read_bytes()) + params = source["items"][1]["content"]["parameters"] + params[0]["query"] += " | where subscriptionId in ({Subscription})" + params.append({"name": "Subscription", "type": 6, "value": "all"}) + params.append({ + "name": "ContextFree", "type": 1, "isHiddenWhenLocked": True, + "queryType": 1, "query": "print context='{Workspace}'", + }) + source["items"][0]["content"]["resourceContextFromParameter"] = "Workspace" + _, removed, retained = workbook._scope_parameters(source) + self.assertEqual({"Workspace", "Subscription", "ContextFree"}, removed) + self.assertFalse(retained) + self.source.write_text(json.dumps(source), encoding="utf-8") + provenance = self.notebook["contentProvenance"] + provenance["source"]["sha256"] = hashlib.sha256(self.source.read_bytes()).hexdigest() + provenance["conversion"]["queries"][-1]["sourceSha256"] = hashlib.sha256(params[0]["query"].encode()).hexdigest() + provenance["conversion"]["queries"].append({ + "sourcePath": "/items/1/content/parameters/2/query", + "sourceSha256": hashlib.sha256(params[2]["query"].encode()).hexdigest(), + "outputPath": None, "targetSha256": None, "status": "removed-scope-helper", + }) + self.invoke_prerequisites() + source["items"].append({"content": {"json": "{ContextFree}"}}) + _, removed, retained = workbook._scope_parameters(source) + self.assertFalse(removed) + self.assertEqual({"Workspace", "Subscription", "ContextFree"}, retained) + self.source.write_text(json.dumps(source), encoding="utf-8") + provenance["source"]["sha256"] = hashlib.sha256(self.source.read_bytes()).hexdigest() + self.invoke_prerequisites(success=False) + + def test_prerequisite_scanner_matches_reviewed_python_query_contracts(self) -> None: + from sentinel_xdr_migration._workbook_core import workbook + queries = [ + "let gran = {Granularity}; EmailEvents | summarize count() by bin(Timestamp, gran)", + "let gran = {Granularity}; EmailEvents | make-series count() on Timestamp step gran", + "union withsource=Table (print value=1), (EmailEvents | take 1)", + "union kind=outer withsource=Table (print value=1), (HiddenParser())", + "union withsource=totable (print value=1), (MessageEvents | take 1)", + "union withsource=Table (print value=1), table(Name)", + "EmailPostDeliveryEvents | take 1", + "MessagePostDeliveryEvents | take 1", + "MessageUrlInfo | take 1", + "FileMaliciousContentInfo | take 1", + "datatable(label:string, value:long)['EmailEvents', 1] | extend doubled=value*2", + "let gran = {Granularity}; DeviceEvents | take 1", + "let gran = {Other}; EmailEvents | summarize count() by bin(Timestamp, gran)", + "{Granularity}() | take 1", + "{Granularity} | take 1", + "table({Granularity}) | take 1", + "totable(strcat('Email', 'Events')) | take 1", + "resources | take 1", + "['resources'] | take 1", + "EmailEvents() | take 1", + "HiddenParser() | take 1", + "DeviceEvents | invoke HiddenParser()", + "let Data = HiddenParser(); Data", + "table('HiddenParser') | take 1", + "table('EmailEvents') | take 1", + "datatable(label:string)['constant'] | join (HiddenParser()) on label", + "let gran = {Granularity}; EmailEvents | summarize count() by bin(Timestamp, gran); table(Name)", + "union withsource=Other (table(Name)), HiddenParser", + "range x from 0 to 10 step 1", + "range x from 0 to 20 step 1", + "range x from 0 to HiddenParser() step 1", + "range x from 0 to toscalar(HiddenParser() | count) step 1", + "range x from 0 to 10 step 1 | union HiddenParser", + "union (range x from 0 to 10 step 1), (HiddenParser())", + "range x from 0 to 10 step 1 | union EmailEvents", + ] + payload = self.root / "scanner-queries.json" + payload.write_text(json.dumps(queries), encoding="utf-8") + result = subprocess.run([ + "pwsh", "-NoLogo", "-NoProfile", "-NonInteractive", "-Command", + "$ErrorActionPreference='Stop'; " + f". {literal(COMMON)}; " + f"$queries=Get-Content -Raw -LiteralPath {literal(payload)} | ConvertFrom-Json; " + "$rows=@(foreach ($q in $queries) { ,@(Get-DefenderWorkbookImplicitFunctions " + "-Value ([pscustomobject]@{query=$q}) | Sort-Object -Unique) }); " + "ConvertTo-Json -InputObject $rows -Depth 10 -Compress", + ], capture_output=True, text=True, check=False) + self.assertEqual(0, result.returncode, result.stderr) + actual = json.loads(result.stdout) + for query, aliases in zip(queries, actual, strict=True): + with self.subTest(query=query): + self.assertEqual(sorted(workbook.opaque_functions(query)), sorted(aliases)) + + def test_reviewed_literal_range_dropdowns_package_without_dependencies(self) -> None: + from sentinel_xdr_migration import workbook_dependencies, workbooks + from test_workbook_dependencies import decisions, review + + source = json.loads(self.source.read_bytes()) + source["items"] = [{"type": 9, "content": {"parameters": [ + {"name": f"Range{i}", "type": 2, "queryType": 0, "query": query} + for i, query in enumerate(( + "range x from 0 to 10 step 1", "range x from 0 to 20 step 1", + )) + ]}}] + self.source.write_text(json.dumps(source), encoding="utf-8") + data = self.solution / "Data/Solution_Sample.json" + data.parent.mkdir() + data.write_text(json.dumps({"Version": "3.2.7"}), encoding="utf-8") + plan = workbook_dependencies.plan_workbook_dependencies(self.solution) + reviewed = decisions(plan) + for row in reviewed["queries"]: + row["tables"] = [] + reviewed["workbooks"][0]["requiredWorkloads"] = [] + receipt = review(self.solution, plan, reviewed) + self.assertFalse(receipt["dependencyReviewRequired"]) + settings = json.loads((self.solution / "Logs/review-1/conversion-config.json").read_bytes()) + result = workbooks.convert_workbooks( + self.solution, config=settings, report_directory=self.solution / "Logs/conversion") + self.assertEqual(1, result["converted"]) + self.target = self.solution / "XDR Workbooks/nested/source.json" + self.reference = self.target.relative_to(self.solution).as_posix() + self.data["XDR Workbooks"] = [self.reference] + self.notebook = json.loads(self.target.read_bytes()) + self.assertEqual(1, workbooks.validate_workbooks(self.solution)["valid"]) + emitted = self.invoke(package=True) + self.assertEqual(1, emitted["count"]) + resources = [r for r in emitted["template"]["resources"] + if r.get("properties", {}).get("contentKind") == "DefenderWorkbook"] + self.assertEqual(1, len(resources)) + install = next(r for r in emitted["template"]["resources"] + if r["type"] == "Microsoft.Resources/deployments") + installed = install["properties"]["template"]["resources"]["workbook"]["properties"]["content"] + self.assertEqual( + ["range x from 0 to 10 step 1", "range x from 0 to 20 step 1"], + [p["query"] for p in installed["items"][0]["content"]["parameters"]], + ) + provenance = self.notebook["contentProvenance"] + self.assertEqual([], provenance["requiredWorkloads"]) + self.assertEqual([], provenance["requiredDataConnectors"]) + self.assertEqual([], provenance.get("requiredWorkspaceFunctions", [])) + + def test_prerequisite_scanner_preserves_known_saved_table_aliases(self) -> None: + self.add_function("EmailEvents") + self.notebook["items"][0]["content"]["query"] = "EmailEvents | take 1" + query = self.notebook["contentProvenance"]["conversion"]["queries"][0] + query["targetSha256"] = hashlib.sha256(b"EmailEvents | take 1").hexdigest() + query["workspaceFunctions"] = [] + self.notebook["contentProvenance"]["requiredWorkspaceFunctions"] = [] + error = self.invoke_prerequisites(success=False) + self.assertIn("unreviewed implicit workspace function 'EmailEvents'", error) + def test_required_parser_dependencies_use_emitted_names_and_preserve_provenance(self) -> None: self.add_function() self.add_function("SecondParser", folder="Workspace Functions", name="lab/arm-name-not-alias") @@ -796,6 +1042,36 @@ def test_version_bounds_and_package_boundary(self) -> None: self.template["variables"]["_solutionVersion"] = "3.2.6" self.invoke(success=False) + def test_source_passthrough_validates_all_dependencies_and_handles_empty_arrays(self) -> None: + baseline = copy.deepcopy(self.notebook) + cases = ( + ({}, True), + ({"requiredWorkloads": [], "requiredDataConnectors": []}, True), + ({"requiredDataConnectors": "[]"}, False), + ({"requiredWorkloads": [ + {"workload": "mde", "dataTypes": ["DeviceEvents"]}, + {"workload": "unknown", "dataTypes": []}, + ]}, False), + ) + for changes, success in cases: + with self.subTest(changes=changes): + self.notebook = copy.deepcopy(baseline) + self.notebook["contentProvenance"].update(changes) + self.write() + result = subprocess.run([ + "pwsh", "-NoLogo", "-NoProfile", "-NonInteractive", "-Command", + "$ErrorActionPreference='Stop'; " + f". {literal(COMMON)}; " + f"$doc=Get-Content -Raw -LiteralPath {literal(self.target)} | ConvertFrom-Json; " + "$source=Assert-DefenderWorkbookDocument -Document $doc " + f"-SolutionRoot {literal(self.solution)} -SolutionName Sample " + "-ContentPath 'fixture' -PassThruSource; " + "ConvertTo-Json -InputObject $source -Depth 100 -Compress", + ], capture_output=True, text=True, check=False) + self.assertEqual(success, result.returncode == 0, result.stderr) + if success: + self.assertEqual(json.loads(self.source.read_bytes()), json.loads(result.stdout)) + def test_review_missing_arrays_unknown_workload_and_stale_source_block(self) -> None: baseline = copy.deepcopy(self.notebook) mutations = [ @@ -882,10 +1158,7 @@ def test_query_current_hash_blockers_and_pruned_scope_record(self) -> None: self.write() self.invoke(success=False) self.notebook = copy.deepcopy(baseline) - self.notebook["contentProvenance"]["conversion"]["queries"].append({ - "sourcePath": "/items/1/query", "sourceSha256": hashlib.sha256(b"scope").hexdigest(), - "outputPath": None, "targetSha256": None, "status": "removed-scope-helper", - }) + self.scope_fixture() self.write() self.assertEqual(1, self.invoke()["count"]) diff --git a/Tools/SentinelToXDRMigration/tests/test_version_policy.py b/Tools/SentinelToXDRMigration/tests/test_version_policy.py index 8bffd77559a..6bb4a45d0e7 100644 --- a/Tools/SentinelToXDRMigration/tests/test_version_policy.py +++ b/Tools/SentinelToXDRMigration/tests/test_version_policy.py @@ -311,16 +311,16 @@ def test_legacy_context_is_unrestricted_and_checks_cannot_be_swallowed(self): f". {policy}; " "Assert-V31ReleaseVersion -Version '4.0.0' -ContentKind Solution -ContentPath 'Legacy'; " "$v31VersionPolicy = @{ Errors = [System.Collections.Generic.List[string]]::new() }; " - "try { Assert-V31ReleaseVersion -Version '4.0.0' -ContentKind Workbook -ContentPath 'Example.json' } catch {}; " + "try { Assert-V31ReleaseVersion -Version '4.0.0' -ContentKind AnalyticsRule -ContentPath 'Example.yaml' } catch {}; " "Assert-V31VersionChecksPassed" ) self.assertNotEqual(0, result.returncode) - self.assertIn("Workbook 'Example.json'", result.stderr) + self.assertIn("AnalyticsRule 'Example.yaml'", result.stderr) self.assertNotIn("Solution 'Legacy'", result.stderr) def test_numeric_bounds_do_not_apply_floor_to_other_content_kinds(self): policy = attribution.ps_literal(attribution.PACKAGER / "common/versionPolicy.ps1") - kinds = ("AnalyticsRule", "HuntingQuery", "Parser", "Workbook", "Playbook", "DataConnector", "LogicAppsCustomConnector", + kinds = ("AnalyticsRule", "HuntingQuery", "Parser", "Playbook", "DataConnector", "LogicAppsCustomConnector", "AzureFunction", "SummaryRule", "Watchlist", "CCF DataConnector/Connections") for kind in kinds: with self.subTest(kind=kind): @@ -376,7 +376,55 @@ def test_other_generators_reject_their_effective_release_versions(self): result = self.assert_rejected_unchanged(entry, pipeline=pipeline) self.assertIn(kind, result.stdout + result.stderr) - def test_workbook_effective_metadata_version_is_checked(self): + def test_workbook_independent_numeric_versions_and_malformed_versions(self): + policy = attribution.ps_literal(attribution.PACKAGER / "common/versionPolicy.ps1") + result = self.run_ps( + f". {policy}; " + "$v31VersionPolicy = @{ Errors = [System.Collections.Generic.List[string]]::new() }; " + "foreach ($v in @('1.0', '2.0.0', '4.0.0', '4.1.0', '10.2.3.4')) { " + "Assert-V31ReleaseVersion -Version $v -ContentKind Workbook -ContentPath 'Workbooks/Test.json' }; " + "Assert-V31VersionChecksPassed" + ) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + for version in ("", "Notebook/1.0", "4.0.0-preview", "4.0.0\n"): + with self.subTest(version=version): + result = self.run_ps( + f". {policy}; " + "$v31VersionPolicy = @{ Errors = [System.Collections.Generic.List[string]]::new() }; " + f"try {{ Assert-V31ReleaseVersion -Version {attribution.ps_literal(version)} " + "-ContentKind Workbook -ContentPath 'Workbooks/Test.json' } catch {}; " + "Assert-V31VersionChecksPassed" + ) + self.assertNotEqual(0, result.returncode) + self.assertIn("independent", result.stderr) + self.assertIn("numeric content version", result.stderr) + + def test_workbook_version_does_not_relax_solution_or_xdr_versions(self): + common = attribution.ps_literal(attribution.PACKAGER / "common/commonFunctions.ps1") + guards = ( + ("Solution", "Assert-V31SolutionVersion -SolutionName 'Sample' -Version $v", + ("3.1.999", "4.0.0"), ("3.2.0", "3.2.1")), + ("CustomDetection", "Assert-V31ReleaseVersion -ContentKind CustomDetection -ContentPath 'XDR Detections/Test.yaml' -Version $v", + ("3.0.99", "4.0.0", "3.1.0-preview"), ("3.1.0", "3.9.99")), + ("DefenderWorkbook", "Assert-DefenderWorkbookReleaseVersion -Version $v", + ("3.0.99", "4.0.0", "3.1.0-preview"), ("3.1.0", "3.9.99")), + ) + for kind, guard, rejected, accepted in guards: + with self.subTest(kind=kind): + result = self.run_ps( + f". {common}; " + "$v31VersionPolicy = @{ Errors = [System.Collections.Generic.List[string]]::new() }; " + "Assert-V31ReleaseVersion -Version '4.0.0' -ContentKind Workbook -ContentPath 'Workbooks/Test.json'; " + f"foreach ($v in @({','.join(map(attribution.ps_literal, accepted))})) {{ {guard} }}; " + "$rejected = @(); " + f"foreach ($v in @({','.join(map(attribution.ps_literal, rejected))})) {{ " + f"try {{ {guard} }} catch {{ $rejected += $v }} }}; " + "ConvertTo-Json -InputObject $rejected -Compress" + ) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertEqual(list(rejected), json.loads(result.stdout)) + + def test_workbook_effective_metadata_version_is_preserved(self): common = attribution.ps_literal(attribution.PACKAGER / "common/commonFunctions.ps1") metadata = self.solution / "Workbooks/WorkbooksMetadata.json" metadata.parent.mkdir() @@ -385,19 +433,36 @@ def test_workbook_effective_metadata_version_is_checked(self): "title": "Test", "description": "Test", "version": "4.0.0", "dataTypesDependencies": [], "dataConnectorsDependencies": [], }])) + workbook = metadata.with_name("Test.json") + workbook.write_text(json.dumps({ + "version": "Notebook/1.0", + "items": [{"type": 1, "content": {"json": "# Test"}}], + })) + before = self.snapshot() result = self.run_ps( f". {common}; " "$v31VersionPolicy = @{ Errors = [System.Collections.Generic.List[string]]::new() }; " - "$contentToImport = [pscustomobject]@{TemplateSpec=$true;Name='Sample';Version='3.1.0'}; " - "$rawData = '{\"version\":\"Notebook/1.0\",\"items\":[]}'; " + "$solutionName = 'Sample'; " + "$global:workbookCounter = 1; $global:DependencyCriteria = @(); " + "$global:baseMainTemplate = [pscustomobject]@{variables=[pscustomobject]@{};parameters=[pscustomobject]@{};resources=@()}; " + "$global:baseCreateUiDefinition = [pscustomobject]@{parameters=[pscustomobject]@{steps=@()}}; " + "$contentToImport = [pscustomobject]@{TemplateSpec=$true;Name='Sample';Version='3.2.1';Author='Microsoft'}; " + f"$rawData = Get-Content -LiteralPath {attribution.ps_literal(workbook)} -Raw; " "GetWorkbookDataMetadata -file 'Workbooks/Test.json' -isPipelineRun $false " - "-contentResourceDetails (returnContentResources '3.1.0') " - f"-baseFolderPath {attribution.ps_literal(str(self.solution) + '/')} " - "-contentToImport $contentToImport" + "-contentResourceDetails (returnContentResources '3.2.1') " + f"-baseFolderPath {attribution.ps_literal(str(self.solution) + '\\')} " + "-contentToImport $contentToImport; " + "Assert-V31VersionChecksPassed; " + "$global:baseMainTemplate | ConvertTo-Json -Depth 50 -Compress" ) - self.assertNotEqual(0, result.returncode, result.stdout + result.stderr) - self.assertIn("Workbook 'Workbooks/Test.json'", result.stderr) - self.assertIn("'4.0.0'", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + template = json.loads(result.stdout.splitlines()[-1]) + self.assertEqual("4.0.0", template["variables"]["workbookVersion1"]) + registered = template["resources"][0] + self.assertEqual("[variables('workbookVersion1')]", registered["properties"]["version"]) + self.assertEqual(json.loads(workbook.read_text()), + json.loads(registered["properties"]["mainTemplate"]["resources"][0]["properties"]["serializedData"])) + self.assertEqual(before, self.snapshot()) def test_ccf_fallback_and_shared_connection_version(self): common = attribution.ps_literal(attribution.PACKAGER / "common/commonFunctions.ps1") diff --git a/Tools/SentinelToXDRMigration/tests/test_workbook_dependencies.py b/Tools/SentinelToXDRMigration/tests/test_workbook_dependencies.py index d5161a4049d..cd07aefb8e2 100644 --- a/Tools/SentinelToXDRMigration/tests/test_workbook_dependencies.py +++ b/Tools/SentinelToXDRMigration/tests/test_workbook_dependencies.py @@ -1,12 +1,14 @@ from copy import deepcopy import json +from importlib.resources import files import pytest from sentinel_xdr_migration import cli, workbooks +from sentinel_xdr_migration.dependencies import dependency_plan, workload_records from sentinel_xdr_migration import workbook_dependencies as dependencies from sentinel_xdr_migration.workbook_functions import candidates -from sentinel_xdr_migration._workbook_core import workbook +from sentinel_xdr_migration._workbook_core import workbook, mapping_catalog, migrator, rewrite from test_workbooks import solution, config, convert, authored SOURCE = "Workbooks/nested/fixture.json" @@ -86,6 +88,215 @@ def test_plan_is_offline_hash_bound_and_never_accepts_heuristics(solution): assert before == {str(path): path.read_bytes() for path in solution.rglob("*") if path.is_file()} +def test_office_native_tables_have_documented_identity_mappings_and_mdo_workload(): + names = ( + "MessageEvents", "MessageUrlInfo", "MessagePostDeliveryEvents", + "FileMaliciousContentInfo", + ) + tables = mapping_catalog.load_mapping_catalog()["tables"] + for name in names: + assert tables[name]["ahTable"] == name + assert tables[name]["tier"] == "1to1" + schemas = json.loads(files("sentinel_xdr_migration._workbook_core") + .joinpath("data/ah_schema_catalog.json").read_text()) + assert not {"MessageEvents", "MessageUrlInfo", "MessagePostDeliveryEvents"} & set(schemas) + records, unknown = workload_records(list(names)) + assert records == [{"workload": "mdo", "dataTypes": sorted(names)}] + assert unknown == [] + + +def test_granularity_scalar_parameter_is_not_a_source_but_table_parameters_are(): + scalar = ( + "let gran = {Granularity}; " + "EmailEvents | make-series Total = count() on Timestamp step gran" + ) + assert workbook.opaque_functions(scalar) == set() + assert workbook.opaque_functions("let source = {TableName}; source | take 1") == {"{TableName}"} + assert workbook.opaque_functions("{TableName} | take 1") == {"{TableName}"} + + +def test_static_datatable_review_converts_without_phantom_dependencies(solution): + query = '''datatable(Index:int, Name:string, Value:int) +[ + 1, "Defender for Office 365", 194, + 2, "Mimecast", 478 +] +| project ["Email security solution"] = Name, ["Threats missed"] = Value +| order by ["Threats missed"] asc''' + assert rewrite.is_source_free_literal_datatable(query) + assert not rewrite.is_source_free_literal_datatable( + 'datatable(Index:int, Name:string)[1, "x"] | join DeviceEvents on Index' + ) + assert not rewrite.is_source_free_literal_datatable( + 'datatable(Index:int)[1] | project MissingColumn' + ) + assert migrator.convert(query, tables=[])["status"] == "converted" + assert migrator.convert("print 1", tables=[])["status"] == "needs-review" + assert dependency_plan(query, metadata=[{"tables": ["MessageEvents"]}])["dataTypes"] == [] + + write_query(solution, query) + plan = dependencies.plan_workbook_dependencies(solution) + row = plan["queries"][0] + assert row["workspaceFunctions"] == [] + assert row["proposal"]["dataTypes"] == [] + assert row["proposal"]["reviewRequired"] is False + + reviewed = decisions(plan) + reviewed["queries"][0]["tables"] = [] + reviewed["workbooks"][0]["requiredWorkloads"] = [] + reviewed["workbooks"][0]["requiredDataConnectors"] = [] + review(solution, plan, reviewed) + settings = json.loads((solution / "Logs/review-1/conversion-config.json").read_bytes()) + result = convert(solution, settings) + assert result["converted"] == 1 and result["blocked"] == 0 + assert workbooks.validate_workbooks(solution)["valid"] == 1 + + authored_document = json.loads(authored(solution).read_bytes()) + provenance = authored_document["contentProvenance"] + assert provenance["requiredWorkloads"] == [] + assert provenance["requiredDataConnectors"] == [] + record = provenance["conversion"]["queries"][0] + assert record["conversion"]["resolvedTables"] == [] + assert record["conversion"]["status"] == "converted" + + write_query(solution, "print 1") + unsupported_plan = dependencies.plan_workbook_dependencies(solution) + unsupported_decisions = decisions(unsupported_plan) + unsupported_decisions["queries"][0]["tables"] = [] + unsupported_decisions["workbooks"][0]["requiredWorkloads"] = [] + unsupported_decisions["workbooks"][0]["requiredDataConnectors"] = [] + with pytest.raises(ValueError, match="requires explicit underlying source tables"): + review(solution, unsupported_plan, unsupported_decisions, output="Logs/review-unsupported") + + +@pytest.mark.parametrize("query", [ + "range x from 0 to 10 step 1", + "range x from 0 to 20 step 1", + "\n range Value_1 from -10 to 0 step 2 \r\n", + "range x from 10 to -10 step -1", + "range x from 0.0 to 1.0 step 0.25", + "range x from -1e1 to 1e1 step 5e-1", + "range DeviceEvents from 0 to 10 step 1", + "range x from 1 to 1 step -1", + "range x from -9223372036854775808 to 9223372036854775807 step 1", +]) +def test_literal_numeric_range_conversion_is_identity_and_dependency_free(query): + assert rewrite.is_source_free_literal_range(query) + assert rewrite.is_source_free_literal_query(query) + assert not rewrite.is_source_free_literal_datatable(query) + for tables in ([], None): + contract = migrator.convert(query, tables=tables) + assert contract["status"] == "converted" + assert contract["convertedKql"] == query + assert contract["resolvedTables"] == [] + assert contract["unsupported"] == [] + assert contract["diff"] == [] + proposal = dependency_plan(query, metadata=[{"tables": ["MessageEvents"]}]) + assert proposal["dataTypes"] == [] + assert proposal["requiredWorkloads"] == [] + assert proposal["requiredDataConnectors"] == [] + assert proposal["reviewRequired"] is False + + +@pytest.mark.parametrize("query", [ + "range(0, 10, 1)", + "range x from 0 to Limit step 1", + "range x from 0 to {Limit} step 1", + "range x from 0 to toscalar(DeviceEvents | count) step 1", + "range x from 0 to HiddenParser() step 1", + "range x from 0 to 10+1 step 1", + "range x from 0 to long(10) step 1", + "range x from datetime(2026-01-01) to now() step 1d", + "range x from 0 to 10 step 0", + "range x from 0 to 10 step -1", + "range x from 10 to 0 step 1", + "range x from 0 to 10 step 0e1", + "range x from 0 to 10 step 1e-999", + "range x from 0 to 1e999 step 1", + "range x from 0 to 9223372036854775808 step 1", + "range x from -9223372036854775809 to 0 step 1", + "range x from NaN to 10 step 1", + "range x from 0 to 10 step 1;", + "range x from 0 to 10 step 1 // comment", + "// range x from 0 to 10 step 1", + "'range x from 0 to 10 step 1'", + "range x from 0 to 10 step 1 | project x", + "range x from 0 to 10 step 1 | union DeviceEvents", + "union (range x from 0 to 10 step 1), (HiddenParser())", + "range x from 0 to 10 step 1; DeviceEvents | count", + "let r = range x from 0 to 10 step 1; r", + "RANGE x FROM 0 TO 10 STEP 1", +]) +def test_nonliteral_or_composite_ranges_do_not_clear_empty_lineage_review(query): + assert not rewrite.is_source_free_literal_range(query) + assert not rewrite.is_source_free_literal_query(query) + assert migrator.convert(query, tables=[])["status"] != "converted" + assert migrator.convert(query)["status"] != "converted" + proposal = dependency_plan(query) + if proposal["dataTypes"]: + assert proposal["dataTypes"] == ["DeviceEvents"] + assert proposal["requiredWorkloads"] == [{"workload": "mde", "dataTypes": ["DeviceEvents"]}] + else: + assert proposal["reviewRequired"] is True + + +def test_range_review_preserves_both_hash_bound_parameter_dropdowns(solution): + path = solution / SOURCE + document = json.loads(path.read_bytes()) + queries = ["range x from 0 to 10 step 1", "range x from 0 to 20 step 1"] + hashes = [ + "2ccdaa784725b2a65a8f381ed39f1d053eb36879948ab0bc52508acdc47dbfea", + "e19eb4f4d503984f28cd04aba8a7975c0ff909046d9c3f980c2033c16aaf61d4", + ] + document["items"] = [{"type": 1, "content": {"json": "layout"}} for _ in range(4)] + parameters = [{"name": f"Placeholder{i}", "type": 1} for i in range(6)] + parameters.extend({"name": f"Range{i}", "type": 2, "query": query, "queryType": 0} + for i, query in enumerate(queries)) + document["items"].append({"type": 9, "content": {"parameters": parameters}}) + path.write_text(json.dumps(document)) + before = path.read_bytes() + plan = dependencies.plan_workbook_dependencies(solution) + assert [row["path"] for row in plan["queries"]] == [ + "/items/4/content/parameters/6/query", "/items/4/content/parameters/7/query"] + assert [row["querySha256"] for row in plan["queries"]] == hashes + assert all(not row["proposal"]["reviewRequired"] for row in plan["queries"]) + reviewed = decisions(plan) + for row in reviewed["queries"]: + row["tables"] = [] + reviewed["workbooks"][0]["requiredWorkloads"] = [] + receipt = review(solution, plan, reviewed) + assert receipt["unresolved"] == [] + settings = json.loads((solution / "Logs/review-1/conversion-config.json").read_bytes()) + assert convert(solution, settings)["converted"] == 1 + assert workbooks.validate_workbooks(solution)["valid"] == 1 + assert path.read_bytes() == before + target = json.loads(authored(solution).read_bytes()) + assert [row["node"]["query"] for row in workbook.query_surfaces(target)] == queries + provenance = target["contentProvenance"] + assert provenance["requiredWorkloads"] == [] + assert provenance["requiredDataConnectors"] == [] + assert provenance.get("requiredWorkspaceFunctions", []) == [] + assert all(row["conversion"]["resolvedTables"] == [] + for row in provenance["conversion"]["queries"]) + + +@pytest.mark.parametrize("query", [ + "range x from 0 to 10 step 1 | union DeviceEvents", + "range x from 0 to HiddenParser() step 1", + "range x from 0 to 10 step 0", +]) +def test_range_review_rejects_empty_lineage_for_unsupported_branches(solution, query): + write_query(solution, query) + plan = dependencies.plan_workbook_dependencies(solution) + reviewed = decisions(plan) + reviewed["queries"][0]["tables"] = [] + reviewed["workbooks"][0]["requiredWorkloads"] = [] + with pytest.raises(ValueError, match="explicit underlying source tables|vetted prerequisites"): + review(solution, plan, reviewed) + assert not authored(solution).exists() + assert not (solution / "Logs/review-1").exists() + + def test_saved_function_lineage_static_proposal_remains_unresolved(solution): record = function(solution) write_query(solution, "let DeviceEvents = Syslog; ReviewedFunction() | count") @@ -471,7 +682,97 @@ def test_vendored_component_manifest_hashes_are_current(): root = files("sentinel_xdr_migration._workbook_core") manifest = json.loads(root.joinpath("import-manifest.json").read_text()) for item in manifest["components"]: - assert hashlib.sha256(root.joinpath(item["component"]).read_bytes()).hexdigest() == item["sha256"] + contents = root.joinpath(item["component"]).read_bytes().replace(b"\r\n", b"\n") + assert hashlib.sha256(contents).hexdigest() == item["sha256"] + + +@pytest.fixture +def connector_solution(solution): + root = solution.parent / "Solutions" / "Microsoft Defender for Office 365" + root.parent.mkdir() + solution.rename(root) + folder = root / "Data Connectors" + folder.mkdir() + (folder / "template_OfficeATP.json").write_text(json.dumps({ + "id": "ReviewedConnector", "dataTypes": [{"name": "Syslog"}]})) + write_query(root, "Syslog | count") + return root + + +@pytest.mark.parametrize("repository_relative", [False, True]) +def test_connector_paths_are_canonical_read_only_and_review_hash_bound(connector_solution, repository_relative): + root = connector_solution + relative = "Data Connectors/template_OfficeATP.json" + reference = f"Solutions/{root.name}/{relative}" if repository_relative else relative + data_path = root / "Data/Solution_Fixture.json" + data_path.write_text(json.dumps({"Version": "3.2.2", "Data Connectors": [reference]})) + before = data_path.read_bytes() + plan = dependencies.plan_workbook_dependencies(root) + connector = plan["connectorAlternatives"][0] + assert connector["sourcePath"] == relative + assert connector["sourceSha256"] == workbooks._sha((root / relative).read_bytes()) + assert plan["solutionDataSha256"] == dependencies.digest(json.loads(before)) + assert review(root, plan, decisions(plan, tables=["Syslog"]))["dependencyReviewRequired"] is False + assert data_path.read_bytes() == before + + altered = deepcopy(plan) + altered["connectorAlternatives"][0]["sourcePath"] = f"Solutions/{root.name}/{relative}" + altered["planSha256"] = dependencies.digest({k: v for k, v in altered.items() if k != "planSha256"}) + with pytest.raises(ValueError, match="malformed or stale"): + review(root, altered, decisions(altered, tables=["Syslog"]), output="Logs/altered-review") + (root / relative).write_text(json.dumps({"id": "Changed", "dataTypes": ["Syslog"]})) + with pytest.raises(ValueError, match="malformed or stale"): + review(root, plan, decisions(plan, tables=["Syslog"]), output="Logs/stale-review") + assert not (root / "Logs/altered-review").exists() + assert not (root / "Logs/stale-review").exists() + + +@pytest.mark.parametrize("reference", [ + "Solutions/Other/Data Connectors/template_OfficeATP.json", + "Solutions/microsoft defender for office 365/Data Connectors/template_OfficeATP.json", + "Solutions/Microsoft Defender for Office 365/../Data Connectors/template_OfficeATP.json", + "Solutions/Microsoft Defender for Office 365/Workbooks/nested/fixture.json", + "Solutions/Microsoft Defender for Office 365/Data Connectors/missing.json", + "Data Connectors/../Data/Solution_Fixture.json", + "Data Connectors/./template_OfficeATP.json", + "Data Connectors//template_OfficeATP.json", + "Data Connectors/", + "/Data Connectors/template_OfficeATP.json", + "C:/Data Connectors/template_OfficeATP.json", + "C:Data Connectors/template_OfficeATP.json", + "Data Connectors\\template_OfficeATP.json", + "../Data Connectors/template_OfficeATP.json", + None, +]) +def test_connector_paths_reject_unsafe_or_non_connector_references(connector_solution, reference): + data_path = connector_solution / "Data/Solution_Fixture.json" + data_path.write_text(json.dumps({"Data Connectors": [reference]})) + with pytest.raises(ValueError): + dependencies.plan_workbook_dependencies(connector_solution) + + +def test_repository_relative_connector_requires_solutions_parent(solution): + with pytest.raises(ValueError): + dependencies._connectors(solution, { + "Data Connectors": [f"Solutions/{solution.name}/Data Connectors/connector.json"]}) + + +@pytest.mark.parametrize("repository_relative", [False, True]) +@pytest.mark.parametrize("directory", [False, True]) +def test_connector_paths_reject_symlinks(connector_solution, repository_relative, directory): + root = connector_solution + folder = root / "Data Connectors" + if directory: + target = root / "Actual Connectors" + folder.rename(target) + folder.symlink_to(target, target_is_directory=True) + relative = "Data Connectors/template_OfficeATP.json" + else: + (folder / "linked.json").symlink_to(folder / "template_OfficeATP.json") + relative = "Data Connectors/linked.json" + reference = f"Solutions/{root.name}/{relative}" if repository_relative else relative + with pytest.raises(ValueError, match="Symlink"): + dependencies._connectors(root, {"Data Connectors": [reference]}) def test_connector_alternatives_are_not_cumulative_or_title_recommendations(solution): @@ -755,3 +1056,22 @@ def test_decoy_functions_cannot_satisfy_hint_flags_or_unsupported_table_indirect def test_benign_member_indexing_and_catalog_table_controls(query): assert workbook.opaque_functions(query) == set() + +@pytest.mark.parametrize("query", [ + "union withsource=Table \n (print X=1)", + "union ISFUZZY=true WITHSOURCE = tAbLe\r\n (print X=1)", + "union kind=outer withsource=ToTable\n(print X=1)", +]) +def test_union_withsource_output_names_are_not_dynamic_table_calls(query): + assert workbook._quoted_calls_and_table_sources(query) == (set(), set()) + assert workbook.opaque_functions(query) == set() + + +@pytest.mark.parametrize(("query", "calls", "bare"), [ + ("table(strcat('Device', 'Events'))", {"{__unsupported_table_indirection__}"}, set()), + ("totable(strcat('Device', 'Events'))", {"{__unsupported_table_indirection__}"}, set()), + ("table('DeviceEvents')", set(), {"DeviceEvents"}), + ("totable('DeviceEvents')", set(), {"DeviceEvents"}), +]) +def test_dynamic_and_literal_table_calls_keep_existing_scanner_behavior(query, calls, bare): + assert workbook._quoted_calls_and_table_sources(query) == (calls, bare) diff --git a/Tools/SentinelToXDRMigration/tests/test_workbooks.py b/Tools/SentinelToXDRMigration/tests/test_workbooks.py index e677ca8d6b5..8f83e3e1590 100644 --- a/Tools/SentinelToXDRMigration/tests/test_workbooks.py +++ b/Tools/SentinelToXDRMigration/tests/test_workbooks.py @@ -11,10 +11,49 @@ import pytest from sentinel_xdr_migration import workbooks -from sentinel_xdr_migration._workbook_core import workbook, mapping_catalog, migrate +from sentinel_xdr_migration._workbook_core import workbook, mapping_catalog, migrate, migrator, rewrite FIXTURES = Path(__file__).parent / "fixtures/workbooks" SCHEMA = Path(__file__).parents[1] / "schema/xdr-workbook.schema.json" +ENDPOINT_TABLES = [ + "DeviceInfo", "DeviceNetworkInfo", "DeviceProcessEvents", "DeviceNetworkEvents", + "DeviceFileEvents", "DeviceRegistryEvents", "DeviceLogonEvents", + "DeviceImageLoadEvents", "DeviceEvents", "DeviceFileCertificateInfo", +] +TYPE_LABEL_QUERY = '''union\x20 +isfuzzy = true +(DeviceInfo | summarize arg_max(TimeGenerated,*) | project TimeGenerated, Type= "DeviceInfo" | extend Description = "Machine information (including OS information)"), +(DeviceNetworkInfo | summarize arg_max(TimeGenerated,*) | project TimeGenerated, Type="DeviceNetworkInfo" | extend Description = "Network properties of machines"), +(DeviceProcessEvents | summarize arg_max(TimeGenerated,*) | project TimeGenerated, Type="DeviceProcessEvents" | extend Description = "Process creation and related events"), +(DeviceNetworkEvents | summarize arg_max(TimeGenerated,*) | project TimeGenerated, Type="DeviceNetworkEvents" | extend Description = "Network connection and related events"), +(DeviceFileEvents | summarize arg_max(TimeGenerated,*) | project TimeGenerated, Type="DeviceFileEvents" | extend Description = "File creation, modification, and other file system events"), +(DeviceRegistryEvents | summarize arg_max(TimeGenerated,*) | project TimeGenerated, Type="DeviceRegistryEvents" | extend Description = "Creation and modification of registry entries"), +(DeviceLogonEvents | summarize arg_max(TimeGenerated,*) | project TimeGenerated, Type="DeviceLogonEvents" | extend Description = "Sign-ins and other authentication events"), +(DeviceImageLoadEvents | summarize arg_max(TimeGenerated,*) | project TimeGenerated, Type="DeviceImageLoadEvents" | extend Description = "DLL loading events"), +(DeviceEvents | summarize arg_max(TimeGenerated,*) | project TimeGenerated, Type="DeviceEvents" | extend Description = "Additional events types"), +(DeviceFileCertificateInfo | summarize arg_max(TimeGenerated,*) | project TimeGenerated, Type="DeviceFileCertificateInfo" | extend Description = "Certificate information of signed files)") +| extend ["Last Log Received At (Local Time)"] = TimeGenerated +| project Type, Description, ["Last Log Received At (Local Time)"]'''.replace("\n", "\r\n") +TYPE_SUMMARY_QUERY = '''let AllDeviceNames = DeviceInfo | distinct DeviceId, DeviceName; +let DeviceEventSummary = DeviceEvents | summarize count() by DeviceId, Type, bin(TimeGenerated,1d); +let DeviceNetworkEventsSummary = DeviceNetworkEvents | summarize count() by DeviceId,Type, bin(TimeGenerated,1d); +let DeviceNetworkInfoSummary = DeviceNetworkInfo | summarize count() by DeviceId,Type, bin(TimeGenerated,1d); +let DeviceLogonEventsSummary = DeviceLogonEvents | summarize count() by DeviceId,Type, bin(TimeGenerated,1d); +let DeviceRegistryEventsSummary = DeviceRegistryEvents | summarize count() by DeviceId,Type, bin(TimeGenerated,1d); +let DeviceProcessEventsSummary = DeviceProcessEvents | summarize count() by DeviceId,Type, bin(TimeGenerated,1d); +(DeviceEventSummary +| union DeviceNetworkEventsSummary, +DeviceNetworkEventsSummary, +DeviceNetworkInfoSummary, +DeviceLogonEventsSummary, +DeviceRegistryEventsSummary, +DeviceProcessEventsSummary) +| join kind=inner (\x20 +AllDeviceNames +) +on $left.DeviceId == $right.DeviceId +| project-reorder Type, TimeGenerated, count_ +//| project-away DeviceId, DeviceId1''' @pytest.fixture @@ -55,6 +94,81 @@ def authored(solution): return solution / "XDR Workbooks/nested/fixture.json" +def test_endpoint_type_labels_and_summary_groupings_are_deterministic(): + labels = rewrite.rewrite(TYPE_LABEL_QUERY, ENDPOINT_TABLES) + assert labels["deterministic"] is True + assert labels["convertedKql"] == TYPE_LABEL_QUERY + assert labels["flags"] == [] + + summary_tables = [ + "DeviceInfo", "DeviceEvents", "DeviceNetworkEvents", "DeviceNetworkInfo", + "DeviceLogonEvents", "DeviceRegistryEvents", "DeviceProcessEvents", + ] + summary = rewrite.rewrite(TYPE_SUMMARY_QUERY, summary_tables) + assert summary["deterministic"] is True + assert summary["flags"] == [] + assert summary["convertedKql"].count('extend Type = "Device') == 6 + assert "| project-reorder Type, TimeGenerated, count_" in summary["convertedKql"] + assert "| join kind=inner" in summary["convertedKql"] + replay = rewrite.rewrite(summary["convertedKql"], summary_tables) + assert replay["deterministic"] is True + assert replay["convertedKql"] == summary["convertedKql"] + + +@pytest.mark.parametrize("query", [ + 'DeviceEvents | where Type == "ProcessCreated" | project Type', + "DeviceEvents | summarize count() by DeviceId, Type", + 'let Labelled = DeviceEvents | project Type = "label"; ' + "let Raw = DeviceEvents | project Type; " + "union Labelled, Raw | project Type", +]) +def test_ambiguous_type_column_references_remain_review_findings(query): + result = rewrite.rewrite(query, ["DeviceEvents"]) + assert result["deterministic"] is False + assert result["convertedKql"] == query + assert any(flag["kind"] == "la-only-column" and flag["name"] == "Type" + for flag in result["flags"]) + + +def test_endpoint_type_conversion_passes_lineage_replay_and_validation(solution): + relative = "Workbooks/nested/fixture.json" + source_path = solution / relative + source = json.loads(source_path.read_text()) + queries = [TYPE_LABEL_QUERY, TYPE_SUMMARY_QUERY] + tables = [ENDPOINT_TABLES, [ + "DeviceInfo", "DeviceEvents", "DeviceNetworkEvents", "DeviceNetworkInfo", + "DeviceLogonEvents", "DeviceRegistryEvents", "DeviceProcessEvents", + ]] + source["items"] = [{ + "type": 3, + "content": {"items": [ + {"type": 3, "content": {"query": query, "queryType": 0, + "resourceType": workbook.LA}} + for query in queries + ]}, + }] + source_path.write_text(json.dumps(source)) + settings = {"workbookResolutions": {relative: { + surface["path"]: { + "querySha256": workbook.query_hash(surface["node"]["query"]), + "tables": tables[index], + "evidence": "Reviewed exact Endpoint workbook query", + } + for index, surface in enumerate(workbook.query_surfaces(source)) + }}} + + result = convert(solution, settings) + assert result["converted"] == 1 and result["blocked"] == 0 + assert workbooks.validate_workbooks(solution)["valid"] == 1 + output = json.loads(authored(solution).read_text()) + records = output["contentProvenance"]["conversion"]["queries"] + assert [record["status"] for record in records] == ["converted", "converted"] + assert all(record["sourceSha256"] and record["targetSha256"] for record in records) + converted_queries = [surface["node"]["query"] for surface in workbook.query_surfaces(output)] + assert converted_queries[0] == TYPE_LABEL_QUERY + assert converted_queries[1].count('extend Type = "Device') == 6 + + @pytest.mark.parametrize("case", json.loads((FIXTURES / "upstream-cases.json").read_text()), ids=lambda case: Path(case["path"]).stem) def test_six_public_source_contracts(case): From 7171cc9b3b95736d8a56b038244e4a61d627322c Mon Sep 17 00:00:00 2001 From: kingwil <5839248+kingwil@users.noreply.github.com> Date: Wed, 14 Oct 2026 21:27:54 +0100 Subject: [PATCH 2/2] Make version-policy diagnostic assertions independent of PowerShell wrapping Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Tools/SentinelToXDRMigration/tests/test_version_policy.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Tools/SentinelToXDRMigration/tests/test_version_policy.py b/Tools/SentinelToXDRMigration/tests/test_version_policy.py index 6bb4a45d0e7..4d2071f7963 100644 --- a/Tools/SentinelToXDRMigration/tests/test_version_policy.py +++ b/Tools/SentinelToXDRMigration/tests/test_version_policy.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import re import shutil import tempfile import unittest @@ -164,7 +165,10 @@ def test_xdr_boundaries_even_without_registration(self): self.assertEqual("3.0.0", properties["contentSchemaVersion"]) else: result = self.assert_rejected_unchanged(entry, pipeline=pipeline) - self.assertIn("CustomDetection 'XDR Detections/Test.yaml'", result.stdout + result.stderr) + diagnostic = re.sub(r"\x1b\[[0-9;]*m", "", result.stdout + result.stderr) + diagnostic = re.sub(r"\n\s*\|\s*", " ", diagnostic) + self.assertIn("CustomDetection 'XDR Detections/Test.yaml'", + " ".join(diagnostic.split())) def test_analytic_versions_keep_legacy_floor(self): entry = self.fixture()