From e37203ef6ebc3cd3ed70914b7f81f0fb1f12e12c Mon Sep 17 00:00:00 2001 From: kingwil <5839248+kingwil@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:20:50 +0100 Subject: [PATCH 1/3] Support explicitly reviewed post-import workbook corrections Preserve original conversion and native review receipts, reconstruct and validate the historical baseline, and bind current payload/query dependency reviews without changing IDs or existing prerequisites. Integrate offline plan/review/apply with authored schemas and V3.1 validation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../common/defenderWorkbooks.ps1 | 18 +- Tools/SentinelToXDRMigration/README.md | 71 ++++ .../schema/xdr-workbook.schema.json | 69 ++++ .../data/xdr-workbook.schema.json | 69 ++++ .../_workbook_core/import-manifest.json | 2 +- .../sentinel_xdr_migration/cli.py | 22 ++ .../native_workbooks.py | 22 +- .../workbook_updates.py | 271 +++++++++++++++ .../sentinel_xdr_migration/workbooks.py | 26 +- .../tests/test_native_workbooks.py | 21 +- .../tests/test_workbook_updates.py | 328 ++++++++++++++++++ 11 files changed, 898 insertions(+), 21 deletions(-) create mode 100644 Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_updates.py create mode 100644 Tools/SentinelToXDRMigration/tests/test_workbook_updates.py diff --git a/Tools/Create-Azure-Sentinel-Solution/common/defenderWorkbooks.ps1 b/Tools/Create-Azure-Sentinel-Solution/common/defenderWorkbooks.ps1 index 3f2f3e6dfd7..af931b07fc5 100644 --- a/Tools/Create-Azure-Sentinel-Solution/common/defenderWorkbooks.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/common/defenderWorkbooks.ps1 @@ -45,7 +45,8 @@ function Assert-DefenderWorkbookReleaseVersion { } function Assert-NativeDefenderWorkbookDocument { - param([psobject]$Document, [string]$SolutionRoot) + param([psobject]$Document, [string]$SolutionRoot, + [ValidateSet('native_workbooks', 'workbook_updates')][string]$ValidationModule = 'native_workbooks') $python = if ($env:SENTINEL_XDR_PYTHON) { $env:SENTINEL_XDR_PYTHON } else { 'python' } $start = [System.Diagnostics.ProcessStartInfo]::new() @@ -56,7 +57,7 @@ function Assert-NativeDefenderWorkbookDocument { $start.RedirectStandardError = $true $start.StandardInputEncoding = [System.Text.UTF8Encoding]::new($false) $start.StandardOutputEncoding = [System.Text.UTF8Encoding]::new($false) - foreach ($argument in @('-m', 'sentinel_xdr_migration.native_workbooks', $SolutionRoot)) { + foreach ($argument in @('-m', "sentinel_xdr_migration.$ValidationModule", $SolutionRoot)) { $start.ArgumentList.Add($argument) } $start.Environment['PYTHONPATH'] = [System.IO.Path]::GetFullPath( @@ -104,6 +105,19 @@ function Assert-DefenderWorkbookDocument { } Assert-DefenderWorkbookReleaseVersion -Version $metadata.version $provenance = $Document.contentProvenance + if ($null -ne $provenance -and $null -ne $provenance.PSObject.Properties['authoredReview']) { + Assert-NativeDefenderWorkbookDocument -Document $Document -SolutionRoot $SolutionRoot ` + -ValidationModule workbook_updates + if ($PassThruSource) { + if ($provenance.origin -ceq 'native-xdr') { + return $Document + } + $resolved = Resolve-SolutionContentPath -SolutionRoot $SolutionRoot -SolutionName $SolutionName ` + -ContentPath $provenance.source.path -RequireFile + return (Get-Content -LiteralPath $resolved.FullPath -Raw | ConvertFrom-Json -ErrorAction Stop) + } + return + } if ($null -ne $provenance -and $provenance.origin -ceq 'native-xdr') { Assert-NativeDefenderWorkbookDocument -Document $Document -SolutionRoot $SolutionRoot if ($PassThruSource) { diff --git a/Tools/SentinelToXDRMigration/README.md b/Tools/SentinelToXDRMigration/README.md index 70c3c8122da..bf96f70c0c1 100644 --- a/Tools/SentinelToXDRMigration/README.md +++ b/Tools/SentinelToXDRMigration/README.md @@ -414,6 +414,77 @@ native reviews and authoring metadata are not installed. Registration keeps the existing reviewed dependency contract. Original Sentinel selections remain independent and no native source is deployed twice. +### Reviewed post-import workbook corrections (offline) + +Use `workbook-update-plan`, `workbook-update-review`, and +`workbook-update-apply` for intentional authored Notebook corrections after +migration/import. These commands do not migrate, deploy, invoke queries, or +establish runtime/UI qualification. They require an explicit original reviewed +**authored** baseline file, for example extracted with `git show` from the +accepted content commit into a review workspace outside deployable content. +An update-reviewed artifact is not an original baseline: subsequent corrections +continue to use the original artifact. + +```powershell +# $Python, $Solution and PYTHONPATH use the same setup as native import above. +$Content = 'XDR Workbooks\example.json'.Replace('\', '/') +$Baseline = 'C:\review-workspace\baseline-example.json' +& $Python -m sentinel_xdr_migration.cli workbook-update-plan ` + --solution $Solution --content $Content --baseline $Baseline | + Set-Content -LiteralPath '.\update-plan.json' -Encoding utf8 +``` + +Inspect the exact current Notebook and plan, including all current `queries` +with discovered tables/functions and `changedQueries` (added/removed queries +use null previous/current hashes). Explicitly set nonempty `actor` and +`reference`, whole-document `decision: "accept"`, and each changed query's +`decision: "accept"` and nonempty `evidence`. Do not edit generated hashes, +dependencies or the reverse baseline delta. Save as `update-decisions.json`. + +```powershell +& $Python -m sentinel_xdr_migration.cli workbook-update-review ` + --solution $Solution --content $Content --baseline $Baseline ` + --review '.\update-decisions.json' | + Set-Content -LiteralPath '.\update-reviewed.json' -Encoding utf8 +& $Python -m sentinel_xdr_migration.cli workbook-update-apply ` + --solution $Solution --content $Content --baseline $Baseline ` + --review '.\update-reviewed.json' +& $Python -m sentinel_xdr_migration.cli native-workbook-validate --solution $Solution +``` + +Apply preserves the edited payload and metadata/IDs, restores historical +provenance from the independently validated baseline (including any accidentally +hand-edited conversion records), and adds `contentProvenance.authoredReview`. +It refuses identity or prerequisite changes. Existing reviewed workloads, +connectors and hash-bound workspace functions must cover discovered current +dependencies; native updates retain native-only lineage, while migrated +updates also support their existing Sentinel tables and workspace functions. +Unknown/new dependencies and unsupported dynamic/cross-resource inputs fail; +reviewer assertions cannot make them supported. + +The review binds the original file SHA, canonical baseline artifact, complete +current canonical Notebook and authored document, exact query/dependency +inventories, actor/reference, and decisions. Only changed baseline values and +historical metadata are retained as a reverse payload delta. Validation +reconstructs that original artifact and reruns the original deterministic +conversion/native-review gates, unchanged. No duplicate raw Notebook or +invented Sentinel parent is retained, and the external baseline is not needed +after apply. Historical receipts describe the original content, not current +queries. Review hashes detect staleness; they are **not cryptographic signatures** +or evidence of who reviewed content. Git history and review remain the trust +boundary. Re-review using the original baseline after any further authored edit. + +The Python validator, PowerShell assertions/dependency scan, static adapter, +and normal V3.1 packaging honor this explicit update branch. Review material +is authoring-only and is not included in installed Notebook provenance. +Keep existing version/release and candidate gates: package through +`package-v3-1 --solution $Solution --toolkit-root $Toolkit --version-bump none` +(for an already prepared release), then run the official +`.script\local-validation\build-and-validate.ps1 -SolutionName +-ContentRoot -SkipPackaging -PackageReportPath `. +That wrapper otherwise defaults to legacy V3 packaging and disallows +cross-checkout packaging; do not use it to create this candidate. + ### Toolkit-owner managed update policy Toolkit-managed Defender Workbooks and XDR detections are updated by stable ID. diff --git a/Tools/SentinelToXDRMigration/schema/xdr-workbook.schema.json b/Tools/SentinelToXDRMigration/schema/xdr-workbook.schema.json index 7825fb41cc8..5b14e60de67 100644 --- a/Tools/SentinelToXDRMigration/schema/xdr-workbook.schema.json +++ b/Tools/SentinelToXDRMigration/schema/xdr-workbook.schema.json @@ -36,6 +36,7 @@ } }], "properties": { + "authoredReview": {"$ref": "#/$defs/authoredReview"}, "origin": {"enum": ["native-xdr", "sentinel-migration"]}, "canonicalPayloadSha256": {"$ref": "#/$defs/sha256"}, "nativeReview": { @@ -118,6 +119,74 @@ } }, "$defs": { + "authoredReview": { + "type": "object", + "required": ["schemaVersion", "baseline", "baselineMetadata", "reversePayloadChanges", "canonicalPayloadSha256", "canonicalDocumentSha256", "queries", "changedQueries", "actor", "reference", "decision", "reviewSha256"], + "properties": { + "schemaVersion": {"const": "1.0.0"}, + "reviewSha256": {"$ref": "#/$defs/sha256"}, + "baseline": { + "type": "object", + "required": ["reference", "sha256", "canonicalDocumentSha256"], + "properties": { + "reference": {"type": "string", "minLength": 1}, + "sha256": {"$ref": "#/$defs/sha256"}, + "canonicalDocumentSha256": {"$ref": "#/$defs/sha256"} + }, + "additionalProperties": false + }, + "baselineMetadata": {"$ref": "#/properties/xdrMetadata"}, + "reversePayloadChanges": { + "type": "array", + "items": { + "type": "object", + "required": ["path", "op"], + "properties": { + "path": {"type": "array", "minItems": 1, "items": {"type": ["string", "integer"]}}, + "op": {"enum": ["add", "remove", "replace"]}, + "value": {} + }, + "additionalProperties": false + } + }, + "canonicalPayloadSha256": {"$ref": "#/$defs/sha256"}, + "canonicalDocumentSha256": {"$ref": "#/$defs/sha256"}, + "queries": { + "type": "array", + "items": { + "type": "object", + "required": ["path", "querySha256", "tables", "workspaceFunctions", "builtinAsimParsers"], + "properties": { + "path": {"type": "string", "pattern": "^/"}, + "querySha256": {"$ref": "#/$defs/sha256"}, + "tables": {"type": "array", "uniqueItems": true, "items": {"type": "string"}}, + "workspaceFunctions": {"type": "array", "uniqueItems": true, "items": {"type": "string"}}, + "builtinAsimParsers": {"type": "array", "uniqueItems": true, "items": {"type": "string"}} + }, + "additionalProperties": false + } + }, + "changedQueries": { + "type": "array", + "items": { + "type": "object", + "required": ["path", "baselineQuerySha256", "querySha256", "decision", "evidence"], + "properties": { + "path": {"type": "string", "pattern": "^/"}, + "baselineQuerySha256": {"anyOf": [{"$ref": "#/$defs/sha256"}, {"type": "null"}]}, + "querySha256": {"anyOf": [{"$ref": "#/$defs/sha256"}, {"type": "null"}]}, + "decision": {"const": "accept"}, + "evidence": {"type": "string", "minLength": 1} + }, + "additionalProperties": false + } + }, + "actor": {"type": "string", "minLength": 1}, + "reference": {"type": "string", "minLength": 1}, + "decision": {"const": "accept"} + }, + "additionalProperties": false + }, "workspaceFunction": { "type": "object", "required": ["functionAlias", "sourcePath", "sourceSha256"], diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/xdr-workbook.schema.json b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/xdr-workbook.schema.json index 7825fb41cc8..5b14e60de67 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/xdr-workbook.schema.json +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/xdr-workbook.schema.json @@ -36,6 +36,7 @@ } }], "properties": { + "authoredReview": {"$ref": "#/$defs/authoredReview"}, "origin": {"enum": ["native-xdr", "sentinel-migration"]}, "canonicalPayloadSha256": {"$ref": "#/$defs/sha256"}, "nativeReview": { @@ -118,6 +119,74 @@ } }, "$defs": { + "authoredReview": { + "type": "object", + "required": ["schemaVersion", "baseline", "baselineMetadata", "reversePayloadChanges", "canonicalPayloadSha256", "canonicalDocumentSha256", "queries", "changedQueries", "actor", "reference", "decision", "reviewSha256"], + "properties": { + "schemaVersion": {"const": "1.0.0"}, + "reviewSha256": {"$ref": "#/$defs/sha256"}, + "baseline": { + "type": "object", + "required": ["reference", "sha256", "canonicalDocumentSha256"], + "properties": { + "reference": {"type": "string", "minLength": 1}, + "sha256": {"$ref": "#/$defs/sha256"}, + "canonicalDocumentSha256": {"$ref": "#/$defs/sha256"} + }, + "additionalProperties": false + }, + "baselineMetadata": {"$ref": "#/properties/xdrMetadata"}, + "reversePayloadChanges": { + "type": "array", + "items": { + "type": "object", + "required": ["path", "op"], + "properties": { + "path": {"type": "array", "minItems": 1, "items": {"type": ["string", "integer"]}}, + "op": {"enum": ["add", "remove", "replace"]}, + "value": {} + }, + "additionalProperties": false + } + }, + "canonicalPayloadSha256": {"$ref": "#/$defs/sha256"}, + "canonicalDocumentSha256": {"$ref": "#/$defs/sha256"}, + "queries": { + "type": "array", + "items": { + "type": "object", + "required": ["path", "querySha256", "tables", "workspaceFunctions", "builtinAsimParsers"], + "properties": { + "path": {"type": "string", "pattern": "^/"}, + "querySha256": {"$ref": "#/$defs/sha256"}, + "tables": {"type": "array", "uniqueItems": true, "items": {"type": "string"}}, + "workspaceFunctions": {"type": "array", "uniqueItems": true, "items": {"type": "string"}}, + "builtinAsimParsers": {"type": "array", "uniqueItems": true, "items": {"type": "string"}} + }, + "additionalProperties": false + } + }, + "changedQueries": { + "type": "array", + "items": { + "type": "object", + "required": ["path", "baselineQuerySha256", "querySha256", "decision", "evidence"], + "properties": { + "path": {"type": "string", "pattern": "^/"}, + "baselineQuerySha256": {"anyOf": [{"$ref": "#/$defs/sha256"}, {"type": "null"}]}, + "querySha256": {"anyOf": [{"$ref": "#/$defs/sha256"}, {"type": "null"}]}, + "decision": {"const": "accept"}, + "evidence": {"type": "string", "minLength": 1} + }, + "additionalProperties": false + } + }, + "actor": {"type": "string", "minLength": 1}, + "reference": {"type": "string", "minLength": 1}, + "decision": {"const": "accept"} + }, + "additionalProperties": false + }, "workspaceFunction": { "type": "object", "required": ["functionAlias", "sourcePath", "sourceSha256"], diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/import-manifest.json b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/import-manifest.json index f3fbb834726..ca1cdf7c41c 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/import-manifest.json +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/import-manifest.json @@ -32,7 +32,7 @@ }, { "component": "data/xdr-workbook.schema.json", - "sha256": "41c1d158185a7a7e6ae16fcf0717c9b6e14fb0ee0b6065b9fa486f2098a0ff07" + "sha256": "f322e55d72e98aba7606174961968855d9c0a83b8a1b0078d12f20c02765394a" }, { "component": "dataframes.py", diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/cli.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/cli.py index 99e5d30df21..ed728a41e2a 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/cli.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/cli.py @@ -177,6 +177,14 @@ def main(argv: list[str] | None = None) -> int: description="Convert Microsoft Sentinel analytic rules into XDR Custom Detection YAML." ) subparsers = parser.add_subparsers(dest="command", required=True) + for update_command in ("workbook-update-plan", "workbook-update-review", "workbook-update-apply"): + update_parser = subparsers.add_parser( + update_command, help="Offline explicit post-import edits; preserve historical receipts.") + update_parser.add_argument("--solution", required=True) + update_parser.add_argument("--content", required=True, help="Authored XDR Workbooks/ relative JSON path.") + update_parser.add_argument("--baseline", required=True, help="External original reviewed authored artifact.") + if update_command != "workbook-update-plan": + update_parser.add_argument("--review", required=True) for native_command in ("native-workbook-plan", "native-workbook-review", "native-workbook-import", "native-workbook-validate"): native_parser = subparsers.add_parser( @@ -1181,6 +1189,9 @@ def main(argv: list[str] | None = None) -> int: selection = None try: isolated_permission_commands = { + "workbook-update-plan", + "workbook-update-review", + "workbook-update-apply", "native-workbook-plan", "native-workbook-review", "native-workbook-import", @@ -1554,6 +1565,17 @@ def main(argv: list[str] | None = None) -> int: result = plan_solution_dependencies( args.solution, results_path=args.results, workspace_id=args.workspace_id, ) + elif args.command.startswith("workbook-update-"): + from .workbook_updates import ( + plan_workbook_update, review_workbook_update, apply_workbook_update, _read_external, + ) + if args.command == "workbook-update-plan": + result = plan_workbook_update(args.solution, args.content, args.baseline) + else: + _, review = _read_external(args.review) + operation = (review_workbook_update if args.command == "workbook-update-review" + else apply_workbook_update) + result = operation(args.solution, args.content, args.baseline, review) elif args.command.startswith("native-workbook-"): from .native_workbooks import ( plan_native_workbook, review_native_workbook, import_native_workbook, load_json, diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/native_workbooks.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/native_workbooks.py index 8259faa4e9d..d81ce03bf3e 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/native_workbooks.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/native_workbooks.py @@ -63,7 +63,8 @@ def identities(root, relative): return content, instance -def _query(surface): +def _query(surface, *, allowed_tables=(), allowed_functions=(), allow_sentinel=False, + include_lineage=False): raw = surface["node"]["query"] if not isinstance(raw, str) or surface["queryType"] != workbook.AH: raise ValueError("Every native query must be a string using advancedHunting.") @@ -71,7 +72,8 @@ def _query(surface): for match in re.finditer(r"\b(?:workspace|app|resource|cluster|database|table|totable|externaldata)\s*[\[(]", code, re.I): if not workbook._is_union_withsource_output_binding(code, match.start()): raise ValueError("Native import does not support dynamic, external or cross-resource sources.") - known = set(TABLE_WORKLOADS) | set(mapping_catalog.load_mapping_catalog()["tables"]) + allowed_tables, allowed_functions = set(allowed_tables), set(allowed_functions) + known = set(TABLE_WORKLOADS) | set(mapping_catalog.load_mapping_catalog()["tables"]) | allowed_tables asim_names = {name for name in re.findall(r"\b[A-Za-z_]\w*", code) if workbook.is_standard_builtin_asim_parser(name)} tables = {r["table"] for r in query_table_references(raw, known_tables=known | asim_names)} @@ -84,18 +86,24 @@ def _query(surface): if workbook.is_standard_builtin_asim_parser(name)} # Built-in parsers are supported functions, not proof of their hidden # input tables, workloads or connector prerequisites. - tables -= asim + tables -= asim | allowed_functions local = set(re.findall(r"\blet\s+([A-Za-z_]\w*)\s*=", code)) tables -= local | workbook._BUILTINS | workbook._STATEMENT_PREFIXES - opaque = workbook.opaque_functions(raw) - tables - explicit = workbook.opaque_functions(raw, include_bare=False) | (quoted_calls - asim) + opaque = workbook.opaque_functions(raw) - tables - allowed_functions + explicit = (workbook.opaque_functions(raw, include_bare=False) + | (quoted_calls - asim)) - allowed_functions if opaque or explicit: raise ValueError("Unreviewable native functions/sources: " + ", ".join(sorted(opaque | explicit))) - if any(table not in TABLE_WORKLOADS or TABLE_WORKLOADS[table] == "sentinel" for table in tables): + if any((table not in TABLE_WORKLOADS or TABLE_WORKLOADS[table] == "sentinel") + and not (allow_sentinel and table in allowed_tables) for table in tables): raise ValueError("Unknown or non-native tables: " + ", ".join(sorted(tables))) - if not tables and not asim and not rewrite.is_source_free_literal_query(raw): + known_function_input = allowed_functions & (calls | quoted_calls | workbook.opaque_functions(raw)) + if not tables and not asim and not known_function_input and not rewrite.is_source_free_literal_query(raw): raise ValueError("Empty lineage requires a standard built-in ASIM parser " "or a recognized literal datatable or numeric range query.") + if include_lineage: + return {"tables": sorted(tables), "builtinAsimParsers": sorted(asim), + "workspaceFunctions": sorted(known_function_input)} return sorted(tables) diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_updates.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_updates.py new file mode 100644 index 00000000000..506a0bc1ea4 --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_updates.py @@ -0,0 +1,271 @@ +"""Explicit offline reviews of post-import edits; historical receipts never change. + +The reverse delta retains only changed baseline values, not another raw Notebook. +Hashes are review/freshness evidence, not signatures or runtime qualification. +""" +from __future__ import annotations + +from copy import deepcopy +import json +from pathlib import Path +import sys + +from ._workbook_core import workbook +from .dependencies import dependency_validation_errors +from .native_workbooks import _payload, _canonical_payload_sha256, _query +from .workbook_functions import validate_functions +from .workbooks import ( + _root, _path, _document, _load_json, _sha, _write, validate_workbooks, +) + + +def _canonical(value): + return _sha(json.dumps(value, sort_keys=True, separators=(",", ":"), + ensure_ascii=False, allow_nan=False).encode("utf-8")) + + +def _review_digest(review): + return _canonical({key: value for key, value in review.items() if key != "reviewSha256"}) + + +def _reverse(current, baseline, path=()): + if isinstance(current, dict) and isinstance(baseline, dict): + changes = [] + for key in sorted(current.keys() - baseline.keys()): + changes.append({"path": [*path, key], "op": "remove"}) + for key in sorted(baseline.keys() - current.keys()): + changes.append({"path": [*path, key], "op": "add", "value": baseline[key]}) + for key in sorted(current.keys() & baseline.keys()): + changes.extend(_reverse(current[key], baseline[key], (*path, key))) + return changes + if isinstance(current, list) and isinstance(baseline, list): + changes = [] + for index in range(min(len(current), len(baseline))): + changes.extend(_reverse(current[index], baseline[index], (*path, index))) + for index in reversed(range(len(baseline), len(current))): + changes.append({"path": [*path, index], "op": "remove"}) + for index in range(len(current), len(baseline)): + changes.append({"path": [*path, index], "op": "add", "value": baseline[index]}) + return changes + if type(current) is type(baseline) and current == baseline: + return [] + return [{"path": list(path), "op": "replace", "value": baseline}] + + +def _restore(payload, changes): + value = deepcopy(payload) + for change in changes: + path, op = change["path"], change["op"] + if not path: + raise ValueError("Baseline delta must not replace the entire Notebook.") + parent = value + for part in path[:-1]: + parent = parent[part] + key = path[-1] + if op == "remove": + if isinstance(parent, list): + parent.pop(key) + else: + del parent[key] + elif op == "add" and isinstance(parent, list): + parent.insert(key, deepcopy(change["value"])) + elif op in ("add", "replace"): + parent[key] = deepcopy(change["value"]) + else: + raise ValueError("Unknown baseline delta operation.") + return value + + +def _historical_errors(root, document): + if "authoredReview" in document.get("contentProvenance", {}): + return ["Baseline must be an original reviewed authored artifact, not an update review."] + result = validate_workbooks(root, selected_only=True, _documents=[ + (root / "XDR Workbooks" / "historical-baseline.json", document)]) + return [error for row in result["results"] for error in row["errors"]] + + +def _queries(document, provenance): + native = provenance.get("origin") == "native-xdr" + records = provenance.get("requiredWorkspaceFunctions", []) + aliases = {record["functionAlias"] for record in records} + declared = {table for record in provenance["requiredWorkloads"] for table in record["dataTypes"]} + queries, used = [], set() + for surface in workbook.query_surfaces(document): + raw = surface["node"]["query"] + lineage = _query(surface, allowed_tables=declared if not native else (), + allowed_functions=aliases, allow_sentinel=not native, + include_lineage=True) + tables = lineage["tables"] + if not set(tables) <= declared: + raise ValueError("Current query tables are outside historical reviewed dependencies: " + + ", ".join(sorted(set(tables) - declared))) + used.update(lineage["workspaceFunctions"]) + queries.append({ + "path": surface["path"], "querySha256": workbook.query_hash(raw), + **lineage, + }) + if used != aliases: + raise ValueError("Current queries must exactly cover historical workspace function prerequisites.") + return queries + + +def _plan(root, current, baseline, baseline_record): + errors = _historical_errors(root, baseline) + if errors: + raise ValueError("Historical baseline failed validation: " + "; ".join(errors)) + historical = baseline["contentProvenance"] + if not isinstance(current.get("xdrMetadata"), dict) or not isinstance(current.get("contentProvenance"), dict): + raise ValueError("Current workbook requires authored metadata and historical provenance objects.") + for field in ("contentId", "instanceId"): + if current["xdrMetadata"].get(field) != baseline["xdrMetadata"][field]: + raise ValueError("Workbook identity must remain unchanged.") + for field in ("requiredWorkloads", "requiredDataConnectors", "requiredWorkspaceFunctions", + "dependencyReviewRequired"): + if current["contentProvenance"].get(field) != historical.get(field): + raise ValueError("Updates cannot replace historical reviewed dependencies: " + field) + restored = deepcopy(current) + restored["contentProvenance"] = deepcopy(historical) + errors = dependency_validation_errors(restored) + validate_functions(root, historical.get("requiredWorkspaceFunctions", [])) + if errors: + raise ValueError("; ".join(errors)) + queries = _queries(restored, historical) + old = {q["path"]: q["node"]["query"] for q in workbook.query_surfaces(baseline)} + new = {q["path"]: q["node"]["query"] for q in workbook.query_surfaces(restored)} + changed = [] + for pointer in sorted(old.keys() | new.keys()): + before = workbook.query_hash(old[pointer]) if pointer in old else None + after = workbook.query_hash(new[pointer]) if pointer in new else None + if before != after: + changed.append({"path": pointer, "baselineQuerySha256": before, + "querySha256": after, "decision": None, "evidence": ""}) + return { + "schemaVersion": "1.0.0", "baseline": baseline_record, + "baselineMetadata": deepcopy(baseline["xdrMetadata"]), + "reversePayloadChanges": _reverse(_payload(restored), _payload(baseline)), + "canonicalPayloadSha256": _canonical_payload_sha256(restored), + "canonicalDocumentSha256": _canonical(restored), + "queries": queries, "changedQueries": changed, + "actor": "", "reference": "", "decision": None, "reviewSha256": None, + }, restored + + +def _read_external(path): + path = Path(path).expanduser() + if any(p.is_symlink() for p in (path, *path.parents)): + raise ValueError("Review/baseline files must not use symlinks.") + raw = path.read_bytes() + value = _load_json(raw) + if not isinstance(value, dict): + raise ValueError("Review/baseline JSON must be an object.") + return raw, value + + +def plan_workbook_update(solution, content, baseline_file): + root = _root(solution) + path = _path(root, content) + if not content.startswith("XDR Workbooks/") or path.suffix.lower() != ".json": + raise ValueError("Update content must be authored XDR Workbooks JSON.") + raw, baseline = _read_external(baseline_file) + record = {"reference": str(Path(baseline_file).name), "sha256": _sha(raw), + "canonicalDocumentSha256": _canonical(baseline)} + return _plan(root, _document(path), baseline, record)[0] + + +def review_workbook_update(solution, content, baseline_file, review): + expected = plan_workbook_update(solution, content, baseline_file) + if not isinstance(review, dict) or set(review) != set(expected): + raise ValueError("Update review must have exactly the current plan fields.") + for field in ("actor", "reference"): + if not isinstance(review[field], str) or not review[field].strip(): + raise ValueError("Update review requires " + field) + expected[field] = review[field] + if review["decision"] != "accept": + raise ValueError("Update requires explicit accept of the whole current authored document.") + expected["decision"] = "accept" + if not isinstance(review["changedQueries"], list) or len(review["changedQueries"]) != len(expected["changedQueries"]): + raise ValueError("Review must cover every changed/added/removed query.") + for actual, query in zip(review["changedQueries"], expected["changedQueries"]): + if not isinstance(actual, dict): + raise ValueError("Malformed changed query review.") + if actual.get("decision") != "accept" or not isinstance(actual.get("evidence"), str) or not actual["evidence"].strip(): + raise ValueError("Every changed query requires explicit accept and evidence.") + query.update(decision="accept", evidence=actual["evidence"]) + digest = _review_digest(expected) + if review["reviewSha256"] not in (None, digest): + raise ValueError("Update review evidence hash is stale.") + expected["reviewSha256"] = review["reviewSha256"] + if review != expected: + raise ValueError("Update review is stale or differs from exact baseline/current query dependencies.") + expected["reviewSha256"] = digest + return expected + + +def apply_workbook_update(solution, content, baseline_file, review): + root = _root(solution) + path = _path(root, content) + previous = path.read_bytes() + review = review_workbook_update(root, content, baseline_file, review) + _, baseline = _read_external(baseline_file) + document = _document(path) + document["contentProvenance"] = deepcopy(baseline["contentProvenance"]) + document["contentProvenance"]["authoredReview"] = review + errors = validate_updated_document(root, document) + if errors: + raise ValueError("; ".join(errors)) + from .workbooks import authored_schema_errors + errors = authored_schema_errors(document) + if errors: + raise ValueError("; ".join(errors)) + _write(path, (json.dumps(document, indent=2, ensure_ascii=False) + "\n").encode(), previous) + return {"path": content, "sha256": _sha(path.read_bytes()), "runtimeQualified": False} + + +def validate_updated_document(solution, document): + try: + root = _root(solution) + current = deepcopy(document) + review = current["contentProvenance"].pop("authoredReview") + if not isinstance(review, dict): + return ["authoredReview must be a complete explicit update review."] + if review.get("reviewSha256") != _review_digest(review): + return ["Authored review evidence hash is stale."] + baseline = _restore(_payload(current), review["reversePayloadChanges"]) + baseline["xdrMetadata"] = deepcopy(review["baselineMetadata"]) + baseline["contentProvenance"] = deepcopy(current["contentProvenance"]) + if _canonical(baseline) != review["baseline"]["canonicalDocumentSha256"]: + return ["Historical baseline/proof hash is stale."] + expected, _ = _plan(root, current, baseline, review["baseline"]) + for field in ("actor", "reference"): + if not isinstance(review[field], str) or not review[field].strip(): + return ["Update review requires nonempty " + field] + expected[field] = review[field] + if review["decision"] != "accept": + return ["Update requires explicit whole-document accept."] + expected["decision"] = "accept" + if len(review["changedQueries"]) != len(expected["changedQueries"]): + return ["Changed query review inventory is stale."] + for actual, query in zip(review["changedQueries"], expected["changedQueries"]): + if actual["decision"] != "accept" or not isinstance(actual["evidence"], str) or not actual["evidence"].strip(): + return ["Changed query requires accept and evidence."] + query.update(decision="accept", evidence=actual["evidence"]) + expected["reviewSha256"] = _review_digest(expected) + if review != expected: + return ["Authored review is stale: whole document, queries or dependencies changed."] + return [] + except (OSError, ValueError, TypeError, KeyError, IndexError, AttributeError) as exc: + return ["Invalid authored review: " + str(exc)] + + +def main(): + if not (3, 11) <= sys.version_info[:2] < (3, 13): + raise ValueError("Workbook update validation requires Python 3.11 or 3.12.") + document = _load_json(sys.stdin.read()) + from .workbooks import authored_schema_errors + errors = authored_schema_errors(document) + validate_updated_document(sys.argv[1], document) + print(json.dumps({"errors": errors})) + return int(bool(errors)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbooks.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbooks.py index ec530a2350c..b33ec26f225 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbooks.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbooks.py @@ -424,21 +424,23 @@ def authored_schema_errors(document): for e in Draft202012Validator(schema).iter_errors(document)] -def validate_workbooks(solution, *, selected_only=False) -> dict: +def validate_workbooks(solution, *, selected_only=False, _documents=None) -> dict: root = _root(solution) inventory = {r["path"]: r for r in inventory_workbooks(root)} - files = selected_workbook_files(root) if selected_only else _files(root, "XDR Workbooks") + files = (selected_workbook_files(root) if selected_only else _files(root, "XDR Workbooks")) if _documents is None else [] results, identities, instances = [], set(), set() source_paths = set() - for path in files: - document = _document(path) + documents = ((path, _document(path)) for path in files) if _documents is None else _documents + for path, document in documents: + digest = _sha(path.read_bytes()) if _documents is None else _sha( + json.dumps(document, sort_keys=True, ensure_ascii=False).encode()) schema_errors = authored_schema_errors(document) if schema_errors: provenance = document.get("contentProvenance") if isinstance(provenance, dict) and isinstance(provenance.get("source"), dict): source_paths.add(provenance["source"].get("path")) results.append({"path": path.relative_to(root).as_posix(), - "sha256": _sha(path.read_bytes()), "valid": False, "errors": schema_errors}) + "sha256": digest, "valid": False, "errors": schema_errors}) continue errors = dependency_validation_errors(document) metadata = document.get("xdrMetadata", {}) @@ -466,14 +468,18 @@ def validate_workbooks(solution, *, selected_only=False) -> dict: errors.append("Workbook displayName is required.") if any(key in provenance for key in ("packageId", "packageVersion", "contentId", "contentVersion")): errors.append("Deployment provenance quartet must not be authored.") - if provenance.get("origin") == "native-xdr": - from .native_workbooks import validate_native_document - errors.extend(validate_native_document(root, document)) + if "authoredReview" in provenance or provenance.get("origin") == "native-xdr": + if "authoredReview" in provenance: + from .workbook_updates import validate_updated_document + errors.extend(validate_updated_document(root, document)) + else: + from .native_workbooks import validate_native_document + errors.extend(validate_native_document(root, document)) if source.get("path") in source_paths: errors.append("Duplicate workbook source provenance.") source_paths.add(source.get("path")) results.append({"path": path.relative_to(root).as_posix(), - "sha256": _sha(path.read_bytes()), "valid": not errors, "errors": errors}) + "sha256": digest, "valid": not errors, "errors": errors}) continue row = inventory.get(source.get("path")) if not row or row["sha256"] != source.get("sha256"): @@ -595,7 +601,7 @@ def validate_workbooks(solution, *, selected_only=False) -> dict: if recorded_source != set(original_queries) or recorded_output != set(output_queries): errors.append("Query provenance does not cover complete original/output inventory.") results.append({"path": path.relative_to(root).as_posix(), - "sha256": _sha(path.read_bytes()), "valid": not errors, "errors": errors}) + "sha256": digest, "valid": not errors, "errors": errors}) if not selected_only: for relative in sorted(set(inventory) - source_paths): results.append({"path": relative, "valid": False, "errors": ["Source workbook has no authored candidate."]}) diff --git a/Tools/SentinelToXDRMigration/tests/test_native_workbooks.py b/Tools/SentinelToXDRMigration/tests/test_native_workbooks.py index fb3bfbc5d10..a8af37e4160 100644 --- a/Tools/SentinelToXDRMigration/tests/test_native_workbooks.py +++ b/Tools/SentinelToXDRMigration/tests/test_native_workbooks.py @@ -384,7 +384,8 @@ def test_native_only_v31_package_requires_no_sentinel_parent(solution, monkeypat @pytest.mark.skipif(not shutil.which("pwsh"), reason="PowerShell 7 required") -def test_real_v31_mixed_package_and_static_validator(solution, monkeypatch): +@pytest.mark.parametrize("reviewed_updates", [False, True]) +def test_real_v31_mixed_package_and_static_validator(solution, monkeypatch, reviewed_updates): monkeypatch.setenv("SENTINEL_XDR_PYTHON", sys.executable) sentinel = solution / "Workbooks" / "sentinel.json" sentinel.parent.mkdir() @@ -418,6 +419,24 @@ def test_real_v31_mixed_package_and_static_validator(solution, monkeypatch): imported(solution) (solution / SOURCE).unlink() assert (solution / "XDR Workbooks" / "sentinel.json").read_bytes() == migrated + if reviewed_updates: + from sentinel_xdr_migration.workbook_updates import plan_workbook_update, apply_workbook_update + for path in workbooks.selected_workbook_files(solution): + baseline = solution / ("baseline-" + path.name) + baseline.write_bytes(path.read_bytes()) + document = json.loads(path.read_bytes()) + for surface in workbook.query_surfaces(document): + if not surface["node"]["query"].startswith("datatable("): + surface["node"]["query"] += " | take 2" + document["isLocked"] = True + path.write_text(json.dumps(document), encoding="utf-8") + relative = path.relative_to(solution).as_posix() + review = plan_workbook_update(solution, relative, baseline) + review.update(actor="fixture reviewer", reference="Reviewed post-import fixes", decision="accept") + for query in review["changedQueries"]: + query.update(decision="accept", evidence="Read exact fixture correction and lineage.") + apply_workbook_update(solution, relative, baseline, review) + baseline.unlink() assert workbooks.validate_workbooks(solution)["invalid"] == 0 subprocess.run(["git", "init", "--quiet", str(solution.parent.parent)], check=True) arguments = [ diff --git a/Tools/SentinelToXDRMigration/tests/test_workbook_updates.py b/Tools/SentinelToXDRMigration/tests/test_workbook_updates.py new file mode 100644 index 00000000000..1ee6e421824 --- /dev/null +++ b/Tools/SentinelToXDRMigration/tests/test_workbook_updates.py @@ -0,0 +1,328 @@ +from copy import deepcopy +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +from uuid import uuid4 + +import pytest + +from sentinel_xdr_migration import cli, workbook_updates as updates, workbooks +from test_native_workbooks import ( + solution as native_solution, imported as native_imported, candidate as native_candidate, +) +from test_workbooks import solution as migrated_solution, config, convert, authored + +ROOT = Path(__file__).resolve().parents[3] + + +def powershell_scan(root, path, resources=()): + script = ROOT / "Tools" / "Create-Azure-Sentinel-Solution" / "common" / "defenderWorkbooks.ps1" + def literal(value): + return "'" + str(value).replace("'", "''") + "'" + command = ( + "$ErrorActionPreference='Stop'; . " + literal(script.with_name("commonFunctions.ps1")) + + "; . " + literal(script) + "; " + "$doc=Get-Content -LiteralPath " + literal(path) + " -Raw | ConvertFrom-Json; " + "$data=Get-Content -LiteralPath " + literal(next((root / "Data").glob("Solution_*.json"))) + + " -Raw | ConvertFrom-Json; $template=" + literal(json.dumps({"resources": list(resources)})) + + " | ConvertFrom-Json; " + "Get-DefenderWorkbookParserDependencies -Document $doc -ContentToImport $data " + "-Template $template -SolutionRoot " + literal(root) + " -SolutionName " + literal(root.name)) + return subprocess.run( + ["pwsh", "-NoLogo", "-NoProfile", "-NonInteractive", "-Command", command], + capture_output=True, text=True, encoding="utf-8", + env={**os.environ, "SENTINEL_XDR_PYTHON": sys.executable}) + + +@pytest.fixture(params=["native", "migrated"]) +def update_case(request): + root = request.getfixturevalue(request.param + "_solution") + if request.param == "native": + native_imported(root) + path, index = native_candidate(root), 1 + else: + assert convert(root)["converted"] == 1 + path, index = authored(root), 0 + data_path = next((root / "Data").glob("Solution_*.json")) + data = json.loads(data_path.read_bytes()) + data["XDR Workbooks"] = [path.relative_to(root).as_posix()] + data_path.write_text(json.dumps(data), encoding="utf-8") + baseline = root / "external-baseline.json" + baseline.write_bytes(path.read_bytes()) + old = json.loads(path.read_bytes()) + document = deepcopy(old) + document["items"][index]["content"]["query"] += " | take 3" + document["isLocked"] = True + document["xdrMetadata"]["version"] = "3.2.3" + path.write_text(json.dumps(document), encoding="utf-8") + return root, path, baseline, index, old + + +def accepted(case): + root, path, baseline, _, _ = case + plan = updates.plan_workbook_update(root, path.relative_to(root).as_posix(), baseline) + plan.update(actor="kingwil", reference="Explicit offline fixture correction review", decision="accept") + for query in plan["changedQueries"]: + query.update(decision="accept", evidence="Read exact corrected KQL and current dependency inventory.") + return plan + + +def apply(case): + root, path, baseline, _, _ = case + return updates.apply_workbook_update(root, path.relative_to(root).as_posix(), baseline, accepted(case)) + + +def test_restores_historical_proof_without_changing_edited_notebook(update_case): + root, path, baseline, _, old = update_case + current = json.loads(path.read_bytes()) + if "conversion" in current["contentProvenance"]: + query = current["contentProvenance"]["conversion"]["queries"][0] + query["conversion"]["convertedKql"] = "invented manual replacement" + query["targetSha256"] = "0" * 64 + else: + current["contentProvenance"]["nativeReview"]["actor"] = "manual replacement" + path.write_text(json.dumps(current), encoding="utf-8") + assert workbooks.validate_workbooks(root, selected_only=True)["invalid"] == 1 + assert apply(update_case)["runtimeQualified"] is False + result = json.loads(path.read_bytes()) + review = result["contentProvenance"].pop("authoredReview") + assert result["contentProvenance"] == old["contentProvenance"] + assert result["items"] == current["items"] + assert result["xdrMetadata"] == current["xdrMetadata"] + assert len(review["changedQueries"]) == 1 + # No external baseline or native raw source is required after apply. + baseline.unlink() + for source in (root / "Workbook Sources").rglob("*.json"): + source.unlink() + assert workbooks.validate_workbooks(root, selected_only=True)["invalid"] == 0 + + +@pytest.mark.parametrize("mutation", [ + "query", "settings", "identity", "historical-proof", "baseline-hash", "delta", + "query-inventory", "dependencies", "metadata", "baseline-raw-hash", "actor", +]) +def test_future_tampering_and_staleness_fail(update_case, mutation): + root, path, _, index, _ = update_case + apply(update_case) + document = json.loads(path.read_bytes()) + provenance = document["contentProvenance"] + review = provenance["authoredReview"] + if mutation == "query": + document["items"][index]["content"]["query"] += " | take 4" + elif mutation == "settings": + document["isLocked"] = False + elif mutation == "identity": + document["xdrMetadata"]["contentId"] = str(uuid4()) + elif mutation == "historical-proof": + provenance["source"]["sha256"] = "0" * 64 + elif mutation == "baseline-hash": + review["baseline"]["canonicalDocumentSha256"] = "0" * 64 + elif mutation == "delta": + review["reversePayloadChanges"] = [] + elif mutation == "query-inventory": + review["queries"].pop() + elif mutation == "dependencies": + provenance["requiredWorkloads"] = [] + elif mutation == "baseline-raw-hash": + review["baseline"]["sha256"] = "0" * 64 + elif mutation == "actor": + review["actor"] = "changed" + else: + document["xdrMetadata"]["displayName"] += " changed" + path.write_text(json.dumps(document), encoding="utf-8") + assert workbooks.validate_workbooks(root, selected_only=True)["invalid"] == 1 + + +@pytest.mark.parametrize("query", [ + "UnknownTable | take 1", "UnknownFunction() | take 1", + "EmailEvents | union DeviceEvents", "workspace('other').EmailEvents", + "EmailEvents | extend x=UnknownFunction(1)", + "_Im_Authentication | union UnknownTable", +]) +def test_unsupported_current_lineage_cannot_be_approved(update_case, query): + root, path, baseline, index, _ = update_case + document = json.loads(path.read_bytes()) + document["items"][index]["content"]["query"] = query + path.write_text(json.dumps(document), encoding="utf-8") + with pytest.raises(ValueError): + updates.plan_workbook_update(root, path.relative_to(root).as_posix(), baseline) + + +@pytest.mark.parametrize("mutation", ["hash", "actor", "query-hash", "table", "evidence", "decision", "proof"]) +def test_review_does_not_accept_self_asserted_or_stale_evidence(update_case, mutation): + root, path, baseline, _, _ = update_case + review = accepted(update_case) + before = path.read_bytes() + if mutation == "hash": + review["canonicalDocumentSha256"] = "0" * 64 + elif mutation == "actor": + review["actor"] = " " + elif mutation == "query-hash": + review["changedQueries"][0]["querySha256"] = "0" * 64 + elif mutation == "table": + review["queries"][0]["tables"] = ["Invented"] + elif mutation == "evidence": + review["changedQueries"][0]["evidence"] = " " + elif mutation == "decision": + review["decision"] = None + else: + old = json.loads(baseline.read_bytes()) + old["contentProvenance"]["source"]["sha256"] = "0" * 64 + baseline.write_text(json.dumps(old), encoding="utf-8") + with pytest.raises(ValueError): + updates.apply_workbook_update(root, path.relative_to(root).as_posix(), baseline, review) + assert path.read_bytes() == before + + +def test_recomputed_hashes_do_not_override_discovered_dependencies(update_case): + root, path, _, index, old = update_case + apply(update_case) + document = json.loads(path.read_bytes()) + review = document["contentProvenance"].pop("authoredReview") + document["items"][index]["content"]["query"] = "UnknownTable | take 1" + review["reversePayloadChanges"] = updates._reverse(updates._payload(document), updates._payload(old)) + review["canonicalPayloadSha256"] = updates._canonical_payload_sha256(document) + review["canonicalDocumentSha256"] = updates._canonical(document) + digest = updates.workbook.query_hash("UnknownTable | take 1") + review["queries"][0]["querySha256"] = digest + review["changedQueries"][0]["querySha256"] = digest + review["reviewSha256"] = updates._review_digest(review) + document["contentProvenance"]["authoredReview"] = review + path.write_text(json.dumps(document), encoding="utf-8") + result = workbooks.validate_workbooks(root, selected_only=True) + assert result["invalid"] == 1 + assert any("UnknownTable" in error for error in result["results"][0]["errors"]) + + +def test_replacement_conversion_receipt_cannot_be_used_as_original_baseline(migrated_solution): + root = migrated_solution + convert(root) + path = authored(root) + document = json.loads(path.read_bytes()) + query = document["contentProvenance"]["conversion"]["queries"][0] + raw = document["items"][0]["content"]["query"] + " | take 3" + document["items"][0]["content"]["query"] = raw + query["targetSha256"] = updates.workbook.query_hash(raw) + query["conversion"]["convertedKql"] = raw + baseline = root / "fabricated-baseline.json" + baseline.write_text(json.dumps(document), encoding="utf-8") + path.write_text(json.dumps(document), encoding="utf-8") + with pytest.raises(ValueError, match="deterministic conversion"): + updates.plan_workbook_update(root, path.relative_to(root).as_posix(), baseline) + + +def test_builtin_asim_lineage_is_explicit_without_inventing_hidden_tables(update_case): + root, path, _, index, _ = update_case + document = json.loads(path.read_bytes()) + document["items"][index]["content"]["query"] = "_Im_Authentication | take 1" + path.write_text(json.dumps(document), encoding="utf-8") + review = accepted(update_case) + assert review["queries"][0]["builtinAsimParsers"] == ["_Im_Authentication"] + assert review["queries"][0]["tables"] == [] + apply(update_case) + assert workbooks.validate_workbooks(root, selected_only=True)["invalid"] == 0 + + +def test_added_and_removed_queries_and_second_review(update_case): + root, path, baseline, _, old = update_case + document = json.loads(path.read_bytes()) + document["items"].insert(0, {"type": 3, "content": { + "queryType": "advancedHunting", "query": 'datatable(x:string)["reviewed literal"]'}}) + path.write_text(json.dumps(document), encoding="utf-8") + review = accepted(update_case) + assert any(q["baselineQuerySha256"] is None for q in review["changedQueries"]) + apply(update_case) + assert workbooks.validate_workbooks(root, selected_only=True)["invalid"] == 0 + document = json.loads(path.read_bytes()) + document["items"] = [{"type": 1, "content": {"json": "All original query panels removed."}}] + path.write_text(json.dumps(document), encoding="utf-8") + review = accepted(update_case) + assert len(review["changedQueries"]) == len(list( + updates.workbook.query_surfaces(old))) + assert all(q["querySha256"] is None for q in review["changedQueries"]) + apply(update_case) + assert workbooks.validate_workbooks(root, selected_only=True)["invalid"] == 0 + + +def test_cli_plan_review_apply(update_case, capsys): + root, path, baseline, _, _ = update_case + args = ["--solution", str(root), "--content", path.relative_to(root).as_posix(), + "--baseline", str(baseline)] + assert cli.main(["workbook-update-plan", *args]) == 0 + assert json.loads(capsys.readouterr().out)["decision"] is None + review_file = root / "review.json" + review_file.write_text(json.dumps(accepted(update_case)), encoding="utf-8") + for operation in ("review", "apply"): + assert cli.main(["workbook-update-" + operation, *args, "--review", str(review_file)]) == 0 + capsys.readouterr() + assert workbooks.validate_workbooks(root, selected_only=True)["invalid"] == 0 + + +@pytest.mark.parametrize("function_query", [False, True]) +def test_migrated_sentinel_tables_and_workspace_functions_remain_supported(migrated_solution, function_query): + from test_workbook_dependencies import ( + function, write_query, decisions, review, dependencies, + ) + root = migrated_solution + if function_query: + record = function(root) + write_query(root, "ReviewedFunction() | count") + plan = dependencies.plan_workbook_dependencies(root) + review(root, plan, decisions(plan, tables=["Syslog"], functions=[record])) + settings = json.loads((root / "Logs/review-1/conversion-config.json").read_bytes()) + expected_tables, expected_functions = [], ["ReviewedFunction"] + else: + write_query(root, "SecurityIncident | take 1") + settings = config(root, ["SecurityIncident"]) + settings["workbookDependencies"] = {"Workbooks/nested/fixture.json": { + "requiredWorkloads": [{"workload": "sentinel", "dataTypes": ["SecurityIncident"]}], + "requiredDataConnectors": [{"connectorId": "ReviewedSentinel", "dataTypes": ["SecurityIncident"]}], + }} + expected_tables, expected_functions = ["SecurityIncident"], [] + assert convert(root, settings)["converted"] == 1 + path = authored(root) + old = json.loads(path.read_bytes()) + baseline = root / "baseline.json" + baseline.write_bytes(path.read_bytes()) + document = deepcopy(old) + document["items"][0]["content"]["query"] += " | take 2" + path.write_text(json.dumps(document), encoding="utf-8") + case = root, path, baseline, 0, old + plan = accepted(case) + assert plan["queries"][0]["tables"] == expected_tables + assert plan["queries"][0]["workspaceFunctions"] == expected_functions + apply(case) + assert workbooks.validate_workbooks(root)["invalid"] == 0 + if shutil.which("pwsh"): + resources = ([{ + "type": "Microsoft.OperationalInsights/workspaces/savedSearches", + "name": "workspace/ReviewedFunction", + "properties": {"functionAlias": "ReviewedFunction", "query": "Syslog | count", + "functionParameters": ""}, + }] if function_query else []) + result = powershell_scan(root, path, resources) + assert result.returncode == 0, result.stdout + result.stderr + if function_query: + missing = powershell_scan(root, path) + assert missing.returncode != 0 + assert "requires exactly one emitted top-level savedSearch" in missing.stderr + if function_query: + with (root / record["sourcePath"]).open("a", encoding="utf-8") as handle: + handle.write("\n# changed dependency") + assert workbooks.validate_workbooks(root)["invalid"] == 1 + + +@pytest.mark.skipif(not shutil.which("pwsh"), reason="PowerShell 7 required") +def test_powershell_assert_and_dependency_scan_use_current_review(update_case): + root, path, _, index, _ = update_case + apply(update_case) + result = powershell_scan(root, path) + assert result.returncode == 0, result.stdout + result.stderr + document = json.loads(path.read_bytes()) + document["items"][index]["content"]["query"] += " | take 8" + path.write_text(json.dumps(document), encoding="utf-8") + result = powershell_scan(root, path) + assert result.returncode != 0 From 7ee0c0b7b140853872b52d448d54a3887554b88e Mon Sep 17 00:00:00 2001 From: kingwil <5839248+kingwil@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:27:24 +0100 Subject: [PATCH 2/3] Preserve reviewed historical hybrid workbook table lineage Handle accepted migrated baselines with incomplete nested Sentinel lineage without rewriting receipts. Require exact per-query inherited table sets and explicit Sentinel connector coverage; keep native gates strict and feed validated current tables into PowerShell prerequisite scanning. Document standalone V3.1 content maintenance without changing frozen workflow gates. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../common/defenderWorkbooks.ps1 | 7 ++ Tools/SentinelToXDRMigration/README.md | 22 ++++++ .../workbook_updates.py | 34 +++++++-- .../tests/test_workbook_updates.py | 70 +++++++++++++++++++ 4 files changed, 128 insertions(+), 5 deletions(-) diff --git a/Tools/Create-Azure-Sentinel-Solution/common/defenderWorkbooks.ps1 b/Tools/Create-Azure-Sentinel-Solution/common/defenderWorkbooks.ps1 index af931b07fc5..5cd6386936d 100644 --- a/Tools/Create-Azure-Sentinel-Solution/common/defenderWorkbooks.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/common/defenderWorkbooks.ps1 @@ -866,6 +866,13 @@ function Get-DefenderWorkbookParserDependencies { } } $sourceAllowedBareTables = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + if ($null -ne $provenance.PSObject.Properties['authoredReview']) { + # The shared adapter has checked these against the reconstructed + # baseline and current KQL, including exact inherited hybrid lineage. + foreach ($query in $provenance.authoredReview.queries) { + foreach ($table in $query.tables) { [void]$allowedBareTables.Add($table) } + } + } foreach ($item in $allowedBareTables) { [void]$sourceAllowedBareTables.Add($item) } $catalogPath = Join-Path $PSScriptRoot '../../SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/mapping_catalog.json' if (Test-Path -LiteralPath $catalogPath -PathType Leaf) { diff --git a/Tools/SentinelToXDRMigration/README.md b/Tools/SentinelToXDRMigration/README.md index bf96f70c0c1..6fec0cc6460 100644 --- a/Tools/SentinelToXDRMigration/README.md +++ b/Tools/SentinelToXDRMigration/README.md @@ -462,6 +462,14 @@ updates also support their existing Sentinel tables and workspace functions. Unknown/new dependencies and unsupported dynamic/cross-resource inputs fail; reviewer assertions cannot make them supported. +Older accepted migrated artifacts can contain nested hybrid table references +that their historical dependency inventory omitted. Their original receipt is +not rewritten: an existing query may retain that exact baseline table set only, +with existing explicit Sentinel connector coverage. The complete current +inventory records those discovered tables. Adding that undeclared lineage to +another query or changing its table set fails. Native-only restrictions remain +unchanged. + The review binds the original file SHA, canonical baseline artifact, complete current canonical Notebook and authored document, exact query/dependency inventories, actor/reference, and decisions. Only changed baseline values and @@ -484,6 +492,20 @@ Keep existing version/release and candidate gates: package through -ContentRoot -SkipPackaging -PackageReportPath `. That wrapper otherwise defaults to legacy V3 packaging and disallows cross-checkout packaging; do not use it to create this candidate. +If an old migration run's frozen selection predates the maintained content, +do not alter that run or weaken its selection gate. The existing standalone +content-maintenance backend is: + +```powershell +& "$Toolkit\Tools\Create-Azure-Sentinel-Solution\V3\createSolutionV3_1.ps1" ` + -SolutionDataFolderPath "$Solution\Data" -ToolkitRootPath $Toolkit ` + -VersionMode local -VersionBump none ` + -ArmValidationMethod lab-deployment -SkipAttributionLookup +``` + +Use the same official validation-only wrapper afterward. This builds a +candidate; it does not establish migration workflow acceptance or lab/runtime +qualification. ### Toolkit-owner managed update policy diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_updates.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_updates.py index 506a0bc1ea4..369abf9e4ae 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_updates.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_updates.py @@ -12,6 +12,9 @@ from ._workbook_core import workbook from .dependencies import dependency_validation_errors +from .dependencies import TABLE_WORKLOADS +from .catalog import query_table_references +from ._workbook_core import mapping_catalog from .native_workbooks import _payload, _canonical_payload_sha256, _query from .workbook_functions import validate_functions from .workbooks import ( @@ -84,21 +87,42 @@ def _historical_errors(root, document): return [error for row in result["results"] for error in row["errors"]] -def _queries(document, provenance): +def _queries(document, provenance, baseline): native = provenance.get("origin") == "native-xdr" records = provenance.get("requiredWorkspaceFunctions", []) aliases = {record["functionAlias"] for record in records} declared = {table for record in provenance["requiredWorkloads"] for table in record["dataTypes"]} + sentinel = {table for record in provenance["requiredWorkloads"] + if record["workload"] == "sentinel" for table in record["dataTypes"]} + connectors = {table for record in provenance["requiredDataConnectors"] for table in record["dataTypes"]} + known = set(TABLE_WORKLOADS) | set(mapping_catalog.load_mapping_catalog()["tables"]) + historical = {surface["path"]: surface for surface in workbook.query_surfaces(baseline)} queries, used = [], set() for surface in workbook.query_surfaces(document): raw = surface["node"]["query"] - lineage = _query(surface, allowed_tables=declared if not native else (), + inherited = set() + previous = historical.get(surface["path"]) + if not native and previous: + # Some accepted converter receipts predate complete nested-source + # discovery. Preserve only that exact query's proven old lineage; + # never invent dependencies or extend the exception to new queries. + candidates = {row["table"] for row in query_table_references( + previous["node"]["query"], known_tables=known)} & known + inherited = set(_query(previous, allowed_tables=declared | candidates, + allowed_functions=aliases, allow_sentinel=True)) + lineage = _query(surface, allowed_tables=declared | inherited if not native else (), allowed_functions=aliases, allow_sentinel=not native, include_lineage=True) tables = lineage["tables"] - if not set(tables) <= declared: + if inherited - declared and set(tables) != inherited: + raise ValueError("Historically incomplete dependencies require unchanged query table lineage.") + if not set(tables) <= declared | inherited: raise ValueError("Current query tables are outside historical reviewed dependencies: " - + ", ".join(sorted(set(tables) - declared))) + + ", ".join(sorted(set(tables) - declared - inherited))) + if not native and not (set(tables) & sentinel) <= connectors: + raise ValueError("Current Sentinel query tables require historical explicit connector coverage.") + if inherited - declared and not (inherited & sentinel & connectors): + raise ValueError("Inherited hybrid lineage requires historical explicit Sentinel connector coverage.") used.update(lineage["workspaceFunctions"]) queries.append({ "path": surface["path"], "querySha256": workbook.query_hash(raw), @@ -129,7 +153,7 @@ def _plan(root, current, baseline, baseline_record): validate_functions(root, historical.get("requiredWorkspaceFunctions", [])) if errors: raise ValueError("; ".join(errors)) - queries = _queries(restored, historical) + queries = _queries(restored, historical, baseline) old = {q["path"]: q["node"]["query"] for q in workbook.query_surfaces(baseline)} new = {q["path"]: q["node"]["query"] for q in workbook.query_surfaces(restored)} changed = [] diff --git a/Tools/SentinelToXDRMigration/tests/test_workbook_updates.py b/Tools/SentinelToXDRMigration/tests/test_workbook_updates.py index 1ee6e421824..84e63db6764 100644 --- a/Tools/SentinelToXDRMigration/tests/test_workbook_updates.py +++ b/Tools/SentinelToXDRMigration/tests/test_workbook_updates.py @@ -326,3 +326,73 @@ def test_powershell_assert_and_dependency_scan_use_current_review(update_case): path.write_text(json.dumps(document), encoding="utf-8") result = powershell_scan(root, path) assert result.returncode != 0 + + +@pytest.mark.parametrize("complete_dependencies", [False, True]) +def test_migrated_hybrid_queries_preserve_historical_sentinel_lineage(migrated_solution, complete_dependencies): + from test_workbook_dependencies import write_query + root = migrated_solution + raw = ( + 'let alerts = SecurityAlert | project SystemAlertId; ' + 'let incidents = materialize(SecurityIncident ' + '| join kind=inner (alerts) on $left.AlertId == $right.SystemAlertId); ' + 'incidents | count' + ) + write_query(root, raw) + tables = ["SecurityAlert", "SecurityIncident"] if complete_dependencies else ["SecurityAlert"] + settings = config(root, tables) + settings["workbookDependencies"] = {"Workbooks/nested/fixture.json": { + "requiredWorkloads": [{"workload": "sentinel", "dataTypes": tables}], + "requiredDataConnectors": [{"connectorId": "MicrosoftThreatProtection", "dataTypes": tables}], + }} + assert convert(root, settings)["converted"] == 1 + path = authored(root) + baseline = root / "baseline-hybrid.json" + baseline.write_bytes(path.read_bytes()) + old = json.loads(path.read_bytes()) + current = deepcopy(old) + current["items"][0]["content"]["query"] += " | take 2" + path.write_text(json.dumps(current), encoding="utf-8") + case = root, path, baseline, 0, old + plan = accepted(case) + assert plan["queries"][0]["tables"] == ["SecurityAlert", "SecurityIncident"] + apply(case) + updated = json.loads(path.read_bytes()) + historical = deepcopy(updated["contentProvenance"]) + historical.pop("authoredReview") + assert historical == old["contentProvenance"] + assert workbooks.validate_workbooks(root)["invalid"] == 0 + if shutil.which("pwsh"): + result = powershell_scan(root, path) + assert result.returncode == 0, result.stdout + result.stderr + for replacement in ( + raw + " | union SecurityEvent", + raw.replace("SecurityIncident", "SecurityEvent"), + ): + current["items"][0]["content"]["query"] = replacement + path.write_text(json.dumps(current), encoding="utf-8") + with pytest.raises(ValueError): + accepted(case) + if not complete_dependencies: + current["items"][0]["content"]["query"] = raw + current["items"].append(deepcopy(current["items"][0])) + path.write_text(json.dumps(current), encoding="utf-8") + with pytest.raises(ValueError): + accepted(case) + + +def test_migrated_sentinel_query_requires_explicit_connector_coverage(migrated_solution): + from test_workbook_dependencies import write_query + root = migrated_solution + write_query(root, "SecurityAlert | take 1") + settings = config(root, ["SecurityAlert"]) + settings["workbookDependencies"] = {"Workbooks/nested/fixture.json": { + "requiredWorkloads": [{"workload": "sentinel", "dataTypes": ["SecurityAlert"]}], + "requiredDataConnectors": [], + }} + assert convert(root, settings)["converted"] == 1 + path = authored(root) + baseline = root / "baseline-no-connector.json" + baseline.write_bytes(path.read_bytes()) + with pytest.raises(ValueError, match="connector coverage"): + updates.plan_workbook_update(root, path.relative_to(root).as_posix(), baseline) From 8d456482c30054c45be972531f4d69c4b1f59352 Mon Sep 17 00:00:00 2001 From: kingwil <5839248+kingwil@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:05:33 +0100 Subject: [PATCH 3/3] Separate clean public workbook metadata from local review evidence Preserve legacy proof locally, require fresh hash-bound authoring receipts, and support explicit public Git-ref reproduction without private approvals. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../common/defenderWorkbooks.ps1 | 20 +- Tools/SentinelToXDRMigration/README.md | 198 +++++++---- .../agent-workflows/end-to-end-migration.md | 16 + .../schema/xdr-workbook.schema.json | 43 ++- .../data/xdr-workbook.schema.json | 43 ++- .../_workbook_core/import-manifest.json | 2 +- .../sentinel_xdr_migration/cli.py | 43 +++ .../sentinel_xdr_migration/dependencies.py | 2 +- .../sentinel_xdr_migration/packaging.py | 5 + .../workbook_publication.py | 311 ++++++++++++++++++ .../workbook_updates.py | 40 ++- .../workbook_workflow.py | 8 +- .../sentinel_xdr_migration/workbooks.py | 8 + .../tests/test_workbook_publication.py | 251 ++++++++++++++ .../tests/test_workbook_updates.py | 71 ++-- 15 files changed, 950 insertions(+), 111 deletions(-) create mode 100644 Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_publication.py create mode 100644 Tools/SentinelToXDRMigration/tests/test_workbook_publication.py diff --git a/Tools/Create-Azure-Sentinel-Solution/common/defenderWorkbooks.ps1 b/Tools/Create-Azure-Sentinel-Solution/common/defenderWorkbooks.ps1 index 5cd6386936d..516eb2a8aa0 100644 --- a/Tools/Create-Azure-Sentinel-Solution/common/defenderWorkbooks.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/common/defenderWorkbooks.ps1 @@ -46,7 +46,7 @@ function Assert-DefenderWorkbookReleaseVersion { function Assert-NativeDefenderWorkbookDocument { param([psobject]$Document, [string]$SolutionRoot, - [ValidateSet('native_workbooks', 'workbook_updates')][string]$ValidationModule = 'native_workbooks') + [ValidateSet('native_workbooks', 'workbook_updates', 'workbook_publication')][string]$ValidationModule = 'native_workbooks') $python = if ($env:SENTINEL_XDR_PYTHON) { $env:SENTINEL_XDR_PYTHON } else { 'python' } $start = [System.Diagnostics.ProcessStartInfo]::new() @@ -105,6 +105,19 @@ function Assert-DefenderWorkbookDocument { } Assert-DefenderWorkbookReleaseVersion -Version $metadata.version $provenance = $Document.contentProvenance + if ($null -ne $provenance -and $provenance.format -ceq 'public-workbook-v1') { + Assert-NativeDefenderWorkbookDocument -Document $Document -SolutionRoot $SolutionRoot ` + -ValidationModule workbook_publication + if ($PassThruSource) { return $Document } + return + } + throw "Published authored workbooks must be clean; run workbook-publication-plan/clean for '$ContentPath'." +} + +# Legacy diagnostics are not a publication entry point. +function Assert-LegacyDefenderWorkbookDocument { + param([psobject]$Document, [string]$SolutionRoot, [string]$SolutionName, [string]$ContentPath, [switch]$PassThruSource) + $provenance = $Document.contentProvenance if ($null -ne $provenance -and $null -ne $provenance.PSObject.Properties['authoredReview']) { Assert-NativeDefenderWorkbookDocument -Document $Document -SolutionRoot $SolutionRoot ` -ValidationModule workbook_updates @@ -812,6 +825,11 @@ function Get-DefenderWorkbookParserDependencies { } } } + if ($provenance.format -ceq 'public-workbook-v1') { + # Python validated exact current query aliases and hash-bound local proof + # (or an explicitly requested public Git reproduction) above. + foreach ($alias in $declared) { [void]$referenced.Add($alias) } + } if (-not $declared.SetEquals($referenced)) { throw 'Defender Workbook requiredWorkspaceFunctions must exactly cover reviewed query workspaceFunctions.' } diff --git a/Tools/SentinelToXDRMigration/README.md b/Tools/SentinelToXDRMigration/README.md index 6fec0cc6460..a5167dd17bd 100644 --- a/Tools/SentinelToXDRMigration/README.md +++ b/Tools/SentinelToXDRMigration/README.md @@ -332,43 +332,23 @@ hidden parser inputs, workloads or connector coverage, and offline acceptance does not qualify parser availability or execution in the target tenant. The plan exits nonzero when it contains technical blockers. -Authored provenance is additive: +The importer currently produces a legacy review-bearing staging candidate. +Before publication, run `workbook-publication-plan/clean` below: retain that +complete proof locally and publish only this compact native provenance: ```json { "contentProvenance": { + "format": "public-workbook-v1", "origin": "native-xdr", - "source": { - "path": "Workbook Sources/Native/example.json", - "sha256": "" - }, - "canonicalPayloadSha256": "", - "nativeReview": { - "origin": "native-xdr", - "source": {"path": "Workbook Sources/Native/example.json", "sha256": ""}, - "canonicalPayloadSha256": "", - "displayName": "Example", - "actor": "", - "reference": "", - "requiredWorkloads": [{"workload": "mdo", "dataTypes": ["EmailEvents"]}], - "requiredDataConnectors": [], - "queries": [{ - "path": "/items/0/content/query", - "querySha256": "", - "tables": ["EmailEvents"], - "decision": "accept", - "evidence": "", - "blockers": [] - }] - }, "requiredWorkloads": [{"workload": "mdo", "dataTypes": ["EmailEvents"]}], "requiredDataConnectors": [], - "dependencyReviewRequired": false + "requiredWorkspaceFunctions": [] } } ``` -There is no `conversion` object in native provenance. `xdrMetadata` uses the +There is no public `conversion` or `nativeReview` object. `xdrMetadata` uses the normal schema/version/display name fields. Canonical UUIDv5 assignment is `uuid5(NAMESPACE_URL, "native-xdr-workbook::")`; the instance is the existing @@ -380,20 +360,23 @@ them. A deliberate source edit requires a fresh review and removal of the old local candidate and its `XDR Workbooks` selection before reimport, retaining the same identity for the same path. -Native validation compares the **whole original Notebook** with the authored +Historical native proof validation compares the **whole original Notebook** with the staging payload after removing only the two additive authoring fields. Thus query strings (including CRLF/Unicode), items, notices, tabs and other source keys -remain unchanged. Missing/stale sources or query coverage, duplicate JSON +remain unchanged. Stale source proof or query coverage, duplicate JSON keys/IDs/selections, traversal and symlinks fail closed. Native validation is -integrated with `validate_workbooks`, the existing local static XDR adapter, -and the V3.1 emitter. The PowerShell emitter reuses this Python validator -against its in-memory native document and requires that interpreter's +integrated with `validate_workbooks` and the existing local static XDR adapter. +After cleanup, public structural validation is portable without raw source or +private approval; authoring/package validation requires the current local receipt. +The PowerShell emitter reuses shared Python validation +against its in-memory public document and requires that interpreter's dependencies; a missing interpreter/dependency is an error, never a waiver. `SENTINEL_XDR_PYTHON` selects the interpreter; absent it, `python` is used. The subprocess imports only the emitter's toolkit checkout. For candidate package regression testing, use the correct V3.1 entry point -from the **content repository root**, with a separate toolkit root: +from the **content repository root**, with a separate toolkit root, **after +explicit publication cleanup**: ```powershell Set-Location 'C:\content' @@ -422,16 +405,17 @@ migration/import. These commands do not migrate, deploy, invoke queries, or establish runtime/UI qualification. They require an explicit original reviewed **authored** baseline file, for example extracted with `git show` from the accepted content commit into a review workspace outside deployable content. -An update-reviewed artifact is not an original baseline: subsequent corrections -continue to use the original artifact. +Legacy update-reviewed artifacts are not original conversion baselines. A clean +published artifact can instead be the public baseline for a later explicit review. ```powershell # $Python, $Solution and PYTHONPATH use the same setup as native import above. $Content = 'XDR Workbooks\example.json'.Replace('\', '/') $Baseline = 'C:\review-workspace\baseline-example.json' +New-Item -ItemType Directory -Force "$Solution\Logs" | Out-Null & $Python -m sentinel_xdr_migration.cli workbook-update-plan ` --solution $Solution --content $Content --baseline $Baseline | - Set-Content -LiteralPath '.\update-plan.json' -Encoding utf8 + Set-Content -LiteralPath "$Solution\Logs\update-plan.json" -Encoding utf8 ``` Inspect the exact current Notebook and plan, including all current `queries` @@ -439,52 +423,120 @@ with discovered tables/functions and `changedQueries` (added/removed queries use null previous/current hashes). Explicitly set nonempty `actor` and `reference`, whole-document `decision: "accept"`, and each changed query's `decision: "accept"` and nonempty `evidence`. Do not edit generated hashes, -dependencies or the reverse baseline delta. Save as `update-decisions.json`. +dependencies or the reverse baseline delta. Save under gitignored `Logs` as +`update-decisions.json`; never put review material in authored content or a PR. ```powershell & $Python -m sentinel_xdr_migration.cli workbook-update-review ` --solution $Solution --content $Content --baseline $Baseline ` - --review '.\update-decisions.json' | - Set-Content -LiteralPath '.\update-reviewed.json' -Encoding utf8 + --review "$Solution\Logs\update-decisions.json" | + Set-Content -LiteralPath "$Solution\Logs\update-reviewed.json" -Encoding utf8 & $Python -m sentinel_xdr_migration.cli workbook-update-apply ` --solution $Solution --content $Content --baseline $Baseline ` - --review '.\update-reviewed.json' + --review "$Solution\Logs\update-reviewed.json" & $Python -m sentinel_xdr_migration.cli native-workbook-validate --solution $Solution ``` -Apply preserves the edited payload and metadata/IDs, restores historical -provenance from the independently validated baseline (including any accidentally -hand-edited conversion records), and adds `contentProvenance.authoredReview`. -It refuses identity or prerequisite changes. Existing reviewed workloads, -connectors and hash-bound workspace functions must cover discovered current -dependencies; native updates retain native-only lineage, while migrated -updates also support their existing Sentinel tables and workspace functions. -Unknown/new dependencies and unsupported dynamic/cross-resource inputs fail; -reviewer assertions cannot make them supported. - -Older accepted migrated artifacts can contain nested hybrid table references -that their historical dependency inventory omitted. Their original receipt is -not rewritten: an existing query may retain that exact baseline table set only, -with existing explicit Sentinel connector coverage. The complete current -inventory records those discovered tables. Adding that undeclared lineage to -another query or changing its table set fails. Native-only restrictions remain -unchanged. +Apply preserves the edited Notebook and metadata/IDs, independently validates +the baseline's historical proof, and emits **clean public metadata**, not an +embedded `authoredReview`. It refuses identity/prerequisite changes. Complete +current dependencies are mandatory: native-only lineage, known tables, exact +hash-bound workspace functions, built-in ASIM recognition, and dynamic/cross- +resource rejection remain enforced. Reviewer assertions cannot waive them. + +The complete review, baseline bytes, and available historical source bytes +remain in a hash-bound receipt under `Logs\workbook-reviews`. These are local +authoring evidence, not signatures, reviewer-identity proof, runtime qualification, +or publication metadata. Keep external source/baseline evidence; do not delete +it after apply. No duplicate full source Notebook needs to be shipped. + +### Clean public workbooks and private authoring receipts + +Published authored JSON has only the Notebook, existing `xdrMetadata` identity/ +version, and compact `contentProvenance`: + +* `format: "public-workbook-v1"` and factual `origin`; +* public `requiredWorkloads`, `requiredDataConnectors`, and + `requiredWorkspaceFunctions` (including function source paths/hashes). + +It contains **no author/reviewer accounts, acceptance decisions, references, +session identifiers, query inventories, conversion receipts, reverse deltas, or +proof histories**. Dependencies are not an acceptance claim. The schema rejects +extra fields in public provenance and prerequisite records. Legacy proof schemas +remain readable for importing/validating old evidence, not for publication. +V3.1 rejects legacy authored files until cleaned; migration/import candidates +must pass the explicit publication step before publishing. + +To clean an already reviewed legacy artifact (including old `authoredReview`): + +```powershell +$ReviewDirectory = Join-Path $Solution 'Logs\workbook-reviews' +New-Item -ItemType Directory -Force $ReviewDirectory | Out-Null +$Plan = Join-Path $ReviewDirectory 'publication-plan.json' +$Review = Join-Path $ReviewDirectory 'publication-decisions.json' +& $Python -m sentinel_xdr_migration.cli workbook-publication-plan ` + --solution $Solution --content $Content | + Set-Content -LiteralPath $Plan -Encoding utf8 +# Inspect the exact file and complete planned dependencies. Save the plan as +# $Review with explicit nonempty actor/reference and decision "accept". +# Never automatically approve a plan. Do not edit inputSha256. +$Decision = Get-Content -LiteralPath $Review -Raw | ConvertFrom-Json +& $Python -m sentinel_xdr_migration.cli workbook-publication-clean ` + --solution $Solution --content $Content --review $Review ` + --expected-sha256 $Decision.inputSha256 +& $Python -m sentinel_xdr_migration.cli workbook-publication-validate ` + --solution $Solution --content $Content --mode authoring +``` + +Cleanup verifies the original raw SHA and all historical gates before writing, +stores the original artifact/proof **verbatim locally**, and changes only +`contentProvenance`. Notebook settings, KQL, `Notebook/1.0`, content/instance IDs, +and `xdrMetadata.version` remain exact. It does not bump a release. An existing +XDR 3.2.2 or O365 3.2.1 artifact remains at that release; deployment copies use +the selected solution release as `contentVersion`. + +Historical hybrid receipts sometimes omitted a nested table. Their limited +same-query historical exception remains usable to validate old proof, but is +**not a public packaging waiver**. The plan adds known missing workloads; it +never invents connector coverage. Reviewers must explicitly supply missing +connector/table prerequisites in the local `dependencies` review. Existing +prerequisites/functions cannot be removed/replaced by cleanup. For example, +an old `SecurityAlert`-only connector record does not establish coverage for +`SecurityIncident`; cleanup blocks until that additional prerequisite is +explicitly reviewed. The historical receipt itself is retained unchanged. + +`workbook-publication-validate --mode structural`, the schema, Python +`validate_workbooks`, and the static adapter validate portable public structure/ +current source dependencies **without private approvals**. They do not approve +authoring. `--mode authoring`, `native-workbook-validate`, authoring workflow gates, +and normal Python/PowerShell packaging additionally require a current canonical +whole-document hash-bound local receipt. A later query/settings/metadata/ +dependency edit needs fresh explicit review; structural success is not a waiver. +The clean/update command creates the receipt that packaging consumes automatically +by current document hash. Prior receipts remain locally available, including when +the same payload receives another authorization. + +For an **explicit standalone reproduction**, not authoring acceptance, a clean +public Git checkout can package without private receipts: + +```powershell +$env:SENTINEL_XDR_WORKBOOK_PUBLIC_REF = '' +try { + # Use the standalone V3.1 command below, retaining the published release. +} finally { + Remove-Item Env:\SENTINEL_XDR_WORKBOOK_PUBLIC_REF -ErrorAction SilentlyContinue +} +``` + +This checks the actual in-memory workbook against the exact file at that Git +commit, plus current public table/function/source dependencies. Missing refs or +edited workbooks fail. There is **no automatic HEAD fallback** and it is not +available as an authoring/workflow approval. Git history/review is the trust +boundary for selecting a published ref; reproduction never claims personal +approval. Receipts/Logs are never ZIP inputs: the package contains the generated +template and UI only. Installed Notebook provenance remains the deployment +identity/version quartet; registration additionally retains public dependencies. -The review binds the original file SHA, canonical baseline artifact, complete -current canonical Notebook and authored document, exact query/dependency -inventories, actor/reference, and decisions. Only changed baseline values and -historical metadata are retained as a reverse payload delta. Validation -reconstructs that original artifact and reruns the original deterministic -conversion/native-review gates, unchanged. No duplicate raw Notebook or -invented Sentinel parent is retained, and the external baseline is not needed -after apply. Historical receipts describe the original content, not current -queries. Review hashes detect staleness; they are **not cryptographic signatures** -or evidence of who reviewed content. Git history and review remain the trust -boundary. Re-review using the original baseline after any further authored edit. - -The Python validator, PowerShell assertions/dependency scan, static adapter, -and normal V3.1 packaging honor this explicit update branch. Review material -is authoring-only and is not included in installed Notebook provenance. Keep existing version/release and candidate gates: package through `package-v3-1 --solution $Solution --toolkit-root $Toolkit --version-bump none` (for an already prepared release), then run the official @@ -500,12 +552,14 @@ content-maintenance backend is: & "$Toolkit\Tools\Create-Azure-Sentinel-Solution\V3\createSolutionV3_1.ps1" ` -SolutionDataFolderPath "$Solution\Data" -ToolkitRootPath $Toolkit ` -VersionMode local -VersionBump none ` - -ArmValidationMethod lab-deployment -SkipAttributionLookup + -ArmValidationMethod lab-deployment ``` Use the same official validation-only wrapper afterward. This builds a candidate; it does not establish migration workflow acceptance or lab/runtime qualification. +Keep automatic attribution lookup and its fail-closed guard; do not add a skip +flag to recover from an attribution failure. ### Toolkit-owner managed update policy diff --git a/Tools/SentinelToXDRMigration/agent-workflows/end-to-end-migration.md b/Tools/SentinelToXDRMigration/agent-workflows/end-to-end-migration.md index 18e4aca8398..63811920e2d 100644 --- a/Tools/SentinelToXDRMigration/agent-workflows/end-to-end-migration.md +++ b/Tools/SentinelToXDRMigration/agent-workflows/end-to-end-migration.md @@ -76,6 +76,22 @@ target and independently verified deployment evidence before package acceptance. ### Versioned workbook and candidate ARM lifecycle +Published authored workbook JSON must use the clean `public-workbook-v1` +dependency contract described in the toolkit README. Keep reviewer accounts, +acceptance decisions, references, session context, source/conversion proofs +and query/reverse-delta histories only in gitignored local `Logs` receipts. +Use `workbook-publication-plan/clean` to import legacy proof without deleting it. +Cleaning preserves the actual Notebook, managed IDs and authored content version. +Incomplete historical connector coverage must be explicitly reviewed and completed, +not carried forward as a public exception. + +Portable structural validation of public content does not establish authoring +acceptance. Authoring/apply/package operations require current hash-bound local +review evidence. Explicit standalone reproduction against a selected published +Git commit can use public dependencies without private receipts; it is not a +workflow approval and does not relax any candidate, official-check, environment +authorization or qualification gate. + New CLI runs persist `workbookCapabilityVersion=1` and default to `armValidationMethod=lab-deployment`, in either explicitly selected profile. Authoring may explicitly choose `--arm-validation-method arm-ttk`. Existing diff --git a/Tools/SentinelToXDRMigration/schema/xdr-workbook.schema.json b/Tools/SentinelToXDRMigration/schema/xdr-workbook.schema.json index 5b14e60de67..0d1e735b0c4 100644 --- a/Tools/SentinelToXDRMigration/schema/xdr-workbook.schema.json +++ b/Tools/SentinelToXDRMigration/schema/xdr-workbook.schema.json @@ -20,6 +20,45 @@ "additionalProperties": false }, "contentProvenance": { + "oneOf": [{"$ref": "#/$defs/publicProvenance"}, {"$ref": "#/$defs/legacyProvenance"}] + } + }, + "$defs": { + "publicProvenance": { + "type": "object", + "required": ["format", "origin", "requiredWorkloads", "requiredDataConnectors", "requiredWorkspaceFunctions"], + "properties": { + "format": {"const": "public-workbook-v1"}, + "origin": {"enum": ["native-xdr", "sentinel-migration"]}, + "requiredWorkloads": { + "type": "array", + "items": { + "type": "object", + "required": ["workload", "dataTypes"], + "properties": { + "workload": {"enum": ["mde", "mdo", "mdi", "mda", "mdc", "entraid", "purview", "appgovernance", "sentinel", "mtp"]}, + "dataTypes": {"$ref": "#/$defs/dataTypes"} + }, + "additionalProperties": false + } + }, + "requiredDataConnectors": { + "type": "array", + "items": { + "type": "object", + "required": ["connectorId", "dataTypes"], + "properties": { + "connectorId": {"type": "string", "minLength": 1}, + "dataTypes": {"$ref": "#/$defs/dataTypes"} + }, + "additionalProperties": false + } + }, + "requiredWorkspaceFunctions": {"type": "array", "items": {"$ref": "#/$defs/workspaceFunction"}} + }, + "additionalProperties": false + }, + "legacyProvenance": { "type": "object", "required": ["source", "requiredWorkloads", "requiredDataConnectors", "dependencyReviewRequired"], "allOf": [{ @@ -116,9 +155,7 @@ "dependencyReviewRequired": {"const": false} }, "additionalProperties": false - } - }, - "$defs": { + }, "authoredReview": { "type": "object", "required": ["schemaVersion", "baseline", "baselineMetadata", "reversePayloadChanges", "canonicalPayloadSha256", "canonicalDocumentSha256", "queries", "changedQueries", "actor", "reference", "decision", "reviewSha256"], diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/xdr-workbook.schema.json b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/xdr-workbook.schema.json index 5b14e60de67..0d1e735b0c4 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/xdr-workbook.schema.json +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/xdr-workbook.schema.json @@ -20,6 +20,45 @@ "additionalProperties": false }, "contentProvenance": { + "oneOf": [{"$ref": "#/$defs/publicProvenance"}, {"$ref": "#/$defs/legacyProvenance"}] + } + }, + "$defs": { + "publicProvenance": { + "type": "object", + "required": ["format", "origin", "requiredWorkloads", "requiredDataConnectors", "requiredWorkspaceFunctions"], + "properties": { + "format": {"const": "public-workbook-v1"}, + "origin": {"enum": ["native-xdr", "sentinel-migration"]}, + "requiredWorkloads": { + "type": "array", + "items": { + "type": "object", + "required": ["workload", "dataTypes"], + "properties": { + "workload": {"enum": ["mde", "mdo", "mdi", "mda", "mdc", "entraid", "purview", "appgovernance", "sentinel", "mtp"]}, + "dataTypes": {"$ref": "#/$defs/dataTypes"} + }, + "additionalProperties": false + } + }, + "requiredDataConnectors": { + "type": "array", + "items": { + "type": "object", + "required": ["connectorId", "dataTypes"], + "properties": { + "connectorId": {"type": "string", "minLength": 1}, + "dataTypes": {"$ref": "#/$defs/dataTypes"} + }, + "additionalProperties": false + } + }, + "requiredWorkspaceFunctions": {"type": "array", "items": {"$ref": "#/$defs/workspaceFunction"}} + }, + "additionalProperties": false + }, + "legacyProvenance": { "type": "object", "required": ["source", "requiredWorkloads", "requiredDataConnectors", "dependencyReviewRequired"], "allOf": [{ @@ -116,9 +155,7 @@ "dependencyReviewRequired": {"const": false} }, "additionalProperties": false - } - }, - "$defs": { + }, "authoredReview": { "type": "object", "required": ["schemaVersion", "baseline", "baselineMetadata", "reversePayloadChanges", "canonicalPayloadSha256", "canonicalDocumentSha256", "queries", "changedQueries", "actor", "reference", "decision", "reviewSha256"], diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/import-manifest.json b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/import-manifest.json index ca1cdf7c41c..7ccab427daa 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/import-manifest.json +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/import-manifest.json @@ -32,7 +32,7 @@ }, { "component": "data/xdr-workbook.schema.json", - "sha256": "f322e55d72e98aba7606174961968855d9c0a83b8a1b0078d12f20c02765394a" + "sha256": "6de557c34ad020d35294db9d2f1cf7cc6753a09cb676acd88dc11b42f5961dca" }, { "component": "dataframes.py", diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/cli.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/cli.py index ed728a41e2a..694a2a05249 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/cli.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/cli.py @@ -177,6 +177,15 @@ def main(argv: list[str] | None = None) -> int: description="Convert Microsoft Sentinel analytic rules into XDR Custom Detection YAML." ) subparsers = parser.add_subparsers(dest="command", required=True) + for command in ("workbook-publication-plan", "workbook-publication-clean", "workbook-publication-validate"): + publication = subparsers.add_parser(command, help="Clean public workbook; retain local authoring evidence.") + publication.add_argument("--solution", required=True) + publication.add_argument("--content", required=True) + if command.endswith("-clean"): + publication.add_argument("--expected-sha256", required=True) + publication.add_argument("--review", required=True) + if command.endswith("-validate"): + publication.add_argument("--mode", choices=("structural", "authoring"), required=True) for update_command in ("workbook-update-plan", "workbook-update-review", "workbook-update-apply"): update_parser = subparsers.add_parser( update_command, help="Offline explicit post-import edits; preserve historical receipts.") @@ -1189,6 +1198,7 @@ def main(argv: list[str] | None = None) -> int: selection = None try: isolated_permission_commands = { + "workbook-publication-plan", "workbook-publication-clean", "workbook-publication-validate", "workbook-update-plan", "workbook-update-review", "workbook-update-apply", @@ -1565,6 +1575,27 @@ def main(argv: list[str] | None = None) -> int: result = plan_solution_dependencies( args.solution, results_path=args.results, workspace_id=args.workspace_id, ) + elif args.command.startswith("workbook-publication-"): + from .workbook_publication import plan_publication, clean_publication, publication_errors, is_public + from .workbook_updates import _read_external + if args.command.endswith("-plan"): + result = plan_publication(args.solution, args.content) + elif args.command.endswith("-validate"): + from .workbooks import _root, _path, _document, validate_workbooks + root = _root(args.solution) + path = _path(root, args.content) + document = _document(path) + validation = validate_workbooks(root, selected_only=True, _documents=[(path, document)]) + errors = [error for row in validation["results"] for error in row["errors"]] + if not is_public(document): + errors.append("Public workbook validation requires clean publication metadata.") + if not errors and args.mode == "authoring": + errors = publication_errors(root, document) + result = {"valid": not errors, "validationScope": args.mode, + "errors": errors, "runtimeQualified": False} + else: + _, review = _read_external(args.review) + result = clean_publication(args.solution, args.content, args.expected_sha256, review) elif args.command.startswith("workbook-update-"): from .workbook_updates import ( plan_workbook_update, review_workbook_update, apply_workbook_update, _read_external, @@ -1592,6 +1623,16 @@ def main(argv: list[str] | None = None) -> int: result = operation(args.solution, args.source, load_json(review_path.read_bytes())) else: result = validate_workbooks(args.solution, selected_only=True) + from .workbook_publication import is_public, publication_errors + from .workbooks import _root, _path, _document + root = _root(args.solution) + for row in result["results"]: + document = _document(_path(root, row["path"])) + if is_public(document): + row["errors"].extend(publication_errors(root, document)) + row["valid"] = not row["errors"] + result["valid"] = sum(row["valid"] for row in result["results"]) + result["invalid"] = result["total"] - result["valid"] elif args.command == "workbook-dependency-plan": from .workbook_dependencies import plan_workbook_dependencies result = plan_workbook_dependencies( @@ -1989,6 +2030,8 @@ def main(argv: list[str] | None = None) -> int: return 1 if args.command == "package-v3-1" and result.get("status") == "blocked": return 1 + if args.command == "workbook-publication-validate" and not result["valid"]: + return 1 if args.command in {"workflow-arm-execute", "workflow-workbook-contract-import"} and result.get("accepted") is not True: return 1 if args.command == "workflow-arm-preflight-execute" and result.get("status") != "ready": diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/dependencies.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/dependencies.py index dbea438e543..0736b5d57d4 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/dependencies.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/dependencies.py @@ -38,7 +38,7 @@ def telemetry_data_types(tables): **{table: "mtp" for table in ("AlertInfo", "AlertEvidence", "IdentityInfo")}, "CloudAppEvents": "mda", **{table: "sentinel" for table in ( - "Syslog", "SecurityAlert", "SecurityEvent", "WindowsEvent", "Event", + "Syslog", "SecurityAlert", "SecurityIncident", "SecurityEvent", "WindowsEvent", "Event", "AzureActivity", "CommonSecurityLog", "SigninLogs", "AuditLogs", "OfficeActivity", "AWSCloudTrail", "AWSGuardDuty", )}, diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/packaging.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/packaging.py index 349b1078da6..27618b59683 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/packaging.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/packaging.py @@ -1022,6 +1022,11 @@ def package_solution_v3_1( workbook_validation = validate_workbooks(root, selected_only=True) if workbook_validation["invalid"] or workbook_validation["valid"] != len(workbook_files): raise ValueError("selected Defender workbooks have unresolved structural or review blockers") + from .workbook_publication import publication_errors + for workbook_file in workbook_files: + blockers = publication_errors(root, _read_json(workbook_file)) + if blockers: + raise ValueError("; ".join(blockers)) script = ( selected_toolkit_root / "Tools" diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_publication.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_publication.py new file mode 100644 index 00000000000..293c7fa58b4 --- /dev/null +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_publication.py @@ -0,0 +1,311 @@ +"""Public workbook dependencies, with private, hash-bound authoring receipts.""" +from __future__ import annotations + +from copy import deepcopy +import json +import os +from pathlib import Path +import subprocess +import sys + +from ._workbook_core import workbook +from ._workbook_core import mapping_catalog +from .dependencies import TABLE_WORKLOADS, dependency_validation_errors +from .native_workbooks import _query +from .workbook_functions import validate_functions +from .workbooks import _root, _path, _document, _load_json, _sha, _write + +FORMAT = "public-workbook-v1" +DEPENDENCIES = ("requiredWorkloads", "requiredDataConnectors", "requiredWorkspaceFunctions") + + +def digest(value): + return _sha(json.dumps(value, sort_keys=True, separators=(",", ":"), + ensure_ascii=False, allow_nan=False).encode("utf-8")) + + +def is_public(document): + return document.get("contentProvenance", {}).get("format") == FORMAT + + +def public_document(document, dependencies=None): + result = deepcopy(document) + old = document["contentProvenance"] + dependencies = old if dependencies is None else dependencies + result["contentProvenance"] = { + "format": FORMAT, "origin": old.get("origin", "sentinel-migration"), + **{key: deepcopy(dependencies.get(key, [])) for key in DEPENDENCIES}, + } + return result + + +def validate_public_document(solution, document): + """Structure and current source/function dependencies, never acceptance.""" + root = _root(solution) + errors = dependency_validation_errors(document) + provenance = document["contentProvenance"] + try: + records = validate_functions(root, provenance["requiredWorkspaceFunctions"]) + aliases = {record["functionAlias"] for record in records} + declared = {table for row in provenance["requiredWorkloads"] for table in row["dataTypes"]} + sentinel = {table for row in provenance["requiredWorkloads"] + if row["workload"] == "sentinel" for table in row["dataTypes"]} + connectors = {table for row in provenance["requiredDataConnectors"] for table in row["dataTypes"]} + used = set() + known = set(TABLE_WORKLOADS) | set(mapping_catalog.load_mapping_catalog()["tables"]) + if not declared <= known: + errors.append("Unknown public dependency tables: " + ", ".join(sorted(declared - known))) + if not connectors <= declared: + errors.append("Public connector tables must be covered by workload prerequisites.") + native = provenance["origin"] == "native-xdr" + if native and (aliases or connectors or sentinel): + errors.append("Native public workbooks cannot declare Sentinel/function prerequisites.") + for surface in workbook.query_surfaces(document): + lineage = _query(surface, allowed_tables=declared, allowed_functions=aliases, + allow_sentinel=not native, include_lineage=True) + tables = set(lineage["tables"]) + if not tables <= known: + errors.append("Unknown public query tables: " + ", ".join(sorted(tables - known))) + if not tables <= declared: + errors.append("Current query tables are outside public dependencies: " + + ", ".join(sorted(tables - declared))) + if not (tables & sentinel) <= connectors: + errors.append("Current Sentinel query tables require explicit connector coverage: " + + ", ".join(sorted((tables & sentinel) - connectors))) + for table in tables: + expected = TABLE_WORKLOADS.get(table) + if expected and not any(row["workload"] == expected and table in row["dataTypes"] + for row in provenance["requiredWorkloads"]): + errors.append("Incorrect public workload for table: " + table) + used.update(lineage["workspaceFunctions"]) + if used != aliases: + errors.append("Public workspace functions must exactly cover current query aliases.") + except (OSError, ValueError, TypeError, KeyError) as exc: + errors.append(str(exc)) + return errors + + +def _receipt_path(root, document): + path = _path(root, "Logs/workbook-reviews/" + digest(document) + ".json") + # Logs is excluded from publication. Refuse tracked or non-ignored evidence. + probe = subprocess.run(["git", "-C", str(root), "rev-parse", "--show-toplevel"], + capture_output=True, text=True) + if probe.returncode == 0: + tracked = subprocess.run(["git", "-C", str(root), "ls-files", "--", str(path)], + capture_output=True, text=True, check=True) + ignored = subprocess.run(["git", "-C", str(root), "check-ignore", "--quiet", str(path)], + capture_output=True) + if tracked.stdout.strip() or ignored.returncode: + raise ValueError("Local workbook receipts must be untracked and gitignored under Logs.") + return path + + +def retain_receipt(solution, document, evidence): + root = _root(solution) + receipt = {"format": FORMAT, "canonicalDocumentSha256": digest(document), + "evidence": evidence} + receipt["receiptSha256"] = digest(receipt) + path = _receipt_path(root, document) + raw = (json.dumps(receipt, indent=2, ensure_ascii=False) + "\n").encode("utf-8") + if path.exists(): + if path.read_bytes() != raw: + old = path.read_bytes() + archive = path.with_name(path.stem + "-" + _sha(old) + ".json") + _write(archive, old, archive.read_bytes() if archive.exists() else None) + path.parent.mkdir(parents=True, exist_ok=True) + _write(path, raw, path.read_bytes() if path.exists() else None) + return str(path) + + +def source_evidence(root, document): + source = document["contentProvenance"].get("source") + if not source: + return {} + path = _path(root, source["path"]) + if not path.exists(): + return {} + raw = path.read_bytes() + if _sha(raw) != source["sha256"]: + raise ValueError("Historical source changed before retaining local proof.") + return {source["path"]: {"sha256": _sha(raw), "artifact": raw.decode("utf-8")}} + + +def _review_binding_errors(document, evidence): + review = evidence["review"] + original = json.loads(evidence["inputArtifact"]) + if "dependencies" in review: + if review["inputSha256"] != evidence["inputSha256"]: + return ["Publication authorization has a stale original input hash."] + expected = public_document(original, review["dependencies"]) + else: + from .workbook_updates import _canonical, _review_digest + from .native_workbooks import _canonical_payload_sha256 + baseline = json.loads(evidence["baselineArtifact"]) + if (review["reviewSha256"] != _review_digest(review) + or review["baseline"]["sha256"] != evidence["baselineSha256"] + or review["baseline"]["canonicalDocumentSha256"] != _canonical(baseline)): + return ["Local update review or historical baseline binding is stale."] + original["contentProvenance"] = deepcopy(baseline["contentProvenance"]) + if (review["canonicalDocumentSha256"] != _canonical(original) + or review["canonicalPayloadSha256"] != _canonical_payload_sha256(original)): + return ["Local update authorization does not bind the current authored payload."] + expected = public_document(original) + if digest(expected) != digest(document): + return ["Local authorization is stale for the current public artifact/dependencies."] + return [] + + +def publication_errors(solution, document, *, public_ref=None): + """Authoring requires a local receipt; explicit reproduction is not review.""" + root = _root(solution) + if not is_public(document): + return ["Published authored workbooks must be clean; run workbook-publication-plan/clean."] + from .workbooks import validate_workbooks + validation = validate_workbooks(root, selected_only=True, _documents=[ + (root / "XDR Workbooks" / "publication.json", document)]) + errors = [error for row in validation["results"] for error in row["errors"]] + if errors: + return errors + if public_ref is not None: + try: + from .workbooks import selected_workbook_files + commit = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--verify", "--end-of-options", + public_ref + "^{commit}"], capture_output=True, text=True, check=True).stdout.strip() + top = Path(subprocess.run(["git", "-C", str(root), "rev-parse", "--show-toplevel"], + capture_output=True, text=True, check=True).stdout.strip()) + matches = [] + for path in selected_workbook_files(root): + if digest(_document(path)) == digest(document): + relative = path.relative_to(top).as_posix() + raw = subprocess.run(["git", "-C", str(top), "show", commit + ":" + relative], + capture_output=True, check=True).stdout + matches.append(digest(json.loads(raw)) == digest(document)) + if matches == [True]: + return [] + return ["Reproduction requires the exact public workbook at the explicit Git ref."] + except (OSError, ValueError, subprocess.CalledProcessError) as exc: + return ["Cannot verify public reproduction ref: " + str(exc)] + path = _receipt_path(root, document) + try: + receipt = json.loads(path.read_bytes()) + claimed = receipt.pop("receiptSha256") + if claimed != digest(receipt) or receipt["canonicalDocumentSha256"] != digest(document): + return ["Local workbook receipt is stale."] + evidence = receipt["evidence"] + if digest(evidence["publicDocument"]) != digest(document): + return ["Local workbook receipt does not bind the exact current document."] + review = evidence["review"] + if (review["decision"] != "accept" or not review["actor"].strip() + or not review["reference"].strip()): + return ["Local authoring receipt requires explicit authorization."] + if evidence["inputSha256"] != _sha(evidence["inputArtifact"].encode("utf-8")): + return ["Local historical source/proof hash is stale."] + if "baselineArtifact" in evidence and evidence["baselineSha256"] != _sha( + evidence["baselineArtifact"].encode("utf-8")): + return ["Local historical baseline hash is stale."] + for record in evidence.get("sourceArtifacts", {}).values(): + if record["sha256"] != _sha(record["artifact"].encode("utf-8")): + return ["Local historical raw source hash is stale."] + return _review_binding_errors(document, evidence) + except (OSError, ValueError, TypeError, KeyError, AttributeError) as exc: + return ["Current hash-bound local authoring receipt required; run an explicit review: " + str(exc)] + + +def plan_publication(solution, content, *, _raw=None): + root = _root(solution) + path = _path(root, content) + if not content.startswith("XDR Workbooks/") or path.suffix.lower() != ".json": + raise ValueError("Publication content must be XDR Workbooks JSON.") + raw = path.read_bytes() if _raw is None else _raw + document = _load_json(raw) + if (not isinstance(document, dict) or document.get("version") != "Notebook/1.0" + or not isinstance(document.get("items"), list)): + raise ValueError("Workbook must be a complete Notebook/1.0 object.") + from .workbooks import validate_workbooks + validation = validate_workbooks(root, selected_only=True, _documents=[(path, document)]) + if validation["invalid"]: + raise ValueError("Input proof/structure failed: " + "; ".join(validation["results"][0]["errors"])) + dependencies = {key: deepcopy(document["contentProvenance"].get(key, [])) for key in DEPENDENCIES} + # Expose missing current table prerequisites, never invent connector coverage. + tables = set() + provenance = document["contentProvenance"] + if "authoredReview" in provenance: + tables = {table for query in provenance["authoredReview"]["queries"] for table in query["tables"]} + else: + for surface in workbook.query_surfaces(document): + aliases = {row["functionAlias"] for row in dependencies["requiredWorkspaceFunctions"]} + declared = {table for row in dependencies["requiredWorkloads"] for table in row["dataTypes"]} + tables.update(_query(surface, allowed_tables=declared, allowed_functions=aliases, + allow_sentinel=provenance.get("origin") != "native-xdr")) + declared = {table for row in dependencies["requiredWorkloads"] for table in row["dataTypes"]} + for table in sorted(tables - declared): + workload = TABLE_WORKLOADS.get(table) + if workload is None: + raise ValueError("Explicit workload mapping required for current table: " + table) + row = next((row for row in dependencies["requiredWorkloads"] if row["workload"] == workload), None) + if row is None: + row = {"workload": workload, "dataTypes": []} + dependencies["requiredWorkloads"].append(row) + row["dataTypes"].append(table) + return {"inputSha256": _sha(raw), "dependencies": dependencies, + "dependencyErrors": sorted(set(validate_public_document( + root, public_document(document, dependencies)))), + "actor": "", "reference": "", "decision": None} + + +def clean_publication(solution, content, expected_sha256, review): + root = _root(solution) + path = _path(root, content) + raw = path.read_bytes() + expected = plan_publication(root, content, _raw=raw) + if not isinstance(review, dict): + raise ValueError("Publication review must be an object.") + if expected_sha256 != _sha(raw) or review.get("inputSha256") != expected_sha256: + raise ValueError("Original workbook hash changed; fresh publication review required.") + if set(review) != set(expected): + raise ValueError("Publication review requires exactly the plan fields.") + if review["dependencyErrors"] != expected["dependencyErrors"]: + raise ValueError("Publication review diagnostics differ from the current plan.") + if review["decision"] != "accept" or any( + not isinstance(review[field], str) or not review[field].strip() + for field in ("actor", "reference")): + raise ValueError("Publication cleanup requires explicit local actor/reference/accept authorization.") + dependencies = review["dependencies"] + if not isinstance(dependencies, dict) or set(dependencies) != set(DEPENDENCIES): + raise ValueError("Publication dependency fields must match the plan.") + original = _load_json(raw) + document = public_document(original, dependencies) + from .workbooks import authored_schema_errors + errors = authored_schema_errors(document) + validate_public_document(root, document) + if errors: + raise ValueError("; ".join(errors)) + if dependencies["requiredWorkspaceFunctions"] != expected["dependencies"]["requiredWorkspaceFunctions"]: + raise ValueError("Cleanup cannot change workspace function dependencies.") + for key, identity in (("requiredWorkloads", "workload"), ("requiredDataConnectors", "connectorId")): + for row in expected["dependencies"][key]: + if not any(other[identity] == row[identity] and set(row["dataTypes"]) <= set(other["dataTypes"]) + for other in dependencies[key]): + raise ValueError("Cleanup cannot remove existing/discovered dependencies.") + receipt = retain_receipt(root, document, { + "publicDocument": document, "inputSha256": _sha(raw), + "inputArtifact": raw.decode("utf-8"), "review": review, + "sourceArtifacts": source_evidence(root, original), + }) + _write(path, (json.dumps(document, indent=2, ensure_ascii=False) + "\n").encode("utf-8"), raw) + return {"path": content, "sha256": _sha(path.read_bytes()), "localReceipt": receipt, + "runtimeQualified": False} + + +def main(): + if not (3, 11) <= sys.version_info[:2] < (3, 13): + raise ValueError("Workbook publication requires Python 3.11 or 3.12.") + document = json.loads(sys.stdin.read()) + errors = publication_errors(sys.argv[1], document, + public_ref=os.environ.get("SENTINEL_XDR_WORKBOOK_PUBLIC_REF")) + print(json.dumps({"errors": errors})) + return int(bool(errors)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_updates.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_updates.py index 369abf9e4ae..5f2a95a5cde 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_updates.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_updates.py @@ -145,7 +145,11 @@ def _plan(root, current, baseline, baseline_record): raise ValueError("Workbook identity must remain unchanged.") for field in ("requiredWorkloads", "requiredDataConnectors", "requiredWorkspaceFunctions", "dependencyReviewRequired"): - if current["contentProvenance"].get(field) != historical.get(field): + default = [] if field == "requiredWorkspaceFunctions" else None + if (field == "dependencyReviewRequired" + and current["contentProvenance"].get("format") == "public-workbook-v1"): + continue + if current["contentProvenance"].get(field, default) != historical.get(field, default): raise ValueError("Updates cannot replace historical reviewed dependencies: " + field) restored = deepcopy(current) restored["contentProvenance"] = deepcopy(historical) @@ -230,19 +234,39 @@ def apply_workbook_update(solution, content, baseline_file, review): path = _path(root, content) previous = path.read_bytes() review = review_workbook_update(root, content, baseline_file, review) - _, baseline = _read_external(baseline_file) - document = _document(path) + baseline_raw, baseline = _read_external(baseline_file) + if _sha(baseline_raw) != review["baseline"]["sha256"]: + raise ValueError("Historical baseline changed before apply.") + document = _load_json(previous) document["contentProvenance"] = deepcopy(baseline["contentProvenance"]) - document["contentProvenance"]["authoredReview"] = review - errors = validate_updated_document(root, document) + if _canonical(document) != review["canonicalDocumentSha256"]: + raise ValueError("Current workbook changed before apply.") + from .workbook_publication import ( + is_public, public_document, retain_receipt, validate_public_document, source_evidence, + ) + if is_public(baseline): + clean = public_document(document) + errors = validate_public_document(root, clean) + else: + document["contentProvenance"]["authoredReview"] = review + errors = validate_updated_document(root, document) + clean = public_document(document) + errors.extend(validate_public_document(root, clean)) if errors: raise ValueError("; ".join(errors)) from .workbooks import authored_schema_errors - errors = authored_schema_errors(document) + errors = authored_schema_errors(clean) if errors: raise ValueError("; ".join(errors)) - _write(path, (json.dumps(document, indent=2, ensure_ascii=False) + "\n").encode(), previous) - return {"path": content, "sha256": _sha(path.read_bytes()), "runtimeQualified": False} + receipt = retain_receipt(root, clean, { + "publicDocument": clean, "inputSha256": _sha(previous), + "inputArtifact": previous.decode("utf-8"), "review": review, + "baselineArtifact": baseline_raw.decode("utf-8"), "baselineSha256": _sha(baseline_raw), + "sourceArtifacts": source_evidence(root, baseline), + }) + _write(path, (json.dumps(clean, indent=2, ensure_ascii=False) + "\n").encode(), previous) + return {"path": content, "sha256": _sha(path.read_bytes()), "localReceipt": receipt, + "runtimeQualified": False} def validate_updated_document(solution, document): diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_workflow.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_workflow.py index 096feb9fc65..83bae6729c7 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_workflow.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbook_workflow.py @@ -273,6 +273,13 @@ def validate_evidence(solution: str | Path, state: dict[str, Any]) -> dict[str, from .workbooks import validate_workbooks result = validate_workbooks(root, selected_only=True) + from .workbook_publication import is_public, publication_errors + for path in selected_workbook_references(root): + document = json.loads(path.read_bytes()) + if is_public(document): + blockers = publication_errors(root, document) + if blockers: + raise ValueError("; ".join(blockers)) if ( result.get("total") != len(selected) or result.get("valid") != len(selected) @@ -1472,4 +1479,3 @@ def audit_private_config_and_support_export( "exposures": exposed_in_exports, }, } - diff --git a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbooks.py b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbooks.py index b33ec26f225..c0dfac5d006 100644 --- a/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbooks.py +++ b/Tools/SentinelToXDRMigration/sentinel_xdr_migration/workbooks.py @@ -468,6 +468,14 @@ def validate_workbooks(solution, *, selected_only=False, _documents=None) -> dic errors.append("Workbook displayName is required.") if any(key in provenance for key in ("packageId", "packageVersion", "contentId", "contentVersion")): errors.append("Deployment provenance quartet must not be authored.") + from .workbook_publication import is_public, validate_public_document + if is_public(document): + errors.extend(validate_public_document(root, document)) + source_paths.update(row["path"] for row in inventory.values() + if row["identity"] == identity) + results.append({"path": path.relative_to(root).as_posix(), + "sha256": digest, "valid": not errors, "errors": errors}) + continue if "authoredReview" in provenance or provenance.get("origin") == "native-xdr": if "authoredReview" in provenance: from .workbook_updates import validate_updated_document diff --git a/Tools/SentinelToXDRMigration/tests/test_workbook_publication.py b/Tools/SentinelToXDRMigration/tests/test_workbook_publication.py new file mode 100644 index 00000000000..249cdefda18 --- /dev/null +++ b/Tools/SentinelToXDRMigration/tests/test_workbook_publication.py @@ -0,0 +1,251 @@ +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import zipfile + +import pytest + +from sentinel_xdr_migration import cli, workbook_publication as public, workbooks +from test_workbook_updates import ( + update_case, native_solution, migrated_solution, accepted, apply, updates, powershell_scan, +) + + +def legacy(case): + root, path, baseline, _, _ = case + document = json.loads(path.read_bytes()) + document["contentProvenance"]["authoredReview"] = updates.review_workbook_update( + root, path.relative_to(root).as_posix(), baseline, accepted(case)) + path.write_text(json.dumps(document), encoding="utf-8") + return path.read_bytes() + + +def cleanup_review(root, path): + review = public.plan_publication(root, path.relative_to(root).as_posix()) + review.update(actor="private-reviewer-account", reference="private session authorization", + decision="accept") + return review + + +def test_legacy_cleanup_retains_exact_private_proof_not_public_history(update_case): + root, path, _, _, _ = update_case + original = legacy(update_case) + before = json.loads(original) + review = cleanup_review(root, path) + result = public.clean_publication(root, path.relative_to(root).as_posix(), + public._sha(original), review) + clean = json.loads(path.read_bytes()) + assert {k: v for k, v in clean.items() if k != "contentProvenance"} == { + k: v for k, v in before.items() if k != "contentProvenance"} + assert set(clean["contentProvenance"]) == { + "format", "origin", *public.DEPENDENCIES} + serialized = json.dumps(clean) + for forbidden in ("actor", "reference", "decision", "queries", "reversePayloadChanges", + "authoredReview", "nativeReview", "conversion", "private-reviewer-account", + "private session authorization", "kingwil"): + assert forbidden not in serialized + receipt_path = Path(result["localReceipt"]) + assert receipt_path.is_relative_to(root / "Logs") + receipt = json.loads(receipt_path.read_bytes()) + assert receipt["evidence"]["inputArtifact"].encode("utf-8") == original + assert receipt["evidence"]["review"] == review + assert receipt["evidence"]["sourceArtifacts"] + assert public.publication_errors(root, clean) == [] + receipt_path.unlink() + for folder in ("Workbook Sources", "Workbooks"): + if (root / folder).exists(): + shutil.rmtree(root / folder) + # Structure is portable; private acceptance is neither claimed nor inferred. + assert workbooks.validate_workbooks(root, selected_only=True)["invalid"] == 0 + assert public.publication_errors(root, clean) + + +@pytest.mark.parametrize("field", ["actor", "reference", "decision", "queries", "conversion", "authoredReview"]) +def test_public_metadata_rejects_reembedded_review_history(update_case, field): + root, path, _, _, _ = update_case + apply(update_case) + document = json.loads(path.read_bytes()) + document["contentProvenance"][field] = "not public metadata" + assert workbooks.authored_schema_errors(document) + assert public.publication_errors(root, document) + + +@pytest.mark.parametrize("mutation", ["hash", "actor", "decision", "current-payload"]) +def test_cleanup_is_hash_bound_and_never_changes_input_on_failure(update_case, mutation): + root, path, _, index, _ = update_case + original = legacy(update_case) + review = cleanup_review(root, path) + if mutation == "hash": + review["inputSha256"] = "0" * 64 + elif mutation == "actor": + review["actor"] = " " + elif mutation == "decision": + review["decision"] = None + else: + document = json.loads(original) + document["items"][index]["content"]["query"] += " | take 10" + path.write_text(json.dumps(document), encoding="utf-8") + unchanged = path.read_bytes() + with pytest.raises(ValueError): + public.clean_publication(root, path.relative_to(root).as_posix(), + public._sha(original), review) + assert path.read_bytes() == unchanged + + +def test_public_baseline_needs_fresh_explicit_review_for_later_edits(update_case): + root, path, baseline, index, _ = update_case + apply(update_case) + baseline.write_bytes(path.read_bytes()) + document = json.loads(path.read_bytes()) + document["items"][index]["content"]["query"] += " | take 9" + path.write_text(json.dumps(document), encoding="utf-8") + assert workbooks.validate_workbooks(root, selected_only=True)["invalid"] == 0 + assert public.publication_errors(root, document) + apply(update_case) + assert public.publication_errors(root, json.loads(path.read_bytes())) == [] + + +@pytest.mark.parametrize("operation", ["update", "cleanup"]) +def test_rekeyed_receipt_cannot_reuse_old_authorization_for_new_settings(update_case, operation): + root, path, _, _, _ = update_case + if operation == "update": + applied = apply(update_case) + else: + legacy(update_case) + review = cleanup_review(root, path) + applied = public.clean_publication(root, path.relative_to(root).as_posix(), + review["inputSha256"], review) + document = json.loads(path.read_bytes()) + receipt = json.loads(Path(applied["localReceipt"]).read_bytes()) + document["isLocked"] = not document["isLocked"] + receipt["canonicalDocumentSha256"] = public.digest(document) + receipt["evidence"]["publicDocument"] = document + receipt.pop("receiptSha256") + receipt["receiptSha256"] = public.digest(receipt) + public._receipt_path(root, document).write_text(json.dumps(receipt), encoding="utf-8") + path.write_text(json.dumps(document), encoding="utf-8") + assert workbooks.validate_workbooks(root, selected_only=True)["invalid"] == 0 + assert public.publication_errors(root, document) + + +def test_reauthorization_archives_previous_local_receipt_without_shipping_it(update_case): + root, path, _, _, _ = update_case + first = apply(update_case) + receipt_path = Path(first["localReceipt"]) + original = receipt_path.read_bytes() + review = cleanup_review(root, path) + public.clean_publication(root, path.relative_to(root).as_posix(), review["inputSha256"], review) + assert receipt_path.read_bytes() != original + assert any(candidate.read_bytes() == original for candidate in receipt_path.parent.glob("*.json")) + assert public.publication_errors(root, json.loads(path.read_bytes())) == [] + + +def test_reproduction_is_explicit_git_ref_bound_and_not_local_authoring_acceptance(update_case, request, monkeypatch): + root, path, _, index, _ = update_case + result = apply(update_case) + document = json.loads(path.read_bytes()) + receipt_path = Path(result["localReceipt"]) + receipt_path.unlink() + for folder in ("Workbook Sources", "Workbooks"): + if (root / folder).exists(): + shutil.rmtree(root / folder) + subprocess.run(["git", "init", "--quiet", str(root)], check=True) + def writable_git_objects(): + for item in (root / ".git").rglob("*"): + if item.is_file(): + item.chmod(0o700) + request.addfinalizer(writable_git_objects) + (root / ".gitignore").write_text("Logs/\n", encoding="utf-8") + subprocess.run(["git", "-C", str(root), "add", "--", ".gitignore", + path.relative_to(root).as_posix(), "Data"], check=True) + subprocess.run(["git", "-C", str(root), "-c", "user.name=Fixture", + "-c", "user.email=fixture@example.invalid", "commit", "--quiet", "-m", "Public baseline"], + check=True) + assert public.publication_errors(root, document) + assert public.publication_errors(root, document, public_ref="HEAD") == [] + assert public.publication_errors(root, document, public_ref="missing-ref") + monkeypatch.setenv("SENTINEL_XDR_WORKBOOK_PUBLIC_REF", "HEAD") + if shutil.which("pwsh"): + run = powershell_scan(root, path) + assert run.returncode == 0, run.stdout + run.stderr + assert public.publication_errors(root, document) + document["items"][index]["content"]["query"] += " | take 11" + path.write_text(json.dumps(document), encoding="utf-8") + assert public.publication_errors(root, document, public_ref="HEAD") + if shutil.which("pwsh"): + assert powershell_scan(root, path).returncode != 0 + + +@pytest.mark.skipif(not shutil.which("pwsh"), reason="PowerShell 7 required") +def test_package_resource_and_archive_contain_only_public_dependencies(update_case): + root, path, _, _, _ = update_case + result = apply(update_case) + document = json.loads(path.read_bytes()) + # Exercise the actual registration/installation constructors, not a live build. + script = Path(__file__).resolve().parents[3] / "Tools" / "Create-Azure-Sentinel-Solution" / "common" / "commonFunctions.ps1" + def literal(value): + return "'" + str(value).replace("'", "''") + "'" + command = ( + "$ErrorActionPreference='Stop'; . " + literal(script) + "; " + "$doc=Get-Content -LiteralPath " + literal(path) + " -Raw | ConvertFrom-Json; " + "Assert-DefenderWorkbookDocument -Document $doc -SolutionRoot " + literal(root) + + " -SolutionName " + literal(root.name) + " -ContentPath " + literal(path.relative_to(root).as_posix()) + "; " + "$resolved=[pscustomobject]@{packageId='fixture';packageVersion=$doc.xdrMetadata.version;" + "contentId=$doc.xdrMetadata.contentId;contentVersion=$doc.xdrMetadata.version}; " + "$notebook=$doc | ConvertTo-Json -Depth 100 | ConvertFrom-Json; " + "$notebook.PSObject.Properties.Remove('xdrMetadata');$notebook.contentProvenance=$resolved; " + "$resource=[pscustomobject]@{import='MicrosoftSecurity';type='Microsoft.Security/workbooks@2026-06-01-preview';" + "properties=[pscustomobject]@{id=$doc.xdrMetadata.instanceId;content=$notebook}}; " + "$install=New-DefenderWorkbookInstallDeployment -DeploymentName fixture -WorkbookResource $resource -ParserDependencies @(); " + "$registration=New-DefenderWorkbookRegistration -DeploymentName fixture -InstallDeployment $install " + "-Document $doc -ResolvedProvenance $resolved -PackageName fixture; " + "[pscustomobject]@{resources=@($install,$registration)} | ConvertTo-Json -Depth 100 -Compress" + ) + run = subprocess.run(["pwsh", "-NoLogo", "-NoProfile", "-NonInteractive", "-Command", command], + capture_output=True, text=True, encoding="utf-8", + env={**os.environ, "SENTINEL_XDR_PYTHON": sys.executable}) + assert run.returncode == 0, run.stdout + run.stderr + package = json.loads(run.stdout) + archive = root / "fixture-package.zip" + with zipfile.ZipFile(archive, "w") as output: + output.writestr("mainTemplate.json", json.dumps(package)) + with zipfile.ZipFile(archive) as output: + assert output.namelist() == ["mainTemplate.json"] + text = output.read("mainTemplate.json").decode("utf-8") + assert Path(result["localReceipt"]).exists() + for forbidden in ("authoredReview", "nativeReview", "reversePayloadChanges", '"actor"', + '"reference"', '"decision"', "kingwil", '"queries"'): + assert forbidden not in text + registration = package["resources"][1]["properties"] + assert registration["version"] == document["xdrMetadata"]["version"] + assert registration["contentProvenance"]["contentVersion"] == document["xdrMetadata"]["version"] + assert registration["contentId"] == document["xdrMetadata"]["contentId"] + installed = package["resources"][0]["properties"]["template"]["resources"]["workbook"]["properties"] + assert installed["id"] == document["xdrMetadata"]["instanceId"] + assert installed["content"]["version"] == "Notebook/1.0" + assert installed["content"]["items"] == document["items"] + + +def test_clean_cli_preserves_local_authorization(update_case, capsys): + root, path, _, _, _ = update_case + legacy(update_case) + args = ["--solution", str(root), "--content", path.relative_to(root).as_posix()] + assert cli.main(["workbook-publication-plan", *args]) == 0 + plan = json.loads(capsys.readouterr().out) + plan.update(actor="local account", reference="user authorization", decision="accept") + review = root / "Logs" / "publication-review.json" + review.parent.mkdir(exist_ok=True) + review.write_text(json.dumps(plan), encoding="utf-8") + assert cli.main(["workbook-publication-clean", *args, "--review", str(review), + "--expected-sha256", plan["inputSha256"]]) == 0 + receipt = Path(json.loads(capsys.readouterr().out)["localReceipt"]) + assert json.loads(receipt.read_bytes())["evidence"]["review"]["reference"] == "user authorization" + receipt.unlink() + assert cli.main(["workbook-publication-validate", *args, "--mode", "structural"]) == 0 + capsys.readouterr() + assert cli.main(["workbook-publication-validate", *args, "--mode", "authoring"]) == 1 + capsys.readouterr() + assert cli.main(["native-workbook-validate", "--solution", str(root)]) == 1 diff --git a/Tools/SentinelToXDRMigration/tests/test_workbook_updates.py b/Tools/SentinelToXDRMigration/tests/test_workbook_updates.py index 84e63db6764..fd5637e0165 100644 --- a/Tools/SentinelToXDRMigration/tests/test_workbook_updates.py +++ b/Tools/SentinelToXDRMigration/tests/test_workbook_updates.py @@ -10,6 +10,7 @@ import pytest from sentinel_xdr_migration import cli, workbook_updates as updates, workbooks +from sentinel_xdr_migration import workbook_publication as publication from test_native_workbooks import ( solution as native_solution, imported as native_imported, candidate as native_candidate, ) @@ -86,10 +87,14 @@ def test_restores_historical_proof_without_changing_edited_notebook(update_case) current["contentProvenance"]["nativeReview"]["actor"] = "manual replacement" path.write_text(json.dumps(current), encoding="utf-8") assert workbooks.validate_workbooks(root, selected_only=True)["invalid"] == 1 - assert apply(update_case)["runtimeQualified"] is False + applied = apply(update_case) + assert applied["runtimeQualified"] is False result = json.loads(path.read_bytes()) - review = result["contentProvenance"].pop("authoredReview") - assert result["contentProvenance"] == old["contentProvenance"] + receipt = json.loads(Path(applied["localReceipt"]).read_bytes()) + review = receipt["evidence"]["review"] + historical = json.loads(receipt["evidence"]["baselineArtifact"]) + assert historical["contentProvenance"] == old["contentProvenance"] + assert result["contentProvenance"] == publication.public_document(old)["contentProvenance"] assert result["items"] == current["items"] assert result["xdrMetadata"] == current["xdrMetadata"] assert len(review["changedQueries"]) == 1 @@ -98,6 +103,7 @@ def test_restores_historical_proof_without_changing_edited_notebook(update_case) for source in (root / "Workbook Sources").rglob("*.json"): source.unlink() assert workbooks.validate_workbooks(root, selected_only=True)["invalid"] == 0 + assert publication.publication_errors(root, result) == [] @pytest.mark.parametrize("mutation", [ @@ -106,10 +112,12 @@ def test_restores_historical_proof_without_changing_edited_notebook(update_case) ]) def test_future_tampering_and_staleness_fail(update_case, mutation): root, path, _, index, _ = update_case - apply(update_case) + applied = apply(update_case) document = json.loads(path.read_bytes()) provenance = document["contentProvenance"] - review = provenance["authoredReview"] + receipt_path = Path(applied["localReceipt"]) + receipt = json.loads(receipt_path.read_bytes()) + review = receipt["evidence"]["review"] if mutation == "query": document["items"][index]["content"]["query"] += " | take 4" elif mutation == "settings": @@ -117,7 +125,7 @@ def test_future_tampering_and_staleness_fail(update_case, mutation): elif mutation == "identity": document["xdrMetadata"]["contentId"] = str(uuid4()) elif mutation == "historical-proof": - provenance["source"]["sha256"] = "0" * 64 + receipt["evidence"]["baselineArtifact"] += " " elif mutation == "baseline-hash": review["baseline"]["canonicalDocumentSha256"] = "0" * 64 elif mutation == "delta": @@ -133,7 +141,8 @@ def test_future_tampering_and_staleness_fail(update_case, mutation): else: document["xdrMetadata"]["displayName"] += " changed" path.write_text(json.dumps(document), encoding="utf-8") - assert workbooks.validate_workbooks(root, selected_only=True)["invalid"] == 1 + receipt_path.write_text(json.dumps(receipt), encoding="utf-8") + assert publication.publication_errors(root, document) @pytest.mark.parametrize("query", [ @@ -179,22 +188,22 @@ def test_review_does_not_accept_self_asserted_or_stale_evidence(update_case, mut def test_recomputed_hashes_do_not_override_discovered_dependencies(update_case): root, path, _, index, old = update_case - apply(update_case) + applied = apply(update_case) document = json.loads(path.read_bytes()) - review = document["contentProvenance"].pop("authoredReview") + receipt_path = Path(applied["localReceipt"]) + receipt = json.loads(receipt_path.read_bytes()) document["items"][index]["content"]["query"] = "UnknownTable | take 1" - review["reversePayloadChanges"] = updates._reverse(updates._payload(document), updates._payload(old)) - review["canonicalPayloadSha256"] = updates._canonical_payload_sha256(document) - review["canonicalDocumentSha256"] = updates._canonical(document) - digest = updates.workbook.query_hash("UnknownTable | take 1") - review["queries"][0]["querySha256"] = digest - review["changedQueries"][0]["querySha256"] = digest - review["reviewSha256"] = updates._review_digest(review) - document["contentProvenance"]["authoredReview"] = review + receipt["canonicalDocumentSha256"] = publication.digest(document) + receipt["evidence"]["publicDocument"] = document + receipt.pop("receiptSha256") + receipt["receiptSha256"] = publication.digest(receipt) + new_receipt = publication._receipt_path(root, document) + new_receipt.write_text(json.dumps(receipt), encoding="utf-8") path.write_text(json.dumps(document), encoding="utf-8") result = workbooks.validate_workbooks(root, selected_only=True) assert result["invalid"] == 1 assert any("UnknownTable" in error for error in result["results"][0]["errors"]) + assert publication.publication_errors(root, document) def test_replacement_conversion_receipt_cannot_be_used_as_original_baseline(migrated_solution): @@ -356,11 +365,31 @@ def test_migrated_hybrid_queries_preserve_historical_sentinel_lineage(migrated_s case = root, path, baseline, 0, old plan = accepted(case) assert plan["queries"][0]["tables"] == ["SecurityAlert", "SecurityIncident"] - apply(case) + if not complete_dependencies: + with pytest.raises(ValueError, match="Unknown or non-native"): + apply(case) + # Legacy historical exceptions stay in local proof. Publication needs + # explicit complete public prerequisites, not an inherited waiver. + legacy = deepcopy(current) + legacy["contentProvenance"]["authoredReview"] = updates.review_workbook_update( + root, path.relative_to(root).as_posix(), baseline, plan) + assert updates.validate_updated_document(root, legacy) == [] + path.write_text(json.dumps(legacy), encoding="utf-8") + clean_review = publication.plan_publication(root, path.relative_to(root).as_posix()) + clean_review.update(actor="fixture reviewer", reference="Explicit complete prerequisites", decision="accept") + assert clean_review["dependencyErrors"] + unchanged = path.read_bytes() + with pytest.raises(ValueError, match="connector coverage"): + publication.clean_publication(root, path.relative_to(root).as_posix(), + clean_review["inputSha256"], clean_review) + assert path.read_bytes() == unchanged + clean_review["dependencies"]["requiredDataConnectors"][0]["dataTypes"].append("SecurityIncident") + publication.clean_publication(root, path.relative_to(root).as_posix(), + clean_review["inputSha256"], clean_review) + else: + apply(case) updated = json.loads(path.read_bytes()) - historical = deepcopy(updated["contentProvenance"]) - historical.pop("authoredReview") - assert historical == old["contentProvenance"] + assert "authoredReview" not in updated["contentProvenance"] assert workbooks.validate_workbooks(root)["invalid"] == 0 if shutil.which("pwsh"): result = powershell_scan(root, path)