Support reviewed workbook updates with clean public artifacts and local audit evidence

Tools/Create-Azure-Sentinel-Solution/common/defenderWorkbooks.ps1

@@ -45,7 +45,8 @@ function Assert-DefenderWorkbookReleaseVersion {
 }
 
 function Assert-NativeDefenderWorkbookDocument {
-    param([psobject]$Document, [string]$SolutionRoot)
+    param([psobject]$Document, [string]$SolutionRoot,
+        [ValidateSet('native_workbooks', 'workbook_updates', 'workbook_publication')][string]$ValidationModule = 'native_workbooks')
 
     $python = if ($env:SENTINEL_XDR_PYTHON) { $env:SENTINEL_XDR_PYTHON } else { 'python' }
     $start = [System.Diagnostics.ProcessStartInfo]::new()
@@ -56,7 +57,7 @@ function Assert-NativeDefenderWorkbookDocument {
     $start.RedirectStandardError = $true
     $start.StandardInputEncoding = [System.Text.UTF8Encoding]::new($false)
     $start.StandardOutputEncoding = [System.Text.UTF8Encoding]::new($false)
-    foreach ($argument in @('-m', 'sentinel_xdr_migration.native_workbooks', $SolutionRoot)) {
+    foreach ($argument in @('-m', "sentinel_xdr_migration.$ValidationModule", $SolutionRoot)) {
         $start.ArgumentList.Add($argument)
     }
     $start.Environment['PYTHONPATH'] = [System.IO.Path]::GetFullPath(
@@ -104,6 +105,32 @@ function Assert-DefenderWorkbookDocument {
     }
     Assert-DefenderWorkbookReleaseVersion -Version $metadata.version
     $provenance = $Document.contentProvenance
+    if ($null -ne $provenance -and $provenance.format -ceq 'public-workbook-v1') {
+        Assert-NativeDefenderWorkbookDocument -Document $Document -SolutionRoot $SolutionRoot `
+            -ValidationModule workbook_publication
+        if ($PassThruSource) { return $Document }
+        return
+    }
+    throw "Published authored workbooks must be clean; run workbook-publication-plan/clean for '$ContentPath'."
+}
+
+# Legacy diagnostics are not a publication entry point.
+function Assert-LegacyDefenderWorkbookDocument {
+    param([psobject]$Document, [string]$SolutionRoot, [string]$SolutionName, [string]$ContentPath, [switch]$PassThruSource)
+    $provenance = $Document.contentProvenance
+    if ($null -ne $provenance -and $null -ne $provenance.PSObject.Properties['authoredReview']) {
+        Assert-NativeDefenderWorkbookDocument -Document $Document -SolutionRoot $SolutionRoot `
+            -ValidationModule workbook_updates
+        if ($PassThruSource) {
+            if ($provenance.origin -ceq 'native-xdr') {
+                return $Document
+            }
+            $resolved = Resolve-SolutionContentPath -SolutionRoot $SolutionRoot -SolutionName $SolutionName `
+                -ContentPath $provenance.source.path -RequireFile
+            return (Get-Content -LiteralPath $resolved.FullPath -Raw | ConvertFrom-Json -ErrorAction Stop)
+        }
+        return
+    }
     if ($null -ne $provenance -and $provenance.origin -ceq 'native-xdr') {
         Assert-NativeDefenderWorkbookDocument -Document $Document -SolutionRoot $SolutionRoot
         if ($PassThruSource) {
@@ -798,6 +825,11 @@ function Get-DefenderWorkbookParserDependencies {
             }
         }
     }
+    if ($provenance.format -ceq 'public-workbook-v1') {
+        # Python validated exact current query aliases and hash-bound local proof
+        # (or an explicitly requested public Git reproduction) above.
+        foreach ($alias in $declared) { [void]$referenced.Add($alias) }
+    }
     if (-not $declared.SetEquals($referenced)) {
         throw 'Defender Workbook requiredWorkspaceFunctions must exactly cover reviewed query workspaceFunctions.'
     }
@@ -852,6 +884,13 @@ function Get-DefenderWorkbookParserDependencies {
         }
     }
     $sourceAllowedBareTables = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
+    if ($null -ne $provenance.PSObject.Properties['authoredReview']) {
+        # The shared adapter has checked these against the reconstructed
+        # baseline and current KQL, including exact inherited hybrid lineage.
+        foreach ($query in $provenance.authoredReview.queries) {
+            foreach ($table in $query.tables) { [void]$allowedBareTables.Add($table) }
+        }
+    }
     foreach ($item in $allowedBareTables) { [void]$sourceAllowedBareTables.Add($item) }
     $catalogPath = Join-Path $PSScriptRoot '../../SentinelToXDRMigration/sentinel_xdr_migration/_workbook_core/data/mapping_catalog.json'
     if (Test-Path -LiteralPath $catalogPath -PathType Leaf) {

Tools/SentinelToXDRMigration/README.md

@@ -332,43 +332,23 @@ hidden parser inputs, workloads or connector coverage, and offline acceptance
 does not qualify parser availability or execution in the target tenant.
 The plan exits nonzero when it contains technical blockers.
 
-Authored provenance is additive:
+The importer currently produces a legacy review-bearing staging candidate.
+Before publication, run `workbook-publication-plan/clean` below: retain that
+complete proof locally and publish only this compact native provenance:
 
 ```json
 {
   "contentProvenance": {
+    "format": "public-workbook-v1",
     "origin": "native-xdr",
-    "source": {
-      "path": "Workbook Sources/Native/example.json",
-      "sha256": "<SHA256 of exact source file bytes>"
-    },
-    "canonicalPayloadSha256": "<canonical Notebook payload SHA256>",
-    "nativeReview": {
-      "origin": "native-xdr",
-      "source": {"path": "Workbook Sources/Native/example.json", "sha256": "<same hash>"},
-      "canonicalPayloadSha256": "<same canonical Notebook payload SHA256>",
-      "displayName": "Example",
-      "actor": "<reviewer>",
-      "reference": "<review reference>",
-      "requiredWorkloads": [{"workload": "mdo", "dataTypes": ["EmailEvents"]}],
-      "requiredDataConnectors": [],
-      "queries": [{
-        "path": "/items/0/content/query",
-        "querySha256": "<SHA256 of exact UTF-8 query string>",
-        "tables": ["EmailEvents"],
-        "decision": "accept",
-        "evidence": "<complete current query/dependency review>",
-        "blockers": []
-      }]
-    },
     "requiredWorkloads": [{"workload": "mdo", "dataTypes": ["EmailEvents"]}],
     "requiredDataConnectors": [],
-    "dependencyReviewRequired": false
+    "requiredWorkspaceFunctions": []
   }
 }
 ```
 
-There is no `conversion` object in native provenance. `xdrMetadata` uses the
+There is no public `conversion` or `nativeReview` object. `xdrMetadata` uses the
 normal schema/version/display name fields. Canonical UUIDv5 assignment is
 `uuid5(NAMESPACE_URL, "native-xdr-workbook:<solution directory name>:<source POSIX path>")`;
 the instance is the existing
@@ -380,20 +360,23 @@ them. A deliberate source edit requires a fresh review and removal of the old
 local candidate and its `XDR Workbooks` selection before reimport, retaining the
 same identity for the same path.
 
-Native validation compares the **whole original Notebook** with the authored
+Historical native proof validation compares the **whole original Notebook** with the staging
 payload after removing only the two additive authoring fields. Thus query
 strings (including CRLF/Unicode), items, notices, tabs and other source keys
-remain unchanged. Missing/stale sources or query coverage, duplicate JSON
+remain unchanged. Stale source proof or query coverage, duplicate JSON
 keys/IDs/selections, traversal and symlinks fail closed. Native validation is
-integrated with `validate_workbooks`, the existing local static XDR adapter,
-and the V3.1 emitter. The PowerShell emitter reuses this Python validator
-against its in-memory native document and requires that interpreter's
+integrated with `validate_workbooks` and the existing local static XDR adapter.
+After cleanup, public structural validation is portable without raw source or
+private approval; authoring/package validation requires the current local receipt.
+The PowerShell emitter reuses shared Python validation
+against its in-memory public document and requires that interpreter's
 dependencies; a missing interpreter/dependency is an error, never a waiver.
 `SENTINEL_XDR_PYTHON` selects the interpreter; absent it, `python` is used.
 The subprocess imports only the emitter's toolkit checkout.
 
 For candidate package regression testing, use the correct V3.1 entry point
-from the **content repository root**, with a separate toolkit root:
+from the **content repository root**, with a separate toolkit root, **after
+explicit publication cleanup**:
 
 ```powershell
 Set-Location 'C:\content'
@@ -414,6 +397,170 @@ native reviews and authoring metadata are not installed. Registration keeps
 the existing reviewed dependency contract. Original Sentinel selections remain
 independent and no native source is deployed twice.
 
+### Reviewed post-import workbook corrections (offline)
+
+Use `workbook-update-plan`, `workbook-update-review`, and
+`workbook-update-apply` for intentional authored Notebook corrections after
+migration/import. These commands do not migrate, deploy, invoke queries, or
+establish runtime/UI qualification. They require an explicit original reviewed
+**authored** baseline file, for example extracted with `git show` from the
+accepted content commit into a review workspace outside deployable content.
+Legacy update-reviewed artifacts are not original conversion baselines. A clean
+published artifact can instead be the public baseline for a later explicit review.
+
+```powershell
+# $Python, $Solution and PYTHONPATH use the same setup as native import above.
+$Content = 'XDR Workbooks\example.json'.Replace('\', '/')
+$Baseline = 'C:\review-workspace\baseline-example.json'
+New-Item -ItemType Directory -Force "$Solution\Logs" | Out-Null
+& $Python -m sentinel_xdr_migration.cli workbook-update-plan `
+  --solution $Solution --content $Content --baseline $Baseline |
+  Set-Content -LiteralPath "$Solution\Logs\update-plan.json" -Encoding utf8
+```
+
+Inspect the exact current Notebook and plan, including all current `queries`
+with discovered tables/functions and `changedQueries` (added/removed queries
+use null previous/current hashes). Explicitly set nonempty `actor` and
+`reference`, whole-document `decision: "accept"`, and each changed query's
+`decision: "accept"` and nonempty `evidence`. Do not edit generated hashes,
+dependencies or the reverse baseline delta. Save under gitignored `Logs` as
+`update-decisions.json`; never put review material in authored content or a PR.
+
+```powershell
+& $Python -m sentinel_xdr_migration.cli workbook-update-review `
+  --solution $Solution --content $Content --baseline $Baseline `
+  --review "$Solution\Logs\update-decisions.json" |
+  Set-Content -LiteralPath "$Solution\Logs\update-reviewed.json" -Encoding utf8
+& $Python -m sentinel_xdr_migration.cli workbook-update-apply `
+  --solution $Solution --content $Content --baseline $Baseline `
+  --review "$Solution\Logs\update-reviewed.json"
+& $Python -m sentinel_xdr_migration.cli native-workbook-validate --solution $Solution
+```
+
+Apply preserves the edited Notebook and metadata/IDs, independently validates
+the baseline's historical proof, and emits **clean public metadata**, not an
+embedded `authoredReview`. It refuses identity/prerequisite changes. Complete
+current dependencies are mandatory: native-only lineage, known tables, exact
+hash-bound workspace functions, built-in ASIM recognition, and dynamic/cross-
+resource rejection remain enforced. Reviewer assertions cannot waive them.
+
+The complete review, baseline bytes, and available historical source bytes
+remain in a hash-bound receipt under `Logs\workbook-reviews`. These are local
+authoring evidence, not signatures, reviewer-identity proof, runtime qualification,
+or publication metadata. Keep external source/baseline evidence; do not delete
+it after apply. No duplicate full source Notebook needs to be shipped.
+
+### Clean public workbooks and private authoring receipts
+
+Published authored JSON has only the Notebook, existing `xdrMetadata` identity/
+version, and compact `contentProvenance`:
+
+* `format: "public-workbook-v1"` and factual `origin`;
+* public `requiredWorkloads`, `requiredDataConnectors`, and
+  `requiredWorkspaceFunctions` (including function source paths/hashes).
+
+It contains **no author/reviewer accounts, acceptance decisions, references,
+session identifiers, query inventories, conversion receipts, reverse deltas, or
+proof histories**. Dependencies are not an acceptance claim. The schema rejects
+extra fields in public provenance and prerequisite records. Legacy proof schemas
+remain readable for importing/validating old evidence, not for publication.
+V3.1 rejects legacy authored files until cleaned; migration/import candidates
+must pass the explicit publication step before publishing.
+
+To clean an already reviewed legacy artifact (including old `authoredReview`):
+
+```powershell
+$ReviewDirectory = Join-Path $Solution 'Logs\workbook-reviews'
+New-Item -ItemType Directory -Force $ReviewDirectory | Out-Null
+$Plan = Join-Path $ReviewDirectory 'publication-plan.json'
+$Review = Join-Path $ReviewDirectory 'publication-decisions.json'
+& $Python -m sentinel_xdr_migration.cli workbook-publication-plan `
+  --solution $Solution --content $Content |
+  Set-Content -LiteralPath $Plan -Encoding utf8
+# Inspect the exact file and complete planned dependencies. Save the plan as
+# $Review with explicit nonempty actor/reference and decision "accept".
+# Never automatically approve a plan. Do not edit inputSha256.
+$Decision = Get-Content -LiteralPath $Review -Raw | ConvertFrom-Json
+& $Python -m sentinel_xdr_migration.cli workbook-publication-clean `
+  --solution $Solution --content $Content --review $Review `
+  --expected-sha256 $Decision.inputSha256
+& $Python -m sentinel_xdr_migration.cli workbook-publication-validate `
+  --solution $Solution --content $Content --mode authoring
+```
+
+Cleanup verifies the original raw SHA and all historical gates before writing,
+stores the original artifact/proof **verbatim locally**, and changes only
+`contentProvenance`. Notebook settings, KQL, `Notebook/1.0`, content/instance IDs,
+and `xdrMetadata.version` remain exact. It does not bump a release. An existing
+XDR 3.2.2 or O365 3.2.1 artifact remains at that release; deployment copies use
+the selected solution release as `contentVersion`.
+
+Historical hybrid receipts sometimes omitted a nested table. Their limited
+same-query historical exception remains usable to validate old proof, but is
+**not a public packaging waiver**. The plan adds known missing workloads; it
+never invents connector coverage. Reviewers must explicitly supply missing
+connector/table prerequisites in the local `dependencies` review. Existing
+prerequisites/functions cannot be removed/replaced by cleanup. For example,
+an old `SecurityAlert`-only connector record does not establish coverage for
+`SecurityIncident`; cleanup blocks until that additional prerequisite is
+explicitly reviewed. The historical receipt itself is retained unchanged.
+
+`workbook-publication-validate --mode structural`, the schema, Python
+`validate_workbooks`, and the static adapter validate portable public structure/
+current source dependencies **without private approvals**. They do not approve
+authoring. `--mode authoring`, `native-workbook-validate`, authoring workflow gates,
+and normal Python/PowerShell packaging additionally require a current canonical
+whole-document hash-bound local receipt. A later query/settings/metadata/
+dependency edit needs fresh explicit review; structural success is not a waiver.
+The clean/update command creates the receipt that packaging consumes automatically
+by current document hash. Prior receipts remain locally available, including when
+the same payload receives another authorization.
+
+For an **explicit standalone reproduction**, not authoring acceptance, a clean
+public Git checkout can package without private receipts:
+
+```powershell
+$env:SENTINEL_XDR_WORKBOOK_PUBLIC_REF = '<published commit SHA>'
+try {
+  # Use the standalone V3.1 command below, retaining the published release.
+} finally {
+  Remove-Item Env:\SENTINEL_XDR_WORKBOOK_PUBLIC_REF -ErrorAction SilentlyContinue
+}
+```
+
+This checks the actual in-memory workbook against the exact file at that Git
+commit, plus current public table/function/source dependencies. Missing refs or
+edited workbooks fail. There is **no automatic HEAD fallback** and it is not
+available as an authoring/workflow approval. Git history/review is the trust
+boundary for selecting a published ref; reproduction never claims personal
+approval. Receipts/Logs are never ZIP inputs: the package contains the generated
+template and UI only. Installed Notebook provenance remains the deployment
+identity/version quartet; registration additionally retains public dependencies.
+
+Keep existing version/release and candidate gates: package through
+`package-v3-1 --solution $Solution --toolkit-root $Toolkit --version-bump none`
+(for an already prepared release), then run the official
+`.script\local-validation\build-and-validate.ps1 -SolutionName <name>
+-ContentRoot <content checkout> -SkipPackaging -PackageReportPath <exact V3.1 report>`.
+That wrapper otherwise defaults to legacy V3 packaging and disallows
+cross-checkout packaging; do not use it to create this candidate.
+If an old migration run's frozen selection predates the maintained content,
+do not alter that run or weaken its selection gate. The existing standalone
+content-maintenance backend is:
+
+```powershell
+& "$Toolkit\Tools\Create-Azure-Sentinel-Solution\V3\createSolutionV3_1.ps1" `
+  -SolutionDataFolderPath "$Solution\Data" -ToolkitRootPath $Toolkit `
+  -VersionMode local -VersionBump none `
+  -ArmValidationMethod lab-deployment
+```
+
+Use the same official validation-only wrapper afterward. This builds a
+candidate; it does not establish migration workflow acceptance or lab/runtime
+qualification.
+Keep automatic attribution lookup and its fail-closed guard; do not add a skip
+flag to recover from an attribution failure.
+
 ### Toolkit-owner managed update policy
 
 Toolkit-managed Defender Workbooks and XDR detections are updated by stable ID.

Tools/SentinelToXDRMigration/agent-workflows/end-to-end-migration.md

@@ -76,6 +76,22 @@ target and independently verified deployment evidence before package acceptance.
 
 ### Versioned workbook and candidate ARM lifecycle
 
+Published authored workbook JSON must use the clean `public-workbook-v1`
+dependency contract described in the toolkit README. Keep reviewer accounts,
+acceptance decisions, references, session context, source/conversion proofs
+and query/reverse-delta histories only in gitignored local `Logs` receipts.
+Use `workbook-publication-plan/clean` to import legacy proof without deleting it.
+Cleaning preserves the actual Notebook, managed IDs and authored content version.
+Incomplete historical connector coverage must be explicitly reviewed and completed,
+not carried forward as a public exception.
+
+Portable structural validation of public content does not establish authoring
+acceptance. Authoring/apply/package operations require current hash-bound local
+review evidence. Explicit standalone reproduction against a selected published
+Git commit can use public dependencies without private receipts; it is not a
+workflow approval and does not relax any candidate, official-check, environment
+authorization or qualification gate.
+
 New CLI runs persist `workbookCapabilityVersion=1` and default to
 `armValidationMethod=lab-deployment`, in either explicitly selected profile.
 Authoring may explicitly choose `--arm-validation-method arm-ttk`. Existing