From ce83f9b45d8dbf97d6b8472bfa9e72e042712a56 Mon Sep 17 00:00:00 2001 From: Simon Gallagher Date: Tue, 10 Nov 2026 16:27:27 +0000 Subject: [PATCH 1/3] Add confidential H100 AKS reproduction and unsupported signed-driver workaround Include provisioning checks, attestation scripts, CUDA smoke test, offline tests and documented investigation results. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- aks-samples/README.md | 3 + aks-samples/aks-gpu-worker/.gitignore | 1 + aks-samples/aks-gpu-worker/Common.ps1 | 124 +++ .../Confidential-GPU-H100-AKS-Onboarding.md | 267 ++++++ aks-samples/aks-gpu-worker/Deploy.ps1 | 183 ++++ .../aks-gpu-worker/Install-SignedDriver.sh | 62 ++ aks-samples/aks-gpu-worker/README.md | 821 ++++++++++++++++++ .../aks-gpu-worker/Test-CvmAttestation.sh | 25 + .../Test-DriverCompatibility.ps1 | 40 + .../aks-gpu-worker/Test-GpuAttestation.sh | 42 + .../gpu-operator-experimental-values.yaml | 23 + .../aks-gpu-worker/gpu-smoke-test.yaml | 36 + .../aks-gpu-worker/signed-driver-probe.yaml | 69 ++ .../aks-gpu-worker/tests/Test-Sample.ps1 | 164 ++++ 14 files changed, 1860 insertions(+) create mode 100644 aks-samples/aks-gpu-worker/.gitignore create mode 100644 aks-samples/aks-gpu-worker/Common.ps1 create mode 100644 aks-samples/aks-gpu-worker/Confidential-GPU-H100-AKS-Onboarding.md create mode 100644 aks-samples/aks-gpu-worker/Deploy.ps1 create mode 100644 aks-samples/aks-gpu-worker/Install-SignedDriver.sh create mode 100644 aks-samples/aks-gpu-worker/README.md create mode 100644 aks-samples/aks-gpu-worker/Test-CvmAttestation.sh create mode 100644 aks-samples/aks-gpu-worker/Test-DriverCompatibility.ps1 create mode 100644 aks-samples/aks-gpu-worker/Test-GpuAttestation.sh create mode 100644 aks-samples/aks-gpu-worker/gpu-operator-experimental-values.yaml create mode 100644 aks-samples/aks-gpu-worker/gpu-smoke-test.yaml create mode 100644 aks-samples/aks-gpu-worker/signed-driver-probe.yaml create mode 100644 aks-samples/aks-gpu-worker/tests/Test-Sample.ps1 diff --git a/aks-samples/README.md b/aks-samples/README.md index ca8a96e..dc843e4 100644 --- a/aks-samples/README.md +++ b/aks-samples/README.md @@ -6,6 +6,9 @@ Deploy Azure Kubernetes Service clusters with AMD SEV-SNP confidential node pools and Customer Managed Keys (CMK). +For a single confidential H100 GPU worker and an exact-kernel NVIDIA driver +compatibility reproduction, see [AKS confidential H100 worker](aks-gpu-worker/README.md). + ## Architecture ``` diff --git a/aks-samples/aks-gpu-worker/.gitignore b/aks-samples/aks-gpu-worker/.gitignore new file mode 100644 index 0000000..a47f082 --- /dev/null +++ b/aks-samples/aks-gpu-worker/.gitignore @@ -0,0 +1 @@ +.local/ diff --git a/aks-samples/aks-gpu-worker/Common.ps1 b/aks-samples/aks-gpu-worker/Common.ps1 new file mode 100644 index 0000000..4e7274c --- /dev/null +++ b/aks-samples/aks-gpu-worker/Common.ps1 @@ -0,0 +1,124 @@ +#Requires -Version 7.2 +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Resolve-AzureCli { + $commands = @(Get-Command az -CommandType Application -ErrorAction SilentlyContinue) + if ($commands.Count) { return $commands[0].Source } + if ($IsWindows) { + foreach ($root in @($env:ProgramFiles, ${env:ProgramFiles(x86)})) { + if ($root) { + $candidate = Join-Path $root 'Microsoft SDKs\Azure\CLI2\wbin\az.cmd' + if (Test-Path $candidate) { return $candidate } + } + } + } + throw 'Azure CLI is required. Install it and run az login before retrying.' +} + +function Invoke-CheckedCommand { + param( + [Parameter(Mandatory)][string]$Command, + [Parameter(Mandatory)][string[]]$ArgumentList + ) + $output = & $Command @ArgumentList + if ($LASTEXITCODE -ne 0) { + throw "$Command $($ArgumentList -join ' ') failed with exit code $LASTEXITCODE." + } + return $output +} + +function Get-DriverManifest { + param( + [Parameter(Mandatory)][ValidatePattern('^[0-9][0-9A-Za-z._-]*$')][string]$Tag + ) + $tokenResponse = Invoke-RestMethod -Uri 'https://nvcr.io/token?service=registry&scope=repository:nvidia/driver:pull' + $accept = @( + 'application/vnd.oci.image.manifest.v1+json' + 'application/vnd.oci.image.index.v1+json' + 'application/vnd.docker.distribution.manifest.v2+json' + 'application/vnd.docker.distribution.manifest.list.v2+json' + ) -join ', ' + $response = Invoke-WebRequest -Uri "https://nvcr.io/v2/nvidia/driver/manifests/$Tag" ` + -Headers @{ Authorization = "Bearer $($tokenResponse.token)"; Accept = $accept } ` + -SkipHttpErrorCheck + $status = [int]$response.StatusCode + $digest = $null + $errorCode = $null + if ($status -eq 200) { + $digest = @($response.Headers['Docker-Content-Digest'])[0] + if ($digest -notmatch '^sha256:[0-9a-f]{64}$') { + throw "Registry did not return a valid digest for $Tag." + } + } + elseif ($status -eq 404) { + $body = $response.Content | ConvertFrom-Json + $errorCode = $body.errors[0].code + if ($errorCode -ne 'MANIFEST_UNKNOWN' -or $body.errors[0].message -ne 'manifest unknown') { + throw "Registry returned an unexpected 404 for ${Tag}: $($response.Content)" + } + } + else { + throw "Registry request for $Tag returned HTTP $status. Check authentication, networking and rate limits; this is not evidence of a missing image." + } + [pscustomobject]@{ + Image = "nvcr.io/nvidia/driver:$Tag" + Tag = $Tag + Available = ($status -eq 200) + HttpStatus = $status + Digest = $digest + ErrorCode = $errorCode + CheckedAtUtc = [DateTime]::UtcNow.ToString('o') + } +} + +function Get-GpuNode { + param( + [Parameter(Mandatory)][string]$KubeConfig, + [string]$PoolName = 'gpunp' + ) + $nodes = Invoke-CheckedCommand kubectl @('--kubeconfig', $KubeConfig, 'get', 'nodes', + '-l', "agentpool=$PoolName", '-o', 'json') | ConvertFrom-Json + if (@($nodes.items).Count -ne 1) { + throw "Expected exactly one node in pool $PoolName; found $(@($nodes.items).Count)." + } + return $nodes.items[0] +} + +function Assert-SamplePools { + param( + [Parameter(Mandatory)][object[]]$Pools, + [Parameter(Mandatory)][string]$SystemVmSize, + [switch]$AllowMissingGpu + ) + $system = @($Pools | Where-Object name -EQ 'sysnp') + $gpu = @($Pools | Where-Object name -EQ 'gpunp') + if ($system.Count -ne 1 -or @($Pools | Where-Object { $_.name -notin @('sysnp', 'gpunp') }).Count -or + $gpu.Count -gt 1 -or (-not $AllowMissingGpu -and $gpu.Count -ne 1)) { + throw 'Expected one system pool and one GPU pool, with no additional pools.' + } + + foreach ($pool in $Pools) { + $size = if ($pool.name -eq 'sysnp') { $SystemVmSize } else { 'Standard_NCC40ads_H100_v5' } + $mode = if ($pool.name -eq 'sysnp') { 'System' } else { 'User' } + if ($pool.vmSize -ne $size -or $pool.count -ne 1 -or $pool.osSku -ne 'Ubuntu2404' -or + $pool.mode -ne $mode -or $pool.enableAutoScaling -eq $true -or $pool.provisioningState -ne 'Succeeded') { + throw "Pool $($pool.name) does not match the single-node sample configuration; it was not modified." + } + if ($pool.name -eq 'gpunp' -and + ($pool.gpuProfile.driver -ne 'None' -or 'sku=gpu:NoSchedule' -notin $pool.nodeTaints)) { + throw 'GPU pool must disable automatic drivers and retain the GPU scheduling taint.' + } + } +} + +function Assert-ConfidentialGpuScaleSet { + param([Parameter(Mandatory)][object[]]$ScaleSets) + $gpu = @($ScaleSets | Where-Object { $_.sku.name -eq 'Standard_NCC40ads_H100_v5' }) + if ($gpu.Count -ne 1) { throw 'Expected exactly one H100 scale set.' } + $security = $gpu[0].virtualMachineProfile.securityProfile + if ($gpu[0].sku.capacity -ne 1 -or $security.securityType -ne 'ConfidentialVM' -or + $security.uefiSettings.secureBootEnabled -ne $true -or $security.uefiSettings.vTpmEnabled -ne $true) { + throw 'H100 scale set must have one ConfidentialVM instance with Secure Boot and vTPM enabled.' + } +} diff --git a/aks-samples/aks-gpu-worker/Confidential-GPU-H100-AKS-Onboarding.md b/aks-samples/aks-gpu-worker/Confidential-GPU-H100-AKS-Onboarding.md new file mode 100644 index 0000000..417ed00 --- /dev/null +++ b/aks-samples/aks-gpu-worker/Confidential-GPU-H100-AKS-Onboarding.md @@ -0,0 +1,267 @@ +# Confidential H100 on AKS: tested signed-package workaround + +This companion to the [upstream AKS onboarding page](https://github.com/Azure/az-cgpu-onboarding/blob/136262ef00ad75b8e0bc789563f043006fb68f4c/docs/Confidential-GPU-H100-AKS-Onboarding.md) +retains its Azure CLI, GPU Operator and workload-validation workflow, but uses +exact-kernel Ubuntu signed packages instead of the failing driver-container +entrypoint. See [investigation history](README.md) for the original blockers. + +**Experimental diagnostic recipe, not a supported production configuration.** +It runs a privileged pod with host devices and firmware access. Do not use on +shared production nodes. The pod holds UVM persistence for at most four hours; +this is not a reconciled installation across reboots, image upgrades or node +replacement. Keep Secure Boot, vTPM and confidential mode enabled. + +## Tested configuration + +Observed in West Europe on 2026-11-10: + +| Component | Observed value | +|---|---| +| Topology | One system node plus one confidential GPU worker (two nodes total) | +| System node | `Standard_D4ds_v6`, Ubuntu2404 | +| GPU worker | `Standard_NCC40ads_H100_v5`, H100 NVL, Ubuntu2404 | +| Kubernetes | `1.36.5` | +| AKS GPU node image | `AKSUbuntu-2404gen2CVMcontainerd-202610.30.1` | +| GPU node OS / kernel | Ubuntu 24.04.5 LTS / `6.8.0-1071-azure-fde` | +| Runtime | `containerd://2.3.7-4` | +| Security | ConfidentialVM, Secure Boot and vTPM enabled; GPU CC ON / PRODUCTION | +| GPU Operator | `v26.7.2`, **driver operand disabled** | +| NVIDIA signed package | `linux-modules-nvidia-615-open-6.8.0-1071-azure-fde=6.8.0-1071.78` | +| NVIDIA driver / matching libraries | `615.71.09` / `615.71.09-0ubuntu0.24.04.2` | +| GPU verifier | Checksum-pinned Azure archive V4.5.1, package `2.3.0`, strict OCSP | +| CPU/CVM verifier | Checksum-pinned Azure guest CLI `v0.1.0`, MAA | + +These are observed versions, not an AKS image pin. AKS may provision a different +kernel later; the installer must fail rather than load mismatched modules. +Ubuntu 26.04 preview explicitly excludes ConfidentialVM; changing container +Ubuntu versions does not change the host kernel. + +## 1. Prerequisites and isolated context + +Use Azure CLI (tested: 2.92.0), kubectl, Helm and Bash. Run the Bash blocks from +this sample directory in a Linux/WSL shell with those tools configured. +The local shell does not need NVIDIA or Python. HTTPS egress is needed for Azure, +Ubuntu repositories, GitHub, NGC and the verifier's certificate/RIM/OCSP services. +Use an authorized subscription with 40 free H100-family vCPUs plus four system +vCPUs and sufficient physical allocation capacity. + +```bash +set -euo pipefail +subscription='' +location=westeurope +resourceGroup=rg-aks-gpu-worker-example +clusterName=aks-gpu-worker-example +mkdir -p .local/cli-extensions-core +export AZURE_EXTENSION_DIR="$PWD/.local/cli-extensions-core" +export AZURE_EXTENSION_USE_DYNAMIC_INSTALL=no +export KUBECONFIG="$PWD/.local/kubeconfig" +az account show --subscription "$subscription" --query '{name:name,state:state}' +az aks get-versions --subscription "$subscription" --location "$location" -o table +``` + +Keep the extension directory empty. This avoids the observed `aks-preview` +18.0.0b17 / core CLI 2.92.0 `hosted_system_profile` client incompatibility without +modifying shared extensions or the default subscription/kubeconfig. +The development authentication model follows the original sample; use Entra +RBAC and restricted API access for production. + +If needed, register the feature from the upstream guide and wait until it is +`Registered` before creation: + +```bash +az feature register --subscription "$subscription" \ + --namespace Microsoft.ContainerService --name Ubuntu2404Preview +az feature show --subscription "$subscription" \ + --namespace Microsoft.ContainerService --name Ubuntu2404Preview \ + --query properties.state -o tsv +az provider register --subscription "$subscription" --namespace Microsoft.ContainerService +``` + +Registration is asynchronous. Do not proceed while the feature is registering. +For automated quota/topology/ownership checks and bounded registration waits, +use [Deploy.ps1](Deploy.ps1) with `-UseCoreCli` as described in the README. + +## 2. Create the disposable cluster and confidential worker + +Use a **new** resource group. These commands incur charges. Automatic upgrades +are disabled only to control this short-lived reproduction, not as a security +policy. AKS requires a system pool; the confidential worker is a user pool. + +```bash +az group create --subscription "$subscription" --name "$resourceGroup" \ + --location "$location" --tags sample=aks-gpu-worker +az aks create --subscription "$subscription" \ + --resource-group "$resourceGroup" --name "$clusterName" \ + --location "$location" --kubernetes-version 1.36.5 --tier free \ + --nodepool-name sysnp --node-count 1 --node-vm-size Standard_D4ds_v6 \ + --os-sku Ubuntu2404 --node-osdisk-type Managed --node-osdisk-size 128 \ + --enable-managed-identity --generate-ssh-keys \ + --auto-upgrade-channel none --node-os-upgrade-channel None \ + --tags sample=aks-gpu-worker +az aks nodepool add --subscription "$subscription" \ + --resource-group "$resourceGroup" --cluster-name "$clusterName" \ + --name gpunp --mode User --node-count 1 \ + --node-vm-size Standard_NCC40ads_H100_v5 --os-sku Ubuntu2404 \ + --gpu-driver None --node-osdisk-type Managed --node-osdisk-size 128 \ + --node-taints sku=gpu:NoSchedule +``` + +For this sample's existing stopped cluster, skip creation and instead use +`az aks start --subscription "$subscription" --resource-group "$resourceGroup" +--name "$clusterName"`. Verify its identity and topology before resuming. + +```bash +az aks get-credentials --subscription "$subscription" \ + --resource-group "$resourceGroup" --name "$clusterName" \ + --file "$KUBECONFIG" --overwrite-existing +kubectl wait --for=condition=Ready node --all --timeout=600s +kubectl get nodes -o wide +test "$(kubectl get nodes -o name | wc -l)" -eq 2 +test "$(kubectl get nodes -l agentpool=gpunp -o name | wc -l)" -eq 1 +test "$(kubectl get nodes -l agentpool=gpunp \ + -o 'jsonpath={.items[0].status.nodeInfo.kernelVersion}')" = 6.8.0-1071-azure-fde +az aks nodepool get-upgrades --subscription "$subscription" \ + --resource-group "$resourceGroup" --cluster-name "$clusterName" \ + --nodepool-name gpunp +nodeResourceGroup=$(az aks show --subscription "$subscription" \ + --resource-group "$resourceGroup" --name "$clusterName" --query nodeResourceGroup -o tsv) +az vmss list --subscription "$subscription" --resource-group "$nodeResourceGroup" \ + --query "[?sku.name=='Standard_NCC40ads_H100_v5'].{name:name,count:sku.capacity,security:virtualMachineProfile.securityProfile}" +test "$(az vmss list --subscription "$subscription" --resource-group "$nodeResourceGroup" \ + --query "[?sku.name=='Standard_NCC40ads_H100_v5' && sku.capacity==\`1\` && virtualMachineProfile.securityProfile.securityType=='ConfidentialVM' && virtualMachineProfile.securityProfile.uefiSettings.secureBootEnabled==\`true\` && virtualMachineProfile.securityProfile.uefiSettings.vTpmEnabled==\`true\`] | length(@)" -o tsv)" = 1 +``` + +Do not continue if effective ConfidentialVM/Secure Boot/vTPM or the kernel differ. +Neither Kubernetes version nor Ubuntu2404 selects the guide's old 1041 image. + +## 3. Install GPU Operator without its broken driver installer + +```bash +helm upgrade --install gpu-operator gpu-operator \ + --repo https://helm.ngc.nvidia.com/nvidia --version v26.7.2 \ + --namespace gpu-operator --create-namespace \ + --values gpu-operator-experimental-values.yaml --set driver.enabled=false +kubectl apply -f signed-driver-probe.yaml +kubectl wait --for=condition=Ready pod/signed-driver-load --timeout=300s +tr -d '\r' < Install-SignedDriver.sh | + kubectl exec -i signed-driver-load -- bash +``` + +Require `SIGNED_DRIVER_LOADED=1` and a zero pipeline status. The script verifies +the exact kernel, Secure Boot, all five module signatures/version/vermagic and +the pinned module-deb SHA-256. It extracts matching packages in the pod, copies +only versioned firmware to the host, loads crypto dependencies and NVIDIA/UVM, +then starts UVM persistence **before the first GPU query**. It requires CC ON, +PRODUCTION and repeated Ready/persistence checks before exposing the coherent +driver tree at `/run/nvidia/driver`. + +No full host metapackage, DKMS, kernel replacement or signing-key enrollment is +performed. A preexisting driver, driver tree or firmware directory is a hard +stop: inspect and cleanly recycle the disposable node instead of reloading +modules on a GPU with a failed CC cleanup. Keep the probe alive during testing. + +On Windows PowerShell, normalize stdin at the receiving end as well: + +```powershell +Get-Content -Raw .\Install-SignedDriver.sh | + kubectl exec -i signed-driver-load -- bash -c 'tr -d "\r" | bash' +if ($LASTEXITCODE -ne 0) { throw 'Signed-driver setup failed' } +``` + +Use the same pattern for the two attestation scripts below. An immutable Pod +spec cannot be updated in place; do not delete/recreate an active probe to add +mounts, since that also kills its persistence daemon. + +## 4. Require a schedulable GPU and a real CUDA workload + +Give the Operator toolkit and device plugin up to ten minutes to initialize: + +```bash +kubectl wait --for=jsonpath='{.status.allocatable.nvidia\.com/gpu}'=1 \ + node -l agentpool=gpunp --timeout=600s +kubectl get pods -n gpu-operator -o wide +tr -d '\r' < Test-GpuAttestation.sh | + kubectl exec -i signed-driver-load -- bash | tee .local/gpu-attestation-before.log +kubectl apply -f gpu-smoke-test.yaml +kubectl wait --for=condition=Complete job/confidential-gpu-smoke --timeout=240s +kubectl logs job/confidential-gpu-smoke | tee .local/gpu-smoke.log +grep -Fx 'Test PASSED' .local/gpu-smoke.log +``` + +The nonprivileged Job requests and limits **one GPU**, uses runtime class +`nvidia`, and runs NVIDIA's vectorAdd. No CPU fallback is supplied. Helm +`deployed`, node Ready, or an optional monitoring operand's state is not a +substitute for these checks. + +## 5. Verify GPU and CPU/CVM attestation separately + +```bash +tr -d '\r' < Test-GpuAttestation.sh | + kubectl exec -i signed-driver-load -- bash | tee .local/gpu-attestation-after.log +tr -d '\r' < Test-CvmAttestation.sh | + kubectl exec -i signed-driver-load -- bash | tee .local/cvm-attestation-summary.log +``` + +Require zero pipeline exits and `GPU_ATTESTATION_VALIDATED=1` / +`CVM_ATTESTATION_VALIDATED=1`. The GPU script uses an isolated Python 3.12 venv, +checksum-pinned V4.5.1 verifier (`2.3.0`), strict OCSP, a fresh verifier nonce, +the exact positive overall verdict, and successful verification of the expected +physical GPU UUID. There is no `--test_no_gpu` or permissive revocation fallback. +The tested flow passed both before and after the CUDA workload. +The published scripts were replayed unchanged on a second fresh worker and +passed again. Optional DCGM exporter readiness remained `0/1`; this recipe +does not claim complete monitoring health. + +The CPU script separately makes a real MAA request using checksum-pinned Azure +guest CLI `v0.1.0`, and requires decoded `sevsnpvm` and +`azure-compliant-cvm` claims. It is not a second independent JWT verification +implementation. Host `/dev`, read-only `/sys/firmware`, and read-only +`/sys/kernel/security` expose the Azure vTPM evidence. Without the TPM event log, +MAA returned `TcgLogs is empty in attestation request`. Sysfs stat size can be +zero: the script checks readable bytes with `wc -c`, not `test -s`. + +Full results stay inside the pod at `/tmp/signed-attestation/gpu-result.log` +and `/tmp/signed-attestation/cpu-result.log`; they may include attestation +tokens. Console summaries omit the GPU token and print only selected CPU +claims. Keep any copied full evidence gitignored and access-controlled. +The GPU verifier reports future deprecation in favour of `nvattest`; migration +to that alternative is not validated by this recipe. + +## 6. Stop and verify deallocation + +Capture needed evidence before cleanup. On the successful probe, remove only +its four versioned firmware files, remove the test Job and pod, then stop +promptly (deleting the pod terminates persistence): + +```bash +kubectl delete job confidential-gpu-smoke --ignore-not-found +kubectl exec signed-driver-load -- bash -c 'set -eu +rm -- /lib/firmware/nvidia/615.71.09/gsp_ga10x.bin \ + /lib/firmware/nvidia/615.71.09/gsp_tu10x.bin \ + /lib/firmware/nvidia/615.71.09/ucodes_ga10x.bin \ + /lib/firmware/nvidia/615.71.09/ucodes_tu10x.bin +rmdir /lib/firmware/nvidia/615.71.09' +kubectl delete pod signed-driver-load +az aks stop --subscription "$subscription" \ + --resource-group "$resourceGroup" --name "$clusterName" +az aks show --subscription "$subscription" \ + --resource-group "$resourceGroup" --name "$clusterName" \ + --query '{power:powerState.code,state:provisioningState}' +az vmss list --subscription "$subscription" --resource-group "$nodeResourceGroup" \ + --query '[].{name:name,capacity:sku.capacity}' -o table +``` + +Require **Stopped / Succeeded** and **both VMSS capacities zero**. If an earlier +step failed, still stop the cluster; inspect which exact probe-owned files +exist before removing them. Do not attempt live recovery by loading mismatched +modules or disabling security. Retained disks/networking may still cost money. +GPU Operator is deliberately left with `driver.enabled=false`. Restarting +compute alone does not recreate the ephemeral driver installation; repeat +clean-node checks and setup. + +When inspection is finished, explicitly deleting this disposable resource +group removes the cluster and its managed resources: + +```bash +az group delete --subscription "$subscription" --name "$resourceGroup" --yes +``` diff --git a/aks-samples/aks-gpu-worker/Deploy.ps1 b/aks-samples/aks-gpu-worker/Deploy.ps1 new file mode 100644 index 0000000..3c1d8bf --- /dev/null +++ b/aks-samples/aks-gpu-worker/Deploy.ps1 @@ -0,0 +1,183 @@ +#Requires -Version 7.2 +[CmdletBinding()] +param( + [Parameter(Mandatory)][guid]$SubscriptionId, + [Parameter(Mandatory)][ValidatePattern('^[a-zA-Z0-9_-]+$')][string]$ResourceGroup, + [Parameter(Mandatory)][ValidatePattern('^[a-zA-Z0-9-]+$')][string]$ClusterName, + [string]$Location = 'westeurope', + [string]$KubernetesVersion = '1.36.5', + [string]$SystemVmSize = 'Standard_D4ds_v6', + [ValidateSet('Ubuntu2404')][string]$SystemOsSku = 'Ubuntu2404', + [string]$GpuZone, + [string]$OutputDirectory = (Join-Path $PSScriptRoot '.local'), + [switch]$PreflightOnly, + [switch]$RegisterPreview, + [switch]$UseCoreCli, + [switch]$Resume +) +. (Join-Path $PSScriptRoot 'Common.ps1') +$previousExtensionDirectory = $env:AZURE_EXTENSION_DIR +$previousDynamicInstall = $env:AZURE_EXTENSION_USE_DYNAMIC_INSTALL +try { +$az = Resolve-AzureCli +$scope = @('--subscription', $SubscriptionId.ToString(), '--only-show-errors') +New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null +$OutputDirectory = (Resolve-Path $OutputDirectory).Path +if ($UseCoreCli) { + $env:AZURE_EXTENSION_DIR = Join-Path $OutputDirectory 'cli-extensions-core' + New-Item -ItemType Directory -Path $env:AZURE_EXTENSION_DIR -Force | Out-Null + if (@(Get-ChildItem -LiteralPath $env:AZURE_EXTENSION_DIR -Force).Count) { + throw 'Core CLI extension directory must be empty; inspect it before continuing.' + } + $env:AZURE_EXTENSION_USE_DYNAMIC_INSTALL = 'no' +} + +$account = Invoke-CheckedCommand $az (@('account', 'show', '-o', 'json') + $scope) | ConvertFrom-Json +if ($account.state -ne 'Enabled') { throw 'The target subscription is not enabled.' } +if ($UseCoreCli) { + $cliVersion = (Invoke-CheckedCommand $az @('version', '-o', 'json') | ConvertFrom-Json).'azure-cli' + Write-Host "Using built-in Azure CLI $cliVersion AKS commands in $Location (extensions isolated)." +} +else { + $extension = Invoke-CheckedCommand $az @('extension', 'show', '--name', 'aks-preview', '-o', 'json') | ConvertFrom-Json + Write-Host "Using Azure CLI aks-preview $($extension.version) in $Location." +} +$versions = Invoke-CheckedCommand $az (@('aks', 'get-versions', '--location', $Location, '-o', 'json') + $scope) | ConvertFrom-Json +$patches = @($versions.values | ForEach-Object { $_.patchVersions.PSObject.Properties.Name }) +if ($KubernetesVersion -notin $patches) { throw "Kubernetes $KubernetesVersion is not offered in $Location." } + +$exists = Invoke-CheckedCommand $az (@('group', 'exists', '--name', $ResourceGroup, '-o', 'tsv') + $scope) +$existingPools = @() +$cluster = @() +if ($exists -eq 'true') { + if (-not $Resume) { throw 'Resource group already exists. Use a new name, or -Resume for this sample.' } + $group = Invoke-CheckedCommand $az (@('group', 'show', '--name', $ResourceGroup, '-o', 'json') + $scope) | ConvertFrom-Json -AsHashtable + if ($group.tags.sample -ne 'aks-gpu-worker' -or $group.location -ne $Location) { + throw 'Existing resource group is not a matching sample group; nothing was modified.' + } + $clusters = @(Invoke-CheckedCommand $az (@('aks', 'list', '--resource-group', $ResourceGroup, '-o', 'json') + $scope) | ConvertFrom-Json -AsHashtable) + if (@($clusters | Where-Object name -NE $ClusterName).Count) { throw 'Resource group contains a different cluster.' } + $cluster = @($clusters | Where-Object name -EQ $ClusterName) + if ($cluster.Count) { + if ($cluster[0].tags.sample -ne 'aks-gpu-worker' -or $cluster[0].location -ne $Location -or + $cluster[0].kubernetesVersion -ne $KubernetesVersion -or $cluster[0].provisioningState -ne 'Succeeded' -or + $cluster[0].autoUpgradeProfile.upgradeChannel -ne 'none' -or + $cluster[0].autoUpgradeProfile.nodeOsUpgradeChannel -ne 'None') { + throw 'Existing cluster is not a successful matching sample cluster; inspect it before continuing.' + } + $existingPools = @(Invoke-CheckedCommand $az (@('aks', 'nodepool', 'list', '--resource-group', $ResourceGroup, + '--cluster-name', $ClusterName, '-o', 'json') + $scope) | ConvertFrom-Json -AsHashtable) + Assert-SamplePools -Pools $existingPools -SystemVmSize $SystemVmSize -AllowMissingGpu + } +} + +$skus = @(Invoke-CheckedCommand $az (@('vm', 'list-skus', '--location', $Location, + '--resource-type', 'virtualMachines', '--all', '-o', 'json') + $scope) | ConvertFrom-Json) +$usage = @(Invoke-CheckedCommand $az (@('vm', 'list-usage', '--location', $Location, '-o', 'json') + $scope) | ConvertFrom-Json) +$required = @{} +$skuEvidence = @(foreach ($size in @($SystemVmSize, 'Standard_NCC40ads_H100_v5')) { + $sku = @($skus | Where-Object name -EQ $size) + if ($sku.Count -ne 1) { throw "SKU $size was not found in $Location." } + $sku = $sku[0] + $regionBlocks = @($sku.restrictions | Where-Object type -EQ 'Location') + if ($regionBlocks.Count) { throw "SKU $size has a regional subscription restriction: $($regionBlocks | ConvertTo-Json -Depth 10 -Compress)" } + if ($size -eq 'Standard_NCC40ads_H100_v5' -and $GpuZone) { + if ($GpuZone -notin @($sku.locationInfo | ForEach-Object zones)) { throw "GPU zone $GpuZone is not offered." } + $zoneBlocks = @($sku.restrictions | Where-Object { + $_.type -eq 'Zone' -and $GpuZone -in $_.restrictionInfo.zones + }) + if ($zoneBlocks.Count) { throw "GPU zone $GpuZone is restricted for this subscription." } + } + $cores = [int]($sku.capabilities | Where-Object name -EQ 'vCPUs').value + if (-not $required.ContainsKey($sku.family)) { $required[$sku.family] = 0 } + $poolName = if ($size -eq 'Standard_NCC40ads_H100_v5') { 'gpunp' } else { 'sysnp' } + if (-not @($existingPools | Where-Object name -EQ $poolName).Count) { + $required[$sku.family] += $cores + } + $sku +}) +$total = 0 +foreach ($family in $required.Keys) { + $quota = @($usage | Where-Object { $_.name.value -eq $family }) + if ($quota.Count -ne 1 -or ([int]$quota[0].limit - [int]$quota[0].currentValue) -lt $required[$family]) { + throw "Insufficient $family quota for $($required[$family]) new vCPUs." + } + $total += $required[$family] +} +$regional = $usage | Where-Object { $_.name.value -eq 'cores' } +if (([int]$regional.limit - [int]$regional.currentValue) -lt $total) { throw "Insufficient regional vCPU quota for $total cores." } + +$featureArgs = @('feature', 'show', '--namespace', 'Microsoft.ContainerService', '--name', 'Ubuntu2404Preview') +$feature = Invoke-CheckedCommand $az ($featureArgs + @('--query', 'properties.state', '-o', 'tsv') + $scope) +if ($feature -ne 'Registered' -and $RegisterPreview -and -not $PreflightOnly) { + Invoke-CheckedCommand $az (@('feature', 'register', '--namespace', 'Microsoft.ContainerService', + '--name', 'Ubuntu2404Preview', '-o', 'none') + $scope) + $deadline = [DateTime]::UtcNow.AddMinutes(20) + do { + Start-Sleep -Seconds 20 + $feature = Invoke-CheckedCommand $az ($featureArgs + @('--query', 'properties.state', '-o', 'tsv') + $scope) + } while ($feature -ne 'Registered' -and [DateTime]::UtcNow -lt $deadline) + if ($feature -eq 'Registered') { + Invoke-CheckedCommand $az (@('provider', 'register', '--namespace', 'Microsoft.ContainerService', + '--wait', '-o', 'none') + $scope) + } +} +$preflight = [ordered]@{ + CheckedAtUtc = [DateTime]::UtcNow.ToString('o') + Location = $Location + KubernetesVersion = $KubernetesVersion + PreviewState = "$feature" + Skus = $skuEvidence + Quota = @($usage | Where-Object { $_.name.value -in @($required.Keys) + @('cores') }) +} +$preflight | ConvertTo-Json -Depth 20 | Set-Content (Join-Path $OutputDirectory 'preflight.json') +if ($feature -ne 'Registered') { throw "Ubuntu2404Preview is $feature. Rerun with -RegisterPreview or arrange subscription registration." } +if ($PreflightOnly) { Write-Host 'Preflight passed; no Azure resources were changed.'; return } + +if ($exists -ne 'true') { + Invoke-CheckedCommand $az (@('group', 'create', '--name', $ResourceGroup, '--location', $Location, + '--tags', 'sample=aks-gpu-worker', '-o', 'none') + $scope) +} +if ($cluster.Count -eq 0) { + Invoke-CheckedCommand $az (@('aks', 'create', '--resource-group', $ResourceGroup, '--name', $ClusterName, + '--location', $Location, '--kubernetes-version', $KubernetesVersion, '--tier', 'free', + '--nodepool-name', 'sysnp', '--node-count', '1', '--node-vm-size', $SystemVmSize, + '--os-sku', $SystemOsSku, '--node-osdisk-type', 'Managed', '--node-osdisk-size', '128', + '--enable-managed-identity', '--generate-ssh-keys', '--auto-upgrade-channel', 'none', + '--node-os-upgrade-channel', 'None', '--tags', 'sample=aks-gpu-worker', '-o', 'none') + $scope) +} +$pools = @(Invoke-CheckedCommand $az (@('aks', 'nodepool', 'list', '--resource-group', $ResourceGroup, + '--cluster-name', $ClusterName, '-o', 'json') + $scope) | ConvertFrom-Json -AsHashtable) +Assert-SamplePools -Pools $pools -SystemVmSize $SystemVmSize -AllowMissingGpu +$gpu = @($pools | Where-Object name -EQ 'gpunp') +if ($gpu.Count -eq 0) { + $arguments = @('aks', 'nodepool', 'add', '--resource-group', $ResourceGroup, '--cluster-name', $ClusterName, + '--name', 'gpunp', '--mode', 'User', '--node-count', '1', '--node-vm-size', 'Standard_NCC40ads_H100_v5', + '--os-sku', 'Ubuntu2404', '--gpu-driver', 'None', '--node-osdisk-type', 'Managed', + '--node-osdisk-size', '128', '--node-taints', 'sku=gpu:NoSchedule', '-o', 'none') + if ($GpuZone) { $arguments += @('--zones', $GpuZone) } + Invoke-CheckedCommand $az ($arguments + $scope) +} +$pools = Invoke-CheckedCommand $az (@('aks', 'nodepool', 'list', '--resource-group', $ResourceGroup, + '--cluster-name', $ClusterName, '-o', 'json') + $scope) +$pools | Set-Content (Join-Path $OutputDirectory 'nodepools.json') +Assert-SamplePools -Pools @($pools | ConvertFrom-Json -AsHashtable) -SystemVmSize $SystemVmSize +$nodeGroup = Invoke-CheckedCommand $az (@('aks', 'show', '--resource-group', $ResourceGroup, + '--name', $ClusterName, '--query', 'nodeResourceGroup', '-o', 'tsv') + $scope) +$scaleSets = Invoke-CheckedCommand $az (@('vmss', 'list', '--resource-group', "$nodeGroup", '-o', 'json') + $scope) +$scaleSets | Set-Content (Join-Path $OutputDirectory 'scalesets.json') +Assert-ConfidentialGpuScaleSet -ScaleSets @($scaleSets | ConvertFrom-Json -AsHashtable) +$kubeconfig = Join-Path $OutputDirectory 'kubeconfig' +Invoke-CheckedCommand $az (@('aks', 'get-credentials', '--resource-group', $ResourceGroup, + '--name', $ClusterName, '--file', $kubeconfig, '--overwrite-existing') + $scope) +Invoke-CheckedCommand kubectl @('--kubeconfig', $kubeconfig, 'wait', '--for=condition=Ready', + 'nodes', '-l', 'agentpool=gpunp', '--timeout=600s') +$node = Get-GpuNode -KubeConfig $kubeconfig +$node | ConvertTo-Json -Depth 30 | Set-Content (Join-Path $OutputDirectory 'gpu-node.json') +& (Join-Path $PSScriptRoot 'Test-DriverCompatibility.ps1') -KubeConfig $kubeconfig ` + -OutputPath (Join-Path $OutputDirectory 'driver-compatibility.json') -RequireAvailable +} +finally { + $env:AZURE_EXTENSION_DIR = $previousExtensionDirectory + $env:AZURE_EXTENSION_USE_DYNAMIC_INSTALL = $previousDynamicInstall +} diff --git a/aks-samples/aks-gpu-worker/Install-SignedDriver.sh b/aks-samples/aks-gpu-worker/Install-SignedDriver.sh new file mode 100644 index 0000000..a6340fa --- /dev/null +++ b/aks-samples/aks-gpu-worker/Install-SignedDriver.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -euo pipefail +trap 'echo "ERROR: signed-driver probe failed at line $LINENO; inspect the node and stop the disposable cluster before retrying." >&2' ERR + +kernel=6.8.0-1071-azure-fde +version=615.71.09 +test "$(uname -r)" = "$kernel" +test ! -d /sys/module/nvidia +test ! -e /tmp/signed-driver +test ! -e "/lib/firmware/nvidia/$version" +test -z "$(ls -A /run/nvidia/driver)" + +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq +apt-get install -y --no-install-recommends ca-certificates kmod mokutil libpciaccess0 +mokutil --sb-state | grep -Fx 'SecureBoot enabled' +mkdir /tmp/signed-driver +cd /tmp/signed-driver +apt-get download "linux-modules-nvidia-615-open-$kernel=6.8.0-1071.78" \ + "nvidia-firmware-615-$version=$version-0ubuntu0.24.04.2" \ + "nvidia-utils-615=$version-0ubuntu0.24.04.2" \ + "libnvidia-compute-615=$version-0ubuntu0.24.04.2" \ + "nvidia-compute-utils-615=$version-0ubuntu0.24.04.2" \ + "libnvidia-cfg1-615=$version-0ubuntu0.24.04.2" +echo "8b3ef898a5a32d08f748f197472a65903bd09a303e00f0bf94435430cc4437e0 linux-modules-nvidia-615-open-${kernel}_6.8.0-1071.78_amd64.deb" | + sha256sum -c - +for deb in ./*.deb; do dpkg-deb -x "$deb" root; done + +module_dir="$PWD/root/lib/modules/$kernel/kernel/nvidia-615-open" +for module in nvidia nvidia-uvm nvidia-modeset nvidia-drm nvidia-peermem; do + test "$(modinfo -F signer "$module_dir/$module.ko")" = 'Canonical Ltd. Kernel Module Signing' + test "$(modinfo -F version "$module_dir/$module.ko")" = "$version" + test "$(modinfo -F vermagic "$module_dir/$module.ko" | cut -d' ' -f1)" = "$kernel" +done +mkdir -p "/lib/firmware/nvidia/$version" +cp -a "root/lib/firmware/nvidia/$version/." "/lib/firmware/nvidia/$version/" +modprobe ecdsa_generic +modprobe ecdh +modprobe ecc +insmod "$module_dir/nvidia.ko" +insmod "$module_dir/nvidia-uvm.ko" + +export LD_LIBRARY_PATH="$PWD/root/usr/lib/x86_64-linux-gnu" +root/usr/bin/nvidia-persistenced --user root --uvm-persistence-mode --verbose \ + --nvidia-cfg-path="$LD_LIBRARY_PATH" +sleep 5 +test -s /var/run/nvidia-persistenced/nvidia-persistenced.pid +kill -0 "$(cat /var/run/nvidia-persistenced/nvidia-persistenced.pid)" +root/usr/bin/nvidia-smi conf-compute -f | grep -Fx 'CC status: ON' +root/usr/bin/nvidia-smi conf-compute -e | grep -Fx 'CC Environment: PRODUCTION' +root/usr/bin/nvidia-smi conf-compute -srs 1 +for attempt in 1 2 3; do + root/usr/bin/nvidia-smi --query-gpu=uuid,persistence_mode --format=csv,noheader | + grep -E '^GPU-[[:xdigit:]-]+, Enabled$' + root/usr/bin/nvidia-smi conf-compute -q | + grep -E 'CC GPUs Ready State[[:space:]]*: Ready$' + sleep 5 +done + +cp -a root/. /run/nvidia/driver/ +echo SIGNED_DRIVER_LOADED=1 +exit 0 diff --git a/aks-samples/aks-gpu-worker/README.md b/aks-samples/aks-gpu-worker/README.md new file mode 100644 index 0000000..517db68 --- /dev/null +++ b/aks-samples/aks-gpu-worker/README.md @@ -0,0 +1,821 @@ +# AKS confidential H100 worker + +## Current workaround - UNSUPPORTED + +> **UNSUPPORTED - experimental diagnostic workaround only.** +> This recipe is not a Microsoft/NVIDIA-supported AKS configuration or a +> production-managed driver installation. Successful workload and attestation +> tests do not establish vendor support. It requires a privileged pod with +> host access and does not survive node lifecycle changes automatically. + +**Tested end to end on two fresh workers on 2026-11-10.** Exact-kernel, +Canonical-signed Ubuntu NVIDIA `615.71.09` packages loaded on +`6.8.0-1071-azure-fde` with Secure Boot enabled. Starting UVM persistence +**before the first GPU query**, then exposing the extracted driver tree to GPU +Operator with its driver installer disabled, produced one allocatable H100. +A GPU-only CUDA vector-add Job passed, strict GPU attestation passed before +and after that workload, and a separate Azure CPU/CVM attestation returned +`sevsnpvm` and `azure-compliant-cvm` claims. + +### Use this path, not the failed driver-container recipes + +Follow the [tested Azure CLI walkthrough](Confidential-GPU-H100-AKS-Onboarding.md) +for the complete commands. The working sequence is: + +1. Provision the disposable system pool and confidential H100 worker with + automatic GPU driver installation disabled. Verify the actual host kernel, + ConfidentialVM, Secure Boot and vTPM before proceeding. +2. Install GPU Operator `v26.7.2` with **`driver.enabled=false`**. Keep the + toolkit/runtime and device plugin; do not use its failing driver installer. +3. Apply [the privileged probe](signed-driver-probe.yaml) on a fresh worker and + run [the exact-package installer](Install-SignedDriver.sh). It extracts + Canonical-signed NVIDIA `615.71.09` modules for + `6.8.0-1071-azure-fde` plus matching firmware and libraries, without changing + the host kernel, disabling Secure Boot or replacing AKS-managed containerd. +4. The installer preloads the crypto dependencies and starts + `nvidia-persistenced --uvm-persistence-mode` **before the first GPU query**, + sets GPU ready state, and exposes the driver tree at `/run/nvidia/driver`. + Require `SIGNED_DRIVER_LOADED=1`, CC ON / PRODUCTION / Ready and one + allocatable `nvidia.com/gpu`. +5. Run [strict GPU attestation](Test-GpuAttestation.sh), then + [the one-GPU CUDA Job](gpu-smoke-test.yaml), then GPU attestation again. + Require zero exits, positive verdicts for the actual GPU UUID, and + `Test PASSED`; no CPU fallback is supplied. +6. Run [separate CPU/CVM attestation](Test-CvmAttestation.sh). The probe mounts + host securityfs read-only for TPM evidence; require the successful MAA + request and `sevsnpvm` / `azure-compliant-cvm` claims. +7. Preserve evidence locally, remove only the probe-owned firmware and test + resources, and stop the cluster using the walkthrough's cleanup commands. + +**Limits:** the probe holds persistence for at most four hours. Deleting it ends +persistence; `/run/nvidia/driver` is ephemeral. Reboots, replacement nodes and +image/kernel upgrades require fresh validation and setup. Do not apply these +exact packages to another kernel. Optional DCGM exporter readiness remained +`0/1`, and the workload/probe persistence displays differed; neither issue is +claimed resolved. Full attestation logs may contain tokens: keep them local and +gitignored. + +The original missing registry tags and generic-driver failure remain +reproducible. [Appendix: everything tried](#appendix-everything-tried) preserves +the failed attempts, alternatives investigated, fixes and verification evidence. + +## Implementation steps - UNSUPPORTED + +The steps below follow the [source guide's section order](https://github.com/Azure/az-cgpu-onboarding/blob/136262ef00ad75b8e0bc789563f043006fb68f4c/docs/Confidential-GPU-H100-AKS-Onboarding.md): +CLI configuration, cluster creation, setup validation, driver installation, +confidential mode, workload and attestation. They replace only the parts that +failed in this reproduction. **This entire procedure is unsupported**, including +the exact OS/kernel/driver combination that passed the tests. + +### 1. Prerequisites and Helm installation + +Use Bash on Linux/WSL with Azure CLI (tested: 2.92.0), Helm and kubectl already +installed. Run all blocks in the **same Bash session from this sample directory**, +with its supplied manifests and scripts present. PowerShell users can use the +alternative provisioning commands below, but must not paste Bash blocks into +PowerShell. + +Use a disposable cluster, permission to provision AKS, 40 available +`StandardNCCads2023Family` vCPUs plus four system-node vCPUs, and sufficient West +Europe allocation capacity. Allow HTTPS egress to Azure, Ubuntu repositories, +GitHub, NGC and the verifier's certificate/RIM/OCSP services. Do not run this +privileged diagnostic on shared production nodes. + +```bash +set -euo pipefail +az version +helm version --short +kubectl version --client +``` + +### 2. Azcli configurations + +Use an explicitly authorized subscription and a new resource group name. +Keep the sample-local extension directory empty: the tested core CLI avoids the +observed `aks-preview` incompatibility without changing shared extensions. +Keep the dedicated kubeconfig rather than overwriting the workstation default. + +```bash +subscription='' +location=westeurope +resourceGroup=rg-aks-gpu-worker-example +clusterName=aks-gpu-worker-example +mkdir -p .local/cli-extensions-core +export AZURE_EXTENSION_DIR="$PWD/.local/cli-extensions-core" +export AZURE_EXTENSION_USE_DYNAMIC_INSTALL=no +export KUBECONFIG="$PWD/.local/kubeconfig" +az login +az account show --subscription "$subscription" --query '{name:name,state:state}' +az aks get-versions --subscription "$subscription" --location "$location" -o table +az feature register --subscription "$subscription" \ + --namespace Microsoft.ContainerService --name Ubuntu2404Preview +az feature show --subscription "$subscription" \ + --namespace Microsoft.ContainerService --name Ubuntu2404Preview \ + --query properties.state -o tsv +``` + +Registration is asynchronous. Wait until the feature reports `Registered`, +rerunning the show command as needed, then propagate registration: + +```bash +test "$(az feature show --subscription "$subscription" \ + --namespace Microsoft.ContainerService --name Ubuntu2404Preview \ + --query properties.state -o tsv)" = Registered +az provider register --subscription "$subscription" \ + --namespace Microsoft.ContainerService +``` + +### 3. Create AKS cluster with CGPU nodepool + +These commands create **one system node plus one confidential GPU worker** and +incur charges. Both pools use Ubuntu2404; autoscaling is not enabled. Disabling +automatic upgrades is only for this short-lived controlled experiment. + +```bash +az group create --subscription "$subscription" --name "$resourceGroup" \ + --location "$location" --tags sample=aks-gpu-worker +az aks create --subscription "$subscription" \ + --resource-group "$resourceGroup" --name "$clusterName" \ + --location "$location" --kubernetes-version 1.36.5 --tier free \ + --nodepool-name sysnp --node-count 1 --node-vm-size Standard_D4ds_v6 \ + --os-sku Ubuntu2404 --node-osdisk-type Managed --node-osdisk-size 128 \ + --enable-managed-identity --generate-ssh-keys \ + --auto-upgrade-channel none --node-os-upgrade-channel None \ + --tags sample=aks-gpu-worker +az aks get-credentials --subscription "$subscription" \ + --resource-group "$resourceGroup" --name "$clusterName" \ + --file "$KUBECONFIG" --overwrite-existing +az aks nodepool add --subscription "$subscription" \ + --resource-group "$resourceGroup" --cluster-name "$clusterName" \ + --name gpunp --mode User --node-count 1 \ + --node-vm-size Standard_NCC40ads_H100_v5 --os-sku Ubuntu2404 \ + --gpu-driver None --node-osdisk-type Managed --node-osdisk-size 128 \ + --node-taints sku=gpu:NoSchedule +``` + +For the retained stopped sample, skip creation, set the variables to its verified +identity, and use `az aks start` instead. Starting it does not restore the +ephemeral driver installation. Do not create additional pools on an unrelated +existing cluster. + +### 4. Setup validation + +```bash +kubectl wait --for=condition=Ready node --all --timeout=600s +kubectl get nodes -o wide +test "$(kubectl get nodes -o name | wc -l)" -eq 2 +test "$(kubectl get nodes -l agentpool=gpunp -o name | wc -l)" -eq 1 +gpu_node_name=$(kubectl get nodes -l agentpool=gpunp \ + -o 'jsonpath={.items[0].metadata.name}') +kernel_version=$(kubectl get node "$gpu_node_name" \ + -o 'jsonpath={.status.nodeInfo.kernelVersion}') +test "$kernel_version" = 6.8.0-1071-azure-fde +az aks nodepool list --subscription "$subscription" \ + --resource-group "$resourceGroup" --cluster-name "$clusterName" \ + --query '[].{Name:name,NodeImageVersion:nodeImageVersion}' -o table +nodeResourceGroup=$(az aks show --subscription "$subscription" \ + --resource-group "$resourceGroup" --name "$clusterName" \ + --query nodeResourceGroup -o tsv) +az vmss list --subscription "$subscription" --resource-group "$nodeResourceGroup" \ + --query "[?sku.name=='Standard_NCC40ads_H100_v5'].{name:name,count:sku.capacity,security:virtualMachineProfile.securityProfile}" +test "$(az vmss list --subscription "$subscription" --resource-group "$nodeResourceGroup" \ + --query "[?sku.name=='Standard_NCC40ads_H100_v5' && sku.capacity==\`1\` && virtualMachineProfile.securityProfile.securityType=='ConfidentialVM' && virtualMachineProfile.securityProfile.uefiSettings.secureBootEnabled==\`true\` && virtualMachineProfile.securityProfile.uefiSettings.vTpmEnabled==\`true\`] | length(@)" -o tsv)" = 1 +``` + +The tested worker reported: + +```text +OS-IMAGE: Ubuntu 24.04.5 LTS +KERNEL-VERSION: 6.8.0-1071-azure-fde +NODE-IMAGE: AKSUbuntu-2404gen2CVMcontainerd-202610.30.1 +``` + +These are observations, **not an AKS image pin or supported combination**. +Stop if the kernel or effective ConfidentialVM/Secure Boot/vTPM checks differ. +Do not substitute 1041 modules, replace the host kernel or disable security. + +### 5. Driver installation + +Unlike the source guide, disable the GPU Operator driver operand. Use its toolkit +and device plugin with the separately extracted signed driver tree. + +```bash +helm upgrade --install gpu-operator gpu-operator \ + --repo https://helm.ngc.nvidia.com/nvidia --version v26.7.2 \ + --namespace gpu-operator --create-namespace \ + --values gpu-operator-experimental-values.yaml --set driver.enabled=false +kubectl apply -f signed-driver-probe.yaml +kubectl wait --for=condition=Ready pod/signed-driver-load --timeout=300s +tr -d '\r' < Install-SignedDriver.sh | + kubectl exec -i signed-driver-load -- bash | tee .local/signed-driver.log +grep -Fx 'SIGNED_DRIVER_LOADED=1' .local/signed-driver.log +``` + +The [installer](Install-SignedDriver.sh) verifies Secure Boot, exact kernel, +module-deb checksum, and five module signers/versions/vermagic values. It extracts +`linux-modules-nvidia-615-open-6.8.0-1071-azure-fde=6.8.0-1071.78` and coherent +`615.71.09` firmware/libraries, loads NVIDIA/UVM, starts UVM persistence **before +querying the GPU**, sets readiness, then copies the tree into `/run/nvidia/driver`. +It does not install a moving kernel metapackage or replace AKS containerd. + +Keep the probe alive. It refuses already-loaded or partially populated nodes: +after failure, inspect and recycle the disposable node rather than repeatedly +unloading/reloading modules or deleting/recreating the persistence pod. + +### 6. Enable confidential GPU mode and verify readiness + +The installer already sets ready state after persistence starts. Confirm CC is +ON and PRODUCTION rather than trying to bypass a failed confidential-mode check: + +```bash +kubectl exec signed-driver-load -- bash -c 'set -euo pipefail +export LD_LIBRARY_PATH=/tmp/signed-driver/root/usr/lib/x86_64-linux-gnu +smi=/tmp/signed-driver/root/usr/bin/nvidia-smi +kill -0 "$(cat /var/run/nvidia-persistenced/nvidia-persistenced.pid)" +"$smi" conf-compute -f | grep -Fx "CC status: ON" +"$smi" conf-compute -e | grep -Fx "CC Environment: PRODUCTION" +"$smi" conf-compute -q | grep -E "CC GPUs Ready State[[:space:]]*: Ready$"' +kubectl wait --for=jsonpath='{.status.allocatable.nvidia\.com/gpu}'=1 \ + node -l agentpool=gpunp --timeout=600s +kubectl get pods -n gpu-operator -o wide +``` + +Require one allocatable GPU. Node Ready and Helm `deployed` alone are insufficient. +The Operator CUDA validator completed during the tested runs; optional DCGM +exporter readiness remained `0/1`. + +### 7. Sample workload + +First attest the GPU so the workload is bracketed by successful checks: + +```bash +tr -d '\r' < Test-GpuAttestation.sh | + kubectl exec -i signed-driver-load -- bash | tee .local/gpu-attestation-before.log +grep -Fx 'GPU_ATTESTATION_VALIDATED=1' .local/gpu-attestation-before.log +kubectl apply -f gpu-smoke-test.yaml +kubectl wait --for=condition=Complete job/confidential-gpu-smoke --timeout=240s +kubectl logs job/confidential-gpu-smoke | tee .local/gpu-smoke.log +grep -Fx 'Test PASSED' .local/gpu-smoke.log +``` + +The [nonprivileged Job](gpu-smoke-test.yaml) requests and limits one GPU, uses +runtime class `nvidia`, and validates CUDA vector addition of 50,000 elements. +No CPU fallback is supplied. + +### 8. GPU attestation and separate CPU/CVM attestation + +```bash +tr -d '\r' < Test-GpuAttestation.sh | + kubectl exec -i signed-driver-load -- bash | tee .local/gpu-attestation-after.log +grep -Fx 'GPU_ATTESTATION_VALIDATED=1' .local/gpu-attestation-after.log +tr -d '\r' < Test-CvmAttestation.sh | + kubectl exec -i signed-driver-load -- bash | tee .local/cvm-attestation-summary.log +grep -Fx 'CVM_ATTESTATION_VALIDATED=1' .local/cvm-attestation-summary.log +``` + +Require zero pipeline exits throughout. [GPU verification](Test-GpuAttestation.sh) +uses checksum-pinned V4.5.1 verifier `2.3.0`, Python 3.12, strict OCSP and a fresh +nonce; it requires `GPU Attestation is Successful.` and verification of the +actual GPU UUID. [CVM verification](Test-CvmAttestation.sh) independently makes +the MAA request and requires returned `sevsnpvm` / `azure-compliant-cvm` claims, +not an additional independent JWT signature validator. + +The probe's read-only securityfs mount supplies the TPM event log needed by MAA. +Keep full logs in the pod or ignored, access-controlled local storage: they may +contain tokens. Preserve needed evidence before cleanup; do not commit it. + +### 9. Troubleshooting and cleanup + +Do not continue after a failed gate. Use the +[investigation appendix](#appendix-everything-tried) for the observed failure +signatures. After a successful run, remove only the four probe-owned firmware +files and test resources, then promptly stop the cluster: + +```bash +kubectl delete job confidential-gpu-smoke --ignore-not-found +kubectl exec signed-driver-load -- bash -c 'set -eu +rm -- /lib/firmware/nvidia/615.71.09/gsp_ga10x.bin \ + /lib/firmware/nvidia/615.71.09/gsp_tu10x.bin \ + /lib/firmware/nvidia/615.71.09/ucodes_ga10x.bin \ + /lib/firmware/nvidia/615.71.09/ucodes_tu10x.bin +rmdir /lib/firmware/nvidia/615.71.09' +kubectl delete pod signed-driver-load +az aks stop --subscription "$subscription" \ + --resource-group "$resourceGroup" --name "$clusterName" +az aks show --subscription "$subscription" \ + --resource-group "$resourceGroup" --name "$clusterName" \ + --query '{power:powerState.code,state:provisioningState}' +az vmss list --subscription "$subscription" --resource-group "$nodeResourceGroup" \ + --query '[].{name:name,capacity:sku.capacity}' -o table +``` + +Require **Stopped / Succeeded** and **both VMSS capacities zero**. If setup failed, +still run `az aks stop`; inspect which exact probe-owned files exist before +cleaning them rather than allowing a cleanup failure to leave compute running. +Stopping retains potentially billable disks/networking. Explicit group deletion +is described in [Cleanup](#cleanup). + +## Deployment plan and scope + +Create a disposable development cluster in West Europe with Kubernetes 1.36.5, +one Ubuntu 24.04 system node (`Standard_D4ds_v6`), and exactly one Ubuntu 24.04 +confidential GPU user node (`Standard_NCC40ads_H100_v5`). AKS requires a system +pool: this is **two nodes total**, not a one-node cluster. Disable automatic +driver installation, autoscaling, and automatic upgrades during the experiment. +Use the existing signed-in Azure subscription only when explicitly supplied. +Do not overwrite other clusters, change the workstation's default subscription +or kubeconfig, downgrade AKS node images, or bypass confidential-computing checks. + +The Azure CLI recipe follows the +[onboarding guide](https://github.com/Azure/az-cgpu-onboarding/blob/136262ef00ad75b8e0bc789563f043006fb68f4c/docs/Confidential-GPU-H100-AKS-Onboarding.md). +The scripts are PowerShell 7 equivalents with quota checks, bounded waits, +native-command error handling, and registry compatibility gates. +This is not a production architecture or a claim of a supported new OS/driver combination. +Unlike the guide's Ubuntu 22.04 system pool, both pools use Ubuntu 24.04 because +the AKS documentation warns against mixed OS versions with GPU Operator. + +## PowerShell alternative prerequisites + +- PowerShell 7.2+, Azure CLI, `kubectl`, and Helm for subsequent GPU Operator setup. + The guide uses `aks-preview`; alternatively, `-UseCoreCli` uses built-in AKS + commands (verified with Azure CLI 2.92.0) without loading installed extensions. +- `az login`, permissions to create AKS and its managed resources, and an enabled + subscription with regional and VM-family quota. One H100 worker needs 40 + `StandardNCCads2023Family` vCPUs, plus four system-node vCPUs. +- `Ubuntu2404Preview` registered in `Microsoft.ContainerService` as required by + the linked guide. Registration may need subscription-owner assistance. +- HTTPS access to Azure, GitHub, NVIDIA NGC, and the Kubernetes API. + +Azure CLI on Windows is also detected at its standard installation path when +the current terminal's PATH has not been refreshed. + +## PowerShell alternative provisioning + +Run from this directory. The supplied resource group must be new unless resuming +this sample's deployment. Resource creation incurs charges. + +```powershell +$subscription = '' +$resourceGroup = 'rg-aks-gpu-worker-20261110' +$cluster = 'aks-gpu-worker-20261110' + +.\Deploy.ps1 -SubscriptionId $subscription -ResourceGroup $resourceGroup ` + -ClusterName $cluster -UseCoreCli -PreflightOnly + +# Registers the documented preview if necessary, with a 20-minute timeout. +.\Deploy.ps1 -SubscriptionId $subscription -ResourceGroup $resourceGroup ` + -ClusterName $cluster -UseCoreCli -RegisterPreview +``` + +For a partial deployment, inspect the resources and rerun with `-Resume`. +Keep the same system-node SKU and `-UseCoreCli` selection when resuming. +Resume validates sample ownership and existing pool configuration before cloud +mutations, and quota checks count only pools still to be created. +Use `-GpuZone` only with a zone reported as available to the subscription. +Quota and SKU listings do not guarantee physical allocation capacity. +No alternate subscription, region, VM family, or kernel is silently substituted. + +The deployment saves local evidence and a dedicated kubeconfig in `.local/` +(gitignored). Do not commit credentials or raw cloud/Kubernetes dumps. +The development cluster follows the guide's authentication model; protect its +API access and kubeconfig. Use Entra RBAC, restricted API access and a resilient +system pool for production. Automatic upgrades are disabled only for this +short-lived experiment, not as a long-term security policy. + +## Exact-kernel container compatibility gate + +This gate diagnoses the original **container-image path**. It is expected to +fail on the tested 1071 kernel and is not an acceptance gate for the unsupported +signed-package workaround above. A successful signed-package run does not +change the missing-image result. + +```powershell +.\Test-DriverCompatibility.ps1 -KubeConfig .\.local\kubeconfig -RequireAvailable +``` + +The worker's **actual** kernel determines the image tag. Selecting Kubernetes +1.36.5 and `Ubuntu2404` does not select an AKS node-image build or kernel. +An unavailable exact-kernel image is an external blocker, not a reason to use +1041 modules on a 1071 kernel. Do not disable Secure Boot, change the host kernel, +or install unsigned modules to get past it. + +The script also verifies the underlying H100 scale set has exactly one instance, +`securityType: ConfidentialVM`, Secure Boot enabled and vTPM enabled. Inspect the +VM scale-set security profile rather than assuming the AKS pool's generic +security flags describe the confidential VM's effective settings. + +## Resource status + +The retained deployment is named `aks-gpu-worker-20261110` in +`rg-aks-gpu-worker-20261110`, West Europe. Resources are retained for inspection. +After the final successful published-script replay, the probe-owned firmware +files, test Job and probe pod were removed. `az aks stop` succeeded: the cluster +reported **Stopped / Succeeded**, and both node scale sets were verified at +capacity **zero**. Full attestation evidence was preserved only in ignored +local storage before the probe was removed. +Stopping AKS deallocates compute but retained disks, networking and other +resources can still incur charges; stopping is not deletion. + +## Local regression checks + +```powershell +.\tests\Test-Sample.ps1 +``` + +These dependency-free tests mock registry and node responses. They check +authentication and OCI/Docker Accept headers, missing tags versus other HTTP +failures, digest validation, compatibility-report shape, the exact pool topology, +effective confidential VM security, and native-process failures. They do not +establish cloud capacity or GPU support. + +## Cleanup + +Resources are deliberately retained for inspection, including after failure. +Review the group and delete it explicitly when finished; this removes the +cluster and its managed node resources and stops their ongoing charges. + +```powershell +az group show --subscription $subscription --name $resourceGroup +az group delete --subscription $subscription --name $resourceGroup --yes +``` + +## References + +- [Confidential GPU AKS onboarding](https://github.com/Azure/az-cgpu-onboarding/blob/136262ef00ad75b8e0bc789563f043006fb68f4c/docs/Confidential-GPU-H100-AKS-Onboarding.md) +- [AKS confidential GPU documentation](https://learn.microsoft.com/azure/aks/use-confidential-gpus) +- [AKS node image upgrades](https://learn.microsoft.com/azure/aks/node-image-upgrade) + +## Appendix: everything tried + +This appendix retains the investigation history, including unsuccessful recipes. +It is **not the current installation procedure**. Use the unsupported workaround +at the top and its linked walkthrough for the tested sequence. + +### Registry-failure reproduction without an Azure cluster + +```powershell +.\Test-DriverCompatibility.ps1 -KernelVersion '6.8.0-1071-azure-fde' +.\Test-DriverCompatibility.ps1 -KernelVersion '6.8.0-1041-azure-fde' +.\Test-DriverCompatibility.ps1 -KernelVersion '6.8.0-1063-azure-fde' +``` + +The JSON distinguishes missing manifests from authentication/network failures. +Add `-RequireAvailable` to fail when neither exact-kernel image exists. +`SupportValidated: false` is intentional: a published image alone does not prove +Microsoft/NVIDIA support, module loading, confidential mode, or attestation. + +### Initial blockers and investigation results + +Live checks on 2026-11-10: + +| Check | Result | Interpretation / action | +|---|---|---| +| West Europe H100 SKU | Listed without subscription restrictions; zone 3 advertised; one worker allocated successfully without an explicit zone | Capacity was sufficient for this attempt, not a future guarantee. | +| H100 quota | 40 available family vCPUs; 140 available regional vCPUs | Sufficient for one worker and the system node. | +| Kubernetes 1.36.5 | Offered | The requested API version can be selected; this does not pin the kernel. | +| System node `Standard_D4ds_v5` | `NotAvailableForSubscription` in West Europe | Changed only the system node to unrestricted `Standard_D4ds_v6`, with 100 family vCPUs available. | +| `Ubuntu2404Preview` | Initially `NotRegistered`; registration succeeded | Registered before cluster creation. | +| Azure CLI 2.92.0 with `aks-preview` 18.0.0b17 | `ManagedCluster` has no attribute `hosted_system_profile` | Client-side failure; retry with `-UseCoreCli`, an empty sample-local extension directory, and dynamic extension installation disabled. The shared extension installation is unchanged. | +| AKS cluster and node pools | Provisioned; exactly one system node and one H100 user node, both Kubernetes `v1.36.5` and Ready | The control-plane/system-pool and H100 allocation stages succeeded. Node Ready is not GPU readiness. | +| Actual H100 node image | `AKSUbuntu-2404gen2CVMcontainerd-202610.30.1` | Matches the reported image build. | +| Actual worker OS / kernel / runtime | Ubuntu `24.04.5 LTS` / `6.8.0-1071-azure-fde` / `containerd://2.3.7-4` | The current provisioning recipe does not produce the guide's documented Ubuntu 24.04.3 / kernel 1041 combination. | +| Effective H100 VM security | `ConfidentialVM`; Secure Boot and vTPM both enabled | Verified on the underlying VM scale set; no security bypass was applied. | +| `580-6.8.0-1041-azure-fde-ubuntu24.04` | HTTP 200 | Published; the guide explicitly describes this kernel. | +| `580-6.8.0-1063-azure-fde-ubuntu24.04` | HTTP 200 with OCI Accept headers | Published; support for this combination has not been established. | +| `580-6.8.0-1071-azure-fde-ubuntu24.04` | HTTP 404, `MANIFEST_UNKNOWN`, `manifest unknown` | Reported missing image reproduced directly. | +| `595-6.8.0-1071-azure-fde-ubuntu24.04` | HTTP 404, `MANIFEST_UNKNOWN`, `manifest unknown` | Changing the driver branch does not fix this missing tag. | + +Unauthenticated registry requests return HTTP 401 even for existing images. +Authenticate using the public anonymous bearer-token flow. Also request both +OCI and Docker manifest media types: otherwise an existing OCI image can return +a misleading HTTP 404 (`OCI manifest found, but accept header does not support +OCI manifests`). Neither error should be classified as an absent tag. + +**Initial outcome: cluster provisioned; exact-kernel container path blocked.** The +exact-kernel compatibility gate deliberately exits with an error after saving +the evidence. This is not a failed H100 allocation: it is a reproducible missing +driver-image blocker on the actual provisioned worker. + +The initial reproduction stopped before GPU Operator installation. The subsequent +generic-driver experiment below did install GPU Operator and attempt driver +loading, unsuccessfully. The later signed-package experiment succeeded through +a different installation path. The 1041 and 1063 images remain positive controls, +not substitutes for 1071. + +To unblock, obtain a Microsoft/NVIDIA-supported pairing for the actual Ubuntu +24.04.5 / Azure-FDE kernel: a published, compatible signed driver image and +confirmation of its support, or an officially supported AKS provisioning path +that yields a compatible OS/kernel. Neither a Kubernetes-version selection nor +`--os-sku Ubuntu2404` pins the node-image build. This reproduction found no +supported way to select the guide's old kernel through these commands, and did +not attempt an unsupported node-image downgrade or host-kernel replacement. + +Once an approved pairing is available, rerun the compatibility gate and follow +the pinned guide's GPU Operator `v25.3.1` steps with the exact driver version. +The worker has `sku=gpu:NoSchedule`: add that toleration to GPU Operator operands +and node-feature-discovery workers before installation. Then verify the signed +module loads, `nvidia-smi` confidential mode/readiness, one allocatable GPU, a +GPU-only workload that fails rather than falling back to CPU, and GPU attestation. +Those are separate acceptance stages, not consequences of a manifest returning +HTTP 200. + +### Latest-image / generic-driver experiment + +This is an **experimental reproduction, not a supported onboarding recipe**. +It preserves the original exact-kernel compatibility gate; generic image +publication does not satisfy that gate. + +On 2026-11-10, the AKS node-pool upgrade API offered +`AKSUbuntu-2404gen2CVMcontainerd-202610.30.1`, identical to the existing image. +Restarting the cluster restored exactly one system node and one H100 worker, +both Ready. The worker still used Ubuntu 24.04.5 and +`6.8.0-1071-azure-fde`. The underlying H100 scale set retained ConfidentialVM, +Secure Boot and vTPM. No pool scaling or image upgrade was needed. + +| Attempt | Observed result | +|---|---| +| GPU Operator `v26.7.2`, default driver `595.91.07`, open modules | Generic `595.91.07-ubuntu24.04` image pulled successfully. Driver initialization failed before module installation. | +| GPU Operator `v26.7.2`, newer published driver `615.71.09`, open modules | Generic `615.71.09-ubuntu24.04` image pulled successfully. The same initialization failure reproduced. | +| Exact-kernel `615-6.8.0-1071-azure-fde-ubuntu24.04` | HTTP 404 / `MANIFEST_UNKNOWN`. | +| Worker GPU resources after generic-container attempts | No `nvidia.com/gpu` allocatable resource; GPU Operator operands remained in initialization. | +| Attestation at this stage | Not run: the generic-container driver prerequisite failed. See the successful signed-package follow-up below. | + +Both generic driver containers successfully obtained the **matching kernel +headers**. They then downloaded the Ubuntu kernel packages and failed with: + +```text +Generating Linux kernel version string... +ls: cannot access 'boot/vmlinuz-*': No such file or directory +Could not locate Linux kernel version string +``` + +The container's `/usr/local/bin/nvidia-driver` expects `boot/vmlinuz-*` after +extracting `linux-image-${KERNEL_VERSION}`. Inspection of the actual Azure-FDE +package showed `usr/lib/linux/efi/kernel.efi-6.8.0-1071-azure-fde` instead. +This is a distinct generic-container packaging/layout blocker, **not missing +kernel headers**, and occurs before NVIDIA modules are built or loaded. + +The Ubuntu package index also lists exact-kernel NVIDIA module packages, +including `linux-modules-nvidia-615-open-6.8.0-1071-azure-fde` +(`6.8.0-1071.78`), with dependency on NVIDIA `615.71.09`. +The signed-package follow-up below tests that route separately; publication is +not proof of a supported AKS integration or successful attestation. +The matching generic NVIDIA image exists (digest +`sha256:0ed39c3e75c219954ba81be558c9aa0e6abbfa838c0f60d61bf9a139776fa186`). +Simply selecting it does not make GPU Operator consume Ubuntu's module packages. + +### Reusing the standalone VM signed-package approach + +The existing [VM deployment script](../../vm-samples/BuildRandomCVM.ps1) was +inspected but **not modified**. Its H100 path forces a confidential Ubuntu +22.04 VM, downloads the checksum-pinned V4.3.3 onboarding archive, prepares a +snapshot kernel, and reboots. Its non-GPU Ubuntu choices do not establish an +AKS confidential GPU image choice. Do not copy its kernel preparation, moving +kernel metapackage, reboot sequence or Docker installation onto a managed AKS +node. + +The reusable part is the **signed module package**, selected for the running +kernel, with coherent firmware and user-space libraries. On 2026-11-10, a +disposable Ubuntu 24.04 pod on the actual H100 worker downloaded and extracted: + +| Exact Ubuntu package | Package version | Module driver version | Inspection | +|---|---|---|---| +| `linux-modules-nvidia-595-server-open-6.8.0-1071-azure-fde` | `6.8.0-1071.78` | `595.91.07` | Five prebuilt modules; correct kernel vermagic; Canonical signer. | +| `linux-modules-nvidia-615-open-6.8.0-1071-azure-fde` | `6.8.0-1071.78` | `615.71.09` | Five prebuilt modules; correct kernel vermagic; Canonical signer. | + +Both trees reported `Canonical Ltd. Kernel Module Signing`, signing-key ID +`E9:DF:13:0F:92:92:A9:B7`, SHA-512 signatures, and vermagic +`6.8.0-1071-azure-fde SMP mod_unload modversions`. Metadata alone does not prove +the node accepts a signature. A subsequent privileged probe **did load the +615 NVIDIA and UVM modules with guest Secure Boot enabled**. The first +`nvidia-smi` identified an NVIDIA H100 NVL and driver `615.71.09`. + +The probe downloaded exact packages and extracted them, rather than installing +the full VM driver metapackage onto the host. A clean-container `apt-get -s` +simulation of that metapackage also selected DKMS, kernel and X11 packages. +It is therefore not a safe managed-AKS installation recipe. Only the matching +615 firmware directory was temporarily copied into the host firmware path; +the host kernel and its module tree were not replaced. + +Initial GPU detection was **not sustained readiness**: persistence was not +started before the first short-lived query. The next CC query returned +`No devices were found` (exit 6). Kernel logs reported: + +```text +kgspCheckGspRmCcCleanup_GH100: CC secret cleanup failed due to timeout! +RmInitAdapter: osInitNvMapping failed, bailing out of RmInitAdapter +RmInitAdapter failed! (0x22:0x56:904) +``` + +Unloading and reloading the same signed module then failed with `No such device`. +This moves the investigation past the generic container's kernel-layout error +to GPU lifecycle/persistence initialization. It does not establish that +persistence ordering is the sole cause, that 615 is supported for this AKS image, +or that attestation succeeds. The clean persistence-first retry below supplies +the sustained-readiness and attestation evidence missing from this first attempt. + +### Successful persistence-first retry + +A fresh-node retry used the same exact signed `615.71.09` packages, preloaded +`ecdsa_generic`, `ecdh` and `ecc`, loaded NVIDIA/UVM, and started +`nvidia-persistenced --uvm-persistence-mode` **before** invoking `nvidia-smi`. +It then set confidential-compute ready state and copied the coherent extracted +driver tree into `/run/nvidia/driver`. GPU Operator `v26.7.2`, installed with +`driver.enabled=false`, supplied the toolkit/runtime and device plugin without +replacing AKS-managed containerd. + +| Acceptance check | Live result | +|---|---| +| Current worker | Ubuntu 24.04.5, AKS image `AKSUbuntu-2404gen2CVMcontainerd-202610.30.1`, kernel `6.8.0-1071-azure-fde`. | +| Trusted driver loading | Canonical-signed NVIDIA/UVM `615.71.09` accepted by the running Secure Boot kernel; H100 NVL detected. | +| Confidential GPU | `CC status: ON`, `CC Environment: PRODUCTION`, `CC GPUs Ready State: Ready`; persistence daemon alive. | +| Kubernetes integration | Exactly one allocatable `nvidia.com/gpu`; Operator CUDA validator passed. | +| Independent GPU workload | Nonprivileged Job requested/limited one GPU using runtime class `nvidia`; vector addition of 50,000 elements returned `Test PASSED`. No CPU fallback. | +| GPU attestation before/after workload | V4.5.1 archive, verifier `2.3.0`, strict OCSP; exit zero, expected GPU UUID verified, exact `GPU Attestation is Successful.` verdict on both runs. | +| Sustained readiness | Three repeated post-workload CC queries succeeded and persistence remained alive. | +| Separate CPU/CVM attestation | Checksum-verified Azure guest CLI `v0.1.0`, MAA provider; exit zero and decoded `sevsnpvm` / `azure-compliant-cvm` claims. | + +The published installer, probe manifest, workload and both attestation scripts +were then replayed unchanged on another fresh worker. The installer emitted +`SIGNED_DRIVER_LOADED=1`; GPU attestation emitted +`GPU_ATTESTATION_VALIDATED=1` before and after the Job's `Test PASSED`; +CVM attestation emitted `CVM_ATTESTATION_VALIDATED=1`. The GPU UUID was +discovered dynamically and differed from the previous allocation. The worker +still had ConfidentialVM, Secure Boot and vTPM enabled, with one allocatable GPU. +All nine Bash blocks in the CLI walkthrough passed syntax checks. Re-running +the installer on the initialized worker failed at its already-loaded-module +guard before mutation, while the existing persistence daemon and GPU remained +ready. Optional DCGM exporter readiness remained `0/1`; monitoring health is +not claimed by these workload and attestation results. + +This comparison supports persistence-first initialization as the useful remedy +for this reproduction, not proof of the sole cause or of vendor support. +The workload container's `nvidia-smi` displayed Persistence Off while the probe +reported Enabled with its UVM persistence daemon alive. That display discrepancy +was not conclusively explained; repeated readiness, CUDA and attestation checks +passed. Do not equate any single persistence display with attestation. + +CPU attestation initially failed with MAA HTTP 400, `TcgLogs is empty in +attestation request`. Mounting host `/sys/kernel/security` read-only exposed +the TPM event log and resolved that failure. Its sysfs stat size is zero despite +containing readable data: use `wc -c`, not `test -s`, to check it. GPU attestation +and CPU/CVM attestation are separate results; the latter records the successful +MAA request and returned claims, not an additional independent JWT verifier. + +The reusable artifacts are [the probe manifest](signed-driver-probe.yaml), +[exact-package installer](Install-SignedDriver.sh), +[GPU-only workload](gpu-smoke-test.yaml), +[strict GPU attestation](Test-GpuAttestation.sh), and +[CPU/CVM attestation](Test-CvmAttestation.sh). The verifier warns of future +deprecation in favour of `nvattest`; that migration has not been tested here. +Full attestation logs can include tokens and must remain local, not in Git. + +**Lifecycle boundary:** the privileged probe holds the persistence daemon and +expires after four hours. Deleting it ends persistence; `/run/nvidia/driver` is +ephemeral. Reboot, node replacement or a new image/kernel requires fresh matching +package validation and setup. The installer intentionally rejects an already +initialized node rather than guessing how to reconcile it. No DKMS, host kernel +replacement, unsigned modules, key enrollment or Secure Boot bypass was used. + +### Clues from the full upstream onboarding flow + +The [full upstream repository](https://github.com/Azure/az-cgpu-onboarding/tree/136262ef00ad75b8e0bc789563f043006fb68f4c), +not just its AKS page, provides several useful distinctions: + +- **Signed exact-ABI modules:** the V4.3.3 VM installer uses + `nvidia-driver-595-server-open` with a moving + `linux-modules-nvidia-595-server-open-azure-fde` metapackage. Reuse the + packaging principle, not the unmodified host installer or its moving kernel + selection. +- **Crypto dependencies:** the + [driver installer](https://github.com/Azure/az-cgpu-onboarding/blob/136262ef00ad75b8e0bc789563f043006fb68f4c/src/step-1-install-gpu-driver.sh) + preloads `ecdsa_generic` and `ecdh`; the inspected NVIDIA module also depends + on `ecc`. The live signed-module probe loaded these before NVIDIA. +- **Persistence before validation:** the + [persistence override](https://github.com/Azure/az-cgpu-onboarding/blob/136262ef00ad75b8e0bc789563f043006fb68f4c/src/nvidia-persistenced.override.conf) + enables `--uvm-persistence-mode`, then sets GPU ready state with + `nvidia-smi conf-compute -srs 1`. The + [attestation script](https://github.com/Azure/az-cgpu-onboarding/blob/136262ef00ad75b8e0bc789563f043006fb68f4c/src/step-2-attestation.sh) + waits for persistence and checks that it is enabled. A single successful + `nvidia-smi` invocation is not an adequate substitute. +- **Runtime integration is separate:** the + [VM tools installer](https://github.com/Azure/az-cgpu-onboarding/blob/136262ef00ad75b8e0bc789563f043006fb68f4c/src/step-3-install-gpu-tools.sh) + installs Docker/containerd and configures the Docker NVIDIA runtime. Do not + overwrite AKS-managed containerd with that installer. A functioning module + still needs AKS-compatible toolkit, device-plugin and GPU scheduling checks. +- **Standalone VM alternative:** the root README recommends a prebuilt + community-gallery VM image with drivers, CUDA, Docker and verifier included. + The current automatic VM onboarding also has an Ubuntu 24.04 path. These + are alternatives if AKS is not mandatory, not AKS node-image selections or + evidence of a working confidential AKS configuration. +- **Different failures need different fixes:** the + [FAQ](https://github.com/Azure/az-cgpu-onboarding/blob/136262ef00ad75b8e0bc789563f043006fb68f4c/Frequently-Asked-Questions.md) + describes driver/library mismatches after updates; that is distinct from + the initial generic-container `vmlinuz` extraction failure. The FAQ also + explains the Azure vTPM CPU attestation route; absence of `/dev/sev-guest` + alone is not proof that an Azure SNP VM lacks confidential protection. +- **Verifier alternatives do not fix drivers:** the optional preview + `nvattest` path is a verifier choice, not a way around missing modules, + failed GPU initialization or the requirement for positive attestation. + +### Newer Ubuntu node OS availability + +Checked on 2026-11-10 against the published AKS documentation and the existing +West Europe confidential H100 pool's node-image upgrade API: + +| Ubuntu version | Published AKS availability | Applicability to this reproduction | +|---|---|---| +| 25.x | Not listed as a supported AKS node OS. | No documented AKS node OS option established. | +| 26.04 | Preview for Kubernetes 1.36 and later, using OS SKU `Ubuntu2604` and feature flag `Ubuntu2604Preview`. Requires Generation 2 VMs; Confidential VM, Trusted Launch and FIPS are explicitly unsupported. | Not a supported replacement for the confidential H100 worker, even though this cluster uses Kubernetes 1.36.5. | +| 24.04 | Supported Ubuntu family for AKS Confidential VM node images. | The existing pool's latest offered image remains `AKSUbuntu-2404gen2CVMcontainerd-202610.30.1`; the worker observed during the experiment used Ubuntu 24.04.5. | + +Sources: [AKS OS version support and Ubuntu 26.04 preview restrictions](https://learn.microsoft.com/azure/aks/upgrade-os-version#migrate-to-ubuntu-2604-preview) +and [AKS node-image variants](https://learn.microsoft.com/azure/aks/node-images). +General AKS preview availability is distinct from support for a confidential +GPU node and does not prove availability for every VM size or region. Likewise, +the existing pool's upgrade response describes that pool's image track, not all +AKS OS SKUs. + +Ubuntu 26.04 is therefore not a documented workaround for this driver blocker. +A standard, non-confidential node would not validate the required environment. +Changing a workload container's Ubuntu base image would not change the managed +host kernel or resolve the driver container's Azure-FDE package-layout failure. +This availability check was read-only; the cluster remained stopped. + +### Reproduce the failed generic-container stage with Azure CLI + +Use the existing subscription, resource group and cluster from the deployment +stage. Keep the sample kubeconfig separate from the default context. These Bash +commands follow the upstream guide's Azure CLI style; they resume billable nodes. +The local values file adds both GPU-taint tolerations and open-module selection. +**This block deliberately reenables the failed generic installer. Do not run it +over a successful signed-package probe.** Use the updated walkthrough for the +working experimental route. + +```bash +az aks nodepool get-upgrades --subscription "$subscription" \ + --resource-group "$resourceGroup" --cluster-name "$clusterName" \ + --nodepool-name gpunp + +az aks start --subscription "$subscription" \ + --resource-group "$resourceGroup" --name "$clusterName" + +az aks get-credentials --subscription "$subscription" \ + --resource-group "$resourceGroup" --name "$clusterName" \ + --file .local/kubeconfig --overwrite-existing + +export KUBECONFIG="$PWD/.local/kubeconfig" +kubectl get nodes -o wide + +helm upgrade --install gpu-operator gpu-operator \ + --repo https://helm.ngc.nvidia.com/nvidia --version v26.7.2 \ + --namespace gpu-operator --create-namespace \ + --values gpu-operator-experimental-values.yaml --set driver.enabled=true + +kubectl --namespace gpu-operator get pods -o wide +kubectl --namespace gpu-operator logs \ + --selector app=nvidia-driver-daemonset --container nvidia-driver-ctr \ + --tail=150 +# After a container restart, inspect the failed attempt: +kubectl --namespace gpu-operator logs \ + --selector app=nvidia-driver-daemonset --container nvidia-driver-ctr \ + --previous --tail=150 +kubectl get nodes --selector agentpool=gpunp \ + -o 'jsonpath={.items[0].status.allocatable}' + +az aks stop --subscription "$subscription" \ + --resource-group "$resourceGroup" --name "$clusterName" +az aks show --subscription "$subscription" \ + --resource-group "$resourceGroup" --name "$clusterName" \ + --query '{power:powerState.code,state:provisioningState}' +``` + +Do not treat Helm's `deployed` status as GPU readiness. In this experiment Helm +succeeded, but the driver did not. Do not work around the layout failure by +replacing the managed host kernel, disabling Secure Boot, installing unsigned +modules, or inventing a driver-ready state. Resolution needs a compatible, +trusted installation path for this Azure-FDE image. The signed-package route +above bypasses the broken container entrypoint without changing the host kernel. + +### Attestation acceptance after driver enablement + +The existing [VM GPU attestation flow](../../vm-samples/BuildRandomCVM.ps1) +provides a reusable checksum-pinned verifier and explicit positive-verdict +checks. Its host-kernel installation and reboot steps do **not** apply to AKS. +Before adapting the verifier Job, require: + +1. A loaded, trusted NVIDIA module and working `nvidia-smi`. +2. `CC status: ON`, `CC Environment: PRODUCTION`, and one allocatable H100 GPU. +3. A GPU-targeted workload that fails rather than falling back to CPU. +4. A checksum-verified, version-pinned verifier, zero exit status, the exact + `GPU Attestation is Successful.` verdict, and successful verification of the + expected GPU UUID. A Completed pod alone is insufficient. + +For the newer reviewed Azure verifier V4.5.1, the release archive SHA-256 is +`075ee785f6d65799d99ae18102fb88bdab4afe31540bf16673a27545c0ba80e8`. +Its Python package requires Python 3.12. Use strict OCSP validation +(`--ocsp_attestation_settings strict`) rather than permissive defaults, and +preserve the verifier evidence. GPU attestation is separate from CPU/CVM +attestation; both have now passed separately through the signed-package route. +The [updated onboarding walkthrough](Confidential-GPU-H100-AKS-Onboarding.md) +preserves the Azure CLI workflow while clearly labeling this experimental path. diff --git a/aks-samples/aks-gpu-worker/Test-CvmAttestation.sh b/aks-samples/aks-gpu-worker/Test-CvmAttestation.sh new file mode 100644 index 0000000..7e236a9 --- /dev/null +++ b/aks-samples/aks-gpu-worker/Test-CvmAttestation.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail +trap 'echo "ERROR: CVM attestation failed at line $LINENO; no success is claimed." >&2' ERR + +test -r /sys/kernel/security/tpm0/binary_bios_measurements +test "$(wc -c < /sys/kernel/security/tpm0/binary_bios_measurements)" -gt 0 +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq +apt-get install -y --no-install-recommends curl ca-certificates +mkdir -p /tmp/signed-attestation +cd /tmp/signed-attestation +curl -fsSL --retry 3 -o azure-guest-attest \ + https://github.com/Azure/azure-guest-attestation-sdk/releases/download/azure-guest-attest-v0.1.0/azure-guest-attest-x86_64-unknown-linux-musl +echo 'c6559e35d588b1dc9074c443beadce395da34b5beba2eb6e276195a694064b43 azure-guest-attest' | + sha256sum -c - +chmod 755 azure-guest-attest +result=/tmp/signed-attestation/cpu-result.log +if ! timeout 180 ./azure-guest-attest guest-attest --provider maa --decode > "$result" 2>&1; then + echo "ERROR: MAA request failed; inspect $result inside the probe." >&2 + exit 1 +fi +grep -E '"x-ms-attestation-type"[[:space:]]*:[[:space:]]*"sevsnpvm"' "$result" +grep -E '"x-ms-compliance-status"[[:space:]]*:[[:space:]]*"azure-compliant-cvm"' "$result" +echo CVM_ATTESTATION_VALIDATED=1 +exit 0 diff --git a/aks-samples/aks-gpu-worker/Test-DriverCompatibility.ps1 b/aks-samples/aks-gpu-worker/Test-DriverCompatibility.ps1 new file mode 100644 index 0000000..8ec7081 --- /dev/null +++ b/aks-samples/aks-gpu-worker/Test-DriverCompatibility.ps1 @@ -0,0 +1,40 @@ +#Requires -Version 7.2 +[CmdletBinding(DefaultParameterSetName = 'Kernel')] +param( + [Parameter(Mandatory, ParameterSetName = 'Kernel')] + [ValidatePattern('^[0-9]+\.[0-9]+\.[0-9]+-[0-9]+-azure-fde$')] + [string]$KernelVersion, + [Parameter(Mandatory, ParameterSetName = 'Node')][string]$KubeConfig, + [Parameter(ParameterSetName = 'Node')][string]$PoolName = 'gpunp', + [ValidateNotNullOrEmpty()][string[]]$DriverBranches = @('580', '595'), + [string]$OutputPath, + [switch]$RequireAvailable +) +. (Join-Path $PSScriptRoot 'Common.ps1') + +if ($PSCmdlet.ParameterSetName -eq 'Node') { + $node = Get-GpuNode -KubeConfig $KubeConfig -PoolName $PoolName + if ($node.status.nodeInfo.osImage -notlike 'Ubuntu 24.04*') { + throw "Unsupported worker OS: $($node.status.nodeInfo.osImage). Expected Ubuntu 24.04." + } + $KernelVersion = $node.status.nodeInfo.kernelVersion +} +if ($KernelVersion -notmatch '^[0-9]+\.[0-9]+\.[0-9]+-[0-9]+-azure-fde$') { + throw "Unsupported confidential worker kernel: $KernelVersion." +} +$results = @(foreach ($branch in $DriverBranches) { + if ($branch -notmatch '^[0-9]+$') { throw "Invalid driver branch: $branch" } + Get-DriverManifest -Tag "$branch-$KernelVersion-ubuntu24.04" +}) +$report = [pscustomobject]@{ + KernelVersion = $KernelVersion + Manifests = $results + MatchingImageAvailable = (@($results | Where-Object Available).Count -gt 0) + SupportValidated = $false +} +$json = $report | ConvertTo-Json -Depth 10 +if ($OutputPath) { $json | Set-Content -LiteralPath $OutputPath } +$json +if ($RequireAvailable -and -not $report.MatchingImageAvailable) { + throw "BLOCKED: no exact-kernel driver image for $KernelVersion. Do not install a different kernel's module or downgrade the AKS image." +} diff --git a/aks-samples/aks-gpu-worker/Test-GpuAttestation.sh b/aks-samples/aks-gpu-worker/Test-GpuAttestation.sh new file mode 100644 index 0000000..503f6ae --- /dev/null +++ b/aks-samples/aks-gpu-worker/Test-GpuAttestation.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail +trap 'echo "ERROR: GPU attestation failed at line $LINENO; no success is claimed." >&2' ERR + +export LD_LIBRARY_PATH=/tmp/signed-driver/root/usr/lib/x86_64-linux-gnu +smi=/tmp/signed-driver/root/usr/bin/nvidia-smi +uuid=$("$smi" --query-gpu=uuid --format=csv,noheader) +[[ "$uuid" =~ ^GPU-[[:xdigit:]-]+$ ]] +"$smi" conf-compute -f | grep -Fx 'CC status: ON' +"$smi" conf-compute -e | grep -Fx 'CC Environment: PRODUCTION' +kill -0 "$(cat /var/run/nvidia-persistenced/nvidia-persistenced.pid)" + +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq +apt-get install -y --no-install-recommends curl ca-certificates python3 python3-venv +mkdir -p /tmp/signed-attestation +cd /tmp/signed-attestation +curl -fsSL --retry 3 -o onboarding.tar.gz \ + https://github.com/Azure/az-cgpu-onboarding/releases/download/V4.5.1/cgpu-onboarding-package.tar.gz +echo '075ee785f6d65799d99ae18102fb88bdab4afe31540bf16673a27545c0ba80e8 onboarding.tar.gz' | sha256sum -c - +tar -xzf onboarding.tar.gz +mkdir -p verifier +tar -xf cgpu-onboarding-package/local_gpu_verifier.tar -C verifier +cd verifier +python3 -m venv /tmp/signed-attestation/venv +/tmp/signed-attestation/venv/bin/python -m pip install . > /tmp/signed-attestation/pip-install.log 2>&1 || { + cat /tmp/signed-attestation/pip-install.log >&2 + exit 1 +} +/tmp/signed-attestation/venv/bin/python -m pip show nv-local-gpu-verifier | + grep -Fx 'Version: 2.3.0' +result=/tmp/signed-attestation/gpu-result.log +status=0 +timeout 300 /tmp/signed-attestation/venv/bin/python -m verifier.cc_admin \ + --ocsp_attestation_settings strict > "$result" 2>&1 || status=$? +# Retain the full local evidence without echoing attestation tokens. +sed '/Entity Attestation Token:/,$d' "$result" +test "$status" -eq 0 +grep -Fx 'GPU Attestation is Successful.' "$result" +grep -F "GPU 0 with UUID $uuid verified successfully." "$result" +echo GPU_ATTESTATION_VALIDATED=1 +exit 0 diff --git a/aks-samples/aks-gpu-worker/gpu-operator-experimental-values.yaml b/aks-samples/aks-gpu-worker/gpu-operator-experimental-values.yaml new file mode 100644 index 0000000..4ed1424 --- /dev/null +++ b/aks-samples/aks-gpu-worker/gpu-operator-experimental-values.yaml @@ -0,0 +1,23 @@ +# Experimental generic driver: availability is not proof of Azure-FDE support. +driver: + version: "615.71.09" + kernelModuleType: open +daemonsets: + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + - key: sku + operator: Equal + value: gpu + effect: NoSchedule +node-feature-discovery: + worker: + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + - key: sku + operator: Equal + value: gpu + effect: NoSchedule diff --git a/aks-samples/aks-gpu-worker/gpu-smoke-test.yaml b/aks-samples/aks-gpu-worker/gpu-smoke-test.yaml new file mode 100644 index 0000000..49b44da --- /dev/null +++ b/aks-samples/aks-gpu-worker/gpu-smoke-test.yaml @@ -0,0 +1,36 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: confidential-gpu-smoke +spec: + backoffLimit: 0 + activeDeadlineSeconds: 180 + template: + spec: + restartPolicy: Never + runtimeClassName: nvidia + nodeSelector: + agentpool: gpunp + tolerations: + - key: sku + operator: Equal + value: gpu + effect: NoSchedule + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + containers: + - name: vector-add + image: nvcr.io/nvidia/gpu-operator:v26.7.2 + command: [sh, -c, "nvidia-smi && vectorAdd"] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 1Gi + nvidia.com/gpu: 1 diff --git a/aks-samples/aks-gpu-worker/signed-driver-probe.yaml b/aks-samples/aks-gpu-worker/signed-driver-probe.yaml new file mode 100644 index 0000000..8076572 --- /dev/null +++ b/aks-samples/aks-gpu-worker/signed-driver-probe.yaml @@ -0,0 +1,69 @@ +apiVersion: v1 +kind: Pod +metadata: + name: signed-driver-load +spec: + restartPolicy: Never + nodeSelector: + agentpool: gpunp + tolerations: + - key: sku + operator: Equal + value: gpu + effect: NoSchedule + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + containers: + - name: probe + image: ubuntu:24.04 + command: [bash, -c, "sleep 14400"] + securityContext: + privileged: true + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 4Gi + volumeMounts: + - name: modules + mountPath: /lib/modules + readOnly: true + - name: firmware + mountPath: /lib/firmware + - name: devices + mountPath: /dev + - name: driver + mountPath: /run/nvidia/driver + - name: sysfirmware + mountPath: /sys/firmware + readOnly: true + - name: security + mountPath: /sys/kernel/security + readOnly: true + volumes: + - name: modules + hostPath: + path: /lib/modules + type: Directory + - name: firmware + hostPath: + path: /lib/firmware + type: Directory + - name: devices + hostPath: + path: /dev + type: Directory + - name: driver + hostPath: + path: /run/nvidia/driver + type: DirectoryOrCreate + - name: sysfirmware + hostPath: + path: /sys/firmware + type: Directory + - name: security + hostPath: + path: /sys/kernel/security + type: Directory diff --git a/aks-samples/aks-gpu-worker/tests/Test-Sample.ps1 b/aks-samples/aks-gpu-worker/tests/Test-Sample.ps1 new file mode 100644 index 0000000..ed5b56c --- /dev/null +++ b/aks-samples/aks-gpu-worker/tests/Test-Sample.ps1 @@ -0,0 +1,164 @@ +#Requires -Version 7.2 +$ErrorActionPreference = 'Stop' +$root = Split-Path $PSScriptRoot -Parent +. (Join-Path $root 'Common.ps1') +$script:passed = 0 +$state = @{} + +function Assert($Condition, [string]$Message) { + if (-not $Condition) { throw $Message } +} +function Expect-Failure([scriptblock]$Action, [string]$Pattern) { + try { & $Action | Out-Null } + catch { + Assert ($_.Exception.Message -match $Pattern) "Unexpected error: $_" + return + } + throw "Expected a failure matching '$Pattern'." +} +function Case([string]$Name, [scriptblock]$Action) { + $state.status = 200 + $state.digest = 'sha256:' + ('a' * 64) + $state.content = '{"errors":[{"code":"MANIFEST_UNKNOWN","message":"manifest unknown"}]}' + $state.networkError = $false + & $Action + $script:passed++ + Write-Host "PASS $Name" +} +function Invoke-RestMethod { @{ token = 'offline-token' } } +function Invoke-WebRequest { + param($Uri, $Headers, [switch]$SkipHttpErrorCheck) + Assert ($Headers.Authorization -eq 'Bearer offline-token') 'Anonymous registry token not supplied.' + foreach ($type in @('oci.image.manifest', 'oci.image.index', 'distribution.manifest.v2', 'distribution.manifest.list.v2')) { + Assert ($Headers.Accept.Contains($type)) "Missing manifest media type: $type" + } + Assert $SkipHttpErrorCheck 'HTTP errors must be inspected explicitly.' + if ($state.networkError) { throw 'Mock network failure' } + @{ + StatusCode = $state.status + Headers = @{ 'Docker-Content-Digest' = @($state.digest) } + Content = $state.content + } +} +function New-Pools { + @( + @{ name = 'sysnp'; vmSize = 'Standard_D4ds_v6'; count = 1; osSku = 'Ubuntu2404' + mode = 'System'; enableAutoScaling = $false; provisioningState = 'Succeeded' } + @{ name = 'gpunp'; vmSize = 'Standard_NCC40ads_H100_v5'; count = 1; osSku = 'Ubuntu2404' + mode = 'User'; enableAutoScaling = $false; provisioningState = 'Succeeded' + gpuProfile = @{ driver = 'None' }; nodeTaints = @('sku=gpu:NoSchedule') } + ) +} +$tag = '580-6.8.0-1071-azure-fde-ubuntu24.04' +Case 'available manifests require OCI/Docker Accept headers and a digest' { + $result = Get-DriverManifest $tag + Assert ($result.Available -and $result.Digest -eq $state.digest) 'Incorrect available-image result.' +} +Case 'genuine absent tag is reported, not thrown' { + $state.status = 404 + $result = Get-DriverManifest $tag + Assert (-not $result.Available -and $result.ErrorCode -eq 'MANIFEST_UNKNOWN') 'Missing tag not identified.' +} +Case 'misleading OCI 404 is not a missing tag' { + $state.status = 404 + $state.content = '{"errors":[{"code":"MANIFEST_UNKNOWN","message":"OCI manifest found, but accept header does not support OCI manifests"}]}' + Expect-Failure { Get-DriverManifest $tag } 'unexpected 404' +} +foreach ($status in @(401, 403, 429, 500, 503)) { + Case "HTTP $status throws rather than reporting missing image" { + $state.status = $status + Expect-Failure { Get-DriverManifest $tag } "HTTP $status" + } +} +Case 'network failure propagates' { + $state.networkError = $true + Expect-Failure { Get-DriverManifest $tag } 'network failure' +} +Case 'invalid digest is rejected' { + $state.digest = 'invalid' + Expect-Failure { Get-DriverManifest $tag } 'valid digest' +} +Case 'invalid registry tag is rejected' { + Expect-Failure { Get-DriverManifest '../invalid' } 'pattern' +} +Case 'compatibility report retains shape and does not claim support' { + $report = & (Join-Path $root 'Test-DriverCompatibility.ps1') -KernelVersion '6.8.0-1071-azure-fde' | ConvertFrom-Json + Assert ($report.Manifests.Count -eq 2 -and $report.MatchingImageAvailable -and -not $report.SupportValidated) 'Incorrect report shape.' +} +Case 'missing images fail the required-availability gate' { + $state.status = 404 + Expect-Failure { + & (Join-Path $root 'Test-DriverCompatibility.ps1') -KernelVersion '6.8.0-1071-azure-fde' -RequireAvailable + } 'BLOCKED: no exact-kernel' +} +Case 'invalid kernel and branch are rejected' { + Expect-Failure { & (Join-Path $root 'Test-DriverCompatibility.ps1') -KernelVersion '6.8.0-generic' } 'pattern' + Expect-Failure { + & (Join-Path $root 'Test-DriverCompatibility.ps1') -KernelVersion '6.8.0-1071-azure-fde' -DriverBranches '580/invalid' + } 'Invalid driver branch' +} +Case 'exact topology and partial system-only topology pass' { + $pools = New-Pools + Assert-SamplePools $pools 'Standard_D4ds_v6' + Assert-SamplePools @($pools[0]) 'Standard_D4ds_v6' -AllowMissingGpu + Expect-Failure { Assert-SamplePools @($pools[0]) 'Standard_D4ds_v6' } 'Expected one system' +} +foreach ($field in @('count', 'vmSize', 'osSku', 'mode', 'enableAutoScaling', 'provisioningState')) { + Case "mismatched pool $field is rejected" { + $pools = New-Pools + $pools[1][$field] = switch ($field) { + 'count' { 2 } + 'enableAutoScaling' { $true } + default { 'incorrect' } + } + Expect-Failure { Assert-SamplePools $pools 'Standard_D4ds_v6' } 'does not match' + } +} +Case 'extra pools and automatic GPU drivers are rejected' { + $pools = New-Pools + Expect-Failure { Assert-SamplePools ($pools + @(@{ name = 'unexpected' })) 'Standard_D4ds_v6' } 'Expected one system' + $pools[1].gpuProfile.driver = 'Install' + Expect-Failure { Assert-SamplePools $pools 'Standard_D4ds_v6' } 'disable automatic drivers' +} +Case 'pool checks support case-sensitive Azure JSON dictionaries' { + $pools = @(New-Pools | ConvertTo-Json -Depth 10 | ConvertFrom-Json -AsHashtable) + Assert-SamplePools $pools 'Standard_D4ds_v6' +} +Case 'confidential scale set requires one instance, Secure Boot and vTPM' { + $vm = @{ + sku = @{ name = 'Standard_NCC40ads_H100_v5'; capacity = 1 } + virtualMachineProfile = @{ securityProfile = @{ + securityType = 'ConfidentialVM' + uefiSettings = @{ secureBootEnabled = $true; vTpmEnabled = $true } + } } + } + Assert-ConfidentialGpuScaleSet @($vm) + $vm.virtualMachineProfile.securityProfile.uefiSettings.secureBootEnabled = $false + Expect-Failure { Assert-ConfidentialGpuScaleSet @($vm) } 'Secure Boot' + $vm.virtualMachineProfile.securityProfile.uefiSettings.secureBootEnabled = $true + $vm.virtualMachineProfile.securityProfile.uefiSettings.vTpmEnabled = $false + Expect-Failure { Assert-ConfidentialGpuScaleSet @($vm) } 'vTPM' + $vm.virtualMachineProfile.securityProfile.uefiSettings.vTpmEnabled = $true + $vm.sku.capacity = 2 + Expect-Failure { Assert-ConfidentialGpuScaleSet @($vm) } 'one ConfidentialVM' +} +Case 'native failure propagates' { + $pwsh = @(Get-Command pwsh -CommandType Application)[0].Source + Expect-Failure { + Invoke-CheckedCommand $pwsh @('-NoProfile', '-Command', 'exit 7') + } 'exit code 7' +} + +# Replace the native boundary only for the node-discovery cases. +function Invoke-CheckedCommand { $state.nodes | ConvertTo-Json -Depth 10 } +Case 'node lookup rejects zero and multiple workers' { + foreach ($items in @(@(), @(@{ name = 'one' }, @{ name = 'two' }))) { + $state.nodes = @{ items = $items } + Expect-Failure { Get-GpuNode -KubeConfig 'offline' } 'Expected exactly one node' + } +} +Case 'node lookup returns exactly one worker' { + $state.nodes = @{ items = @(@{ name = 'one' }) } + Assert ((Get-GpuNode -KubeConfig 'offline').name -eq 'one') 'Wrong node returned.' +} +Write-Host "$script:passed tests passed." From 28f5729b3bd70912b343cbe3012783e1bc43a730 Mon Sep 17 00:00:00 2001 From: Simon Gallagher Date: Thu, 12 Nov 2026 15:00:20 +0000 Subject: [PATCH 2/3] Replace experimental GPU worker with guide-only AKS automation Make the validated latest-image sample self-contained, remove historical driver workarounds, and document redacted GPU attestation and automatic shutdown. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- aks-samples/README.md | 5 +- aks-samples/aks-gpu-worker/Common.ps1 | 296 ++++-- .../Confidential-GPU-H100-AKS-Onboarding.md | 267 ----- aks-samples/aks-gpu-worker/Deploy.ps1 | 416 +++++--- aks-samples/aks-gpu-worker/Infrastructure.ps1 | 80 ++ .../aks-gpu-worker/Install-SignedDriver.sh | 62 -- aks-samples/aks-gpu-worker/Preflight.ps1 | 62 ++ aks-samples/aks-gpu-worker/README.md | 986 ++++-------------- .../aks-gpu-worker/Test-CvmAttestation.sh | 25 - .../Test-DriverCompatibility.ps1 | 40 - .../aks-gpu-worker/Test-GpuAttestation.sh | 42 - .../gpu-operator-experimental-values.yaml | 23 - .../aks-gpu-worker/gpu-smoke-test.yaml | 36 - .../aks-gpu-worker/signed-driver-probe.yaml | 69 -- .../aks-gpu-worker/tests/Test-Sample.ps1 | 389 ++++--- 15 files changed, 1031 insertions(+), 1767 deletions(-) delete mode 100644 aks-samples/aks-gpu-worker/Confidential-GPU-H100-AKS-Onboarding.md create mode 100644 aks-samples/aks-gpu-worker/Infrastructure.ps1 delete mode 100644 aks-samples/aks-gpu-worker/Install-SignedDriver.sh create mode 100644 aks-samples/aks-gpu-worker/Preflight.ps1 delete mode 100644 aks-samples/aks-gpu-worker/Test-CvmAttestation.sh delete mode 100644 aks-samples/aks-gpu-worker/Test-DriverCompatibility.ps1 delete mode 100644 aks-samples/aks-gpu-worker/Test-GpuAttestation.sh delete mode 100644 aks-samples/aks-gpu-worker/gpu-operator-experimental-values.yaml delete mode 100644 aks-samples/aks-gpu-worker/gpu-smoke-test.yaml delete mode 100644 aks-samples/aks-gpu-worker/signed-driver-probe.yaml diff --git a/aks-samples/README.md b/aks-samples/README.md index dc843e4..b709d10 100644 --- a/aks-samples/README.md +++ b/aks-samples/README.md @@ -6,8 +6,9 @@ Deploy Azure Kubernetes Service clusters with AMD SEV-SNP confidential node pools and Customer Managed Keys (CMK). -For a single confidential H100 GPU worker and an exact-kernel NVIDIA driver -compatibility reproduction, see [AKS confidential H100 worker](aks-gpu-worker/README.md). +For guide-only automation of a single confidential H100 GPU worker using the +latest offered AKS node image, GPU workload validation and strict GPU attestation, +see [AKS confidential H100 worker](aks-gpu-worker/README.md). ## Architecture diff --git a/aks-samples/aks-gpu-worker/Common.ps1 b/aks-samples/aks-gpu-worker/Common.ps1 index 4e7274c..f62ac42 100644 --- a/aks-samples/aks-gpu-worker/Common.ps1 +++ b/aks-samples/aks-gpu-worker/Common.ps1 @@ -1,124 +1,222 @@ #Requires -Version 7.2 -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Stop' +. (Join-Path $PSScriptRoot 'Infrastructure.ps1') +. (Join-Path $PSScriptRoot 'Preflight.ps1') -function Resolve-AzureCli { - $commands = @(Get-Command az -CommandType Application -ErrorAction SilentlyContinue) - if ($commands.Count) { return $commands[0].Source } - if ($IsWindows) { - foreach ($root in @($env:ProgramFiles, ${env:ProgramFiles(x86)})) { - if ($root) { - $candidate = Join-Path $root 'Microsoft SDKs\Azure\CLI2\wbin\az.cmd' - if (Test-Path $candidate) { return $candidate } - } - } +function New-SampleSshKey { + param([Parameter(Mandatory)][string]$Path) + $start = [System.Diagnostics.ProcessStartInfo]::new() + $start.FileName = (Get-Command ssh-keygen -CommandType Application -ErrorAction Stop).Source + $start.UseShellExecute = $false + $start.RedirectStandardError = $true + # ArgumentList preserves an empty passphrase, including on PowerShell 7.2. + foreach ($argument in @('-t', 'rsa', '-b', '3072', '-N', '', '-f', $Path, '-q')) { + $start.ArgumentList.Add($argument) } - throw 'Azure CLI is required. Install it and run az login before retrying.' + $process = [System.Diagnostics.Process]::Start($start) + try { + $errorText = $process.StandardError.ReadToEnd() + $process.WaitForExit() + if ($process.ExitCode -ne 0) { throw "ssh-keygen failed with exit code $($process.ExitCode): $errorText" } + if (-not (Test-Path $Path) -or -not (Test-Path "$Path.pub")) { throw 'ssh-keygen did not create the key pair.' } + } + finally { $process.Dispose() } +} + +function Assert-MinimumKernel { + param([string]$Kernel, [string]$Minimum = '6.8.0-1071-azure-fde') + $pattern = '^(\d+)\.(\d+)\.(\d+)-(\d+)-azure-fde$' + if ($Kernel -notmatch $pattern) { throw "Not an Azure FDE kernel: $Kernel" } + $actual = [version]::new([int]$Matches[1], [int]$Matches[2], [int]$Matches[3], [int]$Matches[4]) + if ($Minimum -notmatch $pattern) { throw "Invalid minimum FDE kernel: $Minimum" } + $floor = [version]::new([int]$Matches[1], [int]$Matches[2], [int]$Matches[3], [int]$Matches[4]) + if ($actual -lt $floor) { throw "Kernel $Kernel is older than $Minimum." } +} + +function Get-LatestKubernetesPatch { + param([object]$Versions) + $patches = @($Versions.values | Where-Object { $_.isPreview -ne $true } | + ForEach-Object { $_.patchVersions.PSObject.Properties.Name } | + Where-Object { $_ -match '^\d+\.\d+\.\d+$' } | + Sort-Object { [version]$_ } -Descending) + if (-not $patches.Count) { throw 'No stable regional Kubernetes patch was returned.' } + return $patches[0] } -function Invoke-CheckedCommand { +function Wait-SampleAksReady { param( - [Parameter(Mandatory)][string]$Command, - [Parameter(Mandatory)][string[]]$ArgumentList + [string]$Az, [string]$Subscription, [string]$ResourceGroup, [string]$ClusterName, + [int]$TimeoutSeconds = 1200, [int]$PollSeconds = 15 ) - $output = & $Command @ArgumentList - if ($LASTEXITCODE -ne 0) { - throw "$Command $($ArgumentList -join ' ') failed with exit code $LASTEXITCODE." - } - return $output + $scope = @('--subscription', $Subscription, '--only-show-errors') + $deadline = [DateTime]::UtcNow.AddSeconds($TimeoutSeconds) + do { + $cluster = Invoke-CheckedCommand $Az (@('aks', 'show', '-g', $ResourceGroup, + '-n', $ClusterName, '-o', 'json') + $scope) | ConvertFrom-Json + $pools = @(Invoke-CheckedCommand $Az (@('aks', 'nodepool', 'list', '-g', $ResourceGroup, + '--cluster-name', $ClusterName, '-o', 'json') + $scope) | ConvertFrom-Json) + $states = @($cluster.provisioningState) + @($pools | ForEach-Object provisioningState) + if (@($states | Where-Object { $_ -in @('Failed', 'Canceled') }).Count) { + throw "AKS provisioning failed: $($states -join ', ')." + } + if ($pools.Count -gt 0 -and -not @($states | Where-Object { $_ -ne 'Succeeded' }).Count) { return } + if ([DateTime]::UtcNow -ge $deadline) { break } + Start-Sleep -Seconds $PollSeconds + } while ($true) + throw "AKS operations did not settle within $TimeoutSeconds seconds: $($states -join ', ')." } -function Get-DriverManifest { +function Stop-SampleAks { param( - [Parameter(Mandatory)][ValidatePattern('^[0-9][0-9A-Za-z._-]*$')][string]$Tag + [string]$Az, [string]$Subscription, [string]$ResourceGroup, [string]$ClusterName, + [int]$TimeoutSeconds = 1200, [int]$PollSeconds = 15 ) - $tokenResponse = Invoke-RestMethod -Uri 'https://nvcr.io/token?service=registry&scope=repository:nvidia/driver:pull' - $accept = @( - 'application/vnd.oci.image.manifest.v1+json' - 'application/vnd.oci.image.index.v1+json' - 'application/vnd.docker.distribution.manifest.v2+json' - 'application/vnd.docker.distribution.manifest.list.v2+json' - ) -join ', ' - $response = Invoke-WebRequest -Uri "https://nvcr.io/v2/nvidia/driver/manifests/$Tag" ` - -Headers @{ Authorization = "Bearer $($tokenResponse.token)"; Accept = $accept } ` - -SkipHttpErrorCheck - $status = [int]$response.StatusCode - $digest = $null - $errorCode = $null - if ($status -eq 200) { - $digest = @($response.Headers['Docker-Content-Digest'])[0] - if ($digest -notmatch '^sha256:[0-9a-f]{64}$') { - throw "Registry did not return a valid digest for $Tag." + $arguments = @('aks', 'stop', '-g', $ResourceGroup, '-n', $ClusterName, + '-o', 'none', '--subscription', $Subscription, '--only-show-errors') + $deadline = [DateTime]::UtcNow.AddSeconds($TimeoutSeconds) + $PSNativeCommandUseErrorActionPreference = $false + do { + $output = & $Az @arguments 2>&1 + if ($LASTEXITCODE -eq 0) { return } + $message = $output -join "`n" + # AKS can report Succeeded while an addon handler still holds its lock. + if ($message -notmatch 'OperationNotAllowed' -or $message -notmatch "in-progress .* operation") { + throw "AKS stop failed: $message" } - } - elseif ($status -eq 404) { - $body = $response.Content | ConvertFrom-Json - $errorCode = $body.errors[0].code - if ($errorCode -ne 'MANIFEST_UNKNOWN' -or $body.errors[0].message -ne 'manifest unknown') { - throw "Registry returned an unexpected 404 for ${Tag}: $($response.Content)" + if ([DateTime]::UtcNow -ge $deadline) { throw "AKS stop remained blocked for $TimeoutSeconds seconds: $message" } + Write-Warning 'AKS stop is blocked by an in-progress operation; retrying within the shutdown deadline.' + Start-Sleep -Seconds $PollSeconds + } while ($true) +} + +function Get-ExactDriverImages { + param([string]$Kernel, [string[]]$Branches) + Assert-MinimumKernel $Kernel + $token = Invoke-RestMethod 'https://nvcr.io/token?service=registry&scope=repository:nvidia/driver:pull' + foreach ($branch in ($Branches | Sort-Object { [int]$_ } -Descending -Unique)) { + if ($branch -notmatch '^\d+$') { throw "Invalid driver branch: $branch" } + $tag = "$branch-$Kernel-ubuntu24.04" + $response = Invoke-WebRequest "https://nvcr.io/v2/nvidia/driver/manifests/$tag" -Headers @{ + Authorization = "Bearer $($token.token)" + Accept = 'application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json' + } -SkipHttpErrorCheck + $available = [int]$response.StatusCode -eq 200 + $digest = $null + if ($available) { + $digest = @($response.Headers['Docker-Content-Digest'])[0] + if ($digest -notmatch '^sha256:[0-9a-f]{64}$') { throw "Missing registry digest for $tag." } } - } - else { - throw "Registry request for $Tag returned HTTP $status. Check authentication, networking and rate limits; this is not evidence of a missing image." - } - [pscustomobject]@{ - Image = "nvcr.io/nvidia/driver:$Tag" - Tag = $Tag - Available = ($status -eq 200) - HttpStatus = $status - Digest = $digest - ErrorCode = $errorCode - CheckedAtUtc = [DateTime]::UtcNow.ToString('o') + elseif ([int]$response.StatusCode -eq 404) { + $body = $response.Content | ConvertFrom-Json + if ($body.errors[0].code -ne 'MANIFEST_UNKNOWN' -or $body.errors[0].message -ne 'manifest unknown') { + throw "Unexpected registry 404 for $tag." + } + } + else { throw "Registry HTTP $($response.StatusCode) for $tag; not a missing-image result." } + [pscustomobject]@{ Tag = $tag; Available = $available; Digest = $digest; HttpStatus = [int]$response.StatusCode } } } -function Get-GpuNode { - param( - [Parameter(Mandatory)][string]$KubeConfig, - [string]$PoolName = 'gpunp' - ) - $nodes = Invoke-CheckedCommand kubectl @('--kubeconfig', $KubeConfig, 'get', 'nodes', - '-l', "agentpool=$PoolName", '-o', 'json') | ConvertFrom-Json - if (@($nodes.items).Count -ne 1) { - throw "Expected exactly one node in pool $PoolName; found $(@($nodes.items).Count)." - } - return $nodes.items[0] +function Get-GuideOperatorValues { + param([string]$Kernel) + Assert-MinimumKernel $Kernel + @" +driver: + version: 580-$Kernel + kernelModuleType: open +daemonsets: + tolerations: + - key: sku + operator: Equal + value: gpu + effect: NoSchedule +node-feature-discovery: + worker: + tolerations: + - key: sku + operator: Equal + value: gpu + effect: NoSchedule +"@ } -function Assert-SamplePools { - param( - [Parameter(Mandatory)][object[]]$Pools, - [Parameter(Mandatory)][string]$SystemVmSize, - [switch]$AllowMissingGpu - ) - $system = @($Pools | Where-Object name -EQ 'sysnp') - $gpu = @($Pools | Where-Object name -EQ 'gpunp') - if ($system.Count -ne 1 -or @($Pools | Where-Object { $_.name -notin @('sysnp', 'gpunp') }).Count -or - $gpu.Count -gt 1 -or (-not $AllowMissingGpu -and $gpu.Count -ne 1)) { - throw 'Expected one system pool and one GPU pool, with no additional pools.' +function Get-GuideWorkloadManifest { + param([Parameter(Mandatory)][string]$Manifest) + $Manifest = $Manifest.Replace("`r`n", "`n") + foreach ($required in @('name: tf-matmul-demo', 'image: tensorflow/tensorflow:2.15.0-gpu', + "spec:`n template:", ' restartPolicy: OnFailure', ' c = tf.matmul(a, b)')) { + if (-not $Manifest.Contains($required)) { throw "Upstream workload manifest changed: missing $required." } } + $Manifest = $Manifest.Replace("spec:`n template:", "spec:`n backoffLimit: 0`n activeDeadlineSeconds: 900`n template:") + $Manifest = $Manifest.Replace(' restartPolicy: OnFailure', @' + restartPolicy: Never + runtimeClassName: nvidia + nodeSelector: + agentpool: gpunp +'@) + $Manifest = $Manifest.Replace(' c = tf.matmul(a, b)', @' + tf.config.set_soft_device_placement(False) + assert tf.config.list_physical_devices('GPU'), "No GPU available" + with tf.device('/GPU:0'): + c = tf.matmul(a, b) + assert 'GPU:0' in c.device, "Matmul did not execute on GPU" + print("GPU matmul device:", c.device) +'@) + return $Manifest +} - foreach ($pool in $Pools) { - $size = if ($pool.name -eq 'sysnp') { $SystemVmSize } else { 'Standard_NCC40ads_H100_v5' } - $mode = if ($pool.name -eq 'sysnp') { 'System' } else { 'User' } - if ($pool.vmSize -ne $size -or $pool.count -ne 1 -or $pool.osSku -ne 'Ubuntu2404' -or - $pool.mode -ne $mode -or $pool.enableAutoScaling -eq $true -or $pool.provisioningState -ne 'Succeeded') { - throw "Pool $($pool.name) does not match the single-node sample configuration; it was not modified." +function Stop-ExistingNccCompute { + param([string]$Az, [string]$Subscription, [string]$Location, [switch]$Stop) + $scope = @('--subscription', $Subscription, '--only-show-errors') + $clusters = @(Invoke-CheckedCommand $Az (@('aks', 'list', '-o', 'json') + $scope) | ConvertFrom-Json) + $sets = @(Invoke-CheckedCommand $Az (@('vmss', 'list', '-o', 'json') + $scope) | ConvertFrom-Json) + $vms = @(Invoke-CheckedCommand $Az (@('vm', 'list', '--show-details', '-o', 'json') + $scope) | ConvertFrom-Json) + foreach ($set in @($sets | Where-Object { $_.location -eq $Location -and $_.sku.name -like 'Standard_NCC*' })) { + $owner = @($clusters | Where-Object { $_.nodeResourceGroup -eq $set.resourceGroup }) + if ($owner.Count -gt 1) { throw "Ambiguous AKS owner for $($set.name)." } + if ($owner.Count -eq 1) { + $cluster = $owner[0] + Write-Host "Existing NCC: $($cluster.name), power $($cluster.powerState.code), capacity $($set.sku.capacity)." + if ($cluster.powerState.code -ne 'Stopped') { + if (-not $Stop) { throw 'Running NCC AKS found. Pass -StopExistingNcc to authorize shutdown.' } + Stop-SampleAks $Az $Subscription $cluster.resourceGroup $cluster.name + $state = Invoke-CheckedCommand $Az (@('aks', 'show', '-g', $cluster.resourceGroup, '-n', $cluster.name, + '--query', 'powerState.code', '-o', 'tsv') + $scope) + if ($state -ne 'Stopped') { throw "AKS $($cluster.name) did not stop." } + } + $instances = @(Invoke-CheckedCommand $Az (@('vmss', 'list-instances', '-g', $set.resourceGroup, + '-n', $set.name, '--expand', 'instanceView', '-o', 'json') + $scope) | ConvertFrom-Json) + foreach ($instance in $instances) { + $power = @($instance.instanceView.statuses | Where-Object code -Like 'PowerState/*') + if ($power.Count -ne 1 -or $power[0].code -ne 'PowerState/deallocated') { + throw "Existing AKS NCC instance $($instance.name) is not deallocated." + } + } } - if ($pool.name -eq 'gpunp' -and - ($pool.gpuProfile.driver -ne 'None' -or 'sku=gpu:NoSchedule' -notin $pool.nodeTaints)) { - throw 'GPU pool must disable automatic drivers and retain the GPU scheduling taint.' + else { + $instances = @(Invoke-CheckedCommand $Az (@('vmss', 'list-instances', '-g', $set.resourceGroup, + '-n', $set.name, '--expand', 'instanceView', '-o', 'json') + $scope) | ConvertFrom-Json) + $active = @($instances | Where-Object { + 'PowerState/deallocated' -notin @($_.instanceView.statuses.code) + }) + if ($active.Count) { + if (-not $Stop) { throw "Running standalone NCC scale set $($set.name). Pass -StopExistingNcc." } + Invoke-CheckedCommand $Az (@('vmss', 'deallocate', '-g', $set.resourceGroup, '-n', $set.name, '-o', 'none') + $scope) + $verified = @(Invoke-CheckedCommand $Az (@('vmss', 'list-instances', '-g', $set.resourceGroup, + '-n', $set.name, '--expand', 'instanceView', '-o', 'json') + $scope) | ConvertFrom-Json) + if (@($verified | Where-Object { 'PowerState/deallocated' -notin @($_.instanceView.statuses.code) }).Count) { + throw "Standalone NCC scale set $($set.name) did not deallocate." + } + } } } -} - -function Assert-ConfidentialGpuScaleSet { - param([Parameter(Mandatory)][object[]]$ScaleSets) - $gpu = @($ScaleSets | Where-Object { $_.sku.name -eq 'Standard_NCC40ads_H100_v5' }) - if ($gpu.Count -ne 1) { throw 'Expected exactly one H100 scale set.' } - $security = $gpu[0].virtualMachineProfile.securityProfile - if ($gpu[0].sku.capacity -ne 1 -or $security.securityType -ne 'ConfidentialVM' -or - $security.uefiSettings.secureBootEnabled -ne $true -or $security.uefiSettings.vTpmEnabled -ne $true) { - throw 'H100 scale set must have one ConfidentialVM instance with Secure Boot and vTPM enabled.' + foreach ($vm in @($vms | Where-Object { $_.location -eq $Location -and $_.hardwareProfile.vmSize -like 'Standard_NCC*' })) { + Write-Host "Existing NCC VM: $($vm.name), $($vm.powerState)." + if ($vm.powerState -ne 'VM deallocated') { + if (-not $Stop) { throw "Existing NCC VM $($vm.name) needs -StopExistingNcc." } + Invoke-CheckedCommand $Az (@('vm', 'deallocate', '--ids', $vm.id, '-o', 'none') + $scope) + $state = Invoke-CheckedCommand $Az (@('vm', 'get-instance-view', '--ids', $vm.id, + '--query', "instanceView.statuses[?starts_with(code, 'PowerState/')].code | [0]", '-o', 'tsv') + $scope) + if ($state -ne 'PowerState/deallocated') { throw "VM $($vm.name) did not deallocate." } + } } } diff --git a/aks-samples/aks-gpu-worker/Confidential-GPU-H100-AKS-Onboarding.md b/aks-samples/aks-gpu-worker/Confidential-GPU-H100-AKS-Onboarding.md deleted file mode 100644 index 417ed00..0000000 --- a/aks-samples/aks-gpu-worker/Confidential-GPU-H100-AKS-Onboarding.md +++ /dev/null @@ -1,267 +0,0 @@ -# Confidential H100 on AKS: tested signed-package workaround - -This companion to the [upstream AKS onboarding page](https://github.com/Azure/az-cgpu-onboarding/blob/136262ef00ad75b8e0bc789563f043006fb68f4c/docs/Confidential-GPU-H100-AKS-Onboarding.md) -retains its Azure CLI, GPU Operator and workload-validation workflow, but uses -exact-kernel Ubuntu signed packages instead of the failing driver-container -entrypoint. See [investigation history](README.md) for the original blockers. - -**Experimental diagnostic recipe, not a supported production configuration.** -It runs a privileged pod with host devices and firmware access. Do not use on -shared production nodes. The pod holds UVM persistence for at most four hours; -this is not a reconciled installation across reboots, image upgrades or node -replacement. Keep Secure Boot, vTPM and confidential mode enabled. - -## Tested configuration - -Observed in West Europe on 2026-11-10: - -| Component | Observed value | -|---|---| -| Topology | One system node plus one confidential GPU worker (two nodes total) | -| System node | `Standard_D4ds_v6`, Ubuntu2404 | -| GPU worker | `Standard_NCC40ads_H100_v5`, H100 NVL, Ubuntu2404 | -| Kubernetes | `1.36.5` | -| AKS GPU node image | `AKSUbuntu-2404gen2CVMcontainerd-202610.30.1` | -| GPU node OS / kernel | Ubuntu 24.04.5 LTS / `6.8.0-1071-azure-fde` | -| Runtime | `containerd://2.3.7-4` | -| Security | ConfidentialVM, Secure Boot and vTPM enabled; GPU CC ON / PRODUCTION | -| GPU Operator | `v26.7.2`, **driver operand disabled** | -| NVIDIA signed package | `linux-modules-nvidia-615-open-6.8.0-1071-azure-fde=6.8.0-1071.78` | -| NVIDIA driver / matching libraries | `615.71.09` / `615.71.09-0ubuntu0.24.04.2` | -| GPU verifier | Checksum-pinned Azure archive V4.5.1, package `2.3.0`, strict OCSP | -| CPU/CVM verifier | Checksum-pinned Azure guest CLI `v0.1.0`, MAA | - -These are observed versions, not an AKS image pin. AKS may provision a different -kernel later; the installer must fail rather than load mismatched modules. -Ubuntu 26.04 preview explicitly excludes ConfidentialVM; changing container -Ubuntu versions does not change the host kernel. - -## 1. Prerequisites and isolated context - -Use Azure CLI (tested: 2.92.0), kubectl, Helm and Bash. Run the Bash blocks from -this sample directory in a Linux/WSL shell with those tools configured. -The local shell does not need NVIDIA or Python. HTTPS egress is needed for Azure, -Ubuntu repositories, GitHub, NGC and the verifier's certificate/RIM/OCSP services. -Use an authorized subscription with 40 free H100-family vCPUs plus four system -vCPUs and sufficient physical allocation capacity. - -```bash -set -euo pipefail -subscription='' -location=westeurope -resourceGroup=rg-aks-gpu-worker-example -clusterName=aks-gpu-worker-example -mkdir -p .local/cli-extensions-core -export AZURE_EXTENSION_DIR="$PWD/.local/cli-extensions-core" -export AZURE_EXTENSION_USE_DYNAMIC_INSTALL=no -export KUBECONFIG="$PWD/.local/kubeconfig" -az account show --subscription "$subscription" --query '{name:name,state:state}' -az aks get-versions --subscription "$subscription" --location "$location" -o table -``` - -Keep the extension directory empty. This avoids the observed `aks-preview` -18.0.0b17 / core CLI 2.92.0 `hosted_system_profile` client incompatibility without -modifying shared extensions or the default subscription/kubeconfig. -The development authentication model follows the original sample; use Entra -RBAC and restricted API access for production. - -If needed, register the feature from the upstream guide and wait until it is -`Registered` before creation: - -```bash -az feature register --subscription "$subscription" \ - --namespace Microsoft.ContainerService --name Ubuntu2404Preview -az feature show --subscription "$subscription" \ - --namespace Microsoft.ContainerService --name Ubuntu2404Preview \ - --query properties.state -o tsv -az provider register --subscription "$subscription" --namespace Microsoft.ContainerService -``` - -Registration is asynchronous. Do not proceed while the feature is registering. -For automated quota/topology/ownership checks and bounded registration waits, -use [Deploy.ps1](Deploy.ps1) with `-UseCoreCli` as described in the README. - -## 2. Create the disposable cluster and confidential worker - -Use a **new** resource group. These commands incur charges. Automatic upgrades -are disabled only to control this short-lived reproduction, not as a security -policy. AKS requires a system pool; the confidential worker is a user pool. - -```bash -az group create --subscription "$subscription" --name "$resourceGroup" \ - --location "$location" --tags sample=aks-gpu-worker -az aks create --subscription "$subscription" \ - --resource-group "$resourceGroup" --name "$clusterName" \ - --location "$location" --kubernetes-version 1.36.5 --tier free \ - --nodepool-name sysnp --node-count 1 --node-vm-size Standard_D4ds_v6 \ - --os-sku Ubuntu2404 --node-osdisk-type Managed --node-osdisk-size 128 \ - --enable-managed-identity --generate-ssh-keys \ - --auto-upgrade-channel none --node-os-upgrade-channel None \ - --tags sample=aks-gpu-worker -az aks nodepool add --subscription "$subscription" \ - --resource-group "$resourceGroup" --cluster-name "$clusterName" \ - --name gpunp --mode User --node-count 1 \ - --node-vm-size Standard_NCC40ads_H100_v5 --os-sku Ubuntu2404 \ - --gpu-driver None --node-osdisk-type Managed --node-osdisk-size 128 \ - --node-taints sku=gpu:NoSchedule -``` - -For this sample's existing stopped cluster, skip creation and instead use -`az aks start --subscription "$subscription" --resource-group "$resourceGroup" ---name "$clusterName"`. Verify its identity and topology before resuming. - -```bash -az aks get-credentials --subscription "$subscription" \ - --resource-group "$resourceGroup" --name "$clusterName" \ - --file "$KUBECONFIG" --overwrite-existing -kubectl wait --for=condition=Ready node --all --timeout=600s -kubectl get nodes -o wide -test "$(kubectl get nodes -o name | wc -l)" -eq 2 -test "$(kubectl get nodes -l agentpool=gpunp -o name | wc -l)" -eq 1 -test "$(kubectl get nodes -l agentpool=gpunp \ - -o 'jsonpath={.items[0].status.nodeInfo.kernelVersion}')" = 6.8.0-1071-azure-fde -az aks nodepool get-upgrades --subscription "$subscription" \ - --resource-group "$resourceGroup" --cluster-name "$clusterName" \ - --nodepool-name gpunp -nodeResourceGroup=$(az aks show --subscription "$subscription" \ - --resource-group "$resourceGroup" --name "$clusterName" --query nodeResourceGroup -o tsv) -az vmss list --subscription "$subscription" --resource-group "$nodeResourceGroup" \ - --query "[?sku.name=='Standard_NCC40ads_H100_v5'].{name:name,count:sku.capacity,security:virtualMachineProfile.securityProfile}" -test "$(az vmss list --subscription "$subscription" --resource-group "$nodeResourceGroup" \ - --query "[?sku.name=='Standard_NCC40ads_H100_v5' && sku.capacity==\`1\` && virtualMachineProfile.securityProfile.securityType=='ConfidentialVM' && virtualMachineProfile.securityProfile.uefiSettings.secureBootEnabled==\`true\` && virtualMachineProfile.securityProfile.uefiSettings.vTpmEnabled==\`true\`] | length(@)" -o tsv)" = 1 -``` - -Do not continue if effective ConfidentialVM/Secure Boot/vTPM or the kernel differ. -Neither Kubernetes version nor Ubuntu2404 selects the guide's old 1041 image. - -## 3. Install GPU Operator without its broken driver installer - -```bash -helm upgrade --install gpu-operator gpu-operator \ - --repo https://helm.ngc.nvidia.com/nvidia --version v26.7.2 \ - --namespace gpu-operator --create-namespace \ - --values gpu-operator-experimental-values.yaml --set driver.enabled=false -kubectl apply -f signed-driver-probe.yaml -kubectl wait --for=condition=Ready pod/signed-driver-load --timeout=300s -tr -d '\r' < Install-SignedDriver.sh | - kubectl exec -i signed-driver-load -- bash -``` - -Require `SIGNED_DRIVER_LOADED=1` and a zero pipeline status. The script verifies -the exact kernel, Secure Boot, all five module signatures/version/vermagic and -the pinned module-deb SHA-256. It extracts matching packages in the pod, copies -only versioned firmware to the host, loads crypto dependencies and NVIDIA/UVM, -then starts UVM persistence **before the first GPU query**. It requires CC ON, -PRODUCTION and repeated Ready/persistence checks before exposing the coherent -driver tree at `/run/nvidia/driver`. - -No full host metapackage, DKMS, kernel replacement or signing-key enrollment is -performed. A preexisting driver, driver tree or firmware directory is a hard -stop: inspect and cleanly recycle the disposable node instead of reloading -modules on a GPU with a failed CC cleanup. Keep the probe alive during testing. - -On Windows PowerShell, normalize stdin at the receiving end as well: - -```powershell -Get-Content -Raw .\Install-SignedDriver.sh | - kubectl exec -i signed-driver-load -- bash -c 'tr -d "\r" | bash' -if ($LASTEXITCODE -ne 0) { throw 'Signed-driver setup failed' } -``` - -Use the same pattern for the two attestation scripts below. An immutable Pod -spec cannot be updated in place; do not delete/recreate an active probe to add -mounts, since that also kills its persistence daemon. - -## 4. Require a schedulable GPU and a real CUDA workload - -Give the Operator toolkit and device plugin up to ten minutes to initialize: - -```bash -kubectl wait --for=jsonpath='{.status.allocatable.nvidia\.com/gpu}'=1 \ - node -l agentpool=gpunp --timeout=600s -kubectl get pods -n gpu-operator -o wide -tr -d '\r' < Test-GpuAttestation.sh | - kubectl exec -i signed-driver-load -- bash | tee .local/gpu-attestation-before.log -kubectl apply -f gpu-smoke-test.yaml -kubectl wait --for=condition=Complete job/confidential-gpu-smoke --timeout=240s -kubectl logs job/confidential-gpu-smoke | tee .local/gpu-smoke.log -grep -Fx 'Test PASSED' .local/gpu-smoke.log -``` - -The nonprivileged Job requests and limits **one GPU**, uses runtime class -`nvidia`, and runs NVIDIA's vectorAdd. No CPU fallback is supplied. Helm -`deployed`, node Ready, or an optional monitoring operand's state is not a -substitute for these checks. - -## 5. Verify GPU and CPU/CVM attestation separately - -```bash -tr -d '\r' < Test-GpuAttestation.sh | - kubectl exec -i signed-driver-load -- bash | tee .local/gpu-attestation-after.log -tr -d '\r' < Test-CvmAttestation.sh | - kubectl exec -i signed-driver-load -- bash | tee .local/cvm-attestation-summary.log -``` - -Require zero pipeline exits and `GPU_ATTESTATION_VALIDATED=1` / -`CVM_ATTESTATION_VALIDATED=1`. The GPU script uses an isolated Python 3.12 venv, -checksum-pinned V4.5.1 verifier (`2.3.0`), strict OCSP, a fresh verifier nonce, -the exact positive overall verdict, and successful verification of the expected -physical GPU UUID. There is no `--test_no_gpu` or permissive revocation fallback. -The tested flow passed both before and after the CUDA workload. -The published scripts were replayed unchanged on a second fresh worker and -passed again. Optional DCGM exporter readiness remained `0/1`; this recipe -does not claim complete monitoring health. - -The CPU script separately makes a real MAA request using checksum-pinned Azure -guest CLI `v0.1.0`, and requires decoded `sevsnpvm` and -`azure-compliant-cvm` claims. It is not a second independent JWT verification -implementation. Host `/dev`, read-only `/sys/firmware`, and read-only -`/sys/kernel/security` expose the Azure vTPM evidence. Without the TPM event log, -MAA returned `TcgLogs is empty in attestation request`. Sysfs stat size can be -zero: the script checks readable bytes with `wc -c`, not `test -s`. - -Full results stay inside the pod at `/tmp/signed-attestation/gpu-result.log` -and `/tmp/signed-attestation/cpu-result.log`; they may include attestation -tokens. Console summaries omit the GPU token and print only selected CPU -claims. Keep any copied full evidence gitignored and access-controlled. -The GPU verifier reports future deprecation in favour of `nvattest`; migration -to that alternative is not validated by this recipe. - -## 6. Stop and verify deallocation - -Capture needed evidence before cleanup. On the successful probe, remove only -its four versioned firmware files, remove the test Job and pod, then stop -promptly (deleting the pod terminates persistence): - -```bash -kubectl delete job confidential-gpu-smoke --ignore-not-found -kubectl exec signed-driver-load -- bash -c 'set -eu -rm -- /lib/firmware/nvidia/615.71.09/gsp_ga10x.bin \ - /lib/firmware/nvidia/615.71.09/gsp_tu10x.bin \ - /lib/firmware/nvidia/615.71.09/ucodes_ga10x.bin \ - /lib/firmware/nvidia/615.71.09/ucodes_tu10x.bin -rmdir /lib/firmware/nvidia/615.71.09' -kubectl delete pod signed-driver-load -az aks stop --subscription "$subscription" \ - --resource-group "$resourceGroup" --name "$clusterName" -az aks show --subscription "$subscription" \ - --resource-group "$resourceGroup" --name "$clusterName" \ - --query '{power:powerState.code,state:provisioningState}' -az vmss list --subscription "$subscription" --resource-group "$nodeResourceGroup" \ - --query '[].{name:name,capacity:sku.capacity}' -o table -``` - -Require **Stopped / Succeeded** and **both VMSS capacities zero**. If an earlier -step failed, still stop the cluster; inspect which exact probe-owned files -exist before removing them. Do not attempt live recovery by loading mismatched -modules or disabling security. Retained disks/networking may still cost money. -GPU Operator is deliberately left with `driver.enabled=false`. Restarting -compute alone does not recreate the ephemeral driver installation; repeat -clean-node checks and setup. - -When inspection is finished, explicitly deleting this disposable resource -group removes the cluster and its managed resources: - -```bash -az group delete --subscription "$subscription" --name "$resourceGroup" --yes -``` diff --git a/aks-samples/aks-gpu-worker/Deploy.ps1 b/aks-samples/aks-gpu-worker/Deploy.ps1 index 3c1d8bf..e797119 100644 --- a/aks-samples/aks-gpu-worker/Deploy.ps1 +++ b/aks-samples/aks-gpu-worker/Deploy.ps1 @@ -2,182 +2,290 @@ [CmdletBinding()] param( [Parameter(Mandatory)][guid]$SubscriptionId, - [Parameter(Mandatory)][ValidatePattern('^[a-zA-Z0-9_-]+$')][string]$ResourceGroup, - [Parameter(Mandatory)][ValidatePattern('^[a-zA-Z0-9-]+$')][string]$ClusterName, + [ValidatePattern('^[a-zA-Z0-9_-]+$')][string]$ResourceGroup = "rg-aks-gpu-worker-$(Get-Date -Format yyyyMMddHHmmss)", + [ValidatePattern('^[a-zA-Z0-9-]+$')][string]$ClusterName = "aks-gpu-worker-$(Get-Date -Format yyyyMMddHHmmss)", [string]$Location = 'westeurope', - [string]$KubernetesVersion = '1.36.5', - [string]$SystemVmSize = 'Standard_D4ds_v6', - [ValidateSet('Ubuntu2404')][string]$SystemOsSku = 'Ubuntu2404', - [string]$GpuZone, + [string]$KubernetesVersion, + [string]$MinimumKernel = '6.8.0-1071-azure-fde', [string]$OutputDirectory = (Join-Path $PSScriptRoot '.local'), + [switch]$StopExistingNcc, [switch]$PreflightOnly, - [switch]$RegisterPreview, - [switch]$UseCoreCli, - [switch]$Resume + [switch]$KeepRunning ) . (Join-Path $PSScriptRoot 'Common.ps1') -$previousExtensionDirectory = $env:AZURE_EXTENSION_DIR -$previousDynamicInstall = $env:AZURE_EXTENSION_USE_DYNAMIC_INSTALL -try { -$az = Resolve-AzureCli -$scope = @('--subscription', $SubscriptionId.ToString(), '--only-show-errors') -New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null -$OutputDirectory = (Resolve-Path $OutputDirectory).Path -if ($UseCoreCli) { - $env:AZURE_EXTENSION_DIR = Join-Path $OutputDirectory 'cli-extensions-core' - New-Item -ItemType Directory -Path $env:AZURE_EXTENSION_DIR -Force | Out-Null - if (@(Get-ChildItem -LiteralPath $env:AZURE_EXTENSION_DIR -Force).Count) { - throw 'Core CLI extension directory must be empty; inspect it before continuing.' - } - $env:AZURE_EXTENSION_USE_DYNAMIC_INSTALL = 'no' +$oldExtensions = $env:AZURE_EXTENSION_DIR +$oldDynamicInstall = $env:AZURE_EXTENSION_USE_DYNAMIC_INSTALL +$created = $false +$result = [ordered]@{ + StartedAtUtc = [DateTime]::UtcNow.ToString('o') + SubscriptionId = $SubscriptionId.ToString() + ResourceGroup = $ResourceGroup + ClusterName = $ClusterName + Location = $Location + Status = 'Preparing' + VendorSupportValidated = $false + DriverPath = 'Guide GPU Operator v25.3.1, NVIDIA branch 580 only' } -$account = Invoke-CheckedCommand $az (@('account', 'show', '-o', 'json') + $scope) | ConvertFrom-Json -if ($account.state -ne 'Enabled') { throw 'The target subscription is not enabled.' } -if ($UseCoreCli) { - $cliVersion = (Invoke-CheckedCommand $az @('version', '-o', 'json') | ConvertFrom-Json).'azure-cli' - Write-Host "Using built-in Azure CLI $cliVersion AKS commands in $Location (extensions isolated)." +function Invoke-Az { + param([string[]]$Arguments) + Invoke-CheckedCommand $az ($Arguments + $scope) } -else { - $extension = Invoke-CheckedCommand $az @('extension', 'show', '--name', 'aks-preview', '-o', 'json') | ConvertFrom-Json - Write-Host "Using Azure CLI aks-preview $($extension.version) in $Location." + +function Invoke-Kube { + param([string[]]$Arguments) + Invoke-CheckedCommand kubectl (@('--kubeconfig', $kubeConfig) + $Arguments) } -$versions = Invoke-CheckedCommand $az (@('aks', 'get-versions', '--location', $Location, '-o', 'json') + $scope) | ConvertFrom-Json -$patches = @($versions.values | ForEach-Object { $_.patchVersions.PSObject.Properties.Name }) -if ($KubernetesVersion -notin $patches) { throw "Kubernetes $KubernetesVersion is not offered in $Location." } -$exists = Invoke-CheckedCommand $az (@('group', 'exists', '--name', $ResourceGroup, '-o', 'tsv') + $scope) -$existingPools = @() -$cluster = @() -if ($exists -eq 'true') { - if (-not $Resume) { throw 'Resource group already exists. Use a new name, or -Resume for this sample.' } - $group = Invoke-CheckedCommand $az (@('group', 'show', '--name', $ResourceGroup, '-o', 'json') + $scope) | ConvertFrom-Json -AsHashtable - if ($group.tags.sample -ne 'aks-gpu-worker' -or $group.location -ne $Location) { - throw 'Existing resource group is not a matching sample group; nothing was modified.' - } - $clusters = @(Invoke-CheckedCommand $az (@('aks', 'list', '--resource-group', $ResourceGroup, '-o', 'json') + $scope) | ConvertFrom-Json -AsHashtable) - if (@($clusters | Where-Object name -NE $ClusterName).Count) { throw 'Resource group contains a different cluster.' } - $cluster = @($clusters | Where-Object name -EQ $ClusterName) - if ($cluster.Count) { - if ($cluster[0].tags.sample -ne 'aks-gpu-worker' -or $cluster[0].location -ne $Location -or - $cluster[0].kubernetesVersion -ne $KubernetesVersion -or $cluster[0].provisioningState -ne 'Succeeded' -or - $cluster[0].autoUpgradeProfile.upgradeChannel -ne 'none' -or - $cluster[0].autoUpgradeProfile.nodeOsUpgradeChannel -ne 'None') { - throw 'Existing cluster is not a successful matching sample cluster; inspect it before continuing.' +function Wait-GpuJob { + param([string]$Name, [string]$Namespace, [int]$TimeoutSeconds = 600) + $deadline = [DateTime]::UtcNow.AddSeconds($TimeoutSeconds) + do { + $job = Invoke-Kube @('get', 'job', $Name, '-n', $Namespace, '-o', 'json') | ConvertFrom-Json + $property = $job.status.PSObject.Properties['conditions'] + $conditions = @(if ($property) { $property.Value }) + if (@($conditions | Where-Object { $_ -and $_.type -eq 'Failed' -and $_.status -eq 'True' }).Count) { + Save-GpuJobDiagnostics $Name $Namespace + throw "Job $Namespace/$Name failed. Inspect its pods using the dedicated kubeconfig." } - $existingPools = @(Invoke-CheckedCommand $az (@('aks', 'nodepool', 'list', '--resource-group', $ResourceGroup, - '--cluster-name', $ClusterName, '-o', 'json') + $scope) | ConvertFrom-Json -AsHashtable) - Assert-SamplePools -Pools $existingPools -SystemVmSize $SystemVmSize -AllowMissingGpu + if (@($conditions | Where-Object { $_ -and $_.type -eq 'Complete' -and $_.status -eq 'True' }).Count) { return } + Start-Sleep 10 + } while ([DateTime]::UtcNow -lt $deadline) + Save-GpuJobDiagnostics $Name $Namespace + throw "Job $Namespace/$Name did not complete within $TimeoutSeconds seconds." +} + +function Save-GpuJobDiagnostics { + param([string]$Name, [string]$Namespace) + try { + $pods = Invoke-Kube @('get', 'pods', '-n', $Namespace, '-l', "job-name=$Name", '-o', 'json') + $pods | Set-Content (Join-Path $runDirectory "$Name-failed-pods.json") + $log = Invoke-Kube @('logs', '-n', $Namespace, "job/$Name", '--tail=200') + (($log -join "`n") -replace '(?s)Entity Attestation Token:.*', '[Attestation token omitted]') | + Set-Content (Join-Path $runDirectory "$Name-failed-log.txt") } + catch { Write-Warning "Unable to capture complete diagnostics for ${Namespace}/${Name}: $($_.Exception.Message)" } } -$skus = @(Invoke-CheckedCommand $az (@('vm', 'list-skus', '--location', $Location, - '--resource-type', 'virtualMachines', '--all', '-o', 'json') + $scope) | ConvertFrom-Json) -$usage = @(Invoke-CheckedCommand $az (@('vm', 'list-usage', '--location', $Location, '-o', 'json') + $scope) | ConvertFrom-Json) -$required = @{} -$skuEvidence = @(foreach ($size in @($SystemVmSize, 'Standard_NCC40ads_H100_v5')) { - $sku = @($skus | Where-Object name -EQ $size) - if ($sku.Count -ne 1) { throw "SKU $size was not found in $Location." } - $sku = $sku[0] - $regionBlocks = @($sku.restrictions | Where-Object type -EQ 'Location') - if ($regionBlocks.Count) { throw "SKU $size has a regional subscription restriction: $($regionBlocks | ConvertTo-Json -Depth 10 -Compress)" } - if ($size -eq 'Standard_NCC40ads_H100_v5' -and $GpuZone) { - if ($GpuZone -notin @($sku.locationInfo | ForEach-Object zones)) { throw "GPU zone $GpuZone is not offered." } - $zoneBlocks = @($sku.restrictions | Where-Object { - $_.type -eq 'Zone' -and $GpuZone -in $_.restrictionInfo.zones - }) - if ($zoneBlocks.Count) { throw "GPU zone $GpuZone is restricted for this subscription." } +try { + Assert-MinimumKernel $MinimumKernel + $az = Resolve-AzureCli + foreach ($tool in @('kubectl', 'helm', 'ssh-keygen')) { Get-Command $tool -CommandType Application -ErrorAction Stop | Out-Null } + New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null + $OutputDirectory = (Resolve-Path $OutputDirectory).Path + $runDirectory = Join-Path $OutputDirectory "$ClusterName-$(Get-Date -Format yyyyMMddHHmmssfff)" + New-Item -ItemType Directory -Path $runDirectory | Out-Null + $kubeConfig = Join-Path $runDirectory 'kubeconfig' + $scope = @('--subscription', $SubscriptionId.ToString(), '--only-show-errors') + $env:AZURE_EXTENSION_DIR = Join-Path $runDirectory 'cli-extensions-core' + $env:AZURE_EXTENSION_USE_DYNAMIC_INSTALL = 'no' + New-Item -ItemType Directory -Path $env:AZURE_EXTENSION_DIR | Out-Null + $result.EvidenceDirectory = $runDirectory + + $versions = Invoke-Az @('aks', 'get-versions', '-l', $Location, '-o', 'json') | ConvertFrom-Json + if (-not $KubernetesVersion) { + $default = @($versions.values | Where-Object { $_.isDefault -eq $true -and $_.isPreview -ne $true }) + if ($default.Count -ne 1) { throw 'Expected exactly one non-preview default AKS minor version.' } + $KubernetesVersion = Get-LatestKubernetesPatch ([pscustomobject]@{ values = $default }) } - $cores = [int]($sku.capabilities | Where-Object name -EQ 'vCPUs').value - if (-not $required.ContainsKey($sku.family)) { $required[$sku.family] = 0 } - $poolName = if ($size -eq 'Standard_NCC40ads_H100_v5') { 'gpunp' } else { 'sysnp' } - if (-not @($existingPools | Where-Object name -EQ $poolName).Count) { - $required[$sku.family] += $cores + $stable = @($versions.values | Where-Object { $_.isPreview -ne $true } | + ForEach-Object { $_.patchVersions.PSObject.Properties.Name }) + if ($KubernetesVersion -notin $stable) { throw "Non-preview Kubernetes $KubernetesVersion is not offered." } + $result.KubernetesVersion = $KubernetesVersion + $exists = Invoke-Az @('group', 'exists', '-n', $ResourceGroup, '-o', 'tsv') + if ($exists -ne 'false') { throw 'Use a new resource group; this script never adopts or overwrites existing resources.' } + + if (-not $PreflightOnly) { + Stop-ExistingNccCompute -Az $az -Subscription $SubscriptionId -Location $Location -Stop:$StopExistingNcc } - $sku -}) -$total = 0 -foreach ($family in $required.Keys) { - $quota = @($usage | Where-Object { $_.name.value -eq $family }) - if ($quota.Count -ne 1 -or ([int]$quota[0].limit - [int]$quota[0].currentValue) -lt $required[$family]) { - throw "Insufficient $family quota for $($required[$family]) new vCPUs." + Test-SamplePreflight -Az $az -Subscription $SubscriptionId -Location $Location ` + -KubernetesVersion $KubernetesVersion -OutputDirectory (Join-Path $runDirectory 'preflight') + Invoke-CheckedCommand helm @('pull', 'gpu-operator', '--repo', 'https://helm.ngc.nvidia.com/nvidia', + '--version', 'v25.3.1', '--destination', $runDirectory) + $chartPath = Join-Path $runDirectory 'gpu-operator-v25.3.1.tgz' + $result.ChartSha256 = (Get-FileHash $chartPath -Algorithm SHA256).Hash + if ($PreflightOnly) { + $result.Status = 'PreflightPassed' + return } - $total += $required[$family] -} -$regional = $usage | Where-Object { $_.name.value -eq 'cores' } -if (([int]$regional.limit - [int]$regional.currentValue) -lt $total) { throw "Insufficient regional vCPU quota for $total cores." } -$featureArgs = @('feature', 'show', '--namespace', 'Microsoft.ContainerService', '--name', 'Ubuntu2404Preview') -$feature = Invoke-CheckedCommand $az ($featureArgs + @('--query', 'properties.state', '-o', 'tsv') + $scope) -if ($feature -ne 'Registered' -and $RegisterPreview -and -not $PreflightOnly) { - Invoke-CheckedCommand $az (@('feature', 'register', '--namespace', 'Microsoft.ContainerService', - '--name', 'Ubuntu2404Preview', '-o', 'none') + $scope) - $deadline = [DateTime]::UtcNow.AddMinutes(20) + $sshKey = Join-Path $runDirectory 'ssh-key' + New-SampleSshKey -Path $sshKey + Invoke-Az @('group', 'create', '-n', $ResourceGroup, '-l', $Location, '--tags', 'sample=aks-gpu-worker', '-o', 'none') + $created = $true + Invoke-Az @('aks', 'create', '-g', $ResourceGroup, '-n', $ClusterName, '-l', $Location, + '--kubernetes-version', $KubernetesVersion, '--nodepool-name', 'sysnp', '--node-count', '1', + '--node-vm-size', 'Standard_D4ds_v6', '--os-sku', 'Ubuntu2404', '--ssh-key-value', "$sshKey.pub", + '--enable-managed-identity', '--network-plugin', 'azure', '--network-plugin-mode', 'overlay', + '--auto-upgrade-channel', 'none', '--node-os-upgrade-channel', 'None', + '--tags', 'sample=aks-gpu-worker', '-o', 'none') + Wait-SampleAksReady $az $SubscriptionId $ResourceGroup $ClusterName + Invoke-Az @('aks', 'nodepool', 'add', '-g', $ResourceGroup, '--cluster-name', $ClusterName, + '-n', 'gpunp', '--node-count', '1', '--node-vm-size', 'Standard_NCC40ads_H100_v5', + '--os-sku', 'Ubuntu2404', '--gpu-driver', 'None', '--node-taints', 'sku=gpu:NoSchedule', '-o', 'none') + Wait-SampleAksReady $az $SubscriptionId $ResourceGroup $ClusterName + + foreach ($poolName in @('sysnp', 'gpunp')) { + $pool = Invoke-Az @('aks', 'nodepool', 'show', '-g', $ResourceGroup, '--cluster-name', $ClusterName, + '-n', $poolName, '-o', 'json') | ConvertFrom-Json + $upgrades = Invoke-Az @('aks', 'nodepool', 'get-upgrades', '-g', $ResourceGroup, + '--cluster-name', $ClusterName, '-n', $poolName, '-o', 'json') | ConvertFrom-Json + if (-not $upgrades.latestNodeImageVersion) { throw "AKS did not report the latest image for $poolName." } + if ($pool.nodeImageVersion -ne $upgrades.latestNodeImageVersion) { + Invoke-Az @('aks', 'nodepool', 'upgrade', '-g', $ResourceGroup, '--cluster-name', $ClusterName, + '-n', $poolName, '--node-image-only', '--yes', '-o', 'none') + $pool = Invoke-Az @('aks', 'nodepool', 'show', '-g', $ResourceGroup, '--cluster-name', $ClusterName, + '-n', $poolName, '-o', 'json') | ConvertFrom-Json + } + if ($pool.nodeImageVersion -ne $upgrades.latestNodeImageVersion) { throw "$poolName is not on the latest offered image." } + $result["${poolName}Image"] = $pool.nodeImageVersion + } + $pools = @(Invoke-Az @('aks', 'nodepool', 'list', '-g', $ResourceGroup, + '--cluster-name', $ClusterName, '-o', 'json') | ConvertFrom-Json) + Assert-SamplePools -Pools $pools -SystemVmSize Standard_D4ds_v6 + $cluster = Invoke-Az @('aks', 'show', '-g', $ResourceGroup, '-n', $ClusterName, '-o', 'json') | ConvertFrom-Json + $sets = @(Invoke-Az @('vmss', 'list', '-g', $cluster.nodeResourceGroup, '-o', 'json') | ConvertFrom-Json) + Assert-ConfidentialGpuScaleSet $sets + $sets | ConvertTo-Json -Depth 40 | Set-Content (Join-Path $runDirectory 'scale-sets.json') + Invoke-Az @('aks', 'get-credentials', '-g', $ResourceGroup, '-n', $ClusterName, '--file', $kubeConfig, '-o', 'none') + Invoke-Kube @('wait', 'node', '-l', 'agentpool=gpunp', '--for=condition=Ready', '--timeout=900s') + $node = Get-GpuNode -KubeConfig $kubeConfig + $result.Kernel = $node.status.nodeInfo.kernelVersion + $result.OsImage = $node.status.nodeInfo.osImage + if ($result.OsImage -notlike 'Ubuntu 24.04*') { throw "Unexpected GPU OS: $($result.OsImage)." } + Assert-MinimumKernel $result.Kernel $MinimumKernel + $node | ConvertTo-Json -Depth 30 | Set-Content (Join-Path $runDirectory 'gpu-node.json') + + $images = @(Get-ExactDriverImages -Kernel $result.Kernel -Branches @('580')) + $images | ConvertTo-Json -Depth 5 | Set-Content (Join-Path $runDirectory 'driver-images.json') + $image = @($images | Where-Object Available) + if ($image.Count -ne 1) { + $result.Status = 'BlockedMissingGuideDriver' + throw "The guide's image nvcr.io/nvidia/driver:580-$($result.Kernel)-ubuntu24.04 is absent (MANIFEST_UNKNOWN). No alternative driver or kernel will be installed." + } + $result.DriverImage = "nvcr.io/nvidia/driver:$($image[0].Tag)" + $result.DriverDigest = $image[0].Digest + $values = Get-GuideOperatorValues $result.Kernel + $valuesPath = Join-Path $runDirectory 'operator-values.yaml' + $values | Set-Content $valuesPath + Invoke-CheckedCommand helm @('install', 'gpu-operator', $chartPath, '--kubeconfig', $kubeConfig, + '--namespace', 'gpu-operator', '--create-namespace', '-f', $valuesPath, '--wait', '--timeout', '20m') + $deadline = [DateTime]::UtcNow.AddMinutes(10) do { - Start-Sleep -Seconds 20 - $feature = Invoke-CheckedCommand $az ($featureArgs + @('--query', 'properties.state', '-o', 'tsv') + $scope) - } while ($feature -ne 'Registered' -and [DateTime]::UtcNow -lt $deadline) - if ($feature -eq 'Registered') { - Invoke-CheckedCommand $az (@('provider', 'register', '--namespace', 'Microsoft.ContainerService', - '--wait', '-o', 'none') + $scope) + $node = Get-GpuNode -KubeConfig $kubeConfig + $gpu = $node.status.allocatable.PSObject.Properties['nvidia.com/gpu'] + if ($gpu -and $gpu.Value -eq '1') { break } + Start-Sleep 10 + } while ([DateTime]::UtcNow -lt $deadline) + if (-not $gpu -or $gpu.Value -ne '1') { throw 'The worker did not expose exactly one allocatable GPU.' } + $driverPods = (Invoke-Kube @('get', 'pods', '-n', 'gpu-operator', '-l', 'app=nvidia-driver-daemonset', '-o', 'json') | + ConvertFrom-Json).items + $driver = @($driverPods | Where-Object { $_.spec.nodeName -eq $node.metadata.name }) + if ($driver.Count -ne 1) { throw 'Expected one GPU driver pod on the confidential worker.' } + $driverName = $driver[0].metadata.name + foreach ($operation in @('-f', '-e', '-srs', '-q')) { + $command = @('exec', '-n', 'gpu-operator', $driverName, '-c', 'nvidia-driver-ctr', '--', 'nvidia-smi', 'conf-compute', $operation) + if ($operation -eq '-srs') { $command += '1' } + $output = Invoke-Kube $command + $output | Set-Content (Join-Path $runDirectory "cc-$($operation.TrimStart('-')).txt") + if ($operation -eq '-f' -and "$output" -notmatch 'CC status: ON') { throw 'GPU confidential mode is not ON.' } + if ($operation -eq '-e' -and "$output" -notmatch 'CC Environment: PRODUCTION') { throw 'GPU is not in PRODUCTION mode.' } + if ($operation -eq '-q' -and "$output" -notmatch 'CC GPUs Ready State\s*: Ready') { throw 'GPU is not Ready.' } } -} -$preflight = [ordered]@{ - CheckedAtUtc = [DateTime]::UtcNow.ToString('o') - Location = $Location - KubernetesVersion = $KubernetesVersion - PreviewState = "$feature" - Skus = $skuEvidence - Quota = @($usage | Where-Object { $_.name.value -in @($required.Keys) + @('cores') }) -} -$preflight | ConvertTo-Json -Depth 20 | Set-Content (Join-Path $OutputDirectory 'preflight.json') -if ($feature -ne 'Registered') { throw "Ubuntu2404Preview is $feature. Rerun with -RegisterPreview or arrange subscription registration." } -if ($PreflightOnly) { Write-Host 'Preflight passed; no Azure resources were changed.'; return } -if ($exists -ne 'true') { - Invoke-CheckedCommand $az (@('group', 'create', '--name', $ResourceGroup, '--location', $Location, - '--tags', 'sample=aks-gpu-worker', '-o', 'none') + $scope) -} -if ($cluster.Count -eq 0) { - Invoke-CheckedCommand $az (@('aks', 'create', '--resource-group', $ResourceGroup, '--name', $ClusterName, - '--location', $Location, '--kubernetes-version', $KubernetesVersion, '--tier', 'free', - '--nodepool-name', 'sysnp', '--node-count', '1', '--node-vm-size', $SystemVmSize, - '--os-sku', $SystemOsSku, '--node-osdisk-type', 'Managed', '--node-osdisk-size', '128', - '--enable-managed-identity', '--generate-ssh-keys', '--auto-upgrade-channel', 'none', - '--node-os-upgrade-channel', 'None', '--tags', 'sample=aks-gpu-worker', '-o', 'none') + $scope) -} -$pools = @(Invoke-CheckedCommand $az (@('aks', 'nodepool', 'list', '--resource-group', $ResourceGroup, - '--cluster-name', $ClusterName, '-o', 'json') + $scope) | ConvertFrom-Json -AsHashtable) -Assert-SamplePools -Pools $pools -SystemVmSize $SystemVmSize -AllowMissingGpu -$gpu = @($pools | Where-Object name -EQ 'gpunp') -if ($gpu.Count -eq 0) { - $arguments = @('aks', 'nodepool', 'add', '--resource-group', $ResourceGroup, '--cluster-name', $ClusterName, - '--name', 'gpunp', '--mode', 'User', '--node-count', '1', '--node-vm-size', 'Standard_NCC40ads_H100_v5', - '--os-sku', 'Ubuntu2404', '--gpu-driver', 'None', '--node-osdisk-type', 'Managed', - '--node-osdisk-size', '128', '--node-taints', 'sku=gpu:NoSchedule', '-o', 'none') - if ($GpuZone) { $arguments += @('--zones', $GpuZone) } - Invoke-CheckedCommand $az ($arguments + $scope) + $guideCommit = '136262ef00ad75b8e0bc789563f043006fb68f4c' + $manifestUrl = "https://raw.githubusercontent.com/Azure/az-cgpu-onboarding/$guideCommit/docs/cgpu-aks-attestation.yaml" + $attestation = (Invoke-WebRequest $manifestUrl).Content + if ($attestation -notmatch 'python3 -m verifier.cc_admin') { throw 'Upstream attestation manifest changed; review it before continuing.' } + $identity = Invoke-Kube @('exec', '-n', 'gpu-operator', $driverName, '-c', 'nvidia-driver-ctr', + '--', 'nvidia-smi', '--query-gpu=uuid', '--format=csv,noheader') + $uuid = ("$identity").Trim() + if ($uuid -notmatch '^GPU-[0-9a-fA-F-]+$') { throw 'Expected exactly one GPU UUID.' } + $result.GpuUuid = $uuid + $attestation = $attestation.Replace('./prodtest/bin/python3 -m verifier.cc_admin', + './prodtest/bin/python3 -m verifier.cc_admin --ocsp_attestation_settings strict') + # Pin scheduling and deadline while preserving the guide's verifier installation. + $attestation = $attestation.Replace(' backoffLimit: 0', " backoffLimit: 0`n activeDeadlineSeconds: 900") + $attestation = $attestation.Replace(' restartPolicy: Never', @" + restartPolicy: Never + nodeSelector: + agentpool: gpunp + tolerations: + - key: sku + operator: Equal + value: gpu + effect: NoSchedule +"@) + $attestationPath = Join-Path $runDirectory 'attestation.yaml' + $attestation | Set-Content $attestationPath + foreach ($phase in @('before', 'after')) { + Invoke-Kube @('apply', '-f', $attestationPath) + Wait-GpuJob -Name local-gpu-attestation -Namespace attest -TimeoutSeconds 950 + $log = Invoke-Kube @('logs', '-n', 'attest', 'job/local-gpu-attestation') + $text = $log -join "`n" + $redacted = $text -replace '(?s)Entity Attestation Token:.*', '[Attestation token omitted]' + $redacted | Set-Content (Join-Path $runDirectory "attestation-$phase.txt") + if ($text -notmatch 'GPU Attestation is Successful\.') { throw "GPU attestation $phase did not report success." } + if ($text -notmatch "(?i)$([regex]::Escape($uuid))") { throw "Attestation $phase did not identify this worker's GPU." } + $result["Attestation${phase}Passed"] = $true + Invoke-Kube @('delete', 'job', 'local-gpu-attestation', '-n', 'attest', '--wait=true') + if ($phase -eq 'before') { + $workloadUrl = "https://raw.githubusercontent.com/Azure/az-cgpu-onboarding/$guideCommit/docs/samples-tf-matmul-demo.yaml" + $smoke = Get-GuideWorkloadManifest (Invoke-WebRequest $workloadUrl).Content + $smokePath = Join-Path $runDirectory 'gpu-smoke-test.yaml' + $smoke | Set-Content $smokePath + Invoke-Kube @('apply', '-f', $smokePath) + Wait-GpuJob -Name tf-matmul-demo -Namespace default -TimeoutSeconds 950 + $log = Invoke-Kube @('logs', 'job/tf-matmul-demo') + $log | Set-Content (Join-Path $runDirectory 'gpu-smoke-test.txt') + if (($log -join "`n") -notmatch 'GPU matmul device:.*GPU:0') { throw 'Guide TensorFlow workload did not report GPU execution.' } + $result.GpuWorkloadPassed = $true + } + } + $result.Status = 'Succeeded' } -$pools = Invoke-CheckedCommand $az (@('aks', 'nodepool', 'list', '--resource-group', $ResourceGroup, - '--cluster-name', $ClusterName, '-o', 'json') + $scope) -$pools | Set-Content (Join-Path $OutputDirectory 'nodepools.json') -Assert-SamplePools -Pools @($pools | ConvertFrom-Json -AsHashtable) -SystemVmSize $SystemVmSize -$nodeGroup = Invoke-CheckedCommand $az (@('aks', 'show', '--resource-group', $ResourceGroup, - '--name', $ClusterName, '--query', 'nodeResourceGroup', '-o', 'tsv') + $scope) -$scaleSets = Invoke-CheckedCommand $az (@('vmss', 'list', '--resource-group', "$nodeGroup", '-o', 'json') + $scope) -$scaleSets | Set-Content (Join-Path $OutputDirectory 'scalesets.json') -Assert-ConfidentialGpuScaleSet -ScaleSets @($scaleSets | ConvertFrom-Json -AsHashtable) -$kubeconfig = Join-Path $OutputDirectory 'kubeconfig' -Invoke-CheckedCommand $az (@('aks', 'get-credentials', '--resource-group', $ResourceGroup, - '--name', $ClusterName, '--file', $kubeconfig, '--overwrite-existing') + $scope) -Invoke-CheckedCommand kubectl @('--kubeconfig', $kubeconfig, 'wait', '--for=condition=Ready', - 'nodes', '-l', 'agentpool=gpunp', '--timeout=600s') -$node = Get-GpuNode -KubeConfig $kubeconfig -$node | ConvertTo-Json -Depth 30 | Set-Content (Join-Path $OutputDirectory 'gpu-node.json') -& (Join-Path $PSScriptRoot 'Test-DriverCompatibility.ps1') -KubeConfig $kubeconfig ` - -OutputPath (Join-Path $OutputDirectory 'driver-compatibility.json') -RequireAvailable +catch { + $result.Error = $_.Exception.Message + if ($result.Status -notlike 'Blocked*') { $result.Status = 'Failed' } + throw } finally { - $env:AZURE_EXTENSION_DIR = $previousExtensionDirectory - $env:AZURE_EXTENSION_USE_DYNAMIC_INSTALL = $previousDynamicInstall + try { + if ($created -and -not $KeepRunning) { + $clusters = @(Invoke-Az @('aks', 'list', '-g', $ResourceGroup, '-o', 'json') | ConvertFrom-Json) + if ($clusters.Count -eq 1) { + Wait-SampleAksReady $az $SubscriptionId $ResourceGroup $ClusterName + Stop-SampleAks $az $SubscriptionId $ResourceGroup $ClusterName + $state = Invoke-Az @('aks', 'show', '-g', $ResourceGroup, '-n', $ClusterName, + '--query', 'powerState.code', '-o', 'tsv') + $result.FinalPowerState = "$state" + if ($state -ne 'Stopped') { throw 'Disposable AKS did not stop; inspect it immediately for ongoing NCC charges.' } + $nodeGroup = $clusters[0].nodeResourceGroup + $sets = @(Invoke-Az @('vmss', 'list', '-g', $nodeGroup, '-o', 'json') | ConvertFrom-Json) + foreach ($set in $sets) { + $instances = @(Invoke-Az @('vmss', 'list-instances', '-g', $nodeGroup, '-n', $set.name, + '--expand', 'instanceView', '-o', 'json') | ConvertFrom-Json) + if (@($instances | Where-Object { 'PowerState/deallocated' -notin @($_.instanceView.statuses.code) }).Count) { + throw "Scale set $($set.name) still has allocated instances after AKS stop." + } + } + $result.ComputeDeallocated = $true + } + } + elseif ($created) { $result.FinalPowerState = 'KeptRunningByRequest' } + } + catch { + $result.CleanupError = $_.Exception.Message + $result.Status = 'CleanupFailed' + throw + } + finally { + $result.FinishedAtUtc = [DateTime]::UtcNow.ToString('o') + if ($result.Contains('EvidenceDirectory')) { + $result | ConvertTo-Json -Depth 10 | Set-Content (Join-Path $runDirectory 'result.json') + Write-Host "Evidence: $runDirectory" + } + $env:AZURE_EXTENSION_DIR = $oldExtensions + $env:AZURE_EXTENSION_USE_DYNAMIC_INSTALL = $oldDynamicInstall + } } diff --git a/aks-samples/aks-gpu-worker/Infrastructure.ps1 b/aks-samples/aks-gpu-worker/Infrastructure.ps1 new file mode 100644 index 0000000..4214009 --- /dev/null +++ b/aks-samples/aks-gpu-worker/Infrastructure.ps1 @@ -0,0 +1,80 @@ +#Requires -Version 7.2 +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Resolve-AzureCli { + $commands = @(Get-Command az -CommandType Application -ErrorAction SilentlyContinue) + if ($commands.Count) { return $commands[0].Source } + if ($IsWindows) { + foreach ($root in @($env:ProgramFiles, ${env:ProgramFiles(x86)})) { + if ($root) { + $candidate = Join-Path $root 'Microsoft SDKs\Azure\CLI2\wbin\az.cmd' + if (Test-Path $candidate) { return $candidate } + } + } + } + throw 'Azure CLI is required. Install it and run az login before retrying.' +} + +function Invoke-CheckedCommand { + param( + [Parameter(Mandatory)][string]$Command, + [Parameter(Mandatory)][string[]]$ArgumentList + ) + $output = & $Command @ArgumentList + if ($LASTEXITCODE -ne 0) { + throw "$Command $($ArgumentList -join ' ') failed with exit code $LASTEXITCODE." + } + return $output +} + +function Get-GpuNode { + param( + [Parameter(Mandatory)][string]$KubeConfig, + [string]$PoolName = 'gpunp' + ) + $nodes = Invoke-CheckedCommand kubectl @('--kubeconfig', $KubeConfig, 'get', 'nodes', + '-l', "agentpool=$PoolName", '-o', 'json') | ConvertFrom-Json + if (@($nodes.items).Count -ne 1) { + throw "Expected exactly one node in pool $PoolName; found $(@($nodes.items).Count)." + } + return $nodes.items[0] +} + +function Assert-SamplePools { + param( + [Parameter(Mandatory)][object[]]$Pools, + [Parameter(Mandatory)][string]$SystemVmSize, + [switch]$AllowMissingGpu + ) + $system = @($Pools | Where-Object name -EQ 'sysnp') + $gpu = @($Pools | Where-Object name -EQ 'gpunp') + if ($system.Count -ne 1 -or @($Pools | Where-Object { $_.name -notin @('sysnp', 'gpunp') }).Count -or + $gpu.Count -gt 1 -or (-not $AllowMissingGpu -and $gpu.Count -ne 1)) { + throw 'Expected one system pool and one GPU pool, with no additional pools.' + } + + foreach ($pool in $Pools) { + $size = if ($pool.name -eq 'sysnp') { $SystemVmSize } else { 'Standard_NCC40ads_H100_v5' } + $mode = if ($pool.name -eq 'sysnp') { 'System' } else { 'User' } + if ($pool.vmSize -ne $size -or $pool.count -ne 1 -or $pool.osSku -ne 'Ubuntu2404' -or + $pool.mode -ne $mode -or $pool.enableAutoScaling -eq $true -or $pool.provisioningState -ne 'Succeeded') { + throw "Pool $($pool.name) does not match the single-node sample configuration; it was not modified." + } + if ($pool.name -eq 'gpunp' -and + ($pool.gpuProfile.driver -ne 'None' -or 'sku=gpu:NoSchedule' -notin $pool.nodeTaints)) { + throw 'GPU pool must disable automatic drivers and retain the GPU scheduling taint.' + } + } +} + +function Assert-ConfidentialGpuScaleSet { + param([Parameter(Mandatory)][object[]]$ScaleSets) + $gpu = @($ScaleSets | Where-Object { $_.sku.name -eq 'Standard_NCC40ads_H100_v5' }) + if ($gpu.Count -ne 1) { throw 'Expected exactly one H100 scale set.' } + $security = $gpu[0].virtualMachineProfile.securityProfile + if ($gpu[0].sku.capacity -ne 1 -or $security.securityType -ne 'ConfidentialVM' -or + $security.uefiSettings.secureBootEnabled -ne $true -or $security.uefiSettings.vTpmEnabled -ne $true) { + throw 'H100 scale set must have one ConfidentialVM instance with Secure Boot and vTPM enabled.' + } +} diff --git a/aks-samples/aks-gpu-worker/Install-SignedDriver.sh b/aks-samples/aks-gpu-worker/Install-SignedDriver.sh deleted file mode 100644 index a6340fa..0000000 --- a/aks-samples/aks-gpu-worker/Install-SignedDriver.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -trap 'echo "ERROR: signed-driver probe failed at line $LINENO; inspect the node and stop the disposable cluster before retrying." >&2' ERR - -kernel=6.8.0-1071-azure-fde -version=615.71.09 -test "$(uname -r)" = "$kernel" -test ! -d /sys/module/nvidia -test ! -e /tmp/signed-driver -test ! -e "/lib/firmware/nvidia/$version" -test -z "$(ls -A /run/nvidia/driver)" - -export DEBIAN_FRONTEND=noninteractive -apt-get update -qq -apt-get install -y --no-install-recommends ca-certificates kmod mokutil libpciaccess0 -mokutil --sb-state | grep -Fx 'SecureBoot enabled' -mkdir /tmp/signed-driver -cd /tmp/signed-driver -apt-get download "linux-modules-nvidia-615-open-$kernel=6.8.0-1071.78" \ - "nvidia-firmware-615-$version=$version-0ubuntu0.24.04.2" \ - "nvidia-utils-615=$version-0ubuntu0.24.04.2" \ - "libnvidia-compute-615=$version-0ubuntu0.24.04.2" \ - "nvidia-compute-utils-615=$version-0ubuntu0.24.04.2" \ - "libnvidia-cfg1-615=$version-0ubuntu0.24.04.2" -echo "8b3ef898a5a32d08f748f197472a65903bd09a303e00f0bf94435430cc4437e0 linux-modules-nvidia-615-open-${kernel}_6.8.0-1071.78_amd64.deb" | - sha256sum -c - -for deb in ./*.deb; do dpkg-deb -x "$deb" root; done - -module_dir="$PWD/root/lib/modules/$kernel/kernel/nvidia-615-open" -for module in nvidia nvidia-uvm nvidia-modeset nvidia-drm nvidia-peermem; do - test "$(modinfo -F signer "$module_dir/$module.ko")" = 'Canonical Ltd. Kernel Module Signing' - test "$(modinfo -F version "$module_dir/$module.ko")" = "$version" - test "$(modinfo -F vermagic "$module_dir/$module.ko" | cut -d' ' -f1)" = "$kernel" -done -mkdir -p "/lib/firmware/nvidia/$version" -cp -a "root/lib/firmware/nvidia/$version/." "/lib/firmware/nvidia/$version/" -modprobe ecdsa_generic -modprobe ecdh -modprobe ecc -insmod "$module_dir/nvidia.ko" -insmod "$module_dir/nvidia-uvm.ko" - -export LD_LIBRARY_PATH="$PWD/root/usr/lib/x86_64-linux-gnu" -root/usr/bin/nvidia-persistenced --user root --uvm-persistence-mode --verbose \ - --nvidia-cfg-path="$LD_LIBRARY_PATH" -sleep 5 -test -s /var/run/nvidia-persistenced/nvidia-persistenced.pid -kill -0 "$(cat /var/run/nvidia-persistenced/nvidia-persistenced.pid)" -root/usr/bin/nvidia-smi conf-compute -f | grep -Fx 'CC status: ON' -root/usr/bin/nvidia-smi conf-compute -e | grep -Fx 'CC Environment: PRODUCTION' -root/usr/bin/nvidia-smi conf-compute -srs 1 -for attempt in 1 2 3; do - root/usr/bin/nvidia-smi --query-gpu=uuid,persistence_mode --format=csv,noheader | - grep -E '^GPU-[[:xdigit:]-]+, Enabled$' - root/usr/bin/nvidia-smi conf-compute -q | - grep -E 'CC GPUs Ready State[[:space:]]*: Ready$' - sleep 5 -done - -cp -a root/. /run/nvidia/driver/ -echo SIGNED_DRIVER_LOADED=1 -exit 0 diff --git a/aks-samples/aks-gpu-worker/Preflight.ps1 b/aks-samples/aks-gpu-worker/Preflight.ps1 new file mode 100644 index 0000000..c463e4e --- /dev/null +++ b/aks-samples/aks-gpu-worker/Preflight.ps1 @@ -0,0 +1,62 @@ +#Requires -Version 7.2 +function Test-SamplePreflight { + param( + [string]$Az, + [string]$Subscription, + [string]$Location, + [string]$KubernetesVersion, + [string]$OutputDirectory + ) + $scope = @('--subscription', $Subscription, '--only-show-errors') + $account = Invoke-CheckedCommand $Az (@('account', 'show', '-o', 'json') + $scope) | ConvertFrom-Json + if ($account.state -ne 'Enabled') { throw 'The target subscription is not enabled.' } + $cliVersion = (Invoke-CheckedCommand $Az @('version', '-o', 'json') | ConvertFrom-Json).'azure-cli' + Write-Host "Using built-in Azure CLI $cliVersion AKS commands in $Location (extensions isolated)." + $versions = Invoke-CheckedCommand $Az (@('aks', 'get-versions', '--location', $Location, '-o', 'json') + $scope) | ConvertFrom-Json + $patches = @($versions.values | ForEach-Object { $_.patchVersions.PSObject.Properties.Name }) + if ($KubernetesVersion -notin $patches) { throw "Kubernetes $KubernetesVersion is not offered in $Location." } + + $skus = @(Invoke-CheckedCommand $Az (@('vm', 'list-skus', '--location', $Location, + '--resource-type', 'virtualMachines', '--all', '-o', 'json') + $scope) | ConvertFrom-Json) + $usage = @(Invoke-CheckedCommand $Az (@('vm', 'list-usage', '--location', $Location, '-o', 'json') + $scope) | ConvertFrom-Json) + $required = @{} + $skuEvidence = @(foreach ($size in @('Standard_D4ds_v6', 'Standard_NCC40ads_H100_v5')) { + $sku = @($skus | Where-Object name -EQ $size) + if ($sku.Count -ne 1) { throw "SKU $size was not found in $Location." } + $sku = $sku[0] + $regionBlocks = @($sku.restrictions | Where-Object type -EQ 'Location') + if ($regionBlocks.Count) { + throw "SKU $size has a regional subscription restriction: $($regionBlocks | ConvertTo-Json -Depth 10 -Compress)" + } + $cores = [int]($sku.capabilities | Where-Object name -EQ 'vCPUs').value + if (-not $required.ContainsKey($sku.family)) { $required[$sku.family] = 0 } + $required[$sku.family] += $cores + $sku + }) + $total = 0 + foreach ($family in $required.Keys) { + $quota = @($usage | Where-Object { $_.name.value -eq $family }) + if ($quota.Count -ne 1 -or ([int]$quota[0].limit - [int]$quota[0].currentValue) -lt $required[$family]) { + throw "Insufficient $family quota for $($required[$family]) new vCPUs." + } + $total += $required[$family] + } + $regional = $usage | Where-Object { $_.name.value -eq 'cores' } + if (([int]$regional.limit - [int]$regional.currentValue) -lt $total) { + throw "Insufficient regional vCPU quota for $total cores." + } + $feature = Invoke-CheckedCommand $Az (@('feature', 'show', '--namespace', 'Microsoft.ContainerService', + '--name', 'Ubuntu2404Preview', '--query', 'properties.state', '-o', 'tsv') + $scope) + $preflight = [ordered]@{ + CheckedAtUtc = [DateTime]::UtcNow.ToString('o') + Location = $Location + KubernetesVersion = $KubernetesVersion + PreviewState = "$feature" + Skus = $skuEvidence + Quota = @($usage | Where-Object { $_.name.value -in @($required.Keys) + @('cores') }) + } + New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null + $preflight | ConvertTo-Json -Depth 20 | Set-Content (Join-Path $OutputDirectory 'preflight.json') + if ($feature -ne 'Registered') { throw "Ubuntu2404Preview is $feature. Arrange subscription registration before deploying." } + Write-Host 'Preflight passed; no Azure resources were changed.' +} diff --git a/aks-samples/aks-gpu-worker/README.md b/aks-samples/aks-gpu-worker/README.md index 517db68..7fdea86 100644 --- a/aks-samples/aks-gpu-worker/README.md +++ b/aks-samples/aks-gpu-worker/README.md @@ -1,821 +1,195 @@ -# AKS confidential H100 worker - -## Current workaround - UNSUPPORTED - -> **UNSUPPORTED - experimental diagnostic workaround only.** -> This recipe is not a Microsoft/NVIDIA-supported AKS configuration or a -> production-managed driver installation. Successful workload and attestation -> tests do not establish vendor support. It requires a privileged pod with -> host access and does not survive node lifecycle changes automatically. - -**Tested end to end on two fresh workers on 2026-11-10.** Exact-kernel, -Canonical-signed Ubuntu NVIDIA `615.71.09` packages loaded on -`6.8.0-1071-azure-fde` with Secure Boot enabled. Starting UVM persistence -**before the first GPU query**, then exposing the extracted driver tree to GPU -Operator with its driver installer disabled, produced one allocatable H100. -A GPU-only CUDA vector-add Job passed, strict GPU attestation passed before -and after that workload, and a separate Azure CPU/CVM attestation returned -`sevsnpvm` and `azure-compliant-cvm` claims. - -### Use this path, not the failed driver-container recipes - -Follow the [tested Azure CLI walkthrough](Confidential-GPU-H100-AKS-Onboarding.md) -for the complete commands. The working sequence is: - -1. Provision the disposable system pool and confidential H100 worker with - automatic GPU driver installation disabled. Verify the actual host kernel, - ConfidentialVM, Secure Boot and vTPM before proceeding. -2. Install GPU Operator `v26.7.2` with **`driver.enabled=false`**. Keep the - toolkit/runtime and device plugin; do not use its failing driver installer. -3. Apply [the privileged probe](signed-driver-probe.yaml) on a fresh worker and - run [the exact-package installer](Install-SignedDriver.sh). It extracts - Canonical-signed NVIDIA `615.71.09` modules for - `6.8.0-1071-azure-fde` plus matching firmware and libraries, without changing - the host kernel, disabling Secure Boot or replacing AKS-managed containerd. -4. The installer preloads the crypto dependencies and starts - `nvidia-persistenced --uvm-persistence-mode` **before the first GPU query**, - sets GPU ready state, and exposes the driver tree at `/run/nvidia/driver`. - Require `SIGNED_DRIVER_LOADED=1`, CC ON / PRODUCTION / Ready and one - allocatable `nvidia.com/gpu`. -5. Run [strict GPU attestation](Test-GpuAttestation.sh), then - [the one-GPU CUDA Job](gpu-smoke-test.yaml), then GPU attestation again. - Require zero exits, positive verdicts for the actual GPU UUID, and - `Test PASSED`; no CPU fallback is supplied. -6. Run [separate CPU/CVM attestation](Test-CvmAttestation.sh). The probe mounts - host securityfs read-only for TPM evidence; require the successful MAA - request and `sevsnpvm` / `azure-compliant-cvm` claims. -7. Preserve evidence locally, remove only the probe-owned firmware and test - resources, and stop the cluster using the walkthrough's cleanup commands. - -**Limits:** the probe holds persistence for at most four hours. Deleting it ends -persistence; `/run/nvidia/driver` is ephemeral. Reboots, replacement nodes and -image/kernel upgrades require fresh validation and setup. Do not apply these -exact packages to another kernel. Optional DCGM exporter readiness remained -`0/1`, and the workload/probe persistence displays differed; neither issue is -claimed resolved. Full attestation logs may contain tokens: keep them local and -gitignored. - -The original missing registry tags and generic-driver failure remain -reproducible. [Appendix: everything tried](#appendix-everything-tried) preserves -the failed attempts, alternatives investigated, fixes and verification evidence. - -## Implementation steps - UNSUPPORTED - -The steps below follow the [source guide's section order](https://github.com/Azure/az-cgpu-onboarding/blob/136262ef00ad75b8e0bc789563f043006fb68f4c/docs/Confidential-GPU-H100-AKS-Onboarding.md): -CLI configuration, cluster creation, setup validation, driver installation, -confidential mode, workload and attestation. They replace only the parts that -failed in this reproduction. **This entire procedure is unsupported**, including -the exact OS/kernel/driver combination that passed the tests. - -### 1. Prerequisites and Helm installation - -Use Bash on Linux/WSL with Azure CLI (tested: 2.92.0), Helm and kubectl already -installed. Run all blocks in the **same Bash session from this sample directory**, -with its supplied manifests and scripts present. PowerShell users can use the -alternative provisioning commands below, but must not paste Bash blocks into -PowerShell. - -Use a disposable cluster, permission to provision AKS, 40 available -`StandardNCCads2023Family` vCPUs plus four system-node vCPUs, and sufficient West -Europe allocation capacity. Allow HTTPS egress to Azure, Ubuntu repositories, -GitHub, NGC and the verifier's certificate/RIM/OCSP services. Do not run this -privileged diagnostic on shared production nodes. - -```bash -set -euo pipefail -az version -helm version --short -kubectl version --client -``` - -### 2. Azcli configurations - -Use an explicitly authorized subscription and a new resource group name. -Keep the sample-local extension directory empty: the tested core CLI avoids the -observed `aks-preview` incompatibility without changing shared extensions. -Keep the dedicated kubeconfig rather than overwriting the workstation default. - -```bash -subscription='' -location=westeurope -resourceGroup=rg-aks-gpu-worker-example -clusterName=aks-gpu-worker-example -mkdir -p .local/cli-extensions-core -export AZURE_EXTENSION_DIR="$PWD/.local/cli-extensions-core" -export AZURE_EXTENSION_USE_DYNAMIC_INSTALL=no -export KUBECONFIG="$PWD/.local/kubeconfig" -az login -az account show --subscription "$subscription" --query '{name:name,state:state}' -az aks get-versions --subscription "$subscription" --location "$location" -o table -az feature register --subscription "$subscription" \ - --namespace Microsoft.ContainerService --name Ubuntu2404Preview -az feature show --subscription "$subscription" \ - --namespace Microsoft.ContainerService --name Ubuntu2404Preview \ - --query properties.state -o tsv -``` - -Registration is asynchronous. Wait until the feature reports `Registered`, -rerunning the show command as needed, then propagate registration: - -```bash -test "$(az feature show --subscription "$subscription" \ - --namespace Microsoft.ContainerService --name Ubuntu2404Preview \ - --query properties.state -o tsv)" = Registered -az provider register --subscription "$subscription" \ - --namespace Microsoft.ContainerService -``` - -### 3. Create AKS cluster with CGPU nodepool - -These commands create **one system node plus one confidential GPU worker** and -incur charges. Both pools use Ubuntu2404; autoscaling is not enabled. Disabling -automatic upgrades is only for this short-lived controlled experiment. - -```bash -az group create --subscription "$subscription" --name "$resourceGroup" \ - --location "$location" --tags sample=aks-gpu-worker -az aks create --subscription "$subscription" \ - --resource-group "$resourceGroup" --name "$clusterName" \ - --location "$location" --kubernetes-version 1.36.5 --tier free \ - --nodepool-name sysnp --node-count 1 --node-vm-size Standard_D4ds_v6 \ - --os-sku Ubuntu2404 --node-osdisk-type Managed --node-osdisk-size 128 \ - --enable-managed-identity --generate-ssh-keys \ - --auto-upgrade-channel none --node-os-upgrade-channel None \ - --tags sample=aks-gpu-worker -az aks get-credentials --subscription "$subscription" \ - --resource-group "$resourceGroup" --name "$clusterName" \ - --file "$KUBECONFIG" --overwrite-existing -az aks nodepool add --subscription "$subscription" \ - --resource-group "$resourceGroup" --cluster-name "$clusterName" \ - --name gpunp --mode User --node-count 1 \ - --node-vm-size Standard_NCC40ads_H100_v5 --os-sku Ubuntu2404 \ - --gpu-driver None --node-osdisk-type Managed --node-osdisk-size 128 \ - --node-taints sku=gpu:NoSchedule -``` - -For the retained stopped sample, skip creation, set the variables to its verified -identity, and use `az aks start` instead. Starting it does not restore the -ephemeral driver installation. Do not create additional pools on an unrelated -existing cluster. - -### 4. Setup validation - -```bash -kubectl wait --for=condition=Ready node --all --timeout=600s -kubectl get nodes -o wide -test "$(kubectl get nodes -o name | wc -l)" -eq 2 -test "$(kubectl get nodes -l agentpool=gpunp -o name | wc -l)" -eq 1 -gpu_node_name=$(kubectl get nodes -l agentpool=gpunp \ - -o 'jsonpath={.items[0].metadata.name}') -kernel_version=$(kubectl get node "$gpu_node_name" \ - -o 'jsonpath={.status.nodeInfo.kernelVersion}') -test "$kernel_version" = 6.8.0-1071-azure-fde -az aks nodepool list --subscription "$subscription" \ - --resource-group "$resourceGroup" --cluster-name "$clusterName" \ - --query '[].{Name:name,NodeImageVersion:nodeImageVersion}' -o table -nodeResourceGroup=$(az aks show --subscription "$subscription" \ - --resource-group "$resourceGroup" --name "$clusterName" \ - --query nodeResourceGroup -o tsv) -az vmss list --subscription "$subscription" --resource-group "$nodeResourceGroup" \ - --query "[?sku.name=='Standard_NCC40ads_H100_v5'].{name:name,count:sku.capacity,security:virtualMachineProfile.securityProfile}" -test "$(az vmss list --subscription "$subscription" --resource-group "$nodeResourceGroup" \ - --query "[?sku.name=='Standard_NCC40ads_H100_v5' && sku.capacity==\`1\` && virtualMachineProfile.securityProfile.securityType=='ConfidentialVM' && virtualMachineProfile.securityProfile.uefiSettings.secureBootEnabled==\`true\` && virtualMachineProfile.securityProfile.uefiSettings.vTpmEnabled==\`true\`] | length(@)" -o tsv)" = 1 -``` - -The tested worker reported: - -```text -OS-IMAGE: Ubuntu 24.04.5 LTS -KERNEL-VERSION: 6.8.0-1071-azure-fde -NODE-IMAGE: AKSUbuntu-2404gen2CVMcontainerd-202610.30.1 -``` - -These are observations, **not an AKS image pin or supported combination**. -Stop if the kernel or effective ConfidentialVM/Secure Boot/vTPM checks differ. -Do not substitute 1041 modules, replace the host kernel or disable security. - -### 5. Driver installation - -Unlike the source guide, disable the GPU Operator driver operand. Use its toolkit -and device plugin with the separately extracted signed driver tree. - -```bash -helm upgrade --install gpu-operator gpu-operator \ - --repo https://helm.ngc.nvidia.com/nvidia --version v26.7.2 \ - --namespace gpu-operator --create-namespace \ - --values gpu-operator-experimental-values.yaml --set driver.enabled=false -kubectl apply -f signed-driver-probe.yaml -kubectl wait --for=condition=Ready pod/signed-driver-load --timeout=300s -tr -d '\r' < Install-SignedDriver.sh | - kubectl exec -i signed-driver-load -- bash | tee .local/signed-driver.log -grep -Fx 'SIGNED_DRIVER_LOADED=1' .local/signed-driver.log -``` - -The [installer](Install-SignedDriver.sh) verifies Secure Boot, exact kernel, -module-deb checksum, and five module signers/versions/vermagic values. It extracts -`linux-modules-nvidia-615-open-6.8.0-1071-azure-fde=6.8.0-1071.78` and coherent -`615.71.09` firmware/libraries, loads NVIDIA/UVM, starts UVM persistence **before -querying the GPU**, sets readiness, then copies the tree into `/run/nvidia/driver`. -It does not install a moving kernel metapackage or replace AKS containerd. - -Keep the probe alive. It refuses already-loaded or partially populated nodes: -after failure, inspect and recycle the disposable node rather than repeatedly -unloading/reloading modules or deleting/recreating the persistence pod. - -### 6. Enable confidential GPU mode and verify readiness - -The installer already sets ready state after persistence starts. Confirm CC is -ON and PRODUCTION rather than trying to bypass a failed confidential-mode check: - -```bash -kubectl exec signed-driver-load -- bash -c 'set -euo pipefail -export LD_LIBRARY_PATH=/tmp/signed-driver/root/usr/lib/x86_64-linux-gnu -smi=/tmp/signed-driver/root/usr/bin/nvidia-smi -kill -0 "$(cat /var/run/nvidia-persistenced/nvidia-persistenced.pid)" -"$smi" conf-compute -f | grep -Fx "CC status: ON" -"$smi" conf-compute -e | grep -Fx "CC Environment: PRODUCTION" -"$smi" conf-compute -q | grep -E "CC GPUs Ready State[[:space:]]*: Ready$"' -kubectl wait --for=jsonpath='{.status.allocatable.nvidia\.com/gpu}'=1 \ - node -l agentpool=gpunp --timeout=600s -kubectl get pods -n gpu-operator -o wide -``` - -Require one allocatable GPU. Node Ready and Helm `deployed` alone are insufficient. -The Operator CUDA validator completed during the tested runs; optional DCGM -exporter readiness remained `0/1`. - -### 7. Sample workload - -First attest the GPU so the workload is bracketed by successful checks: - -```bash -tr -d '\r' < Test-GpuAttestation.sh | - kubectl exec -i signed-driver-load -- bash | tee .local/gpu-attestation-before.log -grep -Fx 'GPU_ATTESTATION_VALIDATED=1' .local/gpu-attestation-before.log -kubectl apply -f gpu-smoke-test.yaml -kubectl wait --for=condition=Complete job/confidential-gpu-smoke --timeout=240s -kubectl logs job/confidential-gpu-smoke | tee .local/gpu-smoke.log -grep -Fx 'Test PASSED' .local/gpu-smoke.log -``` - -The [nonprivileged Job](gpu-smoke-test.yaml) requests and limits one GPU, uses -runtime class `nvidia`, and validates CUDA vector addition of 50,000 elements. -No CPU fallback is supplied. - -### 8. GPU attestation and separate CPU/CVM attestation - -```bash -tr -d '\r' < Test-GpuAttestation.sh | - kubectl exec -i signed-driver-load -- bash | tee .local/gpu-attestation-after.log -grep -Fx 'GPU_ATTESTATION_VALIDATED=1' .local/gpu-attestation-after.log -tr -d '\r' < Test-CvmAttestation.sh | - kubectl exec -i signed-driver-load -- bash | tee .local/cvm-attestation-summary.log -grep -Fx 'CVM_ATTESTATION_VALIDATED=1' .local/cvm-attestation-summary.log -``` - -Require zero pipeline exits throughout. [GPU verification](Test-GpuAttestation.sh) -uses checksum-pinned V4.5.1 verifier `2.3.0`, Python 3.12, strict OCSP and a fresh -nonce; it requires `GPU Attestation is Successful.` and verification of the -actual GPU UUID. [CVM verification](Test-CvmAttestation.sh) independently makes -the MAA request and requires returned `sevsnpvm` / `azure-compliant-cvm` claims, -not an additional independent JWT signature validator. - -The probe's read-only securityfs mount supplies the TPM event log needed by MAA. -Keep full logs in the pod or ignored, access-controlled local storage: they may -contain tokens. Preserve needed evidence before cleanup; do not commit it. - -### 9. Troubleshooting and cleanup - -Do not continue after a failed gate. Use the -[investigation appendix](#appendix-everything-tried) for the observed failure -signatures. After a successful run, remove only the four probe-owned firmware -files and test resources, then promptly stop the cluster: - -```bash -kubectl delete job confidential-gpu-smoke --ignore-not-found -kubectl exec signed-driver-load -- bash -c 'set -eu -rm -- /lib/firmware/nvidia/615.71.09/gsp_ga10x.bin \ - /lib/firmware/nvidia/615.71.09/gsp_tu10x.bin \ - /lib/firmware/nvidia/615.71.09/ucodes_ga10x.bin \ - /lib/firmware/nvidia/615.71.09/ucodes_tu10x.bin -rmdir /lib/firmware/nvidia/615.71.09' -kubectl delete pod signed-driver-load -az aks stop --subscription "$subscription" \ - --resource-group "$resourceGroup" --name "$clusterName" -az aks show --subscription "$subscription" \ - --resource-group "$resourceGroup" --name "$clusterName" \ - --query '{power:powerState.code,state:provisioningState}' -az vmss list --subscription "$subscription" --resource-group "$nodeResourceGroup" \ - --query '[].{name:name,capacity:sku.capacity}' -o table -``` - -Require **Stopped / Succeeded** and **both VMSS capacities zero**. If setup failed, -still run `az aks stop`; inspect which exact probe-owned files exist before -cleaning them rather than allowing a cleanup failure to leave compute running. -Stopping retains potentially billable disks/networking. Explicit group deletion -is described in [Cleanup](#cleanup). - -## Deployment plan and scope - -Create a disposable development cluster in West Europe with Kubernetes 1.36.5, -one Ubuntu 24.04 system node (`Standard_D4ds_v6`), and exactly one Ubuntu 24.04 -confidential GPU user node (`Standard_NCC40ads_H100_v5`). AKS requires a system -pool: this is **two nodes total**, not a one-node cluster. Disable automatic -driver installation, autoscaling, and automatic upgrades during the experiment. -Use the existing signed-in Azure subscription only when explicitly supplied. -Do not overwrite other clusters, change the workstation's default subscription -or kubeconfig, downgrade AKS node images, or bypass confidential-computing checks. - -The Azure CLI recipe follows the -[onboarding guide](https://github.com/Azure/az-cgpu-onboarding/blob/136262ef00ad75b8e0bc789563f043006fb68f4c/docs/Confidential-GPU-H100-AKS-Onboarding.md). -The scripts are PowerShell 7 equivalents with quota checks, bounded waits, -native-command error handling, and registry compatibility gates. -This is not a production architecture or a claim of a supported new OS/driver combination. -Unlike the guide's Ubuntu 22.04 system pool, both pools use Ubuntu 24.04 because -the AKS documentation warns against mixed OS versions with GPU Operator. - -## PowerShell alternative prerequisites - -- PowerShell 7.2+, Azure CLI, `kubectl`, and Helm for subsequent GPU Operator setup. - The guide uses `aks-preview`; alternatively, `-UseCoreCli` uses built-in AKS - commands (verified with Azure CLI 2.92.0) without loading installed extensions. -- `az login`, permissions to create AKS and its managed resources, and an enabled - subscription with regional and VM-family quota. One H100 worker needs 40 - `StandardNCCads2023Family` vCPUs, plus four system-node vCPUs. -- `Ubuntu2404Preview` registered in `Microsoft.ContainerService` as required by - the linked guide. Registration may need subscription-owner assistance. -- HTTPS access to Azure, GitHub, NVIDIA NGC, and the Kubernetes API. - -Azure CLI on Windows is also detected at its standard installation path when -the current terminal's PATH has not been refreshed. - -## PowerShell alternative provisioning - -Run from this directory. The supplied resource group must be new unless resuming -this sample's deployment. Resource creation incurs charges. - -```powershell -$subscription = '' -$resourceGroup = 'rg-aks-gpu-worker-20261110' -$cluster = 'aks-gpu-worker-20261110' - -.\Deploy.ps1 -SubscriptionId $subscription -ResourceGroup $resourceGroup ` - -ClusterName $cluster -UseCoreCli -PreflightOnly - -# Registers the documented preview if necessary, with a 20-minute timeout. -.\Deploy.ps1 -SubscriptionId $subscription -ResourceGroup $resourceGroup ` - -ClusterName $cluster -UseCoreCli -RegisterPreview -``` - -For a partial deployment, inspect the resources and rerun with `-Resume`. -Keep the same system-node SKU and `-UseCoreCli` selection when resuming. -Resume validates sample ownership and existing pool configuration before cloud -mutations, and quota checks count only pools still to be created. -Use `-GpuZone` only with a zone reported as available to the subscription. -Quota and SKU listings do not guarantee physical allocation capacity. -No alternate subscription, region, VM family, or kernel is silently substituted. - -The deployment saves local evidence and a dedicated kubeconfig in `.local/` -(gitignored). Do not commit credentials or raw cloud/Kubernetes dumps. -The development cluster follows the guide's authentication model; protect its -API access and kubeconfig. Use Entra RBAC, restricted API access and a resilient -system pool for production. Automatic upgrades are disabled only for this -short-lived experiment, not as a long-term security policy. - -## Exact-kernel container compatibility gate - -This gate diagnoses the original **container-image path**. It is expected to -fail on the tested 1071 kernel and is not an acceptance gate for the unsupported -signed-package workaround above. A successful signed-package run does not -change the missing-image result. +# Latest-image confidential H100 AKS worker + +Automates the [Azure confidential H100 AKS onboarding guide](https://github.com/Azure/az-cgpu-onboarding/blob/136262ef00ad75b8e0bc789563f043006fb68f4c/docs/Confidential-GPU-H100-AKS-Onboarding.md) +using **only its GPU Operator driver installation path**: +GPU Operator **v25.3.1**, open kernel modules, and +`nvcr.io/nvidia/driver:580--ubuntu24.04`. +There is no signed-package fallback, generic driver installation, manual host +kernel change, or Secure Boot bypass. + +## What "latest kernel" means + +The script selects the latest **AKS-managed node image offered to the new pool** +in the selected region, updates the pool if necessary, then reads the actual +worker kernel from Kubernetes. It requires `azure-fde` and a numeric version at +least **6.8.0-1071-azure-fde**. It does not select an arbitrary Ubuntu apt kernel: +AKS controls the kernel in its managed image. The default Kubernetes version is +the newest patch of the region's non-preview default minor; it is recorded with +the image versions. + +**Support boundary:** the linked guide explicitly validates Ubuntu 24.04.3 and +kernel `6.8.0-1041-azure-fde`. Following its installation method on a newer image +does not independently establish Microsoft/NVIDIA support for that combination. +The result always records `VendorSupportValidated: false`. If the exact branch +580 image is missing, the script records `BlockedMissingGuideDriver`, stops +the disposable cluster, and exits nonzero. It does not substitute another branch +or repeatedly allocate workers for the same missing image. + +## Run + +Prerequisites: PowerShell 7.2+, authenticated Azure CLI, kubectl, Helm 3, +OpenSSH `ssh-keygen`, registered `Ubuntu2404Preview`, and permissions/quota to +create AKS and its managed resources. Preflight checks the actual regional SKU +restrictions, NCC family quota, system-family quota and total core quota using +the included [preflight helper](Preflight.ps1). Quota does not guarantee capacity. +Run these commands from this sample directory. ```powershell -.\Test-DriverCompatibility.ps1 -KubeConfig .\.local\kubeconfig -RequireAvailable -``` - -The worker's **actual** kernel determines the image tag. Selecting Kubernetes -1.36.5 and `Ubuntu2404` does not select an AKS node-image build or kernel. -An unavailable exact-kernel image is an external blocker, not a reason to use -1041 modules on a 1071 kernel. Do not disable Secure Boot, change the host kernel, -or install unsigned modules to get past it. - -The script also verifies the underlying H100 scale set has exactly one instance, -`securityType: ConfidentialVM`, Secure Boot enabled and vTPM enabled. Inspect the -VM scale-set security profile rather than assuming the AKS pool's generic -security flags describe the confidential VM's effective settings. - -## Resource status - -The retained deployment is named `aks-gpu-worker-20261110` in -`rg-aks-gpu-worker-20261110`, West Europe. Resources are retained for inspection. -After the final successful published-script replay, the probe-owned firmware -files, test Job and probe pod were removed. `az aks stop` succeeded: the cluster -reported **Stopped / Succeeded**, and both node scale sets were verified at -capacity **zero**. Full attestation evidence was preserved only in ignored -local storage before the probe was removed. -Stopping AKS deallocates compute but retained disks, networking and other -resources can still incur charges; stopping is not deletion. - -## Local regression checks +.\Deploy.ps1 -SubscriptionId '' -PreflightOnly + +.\Deploy.ps1 -SubscriptionId '' ` + -ResourceGroup 'rg-aks-gpu-worker-unique' ` + -ClusterName 'aks-gpu-worker-unique' ` + -Location westeurope -StopExistingNcc +``` + +`-StopExistingNcc` explicitly authorizes stopping **all existing NCC compute in +the selected subscription and region**, including whole AKS clusters that own +NCC pools (their system nodes also stop). Standalone NCC VMs/VMSS are deallocated. +Non-NCC compute and other regions are not changed. Without the switch, running +NCC compute is an error; `-PreflightOnly` never stops or creates compute. +Other regions can be selected explicitly; West Europe is the default. + +Uses a fresh resource group, one `Standard_D4ds_v6` system node and one +`Standard_NCC40ads_H100_v5` worker (44 total vCPUs). Both pools use Ubuntu2404 +instead of the guide's older Ubuntu2204 system pool; the confidential GPU pool, +disabled automatic driver and GPU taint match the guide. Autoscaling is omitted +to stay within the single-GPU quota/cost envelope. Automatic image/Kubernetes +upgrades are disabled for reproducible testing. CLI extensions and kubeconfig +are isolated per run without changing shared CLI defaults or kubeconfig. + +After confirming ConfidentialVM, Secure Boot and vTPM, the script gates Helm on +a real registry manifest lookup. Authentication, rate-limit and network errors +are failures, not missing-image evidence. If installation can proceed, it +requires one allocatable GPU, CC ON/PRODUCTION/Ready, successful strict GPU +attestation, the guide's TensorFlow matrix multiplication job, and successful +strict attestation again. The TensorFlow workload retains the guide's image +and computation, but explicitly selects GPU 0, disables CPU fallback, and checks +the result tensor's actual device. Its runtime and retries are bounded, and it +uses the NVIDIA runtime on the GPU pool. The guide's attestation manifest is +pinned to the revision linked above; +it retains the guide's latest onboarding archive download, with strict OCSP, +bounded runtime and scheduling onto the GPU pool added. This is GPU attestation, +not a separate CPU/whole-VM attestation claim. + +The newly created cluster is stopped in `finally`, including after failure. +Bounded readiness waits allow AKS background addon/pool operations to settle +before subsequent provisioning, configuration checks and shutdown. +If an addon operation starts between readiness and shutdown, AKS stop retries +only `OperationNotAllowed` errors identifying an in-progress operation, for up +to 20 minutes. Retries emit warnings; other errors fail immediately. The same +shutdown helper is used for existing NCC AKS clusters. +Stop and VMSS instance deallocation are verified. `-KeepRunning` explicitly +disables this cost safeguard. Failed provisioning can prevent AKS stop; such +cleanup errors are surfaced and require immediate attention. Existing compute +stopped at the beginning is not automatically restarted. + +Run evidence is in ignored `.local/-/`: kernel, node image, +VMSS security, registry response, result and (when reached) workload/attestation +logs. Failed job pod status and redacted logs are captured before shutdown when +available; diagnostic collection errors are surfaced. Attestation tokens are +omitted from persisted logs. This directory also +contains a dedicated kubeconfig and SSH private key: **do not commit or share it**. +No resources are automatically deleted. Stopped AKS retains disks/IPs and other +resources that may still incur charges; delete its resource group when finished. + +## Offline validation ```powershell .\tests\Test-Sample.ps1 ``` -These dependency-free tests mock registry and node responses. They check -authentication and OCI/Docker Accept headers, missing tags versus other HTTP -failures, digest validation, compatibility-report shape, the exact pool topology, -effective confidential VM security, and native-process failures. They do not -establish cloud capacity or GPU support. - -## Cleanup - -Resources are deliberately retained for inspection, including after failure. -Review the group and delete it explicitly when finished; this removes the -cluster and its managed node resources and stops their ongoing charges. - -```powershell -az group show --subscription $subscription --name $resourceGroup -az group delete --subscription $subscription --name $resourceGroup --yes -``` - -## References - -- [Confidential GPU AKS onboarding](https://github.com/Azure/az-cgpu-onboarding/blob/136262ef00ad75b8e0bc789563f043006fb68f4c/docs/Confidential-GPU-H100-AKS-Onboarding.md) -- [AKS confidential GPU documentation](https://learn.microsoft.com/azure/aks/use-confidential-gpus) -- [AKS node image upgrades](https://learn.microsoft.com/azure/aks/node-image-upgrade) - -## Appendix: everything tried - -This appendix retains the investigation history, including unsuccessful recipes. -It is **not the current installation procedure**. Use the unsupported workaround -at the top and its linked walkthrough for the tested sequence. - -### Registry-failure reproduction without an Azure cluster - -```powershell -.\Test-DriverCompatibility.ps1 -KernelVersion '6.8.0-1071-azure-fde' -.\Test-DriverCompatibility.ps1 -KernelVersion '6.8.0-1041-azure-fde' -.\Test-DriverCompatibility.ps1 -KernelVersion '6.8.0-1063-azure-fde' -``` - -The JSON distinguishes missing manifests from authentication/network failures. -Add `-RequireAvailable` to fail when neither exact-kernel image exists. -`SupportValidated: false` is intentional: a published image alone does not prove -Microsoft/NVIDIA support, module loading, confidential mode, or attestation. - -### Initial blockers and investigation results - -Live checks on 2026-11-10: - -| Check | Result | Interpretation / action | -|---|---|---| -| West Europe H100 SKU | Listed without subscription restrictions; zone 3 advertised; one worker allocated successfully without an explicit zone | Capacity was sufficient for this attempt, not a future guarantee. | -| H100 quota | 40 available family vCPUs; 140 available regional vCPUs | Sufficient for one worker and the system node. | -| Kubernetes 1.36.5 | Offered | The requested API version can be selected; this does not pin the kernel. | -| System node `Standard_D4ds_v5` | `NotAvailableForSubscription` in West Europe | Changed only the system node to unrestricted `Standard_D4ds_v6`, with 100 family vCPUs available. | -| `Ubuntu2404Preview` | Initially `NotRegistered`; registration succeeded | Registered before cluster creation. | -| Azure CLI 2.92.0 with `aks-preview` 18.0.0b17 | `ManagedCluster` has no attribute `hosted_system_profile` | Client-side failure; retry with `-UseCoreCli`, an empty sample-local extension directory, and dynamic extension installation disabled. The shared extension installation is unchanged. | -| AKS cluster and node pools | Provisioned; exactly one system node and one H100 user node, both Kubernetes `v1.36.5` and Ready | The control-plane/system-pool and H100 allocation stages succeeded. Node Ready is not GPU readiness. | -| Actual H100 node image | `AKSUbuntu-2404gen2CVMcontainerd-202610.30.1` | Matches the reported image build. | -| Actual worker OS / kernel / runtime | Ubuntu `24.04.5 LTS` / `6.8.0-1071-azure-fde` / `containerd://2.3.7-4` | The current provisioning recipe does not produce the guide's documented Ubuntu 24.04.3 / kernel 1041 combination. | -| Effective H100 VM security | `ConfidentialVM`; Secure Boot and vTPM both enabled | Verified on the underlying VM scale set; no security bypass was applied. | -| `580-6.8.0-1041-azure-fde-ubuntu24.04` | HTTP 200 | Published; the guide explicitly describes this kernel. | -| `580-6.8.0-1063-azure-fde-ubuntu24.04` | HTTP 200 with OCI Accept headers | Published; support for this combination has not been established. | -| `580-6.8.0-1071-azure-fde-ubuntu24.04` | HTTP 404, `MANIFEST_UNKNOWN`, `manifest unknown` | Reported missing image reproduced directly. | -| `595-6.8.0-1071-azure-fde-ubuntu24.04` | HTTP 404, `MANIFEST_UNKNOWN`, `manifest unknown` | Changing the driver branch does not fix this missing tag. | - -Unauthenticated registry requests return HTTP 401 even for existing images. -Authenticate using the public anonymous bearer-token flow. Also request both -OCI and Docker manifest media types: otherwise an existing OCI image can return -a misleading HTTP 404 (`OCI manifest found, but accept header does not support -OCI manifests`). Neither error should be classified as an absent tag. - -**Initial outcome: cluster provisioned; exact-kernel container path blocked.** The -exact-kernel compatibility gate deliberately exits with an error after saving -the evidence. This is not a failed H100 allocation: it is a reproducible missing -driver-image blocker on the actual provisioned worker. - -The initial reproduction stopped before GPU Operator installation. The subsequent -generic-driver experiment below did install GPU Operator and attempt driver -loading, unsuccessfully. The later signed-package experiment succeeded through -a different installation path. The 1041 and 1063 images remain positive controls, -not substitutes for 1071. +Exercises numeric kernel floors, non-preview patch selection, genuine missing +images versus registry failures, digest validation, NCC-only shutdown scope, +explicit subscription propagation, real SSH key generation/error propagation, +guide workload GPU-placement guards and PowerShell parsing. +Also tests AKS background-operation readiness, failure and timeout handling. +Shutdown tests cover operation conflicts, nonretryable failures, deadline +exhaustion and explicit target/subscription propagation. Preflight tests check +read-only command scope, quota and registration failures, and saved evidence. -To unblock, obtain a Microsoft/NVIDIA-supported pairing for the actual Ubuntu -24.04.5 / Azure-FDE kernel: a published, compatible signed driver image and -confirmation of its support, or an officially supported AKS provisioning path -that yields a compatible OS/kernel. Neither a Kubernetes-version selection nor -`--os-sku Ubuntu2404` pins the node-image build. This reproduction found no -supported way to select the guide's old kernel through these commands, and did -not attempt an unsupported node-image downgrade or host-kernel replacement. +## West Europe live validation -Once an approved pairing is available, rerun the compatibility gate and follow -the pinned guide's GPU Operator `v25.3.1` steps with the exact driver version. -The worker has `sku=gpu:NoSchedule`: add that toleration to GPU Operator operands -and node-feature-discovery workers before installation. Then verify the signed -module loads, `nvidia-smi` confidential mode/readiness, one allocatable GPU, a -GPU-only workload that fails rather than falling back to CPU, and GPU attestation. -Those are separate acceptance stages, not consequences of a manifest returning -HTTP 200. +On 2026-11-12, a fresh single-H100 deployment exited zero with +`Status: Succeeded`. It used Kubernetes +1.36.5, Ubuntu 24.04.5 LTS, the latest offered GPU image +`AKSUbuntu-2404gen2CVMcontainerd-202610.30.1`, kernel +`6.8.0-1071-azure-fde`, and the exact guide branch-580 driver image. +ConfidentialVM, Secure Boot, vTPM, CC ON/PRODUCTION/Ready and one allocatable +GPU checks passed. Strict GPU attestation passed before and after the workload. -### Latest-image / generic-driver experiment - -This is an **experimental reproduction, not a supported onboarding recipe**. -It preserves the original exact-kernel compatibility gate; generic image -publication does not satisfy that gate. - -On 2026-11-10, the AKS node-pool upgrade API offered -`AKSUbuntu-2404gen2CVMcontainerd-202610.30.1`, identical to the existing image. -Restarting the cluster restored exactly one system node and one H100 worker, -both Ready. The worker still used Ubuntu 24.04.5 and -`6.8.0-1071-azure-fde`. The underlying H100 scale set retained ConfidentialVM, -Secure Boot and vTPM. No pool scaling or image upgrade was needed. - -| Attempt | Observed result | -|---|---| -| GPU Operator `v26.7.2`, default driver `595.91.07`, open modules | Generic `595.91.07-ubuntu24.04` image pulled successfully. Driver initialization failed before module installation. | -| GPU Operator `v26.7.2`, newer published driver `615.71.09`, open modules | Generic `615.71.09-ubuntu24.04` image pulled successfully. The same initialization failure reproduced. | -| Exact-kernel `615-6.8.0-1071-azure-fde-ubuntu24.04` | HTTP 404 / `MANIFEST_UNKNOWN`. | -| Worker GPU resources after generic-container attempts | No `nvidia.com/gpu` allocatable resource; GPU Operator operands remained in initialization. | -| Attestation at this stage | Not run: the generic-container driver prerequisite failed. See the successful signed-package follow-up below. | - -Both generic driver containers successfully obtained the **matching kernel -headers**. They then downloaded the Ubuntu kernel packages and failed with: +Actual TensorFlow output (informational/startup warnings omitted): ```text -Generating Linux kernel version string... -ls: cannot access 'boot/vmlinuz-*': No such file or directory -Could not locate Linux kernel version string -``` - -The container's `/usr/local/bin/nvidia-driver` expects `boot/vmlinuz-*` after -extracting `linux-image-${KERNEL_VERSION}`. Inspection of the actual Azure-FDE -package showed `usr/lib/linux/efi/kernel.efi-6.8.0-1071-azure-fde` instead. -This is a distinct generic-container packaging/layout blocker, **not missing -kernel headers**, and occurs before NVIDIA modules are built or loaded. - -The Ubuntu package index also lists exact-kernel NVIDIA module packages, -including `linux-modules-nvidia-615-open-6.8.0-1071-azure-fde` -(`6.8.0-1071.78`), with dependency on NVIDIA `615.71.09`. -The signed-package follow-up below tests that route separately; publication is -not proof of a supported AKS integration or successful attestation. -The matching generic NVIDIA image exists (digest -`sha256:0ed39c3e75c219954ba81be558c9aa0e6abbfa838c0f60d61bf9a139776fa186`). -Simply selecting it does not make GPU Operator consume Ubuntu's module packages. - -### Reusing the standalone VM signed-package approach - -The existing [VM deployment script](../../vm-samples/BuildRandomCVM.ps1) was -inspected but **not modified**. Its H100 path forces a confidential Ubuntu -22.04 VM, downloads the checksum-pinned V4.3.3 onboarding archive, prepares a -snapshot kernel, and reboots. Its non-GPU Ubuntu choices do not establish an -AKS confidential GPU image choice. Do not copy its kernel preparation, moving -kernel metapackage, reboot sequence or Docker installation onto a managed AKS -node. - -The reusable part is the **signed module package**, selected for the running -kernel, with coherent firmware and user-space libraries. On 2026-11-10, a -disposable Ubuntu 24.04 pod on the actual H100 worker downloaded and extracted: - -| Exact Ubuntu package | Package version | Module driver version | Inspection | -|---|---|---|---| -| `linux-modules-nvidia-595-server-open-6.8.0-1071-azure-fde` | `6.8.0-1071.78` | `595.91.07` | Five prebuilt modules; correct kernel vermagic; Canonical signer. | -| `linux-modules-nvidia-615-open-6.8.0-1071-azure-fde` | `6.8.0-1071.78` | `615.71.09` | Five prebuilt modules; correct kernel vermagic; Canonical signer. | - -Both trees reported `Canonical Ltd. Kernel Module Signing`, signing-key ID -`E9:DF:13:0F:92:92:A9:B7`, SHA-512 signatures, and vermagic -`6.8.0-1071-azure-fde SMP mod_unload modversions`. Metadata alone does not prove -the node accepts a signature. A subsequent privileged probe **did load the -615 NVIDIA and UVM modules with guest Secure Boot enabled**. The first -`nvidia-smi` identified an NVIDIA H100 NVL and driver `615.71.09`. - -The probe downloaded exact packages and extracted them, rather than installing -the full VM driver metapackage onto the host. A clean-container `apt-get -s` -simulation of that metapackage also selected DKMS, kernel and X11 packages. -It is therefore not a safe managed-AKS installation recipe. Only the matching -615 firmware directory was temporarily copied into the host firmware path; -the host kernel and its module tree were not replaced. - -Initial GPU detection was **not sustained readiness**: persistence was not -started before the first short-lived query. The next CC query returned -`No devices were found` (exit 6). Kernel logs reported: +TensorFlow version: 2.15.0 +GPU Available: [PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')] +Running GPU matmul... +GPU matmul device: /job:localhost/replica:0/task:0/device:GPU:0 +Elapsed: 4.1502110958099365 sec +``` + +TensorFlow logged duplicate CUDA factory registration messages and a compute +capability 9.0 PTX JIT warning, but the job completed with explicit GPU placement. +Both redacted attestation logs reported `GPU Attestation is Successful.` for +the same worker GPU. + +Automatic shutdown completed in this run without manual recovery: +`FinalPowerState: Stopped`, `ComputeDeallocated: true`. An independent Azure +check at 14:43:14 UTC confirmed AKS Stopped/Succeeded and capacity zero / zero +instances in both the GPU and system scale sets. +The AKS identity's Contributor role was verified on exactly its managed node +resource group. +The vendor-support caveat above still applies; retained disks/IPs can still cost money. + +### Redacted GPU attestation example + +The verifier output below is from that successful run. GPU identifiers and +attestation tokens have been removed; package-installation noise is omitted. +Both the pre-workload and post-workload checks passed for the same GPU. +**Version numbers and exact output may change** as AKS images, drivers, firmware +and the upstream verifier evolve. This demonstrates GPU attestation only. ```text -kgspCheckGspRmCcCleanup_GH100: CC secret cleanup failed due to timeout! -RmInitAdapter: osInitNvMapping failed, bailing out of RmInitAdapter -RmInitAdapter failed! (0x22:0x56:904) +== Run attestation == +Generating random nonce in the local GPU Verifier .. +Number of GPUs available : 1 +Fetching GPU 0 information from GPU driver. +All GPU Evidences fetched successfully +----------------------------------- +Verifying GPU: + Driver version fetched : 580.178.04 + VBIOS version fetched : 96.00.9f.00.04 + Validating GPU certificate chains. + The firmware ID in the device certificate chain is matching with the one in the attestation report. + GPU attestation report certificate chain validation successful. + The certificate chain revocation status verification successful. + Authenticating attestation report + The nonce in the SPDM GET MEASUREMENT request message is matching with the generated nonce. + Driver version fetched from the attestation report : 580.178.04 + VBIOS version fetched from the attestation report : 96.00.9f.00.04 + Attestation report signature verification successful. + Attestation report verification successful. + Authenticating the RIMs. + Authenticating Driver RIM + Fetching the driver RIM from the RIM service. + RIM Schema validation passed. + driver RIM certificate chain verification successful. + The certificate chain revocation status verification successful. + driver RIM signature verification successful. + Driver RIM verification successful + Authenticating VBIOS RIM. + Fetching the VBIOS RIM from the RIM service. + RIM Schema validation passed. + vbios RIM certificate chain verification successful. + The certificate chain revocation status verification successful. + vbios RIM signature verification successful. + VBIOS RIM verification successful + Comparing measurements (runtime vs golden) + The runtime measurements are matching with the golden measurements. + GPU is in expected state. + GPU 0 with UUID verified successfully. + GPU Ready State is already READY +GPU Attestation is Successful. + +[Attestation token omitted] ``` - -Unloading and reloading the same signed module then failed with `No such device`. -This moves the investigation past the generic container's kernel-layout error -to GPU lifecycle/persistence initialization. It does not establish that -persistence ordering is the sole cause, that 615 is supported for this AKS image, -or that attestation succeeds. The clean persistence-first retry below supplies -the sustained-readiness and attestation evidence missing from this first attempt. - -### Successful persistence-first retry - -A fresh-node retry used the same exact signed `615.71.09` packages, preloaded -`ecdsa_generic`, `ecdh` and `ecc`, loaded NVIDIA/UVM, and started -`nvidia-persistenced --uvm-persistence-mode` **before** invoking `nvidia-smi`. -It then set confidential-compute ready state and copied the coherent extracted -driver tree into `/run/nvidia/driver`. GPU Operator `v26.7.2`, installed with -`driver.enabled=false`, supplied the toolkit/runtime and device plugin without -replacing AKS-managed containerd. - -| Acceptance check | Live result | -|---|---| -| Current worker | Ubuntu 24.04.5, AKS image `AKSUbuntu-2404gen2CVMcontainerd-202610.30.1`, kernel `6.8.0-1071-azure-fde`. | -| Trusted driver loading | Canonical-signed NVIDIA/UVM `615.71.09` accepted by the running Secure Boot kernel; H100 NVL detected. | -| Confidential GPU | `CC status: ON`, `CC Environment: PRODUCTION`, `CC GPUs Ready State: Ready`; persistence daemon alive. | -| Kubernetes integration | Exactly one allocatable `nvidia.com/gpu`; Operator CUDA validator passed. | -| Independent GPU workload | Nonprivileged Job requested/limited one GPU using runtime class `nvidia`; vector addition of 50,000 elements returned `Test PASSED`. No CPU fallback. | -| GPU attestation before/after workload | V4.5.1 archive, verifier `2.3.0`, strict OCSP; exit zero, expected GPU UUID verified, exact `GPU Attestation is Successful.` verdict on both runs. | -| Sustained readiness | Three repeated post-workload CC queries succeeded and persistence remained alive. | -| Separate CPU/CVM attestation | Checksum-verified Azure guest CLI `v0.1.0`, MAA provider; exit zero and decoded `sevsnpvm` / `azure-compliant-cvm` claims. | - -The published installer, probe manifest, workload and both attestation scripts -were then replayed unchanged on another fresh worker. The installer emitted -`SIGNED_DRIVER_LOADED=1`; GPU attestation emitted -`GPU_ATTESTATION_VALIDATED=1` before and after the Job's `Test PASSED`; -CVM attestation emitted `CVM_ATTESTATION_VALIDATED=1`. The GPU UUID was -discovered dynamically and differed from the previous allocation. The worker -still had ConfidentialVM, Secure Boot and vTPM enabled, with one allocatable GPU. -All nine Bash blocks in the CLI walkthrough passed syntax checks. Re-running -the installer on the initialized worker failed at its already-loaded-module -guard before mutation, while the existing persistence daemon and GPU remained -ready. Optional DCGM exporter readiness remained `0/1`; monitoring health is -not claimed by these workload and attestation results. - -This comparison supports persistence-first initialization as the useful remedy -for this reproduction, not proof of the sole cause or of vendor support. -The workload container's `nvidia-smi` displayed Persistence Off while the probe -reported Enabled with its UVM persistence daemon alive. That display discrepancy -was not conclusively explained; repeated readiness, CUDA and attestation checks -passed. Do not equate any single persistence display with attestation. - -CPU attestation initially failed with MAA HTTP 400, `TcgLogs is empty in -attestation request`. Mounting host `/sys/kernel/security` read-only exposed -the TPM event log and resolved that failure. Its sysfs stat size is zero despite -containing readable data: use `wc -c`, not `test -s`, to check it. GPU attestation -and CPU/CVM attestation are separate results; the latter records the successful -MAA request and returned claims, not an additional independent JWT verifier. - -The reusable artifacts are [the probe manifest](signed-driver-probe.yaml), -[exact-package installer](Install-SignedDriver.sh), -[GPU-only workload](gpu-smoke-test.yaml), -[strict GPU attestation](Test-GpuAttestation.sh), and -[CPU/CVM attestation](Test-CvmAttestation.sh). The verifier warns of future -deprecation in favour of `nvattest`; that migration has not been tested here. -Full attestation logs can include tokens and must remain local, not in Git. - -**Lifecycle boundary:** the privileged probe holds the persistence daemon and -expires after four hours. Deleting it ends persistence; `/run/nvidia/driver` is -ephemeral. Reboot, node replacement or a new image/kernel requires fresh matching -package validation and setup. The installer intentionally rejects an already -initialized node rather than guessing how to reconcile it. No DKMS, host kernel -replacement, unsigned modules, key enrollment or Secure Boot bypass was used. - -### Clues from the full upstream onboarding flow - -The [full upstream repository](https://github.com/Azure/az-cgpu-onboarding/tree/136262ef00ad75b8e0bc789563f043006fb68f4c), -not just its AKS page, provides several useful distinctions: - -- **Signed exact-ABI modules:** the V4.3.3 VM installer uses - `nvidia-driver-595-server-open` with a moving - `linux-modules-nvidia-595-server-open-azure-fde` metapackage. Reuse the - packaging principle, not the unmodified host installer or its moving kernel - selection. -- **Crypto dependencies:** the - [driver installer](https://github.com/Azure/az-cgpu-onboarding/blob/136262ef00ad75b8e0bc789563f043006fb68f4c/src/step-1-install-gpu-driver.sh) - preloads `ecdsa_generic` and `ecdh`; the inspected NVIDIA module also depends - on `ecc`. The live signed-module probe loaded these before NVIDIA. -- **Persistence before validation:** the - [persistence override](https://github.com/Azure/az-cgpu-onboarding/blob/136262ef00ad75b8e0bc789563f043006fb68f4c/src/nvidia-persistenced.override.conf) - enables `--uvm-persistence-mode`, then sets GPU ready state with - `nvidia-smi conf-compute -srs 1`. The - [attestation script](https://github.com/Azure/az-cgpu-onboarding/blob/136262ef00ad75b8e0bc789563f043006fb68f4c/src/step-2-attestation.sh) - waits for persistence and checks that it is enabled. A single successful - `nvidia-smi` invocation is not an adequate substitute. -- **Runtime integration is separate:** the - [VM tools installer](https://github.com/Azure/az-cgpu-onboarding/blob/136262ef00ad75b8e0bc789563f043006fb68f4c/src/step-3-install-gpu-tools.sh) - installs Docker/containerd and configures the Docker NVIDIA runtime. Do not - overwrite AKS-managed containerd with that installer. A functioning module - still needs AKS-compatible toolkit, device-plugin and GPU scheduling checks. -- **Standalone VM alternative:** the root README recommends a prebuilt - community-gallery VM image with drivers, CUDA, Docker and verifier included. - The current automatic VM onboarding also has an Ubuntu 24.04 path. These - are alternatives if AKS is not mandatory, not AKS node-image selections or - evidence of a working confidential AKS configuration. -- **Different failures need different fixes:** the - [FAQ](https://github.com/Azure/az-cgpu-onboarding/blob/136262ef00ad75b8e0bc789563f043006fb68f4c/Frequently-Asked-Questions.md) - describes driver/library mismatches after updates; that is distinct from - the initial generic-container `vmlinuz` extraction failure. The FAQ also - explains the Azure vTPM CPU attestation route; absence of `/dev/sev-guest` - alone is not proof that an Azure SNP VM lacks confidential protection. -- **Verifier alternatives do not fix drivers:** the optional preview - `nvattest` path is a verifier choice, not a way around missing modules, - failed GPU initialization or the requirement for positive attestation. - -### Newer Ubuntu node OS availability - -Checked on 2026-11-10 against the published AKS documentation and the existing -West Europe confidential H100 pool's node-image upgrade API: - -| Ubuntu version | Published AKS availability | Applicability to this reproduction | -|---|---|---| -| 25.x | Not listed as a supported AKS node OS. | No documented AKS node OS option established. | -| 26.04 | Preview for Kubernetes 1.36 and later, using OS SKU `Ubuntu2604` and feature flag `Ubuntu2604Preview`. Requires Generation 2 VMs; Confidential VM, Trusted Launch and FIPS are explicitly unsupported. | Not a supported replacement for the confidential H100 worker, even though this cluster uses Kubernetes 1.36.5. | -| 24.04 | Supported Ubuntu family for AKS Confidential VM node images. | The existing pool's latest offered image remains `AKSUbuntu-2404gen2CVMcontainerd-202610.30.1`; the worker observed during the experiment used Ubuntu 24.04.5. | - -Sources: [AKS OS version support and Ubuntu 26.04 preview restrictions](https://learn.microsoft.com/azure/aks/upgrade-os-version#migrate-to-ubuntu-2604-preview) -and [AKS node-image variants](https://learn.microsoft.com/azure/aks/node-images). -General AKS preview availability is distinct from support for a confidential -GPU node and does not prove availability for every VM size or region. Likewise, -the existing pool's upgrade response describes that pool's image track, not all -AKS OS SKUs. - -Ubuntu 26.04 is therefore not a documented workaround for this driver blocker. -A standard, non-confidential node would not validate the required environment. -Changing a workload container's Ubuntu base image would not change the managed -host kernel or resolve the driver container's Azure-FDE package-layout failure. -This availability check was read-only; the cluster remained stopped. - -### Reproduce the failed generic-container stage with Azure CLI - -Use the existing subscription, resource group and cluster from the deployment -stage. Keep the sample kubeconfig separate from the default context. These Bash -commands follow the upstream guide's Azure CLI style; they resume billable nodes. -The local values file adds both GPU-taint tolerations and open-module selection. -**This block deliberately reenables the failed generic installer. Do not run it -over a successful signed-package probe.** Use the updated walkthrough for the -working experimental route. - -```bash -az aks nodepool get-upgrades --subscription "$subscription" \ - --resource-group "$resourceGroup" --cluster-name "$clusterName" \ - --nodepool-name gpunp - -az aks start --subscription "$subscription" \ - --resource-group "$resourceGroup" --name "$clusterName" - -az aks get-credentials --subscription "$subscription" \ - --resource-group "$resourceGroup" --name "$clusterName" \ - --file .local/kubeconfig --overwrite-existing - -export KUBECONFIG="$PWD/.local/kubeconfig" -kubectl get nodes -o wide - -helm upgrade --install gpu-operator gpu-operator \ - --repo https://helm.ngc.nvidia.com/nvidia --version v26.7.2 \ - --namespace gpu-operator --create-namespace \ - --values gpu-operator-experimental-values.yaml --set driver.enabled=true - -kubectl --namespace gpu-operator get pods -o wide -kubectl --namespace gpu-operator logs \ - --selector app=nvidia-driver-daemonset --container nvidia-driver-ctr \ - --tail=150 -# After a container restart, inspect the failed attempt: -kubectl --namespace gpu-operator logs \ - --selector app=nvidia-driver-daemonset --container nvidia-driver-ctr \ - --previous --tail=150 -kubectl get nodes --selector agentpool=gpunp \ - -o 'jsonpath={.items[0].status.allocatable}' - -az aks stop --subscription "$subscription" \ - --resource-group "$resourceGroup" --name "$clusterName" -az aks show --subscription "$subscription" \ - --resource-group "$resourceGroup" --name "$clusterName" \ - --query '{power:powerState.code,state:provisioningState}' -``` - -Do not treat Helm's `deployed` status as GPU readiness. In this experiment Helm -succeeded, but the driver did not. Do not work around the layout failure by -replacing the managed host kernel, disabling Secure Boot, installing unsigned -modules, or inventing a driver-ready state. Resolution needs a compatible, -trusted installation path for this Azure-FDE image. The signed-package route -above bypasses the broken container entrypoint without changing the host kernel. - -### Attestation acceptance after driver enablement - -The existing [VM GPU attestation flow](../../vm-samples/BuildRandomCVM.ps1) -provides a reusable checksum-pinned verifier and explicit positive-verdict -checks. Its host-kernel installation and reboot steps do **not** apply to AKS. -Before adapting the verifier Job, require: - -1. A loaded, trusted NVIDIA module and working `nvidia-smi`. -2. `CC status: ON`, `CC Environment: PRODUCTION`, and one allocatable H100 GPU. -3. A GPU-targeted workload that fails rather than falling back to CPU. -4. A checksum-verified, version-pinned verifier, zero exit status, the exact - `GPU Attestation is Successful.` verdict, and successful verification of the - expected GPU UUID. A Completed pod alone is insufficient. - -For the newer reviewed Azure verifier V4.5.1, the release archive SHA-256 is -`075ee785f6d65799d99ae18102fb88bdab4afe31540bf16673a27545c0ba80e8`. -Its Python package requires Python 3.12. Use strict OCSP validation -(`--ocsp_attestation_settings strict`) rather than permissive defaults, and -preserve the verifier evidence. GPU attestation is separate from CPU/CVM -attestation; both have now passed separately through the signed-package route. -The [updated onboarding walkthrough](Confidential-GPU-H100-AKS-Onboarding.md) -preserves the Azure CLI workflow while clearly labeling this experimental path. diff --git a/aks-samples/aks-gpu-worker/Test-CvmAttestation.sh b/aks-samples/aks-gpu-worker/Test-CvmAttestation.sh deleted file mode 100644 index 7e236a9..0000000 --- a/aks-samples/aks-gpu-worker/Test-CvmAttestation.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -trap 'echo "ERROR: CVM attestation failed at line $LINENO; no success is claimed." >&2' ERR - -test -r /sys/kernel/security/tpm0/binary_bios_measurements -test "$(wc -c < /sys/kernel/security/tpm0/binary_bios_measurements)" -gt 0 -export DEBIAN_FRONTEND=noninteractive -apt-get update -qq -apt-get install -y --no-install-recommends curl ca-certificates -mkdir -p /tmp/signed-attestation -cd /tmp/signed-attestation -curl -fsSL --retry 3 -o azure-guest-attest \ - https://github.com/Azure/azure-guest-attestation-sdk/releases/download/azure-guest-attest-v0.1.0/azure-guest-attest-x86_64-unknown-linux-musl -echo 'c6559e35d588b1dc9074c443beadce395da34b5beba2eb6e276195a694064b43 azure-guest-attest' | - sha256sum -c - -chmod 755 azure-guest-attest -result=/tmp/signed-attestation/cpu-result.log -if ! timeout 180 ./azure-guest-attest guest-attest --provider maa --decode > "$result" 2>&1; then - echo "ERROR: MAA request failed; inspect $result inside the probe." >&2 - exit 1 -fi -grep -E '"x-ms-attestation-type"[[:space:]]*:[[:space:]]*"sevsnpvm"' "$result" -grep -E '"x-ms-compliance-status"[[:space:]]*:[[:space:]]*"azure-compliant-cvm"' "$result" -echo CVM_ATTESTATION_VALIDATED=1 -exit 0 diff --git a/aks-samples/aks-gpu-worker/Test-DriverCompatibility.ps1 b/aks-samples/aks-gpu-worker/Test-DriverCompatibility.ps1 deleted file mode 100644 index 8ec7081..0000000 --- a/aks-samples/aks-gpu-worker/Test-DriverCompatibility.ps1 +++ /dev/null @@ -1,40 +0,0 @@ -#Requires -Version 7.2 -[CmdletBinding(DefaultParameterSetName = 'Kernel')] -param( - [Parameter(Mandatory, ParameterSetName = 'Kernel')] - [ValidatePattern('^[0-9]+\.[0-9]+\.[0-9]+-[0-9]+-azure-fde$')] - [string]$KernelVersion, - [Parameter(Mandatory, ParameterSetName = 'Node')][string]$KubeConfig, - [Parameter(ParameterSetName = 'Node')][string]$PoolName = 'gpunp', - [ValidateNotNullOrEmpty()][string[]]$DriverBranches = @('580', '595'), - [string]$OutputPath, - [switch]$RequireAvailable -) -. (Join-Path $PSScriptRoot 'Common.ps1') - -if ($PSCmdlet.ParameterSetName -eq 'Node') { - $node = Get-GpuNode -KubeConfig $KubeConfig -PoolName $PoolName - if ($node.status.nodeInfo.osImage -notlike 'Ubuntu 24.04*') { - throw "Unsupported worker OS: $($node.status.nodeInfo.osImage). Expected Ubuntu 24.04." - } - $KernelVersion = $node.status.nodeInfo.kernelVersion -} -if ($KernelVersion -notmatch '^[0-9]+\.[0-9]+\.[0-9]+-[0-9]+-azure-fde$') { - throw "Unsupported confidential worker kernel: $KernelVersion." -} -$results = @(foreach ($branch in $DriverBranches) { - if ($branch -notmatch '^[0-9]+$') { throw "Invalid driver branch: $branch" } - Get-DriverManifest -Tag "$branch-$KernelVersion-ubuntu24.04" -}) -$report = [pscustomobject]@{ - KernelVersion = $KernelVersion - Manifests = $results - MatchingImageAvailable = (@($results | Where-Object Available).Count -gt 0) - SupportValidated = $false -} -$json = $report | ConvertTo-Json -Depth 10 -if ($OutputPath) { $json | Set-Content -LiteralPath $OutputPath } -$json -if ($RequireAvailable -and -not $report.MatchingImageAvailable) { - throw "BLOCKED: no exact-kernel driver image for $KernelVersion. Do not install a different kernel's module or downgrade the AKS image." -} diff --git a/aks-samples/aks-gpu-worker/Test-GpuAttestation.sh b/aks-samples/aks-gpu-worker/Test-GpuAttestation.sh deleted file mode 100644 index 503f6ae..0000000 --- a/aks-samples/aks-gpu-worker/Test-GpuAttestation.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -trap 'echo "ERROR: GPU attestation failed at line $LINENO; no success is claimed." >&2' ERR - -export LD_LIBRARY_PATH=/tmp/signed-driver/root/usr/lib/x86_64-linux-gnu -smi=/tmp/signed-driver/root/usr/bin/nvidia-smi -uuid=$("$smi" --query-gpu=uuid --format=csv,noheader) -[[ "$uuid" =~ ^GPU-[[:xdigit:]-]+$ ]] -"$smi" conf-compute -f | grep -Fx 'CC status: ON' -"$smi" conf-compute -e | grep -Fx 'CC Environment: PRODUCTION' -kill -0 "$(cat /var/run/nvidia-persistenced/nvidia-persistenced.pid)" - -export DEBIAN_FRONTEND=noninteractive -apt-get update -qq -apt-get install -y --no-install-recommends curl ca-certificates python3 python3-venv -mkdir -p /tmp/signed-attestation -cd /tmp/signed-attestation -curl -fsSL --retry 3 -o onboarding.tar.gz \ - https://github.com/Azure/az-cgpu-onboarding/releases/download/V4.5.1/cgpu-onboarding-package.tar.gz -echo '075ee785f6d65799d99ae18102fb88bdab4afe31540bf16673a27545c0ba80e8 onboarding.tar.gz' | sha256sum -c - -tar -xzf onboarding.tar.gz -mkdir -p verifier -tar -xf cgpu-onboarding-package/local_gpu_verifier.tar -C verifier -cd verifier -python3 -m venv /tmp/signed-attestation/venv -/tmp/signed-attestation/venv/bin/python -m pip install . > /tmp/signed-attestation/pip-install.log 2>&1 || { - cat /tmp/signed-attestation/pip-install.log >&2 - exit 1 -} -/tmp/signed-attestation/venv/bin/python -m pip show nv-local-gpu-verifier | - grep -Fx 'Version: 2.3.0' -result=/tmp/signed-attestation/gpu-result.log -status=0 -timeout 300 /tmp/signed-attestation/venv/bin/python -m verifier.cc_admin \ - --ocsp_attestation_settings strict > "$result" 2>&1 || status=$? -# Retain the full local evidence without echoing attestation tokens. -sed '/Entity Attestation Token:/,$d' "$result" -test "$status" -eq 0 -grep -Fx 'GPU Attestation is Successful.' "$result" -grep -F "GPU 0 with UUID $uuid verified successfully." "$result" -echo GPU_ATTESTATION_VALIDATED=1 -exit 0 diff --git a/aks-samples/aks-gpu-worker/gpu-operator-experimental-values.yaml b/aks-samples/aks-gpu-worker/gpu-operator-experimental-values.yaml deleted file mode 100644 index 4ed1424..0000000 --- a/aks-samples/aks-gpu-worker/gpu-operator-experimental-values.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# Experimental generic driver: availability is not proof of Azure-FDE support. -driver: - version: "615.71.09" - kernelModuleType: open -daemonsets: - tolerations: - - key: nvidia.com/gpu - operator: Exists - effect: NoSchedule - - key: sku - operator: Equal - value: gpu - effect: NoSchedule -node-feature-discovery: - worker: - tolerations: - - key: nvidia.com/gpu - operator: Exists - effect: NoSchedule - - key: sku - operator: Equal - value: gpu - effect: NoSchedule diff --git a/aks-samples/aks-gpu-worker/gpu-smoke-test.yaml b/aks-samples/aks-gpu-worker/gpu-smoke-test.yaml deleted file mode 100644 index 49b44da..0000000 --- a/aks-samples/aks-gpu-worker/gpu-smoke-test.yaml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: confidential-gpu-smoke -spec: - backoffLimit: 0 - activeDeadlineSeconds: 180 - template: - spec: - restartPolicy: Never - runtimeClassName: nvidia - nodeSelector: - agentpool: gpunp - tolerations: - - key: sku - operator: Equal - value: gpu - effect: NoSchedule - - key: nvidia.com/gpu - operator: Exists - effect: NoSchedule - containers: - - name: vector-add - image: nvcr.io/nvidia/gpu-operator:v26.7.2 - command: [sh, -c, "nvidia-smi && vectorAdd"] - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: [ALL] - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - memory: 1Gi - nvidia.com/gpu: 1 diff --git a/aks-samples/aks-gpu-worker/signed-driver-probe.yaml b/aks-samples/aks-gpu-worker/signed-driver-probe.yaml deleted file mode 100644 index 8076572..0000000 --- a/aks-samples/aks-gpu-worker/signed-driver-probe.yaml +++ /dev/null @@ -1,69 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: signed-driver-load -spec: - restartPolicy: Never - nodeSelector: - agentpool: gpunp - tolerations: - - key: sku - operator: Equal - value: gpu - effect: NoSchedule - - key: nvidia.com/gpu - operator: Exists - effect: NoSchedule - containers: - - name: probe - image: ubuntu:24.04 - command: [bash, -c, "sleep 14400"] - securityContext: - privileged: true - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - memory: 4Gi - volumeMounts: - - name: modules - mountPath: /lib/modules - readOnly: true - - name: firmware - mountPath: /lib/firmware - - name: devices - mountPath: /dev - - name: driver - mountPath: /run/nvidia/driver - - name: sysfirmware - mountPath: /sys/firmware - readOnly: true - - name: security - mountPath: /sys/kernel/security - readOnly: true - volumes: - - name: modules - hostPath: - path: /lib/modules - type: Directory - - name: firmware - hostPath: - path: /lib/firmware - type: Directory - - name: devices - hostPath: - path: /dev - type: Directory - - name: driver - hostPath: - path: /run/nvidia/driver - type: DirectoryOrCreate - - name: sysfirmware - hostPath: - path: /sys/firmware - type: Directory - - name: security - hostPath: - path: /sys/kernel/security - type: Directory diff --git a/aks-samples/aks-gpu-worker/tests/Test-Sample.ps1 b/aks-samples/aks-gpu-worker/tests/Test-Sample.ps1 index ed5b56c..0791590 100644 --- a/aks-samples/aks-gpu-worker/tests/Test-Sample.ps1 +++ b/aks-samples/aks-gpu-worker/tests/Test-Sample.ps1 @@ -1,164 +1,269 @@ #Requires -Version 7.2 -$ErrorActionPreference = 'Stop' -$root = Split-Path $PSScriptRoot -Parent -. (Join-Path $root 'Common.ps1') +. (Join-Path $PSScriptRoot '..\Common.ps1') $script:passed = 0 -$state = @{} -function Assert($Condition, [string]$Message) { +function Assert-True { + param([bool]$Condition, [string]$Message) if (-not $Condition) { throw $Message } + $script:passed++ } -function Expect-Failure([scriptblock]$Action, [string]$Pattern) { - try { & $Action | Out-Null } - catch { - Assert ($_.Exception.Message -match $Pattern) "Unexpected error: $_" - return + +function Assert-Throws { + param([scriptblock]$Action, [string]$Pattern) + $caught = $false + try { & $Action } catch { + if ($_.Exception.Message -notmatch $Pattern) { throw } + $caught = $true } - throw "Expected a failure matching '$Pattern'." -} -function Case([string]$Name, [scriptblock]$Action) { - $state.status = 200 - $state.digest = 'sha256:' + ('a' * 64) - $state.content = '{"errors":[{"code":"MANIFEST_UNKNOWN","message":"manifest unknown"}]}' - $state.networkError = $false - & $Action + Assert-True $caught "Expected error matching $Pattern." +} + +foreach ($kernel in @('6.8.0-1071-azure-fde', '6.8.0-1072-azure-fde', '6.8.0-11000-azure-fde', '6.11.0-1-azure-fde', '7.0.0-1-azure-fde')) { + Assert-MinimumKernel $kernel $script:passed++ - Write-Host "PASS $Name" } -function Invoke-RestMethod { @{ token = 'offline-token' } } +foreach ($kernel in @('6.8.0-1070-azure-fde', '6.8.0-999-azure-fde', '6.7.0-9999-azure-fde')) { + Assert-Throws { Assert-MinimumKernel $kernel } 'older' +} +foreach ($kernel in @('6.8.0-1071-azure', '6.8.0-1071-generic', 'garbage')) { + Assert-Throws { Assert-MinimumKernel $kernel } 'Not an Azure FDE' +} +Assert-Throws { Assert-MinimumKernel '6.8.0-1071-azure-fde' 'garbage' } 'Invalid minimum' +$versions = @' +{"values":[ + {"isPreview":false,"patchVersions":{"1.36.9":{},"1.36.10":{}}}, + {"isPreview":true,"patchVersions":{"1.99.0":{}}}, + {"isPreview":null,"patchVersions":{"1.35.11":{}}} +]} +'@ | ConvertFrom-Json +Assert-True ((Get-LatestKubernetesPatch $versions) -eq '1.36.10') 'Patch selection must be numerical and exclude preview.' +Assert-Throws { Get-LatestKubernetesPatch ([pscustomobject]@{ values = @() }) } 'No stable' + +$keyDirectory = Join-Path ([System.IO.Path]::GetTempPath()) "aks-gpu-key-test-$([guid]::NewGuid())" +New-Item -ItemType Directory -Path $keyDirectory | Out-Null +$keyPath = Join-Path $keyDirectory 'ssh-key' +try { + New-SampleSshKey -Path $keyPath + Assert-True ((Get-Content "$keyPath.pub" -Raw) -match '^ssh-rsa ') 'Expected a real RSA public key.' + Assert-Throws { New-SampleSshKey -Path (Join-Path $keyDirectory 'missing\ssh-key') } 'ssh-keygen failed' +} +finally { + foreach ($path in @($keyPath, "$keyPath.pub")) { + if (Test-Path $path) { Remove-Item -LiteralPath $path -Force } + } + Remove-Item -LiteralPath $keyDirectory +} + +function Invoke-RestMethod { param($Uri) return @{ token = 'unit-test-token' } } function Invoke-WebRequest { param($Uri, $Headers, [switch]$SkipHttpErrorCheck) - Assert ($Headers.Authorization -eq 'Bearer offline-token') 'Anonymous registry token not supplied.' - foreach ($type in @('oci.image.manifest', 'oci.image.index', 'distribution.manifest.v2', 'distribution.manifest.list.v2')) { - Assert ($Headers.Accept.Contains($type)) "Missing manifest media type: $type" + Assert-True ($Headers.Authorization -eq 'Bearer unit-test-token') 'Registry authentication was not passed.' + Assert-True ($Uri -like '*/580-6.8.0-1071-azure-fde-ubuntu24.04') 'Driver tag differs from the guide.' + return $script:response +} +$script:response = @{ + StatusCode = 404 + Content = '{"errors":[{"code":"MANIFEST_UNKNOWN","message":"manifest unknown"}]}' +} +$image = Get-ExactDriverImages '6.8.0-1071-azure-fde' @('580') +Assert-True (-not $image.Available -and $image.HttpStatus -eq 404 -and -not $image.Digest) 'Missing-image evidence was incorrect.' +foreach ($status in @(401, 403, 429, 500)) { + $script:response = @{ StatusCode = $status } + Assert-Throws { Get-ExactDriverImages '6.8.0-1071-azure-fde' @('580') } "HTTP $status" +} +$script:response = @{ StatusCode = 404; Content = '{"errors":[{"code":"NAME_UNKNOWN","message":"repository unknown"}]}' } +Assert-Throws { Get-ExactDriverImages '6.8.0-1071-azure-fde' @('580') } 'Unexpected registry 404' +$script:response = @{ StatusCode = 200; Headers = @{ 'Docker-Content-Digest' = @('sha256:' + ('a' * 64)) } } +$image = Get-ExactDriverImages '6.8.0-1071-azure-fde' @('580') +Assert-True ($image.Available -and $image.Digest -eq ('sha256:' + ('a' * 64))) 'Available-image evidence was incorrect.' +$script:response = @{ StatusCode = 200; Headers = @{ 'Docker-Content-Digest' = @('not-a-digest') } } +Assert-Throws { Get-ExactDriverImages '6.8.0-1071-azure-fde' @('580') } 'Missing registry digest' +Remove-Item Function:\Invoke-RestMethod +Remove-Item Function:\Invoke-WebRequest + +$workload = @' +apiVersion: batch/v1 +kind: Job +metadata: + name: tf-matmul-demo +spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: tf-matmul-demo + image: tensorflow/tensorflow:2.15.0-gpu + args: + - | + c = tf.matmul(a, b) +'@ +$guardedWorkload = Get-GuideWorkloadManifest $workload +Assert-True ($guardedWorkload.Contains('activeDeadlineSeconds: 900')) 'Workload deadline missing.' +Assert-True ($guardedWorkload.Contains('restartPolicy: Never')) 'Workload should not retry indefinitely.' +Assert-True ($guardedWorkload.Contains('runtimeClassName: nvidia')) 'NVIDIA runtime missing.' +Assert-True ($guardedWorkload.Contains('agentpool: gpunp')) 'Workload must target GPU pool.' +Assert-True ($guardedWorkload.Contains('tf.config.set_soft_device_placement(False)')) 'CPU fallback must be disabled.' +Assert-True ($guardedWorkload.Contains("with tf.device('/GPU:0'):")) 'Matmul must explicitly target GPU.' +Assert-True ($guardedWorkload.Contains("assert 'GPU:0' in c.device")) 'Actual tensor device must be checked.' +Assert-True ($guardedWorkload.Contains('image: tensorflow/tensorflow:2.15.0-gpu')) 'Guide workload image changed.' +Assert-Throws { Get-GuideWorkloadManifest ($workload.Replace('c = tf.matmul(a, b)', 'unexpected')) } 'Upstream workload manifest changed' + +$script:commands = [System.Collections.Generic.List[string]]::new() +function Invoke-CheckedCommand { + param([string]$Command, [string[]]$ArgumentList) + $call = $ArgumentList -join ' ' + $script:commands.Add($call) + if ($call -like 'aks list *') { return '[]' } + if ($call -like 'vmss list *') { return '[]' } + if ($call -like 'vm list *') { + return '[{"name":"ncc","location":"westeurope","powerState":"VM running","id":"/vm/ncc","hardwareProfile":{"vmSize":"Standard_NCC40ads_H100_v5"}},{"name":"untouched","location":"westeurope","powerState":"VM running","hardwareProfile":{"vmSize":"Standard_D2s_v5"}}]' } - Assert $SkipHttpErrorCheck 'HTTP errors must be inspected explicitly.' - if ($state.networkError) { throw 'Mock network failure' } - @{ - StatusCode = $state.status - Headers = @{ 'Docker-Content-Digest' = @($state.digest) } - Content = $state.content + if ($call -like 'vm deallocate *') { return } + if ($call -like 'vm get-instance-view *') { return $script:power } + throw "Unexpected test command: $call" +} +Assert-Throws { Stop-ExistingNccCompute -Az az -Subscription test -Location westeurope } 'needs -StopExistingNcc' +Assert-True (-not @($script:commands | Where-Object { $_ -like '* deallocate *' }).Count) 'Read-only inventory modified compute.' +$script:commands.Clear() +$script:power = 'PowerState/deallocated' +Stop-ExistingNccCompute -Az az -Subscription test -Location westeurope -Stop +Assert-True (@($script:commands | Where-Object { $_ -like 'vm deallocate *' }).Count -eq 1) 'Expected exactly one NCC shutdown.' +Assert-True (-not @($script:commands | Where-Object { $_ -like '*untouched*' }).Count) 'Non-NCC VM was modified.' +Assert-True (-not @($script:commands | Where-Object { $_ -notlike '*--subscription test*' }).Count) 'Missing subscription scope.' +$script:power = 'PowerState/running' +Assert-Throws { Stop-ExistingNccCompute -Az az -Subscription test -Location westeurope -Stop } 'did not deallocate' +Remove-Item Function:\Invoke-CheckedCommand + +$script:readCount = 0 +$script:readyStates = @('Updating', 'Succeeded') +function Invoke-CheckedCommand { + param([string]$Command, [string[]]$ArgumentList) + Assert-True ('--subscription' -in $ArgumentList -and 'test' -in $ArgumentList) 'Readiness lookup lost subscription scope.' + $state = $script:readyStates[[Math]::Min($script:readCount, $script:readyStates.Count - 1)] + if ($ArgumentList[1] -eq 'show') { + return (@{ provisioningState = $state } | ConvertTo-Json) } + $script:readCount++ + return "[{`"name`":`"sysnp`",`"provisioningState`":`"$state`"}]" } -function New-Pools { - @( - @{ name = 'sysnp'; vmSize = 'Standard_D4ds_v6'; count = 1; osSku = 'Ubuntu2404' - mode = 'System'; enableAutoScaling = $false; provisioningState = 'Succeeded' } - @{ name = 'gpunp'; vmSize = 'Standard_NCC40ads_H100_v5'; count = 1; osSku = 'Ubuntu2404' - mode = 'User'; enableAutoScaling = $false; provisioningState = 'Succeeded' - gpuProfile = @{ driver = 'None' }; nodeTaints = @('sku=gpu:NoSchedule') } - ) -} -$tag = '580-6.8.0-1071-azure-fde-ubuntu24.04' -Case 'available manifests require OCI/Docker Accept headers and a digest' { - $result = Get-DriverManifest $tag - Assert ($result.Available -and $result.Digest -eq $state.digest) 'Incorrect available-image result.' -} -Case 'genuine absent tag is reported, not thrown' { - $state.status = 404 - $result = Get-DriverManifest $tag - Assert (-not $result.Available -and $result.ErrorCode -eq 'MANIFEST_UNKNOWN') 'Missing tag not identified.' -} -Case 'misleading OCI 404 is not a missing tag' { - $state.status = 404 - $state.content = '{"errors":[{"code":"MANIFEST_UNKNOWN","message":"OCI manifest found, but accept header does not support OCI manifests"}]}' - Expect-Failure { Get-DriverManifest $tag } 'unexpected 404' -} -foreach ($status in @(401, 403, 429, 500, 503)) { - Case "HTTP $status throws rather than reporting missing image" { - $state.status = $status - Expect-Failure { Get-DriverManifest $tag } "HTTP $status" +Wait-SampleAksReady az test group cluster -PollSeconds 0 +Assert-True ($script:readCount -eq 2) 'Readiness must wait for background AKS operations.' +$script:readyStates = @('Failed') +Assert-Throws { Wait-SampleAksReady az test group cluster -PollSeconds 0 } 'provisioning failed' +$script:readyStates = @('Updating') +Assert-Throws { Wait-SampleAksReady az test group cluster -TimeoutSeconds 0 -PollSeconds 0 } 'did not settle' +Remove-Item Function:\Invoke-CheckedCommand + +$script:stopCalls = 0 +$script:stopFailures = 1 +$script:stopError = '(OperationNotAllowed) There is an in-progress PutExtensionAddonHandler.PUT operation.' +function Invoke-TestAksStop { + Assert-True (($args -join ' ') -eq 'aks stop -g group -n cluster -o none --subscription test --only-show-errors') 'Shutdown lost explicit target/subscription scope.' + $script:stopCalls++ + $global:LASTEXITCODE = 0 + if ($script:stopCalls -le $script:stopFailures) { + $global:LASTEXITCODE = 1 + $script:stopError } } -Case 'network failure propagates' { - $state.networkError = $true - Expect-Failure { Get-DriverManifest $tag } 'network failure' -} -Case 'invalid digest is rejected' { - $state.digest = 'invalid' - Expect-Failure { Get-DriverManifest $tag } 'valid digest' -} -Case 'invalid registry tag is rejected' { - Expect-Failure { Get-DriverManifest '../invalid' } 'pattern' -} -Case 'compatibility report retains shape and does not claim support' { - $report = & (Join-Path $root 'Test-DriverCompatibility.ps1') -KernelVersion '6.8.0-1071-azure-fde' | ConvertFrom-Json - Assert ($report.Manifests.Count -eq 2 -and $report.MatchingImageAvailable -and -not $report.SupportValidated) 'Incorrect report shape.' -} -Case 'missing images fail the required-availability gate' { - $state.status = 404 - Expect-Failure { - & (Join-Path $root 'Test-DriverCompatibility.ps1') -KernelVersion '6.8.0-1071-azure-fde' -RequireAvailable - } 'BLOCKED: no exact-kernel' -} -Case 'invalid kernel and branch are rejected' { - Expect-Failure { & (Join-Path $root 'Test-DriverCompatibility.ps1') -KernelVersion '6.8.0-generic' } 'pattern' - Expect-Failure { - & (Join-Path $root 'Test-DriverCompatibility.ps1') -KernelVersion '6.8.0-1071-azure-fde' -DriverBranches '580/invalid' - } 'Invalid driver branch' -} -Case 'exact topology and partial system-only topology pass' { - $pools = New-Pools - Assert-SamplePools $pools 'Standard_D4ds_v6' - Assert-SamplePools @($pools[0]) 'Standard_D4ds_v6' -AllowMissingGpu - Expect-Failure { Assert-SamplePools @($pools[0]) 'Standard_D4ds_v6' } 'Expected one system' -} -foreach ($field in @('count', 'vmSize', 'osSku', 'mode', 'enableAutoScaling', 'provisioningState')) { - Case "mismatched pool $field is rejected" { - $pools = New-Pools - $pools[1][$field] = switch ($field) { - 'count' { 2 } - 'enableAutoScaling' { $true } - default { 'incorrect' } - } - Expect-Failure { Assert-SamplePools $pools 'Standard_D4ds_v6' } 'does not match' +Stop-SampleAks Invoke-TestAksStop test group cluster -PollSeconds 0 +Assert-True ($script:stopCalls -eq 2) 'In-progress operation must be retried.' +foreach ($errorMessage in @('(AuthorizationFailed) Denied', + '(OperationNotAllowed) Unsupported cluster configuration', + 'There is an in-progress operation, but a different error code')) { + $script:stopCalls = 0 + $script:stopError = $errorMessage + Assert-Throws { Stop-SampleAks Invoke-TestAksStop test group cluster -PollSeconds 0 } 'AKS stop failed' + Assert-True ($script:stopCalls -eq 1) 'Nonretryable shutdown failure was retried.' +} +$script:stopCalls = 0 +$script:stopError = '(OperationNotAllowed) There is an in-progress PutExtensionAddonHandler.PUT operation.' +Assert-Throws { Stop-SampleAks Invoke-TestAksStop test group cluster -TimeoutSeconds 0 -PollSeconds 0 } 'remained blocked' +Assert-True ($script:stopCalls -eq 1) 'Expired shutdown deadline must stop retrying.' +Remove-Item Function:\Invoke-TestAksStop +$global:LASTEXITCODE = 0 + +$script:preflightCommands = [System.Collections.Generic.List[string]]::new() +$script:accountState = 'Enabled' +$script:featureState = 'Registered' +$script:familyLimit = 40 +$script:regionalLimit = 44 +$script:restricted = $false +function Invoke-CheckedCommand { + param([string]$Command, [string[]]$ArgumentList) + $call = $ArgumentList -join ' ' + $script:preflightCommands.Add($call) + if ($call -eq 'version -o json') { return '{"azure-cli":"2.92.0"}' } + if ($call -notlike '*--subscription test*') { throw 'Preflight lost subscription scope.' } + if ($call -like 'account show *') { return "{`"state`":`"$script:accountState`"}" } + if ($call -like 'aks get-versions *') { + return '{"values":[{"patchVersions":{"1.36.5":{}}}]}' + } + if ($call -like 'vm list-skus *') { + $restrictions = @(if ($script:restricted) { @{ type = 'Location' } }) + return (ConvertTo-Json -Depth 10 -InputObject @( + @{ name = 'Standard_D4ds_v6'; family = 'system'; restrictions = @(); capabilities = @(@{ name = 'vCPUs'; value = '4' }) } + @{ name = 'Standard_NCC40ads_H100_v5'; family = 'ncc'; restrictions = $restrictions; capabilities = @(@{ name = 'vCPUs'; value = '40' }) } + )) } + if ($call -like 'vm list-usage *') { + return (ConvertTo-Json -Depth 10 -InputObject @( + @{ name = @{ value = 'system' }; currentValue = 0; limit = 4 } + @{ name = @{ value = 'ncc' }; currentValue = 0; limit = $script:familyLimit } + @{ name = @{ value = 'cores' }; currentValue = 0; limit = $script:regionalLimit } + )) + } + if ($call -like 'feature show *') { return $script:featureState } + throw "Unexpected preflight command: $call" +} +$preflightDirectory = Join-Path ([System.IO.Path]::GetTempPath()) "aks-gpu-preflight-test-$([guid]::NewGuid())" +try { + Test-SamplePreflight az test westeurope '1.36.5' $preflightDirectory + $evidencePath = Join-Path $preflightDirectory 'preflight.json' + $evidence = Get-Content $evidencePath -Raw | ConvertFrom-Json + Assert-True ($evidence.Skus.Count -eq 2 -and $evidence.Quota.Count -eq 3) 'Preflight evidence must include both SKUs and all quotas.' + Assert-True ($evidence.Location -eq 'westeurope' -and $evidence.PreviewState -eq 'Registered') 'Preflight evidence lost location or registration.' + Assert-True (-not @($script:preflightCommands | Where-Object { $_ -notmatch '^(account show|version|aks get-versions|vm list-skus|vm list-usage|feature show) ' }).Count) 'Preflight modified Azure resources.' + Assert-Throws { Test-SamplePreflight az test westeurope '1.99.0' $preflightDirectory } 'not offered' + $script:accountState = 'Disabled' + Assert-Throws { Test-SamplePreflight az test westeurope '1.36.5' $preflightDirectory } 'not enabled' + $script:accountState = 'Enabled' + $script:familyLimit = 39 + Assert-Throws { Test-SamplePreflight az test westeurope '1.36.5' $preflightDirectory } 'Insufficient ncc quota' + $script:familyLimit = 40 + $script:regionalLimit = 43 + Assert-Throws { Test-SamplePreflight az test westeurope '1.36.5' $preflightDirectory } 'Insufficient regional' + $script:regionalLimit = 44 + $script:restricted = $true + Assert-Throws { Test-SamplePreflight az test westeurope '1.36.5' $preflightDirectory } 'regional subscription restriction' + $script:restricted = $false + $script:featureState = 'NotRegistered' + Assert-Throws { Test-SamplePreflight az test westeurope '1.36.5' $preflightDirectory } 'Arrange subscription registration' } -Case 'extra pools and automatic GPU drivers are rejected' { - $pools = New-Pools - Expect-Failure { Assert-SamplePools ($pools + @(@{ name = 'unexpected' })) 'Standard_D4ds_v6' } 'Expected one system' - $pools[1].gpuProfile.driver = 'Install' - Expect-Failure { Assert-SamplePools $pools 'Standard_D4ds_v6' } 'disable automatic drivers' -} -Case 'pool checks support case-sensitive Azure JSON dictionaries' { - $pools = @(New-Pools | ConvertTo-Json -Depth 10 | ConvertFrom-Json -AsHashtable) - Assert-SamplePools $pools 'Standard_D4ds_v6' -} -Case 'confidential scale set requires one instance, Secure Boot and vTPM' { - $vm = @{ - sku = @{ name = 'Standard_NCC40ads_H100_v5'; capacity = 1 } - virtualMachineProfile = @{ securityProfile = @{ - securityType = 'ConfidentialVM' - uefiSettings = @{ secureBootEnabled = $true; vTpmEnabled = $true } - } } +finally { + if (Test-Path $preflightDirectory) { + Remove-Item -LiteralPath (Join-Path $preflightDirectory 'preflight.json') -Force + Remove-Item -LiteralPath $preflightDirectory } - Assert-ConfidentialGpuScaleSet @($vm) - $vm.virtualMachineProfile.securityProfile.uefiSettings.secureBootEnabled = $false - Expect-Failure { Assert-ConfidentialGpuScaleSet @($vm) } 'Secure Boot' - $vm.virtualMachineProfile.securityProfile.uefiSettings.secureBootEnabled = $true - $vm.virtualMachineProfile.securityProfile.uefiSettings.vTpmEnabled = $false - Expect-Failure { Assert-ConfidentialGpuScaleSet @($vm) } 'vTPM' - $vm.virtualMachineProfile.securityProfile.uefiSettings.vTpmEnabled = $true - $vm.sku.capacity = 2 - Expect-Failure { Assert-ConfidentialGpuScaleSet @($vm) } 'one ConfidentialVM' -} -Case 'native failure propagates' { - $pwsh = @(Get-Command pwsh -CommandType Application)[0].Source - Expect-Failure { - Invoke-CheckedCommand $pwsh @('-NoProfile', '-Command', 'exit 7') - } 'exit code 7' + Remove-Item Function:\Invoke-CheckedCommand } -# Replace the native boundary only for the node-discovery cases. -function Invoke-CheckedCommand { $state.nodes | ConvertTo-Json -Depth 10 } -Case 'node lookup rejects zero and multiple workers' { - foreach ($items in @(@(), @(@{ name = 'one' }, @{ name = 'two' }))) { - $state.nodes = @{ items = $items } - Expect-Failure { Get-GpuNode -KubeConfig 'offline' } 'Expected exactly one node' +$root = Split-Path $PSScriptRoot +foreach ($file in Get-ChildItem $root -Filter '*.ps1' -Recurse | Where-Object FullName -NotMatch '[\\/]\.local[\\/]') { + $tokens = $null + $errors = $null + [System.Management.Automation.Language.Parser]::ParseFile($file.FullName, [ref]$tokens, [ref]$errors) | Out-Null + Assert-True ($errors.Count -eq 0) "Syntax error in $($file.Name): $errors" + if ($file.DirectoryName -eq $root) { + Assert-True ((Get-Content $file.FullName -Raw) -notmatch 'aks-gpu-worker-temp|\.\.\\aks-gpu-worker\\') "Stale sibling dependency in $($file.Name)." } } -Case 'node lookup returns exactly one worker' { - $state.nodes = @{ items = @(@{ name = 'one' }) } - Assert ((Get-GpuNode -KubeConfig 'offline').name -eq 'one') 'Wrong node returned.' -} -Write-Host "$script:passed tests passed." +$source = Get-Content (Join-Path $root 'Deploy.ps1') -Raw +Assert-True ($source.IndexOf('BlockedMissingGuideDriver') -lt $source.IndexOf("helm @('install'")) 'Missing-driver gate must precede Helm installation.' +Assert-True ($source -match "-Branches @\('580'\)") 'Only the guide driver branch is allowed.' +Assert-True ($source -notmatch 'Install-SignedDriver|driver.enabled=false|apt-get.*linux') 'An alternative host-driver installation was introduced.' +$readme = Get-Content (Join-Path $root 'README.md') -Raw +Assert-True ($readme -notmatch 'GPU-[0-9a-f]{8}-|aks-gpu-worker-temp|aks-gpunp-\d+|aks-sysnp-\d+') 'README contains deployment identifiers or stale names.' +Assert-True ($readme.Contains('Version numbers and exact output may change')) 'Attestation version caveat missing.' +Assert-True ($readme.Contains('GPU Attestation is Successful.') -and $readme.Contains('[Attestation token omitted]')) 'Redacted attestation example missing.' +Write-Host "Passed $script:passed assertions." From 3768050708f438c76b21d24ea72791db6e659e8b Mon Sep 17 00:00:00 2001 From: Simon Gallagher Date: Thu, 12 Nov 2026 16:26:08 +0000 Subject: [PATCH 3/3] Signpost the confidential H100 AKS worker sample Add discovery links and clarify GPU validation, support boundaries, and cleanup in the repository and AKS guides. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 3 +++ aks-samples/README.md | 32 +++++++++++++++++++++++++++----- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3f2d3d6..3c0fcc9 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ steps. Not every sample requires every tool listed below. | Send and receive messages through a confidential managed service | [Service Bus Premium](service-bus-examples/README.md) | New confidential-enabled namespace, passwordless messaging, and exact message read-back checks. | | Explore attestation in a container | [Visual attestation v2 on ACI](aci-samples/visual-attestation-demo-v2/README.md) | Direct Microsoft Azure Attestation calls and a Confidential-versus-Standard comparison. | | Deploy a confidential VM or H100 confidential GPU VM | [VM samples](vm-samples/README.md) | AMD SEV-SNP, Intel TDX, and H100 paths with sample-specific attestation and prerequisites. | +| Run a confidential H100 GPU workload on Kubernetes | [AKS confidential H100 worker](aks-samples/aks-gpu-worker/README.md) | Guide-only GPU Operator automation on the latest offered AKS node image, TensorFlow GPU validation, and strict GPU attestation before and after the workload. Compute stops automatically; delete the resource group afterward. | | Keep partner datasets separate while computing shared results | [Federated multi-party analytics](multi-party-samples/advanced-app-federated/README.md) | Each partner processes its own data in its own TEE and returns aggregates. | | Explore confidential video processing | [Automotive machine vision](automotive-machine-vision/README.md) | Confidential ACI video redaction with HTTPS, CCE policy, and an attestation gate. | @@ -80,6 +81,7 @@ must not be used to send sensitive data. | Addition | Description | |---|---| +| **[Confidential H100 AKS worker](aks-samples/aks-gpu-worker/README.md)** | Deploy a single confidential H100 worker using the Azure onboarding guide's GPU Operator path and latest offered AKS image. Includes strict GPU attestation, a GPU-verified TensorFlow workload, redacted example output, and automatic compute shutdown. Revalidated in West Europe; successful execution on a newer image is not vendor-support certification. | | **[Confidential Container Apps hello world](azure-container-apps-sample/container-apps-acc/README.md#start-here)** | Deploy Microsoft's HTML quickstart using ARM and PowerShell, explicitly bound to a dedicated DC4 confidential workload profile in UAE North. | | **[Native Azure Functions on confidential Container Apps](azure-container-apps-sample/functions-on-azure-container-apps/README.md#start-here)** | Tested Node.js 24 `GET /api/hello` JSON endpoint with fresh ACR builds, immutable image deployment, managed identity, and private Blob host storage. Includes local Azurite testing, host-health checks, and portal/API access instructions. | | **[Explicit confidential-compute validation](azure-container-apps-sample/README.md#validate-confidential-compute-placement)** | Read live app/environment resources, match the selected profile to DC4, and check active/healthy revisions. Includes expected output and portal cross-checks, with a clear distinction from cryptographic attestation. | @@ -244,6 +246,7 @@ Confidential Virtual Machine (CVM) deployment scripts: ### [AKS Samples](/aks-samples/README.md) Azure Kubernetes Service with AMD SEV-SNP confidential computing: +- **[Confidential H100 GPU worker](aks-samples/aks-gpu-worker/README.md)** - Guide-only deployment on the latest offered AKS node image, strict GPU attestation before and after a TensorFlow GPU workload, and automatic compute shutdown. See the sample for quota, support boundaries, and resource deletion. - **BuildRandomAKS.ps1** - AKS cluster with Customer Managed Keys and confidential node pools - **[AKS Virtual Nodes on Confidential AKS](/aks-samples/virtual%20nodes/README.md)** - End-to-end sample that deploys AKS with virtual nodes backed by ACI, validates hello-world scheduling on the virtual node, and links to a confidential ACI follow-on manifest. diff --git a/aks-samples/README.md b/aks-samples/README.md index b709d10..b27b178 100644 --- a/aks-samples/README.md +++ b/aks-samples/README.md @@ -1,17 +1,39 @@ # Azure Kubernetes Service (AKS) Confidential Computing Samples -**Last Updated:** February 2026 +**Last Updated:** November 2026 ## Overview -Deploy Azure Kubernetes Service clusters with AMD SEV-SNP confidential node pools and Customer Managed Keys (CMK). +Deploy Azure Kubernetes Service clusters with confidential CPU node pools or a +confidential NVIDIA H100 GPU worker. The CPU sample includes Customer Managed +Keys (CMK); the GPU sample follows the Azure confidential H100 onboarding guide. -For guide-only automation of a single confidential H100 GPU worker using the -latest offered AKS node image, GPU workload validation and strict GPU attestation, -see [AKS confidential H100 worker](aks-gpu-worker/README.md). +## Choose a sample + +| Sample | What it covers | +|---|---| +| [Confidential H100 GPU worker](aks-gpu-worker/README.md) | Single H100 worker on the latest offered AKS node image, guide-only GPU Operator installation, TensorFlow execution verified on GPU, and strict GPU attestation before and after the workload. | +| [Confidential CPU node pools with CMK](#buildrandomaksps1) | AMD SEV-SNP confidential node pools, Key Vault, and customer-managed encryption keys. | +| [AKS virtual nodes](virtual%20nodes/README.md) | Virtual nodes backed by ACI, hello-world scheduling validation, and a confidential ACI follow-on manifest. | + +### Start with the confidential H100 worker + +See the [H100 worker guide](aks-gpu-worker/README.md) for prerequisites, +preflight and deployment commands, redacted attestation output, and cleanup. +The sample was revalidated in West Europe using an Azure-FDE kernel at least +`6.8.0-1071-azure-fde`. Successful execution on a newer AKS image is not +Microsoft/NVIDIA support certification; the guide explains that boundary. +GPU attestation is not a separate CPU/whole-VM attestation claim. + +The worker sample stops newly created compute automatically, but does **not** +delete resources. Delete its resource group when finished to remove retained +disks, IPs, and other potentially billable resources. ## Architecture +The following architecture describes the confidential CPU node-pool/CMK sample. +For the H100 configuration, use the [GPU worker guide](aks-gpu-worker/README.md). + ``` ┌─────────────────────────────────────────────────────────────────┐ │ AKS Cluster with Confidential Nodes │