From 4473ae6fd4166ac21dc07bea38b1f7a54d70b1f7 Mon Sep 17 00:00:00 2001 From: simontesar Date: Wed, 16 Sep 2026 05:55:07 +0200 Subject: [PATCH 1/5] WIP: extra poweroff test Signed-off-by: simontesar --- Makefile | 4 +- README.md | 1 + tests/12-force-power-off/chainsaw-test.yaml | 73 +++++++++++++++++++++ 3 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 tests/12-force-power-off/chainsaw-test.yaml diff --git a/Makefile b/Makefile index 6dac2ce..f80860b 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,8 @@ TESTS := \ 07-biossettings-reboot \ 08-bmcsettings \ 09-persistent-boot-order \ - 10-bmcversion + 10-bmcversion \ + 12-force-power-off .PHONY: help help: ## Show available targets @@ -39,6 +40,7 @@ help: ## Show available targets @echo " test/08-bmcsettings (BMCSettings, Manager attribute; needs a Dell BMC)" @echo " test/09-persistent-boot-order (persistent boot order)" @echo " test/10-bmcversion (BMCVersion, version read without upgrade)" + @echo " test/12-force-power-off (forced power off after a graceful shutdown is ignored)" @echo "" .PHONY: require-values diff --git a/README.md b/README.md index 26bc323..b177ef0 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ To run tests against your own BMC, copy these values into a new file: bmcIP: "172.16.100.11" bmcPort: 443 bmcScheme: https +gracefulPowerOffWindow: 10s username: admin password: password model: "Standard PC (Q35 + ICH9, 2009)" diff --git a/tests/12-force-power-off/chainsaw-test.yaml b/tests/12-force-power-off/chainsaw-test.yaml new file mode 100644 index 0000000..a6122cc --- /dev/null +++ b/tests/12-force-power-off/chainsaw-test.yaml @@ -0,0 +1,73 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: 12-force-power-off +spec: + concurrent: false + bindings: + - name: bmcName + value: (join('-', ['compatibility', $namespace])) + - name: serverName + value: (join('-', ['compatibility', $namespace, 'system-0'])) + steps: + - name: register-bmc + try: + - apply: + file: ../shared/bmcsecret.yaml + - apply: + file: ../shared/bmc.yaml + - name: assert-server-available + try: + - assert: + resource: + apiVersion: metal.ironcore.dev/v1alpha1 + kind: Server + metadata: + name: ($serverName) + status: + state: Available + powerState: "Off" + - name: power-on-out-of-band + try: + - patch: + resource: + apiVersion: metal.ironcore.dev/v1alpha1 + kind: Server + metadata: + name: ($serverName) + annotations: + metal.ironcore.dev/operation: force-on-server + - assert: + resource: + apiVersion: metal.ironcore.dev/v1alpha1 + kind: Server + metadata: + name: ($serverName) + (annotations."metal.ironcore.dev/operation" || 'absent'): 'absent' + status: + powerState: "On" + - name: assert-graceful-did-not-take-effect + try: + - script: + timeout: 2m + env: + content: sleep 10s + - assert: + resource: + apiVersion: metal.ironcore.dev/v1alpha1 + kind: Server + metadata: + name: ($serverName) + status: + powerState: "On" + - name: assert-server-force-powered-off + try: + - assert: + resource: + apiVersion: metal.ironcore.dev/v1alpha1 + kind: Server + metadata: + name: ($serverName) + status: + state: Available + powerState: "Off" From 0786a02a70b9c0dbaf3fbe61ffe87f0674f8a413 Mon Sep 17 00:00:00 2001 From: simontesar Date: Fri, 16 Oct 2026 11:06:41 +0300 Subject: [PATCH 2/5] feat: 12: verify forced power-off --- tests/12-force-power-off/chainsaw-test.yaml | 106 +++++++++++++++---- tests/12-force-power-off/force-off-job.yaml | 67 ++++++++++++ tests/12-force-power-off/ignitionsecret.yaml | 10 ++ tests/12-force-power-off/rbac.yaml | 30 ++++++ tests/12-force-power-off/serverclaim.yaml | 11 ++ 5 files changed, 203 insertions(+), 21 deletions(-) create mode 100644 tests/12-force-power-off/force-off-job.yaml create mode 100644 tests/12-force-power-off/ignitionsecret.yaml create mode 100644 tests/12-force-power-off/rbac.yaml create mode 100644 tests/12-force-power-off/serverclaim.yaml diff --git a/tests/12-force-power-off/chainsaw-test.yaml b/tests/12-force-power-off/chainsaw-test.yaml index a6122cc..096e29c 100644 --- a/tests/12-force-power-off/chainsaw-test.yaml +++ b/tests/12-force-power-off/chainsaw-test.yaml @@ -1,3 +1,6 @@ +# Creates a scenario where the metal-operator issues a "ForceOff" reset by +# booting an image that ignores ACPI and verifying that the server still powered +# off after the graceful window. apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: @@ -5,10 +8,17 @@ metadata: spec: concurrent: false bindings: - - name: bmcName - value: (join('-', ['compatibility', $namespace])) - name: serverName value: (join('-', ['compatibility', $namespace, 'system-0'])) + # Time to sleep before assuming the boot has reached the OS + - name: osBootWait + value: ($values."12-force-power-off".osBootWait || '2m') + # The metal-operator's --power-polling-timeout + - name: powerPollingTimeout + value: ($values."12-force-power-off".powerPollingTimeout || `15`) + # Seconds beyond the power polling timeout allowed for the force-off to take effect + - name: forceOffDelay + value: ($values."12-force-power-off".forceOffDelay || `90`) steps: - name: register-bmc try: @@ -16,51 +26,99 @@ spec: file: ../shared/bmcsecret.yaml - apply: file: ../shared/bmc.yaml - - name: assert-server-available + cleanup: + - sleep: + duration: 10s + - delete: + timeout: 2m + ref: + apiVersion: metal.ironcore.dev/v1alpha1 + kind: Server + labels: + metal-operator-test-framework/namespace: ($namespace) + - name: assert-bmc-enabled try: - assert: resource: apiVersion: metal.ironcore.dev/v1alpha1 - kind: Server + kind: BMC metadata: - name: ($serverName) + name: (join('-', ['compatibility', $namespace])) status: - state: Available - powerState: "Off" - - name: power-on-out-of-band + state: Enabled + - name: assert-server-available try: - - patch: + - assert: resource: apiVersion: metal.ironcore.dev/v1alpha1 kind: Server metadata: name: ($serverName) - annotations: - metal.ironcore.dev/operation: force-on-server + status: + state: Available + powerState: "Off" + # Boots an image ignoring ACPI to avoid graceful shutdown + - name: claim-server + try: + - apply: + file: ignitionsecret.yaml + - apply: + file: serverclaim.yaml - assert: resource: apiVersion: metal.ironcore.dev/v1alpha1 kind: Server metadata: name: ($serverName) - (annotations."metal.ironcore.dev/operation" || 'absent'): 'absent' status: + state: Reserved powerState: "On" - - name: assert-graceful-did-not-take-effect + # Wait for the OS to actually boot, otherwise firmware handles ACPI and might + # respect graceful shutdown + - name: wait-for-os try: - script: - timeout: 2m + timeout: 10m env: - content: sleep 10s + - name: OS_BOOT_WAIT + value: ($osBootWait) + content: sleep "$OS_BOOT_WAIT" + # Create a Job pataching the Server's power state to off and asserts via + # Redfish API that the power state chagnged in a time window that is outside + # the window for a graceful shutdown, i.e. the force-off took effect + - name: force-power-off + bindings: + - name: systemURI + value: (x_k8s_get($client, 'metal.ironcore.dev/v1alpha1', 'Server', '', $serverName).spec.systemURI) + try: + - apply: + file: rbac.yaml + - apply: + file: force-off-job.yaml + # Require the Job to finish either way but fail fast when it failed for + # internal reasons - assert: + timeout: 10m resource: - apiVersion: metal.ironcore.dev/v1alpha1 - kind: Server + apiVersion: batch/v1 + kind: Job metadata: - name: ($serverName) + name: force-off-check status: - powerState: "On" - - name: assert-server-force-powered-off + (conditions[?type == 'Complete' || type == 'Failed'] | [0].status): "True" + - assert: + timeout: 5s + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: force-off-check + status: + (conditions[?type == 'Complete'] | [0].status): "True" + finally: + - podLogs: + selector: job-name=force-off-check + - name: assert-server-off try: - assert: resource: @@ -69,5 +127,11 @@ spec: metadata: name: ($serverName) status: - state: Available powerState: "Off" + - name: release-server + try: + - delete: + ref: + apiVersion: metal.ironcore.dev/v1alpha1 + kind: ServerClaim + name: (join('-', ['compatibility', $namespace])) diff --git a/tests/12-force-power-off/force-off-job.yaml b/tests/12-force-power-off/force-off-job.yaml new file mode 100644 index 0000000..43f0f4c --- /dev/null +++ b/tests/12-force-power-off/force-off-job.yaml @@ -0,0 +1,67 @@ +# Requests the power off by patching the ServerClaim and times the transition +# to Off directly on the BMC's redfish API. The metal-operator first issues a +# GracefulShutdown, waits for --power-polling-timeout and only then issues a +# ForceOff. As the booted image ignores the power button, the system turning +# Off no earlier than the polling timeout proves that ForceOff took effect. +apiVersion: batch/v1 +kind: Job +metadata: + name: force-off-check + namespace: ($namespace) +spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + serviceAccountName: force-off-check + containers: + - name: check + image: curlimages/curl:8.11.1 + env: + - name: URL + value: (join('', [$values.bmcScheme, '://', $values.bmcIP, ':', to_string($values.bmcPort), $systemURI])) + - name: CREDENTIALS + value: (join(':', [$values.username, $values.password])) + - name: NS + value: ($namespace) + - name: CLAIM + value: (join('-', ['compatibility', $namespace])) + - name: POLL_TIMEOUT + value: (to_string($powerPollingTimeout)) + - name: SLACK + value: (to_string($forceOffDelay)) + command: ["/bin/sh", "-c"] + args: + - | + set -eu + SA=/var/run/secrets/kubernetes.io/serviceaccount + MIN=$((POLL_TIMEOUT - 1)) + MAX=$((POLL_TIMEOUT + SLACK)) + + state() { + curl -fsS -k -u "$CREDENTIALS" "$URL" | sed -n 's/.*"PowerState" *: *"\([A-Za-z]*\)".*/\1/p' + } + + s=$(state) + [ "$s" = On ] || { echo "precondition failed: expected PowerState On, got '$s'"; exit 1; } + + curl -fsS -o /dev/null -X PATCH \ + --cacert "$SA/ca.crt" \ + -H "Authorization: Bearer $(cat "$SA/token")" \ + -H 'Content-Type: application/merge-patch+json' \ + -d '{"spec":{"power":"Off"}}' \ + "https://kubernetes.default.svc/apis/metal.ironcore.dev/v1alpha1/namespaces/$NS/serverclaims/$CLAIM" + start=$(date +%s) + echo "requested power off, expecting Off between ${MIN}s and ${MAX}s" + + while :; do + s=$(state || echo unknown) + t=$(( $(date +%s) - start )) + echo "t=${t}s PowerState=$s" + [ "$s" = Off ] && break + [ "$t" -le "$MAX" ] || { echo "FAIL: still $s after ${MAX}s, ForceOff not issued or not honoured"; exit 1; } + sleep 2 + done + + [ "$t" -ge "$MIN" ] || { echo "FAIL: Off after ${t}s (< ${MIN}s), GracefulShutdown took effect, the booted image does not ignore the power button"; exit 1; } + echo "PASS: Off after ${t}s, only reachable via ForceOff" diff --git a/tests/12-force-power-off/ignitionsecret.yaml b/tests/12-force-power-off/ignitionsecret.yaml new file mode 100644 index 0000000..fc3316b --- /dev/null +++ b/tests/12-force-power-off/ignitionsecret.yaml @@ -0,0 +1,10 @@ +# Creates an empty file that serves as a flag to the metalprobe-image to ignore +# ACPI. See metalprobe-image/cmd/metalprobe-launcher/main.go. +apiVersion: v1 +kind: Secret +metadata: + name: (join('-', ['compatibility', $namespace, 'ignition'])) +type: Opaque +stringData: + ignition: | + {"ignition":{"version":"3.4.0"},"storage":{"files":[{"path":"/metalprobe/ignore-power-button","contents":{"source":"data:,"}}]}} diff --git a/tests/12-force-power-off/rbac.yaml b/tests/12-force-power-off/rbac.yaml new file mode 100644 index 0000000..475efa8 --- /dev/null +++ b/tests/12-force-power-off/rbac.yaml @@ -0,0 +1,30 @@ +# Allows the force-off-check Job to patch the ServerClaim +apiVersion: v1 +kind: ServiceAccount +metadata: + name: force-off-check + namespace: ($namespace) +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: force-off-check + namespace: ($namespace) +rules: + - apiGroups: ["metal.ironcore.dev"] + resources: ["serverclaims"] + verbs: ["get", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: force-off-check + namespace: ($namespace) +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: force-off-check +subjects: + - kind: ServiceAccount + name: force-off-check + namespace: ($namespace) diff --git a/tests/12-force-power-off/serverclaim.yaml b/tests/12-force-power-off/serverclaim.yaml new file mode 100644 index 0000000..cf1df0f --- /dev/null +++ b/tests/12-force-power-off/serverclaim.yaml @@ -0,0 +1,11 @@ +apiVersion: metal.ironcore.dev/v1alpha1 +kind: ServerClaim +metadata: + name: (join('-', ['compatibility', $namespace])) +spec: + power: "On" + serverRef: + name: (join('-', ['compatibility', $namespace, 'system-0'])) + ignitionSecretRef: + name: (join('-', ['compatibility', $namespace, 'ignition'])) + image: ghcr.io/simontesar/metal-lab/metalprobe:force-off-dev From 51d3826bdac2db3ef93eee73ca41a2b11bc7f798 Mon Sep 17 00:00:00 2001 From: simontesar Date: Fri, 16 Oct 2026 12:07:27 +0300 Subject: [PATCH 3/5] feat: 12: use main metalprobe-image --- tests/12-force-power-off/serverclaim.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/12-force-power-off/serverclaim.yaml b/tests/12-force-power-off/serverclaim.yaml index cf1df0f..db78851 100644 --- a/tests/12-force-power-off/serverclaim.yaml +++ b/tests/12-force-power-off/serverclaim.yaml @@ -8,4 +8,4 @@ spec: name: (join('-', ['compatibility', $namespace, 'system-0'])) ignitionSecretRef: name: (join('-', ['compatibility', $namespace, 'ignition'])) - image: ghcr.io/simontesar/metal-lab/metalprobe:force-off-dev + image: ghcr.io/simontesar/metal-lab/metalprobe:main From 249accb707bb77cb21b6c4fe32d1b037e556611e Mon Sep 17 00:00:00 2001 From: simontesar Date: Fri, 16 Oct 2026 12:07:52 +0300 Subject: [PATCH 4/5] doc: 12: update README --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 168624c..dbd1bb0 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,6 @@ To run tests against your own BMC, copy these values into a new file: bmcIP: "172.16.100.11" bmcPort: 443 bmcScheme: https -gracefulPowerOffWindow: 10s username: admin password: password firmwareVersion: "1.0.0" @@ -50,6 +49,10 @@ biosVersion: "1.0.0" key: "EmailAlert.1.Address" value: "alerts@example.com" baselineValue: "compat-base@example.com" +12-force-power-off: # optionally override the defaults + osBootWait: 120 # time to sleep before assuming the boot has reached the OS + powerPollingTimeout: 15 # must equal the metal-operator's --power-polling-timeout to the the Job + forceOffDelay: 90 # seconds allowed beyond powerPollingTimeout for the ForceOff to take effect 13-event-subscriptions: testMessageId: "TestEventId" # a MessageId your BMC accepts in SubmitTestEvent, e.g. "SYS1000" on Dell iDRAC testSeverity: "OK" From dbef1a66de78c4b2b26b18b7ffd1310050fedac1 Mon Sep 17 00:00:00 2001 From: simontesar Date: Fri, 16 Oct 2026 12:08:10 +0300 Subject: [PATCH 5/5] feat: 12: update coverage --- coverage.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coverage.md b/coverage.md index 92f9c1c..6c676da 100644 --- a/coverage.md +++ b/coverage.md @@ -32,7 +32,7 @@ These rows/capabilities include the following placeholders in their `test` field | `tests/09-persistent-boot-order` | `BMC Registration`, `Discovery`, `boot.order-get`, `boot.order-set`, `power.on`, `power.off-graceful` | | `tests/10-bmcversion` | `BMC Registration`, `Discovery`, `bmc.version` | | `tests/11-bmcuser` | `BMC Registration`, `account.service`, `account.list`, `account.create-update`, `account.delete` | -| `tests/12-force-power-off` | `BMC Registration`, `Discovery`, `power.off-graceful`, `power.off-force`, `power.wait-state` | +| `tests/12-force-power-off` | `BMC Registration`, `Discovery`, `power.on`, `power.off-graceful`, `power.off-force`, `power.wait-state`, `boot.override-pxe` | | `tests/13-event-subscriptions` | `BMC Registration`, `events.subscribe`, `events.unsubscribe`, `events.list`, `events.test-event`, `events.delivery` | ## Coverage by capability @@ -41,7 +41,7 @@ These rows/capabilities include the following placeholders in their `test` field |------------------------------|-------------------------------------------------------------------------------|:--------:| | `power.on` | `Discovery` | yes | | `power.off-graceful` | `Discovery` | yes | -| `power.off-force` | `tests/03-power-annotation` See **[0]** | no | +| `power.off-force` | `tests/12-force-power-off`, `tests/03-power-annotation` See **[0]** | yes | | `power.reset` | `tests/03-power-annotation` | no | | `power.wait-state` | `Discovery` | yes | | `boot.override-pxe` | `Discovery` | yes |