Test forced poweroff

Makefile

@@ -18,6 +18,7 @@ TESTS := \
 	08-bmcsettings \
 	09-persistent-boot-order \
 	10-bmcversion \
+	12-force-power-off \
 	11-bmcuser \
 	13-event-subscriptions
 
@@ -41,6 +42,7 @@ help: ## Show available targets
 	@echo "    test/08-bmcsettings              (BMCSettings, Manager attribute)"
 	@echo "    test/09-persistent-boot-order    (persistent boot order)"
 	@echo "    test/10-bmcversion               (BMCVersion, version read without upgrade)"
+	@echo "    test/12-force-power-off          (forced power off after a graceful shutdown is ignored)"
 	@echo "    test/11-bmcuser                  (BMCUser, BMC account create/rotate/delete)"
 	@echo "    test/13-event-subscriptions      (Redfish event subscriptions via metal-maintenance-operator telemetry)"
 	@echo ""

README.md

@@ -49,6 +49,10 @@ biosVersion: "1.0.0"
   key: "EmailAlert.1.Address"
   value: "alerts@example.com"
   baselineValue: "compat-base@example.com"
+12-force-power-off: # optionally override the defaults
+  osBootWait: 120 # time to sleep before assuming the boot has reached the OS
+  powerPollingTimeout: 15 #  must equal the metal-operator's --power-polling-timeout to the the Job
+  forceOffDelay: 90 # seconds allowed beyond powerPollingTimeout for the ForceOff to take effect
 13-event-subscriptions:
   testMessageId: "TestEventId" # a MessageId your BMC accepts in SubmitTestEvent, e.g. "SYS1000" on Dell iDRAC
   testSeverity: "OK"

coverage.md

@@ -32,7 +32,7 @@ These rows/capabilities include the following placeholders in their `test` field
 | `tests/09-persistent-boot-order` | `BMC Registration`, `Discovery`, `boot.order-get`, `boot.order-set`, `power.on`, `power.off-graceful`                                          |
 | `tests/10-bmcversion`            | `BMC Registration`, `Discovery`, `bmc.version`                                                                                                 |
 | `tests/11-bmcuser`               | `BMC Registration`, `account.service`, `account.list`, `account.create-update`, `account.delete`                                               |
-| `tests/12-force-power-off`       | `BMC Registration`, `Discovery`, `power.off-graceful`, `power.off-force`, `power.wait-state`                                                   |
+| `tests/12-force-power-off`       | `BMC Registration`, `Discovery`, `power.on`, `power.off-graceful`, `power.off-force`, `power.wait-state`, `boot.override-pxe`                  |
 | `tests/13-event-subscriptions`   | `BMC Registration`, `events.subscribe`, `events.unsubscribe`, `events.list`, `events.test-event`, `events.delivery`                             |
 
 ## Coverage by capability
@@ -41,7 +41,7 @@ These rows/capabilities include the following placeholders in their `test` field
 |------------------------------|-------------------------------------------------------------------------------|:--------:|
 | `power.on`                   | `Discovery`                                                                   |   yes    |
 | `power.off-graceful`         | `Discovery`                                                                   |   yes    |
-| `power.off-force`            | `tests/03-power-annotation` See **[0]**                                       |    no    |
+| `power.off-force`            | `tests/12-force-power-off`, `tests/03-power-annotation` See **[0]**           |   yes    |
 | `power.reset`                | `tests/03-power-annotation`                                                   |    no    |
 | `power.wait-state`           | `Discovery`                                                                   |   yes    |
 | `boot.override-pxe`          | `Discovery`                                                                   |   yes    |

tests/12-force-power-off/chainsaw-test.yaml

@@ -0,0 +1,137 @@
+# Creates a scenario where the metal-operator issues a "ForceOff" reset by
+# booting an image that ignores ACPI and verifying that the server still powered
+# off after the graceful window.
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: 12-force-power-off
+spec:
+  concurrent: false
+  bindings:
+    - name: serverName
+      value: (join('-', ['compatibility', $namespace, 'system-0']))
+    # Time to sleep before assuming the boot has reached the OS
+    - name: osBootWait
+      value: ($values."12-force-power-off".osBootWait || '2m')
+    # The metal-operator's --power-polling-timeout
+    - name: powerPollingTimeout
+      value: ($values."12-force-power-off".powerPollingTimeout || `15`)
+    # Seconds beyond the power polling timeout allowed for the force-off to take effect
+    - name: forceOffDelay
+      value: ($values."12-force-power-off".forceOffDelay || `90`)
+  steps:
+    - name: register-bmc
+      try:
+        - apply:
+            file: ../shared/bmcsecret.yaml
+        - apply:
+            file: ../shared/bmc.yaml
+      cleanup:
+        - sleep:
+            duration: 10s
+        - delete:
+            timeout: 2m
+            ref:
+              apiVersion: metal.ironcore.dev/v1alpha1
+              kind: Server
+              labels:
+                metal-operator-test-framework/namespace: ($namespace)
+    - name: assert-bmc-enabled
+      try:
+        - assert:
+            resource:
+              apiVersion: metal.ironcore.dev/v1alpha1
+              kind: BMC
+              metadata:
+                name: (join('-', ['compatibility', $namespace]))
+              status:
+                state: Enabled
+    - name: assert-server-available
+      try:
+        - assert:
+            resource:
+              apiVersion: metal.ironcore.dev/v1alpha1
+              kind: Server
+              metadata:
+                name: ($serverName)
+              status:
+                state: Available
+                powerState: "Off"
+    # Boots an image ignoring ACPI to avoid graceful shutdown
+    - name: claim-server
+      try:
+        - apply:
+            file: ignitionsecret.yaml
+        - apply:
+            file: serverclaim.yaml
+        - assert:
+            resource:
+              apiVersion: metal.ironcore.dev/v1alpha1
+              kind: Server
+              metadata:
+                name: ($serverName)
+              status:
+                state: Reserved
+                powerState: "On"
+    # Wait for the OS to actually boot, otherwise firmware handles ACPI and might
+    # respect graceful shutdown
+    - name: wait-for-os
+      try:
+        - script:
+            timeout: 10m
+            env:
+              - name: OS_BOOT_WAIT
+                value: ($osBootWait)
+            content: sleep "$OS_BOOT_WAIT"
+    # Create a Job pataching the Server's power state to off and asserts via
+    # Redfish API that the power state chagnged in a time window that is outside
+    # the window for a graceful shutdown, i.e. the force-off took effect
+    - name: force-power-off
+      bindings:
+        - name: systemURI
+          value: (x_k8s_get($client, 'metal.ironcore.dev/v1alpha1', 'Server', '', $serverName).spec.systemURI)
+      try:
+        - apply:
+            file: rbac.yaml
+        - apply:
+            file: force-off-job.yaml
+        # Require the Job to finish either way but fail fast when it failed for
+        # internal reasons
+        - assert:
+            timeout: 10m
+            resource:
+              apiVersion: batch/v1
+              kind: Job
+              metadata:
+                name: force-off-check
+              status:
+                (conditions[?type == 'Complete' || type == 'Failed'] | [0].status): "True"
+        - assert:
+            timeout: 5s
+            resource:
+              apiVersion: batch/v1
+              kind: Job
+              metadata:
+                name: force-off-check
+              status:
+                (conditions[?type == 'Complete'] | [0].status): "True"
+      finally:
+        - podLogs:
+            selector: job-name=force-off-check
+    - name: assert-server-off
+      try:
+        - assert:
+            resource:
+              apiVersion: metal.ironcore.dev/v1alpha1
+              kind: Server
+              metadata:
+                name: ($serverName)
+              status:
+                powerState: "Off"
+    - name: release-server
+      try:
+        - delete:
+            ref:
+              apiVersion: metal.ironcore.dev/v1alpha1
+              kind: ServerClaim
+              name: (join('-', ['compatibility', $namespace]))

tests/12-force-power-off/force-off-job.yaml

@@ -0,0 +1,67 @@
+# Requests the power off by patching the ServerClaim and times the transition
+# to Off directly on the BMC's redfish API. The metal-operator first issues a
+# GracefulShutdown, waits for --power-polling-timeout and only then issues a
+# ForceOff. As the booted image ignores the power button, the system turning
+# Off no earlier than the polling timeout proves that ForceOff took effect.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: force-off-check
+  namespace: ($namespace)
+spec:
+  backoffLimit: 0
+  template:
+    spec:
+      restartPolicy: Never
+      serviceAccountName: force-off-check
+      containers:
+        - name: check
+          image: curlimages/curl:8.11.1
+          env:
+            - name: URL
+              value: (join('', [$values.bmcScheme, '://', $values.bmcIP, ':', to_string($values.bmcPort), $systemURI]))
+            - name: CREDENTIALS
+              value: (join(':', [$values.username, $values.password]))
+            - name: NS
+              value: ($namespace)
+            - name: CLAIM
+              value: (join('-', ['compatibility', $namespace]))
+            - name: POLL_TIMEOUT
+              value: (to_string($powerPollingTimeout))
+            - name: SLACK
+              value: (to_string($forceOffDelay))
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              set -eu
+              SA=/var/run/secrets/kubernetes.io/serviceaccount
+              MIN=$((POLL_TIMEOUT - 1))
+              MAX=$((POLL_TIMEOUT + SLACK))
+
+              state() {
+                curl -fsS -k -u "$CREDENTIALS" "$URL" | sed -n 's/.*"PowerState" *: *"\([A-Za-z]*\)".*/\1/p'
+              }
+
+              s=$(state)
+              [ "$s" = On ] || { echo "precondition failed: expected PowerState On, got '$s'"; exit 1; }
+
+              curl -fsS -o /dev/null -X PATCH \
+                --cacert "$SA/ca.crt" \
+                -H "Authorization: Bearer $(cat "$SA/token")" \
+                -H 'Content-Type: application/merge-patch+json' \
+                -d '{"spec":{"power":"Off"}}' \
+                "https://kubernetes.default.svc/apis/metal.ironcore.dev/v1alpha1/namespaces/$NS/serverclaims/$CLAIM"
+              start=$(date +%s)
+              echo "requested power off, expecting Off between ${MIN}s and ${MAX}s"
+
+              while :; do
+                s=$(state || echo unknown)
+                t=$(( $(date +%s) - start ))
+                echo "t=${t}s PowerState=$s"
+                [ "$s" = Off ] && break
+                [ "$t" -le "$MAX" ] || { echo "FAIL: still $s after ${MAX}s, ForceOff not issued or not honoured"; exit 1; }
+                sleep 2
+              done
+
+              [ "$t" -ge "$MIN" ] || { echo "FAIL: Off after ${t}s (< ${MIN}s), GracefulShutdown took effect, the booted image does not ignore the power button"; exit 1; }
+              echo "PASS: Off after ${t}s, only reachable via ForceOff"

tests/12-force-power-off/ignitionsecret.yaml

@@ -0,0 +1,10 @@
+# Creates an empty file that serves as a flag to the metalprobe-image to ignore
+# ACPI. See metalprobe-image/cmd/metalprobe-launcher/main.go.
+apiVersion: v1
+kind: Secret
+metadata:
+  name: (join('-', ['compatibility', $namespace, 'ignition']))
+type: Opaque
+stringData:
+  ignition: |
+    {"ignition":{"version":"3.4.0"},"storage":{"files":[{"path":"/metalprobe/ignore-power-button","contents":{"source":"data:,"}}]}}

tests/12-force-power-off/rbac.yaml

@@ -0,0 +1,30 @@
+# Allows the force-off-check Job to patch the ServerClaim
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: force-off-check
+  namespace: ($namespace)
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: force-off-check
+  namespace: ($namespace)
+rules:
+  - apiGroups: ["metal.ironcore.dev"]
+    resources: ["serverclaims"]
+    verbs: ["get", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: force-off-check
+  namespace: ($namespace)
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: force-off-check
+subjects:
+  - kind: ServiceAccount
+    name: force-off-check
+    namespace: ($namespace)

tests/12-force-power-off/serverclaim.yaml

@@ -0,0 +1,11 @@
+apiVersion: metal.ironcore.dev/v1alpha1
+kind: ServerClaim
+metadata:
+  name: (join('-', ['compatibility', $namespace]))
+spec:
+  power: "On"
+  serverRef:
+    name: (join('-', ['compatibility', $namespace, 'system-0']))
+  ignitionSecretRef:
+    name: (join('-', ['compatibility', $namespace, 'ignition']))
+  image: ghcr.io/simontesar/metal-lab/metalprobe:main