diff --git a/Makefile b/Makefile index a2373a0..1c1aa13 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,7 @@ TESTS := \ 08-bmcsettings \ 09-persistent-boot-order \ 10-bmcversion \ + 12-force-power-off \ 11-bmcuser \ 13-event-subscriptions @@ -41,6 +42,7 @@ help: ## Show available targets @echo " test/08-bmcsettings (BMCSettings, Manager attribute)" @echo " test/09-persistent-boot-order (persistent boot order)" @echo " test/10-bmcversion (BMCVersion, version read without upgrade)" + @echo " test/12-force-power-off (forced power off after a graceful shutdown is ignored)" @echo " test/11-bmcuser (BMCUser, BMC account create/rotate/delete)" @echo " test/13-event-subscriptions (Redfish event subscriptions via metal-maintenance-operator telemetry)" @echo "" diff --git a/README.md b/README.md index 57eb864..dbd1bb0 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,10 @@ biosVersion: "1.0.0" key: "EmailAlert.1.Address" value: "alerts@example.com" baselineValue: "compat-base@example.com" +12-force-power-off: # optionally override the defaults + osBootWait: 120 # time to sleep before assuming the boot has reached the OS + powerPollingTimeout: 15 # must equal the metal-operator's --power-polling-timeout to the the Job + forceOffDelay: 90 # seconds allowed beyond powerPollingTimeout for the ForceOff to take effect 13-event-subscriptions: testMessageId: "TestEventId" # a MessageId your BMC accepts in SubmitTestEvent, e.g. "SYS1000" on Dell iDRAC testSeverity: "OK" diff --git a/coverage.md b/coverage.md index 92f9c1c..6c676da 100644 --- a/coverage.md +++ b/coverage.md @@ -32,7 +32,7 @@ These rows/capabilities include the following placeholders in their `test` field | `tests/09-persistent-boot-order` | `BMC Registration`, `Discovery`, `boot.order-get`, `boot.order-set`, `power.on`, `power.off-graceful` | | `tests/10-bmcversion` | `BMC Registration`, `Discovery`, `bmc.version` | | `tests/11-bmcuser` | `BMC Registration`, `account.service`, `account.list`, `account.create-update`, `account.delete` | -| `tests/12-force-power-off` | `BMC Registration`, `Discovery`, `power.off-graceful`, `power.off-force`, `power.wait-state` | +| `tests/12-force-power-off` | `BMC Registration`, `Discovery`, `power.on`, `power.off-graceful`, `power.off-force`, `power.wait-state`, `boot.override-pxe` | | `tests/13-event-subscriptions` | `BMC Registration`, `events.subscribe`, `events.unsubscribe`, `events.list`, `events.test-event`, `events.delivery` | ## Coverage by capability @@ -41,7 +41,7 @@ These rows/capabilities include the following placeholders in their `test` field |------------------------------|-------------------------------------------------------------------------------|:--------:| | `power.on` | `Discovery` | yes | | `power.off-graceful` | `Discovery` | yes | -| `power.off-force` | `tests/03-power-annotation` See **[0]** | no | +| `power.off-force` | `tests/12-force-power-off`, `tests/03-power-annotation` See **[0]** | yes | | `power.reset` | `tests/03-power-annotation` | no | | `power.wait-state` | `Discovery` | yes | | `boot.override-pxe` | `Discovery` | yes | diff --git a/tests/12-force-power-off/chainsaw-test.yaml b/tests/12-force-power-off/chainsaw-test.yaml new file mode 100644 index 0000000..096e29c --- /dev/null +++ b/tests/12-force-power-off/chainsaw-test.yaml @@ -0,0 +1,137 @@ +# Creates a scenario where the metal-operator issues a "ForceOff" reset by +# booting an image that ignores ACPI and verifying that the server still powered +# off after the graceful window. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: 12-force-power-off +spec: + concurrent: false + bindings: + - name: serverName + value: (join('-', ['compatibility', $namespace, 'system-0'])) + # Time to sleep before assuming the boot has reached the OS + - name: osBootWait + value: ($values."12-force-power-off".osBootWait || '2m') + # The metal-operator's --power-polling-timeout + - name: powerPollingTimeout + value: ($values."12-force-power-off".powerPollingTimeout || `15`) + # Seconds beyond the power polling timeout allowed for the force-off to take effect + - name: forceOffDelay + value: ($values."12-force-power-off".forceOffDelay || `90`) + steps: + - name: register-bmc + try: + - apply: + file: ../shared/bmcsecret.yaml + - apply: + file: ../shared/bmc.yaml + cleanup: + - sleep: + duration: 10s + - delete: + timeout: 2m + ref: + apiVersion: metal.ironcore.dev/v1alpha1 + kind: Server + labels: + metal-operator-test-framework/namespace: ($namespace) + - name: assert-bmc-enabled + try: + - assert: + resource: + apiVersion: metal.ironcore.dev/v1alpha1 + kind: BMC + metadata: + name: (join('-', ['compatibility', $namespace])) + status: + state: Enabled + - name: assert-server-available + try: + - assert: + resource: + apiVersion: metal.ironcore.dev/v1alpha1 + kind: Server + metadata: + name: ($serverName) + status: + state: Available + powerState: "Off" + # Boots an image ignoring ACPI to avoid graceful shutdown + - name: claim-server + try: + - apply: + file: ignitionsecret.yaml + - apply: + file: serverclaim.yaml + - assert: + resource: + apiVersion: metal.ironcore.dev/v1alpha1 + kind: Server + metadata: + name: ($serverName) + status: + state: Reserved + powerState: "On" + # Wait for the OS to actually boot, otherwise firmware handles ACPI and might + # respect graceful shutdown + - name: wait-for-os + try: + - script: + timeout: 10m + env: + - name: OS_BOOT_WAIT + value: ($osBootWait) + content: sleep "$OS_BOOT_WAIT" + # Create a Job pataching the Server's power state to off and asserts via + # Redfish API that the power state chagnged in a time window that is outside + # the window for a graceful shutdown, i.e. the force-off took effect + - name: force-power-off + bindings: + - name: systemURI + value: (x_k8s_get($client, 'metal.ironcore.dev/v1alpha1', 'Server', '', $serverName).spec.systemURI) + try: + - apply: + file: rbac.yaml + - apply: + file: force-off-job.yaml + # Require the Job to finish either way but fail fast when it failed for + # internal reasons + - assert: + timeout: 10m + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: force-off-check + status: + (conditions[?type == 'Complete' || type == 'Failed'] | [0].status): "True" + - assert: + timeout: 5s + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: force-off-check + status: + (conditions[?type == 'Complete'] | [0].status): "True" + finally: + - podLogs: + selector: job-name=force-off-check + - name: assert-server-off + try: + - assert: + resource: + apiVersion: metal.ironcore.dev/v1alpha1 + kind: Server + metadata: + name: ($serverName) + status: + powerState: "Off" + - name: release-server + try: + - delete: + ref: + apiVersion: metal.ironcore.dev/v1alpha1 + kind: ServerClaim + name: (join('-', ['compatibility', $namespace])) diff --git a/tests/12-force-power-off/force-off-job.yaml b/tests/12-force-power-off/force-off-job.yaml new file mode 100644 index 0000000..43f0f4c --- /dev/null +++ b/tests/12-force-power-off/force-off-job.yaml @@ -0,0 +1,67 @@ +# Requests the power off by patching the ServerClaim and times the transition +# to Off directly on the BMC's redfish API. The metal-operator first issues a +# GracefulShutdown, waits for --power-polling-timeout and only then issues a +# ForceOff. As the booted image ignores the power button, the system turning +# Off no earlier than the polling timeout proves that ForceOff took effect. +apiVersion: batch/v1 +kind: Job +metadata: + name: force-off-check + namespace: ($namespace) +spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + serviceAccountName: force-off-check + containers: + - name: check + image: curlimages/curl:8.11.1 + env: + - name: URL + value: (join('', [$values.bmcScheme, '://', $values.bmcIP, ':', to_string($values.bmcPort), $systemURI])) + - name: CREDENTIALS + value: (join(':', [$values.username, $values.password])) + - name: NS + value: ($namespace) + - name: CLAIM + value: (join('-', ['compatibility', $namespace])) + - name: POLL_TIMEOUT + value: (to_string($powerPollingTimeout)) + - name: SLACK + value: (to_string($forceOffDelay)) + command: ["/bin/sh", "-c"] + args: + - | + set -eu + SA=/var/run/secrets/kubernetes.io/serviceaccount + MIN=$((POLL_TIMEOUT - 1)) + MAX=$((POLL_TIMEOUT + SLACK)) + + state() { + curl -fsS -k -u "$CREDENTIALS" "$URL" | sed -n 's/.*"PowerState" *: *"\([A-Za-z]*\)".*/\1/p' + } + + s=$(state) + [ "$s" = On ] || { echo "precondition failed: expected PowerState On, got '$s'"; exit 1; } + + curl -fsS -o /dev/null -X PATCH \ + --cacert "$SA/ca.crt" \ + -H "Authorization: Bearer $(cat "$SA/token")" \ + -H 'Content-Type: application/merge-patch+json' \ + -d '{"spec":{"power":"Off"}}' \ + "https://kubernetes.default.svc/apis/metal.ironcore.dev/v1alpha1/namespaces/$NS/serverclaims/$CLAIM" + start=$(date +%s) + echo "requested power off, expecting Off between ${MIN}s and ${MAX}s" + + while :; do + s=$(state || echo unknown) + t=$(( $(date +%s) - start )) + echo "t=${t}s PowerState=$s" + [ "$s" = Off ] && break + [ "$t" -le "$MAX" ] || { echo "FAIL: still $s after ${MAX}s, ForceOff not issued or not honoured"; exit 1; } + sleep 2 + done + + [ "$t" -ge "$MIN" ] || { echo "FAIL: Off after ${t}s (< ${MIN}s), GracefulShutdown took effect, the booted image does not ignore the power button"; exit 1; } + echo "PASS: Off after ${t}s, only reachable via ForceOff" diff --git a/tests/12-force-power-off/ignitionsecret.yaml b/tests/12-force-power-off/ignitionsecret.yaml new file mode 100644 index 0000000..fc3316b --- /dev/null +++ b/tests/12-force-power-off/ignitionsecret.yaml @@ -0,0 +1,10 @@ +# Creates an empty file that serves as a flag to the metalprobe-image to ignore +# ACPI. See metalprobe-image/cmd/metalprobe-launcher/main.go. +apiVersion: v1 +kind: Secret +metadata: + name: (join('-', ['compatibility', $namespace, 'ignition'])) +type: Opaque +stringData: + ignition: | + {"ignition":{"version":"3.4.0"},"storage":{"files":[{"path":"/metalprobe/ignore-power-button","contents":{"source":"data:,"}}]}} diff --git a/tests/12-force-power-off/rbac.yaml b/tests/12-force-power-off/rbac.yaml new file mode 100644 index 0000000..475efa8 --- /dev/null +++ b/tests/12-force-power-off/rbac.yaml @@ -0,0 +1,30 @@ +# Allows the force-off-check Job to patch the ServerClaim +apiVersion: v1 +kind: ServiceAccount +metadata: + name: force-off-check + namespace: ($namespace) +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: force-off-check + namespace: ($namespace) +rules: + - apiGroups: ["metal.ironcore.dev"] + resources: ["serverclaims"] + verbs: ["get", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: force-off-check + namespace: ($namespace) +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: force-off-check +subjects: + - kind: ServiceAccount + name: force-off-check + namespace: ($namespace) diff --git a/tests/12-force-power-off/serverclaim.yaml b/tests/12-force-power-off/serverclaim.yaml new file mode 100644 index 0000000..db78851 --- /dev/null +++ b/tests/12-force-power-off/serverclaim.yaml @@ -0,0 +1,11 @@ +apiVersion: metal.ironcore.dev/v1alpha1 +kind: ServerClaim +metadata: + name: (join('-', ['compatibility', $namespace])) +spec: + power: "On" + serverRef: + name: (join('-', ['compatibility', $namespace, 'system-0'])) + ignitionSecretRef: + name: (join('-', ['compatibility', $namespace, 'ignition'])) + image: ghcr.io/simontesar/metal-lab/metalprobe:main