feat: 13: confirm the telemetry setup succeeded by querying metrics

tests/13-event-subscriptions/chainsaw-test.yaml

@@ -63,6 +63,36 @@ spec:
                 name: (join('-', ['compatibility', $namespace]))
                 annotations:
                   metal-operator-test-framework/reconcile: subscribe
+    # Create a Job to read the operator's metrics. When the test event health
+    # check has completed, a BMC-specific Gauge should show up. This is the only
+    # way success is signalled, there is no condition update on the BMC itself.
+    - name: event-verification
+      try:
+        - apply:
+            file: metrics-rbac.yaml
+        - apply:
+            file: event-verification-job.yaml
+        - assert:
+            timeout: 4m
+            resource:
+              apiVersion: batch/v1
+              kind: Job
+              metadata:
+                name: event-verification
+              status:
+                (conditions[?type == 'Complete' || type == 'Failed'] | [0].status): "True"
+        - assert:
+            timeout: 5s
+            resource:
+              apiVersion: batch/v1
+              kind: Job
+              metadata:
+                name: event-verification
+              status:
+                (conditions[?type == 'Complete'] | [0].status): "True"
+      finally:
+        - podLogs:
+            selector: job-name=event-verification
     - name: unsubscribe
       try:
         - apply:

tests/13-event-subscriptions/event-verification-job.yaml

@@ -0,0 +1,50 @@
+# Queries the operator's metrics confirming that a gauge signalling that a BMC's
+# test event arrived appears and is set to 1.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: event-verification
+  namespace: ($namespace)
+spec:
+  backoffLimit: 0
+  template:
+    spec:
+      restartPolicy: Never
+      serviceAccountName: event-verification
+      containers:
+        - name: check
+          image: curlimages/curl:8.11.1
+          env:
+            - name: URL
+              value: (join('', ['https://', $metricsService, '.', $operatorNamespace, '.svc:8443/metrics']))
+            - name: BMC
+              value: (join('-', ['compatibility', $namespace]))
+            - name: TIMEOUT
+              value: "180"
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              set -eu
+              SA=/var/run/secrets/kubernetes.io/serviceaccount
+              METRIC=redfish_event_pipeline_test_last_result
+              start=$(date +%s)
+
+              # The metrics endpoint serves a self-signed certificate
+              metrics() {
+                curl -fsS -k -H "Authorization: Bearer $(cat "$SA/token")" "$URL"
+              }
+
+              while :; do
+                lines=$(metrics | grep "^$METRIC{" || true)
+                t=$(( $(date +%s) - start ))
+                if printf '%s\n' "$lines" | grep -qxF "$METRIC{hostname=\"$BMC\"} 1"; then
+                  echo "PASS: test event from $BMC received after ${t}s"
+                  exit 0
+                fi
+                [ "$t" -le "$TIMEOUT" ] || break
+                sleep 5
+              done
+
+              echo "FAIL: no test event from $BMC received within ${TIMEOUT}s, $METRIC series:"
+              printf '%s\n' "${lines:-<none>}"
+              exit 1

tests/13-event-subscriptions/metrics-rbac.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: event-verification
+  namespace: ($namespace)
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: (join('-', ['event-verification', $namespace]))
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ($metricsReaderRole)
+subjects:
+  - kind: ServiceAccount
+    name: event-verification
+    namespace: ($namespace)