Misc fixes vol9

.trivyignore.yaml

@@ -36,6 +36,7 @@ misconfigurations:
       - dp/dp.yaml
       - dra/daemonset.yaml
       - xpum/xpum.yaml
+      - xpum/xpum-fwupdate-job.yaml
 
   - id: AVD-KSV-0025
     statement: "container_device_plugin_t is a valid SELinux profile"
@@ -88,6 +89,7 @@ misconfigurations:
     statement: "/sys is required for DRA"
     paths:
       - dra/daemonset.yaml
+      - xpum/xpum-fwupdate-job.yaml
 
   - id: AVD-DS-0002
     statement: ""

charts/gpu-base-operator/README.md

@@ -39,6 +39,13 @@ See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_h
 | `operator.resources.limits.memory` | 128Mi | Memory limit for operator pod |
 | `operator.resources.requests.cpu` | 10m | CPU request for operator pod |
 | `operator.resources.requests.memory` | 64Mi | Memory request for operator pod |
+| `metrics.enabled` | false | Serve the operator's own metrics endpoint |
+| `metrics.port` | 8443 | Port for the metrics endpoint |
+| `metrics.secure` | true | Serve metrics over HTTPS behind the authn/authz filter |
+| `metrics.serviceMonitor.enabled` | false | Create a ServiceMonitor for the operator's metrics. Requires the Prometheus Operator CRDs |
+| `metrics.serviceMonitor.interval` | "" | Scrape interval; empty inherits the Prometheus default |
+| `metrics.serviceMonitor.additionalLabels` | {} | Extra ServiceMonitor labels, for a Prometheus with a `serviceMonitorSelector` |
+| `metrics.serviceMonitor.tlsConfig` | {} | Replaces the default `insecureSkipVerify` when `metrics.secure` is true |
 | `privateRegistry.url` | "" | Private registry URL |
 | `privateRegistry.user` | "" | Private registry username |
 | `privateRegistry.token` | "" | Private registry authentication token |

charts/gpu-base-operator/templates/certificates.yaml

@@ -8,9 +8,16 @@ metadata:
   name: intel-gpu-base-operator-metrics-certs
   namespace: {{ .Release.Namespace }}
 spec:
+  # config/certmanager/certificate-metrics.yaml carries SERVICE_NAME/SERVICE_NAMESPACE placeholders
+  # that kustomize substitutes; Helm does not, so they must be spelled out here against the metrics
+  # Service in metrics_service.yaml.
+  #
+  # NOTE: the manager does not mount this secret and is not started with --metrics-cert-path, so it
+  # still serves the self-signed cert it generates at startup and this Certificate goes unused. See
+  # config/default/cert_metrics_manager_patch.yaml for the mount the chart is missing.
   dnsNames:
-  - SERVICE_NAME.SERVICE_NAMESPACE.svc
-  - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+  - {{ .Release.Name }}-controller-manager-metrics-service.{{ .Release.Namespace }}.svc
+  - {{ .Release.Name }}-controller-manager-metrics-service.{{ .Release.Namespace }}.svc.cluster.local
   issuerRef:
     kind: Issuer
     name: intel-gpu-base-operator-selfsigned-issuer

charts/gpu-base-operator/templates/manager.yaml

@@ -59,6 +59,12 @@ spec:
           - --health-probe-bind-address=:8081
           - --webhook-cert-path=/tmp/k8s-webhook-server/serving-certs
           - -v={{ .Values.operator.verbosity }}
+        {{- if .Values.metrics.enabled }}
+          # Without this flag --metrics-bind-address defaults to "0", which disables the
+          # endpoint entirely. The kustomize path sets it via config/default/manager_metrics_patch.yaml.
+          - --metrics-bind-address=:{{ .Values.metrics.port }}
+          - --metrics-secure={{ .Values.metrics.secure }}
+        {{- end }}
         env:
           - name: OPERATOR_NAMESPACE
             valueFrom:
@@ -77,6 +83,11 @@ spec:
         - containerPort: 9443
           name: webhook-server
           protocol: TCP
+        {{- if .Values.metrics.enabled }}
+        - containerPort: {{ .Values.metrics.port }}
+          name: metrics
+          protocol: TCP
+        {{- end }}
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:

charts/gpu-base-operator/templates/metrics_service.yaml

@@ -0,0 +1,22 @@
+{{- if .Values.metrics.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Release.Name }}-controller-manager-metrics-service
+  namespace: {{ .Release.Namespace }}
+  labels:
+    control-plane: {{ .Release.Name }}-controller-manager
+    app.kubernetes.io/name: intel-gpu-base-operator
+spec:
+  ports:
+  # Named "https" or "http" after the scheme actually in use, because the ServiceMonitor
+  # selects the endpoint by port *name*. A fixed name would leave the scrape mismatched
+  # whenever metrics.secure is flipped.
+  - name: {{ if .Values.metrics.secure }}https{{ else }}http{{ end }}
+    port: {{ .Values.metrics.port }}
+    protocol: TCP
+    targetPort: metrics
+  selector:
+    control-plane: {{ .Release.Name }}-controller-manager
+    app.kubernetes.io/name: intel-gpu-base-operator
+{{- end }}

charts/gpu-base-operator/templates/metrics_servicemonitor.yaml

@@ -0,0 +1,54 @@
+{{- if .Values.metrics.serviceMonitor.enabled }}
+{{- if not .Values.metrics.enabled }}
+{{- fail "metrics.serviceMonitor.enabled requires metrics.enabled: a ServiceMonitor scraping a disabled endpoint would report the operator as permanently down" }}
+{{- end }}
+# Scrapes the operator's own metrics endpoint. Opt-in, because it needs the Prometheus
+# Operator CRDs installed: applying a ServiceMonitor without monitoring.coreos.com present
+# fails the whole `helm install` on an unknown kind.
+#
+# Distinct from the ServiceMonitor the operator *deploys for XPU-Manager* (see
+# config/deployments/prometheus/service-monitor.yaml and MiscReconciler) — that one exports GPU
+# telemetry, this one exports the operator's own controller metrics.
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ .Release.Name }}-controller-manager-metrics-monitor
+  namespace: {{ .Release.Namespace }}
+  labels:
+    control-plane: {{ .Release.Name }}-controller-manager
+    app.kubernetes.io/name: intel-gpu-base-operator
+    {{- with .Values.metrics.serviceMonitor.additionalLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  selector:
+    matchLabels:
+      control-plane: {{ .Release.Name }}-controller-manager
+      app.kubernetes.io/name: intel-gpu-base-operator
+  namespaceSelector:
+    matchNames:
+    - {{ .Release.Namespace }}
+  endpoints:
+  - path: /metrics
+    port: {{ if .Values.metrics.secure }}https{{ else }}http{{ end }}
+    scheme: {{ if .Values.metrics.secure }}https{{ else }}http{{ end }}
+    {{- with .Values.metrics.serviceMonitor.interval }}
+    interval: {{ . }}
+    {{- end }}
+    {{- if .Values.metrics.secure }}
+    # The endpoint is protected by authn/authz filters (see cmd/main.go), so the scraper must
+    # present a token. The RBAC allowing it is in metrics_reader_role.yaml — bind Prometheus's
+    # ServiceAccount to that ClusterRole or scrapes come back 401.
+    bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    tlsConfig:
+      {{- if .Values.metrics.serviceMonitor.tlsConfig }}
+      {{- toYaml .Values.metrics.serviceMonitor.tlsConfig | nindent 6 }}
+      {{- else }}
+      # The manager serves a self-signed cert generated at startup, not one from cert-manager,
+      # so there is no CA for Prometheus to verify against and this must be skipped by default.
+      # To verify properly, mount a cert-manager cert into the manager, point
+      # --metrics-cert-path at it, and set metrics.serviceMonitor.tlsConfig here.
+      insecureSkipVerify: true
+      {{- end }}
+    {{- end }}
+{{- end }}

charts/gpu-base-operator/values.yaml

@@ -19,6 +19,27 @@ operator:
       cpu: 100m
       memory: 256Mi
 
+metrics:
+  # Serve the operator's own metrics (the controller-runtime and workqueue metrics).
+  enabled: false
+  port: 8443
+  # Protect the endpoint with TLS and the authn/authz filter. When true, a scraper must present
+  # a bearer token and hold the <release>-metrics-reader ClusterRole.
+  secure: true
+  serviceMonitor:
+    # Off by default: a ServiceMonitor requires the Prometheus Operator CRDs, and applying one
+    # without monitoring.coreos.com installed fails the whole helm install on an unknown kind.
+    # Turn this on only when Prometheus Operator is present.
+    enabled: false
+    # Scrape interval. Empty means inherit the Prometheus default.
+    interval: ""
+    # Extra labels, for a Prometheus whose serviceMonitorSelector requires a specific label
+    # (e.g. release: kube-prometheus-stack) to pick the monitor up.
+    additionalLabels: {}
+    # Replaces the default insecureSkipVerify when metrics.secure is true. Set this only after
+    # pointing the manager's --metrics-cert-path at a real cert; see the template for details.
+    tlsConfig: {}
+
 privateRegistry:
   url: ""
   user: ""

config/deployments/deployments_test.go

@@ -156,6 +156,32 @@ func TestOTelConfig(t *testing.T) {
 	}
 }
 
+// The wedged/survivability mapping is what turns an unusable GPU into a DRA device taint, and
+// it only works if it survives the parse into HWStatusMapping and stays ahead of the unfiltered
+// catch-all rule that would otherwise claim the same states.
+func TestOTelConfig_UnusableGPUStatesMapToFailed(t *testing.T) {
+	mappings := XpuManagerOTelConfig().Exporters.IntelXPUInfo.HWStatusMappings
+	if len(mappings) == 0 {
+		t.Fatal("no hw_status_mappings parsed from otel-config.yaml")
+	}
+
+	first := mappings[0]
+	if first.HealthDomain != "gpu.{{ .hw_state }}" {
+		t.Errorf("expected the per-state GPU mapping first, got health_domain %q", first.HealthDomain)
+	}
+
+	for _, state := range []string{"wedged", "survivability"} {
+		severity, ok := first.StateMapping[state]
+		if !ok {
+			t.Errorf("%s has no state_mapping entry", state)
+			continue
+		}
+		if severity.Severity != "failed" {
+			t.Errorf("%s maps to severity %q, want failed", state, severity.Severity)
+		}
+	}
+}
+
 func TestDevicePluginDaemonset_AutomountServiceAccountToken(t *testing.T) {
 	ds := DevicePluginDaemonset()
 	if ds.Spec.Template.Spec.AutomountServiceAccountToken == nil {

config/deployments/otel_types.go

@@ -91,6 +91,9 @@ type IntelXPUInfoExporter struct {
 }
 
 // HWStatusMapping maps a health domain to state severity entries.
+//
+// The embedded otel-config.yaml is parsed into these types and re-marshalled into the
+// ConfigMap, so a key missing here is silently dropped from what xpumd actually reads.
 type HWStatusMapping struct {
 	HealthDomain string                       `json:"health_domain"`
 	Filters      []KeyValues                  `json:"filters,omitempty"`

config/deployments/xpum/otel-config.yaml

@@ -66,6 +66,26 @@ exporters:
   intel_xpu_info:
     endpoint: "/run/xpumd/intelxpuinfo.sock"
     hw_status_mappings:
+      # A wedged GPU and a card that has fallen into survivability (FDO) mode are both
+      # unusable, so report them as "failed" under their own health domains. Together with
+      # the "health-xpumd-" prefix the DRA driver adds, these become the
+      # health-xpumd-gpu.wedged / health-xpumd-gpu.survivability device taints.
+      #
+      # This has to come first: the mappings are first-match-wins (that is what makes the
+      # ecc filter-out entry below work), and the trailing "{{ .hw_type }}" rule carries no
+      # filters, so it matches every GPU state and would fold these two into its
+      # '"*": warning' bucket instead.
+      - health_domain: "gpu.{{ .hw_state }}"
+        filters:
+          - key: hw.type
+            values: [gpu]
+          - key: hw.state
+            values: [wedged, survivability]
+        state_mapping:
+          wedged:
+            severity: failed
+          survivability:
+            severity: failed
       - health_domain: "memory"
         filters:
           - key: hw.type

config/deployments/xpum/xpum-fwupdate-job.yaml

@@ -18,12 +18,21 @@ spec:
     - action: FailJob
       onPodConditions:
       - type: ConfigIssue
+  # A firmware update that has not finished in ten minutes is not going to. Fail the Job rather
+  # than leave the node tainted and the CR parked in "updating" indefinitely.
+  activeDeadlineSeconds: 600
   template:
     spec:
       volumes:
       - name: update
         emptyDir:
           sizeLimit: 64Mi
+      # The update script resolves the MEI device of the target GPU through
+      # /sys/bus/pci/devices/<bdf>/, so the updater needs the host's sysfs.
+      - name: host-sys
+        hostPath:
+          path: /sys
+          type: Directory
       automountServiceAccountToken: false
       initContainers:
       - name: fw-copy
@@ -68,6 +77,9 @@ spec:
         volumeMounts:
         - name: update
           mountPath: /update
+        - name: host-sys
+          mountPath: /sys
+          readOnly: false
       imagePullSecrets:
       restartPolicy: Never
 

go.mod

@@ -24,7 +24,7 @@ require (
 require github.com/go-openapi/swag/pools v0.27.1 // indirect
 
 require (
-	cel.dev/expr v0.25.1 // indirect
+	cel.dev/expr v0.25.2 // indirect
 	github.com/Masterminds/semver/v3 v3.5.0 // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -85,7 +85,7 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect
 	go.opentelemetry.io/otel/metric v1.44.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.43.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.44.0 // indirect
 	go.opentelemetry.io/otel/trace v1.44.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
@@ -105,7 +105,7 @@ require (
 	gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260610212136-7ab31c22f7ad // indirect
-	google.golang.org/grpc v1.82.1 // indirect
+	google.golang.org/grpc v1.83.1 // indirect
 	google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect

go.sum

@@ -1,5 +1,5 @@
-cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4=
-cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
+cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs=
+cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
 github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE=
 github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
 github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
@@ -194,10 +194,10 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDO
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs=
 go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
 go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
-go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg=
-go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
-go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw=
-go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
+go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
+go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
+go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=
+go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
 go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
 go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
 go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
@@ -240,8 +240,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:
 google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260610212136-7ab31c22f7ad h1:45WmJvIV6C2+O/jjLkPUH+F3aOj/1miDoU2DD0+NWbg=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260610212136-7ab31c22f7ad/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
-google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE=
-google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA=
+google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y=
+google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ=
 google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI=
 google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

internal/controller/openshift.go

@@ -128,10 +128,14 @@ func buildDRASCC(name string) *unstructured.Unstructured {
 
 // buildFWUpdateSCC returns the SCC for the GPU firmware update Job pods.
 // The updater container runs privileged as root to access GPU firmware interfaces.
+//
+// hostPath is allowed because xpum-fwupdate-job.yaml mounts /sys into the updater: the update
+// script resolves the target GPU's MEI device through sysfs. An SCC forbidding it would make
+// every firmware update fail at admission on OpenShift.
 func buildFWUpdateSCC(name string) *unstructured.Unstructured {
 	return buildSCC(name, map[string]interface{}{
 		"allowPrivilegedContainer": true,
-		"allowHostDirVolumePlugin": false,
+		"allowHostDirVolumePlugin": true,
 		"allowHostIPC":             false,
 		"allowHostNetwork":         false,
 		"allowHostPID":             false,
@@ -146,7 +150,7 @@ func buildFWUpdateSCC(name string) *unstructured.Unstructured {
 		"seLinuxContext":           map[string]interface{}{"type": "RunAsAny"},
 		"seccompProfiles":          []interface{}{"*"},
 		"supplementalGroups":       map[string]interface{}{"type": "RunAsAny"},
-		"volumes":                  []interface{}{"emptyDir"},
+		"volumes":                  []interface{}{"hostPath", "emptyDir"},
 		"users":                    []interface{}{},
 		"groups":                   []interface{}{},
 	})