From b37fa1a7b38af1557453d1fe45269b4655d62935 Mon Sep 17 00:00:00 2001 From: Brooke Storm Date: Fri, 23 Oct 2026 17:26:35 -0700 Subject: [PATCH] feat(helm): add certificate hook pod labels and annotations Signed-off-by: Brooke Storm --- deploy/helm/openshell/README.md | 19 +++++ deploy/helm/openshell/README.md.gotmpl | 17 +++++ deploy/helm/openshell/templates/certgen.yaml | 13 +++- deploy/helm/openshell/tests/certgen_test.yaml | 74 +++++++++++++++++++ deploy/helm/openshell/values.yaml | 4 + docs/kubernetes/setup.mdx | 12 +++ skills/debug-openshell-cluster/SKILL.md | 1 + 7 files changed, 138 insertions(+), 2 deletions(-) diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index ffa306fd24..ea52198e28 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -17,6 +17,23 @@ namespace. The gateway and workspace releases can then be upgraded and removed independently. Use Kubernetes `operator` workspace mode when one gateway serves multiple pre-provisioned workspace namespaces. +## Certificate hook metadata + +Use `pkiInitJob.podLabels` and `pkiInitJob.podAnnotations` to add metadata to the +Pod templates of the `-certgen` and `-certgen-backend-ca` +Jobs. These settings apply to both hooks when enabled and do not affect gateway +or sandbox Pods. Chart-owned Pod labels take precedence when custom keys +conflict. + +For example, disable Istio sidecar injection for the hook Pods in an injected +namespace: + +```yaml +pkiInitJob: + podAnnotations: + sidecar.istio.io/inject: "false" +``` + ## Cluster-scoped vs namespaced objects Most objects in this chart are namespaced and land in the release namespace. @@ -450,6 +467,8 @@ discovery endpoint or its TLS CA. | openshiftRoute.host | string | `""` | Hostname for the Route. Must match a SAN on the gateway's server cert. | | pkiInitJob.enabled | bool | `true` | Run a pre-install/pre-upgrade Job that creates gateway and client mTLS Secrets. When certManager.enabled=true, cert-manager owns TLS and this same hook runs in JWT-only mode even if pkiInitJob.enabled remains true. | | pkiInitJob.failOnTimeout | bool | `true` | Fail the helm install/upgrade if cert-manager does not issue the certificate within the polling timeout. When true (default), the install fails immediately if the timeout is reached, providing clear feedback that BackendTLSPolicy is non-functional. When false, the hook succeeds with a warning and you can run `helm upgrade` after cert-manager issues the certificate to create the backend CA ConfigMap. If you set this to false and see "TLS error: Secret is not supplied by SDS" when connecting to the gateway, check if the TLS secret exists and run `helm upgrade` to create the ConfigMap. | +| pkiInitJob.podAnnotations | object | `{}` | Additional annotations on both certificate hook Pods. | +| pkiInitJob.podLabels | object | `{}` | Additional labels on both certificate hook Pods. Chart-owned labels take precedence. | | pkiInitJob.securityContext.allowPrivilegeEscalation | bool | `false` | Whether the certgen hook container can gain additional privileges. | | pkiInitJob.securityContext.capabilities.drop | list | `["ALL"]` | Linux capabilities dropped from the certgen hook container. | | pkiInitJob.securityContext.runAsNonRoot | bool | `true` | Require the certgen hook container to run as a non-root user. | diff --git a/deploy/helm/openshell/README.md.gotmpl b/deploy/helm/openshell/README.md.gotmpl index a98a39f3a6..46b3aa59ba 100644 --- a/deploy/helm/openshell/README.md.gotmpl +++ b/deploy/helm/openshell/README.md.gotmpl @@ -17,6 +17,23 @@ namespace. The gateway and workspace releases can then be upgraded and removed independently. Use Kubernetes `operator` workspace mode when one gateway serves multiple pre-provisioned workspace namespaces. +## Certificate hook metadata + +Use `pkiInitJob.podLabels` and `pkiInitJob.podAnnotations` to add metadata to the +Pod templates of the `-certgen` and `-certgen-backend-ca` +Jobs. These settings apply to both hooks when enabled and do not affect gateway +or sandbox Pods. Chart-owned Pod labels take precedence when custom keys +conflict. + +For example, disable Istio sidecar injection for the hook Pods in an injected +namespace: + +```yaml +pkiInitJob: + podAnnotations: + sidecar.istio.io/inject: "false" +``` + ## Cluster-scoped vs namespaced objects Most objects in this chart are namespaced and land in the release namespace. diff --git a/deploy/helm/openshell/templates/certgen.yaml b/deploy/helm/openshell/templates/certgen.yaml index aea3b8d2cc..c378916ecc 100644 --- a/deploy/helm/openshell/templates/certgen.yaml +++ b/deploy/helm/openshell/templates/certgen.yaml @@ -4,6 +4,7 @@ {{- if or .Values.pkiInitJob.enabled .Values.certManager.enabled }} {{- $hookName := printf "%s-certgen" (include "openshell.fullname" .) }} {{- $ns := .Release.Namespace }} +{{- $podLabels := merge (include "openshell.certgenPodLabels" . | fromYaml) (.Values.pkiInitJob.podLabels | default dict) }} apiVersion: v1 kind: ServiceAccount metadata: @@ -75,7 +76,11 @@ spec: template: metadata: labels: - {{- include "openshell.certgenPodLabels" . | nindent 8 }} + {{- toYaml $podLabels | nindent 8 }} + {{- with .Values.pkiInitJob.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} spec: restartPolicy: OnFailure serviceAccountName: {{ $hookName }} @@ -143,7 +148,11 @@ spec: template: metadata: labels: - {{- include "openshell.certgenPodLabels" . | nindent 8 }} + {{- toYaml $podLabels | nindent 8 }} + {{- with .Values.pkiInitJob.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} spec: restartPolicy: OnFailure serviceAccountName: {{ $hookName }} diff --git a/deploy/helm/openshell/tests/certgen_test.yaml b/deploy/helm/openshell/tests/certgen_test.yaml index 2a21c62ec5..ef0796447d 100644 --- a/deploy/helm/openshell/tests/certgen_test.yaml +++ b/deploy/helm/openshell/tests/certgen_test.yaml @@ -192,3 +192,77 @@ tests: app.kubernetes.io/instance: openshell app.kubernetes.io/component: certgen documentIndex: 4 + + - it: adds configured metadata to both certificate hook Pod templates + template: templates/certgen.yaml + set: + certManager.enabled: true + grpcRoute.backendTLSPolicy.enabled: true + pkiInitJob.podLabels: + example.com/workload: certificate-bootstrap + pkiInitJob.podAnnotations: + sidecar.istio.io/inject: "false" + asserts: + - equal: + path: spec.template.metadata.labels["example.com/workload"] + value: certificate-bootstrap + documentIndex: 3 + - equal: + path: spec.template.metadata.annotations["sidecar.istio.io/inject"] + value: "false" + documentIndex: 3 + - notExists: + path: metadata.labels["example.com/workload"] + documentIndex: 3 + - notExists: + path: metadata.annotations["sidecar.istio.io/inject"] + documentIndex: 3 + - equal: + path: spec.template.metadata.labels["example.com/workload"] + value: certificate-bootstrap + documentIndex: 4 + - equal: + path: spec.template.metadata.annotations["sidecar.istio.io/inject"] + value: "false" + documentIndex: 4 + - notExists: + path: metadata.labels["example.com/workload"] + documentIndex: 4 + - notExists: + path: metadata.annotations["sidecar.istio.io/inject"] + documentIndex: 4 + + - it: preserves chart-owned Pod labels when custom keys conflict + template: templates/certgen.yaml + set: + certManager.enabled: true + grpcRoute.backendTLSPolicy.enabled: true + pkiInitJob.podLabels: + app.kubernetes.io/name: openshell + app.kubernetes.io/instance: custom-release + app.kubernetes.io/component: gateway + asserts: + - equal: + path: spec.template.metadata.labels["app.kubernetes.io/name"] + value: openshell-certgen + documentIndex: 3 + - equal: + path: spec.template.metadata.labels["app.kubernetes.io/instance"] + value: openshell + documentIndex: 3 + - equal: + path: spec.template.metadata.labels["app.kubernetes.io/component"] + value: certgen + documentIndex: 3 + - equal: + path: spec.template.metadata.labels["app.kubernetes.io/name"] + value: openshell-certgen + documentIndex: 4 + - equal: + path: spec.template.metadata.labels["app.kubernetes.io/instance"] + value: openshell + documentIndex: 4 + - equal: + path: spec.template.metadata.labels["app.kubernetes.io/component"] + value: certgen + documentIndex: 4 diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index f1ff0580c8..c234806a14 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -661,6 +661,10 @@ pkiInitJob: # Secrets. When certManager.enabled=true, cert-manager owns TLS and this same # hook runs in JWT-only mode even if pkiInitJob.enabled remains true. enabled: true + # -- Additional labels on both certificate hook Pods. Chart-owned labels take precedence. + podLabels: {} + # -- Additional annotations on both certificate hook Pods. + podAnnotations: {} # -- Extra DNS SANs to append to the server certificate. serverDnsNames: [] # -- Extra IP SANs to append to the server certificate. diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index ae0be7d027..c8c4410750 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -118,6 +118,18 @@ helm upgrade --install openshell \ The chart automatically generates PKI secrets on first install using pre-install Helm hooks. No manual secret creation is required. +To add labels or annotations to both certificate hook Pod templates, use +`pkiInitJob.podLabels` and `pkiInitJob.podAnnotations`. Chart-owned Pod labels +take precedence when custom keys conflict. +For example, disable sidecar injection for hook Pods in an Istio-injected +namespace: + +```yaml +pkiInitJob: + podAnnotations: + sidecar.istio.io/inject: "false" +``` + The default Kubernetes driver puts all workspaces in one sandbox namespace. To have the gateway create a namespace for each workspace, follow [Workspace on Kubernetes](/kubernetes/workspaces). diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 40351c998e..e74e1b1c41 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1279,6 +1279,7 @@ credential failures. | `helm upgrade` fails with an `autoscaling.*` message | HPA values invalid: missing `resources.requests` (or `resources.limits`), `maxReplicas` above 1 without `server.externalDbSecret` (or on a StatefulSet without `workload.allowMultiReplicaStatefulSet`), no metric target, or min/max out of order. "`minReplicas` and `maxReplicas` are not set" means `--reuse-values` kept a release without the chart's autoscaling defaults | Fix the values named in the error; upgrade with `--reset-then-reuse-values` instead of `--reuse-values` | | HPA shows `` targets | No metrics-server for CPU/memory, or the metrics adapter does not serve the custom metric | `kubectl -n openshell describe hpa openshell`, `kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1` | | One replica holds most sessions after a rollout | Expected: sessions stay where they reconnected | `openshell_server_supervisor_sessions` per pod; it fades as sandboxes are recreated | +| Certificate hook Job stays active after `certgen` exits successfully | An injected sidecar is still running in the hook Pod | Inspect all container statuses. Configure `pkiInitJob.podLabels` or `pkiInitJob.podAnnotations` as required by the injector, then retry the Helm install or upgrade to recreate the hook Pods | | Gateway or `-certgen` hook pod rejected with `violates PodSecurity "restricted:latest"` | The rendered container lacks `runAsNonRoot` or `seccompProfile`: an older chart version does not set them, or an override in `securityContext` / `pkiInitJob.securityContext` unset them | `kubectl -n openshell get events --field-selector reason=FailedCreate`; upgrade the chart, then remove overrides that unset those fields | | OpenShift gateway pod fails to start with an SCC/`runAsUser` error (e.g. `unable to validate against any security context constraint`) | Chart's default `podSecurityContext`/`securityContext` hardcodes `runAsUser`/`fsGroup`, which the restricted-v2 SCC rejects; it must instead inject the namespace-assigned UID/GID range | `oc -n openshell describe pod `; deploy with `podSecurityContext: null` and clear `securityContext.runAsUser` (see `deploy/helm/openshell/ci/values-openshift-scc.yaml`) | | OpenShift `-supervisor` Deployment never creates pods after setting `supervisor_topology = "gateway"` (SCC `runAsUser`/`fsGroup` not in the namespace range) | `supervisor.podSecurityContext` defaults `runAsUser`, `runAsGroup`, and `fsGroup` to 65534, outside the namespace's assigned UID/GID ranges | `kubectl -n openshell get events --field-selector reason=FailedCreate`; compare with the `openshift.io/sa.scc.*` annotations from `oc describe ns openshell`; set `supervisor.podSecurityContext.runAsUser`, `.runAsGroup`, and `.fsGroup` to `null`, as `deploy/helm/openshell/ci/values-openshift-scc.yaml` does, and keep `runAsNonRoot` and `seccompProfile` |