From f59dc479cb1555a342fac64818e11cfaa65e877e Mon Sep 17 00:00:00 2001 From: Stephanie Baum Date: Wed, 23 Sep 2026 00:10:32 -0700 Subject: [PATCH 1/5] fix(worker-init): run bootstrap containers as non-root Declare a numeric image user and match it in generated init containers. Set a default fsGroup for shared volumes and run the transport trust installer with the same identity. Refs: #2071 Signed-off-by: Stephanie Baum --- rules/oci/private/common.bzl | 8 +++-- rules/oci/private/go.bzl | 7 +++- .../internal/transporttls/transport_tls.go | 7 ++-- .../transporttls/transport_tls_test.go | 5 +-- .../translate/common/translator.go | 33 +++++++++++++++++++ .../translate/function/translate_container.go | 3 +- .../function/translate_container_utdep.go | 6 ++-- .../translate/function/translate_helm.go | 3 +- .../function/translate_helm_utdep.go | 3 +- .../translate/task/translate.go | 3 +- .../translate/task/translate_helm.go | 3 +- .../worker-init/README.md | 11 +++++++ .../worker-init/cmd/BUILD.bazel | 1 + .../worker-init/cmd/image_contents_test.sh | 5 +++ .../translate/common/translator.go | 33 +++++++++++++++++++ .../translate/common/translator_test.go | 21 ++++++++++++ .../translate/function/translate_container.go | 3 +- .../function/translate_container_utdep.go | 6 ++-- .../translate/function/translate_helm.go | 3 +- .../function/translate_helm_utdep.go | 3 +- .../function/translate_tolerations_test.go | 3 ++ .../translate/task/translate.go | 3 +- .../translate/task/translate_all_test.go | 6 ++++ .../translate/task/translate_helm.go | 3 +- 24 files changed, 159 insertions(+), 23 deletions(-) diff --git a/rules/oci/private/common.bzl b/rules/oci/private/common.bzl index b573cdfe4e..5a23a288b7 100644 --- a/rules/oci/private/common.bzl +++ b/rules/oci/private/common.bzl @@ -39,7 +39,8 @@ def create_oci_image( tags = None, env = None, cmd = None, - workdir = None): + workdir = None, + user = None): """Creates OCI image targets with platform transitions and tarball output. Generates: @@ -57,7 +58,7 @@ def create_oci_image( all_tags = ["manual"] + (tags or []) pre_transitioned = name + "_pre_transitioned" - oci_image( + image_attrs = dict( name = pre_transitioned, base = base, tars = tars + COMMON_LAYERS, @@ -68,6 +69,9 @@ def create_oci_image( visibility = ["//visibility:private"], tags = all_tags, ) + if user: + image_attrs["user"] = user + oci_image(**image_attrs) platform_transition_filegroup( name = name, diff --git a/rules/oci/private/go.bzl b/rules/oci/private/go.bzl index 14f75bd863..3f53de24b1 100644 --- a/rules/oci/private/go.bzl +++ b/rules/oci/private/go.bzl @@ -147,7 +147,7 @@ go_oci_image = macro( }, ) -def _go_oci_multi_binary_image_impl(name, visibility, binaries, base, entrypoint, cmd, extra_layers, registry, extra_registries, tags): +def _go_oci_multi_binary_image_impl(name, visibility, binaries, base, entrypoint, cmd, extra_layers, registry, extra_registries, tags, user): """Pack multiple go_binary targets into a single OCI image layer. Used for images that bundle several binaries (eg nvca-operator's image @@ -192,6 +192,7 @@ def _go_oci_multi_binary_image_impl(name, visibility, binaries, base, entrypoint base = base, entrypoint = entrypoint, cmd = cmd, + user = user, visibility = visibility, registry = registry, extra_registries = extra_registries, @@ -241,5 +242,9 @@ go_oci_multi_binary_image = macro( doc = "Tags for generated targets. 'manual' is always added.", configurable = False, ), + "user": attr.string( + doc = "Numeric UID:GID recorded in the OCI image configuration.", + configurable = False, + ), }, ) diff --git a/src/compute-plane-services/nvca/internal/transporttls/transport_tls.go b/src/compute-plane-services/nvca/internal/transporttls/transport_tls.go index 9db6e14d87..9d55048541 100644 --- a/src/compute-plane-services/nvca/internal/transporttls/transport_tls.go +++ b/src/compute-plane-services/nvca/internal/transporttls/transport_tls.go @@ -23,6 +23,7 @@ import ( "regexp" "strings" + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common" "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function" "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/trustbundle" nvcaconfig "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/types/nvca/config" @@ -255,6 +256,7 @@ func upsertInstallContainer( resources corev1.ResourceRequirements, cfg nvcaconfig.TransportTLSConfig, ) { + common.EnsureWorkerInitFSGroup(podSpec) upsertContainer(&podSpec.InitContainers, corev1.Container{ Name: InstallContainerName, Image: image, @@ -271,10 +273,7 @@ func upsertInstallContainer( {Name: TrustBundleVolumeName, MountPath: TrustBundleMountPath, ReadOnly: true}, {Name: MergedCertsVolumeName, MountPath: MergedCertsMountPath}, }, - SecurityContext: &corev1.SecurityContext{ - RunAsUser: ptr.To[int64](0), - RunAsNonRoot: ptr.To(false), - }, + SecurityContext: common.NewWorkerInitContainerSecurityContext(), }) } diff --git a/src/compute-plane-services/nvca/internal/transporttls/transport_tls_test.go b/src/compute-plane-services/nvca/internal/transporttls/transport_tls_test.go index 0648510173..66120d3ea6 100644 --- a/src/compute-plane-services/nvca/internal/transporttls/transport_tls_test.go +++ b/src/compute-plane-services/nvca/internal/transporttls/transport_tls_test.go @@ -377,9 +377,10 @@ func TestInjectIntoPodSpecUsesRenderedInitImageAndPullPolicy(t *testing.T) { }, installContainer.Args) require.NotNil(t, installContainer.SecurityContext) require.NotNil(t, installContainer.SecurityContext.RunAsUser) - assert.Equal(t, int64(0), *installContainer.SecurityContext.RunAsUser) + assert.Equal(t, int64(1000), *installContainer.SecurityContext.RunAsUser) require.NotNil(t, installContainer.SecurityContext.RunAsNonRoot) - assert.False(t, *installContainer.SecurityContext.RunAsNonRoot) + assert.True(t, *installContainer.SecurityContext.RunAsNonRoot) + assert.Equal(t, int64(1000), *podSpec.SecurityContext.FSGroup) } func TestInjectIntoPodSpecRejectsMissingOrEmptyRegularInitImage(t *testing.T) { diff --git a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common/translator.go b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common/translator.go index 081f134060..a654d7dd91 100644 --- a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common/translator.go +++ b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common/translator.go @@ -237,6 +237,39 @@ func NewWorkloadContainerSecurityContext() *corev1.SecurityContext { } } +// WorkerInitUID is the numeric identity used by the worker-init image. +const WorkerInitUID int64 = 1000 + +// NewWorkerInitContainerSecurityContext matches the worker-init OCI user. +func NewWorkerInitContainerSecurityContext() *corev1.SecurityContext { + uid := WorkerInitUID + nonRoot := true + allowPrivilegeEscalation := false + return &corev1.SecurityContext{ + RunAsUser: &uid, + RunAsGroup: &uid, + RunAsNonRoot: &nonRoot, + AllowPrivilegeEscalation: &allowPrivilegeEscalation, + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + } +} + +// EnsureWorkerInitFSGroup makes pod volumes writable by worker-init while +// preserving an explicitly configured fsGroup. +func EnsureWorkerInitFSGroup(spec *corev1.PodSpec) { + if spec.SecurityContext == nil { + spec.SecurityContext = &corev1.PodSecurityContext{} + } + if spec.SecurityContext.FSGroup == nil { + group := WorkerInitUID + spec.SecurityContext.FSGroup = &group + if spec.SecurityContext.FSGroupChangePolicy == nil { + policy := corev1.FSGroupChangeOnRootMismatch + spec.SecurityContext.FSGroupChangePolicy = &policy + } + } +} + func NewInfraContainerSecurityContext() *corev1.SecurityContext { return &corev1.SecurityContext{ Capabilities: &corev1.Capabilities{ diff --git a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container.go b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container.go index 9312464396..91cac10a8f 100644 --- a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container.go +++ b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container.go @@ -392,10 +392,11 @@ func translateContainer(t CreationQueueMessage, tcfg TranslateConfig) (objs []me Image: initContainerImage, ImagePullPolicy: corev1.PullIfNotPresent, Env: common.SortEnvs(initEnvs), - SecurityContext: common.NewInfraContainerSecurityContext(), + SecurityContext: common.NewWorkerInitContainerSecurityContext(), VolumeMounts: initContainerVolumeMounts, } pod.Spec.InitContainers = append(pod.Spec.InitContainers, initContainer) + common.EnsureWorkerInitFSGroup(&pod.Spec) // The ESS init container needs to be added after the init container, // since the init container creates config.hcl. diff --git a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container_utdep.go b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container_utdep.go index a6fe3beed6..86248c66f2 100644 --- a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container_utdep.go +++ b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container_utdep.go @@ -233,10 +233,11 @@ func translateContainerUtilsDeploy(t CreationQueueMessage, tcfg TranslateConfig) Image: initContainerImage, ImagePullPolicy: corev1.PullIfNotPresent, Env: common.SortEnvs(initEnvs), - SecurityContext: common.NewInfraContainerSecurityContext(), + SecurityContext: common.NewWorkerInitContainerSecurityContext(), VolumeMounts: initContainerVolumeMounts, } inferencePod.Spec.InitContainers = append(inferencePod.Spec.InitContainers, initContainer) + common.EnsureWorkerInitFSGroup(&inferencePod.Spec) objs = append(objs, inferencePod) } @@ -517,10 +518,11 @@ func getUtilsDeploymentAndSecrets(t CreationQueueMessage, tcfg TranslateConfig) Image: initContainerImage, ImagePullPolicy: corev1.PullIfNotPresent, Env: common.SortEnvs(initEnvs), - SecurityContext: common.NewInfraContainerSecurityContext(), + SecurityContext: common.NewWorkerInitContainerSecurityContext(), VolumeMounts: initContainerVolumeMounts, } utilsPod.Spec.InitContainers = append(utilsPod.Spec.InitContainers, initContainer) + common.EnsureWorkerInitFSGroup(&utilsPod.Spec) // The ESS init container needs to be added after the init container, // since the init container creates config.hcl. diff --git a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm.go b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm.go index 54daa6539a..91fbc6ac9b 100644 --- a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm.go +++ b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm.go @@ -316,10 +316,11 @@ func translateHelmChart(t CreationQueueMessage, tcfg TranslateConfig) (objs []me Image: initContainerImage, ImagePullPolicy: corev1.PullIfNotPresent, Env: common.SortEnvs(initEnvs), - SecurityContext: common.NewInfraContainerSecurityContext(), + SecurityContext: common.NewWorkerInitContainerSecurityContext(), VolumeMounts: initContainerVolumeMounts, } utilsPod.Spec.InitContainers = append(utilsPod.Spec.InitContainers, initContainer) + common.EnsureWorkerInitFSGroup(&utilsPod.Spec) // The ESS init container needs to be added after the init container, // since the init container creates config.hcl. diff --git a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm_utdep.go b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm_utdep.go index c98e38d0a8..8571dd384e 100644 --- a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm_utdep.go +++ b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm_utdep.go @@ -288,10 +288,11 @@ func translateHelmChartUtilsDeploy(t CreationQueueMessage, tcfg TranslateConfig) Image: initContainerImage, ImagePullPolicy: corev1.PullIfNotPresent, Env: common.SortEnvs(initEnvs), - SecurityContext: common.NewInfraContainerSecurityContext(), + SecurityContext: common.NewWorkerInitContainerSecurityContext(), VolumeMounts: initContainerVolumeMounts, } utilsPod.Spec.InitContainers = append(utilsPod.Spec.InitContainers, initContainer) + common.EnsureWorkerInitFSGroup(&utilsPod.Spec) // The ESS init container needs to be added after the init container, // since the init container creates config.hcl. diff --git a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/task/translate.go b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/task/translate.go index 3bfd201656..7c3a67aec6 100644 --- a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/task/translate.go +++ b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/task/translate.go @@ -401,10 +401,11 @@ func translateContainer(t CreationQueueMessage, tcfg TranslateConfig) (objs []me Image: initContainerImage, ImagePullPolicy: corev1.PullIfNotPresent, Env: common.SortEnvs(initEnvs), - SecurityContext: common.NewInfraContainerSecurityContext(), + SecurityContext: common.NewWorkerInitContainerSecurityContext(), VolumeMounts: initContainerVolumeMounts, } pod.Spec.InitContainers = append(pod.Spec.InitContainers, initContainer) + common.EnsureWorkerInitFSGroup(&pod.Spec) // The ESS init container needs to be added after the init container, // since the init container creates config.hcl. diff --git a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/task/translate_helm.go b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/task/translate_helm.go index 36a30ee4e8..c928b89147 100644 --- a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/task/translate_helm.go +++ b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/task/translate_helm.go @@ -322,10 +322,11 @@ func translateHelmChart(t CreationQueueMessage, tcfg TranslateConfig) (objs []me Image: initContainerImage, ImagePullPolicy: corev1.PullIfNotPresent, Env: common.SortEnvs(initEnvs), - SecurityContext: common.NewInfraContainerSecurityContext(), + SecurityContext: common.NewWorkerInitContainerSecurityContext(), VolumeMounts: initContainerVolumeMounts, } utilsPod.Spec.InitContainers = append(utilsPod.Spec.InitContainers, initContainer) + common.EnsureWorkerInitFSGroup(&utilsPod.Spec) // The ESS init container needs to be added after the init container, // since the init container creates config.hcl. diff --git a/src/compute-plane-services/worker-init/README.md b/src/compute-plane-services/worker-init/README.md index d4a88a9e70..f85ab8ec6f 100644 --- a/src/compute-plane-services/worker-init/README.md +++ b/src/compute-plane-services/worker-init/README.md @@ -26,3 +26,14 @@ bazel test //src/compute-plane-services/worker-init/... # Or with the Go toolchain, from this directory go test ./... ``` + +## Runtime identity + +The image declares OCI user `1000:1000`. Generated function and task init +containers run with that UID and GID. Their pods use `fsGroup: 1000` when no +other group is configured, so worker-init can write shared volumes. A custom +`INIT_CONTAINER` image must support the same non-root identity and writable +paths. + +The transport trust bundle installer uses the same non-root identity. It writes +the merged CA bundle to an ephemeral volume before worker-init starts. diff --git a/src/compute-plane-services/worker-init/cmd/BUILD.bazel b/src/compute-plane-services/worker-init/cmd/BUILD.bazel index 8b4a77c250..3a01b1ec0c 100644 --- a/src/compute-plane-services/worker-init/cmd/BUILD.bazel +++ b/src/compute-plane-services/worker-init/cmd/BUILD.bazel @@ -51,6 +51,7 @@ go_oci_multi_binary_image( "//src/compute-plane-services/worker-init/cmd/nvcf-trust-bundle-install:nvcf-trust-bundle-install": "/usr/bin/nvcf-trust-bundle-install", }, entrypoint = ["/worker-init"], + user = "1000:1000", # SetupEssAgent reads these files when a function uses secrets. extra_layers = ["//src/compute-plane-services/worker-init/pkg/ess:ess_config_layer"], tags = ["nvcf-worker-init"], diff --git a/src/compute-plane-services/worker-init/cmd/image_contents_test.sh b/src/compute-plane-services/worker-init/cmd/image_contents_test.sh index 30e6102a08..6ce1ecb96f 100755 --- a/src/compute-plane-services/worker-init/cmd/image_contents_test.sh +++ b/src/compute-plane-services/worker-init/cmd/image_contents_test.sh @@ -20,6 +20,11 @@ if [[ -z "${config_rel_path}" || ! -f "${outer_dir}/${config_rel_path}" ]]; then exit 1 fi +if ! grep -Eq '"User"[[:space:]]*:[[:space:]]*"1000:1000"' "${outer_dir}/${config_rel_path}"; then + echo "worker-init image must declare OCI user 1000:1000" >&2 + exit 1 +fi + entrypoint="$(sed -n '/"Entrypoint"/,/\]/p' "${outer_dir}/${config_rel_path}" | tr -d '[:space:]')" case "${entrypoint}" in '"Entrypoint":["/worker-init"],'|'"Entrypoint":["/worker-init"]') ;; diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/common/translator.go b/src/libraries/go/lib/pkg/icms-translate/translate/common/translator.go index 081f134060..a654d7dd91 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/common/translator.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/common/translator.go @@ -237,6 +237,39 @@ func NewWorkloadContainerSecurityContext() *corev1.SecurityContext { } } +// WorkerInitUID is the numeric identity used by the worker-init image. +const WorkerInitUID int64 = 1000 + +// NewWorkerInitContainerSecurityContext matches the worker-init OCI user. +func NewWorkerInitContainerSecurityContext() *corev1.SecurityContext { + uid := WorkerInitUID + nonRoot := true + allowPrivilegeEscalation := false + return &corev1.SecurityContext{ + RunAsUser: &uid, + RunAsGroup: &uid, + RunAsNonRoot: &nonRoot, + AllowPrivilegeEscalation: &allowPrivilegeEscalation, + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + } +} + +// EnsureWorkerInitFSGroup makes pod volumes writable by worker-init while +// preserving an explicitly configured fsGroup. +func EnsureWorkerInitFSGroup(spec *corev1.PodSpec) { + if spec.SecurityContext == nil { + spec.SecurityContext = &corev1.PodSecurityContext{} + } + if spec.SecurityContext.FSGroup == nil { + group := WorkerInitUID + spec.SecurityContext.FSGroup = &group + if spec.SecurityContext.FSGroupChangePolicy == nil { + policy := corev1.FSGroupChangeOnRootMismatch + spec.SecurityContext.FSGroupChangePolicy = &policy + } + } +} + func NewInfraContainerSecurityContext() *corev1.SecurityContext { return &corev1.SecurityContext{ Capabilities: &corev1.Capabilities{ diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/common/translator_test.go b/src/libraries/go/lib/pkg/icms-translate/translate/common/translator_test.go index 38580f9256..9b1f5d9baa 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/common/translator_test.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/common/translator_test.go @@ -391,3 +391,24 @@ func TestNewCommonMetadata(t *testing.T) { }) } } + +func TestNewWorkerInitContainerSecurityContext(t *testing.T) { + ctx := NewWorkerInitContainerSecurityContext() + assert.Equal(t, WorkerInitUID, *ctx.RunAsUser) + assert.Equal(t, WorkerInitUID, *ctx.RunAsGroup) + assert.True(t, *ctx.RunAsNonRoot) + assert.False(t, *ctx.AllowPrivilegeEscalation) + assert.Equal(t, []corev1.Capability{"ALL"}, ctx.Capabilities.Drop) +} + +func TestEnsureWorkerInitFSGroup(t *testing.T) { + spec := &corev1.PodSpec{} + EnsureWorkerInitFSGroup(spec) + assert.Equal(t, WorkerInitUID, *spec.SecurityContext.FSGroup) + assert.Equal(t, corev1.FSGroupChangeOnRootMismatch, *spec.SecurityContext.FSGroupChangePolicy) + + otherGroup := int64(2000) + spec.SecurityContext.FSGroup = &otherGroup + EnsureWorkerInitFSGroup(spec) + assert.Equal(t, otherGroup, *spec.SecurityContext.FSGroup) +} diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container.go b/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container.go index 9312464396..91cac10a8f 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container.go @@ -392,10 +392,11 @@ func translateContainer(t CreationQueueMessage, tcfg TranslateConfig) (objs []me Image: initContainerImage, ImagePullPolicy: corev1.PullIfNotPresent, Env: common.SortEnvs(initEnvs), - SecurityContext: common.NewInfraContainerSecurityContext(), + SecurityContext: common.NewWorkerInitContainerSecurityContext(), VolumeMounts: initContainerVolumeMounts, } pod.Spec.InitContainers = append(pod.Spec.InitContainers, initContainer) + common.EnsureWorkerInitFSGroup(&pod.Spec) // The ESS init container needs to be added after the init container, // since the init container creates config.hcl. diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container_utdep.go b/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container_utdep.go index a6fe3beed6..86248c66f2 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container_utdep.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container_utdep.go @@ -233,10 +233,11 @@ func translateContainerUtilsDeploy(t CreationQueueMessage, tcfg TranslateConfig) Image: initContainerImage, ImagePullPolicy: corev1.PullIfNotPresent, Env: common.SortEnvs(initEnvs), - SecurityContext: common.NewInfraContainerSecurityContext(), + SecurityContext: common.NewWorkerInitContainerSecurityContext(), VolumeMounts: initContainerVolumeMounts, } inferencePod.Spec.InitContainers = append(inferencePod.Spec.InitContainers, initContainer) + common.EnsureWorkerInitFSGroup(&inferencePod.Spec) objs = append(objs, inferencePod) } @@ -517,10 +518,11 @@ func getUtilsDeploymentAndSecrets(t CreationQueueMessage, tcfg TranslateConfig) Image: initContainerImage, ImagePullPolicy: corev1.PullIfNotPresent, Env: common.SortEnvs(initEnvs), - SecurityContext: common.NewInfraContainerSecurityContext(), + SecurityContext: common.NewWorkerInitContainerSecurityContext(), VolumeMounts: initContainerVolumeMounts, } utilsPod.Spec.InitContainers = append(utilsPod.Spec.InitContainers, initContainer) + common.EnsureWorkerInitFSGroup(&utilsPod.Spec) // The ESS init container needs to be added after the init container, // since the init container creates config.hcl. diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm.go b/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm.go index 54daa6539a..91fbc6ac9b 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm.go @@ -316,10 +316,11 @@ func translateHelmChart(t CreationQueueMessage, tcfg TranslateConfig) (objs []me Image: initContainerImage, ImagePullPolicy: corev1.PullIfNotPresent, Env: common.SortEnvs(initEnvs), - SecurityContext: common.NewInfraContainerSecurityContext(), + SecurityContext: common.NewWorkerInitContainerSecurityContext(), VolumeMounts: initContainerVolumeMounts, } utilsPod.Spec.InitContainers = append(utilsPod.Spec.InitContainers, initContainer) + common.EnsureWorkerInitFSGroup(&utilsPod.Spec) // The ESS init container needs to be added after the init container, // since the init container creates config.hcl. diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm_utdep.go b/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm_utdep.go index c98e38d0a8..8571dd384e 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm_utdep.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm_utdep.go @@ -288,10 +288,11 @@ func translateHelmChartUtilsDeploy(t CreationQueueMessage, tcfg TranslateConfig) Image: initContainerImage, ImagePullPolicy: corev1.PullIfNotPresent, Env: common.SortEnvs(initEnvs), - SecurityContext: common.NewInfraContainerSecurityContext(), + SecurityContext: common.NewWorkerInitContainerSecurityContext(), VolumeMounts: initContainerVolumeMounts, } utilsPod.Spec.InitContainers = append(utilsPod.Spec.InitContainers, initContainer) + common.EnsureWorkerInitFSGroup(&utilsPod.Spec) // The ESS init container needs to be added after the init container, // since the init container creates config.hcl. diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_tolerations_test.go b/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_tolerations_test.go index e47b466105..b94acbe3f1 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_tolerations_test.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/function/translate_tolerations_test.go @@ -401,6 +401,9 @@ func TestTranslateContainerWithCacheAndSecrets(t *testing.T) { assert.Equal(t, "value", envSliceToMap(findContainerByName(t, pod.Spec.Containers, inferenceContainerName).Env)["INFERENCE_ENV"]) assert.Equal(t, "ess", findContainerByName(t, pod.Spec.Containers, "ess").Name) require.Len(t, pod.Spec.InitContainers, 2) + assert.Equal(t, common.WorkerInitUID, *pod.Spec.InitContainers[0].SecurityContext.RunAsUser) + assert.True(t, *pod.Spec.InitContainers[0].SecurityContext.RunAsNonRoot) + assert.Equal(t, common.WorkerInitUID, *pod.Spec.SecurityContext.FSGroup) assert.Equal(t, "ess-init", pod.Spec.InitContainers[1].Name) assert.NotNil(t, findObjectByName(t, objs, "rw-pvc-function-cache")) assert.NotNil(t, findObjectByName(t, objs, "writer-job-function-cache")) diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/task/translate.go b/src/libraries/go/lib/pkg/icms-translate/translate/task/translate.go index 3bfd201656..7c3a67aec6 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/task/translate.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/task/translate.go @@ -401,10 +401,11 @@ func translateContainer(t CreationQueueMessage, tcfg TranslateConfig) (objs []me Image: initContainerImage, ImagePullPolicy: corev1.PullIfNotPresent, Env: common.SortEnvs(initEnvs), - SecurityContext: common.NewInfraContainerSecurityContext(), + SecurityContext: common.NewWorkerInitContainerSecurityContext(), VolumeMounts: initContainerVolumeMounts, } pod.Spec.InitContainers = append(pod.Spec.InitContainers, initContainer) + common.EnsureWorkerInitFSGroup(&pod.Spec) // The ESS init container needs to be added after the init container, // since the init container creates config.hcl. diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/task/translate_all_test.go b/src/libraries/go/lib/pkg/icms-translate/translate/task/translate_all_test.go index d9d3d97920..fb60fe2855 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/task/translate_all_test.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/task/translate_all_test.go @@ -45,6 +45,9 @@ func TestTranslateContainerCreatesTaskPod(t *testing.T) { require.Len(t, pod.Spec.InitContainers, 1) assert.Equal(t, common.InitContainerName, pod.Spec.InitContainers[0].Name) assert.Equal(t, "nvcr.io/nvidia/init:latest", pod.Spec.InitContainers[0].Image) + assert.Equal(t, common.WorkerInitUID, *pod.Spec.InitContainers[0].SecurityContext.RunAsUser) + assert.True(t, *pod.Spec.InitContainers[0].SecurityContext.RunAsNonRoot) + assert.Equal(t, common.WorkerInitUID, *pod.Spec.SecurityContext.FSGroup) taskContainer := findTaskContainerByName(t, pod.Spec.Containers, taskContainerName) assert.Equal(t, "nvcr.io/nvidia/task:latest", taskContainer.Image) @@ -109,6 +112,9 @@ func TestTranslateHelmChartCreatesUtilsPod(t *testing.T) { require.Len(t, pod.Spec.InitContainers, 1) assert.Equal(t, common.InitContainerName, pod.Spec.InitContainers[0].Name) assert.Equal(t, "nvcr.io/nvidia/init:latest", pod.Spec.InitContainers[0].Image) + assert.Equal(t, common.WorkerInitUID, *pod.Spec.InitContainers[0].SecurityContext.RunAsUser) + assert.True(t, *pod.Spec.InitContainers[0].SecurityContext.RunAsNonRoot) + assert.Equal(t, common.WorkerInitUID, *pod.Spec.SecurityContext.FSGroup) utilsContainer := findTaskContainerByName(t, pod.Spec.Containers, common.UtilsContainerName) assertTaskUtilsMetricsPort(t, utilsContainer) diff --git a/src/libraries/go/lib/pkg/icms-translate/translate/task/translate_helm.go b/src/libraries/go/lib/pkg/icms-translate/translate/task/translate_helm.go index 36a30ee4e8..c928b89147 100644 --- a/src/libraries/go/lib/pkg/icms-translate/translate/task/translate_helm.go +++ b/src/libraries/go/lib/pkg/icms-translate/translate/task/translate_helm.go @@ -322,10 +322,11 @@ func translateHelmChart(t CreationQueueMessage, tcfg TranslateConfig) (objs []me Image: initContainerImage, ImagePullPolicy: corev1.PullIfNotPresent, Env: common.SortEnvs(initEnvs), - SecurityContext: common.NewInfraContainerSecurityContext(), + SecurityContext: common.NewWorkerInitContainerSecurityContext(), VolumeMounts: initContainerVolumeMounts, } utilsPod.Spec.InitContainers = append(utilsPod.Spec.InitContainers, initContainer) + common.EnsureWorkerInitFSGroup(&utilsPod.Spec) // The ESS init container needs to be added after the init container, // since the init container creates config.hcl. From dbcdfc38d5367814cdfaecf12fbfa7b96989350b Mon Sep 17 00:00:00 2001 From: Stephanie Baum Date: Wed, 23 Sep 2026 10:14:06 -0700 Subject: [PATCH 2/5] fix(self-managed): set non-root runtime identities Set numeric image users where this repository builds the image, and enforce non-root security contexts for chart-managed workloads and migration hooks. Refs: #2071 Signed-off-by: Stephanie Baum --- .../api-keys/templates/deployment.yaml | 3 ++ .../templates/account-bootstrap-hook-job.yaml | 11 ++++++ .../tests/account_bootstrap_image_test.sh | 2 + .../templates/hook-lls-migrations.yaml | 9 +++++ .../templates/hook-llm-migrations.yaml | 9 +++++ .../nats/scripts/test-render-auth-callout.sh | 6 +++ deploy/helm/nats/values.yaml | 25 +++++++++++++ .../templates/hook-post-01-initcluster.yaml | 9 +++++ ...ok-post-01-refresh-jwt-plugin-catalog.yaml | 9 +++++ .../templates/hook-post-02-migrations.yaml | 9 +++++ .../tests/plugin-catalog-refresh-hook.sh | 5 +++ deploy/stacks/self-managed/Makefile | 1 + .../nvcf-upgrade-receipt/templates/job.yaml | 11 ++++++ deploy/stacks/self-managed/global.yaml.gotmpl | 26 +++++++++++-- .../tests/api-keys-startup-probe.sh | 21 +++++++++++ .../tests/image-override-wiring.sh | 4 ++ .../tests/nonroot-migration-addons.sh | 37 +++++++++++++++++++ .../tests/upgrade-receipt-wiring.sh | 2 + infra/openbao/Dockerfile | 6 +-- migrations/openbao/Dockerfile | 2 +- rules/oci/private/java.bzl | 15 ++++++-- rules/oci/private/java_image_contract_test.sh | 6 +++ .../api-keys/BUILD.bazel | 2 + .../api-keys/Dockerfile | 1 + .../templates/hook-openbao-migrations.yaml | 4 ++ src/uis/nvcf-ui/helm/values.yaml | 14 ++++--- 26 files changed, 233 insertions(+), 16 deletions(-) create mode 100755 deploy/stacks/self-managed/tests/nonroot-migration-addons.sh diff --git a/deploy/helm/api-keys-colocated/api-keys/templates/deployment.yaml b/deploy/helm/api-keys-colocated/api-keys/templates/deployment.yaml index 23555f83d7..ff5abefe02 100644 --- a/deploy/helm/api-keys-colocated/api-keys/templates/deployment.yaml +++ b/deploy/helm/api-keys-colocated/api-keys/templates/deployment.yaml @@ -84,6 +84,9 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} securityContext: + runAsNonRoot: true + runAsGroup: 1000 + allowPrivilegeEscalation: false capabilities: drop: - ALL diff --git a/deploy/helm/cloud-functions/nvcf-api/templates/account-bootstrap-hook-job.yaml b/deploy/helm/cloud-functions/nvcf-api/templates/account-bootstrap-hook-job.yaml index 210bd80618..e33a2e43db 100644 --- a/deploy/helm/cloud-functions/nvcf-api/templates/account-bootstrap-hook-job.yaml +++ b/deploy/helm/cloud-functions/nvcf-api/templates/account-bootstrap-hook-job.yaml @@ -39,6 +39,10 @@ spec: app.kubernetes.io/component: account-bootstrap spec: restartPolicy: OnFailure + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 serviceAccountName: nvcf-api-account-bootstrap {{- with .Values.api.accountBootstrap.schedulingGates }} schedulingGates: @@ -64,6 +68,13 @@ spec: - name: account-bootstrap-creator image: "{{ include "nvcf-api.accountBootstrapImage" . }}" imagePullPolicy: {{ .Values.api.accountBootstrap.image.pullPolicy }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] command: ["/bin/bash"] args: ["/scripts/account-bootstrap.sh"] volumeMounts: diff --git a/deploy/helm/cloud-functions/tests/account_bootstrap_image_test.sh b/deploy/helm/cloud-functions/tests/account_bootstrap_image_test.sh index 092df98656..b2a9a645a2 100755 --- a/deploy/helm/cloud-functions/tests/account_bootstrap_image_test.sh +++ b/deploy/helm/cloud-functions/tests/account_bootstrap_image_test.sh @@ -32,6 +32,8 @@ assert_image() { render_job "$work_dir/default.yaml" assert_image "$work_dir/default.yaml" "docker.io/alpine/k8s:1.37.0" +test "$(yq -r '.spec.template.spec.containers[0].securityContext.runAsNonRoot' "$work_dir/default.yaml")" = true +test "$(yq -r '.spec.template.spec.containers[0].securityContext.runAsUser' "$work_dir/default.yaml")" = 1000 render_job "$work_dir/partial-override.yaml" \ --set-string api.accountBootstrap.image.repository=mirror/alpine-k8s diff --git a/deploy/helm/icms/icms-api/templates/hook-lls-migrations.yaml b/deploy/helm/icms/icms-api/templates/hook-lls-migrations.yaml index ff1b75d78d..e56071ea83 100644 --- a/deploy/helm/icms/icms-api/templates/hook-lls-migrations.yaml +++ b/deploy/helm/icms/icms-api/templates/hook-lls-migrations.yaml @@ -35,6 +35,10 @@ spec: {{- end }} spec: restartPolicy: OnFailure + securityContext: + runAsNonRoot: true + runAsUser: 100 + runAsGroup: 1000 serviceAccountName: {{ .Values.sis.lls.hmacRotation.serviceAccountName }} {{- with (include "sis.imagePullSecrets" .) }} imagePullSecrets: @@ -44,6 +48,11 @@ spec: - name: addons-lls-migrations image: "{{ include "sis.image.full" (dict "image" .Values.sis.lls.hmacRotation.image "name" "sis.lls.hmacRotation.image") }}" imagePullPolicy: {{ .Values.sis.lls.hmacRotation.image.pullPolicy }} + securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] env: - name: CORE_MIGRATIONS_ENABLED value: "false" diff --git a/deploy/helm/llm-request-router/llm-request-router/templates/hook-llm-migrations.yaml b/deploy/helm/llm-request-router/llm-request-router/templates/hook-llm-migrations.yaml index 522989f04a..4b231d2a5d 100644 --- a/deploy/helm/llm-request-router/llm-request-router/templates/hook-llm-migrations.yaml +++ b/deploy/helm/llm-request-router/llm-request-router/templates/hook-llm-migrations.yaml @@ -44,6 +44,10 @@ spec: {{- end }} spec: restartPolicy: OnFailure + securityContext: + runAsNonRoot: true + runAsUser: 100 + runAsGroup: 1000 serviceAccountName: {{ $pki.serviceAccountName }} {{- with $pki.imagePullSecrets }} imagePullSecrets: @@ -53,6 +57,11 @@ spec: - name: addons-llm-migrations image: "{{ include "llm-request-router.pkiMigrationsImage" . }}" imagePullPolicy: {{ $pki.image.pullPolicy }} + securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] env: - name: CORE_MIGRATIONS_ENABLED value: "false" diff --git a/deploy/helm/nats/scripts/test-render-auth-callout.sh b/deploy/helm/nats/scripts/test-render-auth-callout.sh index d49ff9f722..9f54cc188a 100755 --- a/deploy/helm/nats/scripts/test-render-auth-callout.sh +++ b/deploy/helm/nats/scripts/test-render-auth-callout.sh @@ -59,6 +59,12 @@ assert_not_contains() { default_render="$tmpdir/default.yaml" render > "$default_render" +test "$(yq -r 'select(.kind == "StatefulSet") | .spec.template.spec.securityContext.runAsUser' "$default_render")" = 1000 +test "$(yq -r 'select(.kind == "StatefulSet") | .spec.template.spec.securityContext.fsGroup' "$default_render")" = 1000 +for container in nats reloader; do + test "$(yq -r "select(.kind == \"StatefulSet\") | .spec.template.spec.containers[] | select(.name == \"$container\") | .securityContext.runAsNonRoot" "$default_render")" = true + test "$(yq -r "select(.kind == \"StatefulSet\") | .spec.template.spec.containers[] | select(.name == \"$container\") | .securityContext.runAsUser" "$default_render")" = 1000 +done reloader_image="$(yq -r '.nats.reloader.image.registry + "/" + .nats.reloader.image.repository + ":" + .nats.reloader.image.tag' "$chart_dir/values.yaml")" assert_contains "$default_render" "image: $reloader_image" assert_contains "$default_render" "# Source: helm-nvcf-nats/templates/nats-auth-callout-nkeys-secret.yaml" diff --git a/deploy/helm/nats/values.yaml b/deploy/helm/nats/values.yaml index 77ef3f576b..36a90a3ac1 100644 --- a/deploy/helm/nats/values.yaml +++ b/deploy/helm/nats/values.yaml @@ -30,6 +30,14 @@ nats: registry: "" pullPolicy: IfNotPresent container: + merge: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] image: registry: "" repository: "" @@ -77,6 +85,14 @@ nats: value: "/healthz?js-server-only=true" reloader: enabled: true + merge: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] image: registry: docker.io repository: natsio/nats-server-config-reloader @@ -96,6 +112,15 @@ nats: nkey: secretName: nats-nkeys key: user.key + podTemplate: + merge: + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch # Auth callout NKey bootstrap. # The default secret name is shared with openbao-migrations; override it only # when every downstream consumer is configured to read the same Secret. diff --git a/deploy/helm/openbao/helm/templates/hook-post-01-initcluster.yaml b/deploy/helm/openbao/helm/templates/hook-post-01-initcluster.yaml index 086eb853f4..2e98da6909 100644 --- a/deploy/helm/openbao/helm/templates/hook-post-01-initcluster.yaml +++ b/deploy/helm/openbao/helm/templates/hook-post-01-initcluster.yaml @@ -37,6 +37,10 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: + securityContext: + runAsNonRoot: true + runAsUser: 100 + runAsGroup: 1000 {{- with .Values.openbao.global.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} @@ -48,6 +52,11 @@ spec: containers: - name: bao-init image: "{{ include "openbao.image" (dict "image" .Values.openbao.migrations.image "name" "openbao.migrations.image") }}" + securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] command: ["/bin/bash"] args: - -c diff --git a/deploy/helm/openbao/helm/templates/hook-post-01-refresh-jwt-plugin-catalog.yaml b/deploy/helm/openbao/helm/templates/hook-post-01-refresh-jwt-plugin-catalog.yaml index 1e885b985d..8c561b7122 100644 --- a/deploy/helm/openbao/helm/templates/hook-post-01-refresh-jwt-plugin-catalog.yaml +++ b/deploy/helm/openbao/helm/templates/hook-post-01-refresh-jwt-plugin-catalog.yaml @@ -34,6 +34,10 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: + securityContext: + runAsNonRoot: true + runAsUser: 100 + runAsGroup: 1000 automountServiceAccountToken: false {{- with .Values.openbao.global.imagePullSecrets }} imagePullSecrets: @@ -50,6 +54,11 @@ spec: # registered before the OnDelete StatefulSet is manually rotated. image: "{{ include "openbao.image" (dict "image" .Values.openbao.server.image "name" "openbao.server.image") }}" imagePullPolicy: {{ .Values.openbao.server.image.pullPolicy }} + securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] command: ["/bin/sh", "-ec"] args: - |- diff --git a/deploy/helm/openbao/helm/templates/hook-post-02-migrations.yaml b/deploy/helm/openbao/helm/templates/hook-post-02-migrations.yaml index d0d7f54295..fe57d534fe 100644 --- a/deploy/helm/openbao/helm/templates/hook-post-02-migrations.yaml +++ b/deploy/helm/openbao/helm/templates/hook-post-02-migrations.yaml @@ -30,6 +30,10 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: + securityContext: + runAsNonRoot: true + runAsUser: 100 + runAsGroup: 1000 {{- with .Values.openbao.global.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} @@ -45,6 +49,11 @@ spec: containers: - name: bao-migrations image: "{{ include "openbao.image" (dict "image" .Values.openbao.migrations.image "name" "openbao.migrations.image") }}" + securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] imagePullPolicy: {{ .Values.openbao.migrations.image.pullPolicy }} volumeMounts: - name: {{ $serverFullname }}-root-token diff --git a/deploy/helm/openbao/tests/plugin-catalog-refresh-hook.sh b/deploy/helm/openbao/tests/plugin-catalog-refresh-hook.sh index 5286282c13..ebc4f712c0 100644 --- a/deploy/helm/openbao/tests/plugin-catalog-refresh-hook.sh +++ b/deploy/helm/openbao/tests/plugin-catalog-refresh-hook.sh @@ -22,6 +22,11 @@ helm dependency build "${chart_dir}" >/dev/null rendered="${test_root}/rendered.yaml" helm template openbao "${chart_dir}" -f "${values_file}" >"${rendered}" +for hook in openbao-server-initialize-cluster openbao-server-refresh-jwt-plugin-catalog openbao-server-migrations; do + test "$(yq -r "select(.kind == \"Job\" and .metadata.name == \"$hook\") | .spec.template.spec.securityContext.runAsNonRoot" "${rendered}")" = true + test "$(yq -r "select(.kind == \"Job\" and .metadata.name == \"$hook\") | .spec.template.spec.securityContext.runAsUser" "${rendered}")" = 100 +done + # Helm upgrades using --reuse-values do not add newly introduced default # subtrees. Rendering must remain compatible with persisted pre-change values. legacy_values="${test_root}/legacy-values.yaml" diff --git a/deploy/stacks/self-managed/Makefile b/deploy/stacks/self-managed/Makefile index 4b23b58b44..b8a772f4da 100644 --- a/deploy/stacks/self-managed/Makefile +++ b/deploy/stacks/self-managed/Makefile @@ -33,6 +33,7 @@ test: @tests/nats-auth-callout-wiring.sh @tests/notary-image-repository.sh @tests/nvcf-ui-migrations-image-tag.sh + @tests/nonroot-migration-addons.sh test-published-charts: @: "$${NVCF_PUBLISHED_CHART_REGISTRY:?NVCF_PUBLISHED_CHART_REGISTRY is required}" diff --git a/deploy/stacks/self-managed/charts/nvcf-upgrade-receipt/templates/job.yaml b/deploy/stacks/self-managed/charts/nvcf-upgrade-receipt/templates/job.yaml index 3b2e6848b0..f6f0254a24 100644 --- a/deploy/stacks/self-managed/charts/nvcf-upgrade-receipt/templates/job.yaml +++ b/deploy/stacks/self-managed/charts/nvcf-upgrade-receipt/templates/job.yaml @@ -25,6 +25,10 @@ spec: name: {{ .Release.Name }} spec: restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 serviceAccountName: {{ .Release.Name }} {{- with .Values.imagePullSecrets }} imagePullSecrets: @@ -34,6 +38,13 @@ spec: - name: receipt image: "{{ with .Values.image.registry }}{{ . }}/{{ end }}{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] env: - name: RECEIPT_CONFIGMAP value: {{ .Values.configMapName | quote }} diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl index 0640092542..03eed70cbf 100644 --- a/deploy/stacks/self-managed/global.yaml.gotmpl +++ b/deploy/stacks/self-managed/global.yaml.gotmpl @@ -225,6 +225,14 @@ nats: {{- end }} {{- end }} container: + merge: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] image: registry: {{ .Values.global.image.registry }} repository: {{ .Values.global.image.repository }}/nats-server @@ -256,18 +264,22 @@ nats: registry: {{ .Values.global.image.registry }} repository: {{ .Values.global.image.repository }}/alpine-k8s {{- $natsNs := include "nvcf.nodeSelector" (dict "type" "controlplane" "selectors" .Values.global.nodeSelectors) -}} - {{- $natsTol := include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) -}} - {{- if or $natsNs $natsTol }} + {{- $natsTol := include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }} podTemplate: merge: spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch {{- with $natsNs }} {{- . | nindent 8 }} {{- end }} {{- with $natsTol }} {{- . | nindent 8 }} {{- end }} - {{- end }} # One merged config: block -- a second config: key under nats: would be # silently collapsed. Surfaces server TLS for split-plane workers that reach @@ -324,6 +336,14 @@ apikeys: startupProbe: {{- toYaml . | nindent 4 }} {{- end }} + {{- with dig "apikeys" "livenessProbe" dict .Values }} + livenessProbe: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with dig "apikeys" "resources" dict .Values }} + resources: + {{- toYaml . | nindent 4 }} + {{- end }} {{- with dig "apikeys" "env" dict .Values }} env: {{- toYaml . | nindent 4 }} diff --git a/deploy/stacks/self-managed/tests/api-keys-startup-probe.sh b/deploy/stacks/self-managed/tests/api-keys-startup-probe.sh index 6645bb8800..3a0b14ac87 100755 --- a/deploy/stacks/self-managed/tests/api-keys-startup-probe.sh +++ b/deploy/stacks/self-managed/tests/api-keys-startup-probe.sh @@ -19,6 +19,11 @@ printf '%s\n' \ 'apikeys:' \ ' startupProbe:' \ ' failureThreshold: 60' \ + ' livenessProbe:' \ + ' timeoutSeconds: 5' \ + ' resources:' \ + ' limits:' \ + ' cpu: 1' \ >"$test_stack_dir/environments/$environment_name.yaml" values_file="$work_dir/api-keys-values.yaml" @@ -33,6 +38,8 @@ HELMFILE_ENV="$environment_name" \ --state-values-set ingress.gatewayApi.gateways.grpc.name=grpc-gw \ --state-values-set ingress.gatewayApi.gateways.grpc.namespace=envoy-gateway-system \ --state-values-set apikeys.startupProbe.failureThreshold=60 \ + --state-values-set apikeys.livenessProbe.timeoutSeconds=5 \ + --state-values-set apikeys.resources.limits.cpu=1 \ --selector name=api-keys \ write-values \ --output-file-template "$values_file" @@ -40,5 +47,19 @@ HELMFILE_ENV="$environment_name" \ actual="$(yq -r '.apikeys.startupProbe.failureThreshold // "missing"' "$values_file")" test "$actual" = "60" || fail "expected apikeys.startupProbe.failureThreshold=60, got $actual" +test "$(yq -r '.apikeys.livenessProbe.timeoutSeconds // "missing"' "$values_file")" = "5" || + fail "API Keys liveness timeout override was not forwarded" +test "$(yq -r '.apikeys.resources.limits.cpu // "missing"' "$values_file")" = "1" || + fail "API Keys CPU limit override was not forwarded" + +chart_dir="$stack_dir/../../helm/api-keys-colocated/api-keys" +manifest="$work_dir/api-keys.yaml" +helm template api-keys "$chart_dir" --namespace api-keys \ + --set apikeys.image.registry=example.com \ + --set apikeys.image.repository=api-keys >"$manifest" +test "$(yq -r 'select(.kind == "Deployment") | .spec.template.spec.containers[0].securityContext.runAsNonRoot' "$manifest")" = true || + fail "API Keys container must run as non-root" +test "$(yq -r 'select(.kind == "Deployment") | .spec.template.spec.containers[0].securityContext.runAsUser' "$manifest")" = 1000 || + fail "API Keys container must use UID 1000" echo "api-keys-startup-probe: all checks passed" diff --git a/deploy/stacks/self-managed/tests/image-override-wiring.sh b/deploy/stacks/self-managed/tests/image-override-wiring.sh index cc97518a3c..97acbc1be3 100755 --- a/deploy/stacks/self-managed/tests/image-override-wiring.sh +++ b/deploy/stacks/self-managed/tests/image-override-wiring.sh @@ -162,6 +162,10 @@ global: EOF render_values "$work_dir/default-values.yaml" +test "$(yq -r '.nats.podTemplate.merge.spec.securityContext.runAsUser' "$work_dir/default-values.yaml")" = 1000 || + fail "nats pod user must be non-root in stack values" +test "$(yq -r '.nats.container.merge.securityContext.runAsNonRoot' "$work_dir/default-values.yaml")" = true || + fail "nats container must run as non-root in stack values" assert_absent "$work_dir/default-values.yaml" \ natsio/nats-server-config-reloader "nats.reloader chart default" assert_yaml_path_absent "$work_dir/default-values.yaml" \ diff --git a/deploy/stacks/self-managed/tests/nonroot-migration-addons.sh b/deploy/stacks/self-managed/tests/nonroot-migration-addons.sh new file mode 100755 index 0000000000..8b441cc2b2 --- /dev/null +++ b/deploy/stacks/self-managed/tests/nonroot-migration-addons.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" +work_dir="$(mktemp -d)" +trap 'rm -rf "$work_dir"' EXIT + +helm template sis "$repo_root/deploy/helm/icms/icms-api" -n sis \ + --set sis.image.registry=example.com \ + --set sis.image.repository=sis \ + --set sis.lls.enabled=true \ + --set sis.lls.hmacRotation.image.registry=example.com \ + --set sis.lls.hmacRotation.image.repository=nvcf-openbao-migrations \ + --set sis.lls.hmacRotation.image.tag=test >"$work_dir/sis.yaml" + +helm template llm "$repo_root/deploy/helm/llm-request-router/llm-request-router" -n nvcf \ + --set llmRequestRouter.image.registry=example.com \ + --set llmRequestRouter.image.repository=llm \ + --set llmRequestRouter.pki.enabled=true \ + --set llmRequestRouter.pki.allowedDomains=cluster.local \ + --set llmRequestRouter.pki.image.registry=example.com \ + --set llmRequestRouter.pki.image.repository=nvcf-openbao-migrations \ + --set llmRequestRouter.pki.image.tag=test >"$work_dir/llm.yaml" + +helm template ui "$repo_root/src/uis/nvcf-ui/helm" -n nvcf \ + --set nvcfUi.image.registry=example.com \ + --set nvcfUi.openbaoMigrations.image.registry=example.com >"$work_dir/ui.yaml" + +for pair in sis:addons-lls-migrations llm:addons-llm-migrations ui:ui-helm-nvcf-ui-openbao-migrations; do + file="${pair%%:*}" + name="${pair#*:}" + selector="select(.kind == \"Job\" and .metadata.name == \"$name\")" + test "$(yq -r "$selector | .spec.template.spec.securityContext.runAsUser" "$work_dir/$file.yaml")" = 100 + test "$(yq -r "$selector | .spec.template.spec.containers[0].securityContext.runAsNonRoot" "$work_dir/$file.yaml")" = true +done + +echo "nonroot-migration-addons: all checks passed" diff --git a/deploy/stacks/self-managed/tests/upgrade-receipt-wiring.sh b/deploy/stacks/self-managed/tests/upgrade-receipt-wiring.sh index f59ec3bc80..7af4a6e8e0 100755 --- a/deploy/stacks/self-managed/tests/upgrade-receipt-wiring.sh +++ b/deploy/stacks/self-managed/tests/upgrade-receipt-wiring.sh @@ -21,6 +21,8 @@ rendered="$(cd "$stack_dir" && HELMFILE_ENV=base helmfile \ fail() { echo "FAIL: $1" >&2; exit 1; } grep -q 'kind: Job' <<<"$rendered" || fail "no Job rendered" +test "$(yq -r 'select(.kind == "Job") | .spec.template.spec.containers[0].securityContext.runAsNonRoot' <<<"$rendered")" = true \ + || fail "receipt must run as non-root" grep -q '"helm.sh/hook": post-install,post-upgrade' <<<"$rendered" \ || fail "receipt must run on both install and upgrade, after the release it describes" grep -q "value: \"${expected_version}\"" <<<"$rendered" \ diff --git a/infra/openbao/Dockerfile b/infra/openbao/Dockerfile index 51d15f92d1..0a4d75c459 100644 --- a/infra/openbao/Dockerfile +++ b/infra/openbao/Dockerfile @@ -49,8 +49,8 @@ RUN GOOS=linux GOARCH="${TARGETARCH}" CGO_ENABLED=0 \ FROM ${BAO_RUNTIME_IMAGE} ARG TARGETARCH -# OpenBao 2.6.2 runs as the non-root openbao user by default. Escalate only -# while installing the image's runtime packages, then restore that identity. +# OpenBao 2.6.2 uses UID 100 and GID 1000. Escalate only while installing +# runtime packages, then restore that identity as numeric OCI metadata. USER root # Upgrade packages inherited from the upstream image before installing the # runtime tools. This ensures security fixes from the pinned Alpine branch are @@ -63,4 +63,4 @@ RUN apk upgrade --no-cache && \ COPY --from=bao-build --chmod=0555 /out/bao-linux-${TARGETARCH} /usr/bin/bao COPY --from=plugin-build --chmod=0555 /out/vault-plugin-secrets-jwt /openbao/plugins/vault-plugin-secrets-jwt -USER openbao +USER 100:1000 diff --git a/migrations/openbao/Dockerfile b/migrations/openbao/Dockerfile index d79e22b4b3..36e60fa4d1 100644 --- a/migrations/openbao/Dockerfile +++ b/migrations/openbao/Dockerfile @@ -116,6 +116,6 @@ COPY --chmod=775 migrations/ /app/migrations/ COPY --chmod=775 addons/ /app/addons/ COPY --chmod=775 entrypoint.sh /app/ -USER openbao +USER 100:1000 CMD ["/app/entrypoint.sh"] diff --git a/rules/oci/private/java.bzl b/rules/oci/private/java.bzl index 8c7dea51e1..24ba153b5b 100644 --- a/rules/oci/private/java.bzl +++ b/rules/oci/private/java.bzl @@ -41,7 +41,7 @@ ULIMIT_SHIM = "/usr/bin/shelless_ulimit" # /usr/bin/java is present on both arches. JAVA_BIN = "/usr/bin/java" -def _java_oci_image_impl(name, visibility, jar, base, jar_path, java_bin, entrypoint, jvm_flags, env, workdir, registry, tags): +def _java_oci_image_impl(name, visibility, jar, base, jar_path, java_bin, entrypoint, jvm_flags, env, workdir, user, registry, tags): layer_name = name + "_layer" files_name = name + "_files" @@ -85,6 +85,7 @@ def _java_oci_image_impl(name, visibility, jar, base, jar_path, java_bin, entryp entrypoint = entry, env = env, workdir = workdir, + user = user, visibility = visibility, registry = registry, tags = tags, @@ -139,6 +140,10 @@ java_oci_image = macro( doc = "Container working directory. Unset leaves the base's value.", configurable = False, ), + "user": attr.string( + doc = "Runtime UID:GID. Unset leaves the base image's user unchanged.", + configurable = False, + ), "jvm_flags": attr.string_list( doc = "JVM flags inserted before -jar. Ignored when entrypoint is set.", configurable = False, @@ -161,7 +166,7 @@ java_oci_image = macro( }, ) -def _java_image_contract_test_impl(name, visibility, image, jar_path, env, workdir, **kwargs): +def _java_image_contract_test_impl(name, visibility, image, jar_path, env, workdir, user, **kwargs): sh_test( name = name, srcs = ["//rules/oci/private:java_image_contract_test.sh"], @@ -176,7 +181,7 @@ def _java_image_contract_test_impl(name, visibility, image, jar_path, env, workd # JDK_JAVA_OPTIONS would arrive truncated at its first space and the # test would silently assert only the first fragment. env = dict( - [("EXPECT_ENV_COUNT", str(len(env)))] + + [("EXPECT_ENV_COUNT", str(len(env))), ("EXPECT_USER", user)] + [ ("EXPECT_ENV_{}".format(i), "{}={}".format(k, env[k])) for i, k in enumerate(sorted(env.keys())) @@ -224,5 +229,9 @@ java_image_contract_test = macro( default = "/home/app", doc = "Expected working directory.", ), + "user": attr.string( + configurable = False, + doc = "Expected runtime UID:GID when the image sets one.", + ), }, ) diff --git a/rules/oci/private/java_image_contract_test.sh b/rules/oci/private/java_image_contract_test.sh index 7d897b67e4..a9162cbb66 100755 --- a/rules/oci/private/java_image_contract_test.sh +++ b/rules/oci/private/java_image_contract_test.sh @@ -185,6 +185,12 @@ assert_config() { # assert_config