From 7b88e4e054b3aff43b70ac3a2fa2f70bf3c3a794 Mon Sep 17 00:00:00 2001 From: chokevin Date: Tue, 20 Oct 2026 14:37:45 -0700 Subject: [PATCH 01/36] feat(signal): add generic certification controller --- .dockerignore | 1 + Makefile | 58 +- .../v1alpha1/certificationprofile_types.go | 126 ++ api/signal/v1alpha1/certificationrun_types.go | 134 ++ api/signal/v1alpha1/groupversion_info.go | 9 + api/signal/v1alpha1/register.go | 29 + api/signal/v1alpha1/zz_generated.deepcopy.go | 528 ++++++ cmd/project-signal-controller/main.go | 160 ++ deploy/signal/00-namespace.yaml.tmpl | 6 + deploy/signal/01-serviceaccounts.yaml.tmpl | 8 + deploy/signal/02-rbac.yaml.tmpl | 107 ++ deploy/signal/06-deployment.yaml.tmpl | 50 + deploy/signal/README.md | 169 ++ ...ounded-cloud.io_certificationprofiles.yaml | 391 +++++ ....unbounded-cloud.io_certificationruns.yaml | 366 +++++ deploy/signal/manifest_test.go | 60 + deploy/signal/rendered/.gitignore | 2 + .../project-signal-controller/Containerfile | 29 + internal/signal/controller.go | 1409 +++++++++++++++++ internal/signal/controller_test.go | 1122 +++++++++++++ internal/signal/evidence.go | 107 ++ internal/signal/evidence_test.go | 78 + internal/signal/policy.go | 623 ++++++++ internal/signal/policy_test.go | 264 +++ internal/signal/projection.go | 249 +++ internal/signal/projection_test.go | 227 +++ 26 files changed, 6303 insertions(+), 9 deletions(-) create mode 100644 api/signal/v1alpha1/certificationprofile_types.go create mode 100644 api/signal/v1alpha1/certificationrun_types.go create mode 100644 api/signal/v1alpha1/groupversion_info.go create mode 100644 api/signal/v1alpha1/register.go create mode 100644 api/signal/v1alpha1/zz_generated.deepcopy.go create mode 100644 cmd/project-signal-controller/main.go create mode 100644 deploy/signal/00-namespace.yaml.tmpl create mode 100644 deploy/signal/01-serviceaccounts.yaml.tmpl create mode 100644 deploy/signal/02-rbac.yaml.tmpl create mode 100644 deploy/signal/06-deployment.yaml.tmpl create mode 100644 deploy/signal/README.md create mode 100644 deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml create mode 100644 deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml create mode 100644 deploy/signal/manifest_test.go create mode 100644 deploy/signal/rendered/.gitignore create mode 100644 images/project-signal-controller/Containerfile create mode 100644 internal/signal/controller.go create mode 100644 internal/signal/controller_test.go create mode 100644 internal/signal/evidence.go create mode 100644 internal/signal/evidence_test.go create mode 100644 internal/signal/policy.go create mode 100644 internal/signal/policy_test.go create mode 100644 internal/signal/projection.go create mode 100644 internal/signal/projection_test.go diff --git a/.dockerignore b/.dockerignore index 9bbe7a5f07..5d20bc8b6b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,6 +6,7 @@ __debug* tmp/ .tmp/ dist/ +deploy/signal/rendered/ # Docker .dockerignore diff --git a/Makefile b/Makefile index 10bfc1e0bb..f401adc6b9 100644 --- a/Makefile +++ b/Makefile @@ -92,6 +92,13 @@ MACHINE_OPS_CONTROLLER_NAME ?= machine-ops-controller MACHINE_OPS_PROVIDER ?= MACHINE_OPS_SITE ?= +PROJECT_SIGNAL_CONTROLLER_BIN=bin/project-signal-controller +PROJECT_SIGNAL_CONTROLLER_CMD=./cmd/project-signal-controller +SIGNAL_CONTROLLER_IMAGE ?= example.invalid/project-signal-controller:replace-me +SIGNAL_NAMESPACE ?= project-signal +SIGNAL_MANIFEST_TEMPLATES_DIR := deploy/signal +SIGNAL_MANIFEST_RENDERED_DIR := deploy/signal/rendered + METALMAN_BIN=bin/metalman METALMAN_CMD=./cmd/metalman NETBOOT_IMAGE ?= $(CONTAINER_REGISTRY)/netboot:$(VERSION_TAG) @@ -230,14 +237,14 @@ NET_FRONTEND_CACHE_FILE := $(NET_FRONTEND_DIST_DIR)/.frontend-build-key # Frontend build toggle (dev builds produce unminified output with sourcemaps). REACT_DEV ?= false -.PHONY: all help fmt lint lint-actions test build vulncheck check-deps kubectl-unbounded kubectl-unbounded-build install-tools install-protoc install-helm generate kubectl-unbounded forge relctl relctl-build agent-artifacts-builder agent-artifacts-builder-build orcadev unbounded-agent machina machina-build machina-oci machina-oci-push machina-manifests machine-ops-controller machine-ops-controller-build machine-ops-controller-oci machine-ops-controller-oci-push machine-ops-manifests metalman metalman-build metalman-oci metalman-oci-push unbounded-operator unbounded-operator-build unbounded-operator-manifests playpen-manifests e2e-gantry e2e-playpen e2e-racer gomod docs-serve unbounded-net-controller unbounded-net-controller-build unbounded-net-node unbounded-net-node-build unbounded-net-routeplan-debug unping unping-build unroute unroute-build license-check notice notice-check gantry gantry-build gantry-manifests inventory-manifests +.PHONY: all help fmt lint lint-actions test build vulncheck check-deps kubectl-unbounded kubectl-unbounded-build install-tools install-protoc install-helm generate kubectl-unbounded forge relctl relctl-build agent-artifacts-builder agent-artifacts-builder-build orcadev unbounded-agent machina machina-build machina-oci machina-oci-push machina-manifests machine-ops-controller machine-ops-controller-build machine-ops-controller-oci machine-ops-controller-oci-push machine-ops-manifests project-signal-controller project-signal-controller-build signal-manifests image-project-signal-controller-local metalman metalman-build metalman-oci metalman-oci-push unbounded-operator unbounded-operator-build unbounded-operator-manifests playpen-manifests e2e-gantry e2e-playpen e2e-racer gomod docs-serve unbounded-net-controller unbounded-net-controller-build unbounded-net-node unbounded-net-node-build unbounded-net-routeplan-debug unping unping-build unroute unroute-build license-check notice notice-check gantry gantry-build gantry-manifests inventory-manifests .PHONY: net-frontend net-frontend-clean net-ebpf-build net-ebpf-generate net-ebpf-verify net-manifests gantry-chart-lint gantry-chart-package release-bom release-manifests unbounded-operator-release-manifest .PHONY: image-machina-local image-token-refresher-local image-machine-ops-controller-local image-metalman-local image-unbounded-operator-local image-unbounded-operator-push image-playpen-local image-net-controller-local image-net-node-local image-gantry-local image-gantry-push images-local .PHONY: image-net-controller-push image-net-node-push images-net-all images-net-all-push ##@ General -all: kubectl-unbounded forge relctl machina machine-ops-controller token-refresher unbounded-operator unbounded-net-controller unbounded-net-node unbounded-net-routeplan-debug unping unroute gantry ## Build all binaries (default) +all: kubectl-unbounded forge relctl machina machine-ops-controller project-signal-controller token-refresher unbounded-operator unbounded-net-controller unbounded-net-node unbounded-net-routeplan-debug unping unroute gantry ## Build all binaries (default) help: ## Show this help @echo "" @@ -289,6 +296,8 @@ help: ## Show this help @echo " unbounded-agent Build unbounded-agent (linux)" @echo " machina | machina-build Build machina controller (with/without lint/test)" @echo " machine-ops-controller Build machine-ops-controller" + @echo " project-signal-controller Build Project Signal controller" + @echo " signal-manifests Render Project Signal manifests" @echo " metalman | metalman-build Build metalman controller (with/without lint/test)" @echo " unbounded-operator | unbounded-operator-build Build the top-level Site operator" @echo " unbounded-net-controller Build net controller" @@ -309,6 +318,10 @@ help: ## Show this help @echo " image-machina-local Build machina image with \$$(CONTAINER_ENGINE)" @echo " image-token-refresher-local Build token-refresher image" @echo " image-machine-ops-controller-local Build machine-ops-controller image" + @echo " image-project-signal-controller-local Build Project Signal controller image" + @echo " Build Project Signal recovery verifier image" + @echo " Build Project Signal GPU runner image" + @echo " Build and push Project Signal GPU runner image" @echo " image-metalman-local Build metalman image" @echo " image-unbounded-operator-local Build unbounded-operator image" @echo " image-unbounded-operator-push Build and push unbounded-operator image" @@ -543,13 +556,13 @@ lint-actions: ## Run actionlint over .github/workflows ifdef CI # In CI each job is independent; skip chained prerequisites. -test: machina-manifests token-refresher-manifests machine-ops-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Run all tests with race detector +test: machina-manifests token-refresher-manifests machine-ops-manifests signal-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Run all tests with race detector $(GOTEST) -race ./... else # Locally, chain test -> lint for convenience. -test: lint machina-manifests token-refresher-manifests machine-ops-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Run all tests (implies lint) +test: lint machina-manifests token-refresher-manifests machine-ops-manifests signal-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Run all tests (implies lint) $(GOTEST) ./... endif @@ -654,16 +667,16 @@ image-racer-object-local: ## Build the racer-object container image locally (sin --build-arg GIT_COMMIT=$(GIT_COMMIT) \ --build-arg BUILD_TIME=$(BUILD_TIME) \ -t racer-object:$(VERSION_TAG) -t $(RACER_OBJECT_IMAGE) \ - -f ./images/racer-object/Containerfile . - $(call trivy-maybe,$(RACER_OBJECT_IMAGE)) + -f ./images/racer-object/Containerfile . + $(call trivy-maybe,$(RACER_OBJECT_IMAGE)) -build: machina-manifests token-refresher-manifests machine-ops-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Build all Go packages - $(GOBUILD) ./... +build: machina-manifests token-refresher-manifests machine-ops-manifests signal-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Build all Go packages + $(GOBUILD) ./... generate: install-protoc ## Run go generate for API types (deepcopy, CRDs) and protobuf PATH="$(PROTOC_DIR)/bin:$$PATH" $(GOCMD) generate $(GO_PACKAGES) -vulncheck: machina-manifests token-refresher-manifests machine-ops-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Run govulncheck; fails only on reachable vulnerabilities that have a final-release fix +vulncheck: machina-manifests token-refresher-manifests machine-ops-manifests signal-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Run govulncheck; fails only on reachable vulnerabilities that have a final-release fix @# The JSON stream is the documented programmatic interface. The gate owns @# the verdict, so govulncheck is not asked for one: in JSON mode it exits 0 @# whether or not it found anything, and a non-zero exit here means the scan @@ -1075,6 +1088,21 @@ machine-ops-controller-oci: image-machine-ops-controller-local ## Alias for imag machine-ops-controller-oci-push: machine-ops-controller-oci ## Build and push the machine-ops-controller image $(CONTAINER_ENGINE) push $(MACHINE_OPS_CONTROLLER_IMAGE) +project-signal-controller-build: signal-manifests ## Build Project Signal controller without lint/test + @mkdir -p bin + $(GOBUILD) -trimpath -ldflags '$(STAMP_LDFLAGS)' -o $(PROJECT_SIGNAL_CONTROLLER_BIN) $(PROJECT_SIGNAL_CONTROLLER_CMD) + +project-signal-controller: test project-signal-controller-build ## Build Project Signal controller (implies test) + +image-project-signal-controller-local: ## Build Project Signal controller image locally + $(CONTAINER_ENGINE) build \ + --build-arg VERSION=$(VERSION) \ + --build-arg GIT_COMMIT=$(GIT_COMMIT) \ + --build-arg BUILD_TIME=$(BUILD_TIME) \ + -t project-signal-controller:$(VERSION_TAG) -t $(SIGNAL_CONTROLLER_IMAGE) \ + -f ./images/project-signal-controller/Containerfile . + $(call trivy-maybe,$(SIGNAL_CONTROLLER_IMAGE)) + MACHINA_NAMESPACE ?= $(UNBOUNDED_NAMESPACE) MACHINA_API_SERVER_ENDPOINT ?= MACHINA_MANIFEST_TEMPLATES_DIR := deploy/machina @@ -1147,6 +1175,18 @@ machine-ops-manifests: ## Render machine-ops-controller manifests into deploy/ma --set APIServerEndpoint=$(MACHINE_OPS_API_SERVER_ENDPOINT) @echo "Rendered machine-ops manifests into $(MACHINE_OPS_MANIFEST_RENDERED_DIR) (image: $(MACHINE_OPS_CONTROLLER_IMAGE))" +signal-manifests: ## Render Project Signal manifests into deploy/signal/rendered + @mkdir -p $(SIGNAL_MANIFEST_RENDERED_DIR)/crd + @find $(SIGNAL_MANIFEST_RENDERED_DIR) -mindepth 1 -not -name .gitignore -delete + $(GOCMD) run ./hack/cmd/render-manifests \ + --templates-dir $(SIGNAL_MANIFEST_TEMPLATES_DIR) \ + --output-dir $(SIGNAL_MANIFEST_RENDERED_DIR) \ + --set Namespace=$(SIGNAL_NAMESPACE) \ + --set ControllerImage=$(SIGNAL_CONTROLLER_IMAGE) + @mkdir -p $(SIGNAL_MANIFEST_RENDERED_DIR)/crd + @cp $(SIGNAL_MANIFEST_TEMPLATES_DIR)/crd/*.yaml $(SIGNAL_MANIFEST_RENDERED_DIR)/crd/ + @echo "Rendered Project Signal manifests into $(SIGNAL_MANIFEST_RENDERED_DIR) (image: $(SIGNAL_CONTROLLER_IMAGE))" + playpen-manifests: ## Render playpen operator and runner manifests into deploy/playpen/rendered @mkdir -p $(PLAYPEN_MANIFEST_RENDERED_DIR) @find $(PLAYPEN_MANIFEST_RENDERED_DIR) -mindepth 1 -not -name .gitignore -delete diff --git a/api/signal/v1alpha1/certificationprofile_types.go b/api/signal/v1alpha1/certificationprofile_types.go new file mode 100644 index 0000000000..553c6f2b59 --- /dev/null +++ b/api/signal/v1alpha1/certificationprofile_types.go @@ -0,0 +1,126 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type ScriptReference struct { + Name string `json:"name"` + Key string `json:"key"` +} + +type WorkflowReference struct { + Name string `json:"name"` +} + +type CertificationPurpose string + +const ( + PurposeFullCertification CertificationPurpose = "FullCertification" + PurposeQuickCertification CertificationPurpose = "QuickCertification" + PurposePreflight CertificationPurpose = "Preflight" + PurposeTargetedVerification CertificationPurpose = "TargetedVerification" + PurposeDiagnostic CertificationPurpose = "Diagnostic" +) + +type CertificationStage struct { + ID string `json:"id"` + DisplayName string `json:"displayName"` + Category string `json:"category"` + Description string `json:"description,omitempty"` + ExpectedDuration metav1.Duration `json:"expectedDuration,omitempty"` + Required bool `json:"required"` + Conditional bool `json:"conditional,omitempty"` +} + +type ExecutionBackend struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Queue string `json:"queue,omitempty"` +} + +type ParticipantPolicy struct { + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=1 + Count int32 `json:"count"` + + Selector metav1.LabelSelector `json:"selector"` + + RequiredResources corev1.ResourceList `json:"requiredResources,omitempty"` +} + +type CertificationProfileSpec struct { + Participants ParticipantPolicy `json:"participants"` + Script ScriptReference `json:"script"` + Workflow WorkflowReference `json:"workflow"` + Backend ExecutionBackend `json:"backend"` + RunnerImage string `json:"runnerImage"` + // +kubebuilder:validation:Enum=FullCertification;QuickCertification;Preflight;TargetedVerification;Diagnostic + Purpose CertificationPurpose `json:"purpose,omitempty"` + StagePlan []CertificationStage `json:"stagePlan,omitempty"` + Parameters map[string]string `json:"parameters,omitempty"` + EvidencePath string `json:"evidencePath"` + Deadline metav1.Duration `json:"deadline"` + MaxAttempts int32 `json:"maxAttempts,omitempty"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + Recurring *RecurringPolicy `json:"recurring,omitempty"` +} + +type RecurringPolicy struct { + Enabled bool `json:"enabled"` + NodeNames []string `json:"nodeNames"` + MinIdleDuration metav1.Duration `json:"minIdleDuration"` + ValidFor metav1.Duration `json:"validFor"` + NotBefore *metav1.Time `json:"notBefore,omitempty"` + ExpiresAt metav1.Time `json:"expiresAt"` + EvaluationInterval metav1.Duration `json:"evaluationInterval,omitempty"` + FailureBackoff metav1.Duration `json:"failureBackoff"` + MaxConsecutiveFailures int32 `json:"maxConsecutiveFailures"` + MaxConcurrent int32 `json:"maxConcurrent"` + FailureDomainLabel string `json:"failureDomainLabel,omitempty"` + MaxConcurrentPerFailureDomain int32 `json:"maxConcurrentPerFailureDomain,omitempty"` + ExclusiveProfiles []string `json:"exclusiveProfiles,omitempty"` +} + +type RecurringNodeStatus struct { + NodeName string `json:"nodeName"` + IdleSince *metav1.Time `json:"idleSince,omitempty"` + LastEvaluatedAt *metav1.Time `json:"lastEvaluatedAt,omitempty"` + LastSuccessfulRun string `json:"lastSuccessfulRun,omitempty"` + LastSuccessfulAt *metav1.Time `json:"lastSuccessfulAt,omitempty"` + NextDueAt *metav1.Time `json:"nextDueAt,omitempty"` + ConsecutiveFailures int32 `json:"consecutiveFailures,omitempty"` + Suspended bool `json:"suspended,omitempty"` + Reason string `json:"reason,omitempty"` +} + +type CertificationProfileStatus struct { + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + Nodes []RecurringNodeStatus `json:"nodes,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster,shortName=certprofile +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Backend",type=string,JSONPath=`.spec.backend.kind` +// +kubebuilder:printcolumn:name="Participants",type=integer,JSONPath=`.spec.participants.count` +// +kubebuilder:printcolumn:name="Recurring",type=boolean,JSONPath=`.spec.recurring.enabled` +type CertificationProfile struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CertificationProfileSpec `json:"spec"` + Status CertificationProfileStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true +type CertificationProfileList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CertificationProfile `json:"items"` +} diff --git a/api/signal/v1alpha1/certificationrun_types.go b/api/signal/v1alpha1/certificationrun_types.go new file mode 100644 index 0000000000..14a796998d --- /dev/null +++ b/api/signal/v1alpha1/certificationrun_types.go @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +type RunPhase string + +const ( + RunPhasePending RunPhase = "Pending" + RunPhaseRunning RunPhase = "Running" + RunPhaseSucceeded RunPhase = "Succeeded" + RunPhaseFailed RunPhase = "Failed" + RunPhaseInconclusive RunPhase = "Inconclusive" + RunPhaseCancelled RunPhase = "Canceled" +) + +func (p RunPhase) Terminal() bool { + switch p { + case RunPhaseSucceeded, RunPhaseFailed, RunPhaseInconclusive, RunPhaseCancelled: + return true + default: + return false + } +} + +type Participant struct { + NodeName string `json:"nodeName"` + NodeUID types.UID `json:"nodeUID"` +} + +type PinnedProfile struct { + Name string `json:"name"` + UID types.UID `json:"uid"` + Generation int64 `json:"generation"` +} + +type ScriptSnapshot struct { + ConfigMapName string `json:"configMapName"` + Key string `json:"key"` + Digest string `json:"digest"` +} + +// +kubebuilder:validation:XValidation:rule="self.profile == oldSelf.profile",message="profile is immutable" +// +kubebuilder:validation:XValidation:rule="self.participants == oldSelf.participants",message="participants are immutable" +// +kubebuilder:validation:XValidation:rule="self.script == oldSelf.script",message="script is immutable" +// +kubebuilder:validation:XValidation:rule="self.workflow == oldSelf.workflow",message="workflow is immutable" +// +kubebuilder:validation:XValidation:rule="self.backend == oldSelf.backend",message="backend is immutable" +// +kubebuilder:validation:XValidation:rule="self.runnerImage == oldSelf.runnerImage",message="runnerImage is immutable" +// +kubebuilder:validation:XValidation:rule="self.parameters == oldSelf.parameters",message="parameters are immutable" +// +kubebuilder:validation:XValidation:rule="self.evidencePath == oldSelf.evidencePath",message="evidencePath is immutable" +// +kubebuilder:validation:XValidation:rule="self.deadline == oldSelf.deadline",message="deadline is immutable" +// +kubebuilder:validation:XValidation:rule="self.maxAttempts == oldSelf.maxAttempts",message="maxAttempts is immutable" +// +kubebuilder:validation:XValidation:rule="self.tolerations == oldSelf.tolerations",message="tolerations are immutable" +// +kubebuilder:validation:XValidation:rule="self.requestNodeName == oldSelf.requestNodeName",message="requestNodeName is immutable" +type CertificationRunSpec struct { + Profile PinnedProfile `json:"profile"` + Participants []Participant `json:"participants"` + Script ScriptSnapshot `json:"script"` + Workflow WorkflowReference `json:"workflow"` + Backend ExecutionBackend `json:"backend"` + RunnerImage string `json:"runnerImage"` + // +kubebuilder:validation:Enum=FullCertification;QuickCertification;Preflight;TargetedVerification;Diagnostic + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="purpose is immutable" + Purpose CertificationPurpose `json:"purpose,omitempty"` + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="stagePlan is immutable" + StagePlan []CertificationStage `json:"stagePlan,omitempty"` + Parameters map[string]string `json:"parameters,omitempty"` + EvidencePath string `json:"evidencePath"` + Deadline metav1.Duration `json:"deadline"` + MaxAttempts int32 `json:"maxAttempts"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + RequestNodeName string `json:"requestNodeName"` + Cancel bool `json:"cancel,omitempty"` +} + +type WorkflowStatus struct { + Name string `json:"name,omitempty"` + UID string `json:"uid,omitempty"` + Attempt int32 `json:"attempt,omitempty"` + Phase string `json:"phase,omitempty"` + Outputs map[string]string `json:"outputs,omitempty"` +} + +type EvidenceStatus struct { + URI string `json:"uri,omitempty"` + Digest string `json:"digest,omitempty"` +} + +type CleanupStatus struct { + ReservationsReleased bool `json:"reservationsReleased,omitempty"` + SnapshotDeleted bool `json:"snapshotDeleted,omitempty"` +} + +type CertificationRunStatus struct { + Phase RunPhase `json:"phase,omitempty"` + ExecutionState string `json:"executionState,omitempty"` + Verdict string `json:"verdict,omitempty"` + Reason string `json:"reason,omitempty"` + Workflow WorkflowStatus `json:"workflow,omitempty"` + Evidence EvidenceStatus `json:"evidence,omitempty"` + Cleanup CleanupStatus `json:"cleanup,omitempty"` + StartedAt *metav1.Time `json:"startedAt,omitempty"` + FinishedAt *metav1.Time `json:"finishedAt,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Namespaced,shortName=certrun +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Profile",type=string,JSONPath=`.spec.profile.name` +// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` +// +kubebuilder:printcolumn:name="Verdict",type=string,JSONPath=`.status.verdict` +// +kubebuilder:printcolumn:name="Workflow",type=string,JSONPath=`.status.workflow.name` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` +type CertificationRun struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CertificationRunSpec `json:"spec"` + Status CertificationRunStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true +type CertificationRunList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CertificationRun `json:"items"` +} diff --git a/api/signal/v1alpha1/groupversion_info.go b/api/signal/v1alpha1/groupversion_info.go new file mode 100644 index 0000000000..0ce40be47d --- /dev/null +++ b/api/signal/v1alpha1/groupversion_info.go @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +// +kubebuilder:object:generate=true +// +groupName=signal.unbounded-cloud.io +package v1alpha1 + +//go:generate go tool controller-gen object:headerFile=../../../hack/boilerplate.go.txt paths=. +//go:generate go tool controller-gen crd:headerFile=../../../hack/boilerplate.yaml.txt paths=. output:crd:dir=../../../deploy/signal/crd diff --git a/api/signal/v1alpha1/register.go b/api/signal/v1alpha1/register.go new file mode 100644 index 0000000000..11130bcf55 --- /dev/null +++ b/api/signal/v1alpha1/register.go @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +const GroupName = "signal.unbounded-cloud.io" + +var ( + GroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha1"} + SchemeBuilder = runtime.NewSchemeBuilder(func(s *runtime.Scheme) error { + s.AddKnownTypes( + GroupVersion, + &CertificationProfile{}, + &CertificationProfileList{}, + &CertificationRun{}, + &CertificationRunList{}, + ) + metav1.AddToGroupVersion(s, GroupVersion) + + return nil + }) + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/api/signal/v1alpha1/zz_generated.deepcopy.go b/api/signal/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000000..36e0b8c25f --- /dev/null +++ b/api/signal/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,528 @@ +//go:build !ignore_autogenerated + +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationProfile) DeepCopyInto(out *CertificationProfile) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationProfile. +func (in *CertificationProfile) DeepCopy() *CertificationProfile { + if in == nil { + return nil + } + out := new(CertificationProfile) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CertificationProfile) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationProfileList) DeepCopyInto(out *CertificationProfileList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CertificationProfile, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationProfileList. +func (in *CertificationProfileList) DeepCopy() *CertificationProfileList { + if in == nil { + return nil + } + out := new(CertificationProfileList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CertificationProfileList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationProfileSpec) DeepCopyInto(out *CertificationProfileSpec) { + *out = *in + in.Participants.DeepCopyInto(&out.Participants) + out.Script = in.Script + out.Workflow = in.Workflow + out.Backend = in.Backend + if in.StagePlan != nil { + in, out := &in.StagePlan, &out.StagePlan + *out = make([]CertificationStage, len(*in)) + copy(*out, *in) + } + if in.Parameters != nil { + in, out := &in.Parameters, &out.Parameters + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + out.Deadline = in.Deadline + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]v1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Recurring != nil { + in, out := &in.Recurring, &out.Recurring + *out = new(RecurringPolicy) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationProfileSpec. +func (in *CertificationProfileSpec) DeepCopy() *CertificationProfileSpec { + if in == nil { + return nil + } + out := new(CertificationProfileSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationProfileStatus) DeepCopyInto(out *CertificationProfileStatus) { + *out = *in + if in.Nodes != nil { + in, out := &in.Nodes, &out.Nodes + *out = make([]RecurringNodeStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationProfileStatus. +func (in *CertificationProfileStatus) DeepCopy() *CertificationProfileStatus { + if in == nil { + return nil + } + out := new(CertificationProfileStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationRun) DeepCopyInto(out *CertificationRun) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationRun. +func (in *CertificationRun) DeepCopy() *CertificationRun { + if in == nil { + return nil + } + out := new(CertificationRun) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CertificationRun) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationRunList) DeepCopyInto(out *CertificationRunList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CertificationRun, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationRunList. +func (in *CertificationRunList) DeepCopy() *CertificationRunList { + if in == nil { + return nil + } + out := new(CertificationRunList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CertificationRunList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationRunSpec) DeepCopyInto(out *CertificationRunSpec) { + *out = *in + out.Profile = in.Profile + if in.Participants != nil { + in, out := &in.Participants, &out.Participants + *out = make([]Participant, len(*in)) + copy(*out, *in) + } + out.Script = in.Script + out.Workflow = in.Workflow + out.Backend = in.Backend + if in.StagePlan != nil { + in, out := &in.StagePlan, &out.StagePlan + *out = make([]CertificationStage, len(*in)) + copy(*out, *in) + } + if in.Parameters != nil { + in, out := &in.Parameters, &out.Parameters + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + out.Deadline = in.Deadline + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]v1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationRunSpec. +func (in *CertificationRunSpec) DeepCopy() *CertificationRunSpec { + if in == nil { + return nil + } + out := new(CertificationRunSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationRunStatus) DeepCopyInto(out *CertificationRunStatus) { + *out = *in + in.Workflow.DeepCopyInto(&out.Workflow) + out.Evidence = in.Evidence + out.Cleanup = in.Cleanup + if in.StartedAt != nil { + in, out := &in.StartedAt, &out.StartedAt + *out = (*in).DeepCopy() + } + if in.FinishedAt != nil { + in, out := &in.FinishedAt, &out.FinishedAt + *out = (*in).DeepCopy() + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationRunStatus. +func (in *CertificationRunStatus) DeepCopy() *CertificationRunStatus { + if in == nil { + return nil + } + out := new(CertificationRunStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationStage) DeepCopyInto(out *CertificationStage) { + *out = *in + out.ExpectedDuration = in.ExpectedDuration +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationStage. +func (in *CertificationStage) DeepCopy() *CertificationStage { + if in == nil { + return nil + } + out := new(CertificationStage) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CleanupStatus) DeepCopyInto(out *CleanupStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CleanupStatus. +func (in *CleanupStatus) DeepCopy() *CleanupStatus { + if in == nil { + return nil + } + out := new(CleanupStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EvidenceStatus) DeepCopyInto(out *EvidenceStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EvidenceStatus. +func (in *EvidenceStatus) DeepCopy() *EvidenceStatus { + if in == nil { + return nil + } + out := new(EvidenceStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExecutionBackend) DeepCopyInto(out *ExecutionBackend) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExecutionBackend. +func (in *ExecutionBackend) DeepCopy() *ExecutionBackend { + if in == nil { + return nil + } + out := new(ExecutionBackend) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Participant) DeepCopyInto(out *Participant) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Participant. +func (in *Participant) DeepCopy() *Participant { + if in == nil { + return nil + } + out := new(Participant) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ParticipantPolicy) DeepCopyInto(out *ParticipantPolicy) { + *out = *in + in.Selector.DeepCopyInto(&out.Selector) + if in.RequiredResources != nil { + in, out := &in.RequiredResources, &out.RequiredResources + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ParticipantPolicy. +func (in *ParticipantPolicy) DeepCopy() *ParticipantPolicy { + if in == nil { + return nil + } + out := new(ParticipantPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PinnedProfile) DeepCopyInto(out *PinnedProfile) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PinnedProfile. +func (in *PinnedProfile) DeepCopy() *PinnedProfile { + if in == nil { + return nil + } + out := new(PinnedProfile) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RecurringNodeStatus) DeepCopyInto(out *RecurringNodeStatus) { + *out = *in + if in.IdleSince != nil { + in, out := &in.IdleSince, &out.IdleSince + *out = (*in).DeepCopy() + } + if in.LastEvaluatedAt != nil { + in, out := &in.LastEvaluatedAt, &out.LastEvaluatedAt + *out = (*in).DeepCopy() + } + if in.LastSuccessfulAt != nil { + in, out := &in.LastSuccessfulAt, &out.LastSuccessfulAt + *out = (*in).DeepCopy() + } + if in.NextDueAt != nil { + in, out := &in.NextDueAt, &out.NextDueAt + *out = (*in).DeepCopy() + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RecurringNodeStatus. +func (in *RecurringNodeStatus) DeepCopy() *RecurringNodeStatus { + if in == nil { + return nil + } + out := new(RecurringNodeStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RecurringPolicy) DeepCopyInto(out *RecurringPolicy) { + *out = *in + if in.NodeNames != nil { + in, out := &in.NodeNames, &out.NodeNames + *out = make([]string, len(*in)) + copy(*out, *in) + } + out.MinIdleDuration = in.MinIdleDuration + out.ValidFor = in.ValidFor + if in.NotBefore != nil { + in, out := &in.NotBefore, &out.NotBefore + *out = (*in).DeepCopy() + } + in.ExpiresAt.DeepCopyInto(&out.ExpiresAt) + out.EvaluationInterval = in.EvaluationInterval + out.FailureBackoff = in.FailureBackoff + if in.ExclusiveProfiles != nil { + in, out := &in.ExclusiveProfiles, &out.ExclusiveProfiles + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RecurringPolicy. +func (in *RecurringPolicy) DeepCopy() *RecurringPolicy { + if in == nil { + return nil + } + out := new(RecurringPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ScriptReference) DeepCopyInto(out *ScriptReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScriptReference. +func (in *ScriptReference) DeepCopy() *ScriptReference { + if in == nil { + return nil + } + out := new(ScriptReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ScriptSnapshot) DeepCopyInto(out *ScriptSnapshot) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScriptSnapshot. +func (in *ScriptSnapshot) DeepCopy() *ScriptSnapshot { + if in == nil { + return nil + } + out := new(ScriptSnapshot) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkflowReference) DeepCopyInto(out *WorkflowReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkflowReference. +func (in *WorkflowReference) DeepCopy() *WorkflowReference { + if in == nil { + return nil + } + out := new(WorkflowReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkflowStatus) DeepCopyInto(out *WorkflowStatus) { + *out = *in + if in.Outputs != nil { + in, out := &in.Outputs, &out.Outputs + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkflowStatus. +func (in *WorkflowStatus) DeepCopy() *WorkflowStatus { + if in == nil { + return nil + } + out := new(WorkflowStatus) + in.DeepCopyInto(out) + return out +} diff --git a/cmd/project-signal-controller/main.go b/cmd/project-signal-controller/main.go new file mode 100644 index 0000000000..9002e90ddb --- /dev/null +++ b/cmd/project-signal-controller/main.go @@ -0,0 +1,160 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "errors" + "fmt" + "os" + osSignal "os/signal" + "syscall" + + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + machinav1 "github.com/Azure/unbounded/api/machina/v1alpha3" + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" + projectsignal "github.com/Azure/unbounded/internal/signal" + "github.com/Azure/unbounded/internal/version" +) + +const controllerName = "project-signal-controller" + +type config struct { + namespace string + scriptNamespace string + metricsAddress string + probeAddress string + leaderElect bool + maxConcurrentRuns int +} + +func main() { + var cfg config + + cmd := &cobra.Command{ + Use: controllerName, + Short: "Project Signal node certification controller", + RunE: func(cmd *cobra.Command, _ []string) error { + ctx, cancel := osSignal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + return run(ctx, cfg) + }, + Version: version.Version + " (commit: " + version.GitCommit + ")", + } + + cmd.Flags().StringVar(&cfg.namespace, "namespace", "project-signal", "Namespace for CertificationRuns and Workflows") + cmd.Flags().StringVar(&cfg.scriptNamespace, "script-namespace", "project-signal", "Namespace containing mutable source script ConfigMaps") + cmd.Flags().StringVar(&cfg.metricsAddress, "metrics-bind-address", ":8080", "Address for metrics") + cmd.Flags().StringVar(&cfg.probeAddress, "health-probe-bind-address", ":8081", "Address for health probes") + cmd.Flags().BoolVar(&cfg.leaderElect, "leader-elect", true, "Enable leader election") + cmd.Flags().IntVar(&cfg.maxConcurrentRuns, "max-concurrent-runs", 4, "Maximum concurrent CertificationRun reconciles") + cmd.CompletionOptions.DisableDefaultCmd = true + + if err := cmd.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } +} + +func run(ctx context.Context, cfg config) error { + ctrl.SetLogger(zap.New()) + + if cfg.namespace == "" || cfg.scriptNamespace == "" { + return errors.New("namespace and script namespace are required") + } + + scheme := runtime.NewScheme() + if err := clientgoscheme.AddToScheme(scheme); err != nil { + return err + } + + if err := signalv1.AddToScheme(scheme); err != nil { + return err + } + + if err := machinav1.AddToScheme(scheme); err != nil { + return err + } + + watchedNamespaces := map[string]cache.Config{cfg.namespace: {}} + watchedNamespaces[cfg.scriptNamespace] = cache.Config{} + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + Cache: cache.Options{DefaultNamespaces: watchedNamespaces}, + Metrics: metricsserver.Options{BindAddress: cfg.metricsAddress}, + HealthProbeBindAddress: cfg.probeAddress, + LeaderElection: cfg.leaderElect, + LeaderElectionID: controllerName, + LeaderElectionNamespace: cfg.namespace, + LeaderElectionReleaseOnCancel: true, + }) + if err != nil { + return fmt.Errorf("create manager: %w", err) + } + + evidence, err := projectsignal.NewKubernetesEvidenceStore(mgr.GetClient(), cfg.namespace) + if err != nil { + return err + } + + if _, err := mgr.GetRESTMapper().RESTMapping(schema.GroupKind{Group: "argoproj.io", Kind: "Workflow"}, "v1alpha1"); err != nil { + return fmt.Errorf("argo Workflows is required but Workflow.argoproj.io/v1alpha1 is unavailable: %w", err) + } + + admission := &projectsignal.AdmissionReconciler{ + Client: mgr.GetClient(), + APIReader: mgr.GetAPIReader(), + Namespace: cfg.namespace, + ScriptNamespace: cfg.scriptNamespace, + Recorder: mgr.GetEventRecorder(controllerName), + } + if err := admission.SetupWithManager(mgr); err != nil { + return fmt.Errorf("register admission reconciler: %w", err) + } + + runs := &projectsignal.RunReconciler{ + Client: mgr.GetClient(), + APIReader: mgr.GetAPIReader(), + Namespace: cfg.namespace, + Evidence: evidence, + MaxConcurrentRuns: cfg.maxConcurrentRuns, + Recorder: mgr.GetEventRecorder(controllerName), + } + if err := runs.SetupWithManager(mgr); err != nil { + return fmt.Errorf("register run reconciler: %w", err) + } + + recurring := &projectsignal.RecurringReconciler{ + Client: mgr.GetClient(), + Reader: mgr.GetAPIReader(), + Namespace: cfg.namespace, + Admission: admission, + } + if err := recurring.SetupWithManager(mgr); err != nil { + return fmt.Errorf("register recurring reconciler: %w", err) + } + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + return err + } + + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + return err + } + + ctrl.Log.Info("starting Project Signal controller", "version", version.String(), "namespace", cfg.namespace) + + return mgr.Start(ctx) +} diff --git a/deploy/signal/00-namespace.yaml.tmpl b/deploy/signal/00-namespace.yaml.tmpl new file mode 100644 index 0000000000..b47ae5cc04 --- /dev/null +++ b/deploy/signal/00-namespace.yaml.tmpl @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: {{ default "project-signal" .Namespace }} + labels: + app.kubernetes.io/part-of: project-signal diff --git a/deploy/signal/01-serviceaccounts.yaml.tmpl b/deploy/signal/01-serviceaccounts.yaml.tmpl new file mode 100644 index 0000000000..e0efbbe38e --- /dev/null +++ b/deploy/signal/01-serviceaccounts.yaml.tmpl @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: project-signal-controller + namespace: {{ default "project-signal" .Namespace }} + labels: + app.kubernetes.io/name: project-signal-controller +automountServiceAccountToken: true diff --git a/deploy/signal/02-rbac.yaml.tmpl b/deploy/signal/02-rbac.yaml.tmpl new file mode 100644 index 0000000000..1b2bc29d89 --- /dev/null +++ b/deploy/signal/02-rbac.yaml.tmpl @@ -0,0 +1,107 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: project-signal-controller +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list"] + - apiGroups: ["signal.unbounded-cloud.io"] + resources: ["certificationprofiles"] + verbs: ["get", "list", "watch"] + - apiGroups: ["signal.unbounded-cloud.io"] + resources: ["certificationprofiles/status"] + verbs: ["get", "update", "patch"] + - apiGroups: ["signal.unbounded-cloud.io"] + resources: ["certificationruns"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["signal.unbounded-cloud.io"] + resources: ["certificationruns/status"] + verbs: ["get", "update", "patch"] + - apiGroups: ["signal.unbounded-cloud.io"] + resources: ["certificationruns/finalizers"] + verbs: ["update"] + - apiGroups: ["unbounded-cloud.io"] + resources: ["machines", "machineoperations"] + verbs: ["get", "list"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: project-signal-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: project-signal-controller +subjects: + - kind: ServiceAccount + name: project-signal-controller + namespace: {{ default "project-signal" .Namespace }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: project-signal-controller + namespace: {{ default "project-signal" .Namespace }} +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] + - apiGroups: ["events.k8s.io"] + resources: ["events"] + verbs: ["create", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["argoproj.io"] + resources: ["workflows"] + verbs: ["get", "list", "watch", "create", "delete"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: project-signal-controller + namespace: {{ default "project-signal" .Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: project-signal-controller +subjects: + - kind: ServiceAccount + name: project-signal-controller + namespace: {{ default "project-signal" .Namespace }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: project-signal-script-reader + namespace: {{ default (default "project-signal" .Namespace) .ScriptNamespace }} +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: project-signal-script-reader + namespace: {{ default (default "project-signal" .Namespace) .ScriptNamespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: project-signal-script-reader +subjects: + - kind: ServiceAccount + name: project-signal-controller + namespace: {{ default "project-signal" .Namespace }} diff --git a/deploy/signal/06-deployment.yaml.tmpl b/deploy/signal/06-deployment.yaml.tmpl new file mode 100644 index 0000000000..d58ea202ba --- /dev/null +++ b/deploy/signal/06-deployment.yaml.tmpl @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: project-signal-controller + namespace: {{ default "project-signal" .Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: project-signal-controller + template: + metadata: + labels: + app.kubernetes.io/name: project-signal-controller + spec: + serviceAccountName: project-signal-controller + containers: + - name: controller + image: {{ default "example.invalid/project-signal-controller:replace-me" .ControllerImage }} + imagePullPolicy: IfNotPresent + args: + - --namespace={{ default "project-signal" .Namespace }} + - --script-namespace={{ default "project-signal" .Namespace }} + - --leader-elect=true + ports: + - name: metrics + containerPort: 8080 + - name: health + containerPort: 8081 + resources: + requests: + cpu: 50m + memory: 96Mi + limits: + cpu: 500m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + capabilities: + drop: [ALL] + livenessProbe: + httpGet: + path: /healthz + port: health + readinessProbe: + httpGet: + path: /readyz + port: health diff --git a/deploy/signal/README.md b/deploy/signal/README.md new file mode 100644 index 0000000000..91fc4f1943 --- /dev/null +++ b/deploy/signal/README.md @@ -0,0 +1,169 @@ +# Project Signal controller + +Project Signal is a Kubernetes controller for admitting approved idle Nodes to +operator-supplied certification workflows. The upstream integration provides +the API, controller, CRDs, RBAC, and controller Deployment. It intentionally +does not publish certification scripts, runner images, CertificationProfiles, +or WorkflowTemplates. + +## Architecture + +```text +approved Node or recurring policy + | + v + CertificationProfile + | + v + reserve Nodes and snapshot the selected ConfigMap key + | + v + immutable CertificationRun input + SHA-256 digest + | + v + operator-supplied Argo WorkflowTemplate + | + v + typed result -> retained evidence -> Node projection -> cleanup +``` + +The controller snapshots one selected script key from a source ConfigMap into +a run-owned immutable ConfigMap. Updating the source affects only later runs. +The controller does not contain or publish certification script bodies. + +## Prerequisites + +- Kubernetes with the `Node`, `Pod`, and `ConfigMap` APIs. +- Argo Workflows `argoproj.io/v1alpha1` Workflow API. +- A controller image built from `cmd/project-signal-controller`. +- An operator-owned script ConfigMap and execution image. +- An operator-owned WorkflowTemplate implementing the interface below. +- Explicitly reviewed Node selectors, resource requests, tolerations, policy + values, and cleanup behavior. + +Backends such as Jobs, MPIJobs, or other workload APIs are operator choices. +Install their controllers and grant their runner permissions separately. +Project Signal does not install them or grant generic workload-runner RBAC. + +## WorkflowTemplate interface + +The referenced WorkflowTemplate must accept these string parameters: + +| Parameter | Meaning | +| --- | --- | +| `run-name` | CertificationRun name | +| `run-uid` | CertificationRun UID | +| `participants` | Comma-separated admitted Node names | +| `script-configmap` | Immutable run-owned ConfigMap | +| `script-key` | Selected ConfigMap key | +| `script-digest` | SHA-256 of the selected script bytes | +| `backend-api-version` | Backend API version from the profile | +| `backend-kind` | Backend kind from the profile | +| `queue` | Optional backend queue | +| `runner-image` | Digest-pinned execution image | +| `tolerations` | JSON-encoded toleration list | +| `script-environment` | JSON-encoded, validated environment map | +| `mpi-payload-mode` | Optional operator-defined execution mode | +| `expected-gpu-product-regex` | Optional operator policy | +| `expected-gpu-count` | Optional operator policy | +| `observed-gpu-capacity` | Capacity copied from the admitted Node | +| `observed-gpu-allocatable` | Allocatable count copied from the admitted Node | +| `gpu-resource-count` | Optional operator policy | +| `privileged` | Optional operator policy | +| `attempt` | One-based attempt number | + +The Workflow must publish only these output parameters: + +| Output | Limit | Meaning | +| --- | ---: | --- | +| `verdict` | 32 bytes | `Pass`, `Fail`, or `Inconclusive` | +| `reason` | 4 KiB | Bounded diagnostic reason | +| `measurements` | 16 KiB | Bounded measurement summary | + +Unexpected, duplicate, malformed, or oversized outputs are rejected and the +run remains fail-closed. + +## Render and install + +Render the controller manifests: + +```bash +make signal-manifests \ + SIGNAL_CONTROLLER_IMAGE= +``` + +Generated output is written to `deploy/signal/rendered/`. Validate it with the +target cluster before applying: + +```bash +kubectl apply --dry-run=server -f deploy/signal/rendered/crd/ +kubectl apply --dry-run=server -f deploy/signal/rendered/ +kubectl apply -f deploy/signal/rendered/crd/ +kubectl apply -f deploy/signal/rendered/ +``` + +The upstream manifests install no profiles. Operators must create a +CertificationProfile that references their own script ConfigMap, +WorkflowTemplate, backend, and digest-pinned runner image. + +## Results and evidence + +Workflow infrastructure state and certification verdicts are separate: + +```text +executionState=Completed, verdict=Pass|Fail, reason= +executionState=Error, verdict=Inconclusive, reason= +``` + +Ambiguous or incomplete workflow execution remains `Inconclusive`. The +controller does not infer a successful result from partial evidence. + +Cluster-local evidence consists of CertificationRun status, the referenced +Workflow and backend status, Events, pod logs, termination messages, and the +SHA-256 digest of the controller's terminal evidence document. + +## Node metadata + +Controller-owned labels: + +| Key | Values | +| --- | --- | +| `signal.unbounded-cloud.io/state` | `pending`, `testing`, `passed`, `failed`, `inconclusive` | +| `signal.unbounded-cloud.io/eligibility` | `blocked`, `production` | +| `signal.unbounded-cloud.io/profile` | bounded profile identifier | + +Pending, testing, failed, and inconclusive states use +`eligibility=blocked`. A successful run projects `passed/production`. Older +runs cannot overwrite a newer terminal projection. + +## Operations + +Request a run by applying the stable trigger label: + +```bash +kubectl label node \ + signal.unbounded-cloud.io/certification-request= +``` + +Cancel an active run: + +```bash +kubectl -n project-signal patch certificationrun \ + --type=merge -p '{"spec":{"cancel":true}}' +``` + +Before removing the controller, disable recurring policies and wait for active +runs to become terminal. Argo Workflows and any workload backend may be shared +cluster services and must not be removed as part of Project Signal cleanup. + +## Non-goals + +This upstream integration does not provide: + +- certification scripts or proprietary thresholds; +- a runner image; +- bundled CertificationProfiles or WorkflowTemplates; +- workload-backend installation or runner RBAC; +- destructive stress, reboot, cordon, drain, or remediation; +- an external evidence service; +- authoritative recovery of ambiguous external mutations. diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml new file mode 100644 index 0000000000..c1067fd453 --- /dev/null +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml @@ -0,0 +1,391 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: certificationprofiles.signal.unbounded-cloud.io +spec: + group: signal.unbounded-cloud.io + names: + kind: CertificationProfile + listKind: CertificationProfileList + plural: certificationprofiles + shortNames: + - certprofile + singular: certificationprofile + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.backend.kind + name: Backend + type: string + - jsonPath: .spec.participants.count + name: Participants + type: integer + - jsonPath: .spec.recurring.enabled + name: Recurring + type: boolean + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + backend: + properties: + apiVersion: + type: string + kind: + type: string + queue: + type: string + required: + - apiVersion + - kind + type: object + deadline: + type: string + evidencePath: + type: string + maxAttempts: + format: int32 + type: integer + parameters: + additionalProperties: + type: string + type: object + participants: + properties: + count: + format: int32 + maximum: 1 + minimum: 1 + type: integer + requiredResources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: ResourceList is a set of (resource name, quantity) + pairs. + type: object + selector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - count + - selector + type: object + purpose: + enum: + - FullCertification + - QuickCertification + - Preflight + - TargetedVerification + - Diagnostic + type: string + recurring: + properties: + enabled: + type: boolean + evaluationInterval: + type: string + exclusiveProfiles: + items: + type: string + type: array + expiresAt: + format: date-time + type: string + failureBackoff: + type: string + failureDomainLabel: + type: string + maxConcurrent: + format: int32 + type: integer + maxConcurrentPerFailureDomain: + format: int32 + type: integer + maxConsecutiveFailures: + format: int32 + type: integer + minIdleDuration: + type: string + nodeNames: + items: + type: string + type: array + notBefore: + format: date-time + type: string + validFor: + type: string + required: + - enabled + - expiresAt + - failureBackoff + - maxConcurrent + - maxConsecutiveFailures + - minIdleDuration + - nodeNames + - validFor + type: object + runnerImage: + type: string + script: + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + stagePlan: + items: + properties: + category: + type: string + conditional: + type: boolean + description: + type: string + displayName: + type: string + expectedDuration: + type: string + id: + type: string + required: + type: boolean + required: + - category + - displayName + - id + - required + type: object + type: array + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + workflow: + properties: + name: + type: string + required: + - name + type: object + required: + - backend + - deadline + - evidencePath + - participants + - runnerImage + - script + - workflow + type: object + status: + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + nodes: + items: + properties: + consecutiveFailures: + format: int32 + type: integer + idleSince: + format: date-time + type: string + lastEvaluatedAt: + format: date-time + type: string + lastSuccessfulAt: + format: date-time + type: string + lastSuccessfulRun: + type: string + nextDueAt: + format: date-time + type: string + nodeName: + type: string + reason: + type: string + suspended: + type: boolean + required: + - nodeName + type: object + type: array + observedGeneration: + format: int64 + type: integer + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml new file mode 100644 index 0000000000..2721ba6b89 --- /dev/null +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml @@ -0,0 +1,366 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: certificationruns.signal.unbounded-cloud.io +spec: + group: signal.unbounded-cloud.io + names: + kind: CertificationRun + listKind: CertificationRunList + plural: certificationruns + shortNames: + - certrun + singular: certificationrun + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.profile.name + name: Profile + type: string + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.verdict + name: Verdict + type: string + - jsonPath: .status.workflow.name + name: Workflow + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + backend: + properties: + apiVersion: + type: string + kind: + type: string + queue: + type: string + required: + - apiVersion + - kind + type: object + cancel: + type: boolean + deadline: + type: string + evidencePath: + type: string + maxAttempts: + format: int32 + type: integer + parameters: + additionalProperties: + type: string + type: object + participants: + items: + properties: + nodeName: + type: string + nodeUID: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + required: + - nodeName + - nodeUID + type: object + type: array + profile: + properties: + generation: + format: int64 + type: integer + name: + type: string + uid: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + required: + - generation + - name + - uid + type: object + purpose: + enum: + - FullCertification + - QuickCertification + - Preflight + - TargetedVerification + - Diagnostic + type: string + x-kubernetes-validations: + - message: purpose is immutable + rule: self == oldSelf + requestNodeName: + type: string + runnerImage: + type: string + script: + properties: + configMapName: + type: string + digest: + type: string + key: + type: string + required: + - configMapName + - digest + - key + type: object + stagePlan: + items: + properties: + category: + type: string + conditional: + type: boolean + description: + type: string + displayName: + type: string + expectedDuration: + type: string + id: + type: string + required: + type: boolean + required: + - category + - displayName + - id + - required + type: object + type: array + x-kubernetes-validations: + - message: stagePlan is immutable + rule: self == oldSelf + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + workflow: + properties: + name: + type: string + required: + - name + type: object + required: + - backend + - deadline + - evidencePath + - maxAttempts + - participants + - profile + - requestNodeName + - runnerImage + - script + - workflow + type: object + x-kubernetes-validations: + - message: profile is immutable + rule: self.profile == oldSelf.profile + - message: participants are immutable + rule: self.participants == oldSelf.participants + - message: script is immutable + rule: self.script == oldSelf.script + - message: workflow is immutable + rule: self.workflow == oldSelf.workflow + - message: backend is immutable + rule: self.backend == oldSelf.backend + - message: runnerImage is immutable + rule: self.runnerImage == oldSelf.runnerImage + - message: parameters are immutable + rule: self.parameters == oldSelf.parameters + - message: evidencePath is immutable + rule: self.evidencePath == oldSelf.evidencePath + - message: deadline is immutable + rule: self.deadline == oldSelf.deadline + - message: maxAttempts is immutable + rule: self.maxAttempts == oldSelf.maxAttempts + - message: tolerations are immutable + rule: self.tolerations == oldSelf.tolerations + - message: requestNodeName is immutable + rule: self.requestNodeName == oldSelf.requestNodeName + status: + properties: + cleanup: + properties: + reservationsReleased: + type: boolean + snapshotDeleted: + type: boolean + type: object + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + evidence: + properties: + digest: + type: string + uri: + type: string + type: object + executionState: + type: string + finishedAt: + format: date-time + type: string + phase: + type: string + reason: + type: string + startedAt: + format: date-time + type: string + verdict: + type: string + workflow: + properties: + attempt: + format: int32 + type: integer + name: + type: string + outputs: + additionalProperties: + type: string + type: object + phase: + type: string + uid: + type: string + type: object + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/signal/manifest_test.go b/deploy/signal/manifest_test.go new file mode 100644 index 0000000000..e0f97d395b --- /dev/null +++ b/deploy/signal/manifest_test.go @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signalmanifest_test + +import ( + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" +) + +func TestUpstreamManifestsContainOnlyControllerSurface(t *testing.T) { + t.Parallel() + + templates, err := filepath.Glob("*.yaml.tmpl") + if err != nil { + t.Fatal(err) + } + + sort.Strings(templates) + + want := []string{ + "00-namespace.yaml.tmpl", + "01-serviceaccounts.yaml.tmpl", + "02-rbac.yaml.tmpl", + "06-deployment.yaml.tmpl", + } + if !reflect.DeepEqual(templates, want) { + t.Fatalf("manifest templates = %v, want %v", templates, want) + } + + for _, path := range []string{"01-serviceaccounts.yaml.tmpl", "02-rbac.yaml.tmpl"} { + content, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + + manifest := string(content) + if strings.Contains(manifest, "project-signal-runner") || + strings.Contains(manifest, "kubeflow.org") { + t.Fatalf("%s contains bundled runner surface", path) + } + } +} + +func TestUpstreamDoesNotPublishCertificationPackage(t *testing.T) { + t.Parallel() + + for _, path := range []string{ + "../../project-signal-certification", + "../../project-signal-certification-runner", + } { + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("%s must not be published upstream: %v", path, err) + } + } +} diff --git a/deploy/signal/rendered/.gitignore b/deploy/signal/rendered/.gitignore new file mode 100644 index 0000000000..d6b7ef32c8 --- /dev/null +++ b/deploy/signal/rendered/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/images/project-signal-controller/Containerfile b/images/project-signal-controller/Containerfile new file mode 100644 index 0000000000..dd52f42b80 --- /dev/null +++ b/images/project-signal-controller/Containerfile @@ -0,0 +1,29 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +FROM docker.io/library/golang:1.27.0-trixie AS builder + +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates git make \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . + +ARG TARGETOS +ARG TARGETARCH +ARG VERSION=dev +ARG GIT_COMMIT= +ARG BUILD_TIME= + +RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 \ + make project-signal-controller-build VERSION=${VERSION} \ + ${GIT_COMMIT:+GIT_COMMIT=${GIT_COMMIT}} ${BUILD_TIME:+BUILD_TIME=${BUILD_TIME}} + +FROM gcr.io/distroless/static-debian12:nonroot + +COPY --from=builder /src/bin/project-signal-controller /project-signal-controller +COPY LICENSE NOTICE /licenses/ + +ENTRYPOINT ["/project-signal-controller"] diff --git a/internal/signal/controller.go b/internal/signal/controller.go new file mode 100644 index 0000000000..3c2ff64633 --- /dev/null +++ b/internal/signal/controller.go @@ -0,0 +1,1409 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signal + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "reflect" + "regexp" + "sort" + "strings" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/rand" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/recorder" + + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" +) + +const ( + RequestLabel = "signal.unbounded-cloud.io/certification-request" + ReservationLabel = "signal.unbounded-cloud.io/reserved-by" + RunNodeLabel = "signal.unbounded-cloud.io/request-node" + RunProfileLabel = "signal.unbounded-cloud.io/profile" + RunTriggerAnnotation = "signal.unbounded-cloud.io/trigger" + RunFinalizer = "signal.unbounded-cloud.io/cleanup" + SnapshotRunLabel = "signal.unbounded-cloud.io/run" + DefaultMaxAttempts = int32(1) +) + +var runnerImageDigestPattern = regexp.MustCompile(`^[^@\s]+@sha256:[0-9a-f]{64}$`) + +var reservedScriptEnvironmentNames = map[string]struct{}{ + "BASH_ENV": {}, + "BASHOPTS": {}, + "CDPATH": {}, + "ENV": {}, + "GLOBIGNORE": {}, + "HOST_PROC_ROOT": {}, + "HOST_SYSFS_ROOT": {}, + "IFS": {}, + "LD_LIBRARY_PATH": {}, + "LD_PRELOAD": {}, + "PATH": {}, + "PROJECT_SIGNAL_PARAMETERS_JSON": {}, + "PROJECT_SIGNAL_RESULTS_DIR": {}, + "PYTHONHOME": {}, + "PYTHONPATH": {}, + "SHELLOPTS": {}, +} + +var workflowOutputLimits = map[string]int{ + "measurements": 16 * 1024, + "reason": 4 * 1024, + "verdict": 32, +} + +var workflowGVK = schema.GroupVersionKind{ + Group: "argoproj.io", + Version: "v1alpha1", + Kind: "Workflow", +} + +type AdmissionReconciler struct { + client.Client + APIReader client.Reader + Namespace string + ScriptNamespace string + Recorder recorder.EventRecorder + Now func() time.Time + admissionMu sync.Mutex +} + +type RunReconciler struct { + client.Client + APIReader client.Reader + Namespace string + Evidence EvidenceStore + Now func() time.Time + MaxConcurrentRuns int + WorkflowAPIVersion string + Recorder recorder.EventRecorder +} + +func (r *AdmissionReconciler) SetupWithManager(mgr ctrl.Manager) error { + if r.APIReader == nil { + r.APIReader = mgr.GetAPIReader() + } + + return ctrl.NewControllerManagedBy(mgr). + For(&corev1.Node{}, builder.WithPredicates(predicate.LabelChangedPredicate{})). + WithOptions(controller.Options{MaxConcurrentReconciles: 1}). + Complete(r) +} + +func (r *RunReconciler) SetupWithManager(mgr ctrl.Manager) error { + if r.APIReader == nil { + r.APIReader = mgr.GetAPIReader() + } + + workflow := &unstructured.Unstructured{} + workflow.SetGroupVersionKind(workflowGVK) + + return ctrl.NewControllerManagedBy(mgr). + For(&signalv1.CertificationRun{}). + Watches(workflow, handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &signalv1.CertificationRun{})). + WithOptions(controller.Options{MaxConcurrentReconciles: max(1, r.MaxConcurrentRuns)}). + Complete(r) +} + +func (r *AdmissionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var node corev1.Node + if err := r.Get(ctx, client.ObjectKey{Name: req.Name}, &node); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + profileName := node.Labels[RequestLabel] + if profileName == "" { + return ctrl.Result{}, nil + } + + return ctrl.Result{}, r.admitNode(ctx, &node, profileName) +} + +func (r *AdmissionReconciler) admitNode(ctx context.Context, node *corev1.Node, profileName string) error { + return r.admitNodeWithTrigger(ctx, node, profileName, "manual") +} + +func (r *AdmissionReconciler) admitNodeWithTrigger( + ctx context.Context, + node *corev1.Node, + profileName string, + trigger string, +) error { + r.admissionMu.Lock() + defer r.admissionMu.Unlock() + + return r.admitNodeWithTriggerLocked(ctx, node, profileName, trigger) +} + +func (r *AdmissionReconciler) admitNodeWithTriggerLocked( + ctx context.Context, + node *corev1.Node, + profileName string, + trigger string, +) error { + if runName := node.Annotations[ReservationLabel]; runName != "" { + var run signalv1.CertificationRun + + reader := r.APIReader + if reader == nil { + reader = r.Client + } + + err := reader.Get(ctx, client.ObjectKey{Namespace: r.Namespace, Name: runName}, &run) + if err == nil { + return nil + } + + if !apierrors.IsNotFound(err) { + return err + } + + var snapshot corev1.ConfigMap + + err = reader.Get( + ctx, + client.ObjectKey{ + Namespace: r.Namespace, + Name: runName + "-scripts", + }, + &snapshot, + ) + if err == nil { + if snapshot.Labels[SnapshotRunLabel] != runName { + return fmt.Errorf( + "orphaned snapshot %s is not bound to reservation %q", + snapshot.Name, + runName, + ) + } + + if err := r.Delete(ctx, &snapshot); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("delete orphaned script snapshot: %w", err) + } + } else if !apierrors.IsNotFound(err) { + return fmt.Errorf("get orphaned script snapshot: %w", err) + } + + before := node.DeepCopy() + delete(node.Annotations, ReservationLabel) + + if err := r.Patch(ctx, node, client.MergeFrom(before)); err != nil { + return fmt.Errorf("release orphaned Project Signal reservation: %w", err) + } + } + + var profile signalv1.CertificationProfile + if err := r.Get(ctx, client.ObjectKey{Name: profileName}, &profile); err != nil { + return fmt.Errorf("get CertificationProfile %s: %w", profileName, err) + } + + if err := validateStagePlan(profile.Spec.StagePlan); err != nil { + return fmt.Errorf("profile %s stage plan: %w", profile.Name, err) + } + + if !runnerImageDigestPattern.MatchString(profile.Spec.RunnerImage) { + return fmt.Errorf("profile %s runner image must use an immutable sha256 digest", profile.Name) + } + + if _, err := scriptEnvironment(profile.Spec.Parameters); err != nil { + return fmt.Errorf("profile %s parameters: %w", profile.Name, err) + } + + participants, err := r.selectParticipants(ctx, node, &profile) + if err != nil { + return err + } + + runName := runName(node.Name, profile.Name, r.now()) + if err := r.reserveParticipants(ctx, participants, runName); err != nil { + return err + } + + releaseOnFailure := true + defer func() { + if releaseOnFailure { + if err := r.releaseParticipants(context.WithoutCancel(ctx), participants, runName); err != nil { + log.FromContext(ctx).Error(err, "failed to roll back participant reservations", "run", runName) + } + } + }() + + parameters := make(map[string]string, len(profile.Spec.Parameters)+2) + for key, value := range profile.Spec.Parameters { + parameters[key] = value + } + + if len(participants) == 1 { + var participantNode corev1.Node + + reader := r.APIReader + if reader == nil { + reader = r.Client + } + + if err := reader.Get(ctx, client.ObjectKey{Name: participants[0].NodeName}, &participantNode); err != nil { + return fmt.Errorf("read admitted participant inventory: %w", err) + } + + gpuResource := corev1.ResourceName("nvidia.com/gpu") + gpuCapacity := participantNode.Status.Capacity[gpuResource] + gpuAllocatable := participantNode.Status.Allocatable[gpuResource] + parameters["observed-gpu-capacity"] = gpuCapacity.String() + parameters["observed-gpu-allocatable"] = gpuAllocatable.String() + } + + snapshot, scriptDigest, err := r.createSnapshot(ctx, runName, &profile) + if err != nil { + return err + } + + maxAttempts := profile.Spec.MaxAttempts + if maxAttempts < 1 { + maxAttempts = DefaultMaxAttempts + } + + run := &signalv1.CertificationRun{ + TypeMeta: metav1.TypeMeta{APIVersion: signalv1.GroupVersion.String(), Kind: "CertificationRun"}, + ObjectMeta: metav1.ObjectMeta{ + Name: runName, + Namespace: r.Namespace, + Labels: map[string]string{ + RunNodeLabel: node.Name, + RunProfileLabel: profile.Name, + }, + Annotations: map[string]string{ + RunTriggerAnnotation: trigger, + }, + Finalizers: []string{RunFinalizer}, + }, + Spec: signalv1.CertificationRunSpec{ + Profile: signalv1.PinnedProfile{ + Name: profile.Name, + UID: profile.UID, + Generation: profile.Generation, + }, + Participants: participants, + Script: signalv1.ScriptSnapshot{ConfigMapName: snapshot.Name, Key: profile.Spec.Script.Key, Digest: scriptDigest}, + Workflow: profile.Spec.Workflow, + Backend: profile.Spec.Backend, + RunnerImage: profile.Spec.RunnerImage, + Purpose: profile.Spec.Purpose, + StagePlan: append([]signalv1.CertificationStage(nil), profile.Spec.StagePlan...), + Parameters: parameters, + EvidencePath: strings.Trim(profile.Spec.EvidencePath, "/") + "/" + runName + ".json", + Deadline: profile.Spec.Deadline, + MaxAttempts: maxAttempts, + Tolerations: profile.Spec.Tolerations, + RequestNodeName: node.Name, + }, + } + + if err := r.Create(ctx, run); err != nil { + deleteErr := r.Delete(context.WithoutCancel(ctx), snapshot) + + return errors.Join(fmt.Errorf("create CertificationRun: %w", err), client.IgnoreNotFound(deleteErr)) + } + + cleanupFailedAdmission := func(cause error) error { + deleteRunErr := r.Delete(context.WithoutCancel(ctx), run) + if deleteRunErr != nil && !apierrors.IsNotFound(deleteRunErr) { + // Keep the reservation if the run could not be made inert. This + // prevents another admission from overlapping the surviving run. + releaseOnFailure = false + } + + deleteSnapshotErr := r.Delete(context.WithoutCancel(ctx), snapshot) + + return errors.Join(cause, client.IgnoreNotFound(deleteRunErr), client.IgnoreNotFound(deleteSnapshotErr)) + } + + snapshot.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(run, signalv1.GroupVersion.WithKind("CertificationRun"))} + if err := r.Update(ctx, snapshot); err != nil { + return cleanupFailedAdmission(fmt.Errorf("bind script snapshot to run: %w", err)) + } + + if err := projectPending(ctx, r.Client, participants, run.Spec.Profile); err != nil { + return cleanupFailedAdmission(fmt.Errorf("project pending certification state: %w", err)) + } + + releaseOnFailure = false + + if r.Recorder != nil { + r.Recorder.Eventf(run, nil, corev1.EventTypeNormal, "CertificationAdmitted", "AdmitCertification", + "Created CertificationRun %s with immutable script digest %s", run.Name, scriptDigest) + } + + return nil +} + +func (r *AdmissionReconciler) selectParticipants( + ctx context.Context, + requestNode *corev1.Node, + profile *signalv1.CertificationProfile, +) ([]signalv1.Participant, error) { + count := profile.Spec.Participants.Count + if count < 1 { + return nil, fmt.Errorf("profile %s participant count must be positive", profile.Name) + } + + if count > 1 { + return nil, fmt.Errorf("profile %s requests %d participants; multi-participant execution is not implemented", profile.Name, count) + } + + selector, err := metav1.LabelSelectorAsSelector(&profile.Spec.Participants.Selector) + if err != nil { + return nil, fmt.Errorf("profile %s participant selector: %w", profile.Name, err) + } + + if !selector.Matches(labels.Set(requestNode.Labels)) || !nodeEligible(requestNode, profile.Spec.Participants.RequiredResources) { + return nil, fmt.Errorf("request node %s does not satisfy profile %s", requestNode.Name, profile.Name) + } + + var nodes corev1.NodeList + if err := r.List(ctx, &nodes); err != nil { + return nil, fmt.Errorf("list participant nodes: %w", err) + } + + sort.Slice(nodes.Items, func(i, j int) bool { return nodes.Items[i].Name < nodes.Items[j].Name }) + + selected := []signalv1.Participant{{NodeName: requestNode.Name, NodeUID: requestNode.UID}} + + for i := range nodes.Items { + candidate := &nodes.Items[i] + + if int32(len(selected)) == count { + break + } + + if candidate.Name == requestNode.Name || + !selector.Matches(labels.Set(candidate.Labels)) || + !nodeEligible(candidate, profile.Spec.Participants.RequiredResources) || + candidate.Annotations[ReservationLabel] != "" { + continue + } + + selected = append(selected, signalv1.Participant{NodeName: candidate.Name, NodeUID: candidate.UID}) + } + + if int32(len(selected)) != count { + return nil, fmt.Errorf("profile %s requires %d eligible unlocked participants, found %d", profile.Name, count, len(selected)) + } + + return selected, nil +} + +func nodeEligible(node *corev1.Node, required corev1.ResourceList) bool { + if node.Spec.Unschedulable || !nodeReady(node) { + return false + } + + for name, quantity := range required { + available, ok := node.Status.Allocatable[name] + if !ok || available.Cmp(quantity) < 0 { + return false + } + } + + return true +} + +func nodeReady(node *corev1.Node) bool { + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady { + return condition.Status == corev1.ConditionTrue + } + } + + return false +} + +func (r *AdmissionReconciler) reserveParticipants(ctx context.Context, participants []signalv1.Participant, runName string) error { + reserved := make([]signalv1.Participant, 0, len(participants)) + for _, participant := range participants { + var node corev1.Node + if err := r.Get(ctx, client.ObjectKey{Name: participant.NodeName}, &node); err != nil { + return errors.Join( + fmt.Errorf("get participant %s: %w", participant.NodeName, err), + r.releaseParticipants(context.WithoutCancel(ctx), reserved, runName), + ) + } + + if node.UID != participant.NodeUID { + return errors.Join( + fmt.Errorf("participant %s identity changed", participant.NodeName), + r.releaseParticipants(context.WithoutCancel(ctx), reserved, runName), + ) + } + + if value := node.Annotations[ReservationLabel]; value != "" { + return errors.Join( + fmt.Errorf("participant %s is already reserved by %s", participant.NodeName, value), + r.releaseParticipants(context.WithoutCancel(ctx), reserved, runName), + ) + } + + before := node.DeepCopy() + if node.Annotations == nil { + node.Annotations = map[string]string{} + } + + node.Annotations[ReservationLabel] = runName + if err := r.Patch( + ctx, + &node, + client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{}), + ); err != nil { + return errors.Join( + fmt.Errorf("reserve participant %s: %w", participant.NodeName, err), + r.releaseParticipants(context.WithoutCancel(ctx), reserved, runName), + ) + } + + reserved = append(reserved, participant) + } + + return nil +} + +func (r *AdmissionReconciler) releaseParticipants(ctx context.Context, participants []signalv1.Participant, runName string) error { + var errs []error + + for _, participant := range participants { + var node corev1.Node + if err := r.Get(ctx, client.ObjectKey{Name: participant.NodeName}, &node); err != nil { + if !apierrors.IsNotFound(err) { + errs = append(errs, err) + } + + continue + } + + if participant.NodeUID != "" && node.UID != participant.NodeUID { + continue + } + + if node.Annotations[ReservationLabel] != runName { + continue + } + + before := node.DeepCopy() + delete(node.Annotations, ReservationLabel) + + if err := r.Patch(ctx, &node, client.MergeFrom(before)); err != nil { + errs = append(errs, err) + } + } + + return errors.Join(errs...) +} + +func (r *AdmissionReconciler) createSnapshot( + ctx context.Context, + runName string, + profile *signalv1.CertificationProfile, +) (*corev1.ConfigMap, string, error) { + var source corev1.ConfigMap + if err := r.Get(ctx, client.ObjectKey{Namespace: r.ScriptNamespace, Name: profile.Spec.Script.Name}, &source); err != nil { + return nil, "", fmt.Errorf("get source script ConfigMap: %w", err) + } + + content, ok := source.Data[profile.Spec.Script.Key] + if !ok { + return nil, "", fmt.Errorf("source script ConfigMap %s has no key %s", source.Name, profile.Spec.Script.Key) + } + + digest := digestString(content) + immutable := true + + snapshot := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: runName + "-scripts", + Namespace: r.Namespace, + Labels: map[string]string{SnapshotRunLabel: runName}, + }, + Immutable: &immutable, + Data: map[string]string{ + profile.Spec.Script.Key: content, + "sha256": digest, + }, + } + if err := r.Create(ctx, snapshot); err != nil { + return nil, "", fmt.Errorf("create immutable script snapshot: %w", err) + } + + return snapshot, digest, nil +} + +func (r *RunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var run signalv1.CertificationRun + if err := r.Get(ctx, req.NamespacedName, &run); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + if !run.DeletionTimestamp.IsZero() { + return r.reconcileDeleting(ctx, &run) + } + + if run.Status.Phase.Terminal() { + return r.cleanup(ctx, &run) + } + + if run.Status.FinishedAt != nil { + return r.finish( + ctx, + &run, + nil, + run.Status.ExecutionState, + run.Status.Verdict, + run.Status.Reason, + ) + } + + if run.Spec.Cancel { + return r.cancel(ctx, &run) + } + + if run.Status.Workflow.Name == "" { + return r.startAttempt(ctx, &run, 1) + } + + workflow := &unstructured.Unstructured{} + workflow.SetGroupVersionKind(workflowGVK) + + reader := r.APIReader + if reader == nil { + reader = r.Client + } + + if err := reader.Get(ctx, client.ObjectKey{Namespace: run.Namespace, Name: run.Status.Workflow.Name}, workflow); err != nil { + if apierrors.IsNotFound(err) { + return r.retryOrFinish(ctx, &run, "Error", "WorkflowMissing", "workflow disappeared before terminal status") + } + + return ctrl.Result{}, err + } + + if run.Status.Workflow.UID == "" || + string(workflow.GetUID()) != run.Status.Workflow.UID || + !metav1.IsControlledBy(workflow, &run) { + return ctrl.Result{}, fmt.Errorf( + "workflow %s identity or ownership changed; refusing concurrent retry", + workflow.GetName(), + ) + } + + phase, _, err := unstructured.NestedString(workflow.Object, "status", "phase") + if err != nil { + return ctrl.Result{}, fmt.Errorf("read Workflow phase: %w", err) + } + + switch phase { + case "", "Pending", "Running": + if err := projectTesting(ctx, r.Client, &run); err != nil { + return ctrl.Result{}, err + } + + if run.Status.Phase != signalv1.RunPhaseRunning || run.Status.Workflow.Phase != phase { + before := run.DeepCopy() + run.Status.Phase = signalv1.RunPhaseRunning + + run.Status.Workflow.Phase = phase + if run.Status.StartedAt == nil { + now := metav1.NewTime(r.now()) + run.Status.StartedAt = &now + } + + return ctrl.Result{}, r.Status().Patch(ctx, &run, client.MergeFrom(before)) + } + + return ctrl.Result{RequeueAfter: 15 * time.Second}, nil + case "Succeeded": + verdict := workflowParameter(workflow, "verdict") + reason := workflowParameter(workflow, "reason") + + if verdict == "" { + verdict = "Inconclusive" + reason = "workflow completed without a verdict" + } + + return r.finish(ctx, &run, workflow, "Completed", verdict, reason) + case "Failed", "Error": + return r.retryOrFinish(ctx, &run, phase, "Workflow"+phase, workflowMessage(workflow)) + default: + return ctrl.Result{}, fmt.Errorf("unsupported Workflow phase %q", phase) + } +} + +func (r *RunReconciler) startAttempt(ctx context.Context, run *signalv1.CertificationRun, attempt int32) (ctrl.Result, error) { + workflow, err := buildWorkflow(run, attempt) + if err != nil { + return ctrl.Result{}, err + } + + if err := r.Create(ctx, workflow); err != nil { + if !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, fmt.Errorf("create Workflow attempt %d: %w", attempt, err) + } + + existing := &unstructured.Unstructured{} + existing.SetGroupVersionKind(workflowGVK) + + if getErr := r.Get(ctx, client.ObjectKeyFromObject(workflow), existing); getErr != nil { + return ctrl.Result{}, fmt.Errorf("get existing Workflow attempt %d: %w", attempt, getErr) + } + + if !metav1.IsControlledBy(existing, run) { + return ctrl.Result{}, fmt.Errorf("workflow attempt %d already exists without matching CertificationRun ownership", attempt) + } + + workflow = existing + } + + before := run.DeepCopy() + run.Status.Phase = signalv1.RunPhaseRunning + run.Status.ExecutionState = "Running" + + run.Status.Workflow = signalv1.WorkflowStatus{ + Name: workflow.GetName(), + UID: string(workflow.GetUID()), + Attempt: attempt, + Phase: "Pending", + } + if run.Status.StartedAt == nil { + now := metav1.NewTime(r.now()) + run.Status.StartedAt = &now + } + + if err := r.Status().Patch(ctx, run, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, fmt.Errorf("record Workflow attempt: %w", err) + } + + if r.Recorder != nil { + r.Recorder.Eventf(run, nil, corev1.EventTypeNormal, "WorkflowStarted", "StartWorkflow", + "Created Workflow %s for attempt %d", workflow.GetName(), attempt) + } + + return ctrl.Result{}, nil +} + +func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured.Unstructured, error) { + participantNames := make([]string, 0, len(run.Spec.Participants)) + for _, participant := range run.Spec.Participants { + participantNames = append(participantNames, participant.NodeName) + } + + tolerations := uniqueTolerations(append([]corev1.Toleration{{ + Key: "nvidia.com/gpu", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }}, run.Spec.Tolerations...)) + + tolerationsJSON, err := json.Marshal(tolerations) + if err != nil { + return nil, fmt.Errorf("encode Workflow tolerations: %w", err) + } + + scriptEnvironment, err := scriptEnvironment(run.Spec.Parameters) + if err != nil { + return nil, err + } + + scriptEnvironmentJSON, err := json.Marshal(scriptEnvironment) + if err != nil { + return nil, fmt.Errorf("encode script environment: %w", err) + } + + name := fmt.Sprintf("%s-a%d", run.Name, attempt) + workflow := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": workflowGVK.GroupVersion().String(), + "kind": workflowGVK.Kind, + "metadata": map[string]any{ + "name": name, + "namespace": run.Namespace, + "labels": map[string]any{ + SnapshotRunLabel: run.Name, + }, + }, + "spec": map[string]any{ + "activeDeadlineSeconds": int64(run.Spec.Deadline.Seconds()), + "ttlStrategy": map[string]any{ + "secondsAfterCompletion": int64(6 * time.Hour / time.Second), + "secondsAfterSuccess": int64(6 * time.Hour / time.Second), + "secondsAfterFailure": int64(6 * time.Hour / time.Second), + }, + "podGC": map[string]any{ + "strategy": "OnWorkflowCompletion", + "deleteDelayDuration": "2h", + }, + "workflowTemplateRef": map[string]any{ + "name": run.Spec.Workflow.Name, + }, + "arguments": map[string]any{ + "parameters": []any{ + map[string]any{"name": "run-name", "value": run.Name}, + map[string]any{"name": "run-uid", "value": string(run.UID)}, + map[string]any{"name": "participants", "value": strings.Join(participantNames, ",")}, + map[string]any{"name": "script-configmap", "value": run.Spec.Script.ConfigMapName}, + map[string]any{"name": "script-key", "value": run.Spec.Script.Key}, + map[string]any{"name": "script-digest", "value": run.Spec.Script.Digest}, + map[string]any{"name": "backend-api-version", "value": run.Spec.Backend.APIVersion}, + map[string]any{"name": "backend-kind", "value": run.Spec.Backend.Kind}, + map[string]any{"name": "queue", "value": run.Spec.Backend.Queue}, + map[string]any{"name": "runner-image", "value": run.Spec.RunnerImage}, + map[string]any{"name": "tolerations", "value": string(tolerationsJSON)}, + map[string]any{"name": "script-environment", "value": string(scriptEnvironmentJSON)}, + map[string]any{"name": "mpi-payload-mode", "value": run.Spec.Parameters["mpi-payload-mode"]}, + map[string]any{"name": "expected-gpu-product-regex", "value": run.Spec.Parameters["expected-gpu-product-regex"]}, + map[string]any{"name": "expected-gpu-count", "value": run.Spec.Parameters["expected-gpu-count"]}, + map[string]any{"name": "observed-gpu-capacity", "value": run.Spec.Parameters["observed-gpu-capacity"]}, + map[string]any{"name": "observed-gpu-allocatable", "value": run.Spec.Parameters["observed-gpu-allocatable"]}, + map[string]any{"name": "gpu-resource-count", "value": run.Spec.Parameters["gpu-resource-count"]}, + map[string]any{"name": "privileged", "value": run.Spec.Parameters["privileged"]}, + map[string]any{"name": "attempt", "value": fmt.Sprintf("%d", attempt)}, + }, + }, + }, + }} + workflow.SetOwnerReferences([]metav1.OwnerReference{ + *metav1.NewControllerRef(run, signalv1.GroupVersion.WithKind("CertificationRun")), + }) + + return workflow, nil +} + +func validEnvironmentName(value string) bool { + if value == "" || (value[0] != '_' && (value[0] < 'A' || value[0] > 'Z')) { + return false + } + + for index := 1; index < len(value); index++ { + character := value[index] + if (character < 'A' || character > 'Z') && + (character < '0' || character > '9') && + character != '_' { + return false + } + } + + return true +} + +func scriptEnvironment(parameters map[string]string) (map[string]string, error) { + keys := make([]string, 0, len(parameters)) + for key := range parameters { + keys = append(keys, key) + } + + sort.Strings(keys) + + environment := make(map[string]string, len(parameters)) + parameterByEnvironment := make(map[string]string, len(parameters)) + + for _, key := range keys { + environmentName := strings.ToUpper(strings.ReplaceAll(key, "-", "_")) + if !validEnvironmentName(environmentName) { + return nil, fmt.Errorf("profile parameter %q cannot be mapped to a script environment variable", key) + } + + if strings.HasPrefix(environmentName, "PROJECT_SIGNAL_") { + return nil, fmt.Errorf("profile parameter %q maps to reserved environment variable %s", key, environmentName) + } + + if _, reserved := reservedScriptEnvironmentNames[environmentName]; reserved { + return nil, fmt.Errorf("profile parameter %q maps to reserved environment variable %s", key, environmentName) + } + + if previous, exists := parameterByEnvironment[environmentName]; exists { + return nil, fmt.Errorf("profile parameters %q and %q both map to %s", previous, key, environmentName) + } + + parameterByEnvironment[environmentName] = key + environment[environmentName] = parameters[key] + } + + return environment, nil +} + +func (r *RunReconciler) retryOrFinish( + ctx context.Context, + run *signalv1.CertificationRun, + executionState string, + reason string, + message string, +) (ctrl.Result, error) { + if run.Status.Workflow.Attempt < run.Spec.MaxAttempts { + return r.startAttempt(ctx, run, run.Status.Workflow.Attempt+1) + } + + return r.finish(ctx, run, nil, executionState, "Inconclusive", reason+": "+message) +} + +func (r *RunReconciler) finish( + ctx context.Context, + run *signalv1.CertificationRun, + workflow *unstructured.Unstructured, + executionState string, + verdict string, + reason string, +) (ctrl.Result, error) { + if run.Status.FinishedAt == nil { + before := run.DeepCopy() + run.Status.ExecutionState = executionState + run.Status.Verdict = verdict + finalReason := reason + + if workflow != nil { + outputs, err := workflowParameters(workflow) + if err != nil { + run.Status.Verdict = "Inconclusive" + finalReason = "workflow outputs rejected: " + err.Error() + } else { + run.Status.Workflow.Outputs = outputs + } + } + + run.Status.Reason = finalReason + if executionState == "Completed" { + run.Status.Workflow.Phase = "Succeeded" + } else { + run.Status.Workflow.Phase = executionState + } + + now := metav1.NewTime(r.now()) + run.Status.FinishedAt = &now + + if err := r.Status().Patch(ctx, run, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, err + } + } else { + executionState = run.Status.ExecutionState + verdict = run.Status.Verdict + reason = run.Status.Reason + } + + evidence, err := r.evidenceDocument(ctx, run, executionState, verdict, reason) + if err != nil { + return ctrl.Result{}, err + } + + uri, digest, err := r.Evidence.Put(ctx, run.Spec.EvidencePath, evidence) + if err != nil { + return ctrl.Result{}, err + } + + before := run.DeepCopy() + run.Status.Evidence = signalv1.EvidenceStatus{URI: uri, Digest: digest} + + switch verdict { + case "Pass": + run.Status.Phase = signalv1.RunPhaseSucceeded + case "Fail", "Suspect": + run.Status.Phase = signalv1.RunPhaseFailed + default: + run.Status.Phase = signalv1.RunPhaseInconclusive + } + + if err := r.Status().Patch(ctx, run, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, err + } + + if err := projectTerminal(ctx, r.Client, run); err != nil { + return ctrl.Result{}, err + } + + if r.Recorder != nil { + r.Recorder.Eventf(run, nil, corev1.EventTypeNormal, "CertificationFinished", "FinishCertification", + "ExecutionState=%s Verdict=%s Reason=%s", executionState, verdict, reason) + } + + return ctrl.Result{RequeueAfter: time.Millisecond}, nil +} + +func uniqueTolerations(tolerations []corev1.Toleration) []corev1.Toleration { + unique := make([]corev1.Toleration, 0, len(tolerations)) + for _, toleration := range tolerations { + duplicate := false + + for _, existing := range unique { + if reflect.DeepEqual(existing, toleration) { + duplicate = true + + break + } + } + + if !duplicate { + unique = append(unique, toleration) + } + } + + return unique +} + +func (r *RunReconciler) evidenceDocument( + ctx context.Context, + run *signalv1.CertificationRun, + executionState string, + verdict string, + reason string, +) ([]byte, error) { + var snapshot corev1.ConfigMap + if err := r.Get(ctx, client.ObjectKey{Namespace: run.Namespace, Name: run.Spec.Script.ConfigMapName}, &snapshot); err != nil { + return nil, fmt.Errorf("read script snapshot for evidence: %w", err) + } + + doc := map[string]any{ + "apiVersion": "signal.unbounded-cloud.io/evidence/v1alpha1", + "runID": string(run.UID), + "runName": run.Name, + "profile": run.Spec.Profile, + "participants": run.Spec.Participants, + "workflow": run.Status.Workflow, + "workflowOutputs": run.Status.Workflow.Outputs, + "backend": run.Spec.Backend, + "runnerImage": run.Spec.RunnerImage, + "purpose": run.Spec.Purpose, + "stagePlan": run.Spec.StagePlan, + "parameters": run.Spec.Parameters, + "scriptSnapshot": run.Spec.Script, + "scriptContent": snapshot.Data[run.Spec.Script.Key], + "executionState": executionState, + "verdict": verdict, + "reason": reason, + "startedAt": run.Status.StartedAt, + "finishedAt": run.Status.FinishedAt.UTC().Format(time.RFC3339Nano), + } + + return json.MarshalIndent(doc, "", " ") +} + +func workflowParameters(workflow *unstructured.Unstructured) (map[string]string, error) { + parameters, found, err := unstructured.NestedSlice(workflow.Object, "status", "outputs", "parameters") + if err != nil { + return nil, fmt.Errorf("read parameters: %w", err) + } + + if !found { + return nil, nil + } + + outputs := make(map[string]string, len(parameters)) + for _, raw := range parameters { + parameter, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("parameter entry is not an object") + } + + name, nameOK := parameter["name"].(string) + value, valueOK := parameter["value"].(string) + + if !nameOK || !valueOK { + return nil, fmt.Errorf("parameter name and value must be strings") + } + + limit, allowed := workflowOutputLimits[name] + if !allowed { + return nil, fmt.Errorf("unexpected parameter %q", name) + } + + if len(value) > limit { + return nil, fmt.Errorf("parameter %q exceeds %d bytes", name, limit) + } + + if _, duplicate := outputs[name]; duplicate { + return nil, fmt.Errorf("duplicate parameter %q", name) + } + + outputs[name] = value + } + + return outputs, nil +} + +func (r *RunReconciler) cancel(ctx context.Context, run *signalv1.CertificationRun) (ctrl.Result, error) { + stopped, err := r.ensureWorkflowStopped(ctx, run) + if err != nil { + return ctrl.Result{}, err + } + + if !stopped { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + + before := run.DeepCopy() + run.Status.Phase = signalv1.RunPhaseCancelled + run.Status.ExecutionState = "Interrupted" + run.Status.Verdict = "Inconclusive" + run.Status.Reason = "canceled by operator" + now := metav1.NewTime(r.now()) + + run.Status.FinishedAt = &now + if err := r.Status().Patch(ctx, run, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{RequeueAfter: time.Millisecond}, nil +} + +func (r *RunReconciler) cleanup(ctx context.Context, run *signalv1.CertificationRun) (ctrl.Result, error) { + if err := projectTerminal(ctx, r.Client, run); err != nil { + return ctrl.Result{}, err + } + + released, err := r.releaseRunReservations(ctx, run) + if err != nil { + return ctrl.Result{}, err + } + + deleted := false + + snapshot := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: run.Namespace, Name: run.Spec.Script.ConfigMapName}} + if err := r.Delete(ctx, snapshot); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + + deleted = true + + if err := r.clearRequest(ctx, run); err != nil { + return ctrl.Result{}, err + } + + statusBefore := run.DeepCopy() + + run.Status.Cleanup = signalv1.CleanupStatus{ReservationsReleased: released, SnapshotDeleted: deleted} + if err := r.Status().Patch(ctx, run, client.MergeFrom(statusBefore)); err != nil { + return ctrl.Result{}, err + } + + before := run.DeepCopy() + + run.Finalizers = removeString(run.Finalizers, RunFinalizer) + if err := r.Patch(ctx, run, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, err + } + + if r.Recorder != nil { + r.Recorder.Eventf(run, nil, corev1.EventTypeNormal, "CertificationCleaned", "CleanCertification", + "Released participant reservations, cleared the request label, and deleted the script snapshot") + } + + return ctrl.Result{}, nil +} + +func (r *RunReconciler) reconcileDeleting(ctx context.Context, run *signalv1.CertificationRun) (ctrl.Result, error) { + if !containsString(run.Finalizers, RunFinalizer) { + return ctrl.Result{}, nil + } + + stopped, err := r.ensureWorkflowStopped(ctx, run) + if err != nil { + return ctrl.Result{}, err + } + + if !stopped { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + + if _, err := r.releaseRunReservations(ctx, run); err != nil { + return ctrl.Result{}, err + } + + if err := r.clearRequest(ctx, run); err != nil { + return ctrl.Result{}, err + } + + before := run.DeepCopy() + run.Finalizers = removeString(run.Finalizers, RunFinalizer) + + return ctrl.Result{}, r.Patch(ctx, run, client.MergeFrom(before)) +} + +func (r *RunReconciler) releaseRunReservations(ctx context.Context, run *signalv1.CertificationRun) (bool, error) { + for _, participant := range run.Spec.Participants { + var node corev1.Node + if err := r.Get(ctx, client.ObjectKey{Name: participant.NodeName}, &node); err != nil { + if apierrors.IsNotFound(err) { + continue + } + + return false, err + } + + if participant.NodeUID != "" && node.UID != participant.NodeUID { + continue + } + + if node.Annotations[ReservationLabel] != run.Name { + continue + } + + before := node.DeepCopy() + delete(node.Annotations, ReservationLabel) + + if err := r.Patch(ctx, &node, client.MergeFrom(before)); err != nil { + return false, err + } + } + + return true, nil +} + +func (r *RunReconciler) clearRequest(ctx context.Context, run *signalv1.CertificationRun) error { + var node corev1.Node + if err := r.Get(ctx, client.ObjectKey{Name: run.Spec.RequestNodeName}, &node); err != nil { + return client.IgnoreNotFound(err) + } + + if requestNodeUID := participantNodeUID(run, run.Spec.RequestNodeName); requestNodeUID != "" && node.UID != requestNodeUID { + return nil + } + + if node.Labels[RequestLabel] != run.Spec.Profile.Name { + return nil + } + + before := node.DeepCopy() + delete(node.Labels, RequestLabel) + + return r.Patch(ctx, &node, client.MergeFrom(before)) +} + +func (r *RunReconciler) ensureWorkflowStopped(ctx context.Context, run *signalv1.CertificationRun) (bool, error) { + if run.Status.Workflow.Name == "" { + return true, nil + } + + workflow := &unstructured.Unstructured{} + workflow.SetGroupVersionKind(workflowGVK) + + key := client.ObjectKey{Namespace: run.Namespace, Name: run.Status.Workflow.Name} + reader := r.APIReader + + if reader == nil { + reader = r.Client + } + + if err := reader.Get(ctx, key, workflow); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + + return false, fmt.Errorf("get Workflow before deletion: %w", err) + } + + if run.Status.Workflow.UID == "" || + string(workflow.GetUID()) != run.Status.Workflow.UID || + !metav1.IsControlledBy(workflow, run) { + return false, fmt.Errorf("refusing to delete Workflow with changed identity or ownership") + } + + propagation := metav1.DeletePropagationForeground + uid := workflow.GetUID() + + if err := r.Delete( + ctx, + workflow, + client.PropagationPolicy(propagation), + client.Preconditions{UID: &uid}, + ); err != nil && !apierrors.IsNotFound(err) { + return false, fmt.Errorf("delete Workflow before releasing participants: %w", err) + } + + if err := reader.Get(ctx, key, workflow); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + + return false, fmt.Errorf("confirm Workflow deletion: %w", err) + } + + return false, nil +} + +func participantNodeUID(run *signalv1.CertificationRun, nodeName string) types.UID { + for _, participant := range run.Spec.Participants { + if participant.NodeName == nodeName { + return participant.NodeUID + } + } + + return "" +} + +func workflowParameter(workflow *unstructured.Unstructured, name string) string { + parameters, _, err := unstructured.NestedSlice(workflow.Object, "status", "outputs", "parameters") + if err == nil { + if value := parameterValue(parameters, name); value != "" { + return value + } + } + + nodes, _, err := unstructured.NestedMap(workflow.Object, "status", "nodes") + if err != nil { + return "" + } + + for _, raw := range nodes { + node, ok := raw.(map[string]any) + if !ok { + continue + } + + outputs, ok := node["outputs"].(map[string]any) + if !ok { + continue + } + + parameters, ok := outputs["parameters"].([]any) + if ok { + if value := parameterValue(parameters, name); value != "" { + return value + } + } + } + + return "" +} + +func parameterValue(parameters []any, name string) string { + for _, raw := range parameters { + parameter, ok := raw.(map[string]any) + if !ok || parameter["name"] != name { + continue + } + + value, ok := parameter["value"].(string) + if !ok { + return "" + } + + return value + } + + return "" +} + +func workflowMessage(workflow *unstructured.Unstructured) string { + message, _, err := unstructured.NestedString(workflow.Object, "status", "message") + if err != nil { + return "workflow message is malformed" + } + + if message == "" { + return "workflow did not report a message" + } + + return message +} + +func digestString(content string) string { + sum := sha256.Sum256([]byte(content)) + + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func runName(node, profile string, now time.Time) string { + base := strings.ToLower(node + "-" + profile) + base = strings.Map(func(r rune) rune { + if r >= 'a' && r <= 'z' || r >= '0' && r <= '9' || r == '-' { + return r + } + + return '-' + }, base) + + base = strings.Trim(base, "-") + if len(base) > 40 { + base = strings.TrimRight(base[:40], "-") + } + + return fmt.Sprintf("%s-%x-%s", base, now.Unix(), rand.String(5)) +} + +func (r *AdmissionReconciler) now() time.Time { + if r.Now != nil { + return r.Now() + } + + return time.Now() +} + +func (r *RunReconciler) now() time.Time { + if r.Now != nil { + return r.Now() + } + + return time.Now() +} + +func containsString(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + + return false +} + +func removeString(values []string, remove string) []string { + result := values[:0] + for _, value := range values { + if value != remove { + result = append(result, value) + } + } + + return result +} + +func validateStagePlan(plan []signalv1.CertificationStage) error { + seen := map[string]struct{}{} + + for i := range plan { + stage := &plan[i] + if stage.ID == "" { + return fmt.Errorf("stage %d has an empty ID", i) + } + + if stage.DisplayName == "" { + return fmt.Errorf("stage %q has an empty display name", stage.ID) + } + + if _, ok := seen[stage.ID]; ok { + return fmt.Errorf("duplicate stage ID %q", stage.ID) + } + + seen[stage.ID] = struct{}{} + } + + return nil +} + +func AddToScheme(scheme *runtime.Scheme) error { + return signalv1.AddToScheme(scheme) +} diff --git a/internal/signal/controller_test.go b/internal/signal/controller_test.go new file mode 100644 index 0000000000..89d88c8d49 --- /dev/null +++ b/internal/signal/controller_test.go @@ -0,0 +1,1122 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signal + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" +) + +func TestAdmissionCreatesImmutableSnapshotAndPreventsDuplicate(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + node := readyNode("gpu-0", "example-gpu") + node.Labels[RequestLabel] = "gpu-smoke" + profile := testProfile("gpu-smoke", "example-gpu") + + const payload = "test payload\n" + + source := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, + Data: map[string]string{"payload.txt": payload}, + } + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node, profile, source).Build() + now := time.Unix(100, 0) + r := &AdmissionReconciler{Client: kube, Namespace: "signal", ScriptNamespace: "signal", Now: func() time.Time { return now }} + + if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}); err != nil { + t.Fatalf("Reconcile: %v", err) + } + + if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}); err != nil { + t.Fatalf("duplicate Reconcile: %v", err) + } + + var runs signalv1.CertificationRunList + if err := kube.List(context.Background(), &runs); err != nil { + t.Fatalf("list runs: %v", err) + } + + if len(runs.Items) != 1 { + t.Fatalf("runs = %d, want 1", len(runs.Items)) + } + + run := runs.Items[0] + if run.Spec.Script.Digest != digestString(payload) { + t.Fatalf("script digest = %q", run.Spec.Script.Digest) + } + + if len(run.Spec.StagePlan) != 1 || run.Spec.StagePlan[0].ID != "workload" { + t.Fatalf("stage plan = %#v, want immutable profile plan", run.Spec.StagePlan) + } + + if run.Spec.Purpose != signalv1.PurposeQuickCertification { + t.Fatalf("purpose = %q, want %q", run.Spec.Purpose, signalv1.PurposeQuickCertification) + } + + if run.Spec.Parameters["observed-gpu-capacity"] != "8" || + run.Spec.Parameters["observed-gpu-allocatable"] != "8" { + t.Fatalf("observed GPU inventory = %#v", run.Spec.Parameters) + } + + if run.Annotations[RunTriggerAnnotation] != "manual" { + t.Fatalf("trigger = %q, want manual", run.Annotations[RunTriggerAnnotation]) + } + + var snapshot corev1.ConfigMap + if err := kube.Get(context.Background(), client.ObjectKey{Namespace: "signal", Name: run.Spec.Script.ConfigMapName}, &snapshot); err != nil { + t.Fatalf("get snapshot: %v", err) + } + + if snapshot.Immutable == nil || !*snapshot.Immutable { + t.Fatal("snapshot is mutable") + } + + if snapshot.Data["payload.txt"] != payload { + t.Fatalf("snapshot payload = %q", snapshot.Data["payload.txt"]) + } + + var locked corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &locked); err != nil { + t.Fatalf("get locked node: %v", err) + } + + if locked.Annotations[ReservationLabel] != run.Name { + t.Fatalf("reservation = %q, want %q", locked.Annotations[ReservationLabel], run.Name) + } +} + +func TestAdmissionRejectsMutableRunnerImageBeforeReservation(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + node := readyNode("gpu-0", "example-gpu") + node.Labels[RequestLabel] = "gpu-smoke" + profile := testProfile("gpu-smoke", "example-gpu") + profile.Spec.RunnerImage = "registry.example/runner:latest" + source := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, + Data: map[string]string{"payload.txt": "test payload\n"}, + } + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node, profile, source).Build() + reconciler := &AdmissionReconciler{Client: kube, Namespace: "signal", ScriptNamespace: "signal"} + + _, err := reconciler.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}, + ) + if err == nil || !strings.Contains(err.Error(), "immutable sha256 digest") { + t.Fatalf("Reconcile error = %v", err) + } + + var got corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &got); err != nil { + t.Fatal(err) + } + + if got.Annotations[ReservationLabel] != "" { + t.Fatalf("node was reserved after rejected profile: %#v", got.Annotations) + } +} + +func TestParticipantSelectionRejectsUnsupportedMultiNode(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + request := readyNode("gpu-0", "H200") + peer := readyNode("gpu-1", "H200") + locked := readyNode("gpu-2", "H200") + locked.Annotations = map[string]string{ReservationLabel: "other-run"} + wrong := readyNode("gpu-3", "A100") + profile := testProfile("h200-pair", "H200") + profile.Spec.Participants.Count = 2 + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(request, peer, locked, wrong).Build() + r := &AdmissionReconciler{Client: kube} + + if _, err := r.selectParticipants(context.Background(), request, profile); err == nil || + !strings.Contains(err.Error(), "multi-participant execution is not implemented") { + t.Fatalf("selectParticipants error = %v", err) + } +} + +func TestAdmissionFailureMakesCreatedRunInertAndReleasesReservation(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + failSnapshot bool + failProjection bool + }{ + {name: "snapshot binding", failSnapshot: true}, + {name: "pending projection", failProjection: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + node := readyNode("gpu-0", "example-gpu") + node.Labels[RequestLabel] = "gpu-smoke" + profile := testProfile("gpu-smoke", "example-gpu") + source := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, + Data: map[string]string{"payload.txt": "test payload\n"}, + } + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(node, profile, source). + WithInterceptorFuncs(interceptor.Funcs{ + Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + if snapshot, ok := obj.(*corev1.ConfigMap); ok && + tc.failSnapshot && snapshot.Name != source.Name { + return errors.New("snapshot update failed") + } + + return c.Update(ctx, obj, opts...) + }, + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if projected, ok := obj.(*corev1.Node); ok && + tc.failProjection && projected.Labels[StateLabel] == "pending" { + return errors.New("projection patch failed") + } + + return c.Patch(ctx, obj, patch, opts...) + }, + }). + Build() + r := &AdmissionReconciler{ + Client: kube, + Namespace: "signal", + ScriptNamespace: "signal", + Now: func() time.Time { return time.Unix(100, 0) }, + } + + if _, err := r.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}, + ); err == nil { + t.Fatal("Reconcile error = nil, want failed admission") + } + + var runs signalv1.CertificationRunList + if err := kube.List(context.Background(), &runs); err != nil { + t.Fatalf("list runs: %v", err) + } + + for i := range runs.Items { + if runs.Items[i].DeletionTimestamp.IsZero() { + t.Fatalf("run %s remained active after failed admission", runs.Items[i].Name) + } + } + + var cleaned corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &cleaned); err != nil { + t.Fatalf("get cleaned node: %v", err) + } + + if cleaned.Annotations[ReservationLabel] != "" { + t.Fatalf("reservation remained after inerting failed run: %q", cleaned.Annotations[ReservationLabel]) + } + }) + } +} + +func TestAdmissionRecoversOrphanedReservationAndSnapshot(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + node := readyNode("gpu-0", "example-gpu") + node.Labels[RequestLabel] = "gpu-smoke" + node.Annotations = map[string]string{ReservationLabel: "orphaned-run"} + profile := testProfile("gpu-smoke", "example-gpu") + source := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, + Data: map[string]string{"payload.txt": "test payload\n"}, + } + immutable := true + orphanedSnapshot := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "orphaned-run-scripts", + Namespace: "signal", + Labels: map[string]string{SnapshotRunLabel: "orphaned-run"}, + }, + Immutable: &immutable, + Data: map[string]string{"payload.txt": "orphaned\n"}, + } + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(node, profile, source, orphanedSnapshot). + Build() + r := &AdmissionReconciler{ + Client: kube, + APIReader: kube, + Namespace: "signal", + ScriptNamespace: "signal", + Now: func() time.Time { return time.Unix(100, 0) }, + } + + if _, err := r.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}, + ); err != nil { + t.Fatalf("Reconcile: %v", err) + } + + if err := kube.Get( + context.Background(), + client.ObjectKeyFromObject(orphanedSnapshot), + &corev1.ConfigMap{}, + ); !apierrors.IsNotFound(err) { + t.Fatalf("orphaned snapshot get error = %v, want NotFound", err) + } + + var runs signalv1.CertificationRunList + if err := kube.List(context.Background(), &runs); err != nil { + t.Fatalf("list runs: %v", err) + } + + if len(runs.Items) != 1 { + t.Fatalf("runs = %d, want recovered admission", len(runs.Items)) + } + + var recovered corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &recovered); err != nil { + t.Fatalf("get recovered node: %v", err) + } + + if recovered.Annotations[ReservationLabel] != runs.Items[0].Name { + t.Fatalf("reservation = %q, want %q", recovered.Annotations[ReservationLabel], runs.Items[0].Name) + } +} + +func TestDeletingRunClearsReservationAndRequest(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + run := testRun("deleting-run") + run.Finalizers = []string{RunFinalizer} + node := readyNode(run.Spec.RequestNodeName, "A100") + node.Labels[RequestLabel] = run.Spec.Profile.Name + node.Annotations = map[string]string{ReservationLabel: run.Name} + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(run, node).Build() + reconciler := &RunReconciler{Client: kube} + + if _, err := reconciler.reconcileDeleting(context.Background(), run); err != nil { + t.Fatalf("reconcileDeleting: %v", err) + } + + var cleaned corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &cleaned); err != nil { + t.Fatalf("get cleaned node: %v", err) + } + + if cleaned.Annotations[ReservationLabel] != "" || cleaned.Labels[RequestLabel] != "" { + t.Fatalf("deletion cleanup left trigger metadata: labels=%v annotations=%v", cleaned.Labels, cleaned.Annotations) + } +} + +func TestCancellationWaitsForWorkflowDeletionBeforeReleasingNode(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + run := testRun("canceling-run") + run.Spec.Cancel = true + run.Status.Phase = signalv1.RunPhaseRunning + run.Status.Workflow = signalv1.WorkflowStatus{Name: "canceling-run-a1", Attempt: 1} + node := readyNode(run.Spec.RequestNodeName, "A100") + node.Annotations = map[string]string{ReservationLabel: run.Name} + workflow := workflowObject(run, "Running", "", "") + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationRun{}). + WithObjects(run, node, workflow). + WithInterceptorFuncs(interceptor.Funcs{ + Delete: func( + _ context.Context, + _ client.WithWatch, + _ client.Object, + _ ...client.DeleteOption, + ) error { + return nil + }, + }). + Build() + reconciler := &RunReconciler{Client: kube} + + result, err := reconciler.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}, + ) + if err != nil { + t.Fatalf("Reconcile: %v", err) + } + + if result.RequeueAfter == 0 { + t.Fatal("cancellation did not wait for Workflow deletion") + } + + var gotRun signalv1.CertificationRun + if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &gotRun); err != nil { + t.Fatal(err) + } + + if gotRun.Status.Phase.Terminal() { + t.Fatalf("run became terminal before Workflow deletion: %#v", gotRun.Status) + } + + var gotNode corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &gotNode); err != nil { + t.Fatal(err) + } + + if gotNode.Annotations[ReservationLabel] != run.Name { + t.Fatalf("reservation was released before Workflow deletion: %v", gotNode.Annotations) + } +} + +func TestWorkflowPinsAttemptInputs(t *testing.T) { + t.Parallel() + + run := testRun("run-1") + run.UID = types.UID("run-uid") + run.Spec.Parameters = map[string]string{ + "expected-gpu-count": "8", + "health-mode": "snapshot", + "mpi-payload-mode": "script", + } + run.Spec.Tolerations = []corev1.Toleration{{ + Key: "nvidia.com/gpu", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }} + + workflow, err := buildWorkflow(run, 2) + if err != nil { + t.Fatal(err) + } + + if workflow.GetName() != "run-1-a2" { + t.Fatalf("workflow name = %q", workflow.GetName()) + } + + if got, _, _ := unstructured.NestedInt64(workflow.Object, "spec", "ttlStrategy", "secondsAfterCompletion"); got != 21600 { + t.Fatalf("workflow completion TTL = %d, want 21600", got) + } + + if got, _, _ := unstructured.NestedString(workflow.Object, "spec", "podGC", "deleteDelayDuration"); got != "2h" { + t.Fatalf("workflow pod GC delay = %q, want 2h", got) + } + + if len(workflow.GetOwnerReferences()) != 1 || workflow.GetOwnerReferences()[0].UID != run.UID { + t.Fatalf("owner references = %#v", workflow.GetOwnerReferences()) + } + + parameters, found, err := unstructured.NestedSlice(workflow.Object, "spec", "arguments", "parameters") + if err != nil || !found { + t.Fatalf("workflow parameters: found=%t err=%v", found, err) + } + + values := map[string]string{} + + for _, raw := range parameters { + parameter := raw.(map[string]any) + values[parameter["name"].(string)] = parameter["value"].(string) + } + + if values["script-digest"] != run.Spec.Script.Digest || values["backend-kind"] != "MPIJob" || values["attempt"] != "2" { + t.Fatalf("workflow values = %#v", values) + } + + if values["script-configmap"] != run.Spec.Script.ConfigMapName || values["script-configmap"] == "scripts" { + t.Fatalf("workflow must mount the immutable snapshot, values = %#v", values) + } + + if !strings.Contains(values["tolerations"], `"key":"nvidia.com/gpu"`) { + t.Fatalf("workflow tolerations = %q", values["tolerations"]) + } + + var environment map[string]string + if err := json.Unmarshal([]byte(values["script-environment"]), &environment); err != nil { + t.Fatalf("decode script environment: %v", err) + } + + if environment["EXPECTED_GPU_COUNT"] != "8" || + environment["HEALTH_MODE"] != "snapshot" || + values["mpi-payload-mode"] != "script" { + t.Fatalf("workflow script policy = %#v, parameters = %#v", environment, values) + } + + var tolerations []corev1.Toleration + if err := json.Unmarshal([]byte(values["tolerations"]), &tolerations); err != nil { + t.Fatalf("decode workflow tolerations: %v", err) + } + + if len(tolerations) != 1 { + t.Fatalf("workflow tolerations = %#v, want one deduplicated GPU toleration", tolerations) + } +} + +func TestBuildWorkflowRejectsParameterThatCannotBecomeEnvironmentVariable(t *testing.T) { + t.Parallel() + + run := testRun("run-invalid-parameter") + run.Spec.Parameters = map[string]string{"invalid.parameter": "value"} + + if _, err := buildWorkflow(run, 1); err == nil || + !strings.Contains(err.Error(), "cannot be mapped to a script environment variable") { + t.Fatalf("buildWorkflow error = %v", err) + } +} + +func TestScriptEnvironmentRejectsCollidingParameterNames(t *testing.T) { + t.Parallel() + + _, err := scriptEnvironment(map[string]string{ + "gpu-devices": "0,1", + "gpu_devices": "2,3", + }) + if err == nil || !strings.Contains(err.Error(), "both map to GPU_DEVICES") { + t.Fatalf("scriptEnvironment error = %v", err) + } +} + +func TestScriptEnvironmentRejectsReservedNames(t *testing.T) { + t.Parallel() + + for _, parameter := range []string{"path", "bash-env", "project-signal-run-uid"} { + _, err := scriptEnvironment(map[string]string{parameter: "value"}) + if err == nil || !strings.Contains(err.Error(), "reserved environment variable") { + t.Fatalf("scriptEnvironment(%q) error = %v", parameter, err) + } + } +} + +func TestWorkflowParametersRejectMalformedEvidence(t *testing.T) { + t.Parallel() + + tests := map[string][]any{ + "duplicate": { + map[string]any{"name": "verdict", "value": "Pass"}, + map[string]any{"name": "verdict", "value": "Fail"}, + }, + "unexpected": { + map[string]any{"name": "unbounded", "value": "value"}, + }, + "oversized": { + map[string]any{"name": "measurements", "value": strings.Repeat("x", 16*1024+1)}, + }, + "wrong type": { + map[string]any{"name": "verdict", "value": true}, + }, + } + + for name, parameters := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + workflow := &unstructured.Unstructured{Object: map[string]any{ + "status": map[string]any{ + "outputs": map[string]any{"parameters": parameters}, + }, + }} + + if _, err := workflowParameters(workflow); err == nil { + t.Fatal("workflowParameters succeeded") + } + }) + } +} + +func TestStartAttemptRecoversExistingOwnedWorkflow(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + run := testRun("run-recovered") + + workflow, err := buildWorkflow(run, 1) + if err != nil { + t.Fatal(err) + } + + workflow.SetUID("workflow-uid") + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationRun{}). + WithObjects(run, workflow). + Build() + r := &RunReconciler{Client: kube, Evidence: &memoryEvidenceStore{}} + + if _, err := r.startAttempt(context.Background(), run, 1); err != nil { + t.Fatalf("startAttempt: %v", err) + } + + var got signalv1.CertificationRun + if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { + t.Fatalf("get run: %v", err) + } + + if got.Status.Workflow.Name != workflow.GetName() || + got.Status.Workflow.UID != string(workflow.GetUID()) || + got.Status.Workflow.Attempt != 1 { + t.Fatalf("workflow status = %#v", got.Status.Workflow) + } +} + +func TestWorkflowParameterFallsBackToEntrypointNodeOutputs(t *testing.T) { + t.Parallel() + + workflow := &unstructured.Unstructured{Object: map[string]any{ + "status": map[string]any{ + "nodes": map[string]any{ + "root": map[string]any{ + "outputs": map[string]any{ + "parameters": []any{map[string]any{"name": "verdict", "value": "Pass"}}, + }, + }, + }, + }, + }} + + if got := workflowParameter(workflow, "verdict"); got != "Pass" { + t.Fatalf("verdict = %q", got) + } +} + +func TestValidateStagePlanRejectsInvalidAndDuplicateStages(t *testing.T) { + t.Parallel() + + if err := validateStagePlan([]signalv1.CertificationStage{{DisplayName: "Missing ID"}}); err == nil { + t.Fatal("expected empty stage ID to fail validation") + } + + if err := validateStagePlan([]signalv1.CertificationStage{ + {ID: "workload", DisplayName: "First"}, + {ID: "workload", DisplayName: "Duplicate"}, + }); err == nil { + t.Fatal("expected duplicate stage IDs to fail validation") + } +} + +func TestTerminalVerdictPersistsEvidenceAndCleanupReleasesNode(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + run := testRun("run-pass") + run.Status = signalv1.CertificationRunStatus{ + Phase: signalv1.RunPhaseRunning, + Workflow: signalv1.WorkflowStatus{Name: "run-pass-a1", Attempt: 1}, + StartedAt: ptrTime(metav1.NewTime(time.Unix(100, 0))), + } + node := readyNode("gpu-0", "A100") + node.Labels[RequestLabel] = run.Spec.Profile.Name + node.Annotations = map[string]string{ReservationLabel: run.Name} + snapshot := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Script.ConfigMapName, Namespace: run.Namespace}, + Data: map[string]string{run.Spec.Script.Key: "test payload\n"}, + } + workflow := workflowObject(run, "Succeeded", "Pass", "Healthy") + store := &memoryEvidenceStore{} + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationRun{}). + WithObjects(run, node, snapshot, workflow). + Build() + r := &RunReconciler{Client: kube, Evidence: store, Now: func() time.Time { return time.Unix(200, 0) }} + + if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}); err != nil { + t.Fatalf("terminal Reconcile: %v", err) + } + + if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}); err != nil { + t.Fatalf("cleanup Reconcile: %v", err) + } + + var got signalv1.CertificationRun + if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { + t.Fatalf("get run: %v", err) + } + + if got.Status.Phase != signalv1.RunPhaseSucceeded || + got.Status.Verdict != "Pass" || + got.Status.Workflow.Phase != "Succeeded" { + t.Fatalf("status = %#v", got.Status) + } + + if got.Status.Workflow.Outputs["measurements"] != "checked=14" { + t.Fatalf("retained workflow outputs = %#v", got.Status.Workflow.Outputs) + } + + if got.Status.Evidence.URI == "" || len(store.data) == 0 { + t.Fatal("evidence was not persisted") + } + + var evidence struct { + Workflow signalv1.WorkflowStatus `json:"workflow"` + } + if err := json.Unmarshal(store.data, &evidence); err != nil { + t.Fatalf("decode evidence: %v", err) + } + + if evidence.Workflow.Phase != "Succeeded" || + evidence.Workflow.Outputs["measurements"] != "checked=14" { + t.Fatalf("evidence workflow = %#v", evidence.Workflow) + } + + if !got.Status.Cleanup.ReservationsReleased || !got.Status.Cleanup.SnapshotDeleted { + t.Fatalf("cleanup status = %#v", got.Status.Cleanup) + } + + if containsString(got.Finalizers, RunFinalizer) { + t.Fatal("cleanup finalizer remains") + } + + var released corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &released); err != nil { + t.Fatalf("get released node: %v", err) + } + + if released.Annotations[ReservationLabel] != "" || released.Labels[RequestLabel] != "" { + t.Fatalf("node was not released: labels=%v annotations=%v", released.Labels, released.Annotations) + } + + if released.Labels[StateLabel] != "passed" || + released.Labels[EligibilityLabel] != "production" || + released.Annotations[LastSuccessfulRunAnnotation] != run.Name { + t.Fatalf("terminal projection was not retained: labels=%v annotations=%v", released.Labels, released.Annotations) + } +} + +func TestTerminalFinalizationRetriesWithIdenticalEvidence(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + run := testRun("run-retry-finalization") + run.Status = signalv1.CertificationRunStatus{ + Phase: signalv1.RunPhaseRunning, + Workflow: signalv1.WorkflowStatus{Name: "run-retry-finalization-a1", Attempt: 1}, + StartedAt: ptrTime(metav1.NewTime(time.Unix(100, 0))), + } + node := readyNode("gpu-0", "A100") + node.Annotations = map[string]string{ReservationLabel: run.Name} + snapshot := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Script.ConfigMapName, Namespace: run.Namespace}, + Data: map[string]string{run.Spec.Script.Key: "test payload\n"}, + } + workflow := workflowObject(run, "Succeeded", "Pass", "Healthy") + statusPatches := 0 + base := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationRun{}). + WithObjects(run, node, snapshot, workflow). + Build() + kube := interceptor.NewClient(base, interceptor.Funcs{ + SubResourcePatch: func( + ctx context.Context, + c client.Client, + subResource string, + obj client.Object, + patch client.Patch, + opts ...client.SubResourcePatchOption, + ) error { + if subResource == "status" { + statusPatches++ + if statusPatches == 2 { + return errors.New("status patch interrupted after evidence persistence") + } + } + + return c.SubResource(subResource).Patch(ctx, obj, patch, opts...) + }, + }) + store := &immutableMemoryEvidenceStore{} + r := &RunReconciler{ + Client: kube, + Evidence: store, + Now: func() time.Time { return time.Unix(200+int64(statusPatches), 0) }, + } + key := reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)} + + if _, err := r.Reconcile(context.Background(), key); err == nil { + t.Fatal("first Reconcile error = nil, want interrupted status patch") + } + + var interrupted signalv1.CertificationRun + if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &interrupted); err != nil { + t.Fatalf("get interrupted run: %v", err) + } + + beforeCancel := interrupted.DeepCopy() + interrupted.Spec.Cancel = true + + if err := kube.Patch(context.Background(), &interrupted, client.MergeFrom(beforeCancel)); err != nil { + t.Fatalf("request cancellation: %v", err) + } + + if _, err := r.Reconcile(context.Background(), key); err != nil { + t.Fatalf("retry Reconcile: %v", err) + } + + var got signalv1.CertificationRun + if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { + t.Fatalf("get run: %v", err) + } + + if got.Status.Phase != signalv1.RunPhaseSucceeded { + t.Fatalf("phase = %q, want Succeeded", got.Status.Phase) + } + + if store.puts != 2 { + t.Fatalf("evidence puts = %d, want 2 identical attempts", store.puts) + } +} + +func TestWorkflowReplacementIsRejected(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + run := testRun("run-workflow-replaced") + run.Status = signalv1.CertificationRunStatus{ + Phase: signalv1.RunPhaseRunning, + Workflow: signalv1.WorkflowStatus{ + Name: "run-workflow-replaced-a1", + UID: "original-workflow-uid", + Attempt: 1, + }, + } + replacement := workflowObject(run, "Succeeded", "Pass", "forged") + replacement.SetUID("replacement-workflow-uid") + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationRun{}). + WithObjects(run, replacement). + Build() + r := &RunReconciler{Client: kube, Evidence: &memoryEvidenceStore{}} + + if _, err := r.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}, + ); err == nil || !strings.Contains(err.Error(), "refusing concurrent retry") { + t.Fatalf("Reconcile error = %v, want identity rejection", err) + } + + var got signalv1.CertificationRun + if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { + t.Fatalf("get run: %v", err) + } + + if got.Status.Workflow.Attempt != 1 || + got.Status.Workflow.Name != "run-workflow-replaced-a1" { + t.Fatalf("replacement workflow triggered a retry: %#v", got.Status.Workflow) + } + + var workflows unstructured.UnstructuredList + workflows.SetGroupVersionKind(workflowGVK.GroupVersion().WithKind("WorkflowList")) + + if err := kube.List(context.Background(), &workflows, client.InNamespace(run.Namespace)); err != nil { + t.Fatalf("list Workflows: %v", err) + } + + if len(workflows.Items) != 1 { + t.Fatalf("workflows = %d, want only mismatched original name", len(workflows.Items)) + } +} + +func TestCancellationRefusesToDeleteReplacementWorkflow(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + run := testRun("run-cancel-replacement") + run.Spec.Cancel = true + run.Status = signalv1.CertificationRunStatus{ + Phase: signalv1.RunPhaseRunning, + Workflow: signalv1.WorkflowStatus{ + Name: "run-cancel-replacement-a1", + UID: "original-workflow-uid", + Attempt: 1, + }, + } + replacement := workflowObject(run, "Running", "", "") + replacement.SetUID("replacement-workflow-uid") + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationRun{}). + WithObjects(run, replacement). + Build() + r := &RunReconciler{Client: kube} + + if _, err := r.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}, + ); err == nil || !strings.Contains(err.Error(), "refusing to delete Workflow") { + t.Fatalf("Reconcile error = %v, want identity rejection", err) + } + + var remaining unstructured.Unstructured + remaining.SetGroupVersionKind(workflowGVK) + + if err := kube.Get(context.Background(), client.ObjectKeyFromObject(replacement), &remaining); err != nil { + t.Fatalf("replacement Workflow was deleted: %v", err) + } +} + +func TestWorkflowFailureRetriesWithoutSwitchingPinnedInputs(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + run := testRun("run-retry") + run.Spec.MaxAttempts = 2 + run.Status = signalv1.CertificationRunStatus{ + Phase: signalv1.RunPhaseRunning, + Workflow: signalv1.WorkflowStatus{Name: "run-retry-a1", Attempt: 1}, + } + workflow := workflowObject(run, "Failed", "", "") + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationRun{}). + WithObjects(run, workflow). + Build() + r := &RunReconciler{Client: kube, Evidence: &memoryEvidenceStore{}} + + if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}); err != nil { + t.Fatalf("Reconcile: %v", err) + } + + var got signalv1.CertificationRun + if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { + t.Fatalf("get run: %v", err) + } + + if got.Status.Workflow.Attempt != 2 || got.Status.Workflow.Name != "run-retry-a2" { + t.Fatalf("workflow status = %#v", got.Status.Workflow) + } + + var retry unstructured.Unstructured + retry.SetGroupVersionKind(workflowGVK) + + if err := kube.Get(context.Background(), client.ObjectKey{Namespace: run.Namespace, Name: "run-retry-a2"}, &retry); err != nil { + t.Fatalf("get retry workflow: %v", err) + } + + if value := workflowArgument(&retry, "script-digest"); value != run.Spec.Script.Digest { + t.Fatalf("retry script digest = %q", value) + } + + if value := workflowArgument(&retry, "backend-kind"); value != run.Spec.Backend.Kind { + t.Fatalf("retry backend = %q", value) + } +} + +func TestCanceledRunCleansUpDespiteCancelFlag(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + run := testRun("run-canceled") + run.Spec.Cancel = true + run.Status.Phase = signalv1.RunPhaseCancelled + node := readyNode("gpu-0", "A100") + node.Labels[RequestLabel] = run.Spec.Profile.Name + node.Annotations = map[string]string{ReservationLabel: run.Name} + snapshot := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Script.ConfigMapName, Namespace: run.Namespace}, + Data: map[string]string{run.Spec.Script.Key: "test payload\n"}, + } + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationRun{}). + WithObjects(run, node, snapshot). + Build() + r := &RunReconciler{Client: kube, Evidence: &memoryEvidenceStore{}} + + if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}); err != nil { + t.Fatalf("Reconcile: %v", err) + } + + var got signalv1.CertificationRun + if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { + t.Fatalf("get run: %v", err) + } + + if containsString(got.Finalizers, RunFinalizer) || !got.Status.Cleanup.ReservationsReleased { + t.Fatalf("run was not cleaned up: %#v", got) + } +} + +type memoryEvidenceStore struct { + data []byte +} + +func (s *memoryEvidenceStore) Put(_ context.Context, path string, data []byte) (string, string, error) { + s.data = append([]byte(nil), data...) + + return "memory://" + path, digestString(string(data)), nil +} + +type immutableMemoryEvidenceStore struct { + data []byte + puts int +} + +func (s *immutableMemoryEvidenceStore) Put(_ context.Context, path string, data []byte) (string, string, error) { + s.puts++ + if s.data != nil && string(s.data) != string(data) { + return "", "", errors.New("evidence changed across retry") + } + + s.data = append([]byte(nil), data...) + + return "memory://" + path, digestString(string(data)), nil +} + +func testScheme(t *testing.T) *runtime.Scheme { + t.Helper() + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + if err := signalv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + scheme.AddKnownTypeWithName(workflowGVK, &unstructured.Unstructured{}) + scheme.AddKnownTypeWithName(workflowGVK.GroupVersion().WithKind("WorkflowList"), &unstructured.UnstructuredList{}) + + return scheme +} + +func readyNode(name, sku string) *corev1.Node { + return &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(name + "-uid"), + Labels: map[string]string{ + "node.kubernetes.io/instance-type": sku, + }, + }, + Status: corev1.NodeStatus{ + Capacity: corev1.ResourceList{corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("8")}, + Allocatable: corev1.ResourceList{corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("8")}, + Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}, + }, + } +} + +func testProfile(name, sku string) *signalv1.CertificationProfile { + return &signalv1.CertificationProfile{ + ObjectMeta: metav1.ObjectMeta{Name: name, UID: types.UID(name + "-uid"), Generation: 3}, + Spec: signalv1.CertificationProfileSpec{ + Participants: signalv1.ParticipantPolicy{ + Count: 1, + Selector: metav1.LabelSelector{MatchLabels: map[string]string{ + "node.kubernetes.io/instance-type": sku, + }}, + RequiredResources: corev1.ResourceList{corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("1")}, + }, + Script: signalv1.ScriptReference{Name: "scripts", Key: "payload.txt"}, + Workflow: signalv1.WorkflowReference{Name: "gpu-cert"}, + Backend: signalv1.ExecutionBackend{APIVersion: "kubeflow.org/v2beta1", Kind: "MPIJob"}, + RunnerImage: "runner@sha256:" + strings.Repeat("a", 64), + Purpose: signalv1.PurposeQuickCertification, + StagePlan: []signalv1.CertificationStage{{ID: "workload", DisplayName: "Workload", Category: "compute", Required: true}}, + EvidencePath: "signal/" + sku, + Deadline: metav1.Duration{Duration: 10 * time.Minute}, + MaxAttempts: 2, + }, + } +} + +func testRun(name string) *signalv1.CertificationRun { + return &signalv1.CertificationRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "signal", + UID: types.UID(name + "-uid"), + Finalizers: []string{RunFinalizer}, + }, + Spec: signalv1.CertificationRunSpec{ + Profile: signalv1.PinnedProfile{Name: "gpu-profile", UID: "profile-uid", Generation: 3}, + Participants: []signalv1.Participant{{NodeName: "gpu-0", NodeUID: "gpu-0-uid"}}, + Script: signalv1.ScriptSnapshot{ConfigMapName: name + "-scripts", Key: "payload.txt", Digest: digestString("test payload\n")}, + Workflow: signalv1.WorkflowReference{Name: "gpu-cert"}, + Backend: signalv1.ExecutionBackend{APIVersion: "kubeflow.org/v2beta1", Kind: "MPIJob"}, + RunnerImage: "runner@sha256:" + strings.Repeat("a", 64), + EvidencePath: "signal/" + name + ".json", + Deadline: metav1.Duration{Duration: 10 * time.Minute}, + MaxAttempts: 2, + RequestNodeName: "gpu-0", + }, + } +} + +func workflowObject(run *signalv1.CertificationRun, phase, verdict, reason string) *unstructured.Unstructured { + const workflowUID = "workflow-uid" + + run.Status.Workflow.UID = workflowUID + workflow := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": workflowGVK.GroupVersion().String(), + "kind": workflowGVK.Kind, + "metadata": map[string]any{ + "name": run.Status.Workflow.Name, + "namespace": run.Namespace, + "uid": workflowUID, + "ownerReferences": []any{ + map[string]any{ + "apiVersion": signalv1.GroupVersion.String(), + "kind": "CertificationRun", + "name": run.Name, + "uid": string(run.UID), + "controller": true, + "blockOwnerDeletion": true, + }, + }, + }, + "status": map[string]any{ + "phase": phase, + "message": "test workflow message", + "outputs": map[string]any{"parameters": []any{ + map[string]any{"name": "verdict", "value": verdict}, + map[string]any{"name": "reason", "value": reason}, + map[string]any{"name": "measurements", "value": "checked=14"}, + }}, + }, + }} + + return workflow +} + +func workflowArgument(workflow *unstructured.Unstructured, name string) string { + parameters, _, _ := unstructured.NestedSlice(workflow.Object, "spec", "arguments", "parameters") + for _, raw := range parameters { + parameter := raw.(map[string]any) + if parameter["name"] == name { + return fmt.Sprint(parameter["value"]) + } + } + + return "" +} + +func ptrTime(value metav1.Time) *metav1.Time { + return &value +} diff --git a/internal/signal/evidence.go b/internal/signal/evidence.go new file mode 100644 index 0000000000..dec8e50152 --- /dev/null +++ b/internal/signal/evidence.go @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signal + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "net/url" + "strings" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type EvidenceStore interface { + Put(context.Context, string, []byte) (string, string, error) +} + +type KubernetesEvidenceStore struct { + kube client.Client + namespace string +} + +func NewKubernetesEvidenceStore(kube client.Client, namespace string) (*KubernetesEvidenceStore, error) { + if kube == nil { + return nil, fmt.Errorf("kubernetes client is required") + } + + if namespace == "" { + return nil, fmt.Errorf("evidence namespace is required") + } + + return &KubernetesEvidenceStore{kube: kube, namespace: namespace}, nil +} + +func (s *KubernetesEvidenceStore) Put(ctx context.Context, path string, data []byte) (string, string, error) { + name := strings.TrimSuffix(url.PathEscape(strings.TrimSuffix(path[strings.LastIndex(path, "/")+1:], ".json")), ".json") + if name == "" { + return "", "", fmt.Errorf("evidence path is required") + } + + sum := sha256.Sum256(data) + digest := "sha256:" + hex.EncodeToString(sum[:]) + + compressed, err := compressEvidence(data) + if err != nil { + return "", "", err + } + + immutable := true + + evidence := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name + "-evidence", + Namespace: s.namespace, + Labels: map[string]string{ + SnapshotRunLabel: name, + }, + Annotations: map[string]string{ + "signal.unbounded-cloud.io/evidence-digest": digest, + }, + }, + Immutable: &immutable, + BinaryData: map[string][]byte{ + "evidence.json.gz": compressed, + }, + } + if err := s.kube.Create(ctx, evidence); err != nil { + if !apierrors.IsAlreadyExists(err) { + return "", "", fmt.Errorf("create evidence ConfigMap: %w", err) + } + + var existing corev1.ConfigMap + if getErr := s.kube.Get(ctx, client.ObjectKeyFromObject(evidence), &existing); getErr != nil { + return "", "", fmt.Errorf("get existing evidence ConfigMap: %w", getErr) + } + + if existing.Annotations["signal.unbounded-cloud.io/evidence-digest"] != digest || + !bytes.Equal(existing.BinaryData["evidence.json.gz"], compressed) { + return "", "", fmt.Errorf("evidence ConfigMap %s already exists with different content", evidence.Name) + } + } + + return fmt.Sprintf("kubernetes://%s/configmaps/%s/evidence.json.gz", s.namespace, evidence.Name), digest, nil +} + +func compressEvidence(data []byte) ([]byte, error) { + var compressed bytes.Buffer + + writer := gzip.NewWriter(&compressed) + if _, err := writer.Write(data); err != nil { + return nil, fmt.Errorf("compress evidence: %w", err) + } + + if err := writer.Close(); err != nil { + return nil, fmt.Errorf("finish evidence compression: %w", err) + } + + return compressed.Bytes(), nil +} diff --git a/internal/signal/evidence_test.go b/internal/signal/evidence_test.go new file mode 100644 index 0000000000..b24f269dea --- /dev/null +++ b/internal/signal/evidence_test.go @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signal + +import ( + "bytes" + "compress/gzip" + "context" + "io" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestKubernetesEvidenceStoreReturnsQueryableRunURIAndDigest(t *testing.T) { + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + kube := fake.NewClientBuilder().WithScheme(scheme).Build() + + store, err := NewKubernetesEvidenceStore(kube, "project-signal") + if err != nil { + t.Fatal(err) + } + + data := []byte(`{"verdict":"Pass"}`) + + uri, digest, err := store.Put(context.Background(), "project-signal/example/run-a.json", data) + if err != nil { + t.Fatal(err) + } + + if uri != "kubernetes://project-signal/configmaps/run-a-evidence/evidence.json.gz" { + t.Fatalf("unexpected URI %q", uri) + } + + if !strings.HasPrefix(digest, "sha256:") || len(digest) != len("sha256:")+64 { + t.Fatalf("unexpected digest %q", digest) + } + + var evidence corev1.ConfigMap + if err := kube.Get( + context.Background(), + client.ObjectKey{Namespace: "project-signal", Name: "run-a-evidence"}, + &evidence, + ); err != nil { + t.Fatal(err) + } + + if evidence.Immutable == nil || !*evidence.Immutable { + t.Fatal("evidence ConfigMap is mutable") + } + + reader, err := gzip.NewReader(bytes.NewReader(evidence.BinaryData["evidence.json.gz"])) + if err != nil { + t.Fatal(err) + } + + got, err := io.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + + if err := reader.Close(); err != nil { + t.Fatal(err) + } + + if string(got) != string(data) { + t.Fatalf("persisted evidence = %q", got) + } +} diff --git a/internal/signal/policy.go b/internal/signal/policy.go new file mode 100644 index 0000000000..118f0303bb --- /dev/null +++ b/internal/signal/policy.go @@ -0,0 +1,623 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signal + +import ( + "context" + "fmt" + "slices" + "sort" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + + machinav1 "github.com/Azure/unbounded/api/machina/v1alpha3" + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" +) + +const defaultPolicyEvaluationInterval = time.Minute + +type RecurringReconciler struct { + client.Client + Reader client.Reader + Namespace string + Admission *AdmissionReconciler + Now func() time.Time +} + +func (r *RecurringReconciler) SetupWithManager(mgr ctrl.Manager) error { + if r.Reader == nil { + r.Reader = mgr.GetAPIReader() + } + + return ctrl.NewControllerManagedBy(mgr). + For(&signalv1.CertificationProfile{}). + WithOptions(controller.Options{MaxConcurrentReconciles: 1}). + Complete(r) +} + +func (r *RecurringReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var profile signalv1.CertificationProfile + if err := r.Get(ctx, req.NamespacedName, &profile); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + policy := profile.Spec.Recurring + if policy == nil || !policy.Enabled { + return ctrl.Result{}, nil + } + + now := r.now() + + interval := policy.EvaluationInterval.Duration + if interval <= 0 { + interval = defaultPolicyEvaluationInterval + } + + before := profile.DeepCopy() + profile.Status.ObservedGeneration = profile.Generation + + if err := validateRecurringPolicy(policy); err != nil { + setProfileCondition(&profile, now, metav1.ConditionFalse, "InvalidPolicy", err.Error()) + + return ctrl.Result{RequeueAfter: interval}, r.Status().Patch(ctx, &profile, client.MergeFrom(before)) + } + + if policyExpired(policy, now) { + for index := range profile.Status.Nodes { + profile.Status.Nodes[index].Suspended = true + profile.Status.Nodes[index].Reason = "PolicyExpired" + profile.Status.Nodes[index].NextDueAt = nil + } + + setProfileCondition(&profile, now, metav1.ConditionFalse, "Expired", + fmt.Sprintf("recurring admission expired at %s", policy.ExpiresAt.UTC().Format(time.RFC3339))) + + if err := r.Status().Patch(ctx, &profile, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{}, r.projectStatuses(ctx, &profile) + } + + if r.Admission == nil { + return ctrl.Result{}, fmt.Errorf("admission reconciler is required") + } + + r.Admission.admissionMu.Lock() + defer r.Admission.admissionMu.Unlock() + + var runs signalv1.CertificationRunList + if err := r.Reader.List(ctx, &runs, client.InNamespace(r.Namespace)); err != nil { + return ctrl.Result{}, fmt.Errorf("list CertificationRuns: %w", err) + } + + activeByProfile, activeByDomain := activeBudgets(runs.Items, &profile) + if policy.FailureDomainLabel != "" { + for index := range runs.Items { + run := &runs.Items[index] + if run.Spec.Profile.Name != profile.Name || run.Status.Phase.Terminal() { + continue + } + + for _, participant := range run.Spec.Participants { + var node corev1.Node + if err := r.Reader.Get(ctx, client.ObjectKey{Name: participant.NodeName}, &node); err != nil { + return ctrl.Result{}, fmt.Errorf( + "get active participant %s for failure-domain accounting: %w", + participant.NodeName, + err, + ) + } + + activeByDomain[node.Labels[policy.FailureDomainLabel]]++ + } + } + } + + statuses := make([]signalv1.RecurringNodeStatus, 0, len(policy.NodeNames)) + + for _, nodeName := range policy.NodeNames { + old := recurringNodeStatus(profile.Status.Nodes, nodeName) + + status, admitted, err := r.evaluateNode(ctx, &profile, nodeName, old, runs.Items, activeByProfile, activeByDomain, now) + if err != nil { + status.Reason = "EvaluationError: " + err.Error() + statuses = append(statuses, status) + + continue + } + + statuses = append(statuses, status) + + if admitted { + activeByProfile++ + + var node corev1.Node + if getErr := r.Reader.Get(ctx, client.ObjectKey{Name: nodeName}, &node); getErr != nil { + return ctrl.Result{}, fmt.Errorf( + "get admitted participant %s for failure-domain accounting: %w", + nodeName, + getErr, + ) + } + + activeByDomain[node.Labels[policy.FailureDomainLabel]]++ + } + } + + profile.Status.Nodes = statuses + setProfileCondition(&profile, now, metav1.ConditionTrue, "Evaluated", + fmt.Sprintf("evaluated %d explicitly opted-in nodes", len(statuses))) + + if err := r.Status().Patch(ctx, &profile, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, err + } + + if err := r.projectStatuses(ctx, &profile); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{RequeueAfter: interval}, nil +} + +func (r *RecurringReconciler) projectStatuses(ctx context.Context, profile *signalv1.CertificationProfile) error { + for _, status := range profile.Status.Nodes { + if err := projectRecurring(ctx, r.Client, profile, status); err != nil { + return err + } + } + + return nil +} + +func (r *RecurringReconciler) evaluateNode( + ctx context.Context, + profile *signalv1.CertificationProfile, + nodeName string, + old signalv1.RecurringNodeStatus, + runs []signalv1.CertificationRun, + activeByProfile int32, + activeByDomain map[string]int32, + now time.Time, +) (signalv1.RecurringNodeStatus, bool, error) { + status := old + status.NodeName = nodeName + evaluated := metav1.NewTime(now) + status.LastEvaluatedAt = &evaluated + status.Suspended = false + + var node corev1.Node + if err := r.Reader.Get(ctx, client.ObjectKey{Name: nodeName}, &node); err != nil { + status.IdleSince = nil + + return status, false, fmt.Errorf("get Node: %w", err) + } + + selector, err := metav1.LabelSelectorAsSelector(&profile.Spec.Participants.Selector) + if err != nil { + return status, false, fmt.Errorf("participant selector: %w", err) + } + + if !selector.Matches(labels.Set(node.Labels)) || !nodeEligible(&node, profile.Spec.Participants.RequiredResources) { + status.IdleSince = nil + status.Reason = "NodeIneligible" + + return status, false, nil + } + + busy, reason, err := r.nodeBusy(ctx, &node) + if err != nil { + status.IdleSince = nil + + return status, false, err + } + + if busy { + status.IdleSince = nil + status.Reason = reason + + return status, false, nil + } + + if status.IdleSince == nil { + idleSince := metav1.NewTime(now) + status.IdleSince = &idleSince + } + + nodeRuns := runsForNodeProfile(runs, nodeName, profile.Name) + lastSuccess, consecutiveFailures, latestFailure := runHistory(nodeRuns) + status.ConsecutiveFailures = consecutiveFailures + + if lastSuccess != nil { + status.LastSuccessfulRun = lastSuccess.Name + status.LastSuccessfulAt = lastSuccess.Status.FinishedAt + } + + due := dueAt(profile.Spec.Recurring, status.IdleSince, lastSuccess) + status.NextDueAt = ptrMetaTime(due) + + if suspended, reason := failureGate(profile.Spec.Recurring, consecutiveFailures, latestFailure, now); reason != "" { + status.Suspended = suspended + status.Reason = reason + + return status, false, nil + } + + if now.Before(due) { + status.Reason = "NotDue" + + return status, false, nil + } + + if activeRunForNode(runs, nodeName) || node.Annotations[ReservationLabel] != "" { + status.Reason = "RunActive" + + return status, false, nil + } + + if excludedRunActive(runs, nodeName, profile.Spec.Recurring.ExclusiveProfiles) { + status.Reason = "ExclusiveProfileActive" + + return status, false, nil + } + + if activeByProfile >= profile.Spec.Recurring.MaxConcurrent { + status.Reason = "ConcurrencyBudget" + + return status, false, nil + } + + domain := node.Labels[profile.Spec.Recurring.FailureDomainLabel] + if profile.Spec.Recurring.FailureDomainLabel != "" && + profile.Spec.Recurring.MaxConcurrentPerFailureDomain > 0 && + activeByDomain[domain] >= profile.Spec.Recurring.MaxConcurrentPerFailureDomain { + status.Reason = "FailureDomainBudget" + + return status, false, nil + } + + if err := r.Admission.admitNodeWithTriggerLocked(ctx, &node, profile.Name, "recurring"); err != nil { + if apierrors.IsAlreadyExists(err) || apierrors.IsConflict(err) { + status.Reason = "AdmissionRaced" + + return status, false, nil + } + + return status, false, fmt.Errorf("automatic admission: %w", err) + } + + status.Reason = "Admitted" + status.NextDueAt = nil + + return status, true, nil +} + +func (r *RecurringReconciler) nodeBusy(ctx context.Context, node *corev1.Node) (bool, string, error) { + var pods corev1.PodList + if err := r.Reader.List(ctx, &pods, client.MatchingFields{"spec.nodeName": node.Name}); err != nil { + return true, "WorkloadStateUnavailable", fmt.Errorf("list Node pods: %w", err) + } + + for index := range pods.Items { + pod := &pods.Items[index] + if pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { + continue + } + + if controlledByDaemonSet(pod) { + continue + } + + return true, "WorkloadActive", nil + } + + active, err := r.activeMachineOperation(ctx, node.Name) + if err != nil { + return true, "MaintenanceStateUnavailable", err + } + + if active { + return true, "MaintenanceOperationActive", nil + } + + return false, "", nil +} + +func (r *RecurringReconciler) activeMachineOperation(ctx context.Context, nodeName string) (bool, error) { + var machines machinav1.MachineList + if err := r.Reader.List(ctx, &machines); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + + return false, fmt.Errorf("list Machines: %w", err) + } + + machineNames := map[string]struct{}{} + + for index := range machines.Items { + machine := &machines.Items[index] + if machine.Spec.Kubernetes != nil && + machine.Spec.Kubernetes.NodeRef != nil && + machine.Spec.Kubernetes.NodeRef.Name == nodeName { + machineNames[machine.Name] = struct{}{} + } + } + + if len(machineNames) == 0 { + return false, nil + } + + var operations machinav1.MachineOperationList + if err := r.Reader.List(ctx, &operations); err != nil { + return false, fmt.Errorf("list MachineOperations: %w", err) + } + + for index := range operations.Items { + operation := &operations.Items[index] + if operation.Status.Phase != machinav1.OperationPhasePending && + operation.Status.Phase != machinav1.OperationPhaseInProgress { + continue + } + + if _, ok := machineNames[operation.Spec.MachineRef]; ok { + return true, nil + } + + if operation.Spec.MachineSelector != nil { + selector, err := metav1.LabelSelectorAsSelector(operation.Spec.MachineSelector) + if err != nil { + return false, fmt.Errorf("MachineOperation %s selector: %w", operation.Name, err) + } + + for machineIndex := range machines.Items { + machine := &machines.Items[machineIndex] + if _, ok := machineNames[machine.Name]; ok && selector.Matches(labels.Set(machine.Labels)) { + return true, nil + } + } + } + } + + return false, nil +} + +func controlledByDaemonSet(pod *corev1.Pod) bool { + for _, owner := range pod.OwnerReferences { + if owner.Controller != nil && *owner.Controller && owner.Kind == "DaemonSet" { + return true + } + } + + return false +} + +func validateRecurringPolicy(policy *signalv1.RecurringPolicy) error { + switch { + case len(policy.NodeNames) == 0: + return fmt.Errorf("nodeNames must explicitly opt in at least one Node") + case policy.MinIdleDuration.Duration < 15*time.Minute: + return fmt.Errorf("minIdleDuration must be at least 15 minutes") + case policy.ValidFor.Duration <= 0: + return fmt.Errorf("validFor must be positive") + case policy.ExpiresAt.IsZero(): + return fmt.Errorf("expiresAt is required") + case policy.MaxConcurrent < 1: + return fmt.Errorf("maxConcurrent must be positive") + case policy.MaxConsecutiveFailures < 1: + return fmt.Errorf("maxConsecutiveFailures must be positive") + } + + return nil +} + +func policyExpired(policy *signalv1.RecurringPolicy, now time.Time) bool { + return !now.Before(policy.ExpiresAt.Time) +} + +func failureGate( + policy *signalv1.RecurringPolicy, + consecutiveFailures int32, + latestFailure *time.Time, + now time.Time, +) (bool, string) { + if policy.MaxConsecutiveFailures > 0 && consecutiveFailures >= policy.MaxConsecutiveFailures { + return true, "CircuitOpen" + } + + if latestFailure != nil && now.Before(latestFailure.Add(policy.FailureBackoff.Duration)) { + return false, "FailureBackoff" + } + + return false, "" +} + +func dueAt( + policy *signalv1.RecurringPolicy, + idleSince *metav1.Time, + lastSuccess *signalv1.CertificationRun, +) time.Time { + due := idleSince.Add(policy.MinIdleDuration.Duration) + if policy.NotBefore != nil && due.Before(policy.NotBefore.Time) { + due = policy.NotBefore.Time + } + + if lastSuccess != nil && lastSuccess.Status.FinishedAt != nil { + freshUntil := lastSuccess.Status.FinishedAt.Add(policy.ValidFor.Duration) + if due.Before(freshUntil) { + due = freshUntil + } + } + + return due +} + +func activeBudgets(runs []signalv1.CertificationRun, profile *signalv1.CertificationProfile) (int32, map[string]int32) { + var active int32 + + domains := map[string]int32{} + + for index := range runs { + run := &runs[index] + if run.Spec.Profile.Name != profile.Name || run.Status.Phase.Terminal() { + continue + } + + active++ + } + + return active, domains +} + +func activeRunForNode(runs []signalv1.CertificationRun, nodeName string) bool { + for index := range runs { + run := &runs[index] + if run.Status.Phase.Terminal() { + continue + } + + for _, participant := range run.Spec.Participants { + if participant.NodeName == nodeName { + return true + } + } + } + + return false +} + +func excludedRunActive(runs []signalv1.CertificationRun, nodeName string, profiles []string) bool { + for index := range runs { + run := &runs[index] + if run.Status.Phase.Terminal() || !slices.Contains(profiles, run.Spec.Profile.Name) { + continue + } + + for _, participant := range run.Spec.Participants { + if participant.NodeName == nodeName { + return true + } + } + } + + return false +} + +func runsForNodeProfile( + runs []signalv1.CertificationRun, + nodeName string, + profileName string, +) []signalv1.CertificationRun { + result := make([]signalv1.CertificationRun, 0) + + for index := range runs { + run := runs[index] + if run.Spec.Profile.Name != profileName || !run.Status.Phase.Terminal() { + continue + } + + for _, participant := range run.Spec.Participants { + if participant.NodeName == nodeName { + result = append(result, run) + + break + } + } + } + + sort.Slice(result, func(i, j int) bool { + return completionTime(&result[i]).After(completionTime(&result[j])) + }) + + return result +} + +func runHistory( + runs []signalv1.CertificationRun, +) (*signalv1.CertificationRun, int32, *time.Time) { + var ( + lastSuccess *signalv1.CertificationRun + latestFailure *time.Time + consecutiveFailure int32 + ) + + for index := range runs { + run := &runs[index] + if run.Status.Phase == signalv1.RunPhaseSucceeded { + lastSuccess = run + + break + } + + if run.Status.Phase != signalv1.RunPhaseFailed && run.Status.Phase != signalv1.RunPhaseInconclusive { + continue + } + + consecutiveFailure++ + + if latestFailure == nil { + value := completionTime(run) + latestFailure = &value + } + } + + return lastSuccess, consecutiveFailure, latestFailure +} + +func completionTime(run *signalv1.CertificationRun) time.Time { + if run.Status.FinishedAt != nil { + return run.Status.FinishedAt.Time + } + + return run.CreationTimestamp.Time +} + +func recurringNodeStatus(statuses []signalv1.RecurringNodeStatus, nodeName string) signalv1.RecurringNodeStatus { + for _, status := range statuses { + if status.NodeName == nodeName { + return status + } + } + + return signalv1.RecurringNodeStatus{NodeName: nodeName} +} + +func ptrMetaTime(value time.Time) *metav1.Time { + result := metav1.NewTime(value) + + return &result +} + +func setProfileCondition( + profile *signalv1.CertificationProfile, + now time.Time, + status metav1.ConditionStatus, + reason string, + message string, +) { + apimeta.SetStatusCondition(&profile.Status.Conditions, metav1.Condition{ + Type: "RecurringAdmissionReady", + Status: status, + ObservedGeneration: profile.Generation, + LastTransitionTime: metav1.NewTime(now), + Reason: reason, + Message: message, + }) +} + +func (r *RecurringReconciler) now() time.Time { + if r.Now != nil { + return r.Now() + } + + return time.Now() +} diff --git a/internal/signal/policy_test.go b/internal/signal/policy_test.go new file mode 100644 index 0000000000..933bf8bcc2 --- /dev/null +++ b/internal/signal/policy_test.go @@ -0,0 +1,264 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signal + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" +) + +func TestRecurringRunBudgetsUseAuthoritativeReader(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + if err := signalv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + now := time.Date(2026, 10, 10, 20, 0, 0, 0, time.UTC) + node := readyNode("gpu-0", "wrong-gpu") + profile := testProfile("gpu-smoke", "expected-gpu") + profile.Spec.Recurring = &signalv1.RecurringPolicy{ + Enabled: true, + NodeNames: []string{node.Name}, + MinIdleDuration: metav1.Duration{Duration: 15 * time.Minute}, + ValidFor: metav1.Duration{Duration: time.Hour}, + ExpiresAt: metav1.NewTime(now.Add(time.Hour)), + FailureBackoff: metav1.Duration{Duration: time.Minute}, + MaxConsecutiveFailures: 3, + MaxConcurrent: 1, + } + base := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationProfile{}). + WithObjects(node, profile). + Build() + cached := interceptor.NewClient(base, interceptor.Funcs{ + List: func( + context.Context, + client.WithWatch, + client.ObjectList, + ...client.ListOption, + ) error { + return errors.New("cached list must not define admission budgets") + }, + }) + r := &RecurringReconciler{ + Client: cached, + Reader: base, + Namespace: "signal", + Admission: &AdmissionReconciler{Client: cached}, + Now: func() time.Time { return now }, + } + + if _, err := r.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKey{Name: profile.Name}}, + ); err != nil { + t.Fatalf("Reconcile: %v", err) + } +} + +func TestRecurringFailureDomainAccountingFailsClosed(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + if err := signalv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + now := time.Date(2026, 10, 10, 20, 0, 0, 0, time.UTC) + profile := testProfile("gpu-smoke", "expected-gpu") + profile.Spec.Recurring = &signalv1.RecurringPolicy{ + Enabled: true, + NodeNames: []string{"gpu-0"}, + MinIdleDuration: metav1.Duration{Duration: 15 * time.Minute}, + ValidFor: metav1.Duration{Duration: time.Hour}, + ExpiresAt: metav1.NewTime(now.Add(time.Hour)), + FailureBackoff: metav1.Duration{Duration: time.Minute}, + MaxConsecutiveFailures: 3, + MaxConcurrent: 1, + FailureDomainLabel: "topology.kubernetes.io/zone", + MaxConcurrentPerFailureDomain: 1, + } + active := testRun("active") + active.Spec.Profile.Name = profile.Name + active.Spec.Participants = []signalv1.Participant{{NodeName: "missing-node", NodeUID: "missing-uid"}} + active.Status.Phase = signalv1.RunPhaseRunning + base := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationProfile{}). + WithObjects(profile, active). + Build() + r := &RecurringReconciler{ + Client: base, + Reader: base, + Namespace: "signal", + Admission: &AdmissionReconciler{Client: base}, + Now: func() time.Time { return now }, + } + + if _, err := r.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKey{Name: profile.Name}}, + ); err == nil || !strings.Contains(err.Error(), "failure-domain accounting") { + t.Fatalf("Reconcile error = %v, want fail-closed accounting", err) + } +} + +func TestRecurringDueUsesContinuousIdleAndFreshness(t *testing.T) { + t.Parallel() + + start := time.Date(2026, 10, 10, 20, 0, 0, 0, time.UTC) + idleSince := metav1.NewTime(start) + policy := &signalv1.RecurringPolicy{ + MinIdleDuration: metav1.Duration{Duration: 15 * time.Minute}, + ValidFor: metav1.Duration{Duration: time.Hour}, + } + + if got := dueAt(policy, &idleSince, nil); !got.Equal(start.Add(15 * time.Minute)) { + t.Fatalf("initial due = %s", got) + } + + success := testRun("success") + success.Status.Phase = signalv1.RunPhaseSucceeded + success.Status.FinishedAt = ptrTime(metav1.NewTime(start.Add(10 * time.Minute))) + + if got := dueAt(policy, &idleSince, success); !got.Equal(start.Add(70 * time.Minute)) { + t.Fatalf("freshness due = %s", got) + } +} + +func TestRecurringIdleStateSurvivesRestart(t *testing.T) { + t.Parallel() + + idleSince := metav1.NewTime(time.Date(2026, 10, 10, 20, 0, 0, 0, time.UTC)) + statuses := []signalv1.RecurringNodeStatus{{NodeName: "gpu-0", IdleSince: &idleSince, Reason: "NotDue"}} + + got := recurringNodeStatus(statuses, "gpu-0") + if got.IdleSince == nil || !got.IdleSince.Equal(&idleSince) { + t.Fatalf("idleSince was not recovered: %#v", got) + } +} + +func TestRecurringExpiryFailsClosed(t *testing.T) { + t.Parallel() + + expiry := metav1.NewTime(time.Date(2026, 10, 12, 14, 0, 0, 0, time.UTC)) + policy := &signalv1.RecurringPolicy{ExpiresAt: expiry} + + if policyExpired(policy, expiry.Add(-time.Nanosecond)) { + t.Fatal("policy expired early") + } + + if !policyExpired(policy, expiry.Time) { + t.Fatal("policy remained active at its expiry") + } +} + +func TestRecurringConcurrencyAndDuplicatePrevention(t *testing.T) { + t.Parallel() + + active := *testRun("active") + active.Status.Phase = signalv1.RunPhaseRunning + active.Spec.Profile.Name = "short" + profile := testProfile("short", "A100") + + count, _ := activeBudgets([]signalv1.CertificationRun{active}, profile) + if count != 1 || !activeRunForNode([]signalv1.CertificationRun{active}, "gpu-0") { + t.Fatalf("active budget=%d duplicate=%t", count, activeRunForNode([]signalv1.CertificationRun{active}, "gpu-0")) + } +} + +func TestRecurringShortLongMutualExclusion(t *testing.T) { + t.Parallel() + + longRun := *testRun("long") + longRun.Spec.Profile.Name = "gpu-long" + longRun.Status.Phase = signalv1.RunPhaseRunning + + if !excludedRunActive( + []signalv1.CertificationRun{longRun}, + "gpu-0", + []string{"gpu-long"}, + ) { + t.Fatal("active long run did not exclude the short profile") + } +} + +func TestRecurringFailureBackoffAndCircuitBreaker(t *testing.T) { + t.Parallel() + + now := time.Date(2026, 10, 10, 20, 0, 0, 0, time.UTC) + failed := *testRun("failed") + failed.Status.Phase = signalv1.RunPhaseFailed + failed.Status.FinishedAt = ptrTime(metav1.NewTime(now.Add(-5 * time.Minute))) + inconclusive := *testRun("inconclusive") + inconclusive.Status.Phase = signalv1.RunPhaseInconclusive + inconclusive.Status.FinishedAt = ptrTime(metav1.NewTime(now.Add(-10 * time.Minute))) + runs := []signalv1.CertificationRun{failed, inconclusive} + + _, count, latest := runHistory(runs) + policy := &signalv1.RecurringPolicy{ + FailureBackoff: metav1.Duration{Duration: 30 * time.Minute}, + MaxConsecutiveFailures: 3, + } + + suspended, reason := failureGate(policy, count, latest, now) + if suspended || reason != "FailureBackoff" { + t.Fatalf("backoff gate = (%t, %q)", suspended, reason) + } + + policy.MaxConsecutiveFailures = 2 + + suspended, reason = failureGate(policy, count, latest, now) + if !suspended || reason != "CircuitOpen" { + t.Fatalf("circuit gate = (%t, %q)", suspended, reason) + } +} + +func TestIdleDetectionIgnoresOnlyDaemonSetsAndTerminalPods(t *testing.T) { + t.Parallel() + + controller := true + daemon := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{OwnerReferences: []metav1.OwnerReference{{ + APIVersion: appsv1.SchemeGroupVersion.String(), + Kind: "DaemonSet", + Controller: &controller, + }}}, + } + deployment := daemon.DeepCopy() + deployment.OwnerReferences[0].Kind = "ReplicaSet" + + if !controlledByDaemonSet(daemon) { + t.Fatal("DaemonSet pod was treated as workload activity") + } + + if controlledByDaemonSet(deployment) { + t.Fatal("non-DaemonSet pod was treated as disruptable") + } +} diff --git a/internal/signal/projection.go b/internal/signal/projection.go new file mode 100644 index 0000000000..4ed10d4a30 --- /dev/null +++ b/internal/signal/projection.go @@ -0,0 +1,249 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signal + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "strconv" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" +) + +const ( + StateLabel = "signal.unbounded-cloud.io/state" + EligibilityLabel = "signal.unbounded-cloud.io/eligibility" + ProfileLabel = "signal.unbounded-cloud.io/profile" + + ProfileNameAnnotation = "signal.unbounded-cloud.io/profile-name" + ProfileGenerationAnnotation = "signal.unbounded-cloud.io/profile-generation" + LastRunAnnotation = "signal.unbounded-cloud.io/last-run" + LastRunAtAnnotation = "signal.unbounded-cloud.io/last-run-at" + LastSuccessfulRunAnnotation = "signal.unbounded-cloud.io/last-successful-run" + LastSuccessfulAtAnnotation = "signal.unbounded-cloud.io/last-successful-at" + EvidenceDigestAnnotation = "signal.unbounded-cloud.io/evidence-digest" + ScriptDigestAnnotation = "signal.unbounded-cloud.io/script-digest" + NextDueAtAnnotation = "signal.unbounded-cloud.io/next-due-at" + RecurringReasonAnnotation = "signal.unbounded-cloud.io/recurring-reason" + RecurringSuspendedAnnotation = "signal.unbounded-cloud.io/recurring-suspended" + + maxProjectionAnnotationValue = 1024 +) + +func projectPending( + ctx context.Context, + kube client.Client, + participants []signalv1.Participant, + profile signalv1.PinnedProfile, +) error { + return projectParticipants(ctx, kube, participants, func(node *corev1.Node) error { + setProfileProjection(node, profile) + node.Labels[StateLabel] = "pending" + node.Labels[EligibilityLabel] = "blocked" + + return nil + }) +} + +func projectTesting(ctx context.Context, kube client.Client, run *signalv1.CertificationRun) error { + return projectParticipants(ctx, kube, run.Spec.Participants, func(node *corev1.Node) error { + setProfileProjection(node, run.Spec.Profile) + node.Labels[StateLabel] = "testing" + node.Labels[EligibilityLabel] = "blocked" + + return nil + }) +} + +func projectTerminal(ctx context.Context, kube client.Client, run *signalv1.CertificationRun) error { + if run.Status.FinishedAt == nil { + return nil + } + + finishedAt := run.Status.FinishedAt.UTC().Format(time.RFC3339) + + return projectParticipants(ctx, kube, run.Spec.Participants, func(node *corev1.Node) error { + if projectionTimeAfter(node.Annotations[LastRunAtAnnotation], run.Status.FinishedAt.Time) { + return nil + } + + setProfileProjection(node, run.Spec.Profile) + node.Annotations[LastRunAnnotation] = boundedAnnotationValue(run.Name) + node.Annotations[LastRunAtAnnotation] = finishedAt + node.Annotations[EvidenceDigestAnnotation] = boundedAnnotationValue(run.Status.Evidence.Digest) + node.Annotations[ScriptDigestAnnotation] = boundedAnnotationValue(run.Spec.Script.Digest) + + switch run.Status.Phase { + case signalv1.RunPhaseSucceeded: + node.Labels[StateLabel] = "passed" + node.Labels[EligibilityLabel] = "production" + + if !projectionTimeAfter(node.Annotations[LastSuccessfulAtAnnotation], run.Status.FinishedAt.Time) { + node.Annotations[LastSuccessfulRunAnnotation] = boundedAnnotationValue(run.Name) + node.Annotations[LastSuccessfulAtAnnotation] = finishedAt + } + case signalv1.RunPhaseFailed: + node.Labels[StateLabel] = "failed" + node.Labels[EligibilityLabel] = "blocked" + default: + node.Labels[StateLabel] = "inconclusive" + node.Labels[EligibilityLabel] = "blocked" + } + + return nil + }) +} + +func projectRecurring( + ctx context.Context, + kube client.Client, + profile *signalv1.CertificationProfile, + status signalv1.RecurringNodeStatus, +) error { + participant := signalv1.Participant{NodeName: status.NodeName} + pinned := signalv1.PinnedProfile{Name: profile.Name, UID: profile.UID, Generation: profile.Generation} + + return projectParticipants(ctx, kube, []signalv1.Participant{participant}, func(node *corev1.Node) error { + selectedProfile := node.Annotations[ProfileNameAnnotation] + + active := node.Labels[StateLabel] == "pending" || node.Labels[StateLabel] == "testing" + if active && selectedProfile != profile.Name { + return nil + } + + if status.LastSuccessfulAt != nil && + !projectionTimeAfter(node.Annotations[LastSuccessfulAtAnnotation], status.LastSuccessfulAt.Time) { + setProfileProjection(node, pinned) + } else { + return nil + } + + setOrDeleteAnnotation(node, NextDueAtAnnotation, formatMetaTime(status.NextDueAt)) + setOrDeleteAnnotation(node, RecurringReasonAnnotation, status.Reason) + node.Annotations[RecurringSuspendedAnnotation] = strconv.FormatBool(status.Suspended) + + if status.LastSuccessfulAt != nil && + !projectionTimeAfter(node.Annotations[LastSuccessfulAtAnnotation], status.LastSuccessfulAt.Time) { + node.Annotations[LastSuccessfulRunAnnotation] = boundedAnnotationValue(status.LastSuccessfulRun) + node.Annotations[LastSuccessfulAtAnnotation] = status.LastSuccessfulAt.UTC().Format(time.RFC3339) + + if node.Labels[StateLabel] == "" { + node.Labels[StateLabel] = "passed" + node.Labels[EligibilityLabel] = "production" + } + } + + return nil + }) +} + +func projectParticipants( + ctx context.Context, + kube client.Client, + participants []signalv1.Participant, + mutate func(*corev1.Node) error, +) error { + for _, participant := range participants { + if participant.NodeName == "" { + continue + } + + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + var node corev1.Node + if err := kube.Get(ctx, client.ObjectKey{Name: participant.NodeName}, &node); err != nil { + return client.IgnoreNotFound(err) + } + + if participant.NodeUID != "" && node.UID != participant.NodeUID { + return nil + } + + before := node.DeepCopy() + ensureProjectionMaps(&node) + + if err := mutate(&node); err != nil { + return err + } + + return kube.Patch(ctx, &node, client.MergeFrom(before)) + }); err != nil { + return fmt.Errorf("project Node %s status: %w", participant.NodeName, err) + } + } + + return nil +} + +func setProfileProjection(node *corev1.Node, profile signalv1.PinnedProfile) { + ensureProjectionMaps(node) + node.Labels[ProfileLabel] = projectionLabelValue(profile.Name) + node.Annotations[ProfileNameAnnotation] = boundedAnnotationValue(profile.Name) + node.Annotations[ProfileGenerationAnnotation] = strconv.FormatInt(profile.Generation, 10) +} + +func ensureProjectionMaps(node *corev1.Node) { + if node.Labels == nil { + node.Labels = map[string]string{} + } + + if node.Annotations == nil { + node.Annotations = map[string]string{} + } +} + +func projectionLabelValue(value string) string { + if len(validation.IsValidLabelValue(value)) == 0 { + return value + } + + sum := sha256.Sum256([]byte(value)) + + return "sha256-" + hex.EncodeToString(sum[:8]) +} + +func boundedAnnotationValue(value string) string { + if len(value) <= maxProjectionAnnotationValue { + return value + } + + sum := sha256.Sum256([]byte(value)) + + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func projectionTimeAfter(value string, candidate time.Time) bool { + existing, err := time.Parse(time.RFC3339, value) + if err != nil { + return false + } + + return existing.After(candidate) +} + +func setOrDeleteAnnotation(node *corev1.Node, key, value string) { + if value == "" { + delete(node.Annotations, key) + + return + } + + node.Annotations[key] = boundedAnnotationValue(value) +} + +func formatMetaTime(value *metav1.Time) string { + if value == nil { + return "" + } + + return value.UTC().Format(time.RFC3339) +} diff --git a/internal/signal/projection_test.go b/internal/signal/projection_test.go new file mode 100644 index 0000000000..6af9abcb6b --- /dev/null +++ b/internal/signal/projection_test.go @@ -0,0 +1,227 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signal + +import ( + "context" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" +) + +func TestNodeProjectionTransitionsAndRetainsTerminalSummary(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + node := readyNode("gpu-0", "A100") + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node).Build() + run := testRun("run-pass") + + if err := projectPending(context.Background(), kube, run.Spec.Participants, run.Spec.Profile); err != nil { + t.Fatal(err) + } + + assertNodeProjection(t, kube, node.Name, "pending", "blocked") + + if err := projectTesting(context.Background(), kube, run); err != nil { + t.Fatal(err) + } + + assertNodeProjection(t, kube, node.Name, "testing", "blocked") + + run.Status.Phase = signalv1.RunPhaseSucceeded + run.Status.ExecutionState = "Completed" + run.Status.Verdict = "Pass" + run.Status.Reason = "Healthy" + run.Status.Evidence.Digest = "sha256:evidence" + run.Status.FinishedAt = ptrTime(metav1.NewTime(time.Date(2026, 10, 11, 5, 0, 0, 0, time.UTC))) + + if err := projectTerminal(context.Background(), kube, run); err != nil { + t.Fatal(err) + } + + var got corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &got); err != nil { + t.Fatal(err) + } + + if got.Labels[StateLabel] != "passed" || got.Labels[EligibilityLabel] != "production" { + t.Fatalf("terminal labels = %v", got.Labels) + } + + if got.Annotations[LastRunAnnotation] != run.Name || + got.Annotations[LastSuccessfulRunAnnotation] != run.Name || + got.Annotations[EvidenceDigestAnnotation] != run.Status.Evidence.Digest || + got.Annotations[ScriptDigestAnnotation] != run.Spec.Script.Digest { + t.Fatalf("terminal annotations = %v", got.Annotations) + } +} + +func TestNodeProjectionFailureAndInconclusiveBlockEligibility(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + phase signalv1.RunPhase + state string + }{ + {name: "failed", phase: signalv1.RunPhaseFailed, state: "failed"}, + {name: "inconclusive", phase: signalv1.RunPhaseInconclusive, state: "inconclusive"}, + } { + t.Run(tc.name, func(t *testing.T) { + scheme := testScheme(t) + node := readyNode("gpu-0", "A100") + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node).Build() + run := testRun("run-" + tc.name) + run.Status.Phase = tc.phase + run.Status.FinishedAt = ptrTime(metav1.Now()) + + if err := projectTerminal(context.Background(), kube, run); err != nil { + t.Fatal(err) + } + + assertNodeProjection(t, kube, node.Name, tc.state, "blocked") + }) + } +} + +func TestNodeProjectionDoesNotRegressAfterRestartOrOlderRetry(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + node := readyNode("gpu-0", "A100") + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node).Build() + newer := testRun("newer") + newer.Status.Phase = signalv1.RunPhaseSucceeded + newer.Status.FinishedAt = ptrTime(metav1.NewTime(time.Date(2026, 10, 11, 6, 0, 0, 0, time.UTC))) + newer.Status.Evidence.Digest = "sha256:newer" + + if err := projectTerminal(context.Background(), kube, newer); err != nil { + t.Fatal(err) + } + + older := testRun("older") + older.Status.Phase = signalv1.RunPhaseFailed + older.Status.FinishedAt = ptrTime(metav1.NewTime(time.Date(2026, 10, 11, 5, 0, 0, 0, time.UTC))) + older.Status.Evidence.Digest = "sha256:older" + + if err := projectTerminal(context.Background(), kube, older); err != nil { + t.Fatal(err) + } + + var got corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &got); err != nil { + t.Fatal(err) + } + + if got.Labels[StateLabel] != "passed" || + got.Annotations[LastRunAnnotation] != newer.Name || + got.Annotations[EvidenceDigestAnnotation] != "sha256:newer" { + t.Fatalf("projection regressed: labels=%v annotations=%v", got.Labels, got.Annotations) + } +} + +func TestNodeProjectionDoesNotCertifyReplacementNode(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + replacement := readyNode("gpu-0", "A100") + replacement.UID = "replacement-uid" + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(replacement).Build() + run := testRun("run-pass") + run.Status.Phase = signalv1.RunPhaseSucceeded + run.Status.Verdict = "Pass" + run.Status.FinishedAt = ptrTime(metav1.Now()) + + if err := projectTerminal(context.Background(), kube, run); err != nil { + t.Fatal(err) + } + + var got corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: replacement.Name}, &got); err != nil { + t.Fatal(err) + } + + if got.Labels[StateLabel] != "" || got.Labels[EligibilityLabel] != "" { + t.Fatalf("replacement Node was certified: labels=%v", got.Labels) + } +} + +func TestProjectionValuesAreBoundedAndValid(t *testing.T) { + t.Parallel() + + longValue := strings.Repeat("Invalid/Profile:", 200) + + label := projectionLabelValue(longValue) + if len(label) > 63 { + t.Fatalf("label value is too long: %d", len(label)) + } + + if bounded := boundedAnnotationValue(longValue); len(bounded) > len("sha256:")+64 { + t.Fatalf("annotation value is unbounded: %d", len(bounded)) + } + + if got := projectionLabelValue("gpu-smoke-example"); got != "gpu-smoke-example" { + t.Fatalf("valid profile label changed to %q", got) + } +} + +func TestRecurringProjectionPublishesNextDueAndSuspension(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + node := readyNode("gpu-0", "A100") + profile := testProfile("gpu-smoke-example", "example-gpu") + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node).Build() + nextDue := metav1.NewTime(time.Date(2026, 10, 11, 7, 0, 0, 0, time.UTC)) + status := signalv1.RecurringNodeStatus{ + NodeName: node.Name, + NextDueAt: &nextDue, + LastSuccessfulRun: "run-pass", + LastSuccessfulAt: ptrTime(metav1.NewTime(time.Date(2026, 10, 11, 6, 0, 0, 0, time.UTC))), + Suspended: true, + Reason: "CircuitOpen", + } + + if err := projectRecurring(context.Background(), kube, profile, status); err != nil { + t.Fatal(err) + } + + var got corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &got); err != nil { + t.Fatal(err) + } + + if got.Annotations[NextDueAtAnnotation] != "2026-10-11T07:00:00Z" || + got.Annotations[RecurringReasonAnnotation] != "CircuitOpen" || + got.Annotations[RecurringSuspendedAnnotation] != "true" { + t.Fatalf("recurring annotations = %v", got.Annotations) + } +} + +func assertNodeProjection( + t *testing.T, + kube client.Client, + nodeName string, + state string, + eligibility string, +) { + t.Helper() + + var node corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: nodeName}, &node); err != nil { + t.Fatal(err) + } + + if node.Labels[StateLabel] != state || node.Labels[EligibilityLabel] != eligibility { + t.Fatalf("projection = state %q eligibility %q", node.Labels[StateLabel], node.Labels[EligibilityLabel]) + } +} From 89d5f385dc5916120f3ab70e516f8539863b9622 Mon Sep 17 00:00:00 2001 From: chokevin Date: Tue, 20 Oct 2026 15:22:20 -0700 Subject: [PATCH 02/36] feat(signal): support grouped node certification --- .../v1alpha1/certificationprofile_types.go | 2 +- api/signal/v1alpha1/certificationrun_types.go | 25 +- api/signal/v1alpha1/zz_generated.deepcopy.go | 45 ++- deploy/signal/README.md | 32 +- ...ounded-cloud.io_certificationprofiles.yaml | 2 +- ....unbounded-cloud.io_certificationruns.yaml | 59 ++++ internal/signal/controller.go | 200 ++++++++++- internal/signal/controller_test.go | 314 +++++++++++++++++- internal/signal/policy.go | 113 ++++++- internal/signal/policy_test.go | 59 ++++ 10 files changed, 808 insertions(+), 43 deletions(-) diff --git a/api/signal/v1alpha1/certificationprofile_types.go b/api/signal/v1alpha1/certificationprofile_types.go index 553c6f2b59..4666975ccf 100644 --- a/api/signal/v1alpha1/certificationprofile_types.go +++ b/api/signal/v1alpha1/certificationprofile_types.go @@ -45,7 +45,7 @@ type ExecutionBackend struct { type ParticipantPolicy struct { // +kubebuilder:validation:Minimum=1 - // +kubebuilder:validation:Maximum=1 + // +kubebuilder:validation:Maximum=64 Count int32 `json:"count"` Selector metav1.LabelSelector `json:"selector"` diff --git a/api/signal/v1alpha1/certificationrun_types.go b/api/signal/v1alpha1/certificationrun_types.go index 14a796998d..21a4be518c 100644 --- a/api/signal/v1alpha1/certificationrun_types.go +++ b/api/signal/v1alpha1/certificationrun_types.go @@ -30,8 +30,23 @@ func (p RunPhase) Terminal() bool { } type Participant struct { - NodeName string `json:"nodeName"` - NodeUID types.UID `json:"nodeUID"` + // +kubebuilder:validation:Minimum=0 + Index int32 `json:"index"` + NodeName string `json:"nodeName"` + NodeUID types.UID `json:"nodeUID"` + Capacity corev1.ResourceList `json:"capacity,omitempty"` + Allocatable corev1.ResourceList `json:"allocatable,omitempty"` +} + +type BackendResourceReference struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Namespace string `json:"namespace,omitempty"` + Name string `json:"name"` + UID types.UID `json:"uid"` + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=64 + ParticipantIndexes []int32 `json:"participantIndexes,omitempty"` } type PinnedProfile struct { @@ -59,7 +74,9 @@ type ScriptSnapshot struct { // +kubebuilder:validation:XValidation:rule="self.tolerations == oldSelf.tolerations",message="tolerations are immutable" // +kubebuilder:validation:XValidation:rule="self.requestNodeName == oldSelf.requestNodeName",message="requestNodeName is immutable" type CertificationRunSpec struct { - Profile PinnedProfile `json:"profile"` + Profile PinnedProfile `json:"profile"` + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=64 Participants []Participant `json:"participants"` Script ScriptSnapshot `json:"script"` Workflow WorkflowReference `json:"workflow"` @@ -85,6 +102,8 @@ type WorkflowStatus struct { Attempt int32 `json:"attempt,omitempty"` Phase string `json:"phase,omitempty"` Outputs map[string]string `json:"outputs,omitempty"` + // +kubebuilder:validation:MaxItems=64 + BackendResources []BackendResourceReference `json:"backendResources,omitempty"` } type EvidenceStatus struct { diff --git a/api/signal/v1alpha1/zz_generated.deepcopy.go b/api/signal/v1alpha1/zz_generated.deepcopy.go index 36e0b8c25f..6775dfde0e 100644 --- a/api/signal/v1alpha1/zz_generated.deepcopy.go +++ b/api/signal/v1alpha1/zz_generated.deepcopy.go @@ -13,6 +13,26 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackendResourceReference) DeepCopyInto(out *BackendResourceReference) { + *out = *in + if in.ParticipantIndexes != nil { + in, out := &in.ParticipantIndexes, &out.ParticipantIndexes + *out = make([]int32, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackendResourceReference. +func (in *BackendResourceReference) DeepCopy() *BackendResourceReference { + if in == nil { + return nil + } + out := new(BackendResourceReference) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CertificationProfile) DeepCopyInto(out *CertificationProfile) { *out = *in @@ -211,7 +231,9 @@ func (in *CertificationRunSpec) DeepCopyInto(out *CertificationRunSpec) { if in.Participants != nil { in, out := &in.Participants, &out.Participants *out = make([]Participant, len(*in)) - copy(*out, *in) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } out.Script = in.Script out.Workflow = in.Workflow @@ -345,6 +367,20 @@ func (in *ExecutionBackend) DeepCopy() *ExecutionBackend { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Participant) DeepCopyInto(out *Participant) { *out = *in + if in.Capacity != nil { + in, out := &in.Capacity, &out.Capacity + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + if in.Allocatable != nil { + in, out := &in.Allocatable, &out.Allocatable + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Participant. @@ -515,6 +551,13 @@ func (in *WorkflowStatus) DeepCopyInto(out *WorkflowStatus) { (*out)[key] = val } } + if in.BackendResources != nil { + in, out := &in.BackendResources, &out.BackendResources + *out = make([]BackendResourceReference, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkflowStatus. diff --git a/deploy/signal/README.md b/deploy/signal/README.md index 91fc4f1943..68544d4ced 100644 --- a/deploy/signal/README.md +++ b/deploy/signal/README.md @@ -53,7 +53,10 @@ The referenced WorkflowTemplate must accept these string parameters: | --- | --- | | `run-name` | CertificationRun name | | `run-uid` | CertificationRun UID | -| `participants` | Comma-separated admitted Node names | +| `participants` | Comma-separated admitted Node names retained for compatibility | +| `participants-json` | Ordered JSON array of participant indexes, Node names, immutable UIDs, capacity, and allocatable resources | +| `participant-count` | Number of atomically admitted participants | +| `request-node-name` | Node that triggered the grouped run | | `script-configmap` | Immutable run-owned ConfigMap | | `script-key` | Selected ConfigMap key | | `script-digest` | SHA-256 of the selected script bytes | @@ -79,9 +82,34 @@ The Workflow must publish only these output parameters: | `verdict` | 32 bytes | `Pass`, `Fail`, or `Inconclusive` | | `reason` | 4 KiB | Bounded diagnostic reason | | `measurements` | 16 KiB | Bounded measurement summary | +| `backend-resources` | 16 KiB | Optional JSON array of backend workload identities and covered participant indexes; required for a multi-Node `Pass` | Unexpected, duplicate, malformed, or oversized outputs are rejected and the -run remains fail-closed. +run remains fail-closed. Each `backend-resources` entry must contain +`apiVersion`, `kind`, `name`, `uid`, and the zero-based `participantIndexes` +that the workload executed. `namespace` is optional. A multi-Node successful +result must cover every admitted participant. + +## Multi-Node execution + +Participant admission is deterministic and atomic. The requesting Node is +participant zero, additional eligible Nodes are selected by name, and a failed +reservation rolls back every reservation already acquired. Each participant is +bound by Node UID and an immutable capacity and allocatable snapshot in the +CertificationRun. + +The participant group is a certification boundary, not necessarily one +distributed process group. Operator WorkflowTemplates should create one +independently pinned backend workload per participant unless a test explicitly +requires a cross-Node world. The backend adapter owns launcher and worker +topology, rank environment, rendezvous or SSH configuration, workload RBAC, +deadline propagation, foreground cleanup, and result aggregation. + +The currently validated adapter contract uses ordinary single-Node workloads +for Node diagnostics and `kubeflow.org/v2beta1` MPIJobs for tests that require +an OpenMPI launcher and worker. Kubeflow Trainer v2 TrainJobs can be supplied by +operators through the same generic interface, but require a separately managed, +version-pinned MPI TrainingRuntime and are not installed by Project Signal. ## Render and install diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml index c1067fd453..ab5b7120fe 100644 --- a/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml @@ -78,7 +78,7 @@ spec: properties: count: format: int32 - maximum: 1 + maximum: 64 minimum: 1 type: integer requiredResources: diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml index 2721ba6b89..a95227c6d1 100644 --- a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml @@ -85,6 +85,30 @@ spec: participants: items: properties: + allocatable: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: ResourceList is a set of (resource name, quantity) + pairs. + type: object + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: ResourceList is a set of (resource name, quantity) + pairs. + type: object + index: + format: int32 + minimum: 0 + type: integer nodeName: type: string nodeUID: @@ -94,9 +118,12 @@ spec: intent and helps make sure that UIDs and names do not get conflated. type: string required: + - index - nodeName - nodeUID type: object + maxItems: 64 + minItems: 1 type: array profile: properties: @@ -345,6 +372,38 @@ spec: attempt: format: int32 type: integer + backendResources: + items: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + participantIndexes: + items: + format: int32 + type: integer + maxItems: 64 + minItems: 1 + type: array + uid: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + required: + - apiVersion + - kind + - name + - uid + type: object + maxItems: 64 + type: array name: type: string outputs: diff --git a/internal/signal/controller.go b/internal/signal/controller.go index 3c2ff64633..d91434d75a 100644 --- a/internal/signal/controller.go +++ b/internal/signal/controller.go @@ -10,6 +10,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "reflect" "regexp" "sort" @@ -71,9 +72,10 @@ var reservedScriptEnvironmentNames = map[string]struct{}{ } var workflowOutputLimits = map[string]int{ - "measurements": 16 * 1024, - "reason": 4 * 1024, - "verdict": 32, + "backend-resources": 16 * 1024, + "measurements": 16 * 1024, + "reason": 4 * 1024, + "verdict": 32, } var workflowGVK = schema.GroupVersionKind{ @@ -208,11 +210,8 @@ func (r *AdmissionReconciler) admitNodeWithTriggerLocked( return fmt.Errorf("get orphaned script snapshot: %w", err) } - before := node.DeepCopy() - delete(node.Annotations, ReservationLabel) - - if err := r.Patch(ctx, node, client.MergeFrom(before)); err != nil { - return fmt.Errorf("release orphaned Project Signal reservation: %w", err) + if err := r.releaseOrphanedReservations(ctx, runName); err != nil { + return fmt.Errorf("release orphaned Project Signal reservations: %w", err) } } @@ -370,10 +369,26 @@ func (r *AdmissionReconciler) selectParticipants( return nil, fmt.Errorf("profile %s participant count must be positive", profile.Name) } - if count > 1 { - return nil, fmt.Errorf("profile %s requests %d participants; multi-participant execution is not implemented", profile.Name, count) + if count > 64 { + return nil, fmt.Errorf("profile %s participant count must not exceed 64", profile.Name) + } + + reader := r.APIReader + if reader == nil { + reader = r.Client + } + + var authoritativeRequest corev1.Node + if err := reader.Get(ctx, client.ObjectKey{Name: requestNode.Name}, &authoritativeRequest); err != nil { + return nil, fmt.Errorf("read request node %s: %w", requestNode.Name, err) } + if requestNode.UID != "" && authoritativeRequest.UID != requestNode.UID { + return nil, fmt.Errorf("request node %s identity changed", requestNode.Name) + } + + requestNode = &authoritativeRequest + selector, err := metav1.LabelSelectorAsSelector(&profile.Spec.Participants.Selector) if err != nil { return nil, fmt.Errorf("profile %s participant selector: %w", profile.Name, err) @@ -384,13 +399,14 @@ func (r *AdmissionReconciler) selectParticipants( } var nodes corev1.NodeList - if err := r.List(ctx, &nodes); err != nil { + + if err := reader.List(ctx, &nodes); err != nil { return nil, fmt.Errorf("list participant nodes: %w", err) } sort.Slice(nodes.Items, func(i, j int) bool { return nodes.Items[i].Name < nodes.Items[j].Name }) - selected := []signalv1.Participant{{NodeName: requestNode.Name, NodeUID: requestNode.UID}} + selected := []signalv1.Participant{participantSnapshot(requestNode, 0)} for i := range nodes.Items { candidate := &nodes.Items[i] @@ -406,7 +422,7 @@ func (r *AdmissionReconciler) selectParticipants( continue } - selected = append(selected, signalv1.Participant{NodeName: candidate.Name, NodeUID: candidate.UID}) + selected = append(selected, participantSnapshot(candidate, int32(len(selected)))) } if int32(len(selected)) != count { @@ -416,6 +432,42 @@ func (r *AdmissionReconciler) selectParticipants( return selected, nil } +func (r *AdmissionReconciler) releaseOrphanedReservations(ctx context.Context, runName string) error { + reader := r.APIReader + if reader == nil { + reader = r.Client + } + + var nodes corev1.NodeList + if err := reader.List(ctx, &nodes); err != nil { + return fmt.Errorf("list Nodes reserved by %s: %w", runName, err) + } + + participants := make([]signalv1.Participant, 0) + + for index := range nodes.Items { + node := &nodes.Items[index] + if node.Annotations[ReservationLabel] == runName { + participants = append(participants, signalv1.Participant{ + NodeName: node.Name, + NodeUID: node.UID, + }) + } + } + + return r.releaseParticipants(ctx, participants, runName) +} + +func participantSnapshot(node *corev1.Node, index int32) signalv1.Participant { + return signalv1.Participant{ + Index: index, + NodeName: node.Name, + NodeUID: node.UID, + Capacity: node.Status.Capacity.DeepCopy(), + Allocatable: node.Status.Allocatable.DeepCopy(), + } +} + func nodeEligible(node *corev1.Node, required corev1.ResourceList) bool { if node.Spec.Unschedulable || !nodeReady(node) { return false @@ -737,6 +789,11 @@ func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured return nil, fmt.Errorf("encode script environment: %w", err) } + participantsJSON, err := json.Marshal(run.Spec.Participants) + if err != nil { + return nil, fmt.Errorf("encode Workflow participants: %w", err) + } + name := fmt.Sprintf("%s-a%d", run.Name, attempt) workflow := &unstructured.Unstructured{Object: map[string]any{ "apiVersion": workflowGVK.GroupVersion().String(), @@ -767,6 +824,9 @@ func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured map[string]any{"name": "run-name", "value": run.Name}, map[string]any{"name": "run-uid", "value": string(run.UID)}, map[string]any{"name": "participants", "value": strings.Join(participantNames, ",")}, + map[string]any{"name": "participants-json", "value": string(participantsJSON)}, + map[string]any{"name": "participant-count", "value": fmt.Sprintf("%d", len(run.Spec.Participants))}, + map[string]any{"name": "request-node-name", "value": run.Spec.RequestNodeName}, map[string]any{"name": "script-configmap", "value": run.Spec.Script.ConfigMapName}, map[string]any{"name": "script-key", "value": run.Spec.Script.Key}, map[string]any{"name": "script-digest", "value": run.Spec.Script.Digest}, @@ -883,6 +943,23 @@ func (r *RunReconciler) finish( finalReason = "workflow outputs rejected: " + err.Error() } else { run.Status.Workflow.Outputs = outputs + + backendResources, backendErr := backendResourceReferences( + outputs["backend-resources"], + len(run.Spec.Participants), + ) + if backendErr != nil { + run.Status.Verdict = "Inconclusive" + finalReason = "workflow backend resources rejected: " + backendErr.Error() + } else { + run.Status.Workflow.BackendResources = backendResources + if run.Status.Verdict == "Pass" && + len(run.Spec.Participants) > 1 && + len(backendResources) == 0 { + run.Status.Verdict = "Inconclusive" + finalReason = "workflow did not identify backend resources for all participants" + } + } } } @@ -899,12 +976,12 @@ func (r *RunReconciler) finish( if err := r.Status().Patch(ctx, run, client.MergeFrom(before)); err != nil { return ctrl.Result{}, err } - } else { - executionState = run.Status.ExecutionState - verdict = run.Status.Verdict - reason = run.Status.Reason } + executionState = run.Status.ExecutionState + verdict = run.Status.Verdict + reason = run.Status.Reason + evidence, err := r.evidenceDocument(ctx, run, executionState, verdict, reason) if err != nil { return ctrl.Result{}, err @@ -1001,6 +1078,95 @@ func (r *RunReconciler) evidenceDocument( return json.MarshalIndent(doc, "", " ") } +func backendResourceReferences( + value string, + participantCount int, +) ([]signalv1.BackendResourceReference, error) { + if participantCount < 1 || participantCount > 64 { + return nil, fmt.Errorf("participant count must be between 1 and 64") + } + + if value == "" { + return nil, nil + } + + var references []signalv1.BackendResourceReference + + decoder := json.NewDecoder(strings.NewReader(value)) + decoder.DisallowUnknownFields() + + if err := decoder.Decode(&references); err != nil { + return nil, fmt.Errorf("decode backend-resources: %w", err) + } + + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return nil, fmt.Errorf("backend-resources contains trailing JSON values") + } + + if len(references) == 0 || len(references) > 64 { + return nil, fmt.Errorf("backend-resources must contain between 1 and 64 entries") + } + + identities := make(map[string]struct{}, len(references)) + covered := make([]bool, participantCount) + + for index := range references { + reference := &references[index] + if reference.APIVersion == "" || reference.Kind == "" || reference.Name == "" || reference.UID == "" { + return nil, fmt.Errorf("backend-resources[%d] has an incomplete identity", index) + } + + identity := strings.Join([]string{ + reference.APIVersion, + reference.Kind, + reference.Namespace, + reference.Name, + string(reference.UID), + }, "\x00") + if _, exists := identities[identity]; exists { + return nil, fmt.Errorf("backend-resources[%d] duplicates a workload identity", index) + } + + identities[identity] = struct{}{} + + if len(reference.ParticipantIndexes) == 0 { + return nil, fmt.Errorf("backend-resources[%d] does not identify a participant", index) + } + + seenIndexes := make(map[int32]struct{}, len(reference.ParticipantIndexes)) + for _, participantIndex := range reference.ParticipantIndexes { + if participantIndex < 0 || int(participantIndex) >= participantCount { + return nil, fmt.Errorf( + "backend-resources[%d] participant index %d is out of range", + index, + participantIndex, + ) + } + + if _, exists := seenIndexes[participantIndex]; exists { + return nil, fmt.Errorf( + "backend-resources[%d] repeats participant index %d", + index, + participantIndex, + ) + } + + seenIndexes[participantIndex] = struct{}{} + covered[participantIndex] = true + } + } + + if participantCount > 1 { + for index, isCovered := range covered { + if !isCovered { + return nil, fmt.Errorf("backend-resources does not cover participant index %d", index) + } + } + } + + return references, nil +} + func workflowParameters(workflow *unstructured.Unstructured) (map[string]string, error) { parameters, found, err := unstructured.NestedSlice(workflow.Object, "status", "outputs", "parameters") if err != nil { diff --git a/internal/signal/controller_test.go b/internal/signal/controller_test.go index 89d88c8d49..0e25380400 100644 --- a/internal/signal/controller_test.go +++ b/internal/signal/controller_test.go @@ -140,7 +140,7 @@ func TestAdmissionRejectsMutableRunnerImageBeforeReservation(t *testing.T) { } } -func TestParticipantSelectionRejectsUnsupportedMultiNode(t *testing.T) { +func TestParticipantSelectionIsDeterministicAndSkipsLockedNodes(t *testing.T) { t.Parallel() scheme := testScheme(t) @@ -154,12 +154,145 @@ func TestParticipantSelectionRejectsUnsupportedMultiNode(t *testing.T) { kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(request, peer, locked, wrong).Build() r := &AdmissionReconciler{Client: kube} + participants, err := r.selectParticipants(context.Background(), request, profile) + if err != nil { + t.Fatalf("selectParticipants: %v", err) + } + + if len(participants) != 2 || + participants[0].Index != 0 || + participants[0].NodeName != request.Name || + participants[1].Index != 1 || + participants[1].NodeName != peer.Name { + t.Fatalf("participants = %#v", participants) + } + + gpu := corev1.ResourceName("nvidia.com/gpu") + capacity := participants[1].Capacity[gpu] + + allocatable := participants[1].Allocatable[gpu] + if capacity.Cmp(resource.MustParse("8")) != 0 || + allocatable.Cmp(resource.MustParse("8")) != 0 { + t.Fatalf("participant inventory = %#v", participants[1]) + } +} + +func TestParticipantSelectionRejectsInsufficientEligibleNodes(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + request := readyNode("gpu-0", "H200") + profile := testProfile("h200-pair", "H200") + profile.Spec.Participants.Count = 2 + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(request).Build() + r := &AdmissionReconciler{Client: kube} + if _, err := r.selectParticipants(context.Background(), request, profile); err == nil || - !strings.Contains(err.Error(), "multi-participant execution is not implemented") { + !strings.Contains(err.Error(), "requires 2 eligible unlocked participants, found 1") { t.Fatalf("selectParticipants error = %v", err) } } +func TestMultiNodeAdmissionReservesEveryParticipant(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + request := readyNode("gpu-0", "H200") + request.Labels[RequestLabel] = "h200-pair" + peer := readyNode("gpu-1", "H200") + profile := testProfile("h200-pair", "H200") + profile.Spec.Participants.Count = 2 + source := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, + Data: map[string]string{"payload.txt": "test payload\n"}, + } + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(request, peer, profile, source).Build() + r := &AdmissionReconciler{ + Client: kube, + Namespace: "signal", + ScriptNamespace: "signal", + Now: func() time.Time { return time.Unix(100, 0) }, + } + + if _, err := r.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKey{Name: request.Name}}, + ); err != nil { + t.Fatalf("Reconcile: %v", err) + } + + var runs signalv1.CertificationRunList + if err := kube.List(context.Background(), &runs); err != nil { + t.Fatal(err) + } + + if len(runs.Items) != 1 || len(runs.Items[0].Spec.Participants) != 2 { + t.Fatalf("runs = %#v", runs.Items) + } + + for index, name := range []string{"gpu-0", "gpu-1"} { + participant := runs.Items[0].Spec.Participants[index] + if participant.Index != int32(index) || participant.NodeName != name { + t.Fatalf("participant %d = %#v", index, participant) + } + + var node corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: name}, &node); err != nil { + t.Fatal(err) + } + + if node.Annotations[ReservationLabel] != runs.Items[0].Name { + t.Fatalf("Node %s reservation = %q", name, node.Annotations[ReservationLabel]) + } + } +} + +func TestMultiNodeReservationRollsBackPartialFailure(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + first := readyNode("gpu-0", "H200") + second := readyNode("gpu-1", "H200") + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(first, second). + WithInterceptorFuncs(interceptor.Funcs{ + Patch: func( + ctx context.Context, + c client.WithWatch, + obj client.Object, + patch client.Patch, + opts ...client.PatchOption, + ) error { + node, ok := obj.(*corev1.Node) + if ok && node.Name == second.Name && node.Annotations[ReservationLabel] == "run-pair" { + return errors.New("reservation conflict") + } + + return c.Patch(ctx, obj, patch, opts...) + }, + }). + Build() + r := &AdmissionReconciler{Client: kube} + participants := []signalv1.Participant{ + participantSnapshot(first, 0), + participantSnapshot(second, 1), + } + + if err := r.reserveParticipants(context.Background(), participants, "run-pair"); err == nil { + t.Fatal("reserveParticipants error = nil") + } + + var recovered corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: first.Name}, &recovered); err != nil { + t.Fatal(err) + } + + if recovered.Annotations[ReservationLabel] != "" { + t.Fatalf("partial reservation was not rolled back: %#v", recovered.Annotations) + } +} + func TestAdmissionFailureMakesCreatedRunInertAndReleasesReservation(t *testing.T) { t.Parallel() @@ -248,7 +381,10 @@ func TestAdmissionRecoversOrphanedReservationAndSnapshot(t *testing.T) { node := readyNode("gpu-0", "example-gpu") node.Labels[RequestLabel] = "gpu-smoke" node.Annotations = map[string]string{ReservationLabel: "orphaned-run"} + peer := readyNode("gpu-1", "example-gpu") + peer.Annotations = map[string]string{ReservationLabel: "orphaned-run"} profile := testProfile("gpu-smoke", "example-gpu") + profile.Spec.Participants.Count = 2 source := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, Data: map[string]string{"payload.txt": "test payload\n"}, @@ -265,7 +401,7 @@ func TestAdmissionRecoversOrphanedReservationAndSnapshot(t *testing.T) { } kube := fake.NewClientBuilder(). WithScheme(scheme). - WithObjects(node, profile, source, orphanedSnapshot). + WithObjects(node, peer, profile, source, orphanedSnapshot). Build() r := &AdmissionReconciler{ Client: kube, @@ -299,13 +435,20 @@ func TestAdmissionRecoversOrphanedReservationAndSnapshot(t *testing.T) { t.Fatalf("runs = %d, want recovered admission", len(runs.Items)) } - var recovered corev1.Node - if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &recovered); err != nil { - t.Fatalf("get recovered node: %v", err) - } + for _, nodeName := range []string{node.Name, peer.Name} { + var recovered corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: nodeName}, &recovered); err != nil { + t.Fatalf("get recovered node %s: %v", nodeName, err) + } - if recovered.Annotations[ReservationLabel] != runs.Items[0].Name { - t.Fatalf("reservation = %q, want %q", recovered.Annotations[ReservationLabel], runs.Items[0].Name) + if recovered.Annotations[ReservationLabel] != runs.Items[0].Name { + t.Fatalf( + "Node %s reservation = %q, want %q", + nodeName, + recovered.Annotations[ReservationLabel], + runs.Items[0].Name, + ) + } } } @@ -447,6 +590,21 @@ func TestWorkflowPinsAttemptInputs(t *testing.T) { t.Fatalf("workflow values = %#v", values) } + if values["participant-count"] != "1" || values["request-node-name"] != run.Spec.RequestNodeName { + t.Fatalf("workflow participant metadata = %#v", values) + } + + var participants []signalv1.Participant + if err := json.Unmarshal([]byte(values["participants-json"]), &participants); err != nil { + t.Fatalf("decode participants: %v", err) + } + + if len(participants) != 1 || + participants[0].NodeName != run.Spec.Participants[0].NodeName || + participants[0].NodeUID != run.Spec.Participants[0].NodeUID { + t.Fatalf("workflow participants = %#v", participants) + } + if values["script-configmap"] != run.Spec.Script.ConfigMapName || values["script-configmap"] == "scripts" { t.Fatalf("workflow must mount the immutable snapshot, values = %#v", values) } @@ -547,6 +705,144 @@ func TestWorkflowParametersRejectMalformedEvidence(t *testing.T) { } } +func TestBackendResourceReferencesRequireCompleteMultiNodeCoverage(t *testing.T) { + t.Parallel() + + value := `[ + {"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","namespace":"signal","name":"node-0","uid":"uid-0","participantIndexes":[0]}, + {"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","namespace":"signal","name":"node-1","uid":"uid-1","participantIndexes":[1]} + ]` + + references, err := backendResourceReferences(value, 2) + if err != nil { + t.Fatalf("backendResourceReferences: %v", err) + } + + if len(references) != 2 || references[1].ParticipantIndexes[0] != 1 { + t.Fatalf("references = %#v", references) + } + + for name, malformed := range map[string]string{ + "missing participant": `[{"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","name":"node-0","uid":"uid-0","participantIndexes":[0]}]`, + "out of range": `[{"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","name":"node-0","uid":"uid-0","participantIndexes":[0,2]}]`, + "empty coverage": `[{"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","name":"node-0","uid":"uid-0"}]`, + "missing uid": `[{"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","name":"node-0","participantIndexes":[0,1]}]`, + "unknown field": `[{"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","name":"node-0","uid":"uid-0","participantIndexes":[0,1],"extra":true}]`, + "trailing value": `[{"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","name":"node-0","uid":"uid-0","participantIndexes":[0,1]}] {}`, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + if _, err := backendResourceReferences(malformed, 2); err == nil { + t.Fatalf("backendResourceReferences(%s) error = nil", malformed) + } + }) + } +} + +func TestMultiNodePassWithoutBackendIdentityBecomesInconclusive(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + run := testRun("run-multi") + run.Spec.Participants = []signalv1.Participant{ + participantSnapshot(readyNode("gpu-0", "H200"), 0), + participantSnapshot(readyNode("gpu-1", "H200"), 1), + } + run.Status = signalv1.CertificationRunStatus{ + Phase: signalv1.RunPhaseRunning, + Workflow: signalv1.WorkflowStatus{Name: "run-multi-a1", Attempt: 1}, + StartedAt: ptrTime(metav1.NewTime(time.Unix(100, 0))), + } + snapshot := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Script.ConfigMapName, Namespace: run.Namespace}, + Data: map[string]string{run.Spec.Script.Key: "test payload\n"}, + } + workflow := workflowObject(run, "Succeeded", "Pass", "Healthy") + store := &memoryEvidenceStore{} + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationRun{}). + WithObjects(run, snapshot, workflow). + Build() + r := &RunReconciler{Client: kube, Evidence: store, Now: func() time.Time { return time.Unix(200, 0) }} + + if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}); err != nil { + t.Fatalf("Reconcile: %v", err) + } + + var got signalv1.CertificationRun + if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { + t.Fatal(err) + } + + if got.Status.Phase != signalv1.RunPhaseInconclusive || + got.Status.Verdict != "Inconclusive" || + !strings.Contains(got.Status.Reason, "did not identify backend resources") { + t.Fatalf("status = %#v", got.Status) + } +} + +func TestMultiNodePassRetainsCoveredBackendIdentities(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + run := testRun("run-multi-pass") + run.Spec.Participants = []signalv1.Participant{ + participantSnapshot(readyNode("gpu-0", "H200"), 0), + participantSnapshot(readyNode("gpu-1", "H200"), 1), + } + run.Status = signalv1.CertificationRunStatus{ + Phase: signalv1.RunPhaseRunning, + Workflow: signalv1.WorkflowStatus{Name: "run-multi-pass-a1", Attempt: 1}, + StartedAt: ptrTime(metav1.NewTime(time.Unix(100, 0))), + } + snapshot := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Script.ConfigMapName, Namespace: run.Namespace}, + Data: map[string]string{run.Spec.Script.Key: "test payload\n"}, + } + workflow := workflowObject(run, "Succeeded", "Pass", "Healthy") + + outputs, _, err := unstructured.NestedSlice(workflow.Object, "status", "outputs", "parameters") + if err != nil { + t.Fatal(err) + } + + outputs = append(outputs, map[string]any{ + "name": "backend-resources", + "value": `[ + {"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","namespace":"signal","name":"node-0","uid":"uid-0","participantIndexes":[0]}, + {"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","namespace":"signal","name":"node-1","uid":"uid-1","participantIndexes":[1]} + ]`, + }) + if err := unstructured.SetNestedSlice(workflow.Object, outputs, "status", "outputs", "parameters"); err != nil { + t.Fatal(err) + } + + store := &memoryEvidenceStore{} + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationRun{}). + WithObjects(run, snapshot, workflow). + Build() + r := &RunReconciler{Client: kube, Evidence: store, Now: func() time.Time { return time.Unix(200, 0) }} + + if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}); err != nil { + t.Fatalf("Reconcile: %v", err) + } + + var got signalv1.CertificationRun + if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { + t.Fatal(err) + } + + if got.Status.Phase != signalv1.RunPhaseSucceeded || + len(got.Status.Workflow.BackendResources) != 2 || + got.Status.Workflow.BackendResources[1].ParticipantIndexes[0] != 1 { + t.Fatalf("status = %#v", got.Status) + } +} + func TestStartAttemptRecoversExistingOwnedWorkflow(t *testing.T) { t.Parallel() diff --git a/internal/signal/policy.go b/internal/signal/policy.go index 118f0303bb..bdcd9d39a4 100644 --- a/internal/signal/policy.go +++ b/internal/signal/policy.go @@ -141,16 +141,11 @@ func (r *RecurringReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( if admitted { activeByProfile++ - var node corev1.Node - if getErr := r.Reader.Get(ctx, client.ObjectKey{Name: nodeName}, &node); getErr != nil { - return ctrl.Result{}, fmt.Errorf( - "get admitted participant %s for failure-domain accounting: %w", - nodeName, - getErr, - ) + if policy.FailureDomainLabel != "" { + if err := r.accountAdmittedFailureDomains(ctx, nodeName, policy.FailureDomainLabel, activeByDomain); err != nil { + return ctrl.Result{}, err + } } - - activeByDomain[node.Labels[policy.FailureDomainLabel]]++ } } @@ -169,6 +164,47 @@ func (r *RecurringReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{RequeueAfter: interval}, nil } +func (r *RecurringReconciler) accountAdmittedFailureDomains( + ctx context.Context, + requestNodeName string, + failureDomainLabel string, + activeByDomain map[string]int32, +) error { + var requestNode corev1.Node + if err := r.Reader.Get(ctx, client.ObjectKey{Name: requestNodeName}, &requestNode); err != nil { + return fmt.Errorf("get admitted request Node %s for failure-domain accounting: %w", requestNodeName, err) + } + + runName := requestNode.Annotations[ReservationLabel] + if runName == "" { + return fmt.Errorf("admitted request Node %s has no reservation", requestNodeName) + } + + var run signalv1.CertificationRun + if err := r.Reader.Get(ctx, client.ObjectKey{Namespace: r.Namespace, Name: runName}, &run); err != nil { + return fmt.Errorf("get admitted run %s for failure-domain accounting: %w", runName, err) + } + + for _, participant := range run.Spec.Participants { + var node corev1.Node + if err := r.Reader.Get(ctx, client.ObjectKey{Name: participant.NodeName}, &node); err != nil { + return fmt.Errorf( + "get admitted participant %s for failure-domain accounting: %w", + participant.NodeName, + err, + ) + } + + if participant.NodeUID != "" && node.UID != participant.NodeUID { + return fmt.Errorf("admitted participant %s identity changed", participant.NodeName) + } + + activeByDomain[node.Labels[failureDomainLabel]]++ + } + + return nil +} + func (r *RecurringReconciler) projectStatuses(ctx context.Context, profile *signalv1.CertificationProfile) error { for _, status := range profile.Status.Nodes { if err := projectRecurring(ctx, r.Client, profile, status); err != nil { @@ -285,6 +321,31 @@ func (r *RecurringReconciler) evaluateNode( return status, false, nil } + if profile.Spec.Recurring.FailureDomainLabel != "" && + profile.Spec.Recurring.MaxConcurrentPerFailureDomain > 0 { + participants, err := r.Admission.selectParticipants(ctx, &node, profile) + if err != nil { + return status, false, fmt.Errorf("select recurring participants: %w", err) + } + + fits, err := r.participantsFitFailureDomainBudget( + ctx, + participants, + profile.Spec.Recurring.FailureDomainLabel, + profile.Spec.Recurring.MaxConcurrentPerFailureDomain, + activeByDomain, + ) + if err != nil { + return status, false, err + } + + if !fits { + status.Reason = "FailureDomainBudget" + + return status, false, nil + } + } + if err := r.Admission.admitNodeWithTriggerLocked(ctx, &node, profile.Name, "recurring"); err != nil { if apierrors.IsAlreadyExists(err) || apierrors.IsConflict(err) { status.Reason = "AdmissionRaced" @@ -301,6 +362,40 @@ func (r *RecurringReconciler) evaluateNode( return status, true, nil } +func (r *RecurringReconciler) participantsFitFailureDomainBudget( + ctx context.Context, + participants []signalv1.Participant, + failureDomainLabel string, + maxConcurrent int32, + activeByDomain map[string]int32, +) (bool, error) { + prospective := make(map[string]int32) + + for _, participant := range participants { + var node corev1.Node + if err := r.Reader.Get(ctx, client.ObjectKey{Name: participant.NodeName}, &node); err != nil { + return false, fmt.Errorf( + "get selected participant %s for failure-domain accounting: %w", + participant.NodeName, + err, + ) + } + + if participant.NodeUID != "" && node.UID != participant.NodeUID { + return false, fmt.Errorf("selected participant %s identity changed", participant.NodeName) + } + + domain := node.Labels[failureDomainLabel] + prospective[domain]++ + + if activeByDomain[domain]+prospective[domain] > maxConcurrent { + return false, nil + } + } + + return true, nil +} + func (r *RecurringReconciler) nodeBusy(ctx context.Context, node *corev1.Node) (bool, string, error) { var pods corev1.PodList if err := r.Reader.List(ctx, &pods, client.MatchingFields{"spec.nodeName": node.Name}); err != nil { diff --git a/internal/signal/policy_test.go b/internal/signal/policy_test.go index 933bf8bcc2..f856e4dc76 100644 --- a/internal/signal/policy_test.go +++ b/internal/signal/policy_test.go @@ -129,6 +129,65 @@ func TestRecurringFailureDomainAccountingFailsClosed(t *testing.T) { } } +func TestRecurringMultiNodeFailureDomainBudgetAccountsWholeGroup(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + if err := signalv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + first := readyNode("gpu-0", "H200") + first.Labels["topology.kubernetes.io/zone"] = "zone-a" + second := readyNode("gpu-1", "H200") + second.Labels["topology.kubernetes.io/zone"] = "zone-a" + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(first, second).Build() + r := &RecurringReconciler{Reader: kube} + participants := []signalv1.Participant{ + participantSnapshot(first, 0), + participantSnapshot(second, 1), + } + + fits, err := r.participantsFitFailureDomainBudget( + context.Background(), + participants, + "topology.kubernetes.io/zone", + 1, + map[string]int32{}, + ) + if err != nil { + t.Fatal(err) + } + + if fits { + t.Fatal("two participants in one domain exceeded the per-domain budget") + } + + second.Labels["topology.kubernetes.io/zone"] = "zone-b" + if err := kube.Update(context.Background(), second); err != nil { + t.Fatal(err) + } + + fits, err = r.participantsFitFailureDomainBudget( + context.Background(), + participants, + "topology.kubernetes.io/zone", + 1, + map[string]int32{}, + ) + if err != nil { + t.Fatal(err) + } + + if !fits { + t.Fatal("participants in distinct domains should fit the budget") + } +} + func TestRecurringDueUsesContinuousIdleAndFreshness(t *testing.T) { t.Parallel() From 51ed0a947af1c79fb344165c5f4c46b129ef7c60 Mon Sep 17 00:00:00 2001 From: chokevin Date: Tue, 20 Oct 2026 16:12:02 -0700 Subject: [PATCH 03/36] signal: add participant backend adapter --- Makefile | 28 +- cmd/project-signal-adapter/main.go | 165 ++++ .../03-adapter-serviceaccount-rbac.yaml.tmpl | 46 + .../04-adapter-workflowtemplate.yaml.tmpl | 116 +++ deploy/signal/README.md | 99 +- deploy/signal/manifest_test.go | 38 +- images/project-signal-adapter/Containerfile | 29 + internal/signaladapter/adapter.go | 933 ++++++++++++++++++ internal/signaladapter/adapter_test.go | 555 +++++++++++ internal/signaladapter/quantity.go | 16 + internal/signaladapter/runner.go | 465 +++++++++ internal/signaladapter/runner_test.go | 73 ++ 12 files changed, 2535 insertions(+), 28 deletions(-) create mode 100644 cmd/project-signal-adapter/main.go create mode 100644 deploy/signal/03-adapter-serviceaccount-rbac.yaml.tmpl create mode 100644 deploy/signal/04-adapter-workflowtemplate.yaml.tmpl create mode 100644 images/project-signal-adapter/Containerfile create mode 100644 internal/signaladapter/adapter.go create mode 100644 internal/signaladapter/adapter_test.go create mode 100644 internal/signaladapter/quantity.go create mode 100644 internal/signaladapter/runner.go create mode 100644 internal/signaladapter/runner_test.go diff --git a/Makefile b/Makefile index f401adc6b9..b773e992ad 100644 --- a/Makefile +++ b/Makefile @@ -95,6 +95,10 @@ MACHINE_OPS_SITE ?= PROJECT_SIGNAL_CONTROLLER_BIN=bin/project-signal-controller PROJECT_SIGNAL_CONTROLLER_CMD=./cmd/project-signal-controller SIGNAL_CONTROLLER_IMAGE ?= example.invalid/project-signal-controller:replace-me +PROJECT_SIGNAL_ADAPTER_BIN=bin/project-signal-adapter +PROJECT_SIGNAL_ADAPTER_CMD=./cmd/project-signal-adapter +SIGNAL_ADAPTER_IMAGE ?= example.invalid/project-signal-adapter@sha256:0000000000000000000000000000000000000000000000000000000000000000 +SIGNAL_ADAPTER_BUILD_IMAGE ?= example.invalid/project-signal-adapter:replace-me SIGNAL_NAMESPACE ?= project-signal SIGNAL_MANIFEST_TEMPLATES_DIR := deploy/signal SIGNAL_MANIFEST_RENDERED_DIR := deploy/signal/rendered @@ -237,14 +241,14 @@ NET_FRONTEND_CACHE_FILE := $(NET_FRONTEND_DIST_DIR)/.frontend-build-key # Frontend build toggle (dev builds produce unminified output with sourcemaps). REACT_DEV ?= false -.PHONY: all help fmt lint lint-actions test build vulncheck check-deps kubectl-unbounded kubectl-unbounded-build install-tools install-protoc install-helm generate kubectl-unbounded forge relctl relctl-build agent-artifacts-builder agent-artifacts-builder-build orcadev unbounded-agent machina machina-build machina-oci machina-oci-push machina-manifests machine-ops-controller machine-ops-controller-build machine-ops-controller-oci machine-ops-controller-oci-push machine-ops-manifests project-signal-controller project-signal-controller-build signal-manifests image-project-signal-controller-local metalman metalman-build metalman-oci metalman-oci-push unbounded-operator unbounded-operator-build unbounded-operator-manifests playpen-manifests e2e-gantry e2e-playpen e2e-racer gomod docs-serve unbounded-net-controller unbounded-net-controller-build unbounded-net-node unbounded-net-node-build unbounded-net-routeplan-debug unping unping-build unroute unroute-build license-check notice notice-check gantry gantry-build gantry-manifests inventory-manifests +.PHONY: all help fmt lint lint-actions test build vulncheck check-deps kubectl-unbounded kubectl-unbounded-build install-tools install-protoc install-helm generate kubectl-unbounded forge relctl relctl-build agent-artifacts-builder agent-artifacts-builder-build orcadev unbounded-agent machina machina-build machina-oci machina-oci-push machina-manifests machine-ops-controller machine-ops-controller-build machine-ops-controller-oci machine-ops-controller-oci-push machine-ops-manifests project-signal-controller project-signal-controller-build project-signal-adapter project-signal-adapter-build signal-manifests image-project-signal-controller-local image-project-signal-adapter-local metalman metalman-build metalman-oci metalman-oci-push unbounded-operator unbounded-operator-build unbounded-operator-manifests playpen-manifests e2e-gantry e2e-playpen e2e-racer gomod docs-serve unbounded-net-controller unbounded-net-controller-build unbounded-net-node unbounded-net-node-build unbounded-net-routeplan-debug unping unping-build unroute unroute-build license-check notice notice-check gantry gantry-build gantry-manifests inventory-manifests .PHONY: net-frontend net-frontend-clean net-ebpf-build net-ebpf-generate net-ebpf-verify net-manifests gantry-chart-lint gantry-chart-package release-bom release-manifests unbounded-operator-release-manifest .PHONY: image-machina-local image-token-refresher-local image-machine-ops-controller-local image-metalman-local image-unbounded-operator-local image-unbounded-operator-push image-playpen-local image-net-controller-local image-net-node-local image-gantry-local image-gantry-push images-local .PHONY: image-net-controller-push image-net-node-push images-net-all images-net-all-push ##@ General -all: kubectl-unbounded forge relctl machina machine-ops-controller project-signal-controller token-refresher unbounded-operator unbounded-net-controller unbounded-net-node unbounded-net-routeplan-debug unping unroute gantry ## Build all binaries (default) +all: kubectl-unbounded forge relctl machina machine-ops-controller project-signal-controller project-signal-adapter token-refresher unbounded-operator unbounded-net-controller unbounded-net-node unbounded-net-routeplan-debug unping unroute gantry ## Build all binaries (default) help: ## Show this help @echo "" @@ -297,6 +301,7 @@ help: ## Show this help @echo " machina | machina-build Build machina controller (with/without lint/test)" @echo " machine-ops-controller Build machine-ops-controller" @echo " project-signal-controller Build Project Signal controller" + @echo " project-signal-adapter Build Project Signal participant adapter" @echo " signal-manifests Render Project Signal manifests" @echo " metalman | metalman-build Build metalman controller (with/without lint/test)" @echo " unbounded-operator | unbounded-operator-build Build the top-level Site operator" @@ -319,6 +324,7 @@ help: ## Show this help @echo " image-token-refresher-local Build token-refresher image" @echo " image-machine-ops-controller-local Build machine-ops-controller image" @echo " image-project-signal-controller-local Build Project Signal controller image" + @echo " image-project-signal-adapter-local Build Project Signal adapter image" @echo " Build Project Signal recovery verifier image" @echo " Build Project Signal GPU runner image" @echo " Build and push Project Signal GPU runner image" @@ -1094,6 +1100,12 @@ project-signal-controller-build: signal-manifests ## Build Project Signal contro project-signal-controller: test project-signal-controller-build ## Build Project Signal controller (implies test) +project-signal-adapter-build: signal-manifests ## Build Project Signal adapter without lint/test + @mkdir -p bin + $(GOBUILD) -trimpath -ldflags '$(STAMP_LDFLAGS)' -o $(PROJECT_SIGNAL_ADAPTER_BIN) $(PROJECT_SIGNAL_ADAPTER_CMD) + +project-signal-adapter: test project-signal-adapter-build ## Build Project Signal adapter (implies test) + image-project-signal-controller-local: ## Build Project Signal controller image locally $(CONTAINER_ENGINE) build \ --build-arg VERSION=$(VERSION) \ @@ -1103,6 +1115,15 @@ image-project-signal-controller-local: ## Build Project Signal controller image -f ./images/project-signal-controller/Containerfile . $(call trivy-maybe,$(SIGNAL_CONTROLLER_IMAGE)) +image-project-signal-adapter-local: ## Build Project Signal adapter image locally + $(CONTAINER_ENGINE) build \ + --build-arg VERSION=$(VERSION) \ + --build-arg GIT_COMMIT=$(GIT_COMMIT) \ + --build-arg BUILD_TIME=$(BUILD_TIME) \ + -t project-signal-adapter:$(VERSION_TAG) -t $(SIGNAL_ADAPTER_BUILD_IMAGE) \ + -f ./images/project-signal-adapter/Containerfile . + $(call trivy-maybe,$(SIGNAL_ADAPTER_BUILD_IMAGE)) + MACHINA_NAMESPACE ?= $(UNBOUNDED_NAMESPACE) MACHINA_API_SERVER_ENDPOINT ?= MACHINA_MANIFEST_TEMPLATES_DIR := deploy/machina @@ -1182,7 +1203,8 @@ signal-manifests: ## Render Project Signal manifests into deploy/signal/rendered --templates-dir $(SIGNAL_MANIFEST_TEMPLATES_DIR) \ --output-dir $(SIGNAL_MANIFEST_RENDERED_DIR) \ --set Namespace=$(SIGNAL_NAMESPACE) \ - --set ControllerImage=$(SIGNAL_CONTROLLER_IMAGE) + --set ControllerImage=$(SIGNAL_CONTROLLER_IMAGE) \ + --set AdapterImage=$(SIGNAL_ADAPTER_IMAGE) @mkdir -p $(SIGNAL_MANIFEST_RENDERED_DIR)/crd @cp $(SIGNAL_MANIFEST_TEMPLATES_DIR)/crd/*.yaml $(SIGNAL_MANIFEST_RENDERED_DIR)/crd/ @echo "Rendered Project Signal manifests into $(SIGNAL_MANIFEST_RENDERED_DIR) (image: $(SIGNAL_CONTROLLER_IMAGE))" diff --git a/cmd/project-signal-adapter/main.go b/cmd/project-signal-adapter/main.go new file mode 100644 index 0000000000..7b977d39dc --- /dev/null +++ b/cmd/project-signal-adapter/main.go @@ -0,0 +1,165 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + osSignal "os/signal" + "path/filepath" + "syscall" + + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + "github.com/Azure/unbounded/internal/signaladapter" + "github.com/Azure/unbounded/internal/version" +) + +const adapterName = "project-signal-adapter" + +type config struct { + signaladapter.Config + outputDirectory string +} + +func main() { + var cfg config + + command := &cobra.Command{ + Use: adapterName, + Short: "Run participant-aware Project Signal backend workloads", + Version: version.Version + " (commit: " + version.GitCommit + ")", + RunE: func(cmd *cobra.Command, _ []string) error { + ctx, cancel := osSignal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + return run(ctx, cfg) + }, + } + + flags := command.Flags() + flags.StringVar(&cfg.Namespace, "namespace", "", "Namespace for backend resources") + flags.StringVar(&cfg.WorkflowName, "workflow-name", "", "Owning Argo Workflow name") + flags.StringVar(&cfg.RunName, "run-name", "", "CertificationRun name") + flags.Var(uidValue{target: &cfg.RunUID}, "run-uid", "CertificationRun UID") + flags.StringVar(&cfg.ParticipantsJSON, "participants-json", "", "Ordered participant snapshot JSON") + flags.IntVar(&cfg.ParticipantCount, "participant-count", 0, "Declared participant count") + flags.StringVar(&cfg.RequestNodeName, "request-node-name", "", "Compatibility request Node name") + flags.StringVar(&cfg.LegacyParticipants, "participants", "", "Compatibility comma-separated Node names") + flags.StringVar(&cfg.ScriptConfigMap, "script-configmap", "", "Immutable script ConfigMap") + flags.StringVar(&cfg.ScriptKey, "script-key", "", "Script ConfigMap key") + flags.StringVar(&cfg.ScriptDigest, "script-digest", "", "SHA-256 digest of the script bytes") + flags.StringVar(&cfg.RunnerImage, "runner-image", "", "Digest-pinned runner image") + flags.StringVar(&cfg.BackendAPIVersion, "backend-api-version", "", "Backend API version") + flags.StringVar(&cfg.BackendKind, "backend-kind", "", "Backend kind") + flags.StringVar(&cfg.Queue, "queue", "", "Kueue LocalQueue name") + flags.StringVar(&cfg.TolerationsJSON, "tolerations", "[]", "JSON-encoded Pod tolerations") + flags.StringVar(&cfg.ScriptEnvironmentJSON, "script-environment", "{}", "JSON-encoded script environment") + flags.StringVar(&cfg.GPUResourceCount, "gpu-resource-count", "0", "GPU resource limit per participant") + flags.BoolVar(&cfg.Privileged, "privileged", false, "Run the participant workload with a privileged security context") + flags.IntVar(&cfg.Attempt, "attempt", 0, "One-based Workflow attempt number") + flags.Int64Var(&cfg.BackoffLimit, "backoff-limit", 0, "MPIJob backoff limit") + flags.Int64Var(&cfg.TTLSeconds, "ttl-seconds-after-finished", 3600, "MPIJob TTL after completion") + flags.StringVar(&cfg.outputDirectory, "output-directory", "/outputs", "Argo output directory") + + command.CompletionOptions.DisableDefaultCmd = true + + if err := command.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } +} + +func run(ctx context.Context, cfg config) error { + restConfig, err := rest.InClusterConfig() + if err != nil { + return fmt.Errorf("load in-cluster Kubernetes configuration: %w", err) + } + + dynamicClient, err := dynamic.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("create dynamic Kubernetes client: %w", err) + } + + kubernetesClient, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("create Kubernetes client: %w", err) + } + + runner := &signaladapter.Runner{ + Dynamic: dynamicClient, + Kubernetes: kubernetesClient, + } + + output, err := runner.Execute(ctx, cfg.Config) + if err != nil { + return err + } + + return writeOutputs(cfg.outputDirectory, output) +} + +func writeOutputs(directory string, output signaladapter.AggregateOutput) error { + if directory == "" { + return errors.New("output-directory is required") + } + + if err := os.MkdirAll(directory, 0o750); err != nil { + return fmt.Errorf("create output directory: %w", err) + } + + outputs := map[string]string{ + "verdict": output.Verdict, + "reason": output.Reason, + "measurements": output.Measurements, + "backend-resources": output.BackendJSON, + } + for name, value := range outputs { + if err := os.WriteFile(filepath.Join(directory, name), []byte(value), 0o600); err != nil { + return fmt.Errorf("write %s output: %w", name, err) + } + } + + evidence, err := json.Marshal(output) + if err != nil { + return fmt.Errorf("encode adapter result: %w", err) + } + + fmt.Println(string(evidence)) + + return nil +} + +type uidValue struct { + target *types.UID +} + +func (v uidValue) String() string { + if v.target == nil { + return "" + } + + return string(*v.target) +} + +func (v uidValue) Set(value string) error { + if v.target == nil { + return errors.New("UID target is nil") + } + + *v.target = types.UID(value) + + return nil +} + +func (uidValue) Type() string { + return "uid" +} diff --git a/deploy/signal/03-adapter-serviceaccount-rbac.yaml.tmpl b/deploy/signal/03-adapter-serviceaccount-rbac.yaml.tmpl new file mode 100644 index 0000000000..102402fe25 --- /dev/null +++ b/deploy/signal/03-adapter-serviceaccount-rbac.yaml.tmpl @@ -0,0 +1,46 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: project-signal-adapter + namespace: {{ default "project-signal" .Namespace }} + labels: + app.kubernetes.io/name: project-signal-adapter +automountServiceAccountToken: true + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: project-signal-adapter + namespace: {{ default "project-signal" .Namespace }} +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get"] + - apiGroups: ["kubeflow.org"] + resources: ["mpijobs"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["argoproj.io"] + resources: ["workflows"] + verbs: ["get"] + - apiGroups: ["argoproj.io"] + resources: ["workflowtaskresults"] + verbs: ["create", "patch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: project-signal-adapter + namespace: {{ default "project-signal" .Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: project-signal-adapter +subjects: + - kind: ServiceAccount + name: project-signal-adapter + namespace: {{ default "project-signal" .Namespace }} diff --git a/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl b/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl new file mode 100644 index 0000000000..6d0eaaece9 --- /dev/null +++ b/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl @@ -0,0 +1,116 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: project-signal-participant-adapter + namespace: {{ default "project-signal" .Namespace }} +spec: + serviceAccountName: project-signal-adapter + entrypoint: certification + arguments: + parameters: + - name: run-name + - name: run-uid + - name: participants + - name: participants-json + - name: participant-count + - name: request-node-name + - name: script-configmap + - name: script-key + - name: script-digest + - name: backend-api-version + - name: backend-kind + - name: queue + - name: runner-image + - name: tolerations + value: "[]" + - name: script-environment + value: "{}" + - name: mpi-payload-mode + value: "" + - name: expected-gpu-product-regex + value: "" + - name: expected-gpu-count + value: "0" + - name: observed-gpu-capacity + value: "0" + - name: observed-gpu-allocatable + value: "0" + - name: gpu-resource-count + value: "0" + - name: privileged + value: "false" + - name: attempt + templates: + - name: certification + container: + image: {{ default "example.invalid/project-signal-adapter@sha256:0000000000000000000000000000000000000000000000000000000000000000" .AdapterImage }} + imagePullPolicy: IfNotPresent + args: + - --namespace + - "$(POD_NAMESPACE)" + - --workflow-name + - "$(WORKFLOW_NAME)" + - --run-name + - "{{`{{workflow.parameters.run-name}}`}}" + - --run-uid + - "{{`{{workflow.parameters.run-uid}}`}}" + - --participants + - "{{`{{workflow.parameters.participants}}`}}" + - --participants-json + - "{{`{{workflow.parameters.participants-json}}`}}" + - --participant-count + - "{{`{{workflow.parameters.participant-count}}`}}" + - --request-node-name + - "{{`{{workflow.parameters.request-node-name}}`}}" + - --script-configmap + - "{{`{{workflow.parameters.script-configmap}}`}}" + - --script-key + - "{{`{{workflow.parameters.script-key}}`}}" + - --script-digest + - "{{`{{workflow.parameters.script-digest}}`}}" + - --runner-image + - "{{`{{workflow.parameters.runner-image}}`}}" + - --backend-api-version + - "{{`{{workflow.parameters.backend-api-version}}`}}" + - --backend-kind + - "{{`{{workflow.parameters.backend-kind}}`}}" + - --queue + - "{{`{{workflow.parameters.queue}}`}}" + - --tolerations + - "{{`{{workflow.parameters.tolerations}}`}}" + - --script-environment + - "{{`{{workflow.parameters.script-environment}}`}}" + - --gpu-resource-count + - "{{`{{workflow.parameters.gpu-resource-count}}`}}" + - "--privileged={{`{{workflow.parameters.privileged}}`}}" + - --attempt + - "{{`{{workflow.parameters.attempt}}`}}" + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: WORKFLOW_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['workflows.argoproj.io/workflow'] + volumeMounts: + - name: outputs + mountPath: /outputs + volumes: + - name: outputs + emptyDir: {} + outputs: + parameters: + - name: verdict + valueFrom: + path: /outputs/verdict + - name: reason + valueFrom: + path: /outputs/reason + - name: measurements + valueFrom: + path: /outputs/measurements + - name: backend-resources + valueFrom: + path: /outputs/backend-resources diff --git a/deploy/signal/README.md b/deploy/signal/README.md index 68544d4ced..beb15185a9 100644 --- a/deploy/signal/README.md +++ b/deploy/signal/README.md @@ -1,10 +1,11 @@ # Project Signal controller Project Signal is a Kubernetes controller for admitting approved idle Nodes to -operator-supplied certification workflows. The upstream integration provides -the API, controller, CRDs, RBAC, and controller Deployment. It intentionally -does not publish certification scripts, runner images, CertificationProfiles, -or WorkflowTemplates. +certification workflows. The upstream integration provides the API, controller, +CRDs, controller Deployment, and an optional generic participant adapter for +ordinary Pods and Kubeflow MPIJobs. It intentionally does not publish +certification scripts, runner images, CertificationProfiles, hardware policy, +or environment-specific queue configuration. ## Architecture @@ -21,7 +22,7 @@ approved Node or recurring policy immutable CertificationRun input + SHA-256 digest | v - operator-supplied Argo WorkflowTemplate + operator-supplied or generic adapter WorkflowTemplate | v typed result -> retained evidence -> Node projection -> cleanup @@ -37,13 +38,21 @@ The controller does not contain or publish certification script bodies. - Argo Workflows `argoproj.io/v1alpha1` Workflow API. - A controller image built from `cmd/project-signal-controller`. - An operator-owned script ConfigMap and execution image. -- An operator-owned WorkflowTemplate implementing the interface below. +- An operator-owned WorkflowTemplate implementing the interface below, or the + included `project-signal-participant-adapter` WorkflowTemplate. - Explicitly reviewed Node selectors, resource requests, tolerations, policy values, and cleanup behavior. -Backends such as Jobs, MPIJobs, or other workload APIs are operator choices. -Install their controllers and grant their runner permissions separately. -Project Signal does not install them or grant generic workload-runner RBAC. +The execution image used with the included adapter must contain Bash and the +standard `mkdir`, `chmod`, `base64`, and `tr` utilities, plus every tool required +by the selected script. The adapter image and execution image are separate: the +upstream adapter creates and observes workloads, while the operator-owned +execution image runs the script. + +The included adapter supports `v1` Pods and `kubeflow.org/v2beta1` MPIJobs. +Project Signal does not install MPI Operator or Kueue. Its adapter RBAC is +namespace-scoped and intentionally grants only the Pod, Pod log, MPIJob, +Workflow, and Argo task-result access needed by the included template. ## WorkflowTemplate interface @@ -98,18 +107,26 @@ reservation rolls back every reservation already acquired. Each participant is bound by Node UID and an immutable capacity and allocatable snapshot in the CertificationRun. -The participant group is a certification boundary, not necessarily one -distributed process group. Operator WorkflowTemplates should create one -independently pinned backend workload per participant unless a test explicitly -requires a cross-Node world. The backend adapter owns launcher and worker -topology, rank environment, rendezvous or SSH configuration, workload RBAC, -deadline propagation, foreground cleanup, and result aggregation. - -The currently validated adapter contract uses ordinary single-Node workloads -for Node diagnostics and `kubeflow.org/v2beta1` MPIJobs for tests that require -an OpenMPI launcher and worker. Kubeflow Trainer v2 TrainJobs can be supplied by -operators through the same generic interface, but require a separately managed, -version-pinned MPI TrainingRuntime and are not installed by Project Signal. +The participant group is a certification boundary, not one distributed process +group. The included adapter creates one independently pinned backend workload +per participant. Ordinary diagnostics use one Pod per participant. MPI +diagnostics use one MPIJob per participant with one launcher and one GPU worker; +only the worker is pinned to the participant Node and receives the configured +GPU limit. The adapter propagates the Workflow active deadline, uses the +profile's queue, records API-assigned workload UIDs, reads the final typed result +from bounded workload logs, aggregates diagnostic state separately from backend +execution state, and foreground-deletes every created workload. + +The selected script is invoked with no arguments for a Pod backend, +`mpi-launcher ` in an MPIJob launcher, and `mpi-worker` in its worker. +Scripts write `verdict`, `reason`, and optional `measurements.txt` under +`/results`. A nonzero process exit remains an execution error and cannot be +converted into a diagnostic `Fail`. Missing results, partial backend execution, +identity gaps, malformed output, deadline interruption, and cleanup failure are +all `Inconclusive`. + +Other backend kinds remain operator-supplied and are not installed or granted +permissions by the included adapter. ## Render and install @@ -117,9 +134,14 @@ Render the controller manifests: ```bash make signal-manifests \ - SIGNAL_CONTROLLER_IMAGE= + SIGNAL_CONTROLLER_IMAGE= \ + SIGNAL_ADAPTER_IMAGE= ``` +Local adapter builds use the separate mutable +`SIGNAL_ADAPTER_BUILD_IMAGE=` staging reference because +container engines cannot tag a local build with an `image@sha256:` reference. + Generated output is written to `deploy/signal/rendered/`. Validate it with the target cluster before applying: @@ -131,8 +153,37 @@ kubectl apply -f deploy/signal/rendered/ ``` The upstream manifests install no profiles. Operators must create a -CertificationProfile that references their own script ConfigMap, -WorkflowTemplate, backend, and digest-pinned runner image. +CertificationProfile that references their own script ConfigMap and +digest-pinned runner image. To use the included adapter, set +`spec.workflow.name: project-signal-participant-adapter` and select either: + +```yaml +backend: + apiVersion: v1 + kind: Pod +``` + +or an explicit Kueue queue: + +```yaml +backend: + apiVersion: kubeflow.org/v2beta1 + kind: MPIJob + queue: +``` + +`gpu-resource-count` remains an operator-supplied profile parameter. It may be +zero for Pod diagnostics and must be positive for MPIJobs. The MPIJob policy is +fail-closed: `backoffLimit: 0`, `cleanPodPolicy: All`, a one-hour post-finish +TTL, replica-level `restartPolicy: Never`, `WaitForWorkersReady`, and +`runLauncherAsWorker: false`. + +Operators can override the nonnegative MPI retry and TTL values through profile +parameters `backend-backoff-limit` and +`backend-ttl-seconds-after-finished`. These become +`BACKEND_BACKOFF_LIMIT` and `BACKEND_TTL_SECONDS_AFTER_FINISHED` in the validated +script environment and remain explicit per-profile policy rather than cluster +defaults. ## Results and evidence diff --git a/deploy/signal/manifest_test.go b/deploy/signal/manifest_test.go index e0f97d395b..0fb4cbb6c3 100644 --- a/deploy/signal/manifest_test.go +++ b/deploy/signal/manifest_test.go @@ -12,7 +12,7 @@ import ( "testing" ) -func TestUpstreamManifestsContainOnlyControllerSurface(t *testing.T) { +func TestUpstreamManifestsContainControllerAndGenericAdapter(t *testing.T) { t.Parallel() templates, err := filepath.Glob("*.yaml.tmpl") @@ -26,6 +26,8 @@ func TestUpstreamManifestsContainOnlyControllerSurface(t *testing.T) { "00-namespace.yaml.tmpl", "01-serviceaccounts.yaml.tmpl", "02-rbac.yaml.tmpl", + "03-adapter-serviceaccount-rbac.yaml.tmpl", + "04-adapter-workflowtemplate.yaml.tmpl", "06-deployment.yaml.tmpl", } if !reflect.DeepEqual(templates, want) { @@ -46,6 +48,40 @@ func TestUpstreamManifestsContainOnlyControllerSurface(t *testing.T) { } } +func TestAdapterManifestIsGenericAndParticipantAware(t *testing.T) { + t.Parallel() + + content, err := os.ReadFile("04-adapter-workflowtemplate.yaml.tmpl") + if err != nil { + t.Fatal(err) + } + + manifest := string(content) + for _, required := range []string{ + "participants-json", + "participant-count", + "request-node-name", + "backend-resources", + "backend-api-version", + "backend-kind", + "queue", + } { + if !strings.Contains(manifest, required) { + t.Fatalf("adapter WorkflowTemplate does not contain %q", required) + } + } + + for _, forbidden := range []string{ + "project-signal-certification/", + "REPLACE_WITH_", + "internal-node", + } { + if strings.Contains(manifest, forbidden) { + t.Fatalf("adapter WorkflowTemplate contains downstream-specific value %q", forbidden) + } + } +} + func TestUpstreamDoesNotPublishCertificationPackage(t *testing.T) { t.Parallel() diff --git a/images/project-signal-adapter/Containerfile b/images/project-signal-adapter/Containerfile new file mode 100644 index 0000000000..2317fc7945 --- /dev/null +++ b/images/project-signal-adapter/Containerfile @@ -0,0 +1,29 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +FROM docker.io/library/golang:1.27.0-trixie AS builder + +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates git make \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . + +ARG TARGETOS +ARG TARGETARCH +ARG VERSION=dev +ARG GIT_COMMIT= +ARG BUILD_TIME= + +RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 \ + make project-signal-adapter-build VERSION=${VERSION} \ + ${GIT_COMMIT:+GIT_COMMIT=${GIT_COMMIT}} ${BUILD_TIME:+BUILD_TIME=${BUILD_TIME}} + +FROM gcr.io/distroless/static-debian12:nonroot + +COPY --from=builder /src/bin/project-signal-adapter /project-signal-adapter +COPY LICENSE NOTICE /licenses/ + +ENTRYPOINT ["/project-signal-adapter"] diff --git a/internal/signaladapter/adapter.go b/internal/signaladapter/adapter.go new file mode 100644 index 0000000000..e3b5f754c0 --- /dev/null +++ b/internal/signaladapter/adapter.go @@ -0,0 +1,933 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signaladapter + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "maps" + "regexp" + "slices" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" + + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" +) + +const ( + MaxParticipants = 64 + MaxMeasurementsBytes = 16 * 1024 + MaxBackendOutputBytes = 16 * 1024 + resultPrefix = "PROJECT_SIGNAL_RESULT_V1 " +) + +var ( + runnerImagePattern = regexp.MustCompile(`^[^@\s]+@sha256:[0-9a-f]{64}$`) + scriptDigestPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) +) + +var reservedEnvironmentNames = map[string]struct{}{ + "BASH_ENV": {}, + "BASHOPTS": {}, + "CDPATH": {}, + "ENV": {}, + "GLOBIGNORE": {}, + "HOST_PROC_ROOT": {}, + "HOST_SYSFS_ROOT": {}, + "IFS": {}, + "LD_LIBRARY_PATH": {}, + "LD_PRELOAD": {}, + "PATH": {}, + "PYTHONHOME": {}, + "PYTHONPATH": {}, + "SHELLOPTS": {}, +} + +type Config struct { + Namespace string + WorkflowName string + RunName string + RunUID types.UID + ParticipantsJSON string + ParticipantCount int + RequestNodeName string + LegacyParticipants string + ScriptConfigMap string + ScriptKey string + ScriptDigest string + RunnerImage string + BackendAPIVersion string + BackendKind string + Queue string + TolerationsJSON string + ScriptEnvironmentJSON string + GPUResourceCount string + Privileged bool + Attempt int + BackoffLimit int64 + TTLSeconds int64 +} + +type Result struct { + ParticipantIndex int `json:"participantIndex"` + ExecutionState string `json:"executionState"` + Verdict string `json:"verdict"` + Reason string `json:"reason"` + Measurements string `json:"measurements"` +} + +type AggregateOutput struct { + ExecutionState string `json:"executionState"` + Verdict string `json:"verdict"` + Reason string `json:"reason"` + Measurements string `json:"measurements"` + BackendResources []signalv1.BackendResourceReference `json:"-"` + BackendJSON string `json:"backend-resources"` +} + +func ParseParticipants( + participantsJSON string, + participantCount int, + requestNodeName string, + legacyParticipants string, +) ([]signalv1.Participant, error) { + if participantCount < 1 || participantCount > MaxParticipants { + return nil, fmt.Errorf("participant-count must be between 1 and %d", MaxParticipants) + } + + var participants []signalv1.Participant + + if strings.TrimSpace(participantsJSON) != "" { + decoder := json.NewDecoder(strings.NewReader(participantsJSON)) + decoder.DisallowUnknownFields() + + if err := decoder.Decode(&participants); err != nil { + return nil, fmt.Errorf("decode participants-json: %w", err) + } + + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + if err == nil { + return nil, errors.New("participants-json contains trailing values") + } + + return nil, fmt.Errorf("decode trailing participants-json value: %w", err) + } + } else { + nodeName := requestNodeName + if nodeName == "" { + names := strings.Split(legacyParticipants, ",") + if len(names) > 0 { + nodeName = strings.TrimSpace(names[0]) + } + } + + if nodeName == "" { + return nil, errors.New("participants-json or a compatibility Node name is required") + } + + participants = []signalv1.Participant{{Index: 0, NodeName: nodeName}} + } + + if len(participants) != participantCount { + return nil, fmt.Errorf( + "participant-count %d does not match participants-json length %d", + participantCount, + len(participants), + ) + } + + names := make(map[string]struct{}, len(participants)) + uids := make(map[types.UID]struct{}, len(participants)) + + for index := range participants { + participant := &participants[index] + if participant.Index != int32(index) { + return nil, fmt.Errorf("participants-json[%d].index must equal %d", index, index) + } + + if errors := validation.IsDNS1123Subdomain(participant.NodeName); len(errors) > 0 { + return nil, fmt.Errorf("participants-json[%d].nodeName is invalid: %s", index, strings.Join(errors, ", ")) + } + + if _, duplicate := names[participant.NodeName]; duplicate { + return nil, fmt.Errorf("participants-json[%d].nodeName duplicates %q", index, participant.NodeName) + } + + names[participant.NodeName] = struct{}{} + + if strings.TrimSpace(participantsJSON) == "" { + continue + } + + if participant.NodeUID == "" { + return nil, fmt.Errorf("participants-json[%d].nodeUID is required", index) + } + + if _, duplicate := uids[participant.NodeUID]; duplicate { + return nil, fmt.Errorf("participants-json[%d].nodeUID duplicates %q", index, participant.NodeUID) + } + + uids[participant.NodeUID] = struct{}{} + } + + return participants, nil +} + +func ValidateConfig(config Config) ([]signalv1.Participant, []corev1.Toleration, []corev1.EnvVar, error) { + for name, value := range map[string]string{ + "namespace": config.Namespace, + "workflow-name": config.WorkflowName, + "run-name": config.RunName, + "run-uid": string(config.RunUID), + "script-configmap": config.ScriptConfigMap, + "script-key": config.ScriptKey, + "script-digest": config.ScriptDigest, + "runner-image": config.RunnerImage, + "backend-kind": config.BackendKind, + } { + if value == "" { + return nil, nil, nil, fmt.Errorf("%s is required", name) + } + } + + for name, value := range map[string]string{ + "namespace": config.Namespace, + "workflow-name": config.WorkflowName, + "run-name": config.RunName, + "script-configmap": config.ScriptConfigMap, + } { + if errors := validation.IsDNS1123Subdomain(value); len(errors) > 0 { + return nil, nil, nil, fmt.Errorf("%s is invalid: %s", name, strings.Join(errors, ", ")) + } + } + + if errors := validation.IsConfigMapKey(config.ScriptKey); len(errors) > 0 { + return nil, nil, nil, fmt.Errorf("script-key is invalid: %s", strings.Join(errors, ", ")) + } + + if !runnerImagePattern.MatchString(config.RunnerImage) { + return nil, nil, nil, errors.New("runner-image must be an immutable sha256 digest reference") + } + + if !scriptDigestPattern.MatchString(config.ScriptDigest) { + return nil, nil, nil, errors.New("script-digest must contain 64 lowercase hexadecimal characters") + } + + switch { + case config.BackendKind == "Pod" && config.BackendAPIVersion == "v1": + case config.BackendKind == "MPIJob" && config.BackendAPIVersion == "kubeflow.org/v2beta1": + if config.Queue == "" { + return nil, nil, nil, errors.New("queue is required for MPIJob execution") + } + + if errors := validation.IsDNS1123Subdomain(config.Queue); len(errors) > 0 { + return nil, nil, nil, fmt.Errorf("queue is invalid: %s", strings.Join(errors, ", ")) + } + default: + return nil, nil, nil, fmt.Errorf( + "unsupported backend %s %s", + config.BackendAPIVersion, + config.BackendKind, + ) + } + + if config.BackoffLimit < 0 { + return nil, nil, nil, errors.New("backoff-limit must not be negative") + } + + if config.TTLSeconds < 0 { + return nil, nil, nil, errors.New("ttl-seconds-after-finished must not be negative") + } + + if config.Attempt < 1 { + return nil, nil, nil, errors.New("attempt must be positive") + } + + participants, err := ParseParticipants( + config.ParticipantsJSON, + config.ParticipantCount, + config.RequestNodeName, + config.LegacyParticipants, + ) + if err != nil { + return nil, nil, nil, err + } + + var tolerations []corev1.Toleration + if err := decodeJSONObject(config.TolerationsJSON, &tolerations); err != nil { + return nil, nil, nil, fmt.Errorf("decode tolerations: %w", err) + } + + var environment map[string]string + if err := decodeJSONObject(config.ScriptEnvironmentJSON, &environment); err != nil { + return nil, nil, nil, fmt.Errorf("decode script-environment: %w", err) + } + + if value := environment["BACKEND_BACKOFF_LIMIT"]; value != "" { + parsed, err := strconv.ParseInt(value, 10, 64) + if err != nil || parsed < 0 { + return nil, nil, nil, errors.New("BACKEND_BACKOFF_LIMIT must be a nonnegative integer") + } + + config.BackoffLimit = parsed + } + + if value := environment["BACKEND_TTL_SECONDS_AFTER_FINISHED"]; value != "" { + parsed, err := strconv.ParseInt(value, 10, 64) + if err != nil || parsed < 0 { + return nil, nil, nil, errors.New("BACKEND_TTL_SECONDS_AFTER_FINISHED must be a nonnegative integer") + } + + config.TTLSeconds = parsed + } + + environmentNames := make([]string, 0, len(environment)) + for name := range environment { + environmentNames = append(environmentNames, name) + } + + slices.Sort(environmentNames) + + env := make([]corev1.EnvVar, 0, len(environmentNames)+4) + for _, name := range environmentNames { + if errors := validation.IsEnvVarName(name); len(errors) > 0 { + return nil, nil, nil, fmt.Errorf("script-environment contains invalid name %q", name) + } + + if strings.HasPrefix(name, "PROJECT_SIGNAL_") { + return nil, nil, nil, fmt.Errorf("script-environment contains reserved name %q", name) + } + + if _, reserved := reservedEnvironmentNames[name]; reserved { + return nil, nil, nil, fmt.Errorf("script-environment contains reserved name %q", name) + } + + env = append(env, corev1.EnvVar{Name: name, Value: environment[name]}) + } + + return participants, tolerations, env, nil +} + +func RenderResources( + config Config, + participants []signalv1.Participant, + tolerations []corev1.Toleration, + environment []corev1.EnvVar, + workflow metav1.OwnerReference, + activeDeadlineSeconds int64, +) ([]*unstructured.Unstructured, error) { + resources := make([]*unstructured.Unstructured, 0, len(participants)) + + gpuCount, err := strconv.ParseInt(config.GPUResourceCount, 10, 64) + if err != nil || gpuCount < 0 { + return nil, fmt.Errorf("gpu-resource-count must be a nonnegative integer") + } + + if config.BackendKind == "MPIJob" && gpuCount < 1 { + return nil, errors.New("gpu-resource-count must be positive for MPIJob execution") + } + + var policyEnvironment map[string]string + if err := decodeJSONObject(config.ScriptEnvironmentJSON, &policyEnvironment); err != nil { + return nil, fmt.Errorf("decode adapter policy: %w", err) + } + + if value := policyEnvironment["BACKEND_BACKOFF_LIMIT"]; value != "" { + config.BackoffLimit, err = strconv.ParseInt(value, 10, 64) + if err != nil { + return nil, fmt.Errorf("parse BACKEND_BACKOFF_LIMIT: %w", err) + } + } + + if value := policyEnvironment["BACKEND_TTL_SECONDS_AFTER_FINISHED"]; value != "" { + config.TTLSeconds, err = strconv.ParseInt(value, 10, 64) + if err != nil { + return nil, fmt.Errorf("parse BACKEND_TTL_SECONDS_AFTER_FINISHED: %w", err) + } + } + + for _, participant := range participants { + var object any + + switch config.BackendKind { + case "Pod": + object = ordinaryPod(config, participant, tolerations, environment, workflow, activeDeadlineSeconds, gpuCount) + case "MPIJob": + object = mpiJob(config, participant, tolerations, environment, workflow, activeDeadlineSeconds, gpuCount) + default: + return nil, fmt.Errorf("unsupported backend kind %q", config.BackendKind) + } + + content, err := runtime.DefaultUnstructuredConverter.ToUnstructured(object) + if err != nil { + return nil, fmt.Errorf("convert %s for participant %d: %w", config.BackendKind, participant.Index, err) + } + + resources = append(resources, &unstructured.Unstructured{Object: content}) + } + + return resources, nil +} + +func BackendResource(resource *unstructured.Unstructured, participantIndex int32) (signalv1.BackendResourceReference, error) { + label := resource.GetLabels()["signal.unbounded-cloud.io/participant-index"] + if label != strconv.FormatInt(int64(participantIndex), 10) { + return signalv1.BackendResourceReference{}, fmt.Errorf( + "backend resource participant label %q does not match index %d", + label, + participantIndex, + ) + } + + if resource.GetAPIVersion() == "" || + resource.GetKind() == "" || + resource.GetName() == "" || + resource.GetUID() == "" { + return signalv1.BackendResourceReference{}, errors.New("backend resource has an incomplete identity") + } + + return signalv1.BackendResourceReference{ + APIVersion: resource.GetAPIVersion(), + Kind: resource.GetKind(), + Namespace: resource.GetNamespace(), + Name: resource.GetName(), + UID: resource.GetUID(), + ParticipantIndexes: []int32{participantIndex}, + }, nil +} + +func Aggregate( + participantCount int, + backendResources []signalv1.BackendResourceReference, + results []Result, +) (AggregateOutput, error) { + if participantCount < 1 || participantCount > MaxParticipants { + return AggregateOutput{}, fmt.Errorf("participant-count must be between 1 and %d", MaxParticipants) + } + + if err := validateBackendCoverage(participantCount, backendResources); err != nil { + return AggregateOutput{}, err + } + + if len(results) != participantCount { + return AggregateOutput{}, fmt.Errorf("results must contain exactly %d entries", participantCount) + } + + ordered := make([]Result, participantCount) + seen := make([]bool, participantCount) + executionFailed := false + diagnosticFailed := false + allPassed := true + + for index, result := range results { + if result.ParticipantIndex < 0 || result.ParticipantIndex >= participantCount { + return AggregateOutput{}, fmt.Errorf("results[%d].participantIndex is out of range", index) + } + + if seen[result.ParticipantIndex] { + return AggregateOutput{}, fmt.Errorf("results repeats participant index %d", result.ParticipantIndex) + } + + seen[result.ParticipantIndex] = true + + if result.ExecutionState == "" || result.Reason == "" { + return AggregateOutput{}, fmt.Errorf("results[%d] requires executionState and reason", index) + } + + switch result.Verdict { + case "Pass": + case "Fail": + diagnosticFailed = true + allPassed = false + case "Inconclusive": + allPassed = false + default: + return AggregateOutput{}, fmt.Errorf("results[%d].verdict is invalid", index) + } + + if result.ExecutionState != "Succeeded" { + executionFailed = true + } + + if len(result.Measurements) > MaxMeasurementsBytes { + return AggregateOutput{}, fmt.Errorf("results[%d].measurements exceeds %d bytes", index, MaxMeasurementsBytes) + } + + ordered[result.ParticipantIndex] = result + } + + for index, present := range seen { + if !present { + return AggregateOutput{}, fmt.Errorf("results does not cover participant index %d", index) + } + } + + backendJSON, err := json.Marshal(backendResources) + if err != nil { + return AggregateOutput{}, fmt.Errorf("encode backend-resources: %w", err) + } + + if len(backendJSON) > MaxBackendOutputBytes { + return AggregateOutput{}, fmt.Errorf("backend-resources exceeds %d bytes", MaxBackendOutputBytes) + } + + measurementsJSON, err := json.Marshal(map[string]any{ + "participantCount": participantCount, + "results": ordered, + }) + if err != nil { + return AggregateOutput{}, fmt.Errorf("encode measurements: %w", err) + } + + if len(measurementsJSON) > MaxMeasurementsBytes { + return AggregateOutput{}, fmt.Errorf("aggregate measurements exceeds %d bytes", MaxMeasurementsBytes) + } + + output := AggregateOutput{ + ExecutionState: "Succeeded", + Verdict: "Inconclusive", + Reason: "ParticipantCertificationInconclusive", + Measurements: string(measurementsJSON), + BackendResources: backendResources, + BackendJSON: string(backendJSON), + } + + switch { + case executionFailed: + output.ExecutionState = "Error" + output.Reason = "ParticipantCoverageIncomplete" + case diagnosticFailed: + output.Verdict = "Fail" + output.Reason = "ParticipantCertificationFailed" + case allPassed: + output.Verdict = "Pass" + output.Reason = "AllParticipantsPassed" + } + + return output, nil +} + +func ParseResultLog(logContent string, participantIndex int) (Result, error) { + lines := strings.Split(logContent, "\n") + for index := len(lines) - 1; index >= 0; index-- { + line := lines[index] + if !strings.HasPrefix(line, resultPrefix) { + continue + } + + fields := strings.Fields(strings.TrimPrefix(line, resultPrefix)) + if len(fields) != 3 { + return Result{}, errors.New("result record must contain verdict, reason, and measurements") + } + + values := make([]string, len(fields)) + for fieldIndex, field := range fields { + if field == "-" { + continue + } + + decoded, err := base64.StdEncoding.DecodeString(field) + if err != nil { + return Result{}, fmt.Errorf("decode result field %d: %w", fieldIndex, err) + } + + values[fieldIndex] = string(decoded) + } + + return Result{ + ParticipantIndex: participantIndex, + ExecutionState: "Succeeded", + Verdict: values[0], + Reason: values[1], + Measurements: values[2], + }, nil + } + + return Result{}, errors.New("workload log does not contain a Project Signal result record") +} + +func resultCommand(arguments ...string) []string { + quotedArguments := make([]string, len(arguments)) + for index, argument := range arguments { + quotedArguments[index] = shellQuote(argument) + } + + return []string{ + "/bin/bash", + "-c", + fmt.Sprintf(` +set -u +mkdir -p /results +chmod 0755 /results +printf '%%s' Inconclusive > /results/verdict +printf '%%s' ScriptDidNotProduceReason > /results/reason +: > /results/measurements.txt +set +e +/bin/bash "/scripts/${PROJECT_SIGNAL_SCRIPT_KEY}" %s +status=$? +set -e +if [[ $status -ne 0 ]]; then + [[ -s /results/verdict ]] || printf '%%s' Inconclusive > /results/verdict + [[ -s /results/reason ]] || printf 'ScriptExit%%d' "$status" > /results/reason + printf '\nscript_exit_code=%%d\n' "$status" >> /results/measurements.txt +fi +verdict="$(base64 < /results/verdict | tr -d '\n')" +reason="$(base64 < /results/reason | tr -d '\n')" +measurements="$(base64 < /results/measurements.txt | tr -d '\n')" +[[ -n "${verdict}" ]] || verdict=- +[[ -n "${reason}" ]] || reason=- +[[ -n "${measurements}" ]] || measurements=- +printf '%s%%s %%s %%s\n' "${verdict}" "${reason}" "${measurements}" +exit "$status" +`, strings.Join(quotedArguments, " "), resultPrefix), + } +} + +func ordinaryPod( + config Config, + participant signalv1.Participant, + tolerations []corev1.Toleration, + environment []corev1.EnvVar, + workflow metav1.OwnerReference, + activeDeadlineSeconds int64, + gpuCount int64, +) *corev1.Pod { + labels := resourceLabels(config, participant) + env := workloadEnvironment(config, participant, environment) + + pod := &corev1.Pod{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Pod"}, + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName(config.RunName, participant.Index), + Namespace: config.Namespace, + Labels: labels, + OwnerReferences: []metav1.OwnerReference{workflow}, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + NodeSelector: map[string]string{corev1.LabelHostname: participant.NodeName}, + Tolerations: tolerations, + ActiveDeadlineSeconds: &activeDeadlineSeconds, + AutomountServiceAccountToken: ptr(false), + Containers: []corev1.Container{{ + Name: "main", + Image: config.RunnerImage, + Command: resultCommand(), + Env: env, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr(config.Privileged), + }, + Resources: corev1.ResourceRequirements{ + Limits: gpuResources(gpuCount), + }, + VolumeMounts: resultVolumeMounts(), + }}, + Volumes: resultVolumes(config.ScriptConfigMap), + }, + } + + return pod +} + +func mpiJob( + config Config, + participant signalv1.Participant, + tolerations []corev1.Toleration, + environment []corev1.EnvVar, + workflow metav1.OwnerReference, + activeDeadlineSeconds int64, + gpuCount int64, +) *unstructured.Unstructured { + labels := resourceLabels(config, participant) + jobName := resourceName(config.RunName, participant.Index) + launcherEnv := workloadEnvironment(config, participant, environment) + workerEnv := workloadEnvironment(config, participant, environment) + + object := map[string]any{ + "apiVersion": "kubeflow.org/v2beta1", + "kind": "MPIJob", + "metadata": map[string]any{ + "name": jobName, + "namespace": config.Namespace, + "labels": stringMapAny(withLabel(labels, "kueue.x-k8s.io/queue-name", config.Queue)), + "ownerReferences": []any{ownerReferenceMap(workflow)}, + }, + "spec": map[string]any{ + "launcherCreationPolicy": "WaitForWorkersReady", + "mpiImplementation": "OpenMPI", + "runLauncherAsWorker": false, + "slotsPerWorker": gpuCount, + "runPolicy": map[string]any{ + "activeDeadlineSeconds": activeDeadlineSeconds, + "backoffLimit": config.BackoffLimit, + "cleanPodPolicy": "All", + "ttlSecondsAfterFinished": config.TTLSeconds, + }, + "mpiReplicaSpecs": map[string]any{ + "Launcher": map[string]any{ + "replicas": int64(1), + "template": podTemplate( + labels, + nil, + nil, + corev1.Container{ + Name: "launcher", + Image: config.RunnerImage, + Command: resultCommand("mpi-launcher", strconv.FormatInt(gpuCount, 10)), + Env: launcherEnv, + VolumeMounts: resultVolumeMounts(), + }, + config.ScriptConfigMap, + ), + }, + "Worker": map[string]any{ + "replicas": int64(1), + "template": podTemplate( + labels, + map[string]string{corev1.LabelHostname: participant.NodeName}, + tolerations, + corev1.Container{ + Name: "worker", + Image: config.RunnerImage, + Command: resultCommand("mpi-worker"), + Env: workerEnv, + Resources: corev1.ResourceRequirements{Limits: gpuResources(gpuCount)}, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr(config.Privileged), + }, + VolumeMounts: resultVolumeMounts(), + }, + config.ScriptConfigMap, + ), + }, + }, + }, + } + + return &unstructured.Unstructured{Object: object} +} + +func podTemplate( + labels map[string]string, + nodeSelector map[string]string, + tolerations []corev1.Toleration, + container corev1.Container, + scriptConfigMap string, +) map[string]any { + template := corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + NodeSelector: nodeSelector, + Tolerations: tolerations, + AutomountServiceAccountToken: ptr(false), + Containers: []corev1.Container{container}, + Volumes: resultVolumes(scriptConfigMap), + }, + } + + content, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&template) + if err != nil { + panic(err) + } + + return content +} + +func workloadEnvironment( + config Config, + participant signalv1.Participant, + environment []corev1.EnvVar, +) []corev1.EnvVar { + result := slices.Clone(environment) + + return append(result, + corev1.EnvVar{Name: "PROJECT_SIGNAL_RUN_NAME", Value: config.RunName}, + corev1.EnvVar{Name: "PROJECT_SIGNAL_RUN_UID", Value: string(config.RunUID)}, + corev1.EnvVar{Name: "PROJECT_SIGNAL_PARTICIPANT_INDEX", Value: strconv.FormatInt(int64(participant.Index), 10)}, + corev1.EnvVar{Name: "PROJECT_SIGNAL_NODE_NAME", Value: participant.NodeName}, + corev1.EnvVar{Name: "PROJECT_SIGNAL_NODE_UID", Value: string(participant.NodeUID)}, + corev1.EnvVar{Name: "PROJECT_SIGNAL_SCRIPT_KEY", Value: config.ScriptKey}, + corev1.EnvVar{Name: "PROJECT_SIGNAL_SCRIPT_DIGEST", Value: config.ScriptDigest}, + corev1.EnvVar{Name: "PROJECT_SIGNAL_ATTEMPT", Value: strconv.Itoa(config.Attempt)}, + corev1.EnvVar{Name: "PROJECT_SIGNAL_RESULTS_DIR", Value: "/results"}, + ) +} + +func resultVolumes(scriptConfigMap string) []corev1.Volume { + return []corev1.Volume{ + { + Name: "scripts", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: scriptConfigMap}, + DefaultMode: ptr(int32(0o555)), + }, + }, + }, + {Name: "results", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + } +} + +func resultVolumeMounts() []corev1.VolumeMount { + return []corev1.VolumeMount{ + {Name: "scripts", MountPath: "/scripts", ReadOnly: true}, + {Name: "results", MountPath: "/results"}, + } +} + +func gpuResources(count int64) corev1.ResourceList { + if count == 0 { + return nil + } + + return corev1.ResourceList{ + corev1.ResourceName("nvidia.com/gpu"): *resourceQuantity(count), + } +} + +func validateBackendCoverage( + participantCount int, + resources []signalv1.BackendResourceReference, +) error { + if len(resources) != participantCount { + return fmt.Errorf("backend-resources must contain exactly %d entries", participantCount) + } + + identities := make(map[string]struct{}, len(resources)) + covered := make([]bool, participantCount) + + for index, resource := range resources { + if resource.APIVersion == "" || resource.Kind == "" || resource.Name == "" || resource.UID == "" { + return fmt.Errorf("backend-resources[%d] has an incomplete identity", index) + } + + identity := strings.Join([]string{ + resource.APIVersion, + resource.Kind, + resource.Namespace, + resource.Name, + string(resource.UID), + }, "\x00") + if _, duplicate := identities[identity]; duplicate { + return fmt.Errorf("backend-resources[%d] duplicates a workload identity", index) + } + + identities[identity] = struct{}{} + + if len(resource.ParticipantIndexes) != 1 { + return fmt.Errorf("backend-resources[%d] must identify exactly one participant", index) + } + + participantIndex := resource.ParticipantIndexes[0] + if participantIndex < 0 || participantIndex >= int32(participantCount) { + return fmt.Errorf("backend-resources[%d] participant index %d is out of range", index, participantIndex) + } + + if covered[participantIndex] { + return fmt.Errorf("backend-resources[%d] repeats participant index %d", index, participantIndex) + } + + covered[participantIndex] = true + } + + for index, present := range covered { + if !present { + return fmt.Errorf("backend-resources does not cover participant index %d", index) + } + } + + return nil +} + +func decodeJSONObject[T any](content string, destination *T) error { + if strings.TrimSpace(content) == "" { + content = "[]" + if _, ok := any(destination).(*map[string]string); ok { + content = "{}" + } + } + + decoder := json.NewDecoder(strings.NewReader(content)) + decoder.DisallowUnknownFields() + + if err := decoder.Decode(destination); err != nil { + return err + } + + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + if err == nil { + return errors.New("contains trailing JSON values") + } + + return err + } + + return nil +} + +func resourceLabels(config Config, participant signalv1.Participant) map[string]string { + return map[string]string{ + "app.kubernetes.io/name": "project-signal-adapter", + "signal.unbounded-cloud.io/run": config.RunName, + "signal.unbounded-cloud.io/participant-index": strconv.FormatInt(int64(participant.Index), 10), + } +} + +func resourceName(runName string, participantIndex int32) string { + suffix := fmt.Sprintf("-p%d", participantIndex) + + maximumRunLength := 63 - len(suffix) + if len(runName) > maximumRunLength { + runName = strings.TrimRight(runName[:maximumRunLength], "-.") + } + + return runName + suffix +} + +func stringMapAny(values map[string]string) map[string]any { + result := make(map[string]any, len(values)) + for key, value := range values { + result[key] = value + } + + return result +} + +func withLabel(values map[string]string, key string, value string) map[string]string { + result := maps.Clone(values) + result[key] = value + + return result +} + +func ownerReferenceMap(reference metav1.OwnerReference) map[string]any { + return map[string]any{ + "apiVersion": reference.APIVersion, + "kind": reference.Kind, + "name": reference.Name, + "uid": string(reference.UID), + "controller": reference.Controller, + } +} + +func shellQuote(value string) string { + return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'" +} + +func ptr[T any](value T) *T { + return &value +} diff --git a/internal/signaladapter/adapter_test.go b/internal/signaladapter/adapter_test.go new file mode 100644 index 0000000000..d568e8daa4 --- /dev/null +++ b/internal/signaladapter/adapter_test.go @@ -0,0 +1,555 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signaladapter + +import ( + "encoding/base64" + "encoding/json" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" +) + +func TestRenderOrdinaryPodsPerParticipant(t *testing.T) { + t.Parallel() + + config := testConfig() + config.BackendAPIVersion = "v1" + config.BackendKind = "Pod" + config.Queue = "" + + participants, tolerations, environment, err := ValidateConfig(config) + if err != nil { + t.Fatal(err) + } + + resources, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner(), 900) + if err != nil { + t.Fatal(err) + } + + if len(resources) != 2 { + t.Fatalf("resources = %d, want 2", len(resources)) + } + + for index, resource := range resources { + if resource.GetKind() != "Pod" { + t.Fatalf("resource %d kind = %q", index, resource.GetKind()) + } + + nodeName, _, err := unstructured.NestedString(resource.Object, "spec", "nodeSelector", corev1.LabelHostname) + if err != nil { + t.Fatal(err) + } + + if nodeName != participants[index].NodeName { + t.Fatalf("resource %d node selector = %q", index, nodeName) + } + + restartPolicy, _, err := unstructured.NestedString(resource.Object, "spec", "restartPolicy") + if err != nil { + t.Fatal(err) + } + + if restartPolicy != string(corev1.RestartPolicyNever) { + t.Fatalf("resource %d restart policy = %q", index, restartPolicy) + } + + automount, _, err := unstructured.NestedBool(resource.Object, "spec", "automountServiceAccountToken") + if err != nil { + t.Fatal(err) + } + + if automount { + t.Fatalf("resource %d automounts a service account token", index) + } + } +} + +func TestRenderMPIJobsPerParticipant(t *testing.T) { + t.Parallel() + + config := testConfig() + + participants, tolerations, environment, err := ValidateConfig(config) + if err != nil { + t.Fatal(err) + } + + resources, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner(), 1200) + if err != nil { + t.Fatal(err) + } + + for index, resource := range resources { + if resource.GetAPIVersion() != "kubeflow.org/v2beta1" || resource.GetKind() != "MPIJob" { + t.Fatalf("resource %d = %s %s", index, resource.GetAPIVersion(), resource.GetKind()) + } + + queue, _, err := unstructured.NestedString( + resource.Object, + "metadata", + "labels", + "kueue.x-k8s.io/queue-name", + ) + if err != nil { + t.Fatal(err) + } + + if queue != "certification" { + t.Fatalf("resource %d queue = %q", index, queue) + } + + activeDeadline, _, err := unstructured.NestedInt64(resource.Object, "spec", "runPolicy", "activeDeadlineSeconds") + if err != nil { + t.Fatal(err) + } + + if activeDeadline != 1200 { + t.Fatalf("resource %d active deadline = %d", index, activeDeadline) + } + + cleanPolicy, _, err := unstructured.NestedString(resource.Object, "spec", "runPolicy", "cleanPodPolicy") + if err != nil { + t.Fatal(err) + } + + if cleanPolicy != "All" { + t.Fatalf("resource %d cleanPodPolicy = %q", index, cleanPolicy) + } + + for _, replica := range []string{"Launcher", "Worker"} { + restartPolicy, _, err := unstructured.NestedString( + resource.Object, + "spec", + "mpiReplicaSpecs", + replica, + "template", + "spec", + "restartPolicy", + ) + if err != nil { + t.Fatal(err) + } + + if restartPolicy != string(corev1.RestartPolicyNever) { + t.Fatalf("resource %d %s restart policy = %q", index, replica, restartPolicy) + } + } + + if _, found, err := unstructured.NestedString(resource.Object, "spec", "restartPolicy"); err != nil || found { + t.Fatalf("resource %d has top-level restartPolicy: found=%t err=%v", index, found, err) + } + + workerNode, _, err := unstructured.NestedString( + resource.Object, + "spec", + "mpiReplicaSpecs", + "Worker", + "template", + "spec", + "nodeSelector", + corev1.LabelHostname, + ) + if err != nil { + t.Fatal(err) + } + + if workerNode != participants[index].NodeName { + t.Fatalf("resource %d worker node = %q", index, workerNode) + } + + launcherContainers, found, err := unstructured.NestedSlice( + resource.Object, + "spec", + "mpiReplicaSpecs", + "Launcher", + "template", + "spec", + "containers", + ) + if err != nil || !found { + t.Fatalf("resource %d launcher containers: found=%t err=%v", index, found, err) + } + + launcherJSON, err := json.Marshal(launcherContainers) + if err != nil { + t.Fatal(err) + } + + if strings.Contains(string(launcherJSON), "nvidia.com/gpu") { + t.Fatalf("resource %d launcher requests GPUs: %s", index, launcherJSON) + } + } +} + +func TestRenderMPIJobUsesExplicitLifecycleOverrides(t *testing.T) { + t.Parallel() + + config := testConfig() + config.ScriptEnvironmentJSON = `{ + "BACKEND_BACKOFF_LIMIT":"2", + "BACKEND_TTL_SECONDS_AFTER_FINISHED":"7200" + }` + + participants, tolerations, environment, err := ValidateConfig(config) + if err != nil { + t.Fatal(err) + } + + resources, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner(), 1200) + if err != nil { + t.Fatal(err) + } + + backoff, _, err := unstructured.NestedInt64(resources[0].Object, "spec", "runPolicy", "backoffLimit") + if err != nil { + t.Fatal(err) + } + + ttl, _, err := unstructured.NestedInt64(resources[0].Object, "spec", "runPolicy", "ttlSecondsAfterFinished") + if err != nil { + t.Fatal(err) + } + + if backoff != 2 || ttl != 7200 { + t.Fatalf("run policy backoff=%d ttl=%d", backoff, ttl) + } +} + +func TestParseParticipantsRejectsMalformedIdentity(t *testing.T) { + t.Parallel() + + tests := map[string]string{ + "noncontiguous index": `[{"index":1,"nodeName":"node-a","nodeUID":"uid-a"}]`, + "missing UID": `[{"index":0,"nodeName":"node-a","nodeUID":""}]`, + "duplicate node": `[{"index":0,"nodeName":"node-a","nodeUID":"uid-a"},{"index":1,"nodeName":"node-a","nodeUID":"uid-b"}]`, + "duplicate UID": `[{"index":0,"nodeName":"node-a","nodeUID":"uid-a"},{"index":1,"nodeName":"node-b","nodeUID":"uid-a"}]`, + } + + for name, participantsJSON := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + count := 1 + if strings.Contains(participantsJSON, `"index":1`) && strings.Count(participantsJSON, `"index"`) == 2 { + count = 2 + } + + if _, err := ParseParticipants(participantsJSON, count, "", ""); err == nil { + t.Fatal("ParseParticipants succeeded") + } + }) + } +} + +func TestParseParticipantsRejectsTrailingJSON(t *testing.T) { + t.Parallel() + + participantsJSON := `[{"index":0,"nodeName":"node-a","nodeUID":"uid-a"}] []` + if _, err := ParseParticipants(participantsJSON, 1, "", ""); err == nil { + t.Fatal("ParseParticipants accepted trailing JSON") + } +} + +func TestParseParticipantsCompatibilityFallbacks(t *testing.T) { + t.Parallel() + + for name, compatibility := range map[string][2]string{ + "request Node": {"node-request", ""}, + "legacy list": {"", "node-legacy,node-ignored"}, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + participants, err := ParseParticipants("", 1, compatibility[0], compatibility[1]) + if err != nil { + t.Fatal(err) + } + + if len(participants) != 1 || participants[0].NodeName == "" { + t.Fatalf("participants = %#v", participants) + } + }) + } +} + +func TestValidateConfigRequiresDigestAndMPIQueue(t *testing.T) { + t.Parallel() + + config := testConfig() + + config.RunnerImage = "example.invalid/runner:latest" + if _, _, _, err := ValidateConfig(config); err == nil || !strings.Contains(err.Error(), "immutable sha256") { + t.Fatalf("digest validation error = %v", err) + } + + config = testConfig() + + config.Queue = "" + if _, _, _, err := ValidateConfig(config); err == nil || !strings.Contains(err.Error(), "queue is required") { + t.Fatalf("queue validation error = %v", err) + } +} + +func TestValidateConfigRejectsReservedEnvironment(t *testing.T) { + t.Parallel() + + for _, environment := range []string{ + `{"PATH":"/tmp"}`, + `{"PROJECT_SIGNAL_RUN_UID":"replacement"}`, + } { + config := testConfig() + config.ScriptEnvironmentJSON = environment + + if _, _, _, err := ValidateConfig(config); err == nil || + !strings.Contains(err.Error(), "reserved name") { + t.Fatalf("environment %s error = %v", environment, err) + } + } +} + +func TestAggregatePassRequiresCompleteResultsAndIdentities(t *testing.T) { + t.Parallel() + + resources := testBackendResources() + results := []Result{ + {ParticipantIndex: 0, ExecutionState: "Succeeded", Verdict: "Pass", Reason: "Healthy"}, + {ParticipantIndex: 1, ExecutionState: "Succeeded", Verdict: "Pass", Reason: "Healthy"}, + } + + output, err := Aggregate(2, resources, results) + if err != nil { + t.Fatal(err) + } + + if output.Verdict != "Pass" || output.Reason != "AllParticipantsPassed" { + t.Fatalf("output = %#v", output) + } + + if _, err := Aggregate(2, resources[:1], results); err == nil { + t.Fatal("Aggregate accepted incomplete backend coverage") + } + + if _, err := Aggregate(2, resources, results[:1]); err == nil { + t.Fatal("Aggregate accepted incomplete results") + } +} + +func TestAggregateBackendFailurePrecedesDiagnosticFailure(t *testing.T) { + t.Parallel() + + output, err := Aggregate(2, testBackendResources(), []Result{ + {ParticipantIndex: 0, ExecutionState: "Succeeded", Verdict: "Fail", Reason: "DiagnosticFailed"}, + {ParticipantIndex: 1, ExecutionState: "Error", Verdict: "Inconclusive", Reason: "BackendFailed"}, + }) + if err != nil { + t.Fatal(err) + } + + if output.ExecutionState != "Error" || + output.Verdict != "Inconclusive" || + output.Reason != "ParticipantCoverageIncomplete" { + t.Fatalf("output = %#v", output) + } +} + +func TestAggregateRetainsDiagnosticFailureWhenBackendsSucceed(t *testing.T) { + t.Parallel() + + output, err := Aggregate(2, testBackendResources(), []Result{ + {ParticipantIndex: 0, ExecutionState: "Succeeded", Verdict: "Fail", Reason: "DiagnosticFailed"}, + {ParticipantIndex: 1, ExecutionState: "Succeeded", Verdict: "Pass", Reason: "Healthy"}, + }) + if err != nil { + t.Fatal(err) + } + + if output.Verdict != "Fail" || output.Reason != "ParticipantCertificationFailed" { + t.Fatalf("output = %#v", output) + } +} + +func TestAggregateRejectsDuplicateIdentityAndOversizedMeasurements(t *testing.T) { + t.Parallel() + + resources := testBackendResources() + resources[1] = resources[0] + resources[1].ParticipantIndexes = []int32{1} + + results := []Result{ + {ParticipantIndex: 0, ExecutionState: "Succeeded", Verdict: "Pass", Reason: "Healthy"}, + {ParticipantIndex: 1, ExecutionState: "Succeeded", Verdict: "Pass", Reason: "Healthy"}, + } + if _, err := Aggregate(2, resources, results); err == nil { + t.Fatal("Aggregate accepted duplicate backend identity") + } + + resources = testBackendResources() + + results[1].Measurements = strings.Repeat("x", MaxMeasurementsBytes+1) + if _, err := Aggregate(2, resources, results); err == nil { + t.Fatal("Aggregate accepted oversized measurements") + } +} + +func TestParticipantBounds(t *testing.T) { + t.Parallel() + + if _, err := ParseParticipants("[]", 0, "", ""); err == nil { + t.Fatal("ParseParticipants accepted zero participants") + } + + if _, err := ParseParticipants("[]", MaxParticipants+1, "", ""); err == nil { + t.Fatal("ParseParticipants accepted too many participants") + } +} + +func TestBackendResourceRequiresMatchingParticipantLabel(t *testing.T) { + t.Parallel() + + resource := ordinaryPod( + testConfig(), + testParticipants()[0], + nil, + nil, + testWorkflowOwner(), + 900, + 0, + ) + resource.UID = types.UID("pod-uid") + + content, err := json.Marshal(resource) + if err != nil { + t.Fatal(err) + } + + var object map[string]any + if err := json.Unmarshal(content, &object); err != nil { + t.Fatal(err) + } + + unstructuredResource := &unstructured.Unstructured{Object: object} + if _, err := BackendResource(unstructuredResource, 1); err == nil { + t.Fatal("BackendResource accepted mismatched participant label") + } +} + +func TestParseResultLogUsesLastCompleteRecord(t *testing.T) { + t.Parallel() + + encode := func(value string) string { + return base64.StdEncoding.EncodeToString([]byte(value)) + } + logContent := "noise\n" + resultPrefix + + encode("Pass") + " " + + encode("Healthy") + " " + + encode("temperature=42") + "\n" + + result, err := ParseResultLog(logContent, 1) + if err != nil { + t.Fatal(err) + } + + if result.ParticipantIndex != 1 || + result.Verdict != "Pass" || + result.Measurements != "temperature=42" { + t.Fatalf("result = %#v", result) + } +} + +func TestParseResultLogAllowsEmptyMeasurements(t *testing.T) { + t.Parallel() + + encode := func(value string) string { + return base64.StdEncoding.EncodeToString([]byte(value)) + } + logContent := resultPrefix + encode("Pass") + " " + encode("Healthy") + " -\n" + + result, err := ParseResultLog(logContent, 0) + if err != nil { + t.Fatal(err) + } + + if result.Measurements != "" { + t.Fatalf("measurements = %q", result.Measurements) + } +} + +func testConfig() Config { + participantsJSON, err := json.Marshal(testParticipants()) + if err != nil { + panic(err) + } + + return Config{ + Namespace: "project-signal", + WorkflowName: "run-example-a1", + RunName: "run-example", + RunUID: types.UID("run-uid"), + ParticipantsJSON: string(participantsJSON), + ParticipantCount: 2, + RequestNodeName: "node-a", + LegacyParticipants: "node-a,node-b", + ScriptConfigMap: "run-example-scripts", + ScriptKey: "check.sh", + ScriptDigest: strings.Repeat("b", 64), + RunnerImage: "example.invalid/runner@sha256:" + strings.Repeat("a", 64), + BackendAPIVersion: "kubeflow.org/v2beta1", + BackendKind: "MPIJob", + Queue: "certification", + TolerationsJSON: `[{"key":"nvidia.com/gpu","operator":"Exists","effect":"NoSchedule"}]`, + ScriptEnvironmentJSON: `{"CHECK_MODE":"quick"}`, + GPUResourceCount: "1", + Attempt: 1, + BackoffLimit: 0, + TTLSeconds: 3600, + } +} + +func testParticipants() []signalv1.Participant { + return []signalv1.Participant{ + {Index: 0, NodeName: "node-a", NodeUID: types.UID("uid-a")}, + {Index: 1, NodeName: "node-b", NodeUID: types.UID("uid-b")}, + } +} + +func testBackendResources() []signalv1.BackendResourceReference { + return []signalv1.BackendResourceReference{ + { + APIVersion: "v1", + Kind: "Pod", + Namespace: "project-signal", + Name: "run-example-p0", + UID: types.UID("uid-0"), + ParticipantIndexes: []int32{0}, + }, + { + APIVersion: "v1", + Kind: "Pod", + Namespace: "project-signal", + Name: "run-example-p1", + UID: types.UID("uid-1"), + ParticipantIndexes: []int32{1}, + }, + } +} + +func testWorkflowOwner() metav1.OwnerReference { + controller := true + + return metav1.OwnerReference{ + APIVersion: "argoproj.io/v1alpha1", + Kind: "Workflow", + Name: "run-example-a1", + UID: types.UID("workflow-uid"), + Controller: &controller, + } +} diff --git a/internal/signaladapter/quantity.go b/internal/signaladapter/quantity.go new file mode 100644 index 0000000000..1dc68523ca --- /dev/null +++ b/internal/signaladapter/quantity.go @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signaladapter + +import ( + "strconv" + + "k8s.io/apimachinery/pkg/api/resource" +) + +func resourceQuantity(value int64) *resource.Quantity { + quantity := resource.MustParse(strconv.FormatInt(value, 10)) + + return &quantity +} diff --git a/internal/signaladapter/runner.go b/internal/signaladapter/runner.go new file mode 100644 index 0000000000..cfcd347568 --- /dev/null +++ b/internal/signaladapter/runner.go @@ -0,0 +1,465 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signaladapter + +import ( + "context" + "errors" + "fmt" + "io" + "strconv" + "strings" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" +) + +var ( + workflowGVR = schema.GroupVersionResource{Group: "argoproj.io", Version: "v1alpha1", Resource: "workflows"} + mpiJobGVR = schema.GroupVersionResource{Group: "kubeflow.org", Version: "v2beta1", Resource: "mpijobs"} + podGVR = schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"} +) + +type Runner struct { + Dynamic dynamic.Interface + Kubernetes kubernetes.Interface + PollInterval time.Duration + CleanupWait time.Duration +} + +type createdResource struct { + gvr schema.GroupVersionResource + resource *unstructured.Unstructured + participantIndex int32 +} + +func (r *Runner) Execute(ctx context.Context, config Config) (output AggregateOutput, resultErr error) { + if r.Dynamic == nil || r.Kubernetes == nil { + return AggregateOutput{}, errors.New("kubernetes clients are required") + } + + participants, tolerations, environment, err := ValidateConfig(config) + if err != nil { + return AggregateOutput{}, err + } + + workflow, err := r.Dynamic.Resource(workflowGVR).Namespace(config.Namespace). + Get(ctx, config.WorkflowName, metav1.GetOptions{}) + if err != nil { + return AggregateOutput{}, fmt.Errorf("get owning Workflow: %w", err) + } + + activeDeadlineSeconds, found, err := unstructured.NestedInt64( + workflow.Object, + "spec", + "activeDeadlineSeconds", + ) + if err != nil { + return AggregateOutput{}, fmt.Errorf("read Workflow active deadline: %w", err) + } + + if !found || activeDeadlineSeconds < 1 { + return AggregateOutput{}, errors.New("owning Workflow requires a positive activeDeadlineSeconds") + } + + executionContext, cancelExecution := context.WithTimeout( + ctx, + time.Duration(activeDeadlineSeconds)*time.Second, + ) + defer cancelExecution() + + controller := true + owner := metav1.OwnerReference{ + APIVersion: workflow.GetAPIVersion(), + Kind: workflow.GetKind(), + Name: workflow.GetName(), + UID: workflow.GetUID(), + Controller: &controller, + } + + resources, err := RenderResources( + config, + participants, + tolerations, + environment, + owner, + activeDeadlineSeconds, + ) + if err != nil { + return AggregateOutput{}, err + } + + created := make([]createdResource, 0, len(resources)) + defer func() { + if len(created) == 0 { + return + } + + cleanupContext, cancel := context.WithTimeout(context.Background(), r.cleanupWait()) + defer cancel() + + if err := r.cleanup(cleanupContext, config.Namespace, created); err != nil { + resultErr = errors.Join(resultErr, fmt.Errorf("cleanup after adapter exit: %w", err)) + } + }() + + references := make([]signalv1.BackendResourceReference, 0, len(resources)) + for index, resource := range resources { + gvr := podGVR + if resource.GetKind() == "MPIJob" { + gvr = mpiJobGVR + } + + createdObject, createErr := r.Dynamic.Resource(gvr).Namespace(config.Namespace). + Create(executionContext, resource, metav1.CreateOptions{}) + if createErr != nil { + return AggregateOutput{}, fmt.Errorf("create participant %d %s: %w", index, resource.GetKind(), createErr) + } + + created = append(created, createdResource{ + gvr: gvr, + resource: createdObject, + participantIndex: int32(index), + }) + + reference, referenceErr := BackendResource(createdObject, int32(index)) + if referenceErr != nil { + return AggregateOutput{}, fmt.Errorf("record participant %d backend identity: %w", index, referenceErr) + } + + references = append(references, reference) + } + + results := make([]Result, len(created)) + + var waitGroup sync.WaitGroup + + for index := range created { + waitGroup.Add(1) + + go func() { + defer waitGroup.Done() + + results[index] = r.waitForResult(executionContext, config, created[index]) + }() + } + + waitGroup.Wait() + + output, err = Aggregate(len(participants), references, results) + if err != nil { + return AggregateOutput{}, err + } + + cleanupContext, cancel := context.WithTimeout(context.Background(), r.cleanupWait()) + defer cancel() + + if err := r.cleanup(cleanupContext, config.Namespace, created); err != nil { + output.ExecutionState = "Error" + output.Verdict = "Inconclusive" + output.Reason = "BackendCleanupFailed" + + return output, nil + } + + created = nil + + return output, nil +} + +func (r *Runner) waitForResult(ctx context.Context, config Config, backend createdResource) Result { + result := Result{ + ParticipantIndex: int(backend.participantIndex), + ExecutionState: "Error", + Verdict: "Inconclusive", + Reason: "BackendExecutionFailed", + } + + ticker := time.NewTicker(r.pollInterval()) + defer ticker.Stop() + + var capturedResult *Result + + for { + object, err := r.Dynamic.Resource(backend.gvr).Namespace(config.Namespace). + Get(ctx, backend.resource.GetName(), metav1.GetOptions{}) + if err != nil { + result.Reason = "BackendObservationFailed: " + err.Error() + + return result + } + + terminal, succeeded, stateErr := backendState(object) + if stateErr != nil { + result.Reason = "BackendStatusInvalid: " + stateErr.Error() + + return result + } + + if backend.resource.GetKind() == "MPIJob" && capturedResult == nil { + logContent, logErr := r.resultLog(ctx, config, backend) + if logErr == nil { + parsed, parseErr := ParseResultLog(logContent, int(backend.participantIndex)) + if parseErr == nil { + capturedResult = &parsed + } + } + } + + if terminal { + if !succeeded { + return result + } + + if capturedResult != nil { + return *capturedResult + } + + logContent, logErr := r.resultLog(ctx, config, backend) + if logErr != nil { + result.Reason = "BackendResultUnavailable: " + logErr.Error() + + return result + } + + parsed, parseErr := ParseResultLog(logContent, int(backend.participantIndex)) + if parseErr != nil { + result.Reason = "BackendResultInvalid: " + parseErr.Error() + + return result + } + + return parsed + } + + select { + case <-ctx.Done(): + result.Reason = "BackendDeadlineExceeded" + + return result + case <-ticker.C: + } + } +} + +func backendState(resource *unstructured.Unstructured) (bool, bool, error) { + switch resource.GetKind() { + case "Pod": + phase, found, err := unstructured.NestedString(resource.Object, "status", "phase") + if err != nil { + return false, false, err + } + + if !found { + return false, false, nil + } + + switch corev1.PodPhase(phase) { + case corev1.PodSucceeded: + return true, true, nil + case corev1.PodFailed: + return true, false, nil + default: + return false, false, nil + } + case "MPIJob": + succeeded, _, err := unstructured.NestedInt64( + resource.Object, + "status", + "replicaStatuses", + "Launcher", + "succeeded", + ) + if err != nil { + return false, false, err + } + + if succeeded > 0 { + return true, true, nil + } + + failed, _, err := unstructured.NestedInt64( + resource.Object, + "status", + "replicaStatuses", + "Launcher", + "failed", + ) + if err != nil { + return false, false, err + } + + if failed > 0 { + return true, false, nil + } + + return false, false, nil + default: + return false, false, fmt.Errorf("unsupported backend kind %q", resource.GetKind()) + } +} + +func (r *Runner) resultLog( + ctx context.Context, + config Config, + backend createdResource, +) (string, error) { + podName := backend.resource.GetName() + containerName := "main" + + if backend.resource.GetKind() == "MPIJob" { + pods, err := r.Kubernetes.CoreV1().Pods(config.Namespace).List(ctx, metav1.ListOptions{ + LabelSelector: strings.Join([]string{ + "signal.unbounded-cloud.io/run=" + config.RunName, + "signal.unbounded-cloud.io/participant-index=" + + strconv.FormatInt(int64(backend.participantIndex), 10), + }, ","), + }) + if err != nil { + return "", fmt.Errorf("list MPIJob Pods: %w", err) + } + + podName = "" + + for index := range pods.Items { + pod := &pods.Items[index] + if !ownedBy(pod.OwnerReferences, backend.resource.GetUID()) { + continue + } + + for _, container := range pod.Spec.Containers { + if container.Name == "launcher" { + podName = pod.Name + containerName = container.Name + + break + } + } + } + + if podName == "" { + return "", errors.New("MPIJob launcher Pod was not found") + } + } + + stream, err := r.Kubernetes.CoreV1().Pods(config.Namespace). + GetLogs(podName, &corev1.PodLogOptions{Container: containerName}). + Stream(ctx) + if err != nil { + return "", fmt.Errorf("open result log: %w", err) + } + + content, err := io.ReadAll(io.LimitReader(stream, 64*1024)) + if err != nil { + closeErr := stream.Close() + if closeErr != nil { + return "", errors.Join( + fmt.Errorf("read result log: %w", err), + fmt.Errorf("close result log: %w", closeErr), + ) + } + + return "", fmt.Errorf("read result log: %w", err) + } + + if err := stream.Close(); err != nil { + return "", fmt.Errorf("close result log: %w", err) + } + + return string(content), nil +} + +func (r *Runner) cleanup( + ctx context.Context, + namespace string, + resources []createdResource, +) error { + propagation := metav1.DeletePropagationForeground + + var errs []error + + for _, resource := range resources { + err := r.Dynamic.Resource(resource.gvr).Namespace(namespace).Delete( + ctx, + resource.resource.GetName(), + metav1.DeleteOptions{ + Preconditions: &metav1.Preconditions{UID: ptr(resource.resource.GetUID())}, + PropagationPolicy: &propagation, + }, + ) + if err != nil && !apierrors.IsNotFound(err) { + errs = append(errs, fmt.Errorf("delete %s %s: %w", resource.resource.GetKind(), resource.resource.GetName(), err)) + } + } + + ticker := time.NewTicker(r.pollInterval()) + defer ticker.Stop() + + for _, resource := range resources { + for { + _, err := r.Dynamic.Resource(resource.gvr).Namespace(namespace). + Get(ctx, resource.resource.GetName(), metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + break + } + + if err != nil { + errs = append(errs, fmt.Errorf("observe deletion of %s %s: %w", resource.resource.GetKind(), resource.resource.GetName(), err)) + + break + } + + select { + case <-ctx.Done(): + errs = append(errs, fmt.Errorf( + "wait for deletion of %s %s: %w", + resource.resource.GetKind(), + resource.resource.GetName(), + ctx.Err(), + )) + + return errors.Join(errs...) + case <-ticker.C: + } + } + } + + return errors.Join(errs...) +} + +func (r *Runner) pollInterval() time.Duration { + if r.PollInterval > 0 { + return r.PollInterval + } + + return 2 * time.Second +} + +func (r *Runner) cleanupWait() time.Duration { + if r.CleanupWait > 0 { + return r.CleanupWait + } + + return 30 * time.Second +} + +func ownedBy(references []metav1.OwnerReference, uid types.UID) bool { + for _, reference := range references { + if reference.UID == uid { + return true + } + } + + return false +} diff --git a/internal/signaladapter/runner_test.go b/internal/signaladapter/runner_test.go new file mode 100644 index 0000000000..f3dd8e0f74 --- /dev/null +++ b/internal/signaladapter/runner_test.go @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signaladapter + +import ( + "context" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" +) + +func TestCleanupDeletesCreatedBackends(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + pod := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": map[string]any{ + "name": "run-p0", + "namespace": "project-signal", + "uid": "pod-uid", + }, + }} + job := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "kubeflow.org/v2beta1", + "kind": "MPIJob", + "metadata": map[string]any{ + "name": "run-p1", + "namespace": "project-signal", + "uid": "job-uid", + }, + }} + + client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds( + scheme, + map[schema.GroupVersionResource]string{ + podGVR: "PodList", + mpiJobGVR: "MPIJobList", + }, + pod, + job, + ) + runner := &Runner{Dynamic: client, PollInterval: time.Millisecond} + + err := runner.cleanup(context.Background(), "project-signal", []createdResource{ + {gvr: podGVR, resource: pod, participantIndex: 0}, + {gvr: mpiJobGVR, resource: job, participantIndex: 1}, + }) + if err != nil { + t.Fatal(err) + } + + for _, item := range []struct { + gvr schema.GroupVersionResource + name string + }{ + {gvr: podGVR, name: pod.GetName()}, + {gvr: mpiJobGVR, name: job.GetName()}, + } { + _, err := client.Resource(item.gvr).Namespace("project-signal"). + Get(context.Background(), item.name, metav1.GetOptions{}) + if err == nil { + t.Fatalf("%s still exists", item.name) + } + } +} From 6451a597a86224765157e50ec4dcee064f4553f4 Mon Sep 17 00:00:00 2001 From: chokevin Date: Tue, 20 Oct 2026 17:50:20 -0700 Subject: [PATCH 04/36] signal: fix adapter formatting --- internal/signaladapter/adapter.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/signaladapter/adapter.go b/internal/signaladapter/adapter.go index e3b5f754c0..f6b53947c7 100644 --- a/internal/signaladapter/adapter.go +++ b/internal/signaladapter/adapter.go @@ -753,7 +753,8 @@ func workloadEnvironment( ) []corev1.EnvVar { result := slices.Clone(environment) - return append(result, + return append( + result, corev1.EnvVar{Name: "PROJECT_SIGNAL_RUN_NAME", Value: config.RunName}, corev1.EnvVar{Name: "PROJECT_SIGNAL_RUN_UID", Value: string(config.RunUID)}, corev1.EnvVar{Name: "PROJECT_SIGNAL_PARTICIPANT_INDEX", Value: strconv.FormatInt(int64(participant.Index), 10)}, From dd9626613d2f340d1cdbbc6e08f672e7b2661792 Mon Sep 17 00:00:00 2001 From: chokevin Date: Tue, 20 Oct 2026 18:09:37 -0700 Subject: [PATCH 05/36] signal: satisfy pinned adapter formatting --- internal/signaladapter/adapter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/signaladapter/adapter.go b/internal/signaladapter/adapter.go index f6b53947c7..2d87582106 100644 --- a/internal/signaladapter/adapter.go +++ b/internal/signaladapter/adapter.go @@ -908,7 +908,7 @@ func stringMapAny(values map[string]string) map[string]any { return result } -func withLabel(values map[string]string, key string, value string) map[string]string { +func withLabel(values map[string]string, key, value string) map[string]string { result := maps.Clone(values) result[key] = value From 1f804d60ae9da4c4f343bf6eb5e7884ce4455c2b Mon Sep 17 00:00:00 2001 From: chokevin Date: Wed, 21 Oct 2026 14:57:26 -0700 Subject: [PATCH 06/36] signal: add typed workload execution policy --- .../v1alpha1/certificationprofile_types.go | 30 +- api/signal/v1alpha1/certificationrun_types.go | 12 +- api/signal/v1alpha1/zz_generated.deepcopy.go | 65 +++ cmd/project-signal-adapter/main.go | 1 + .../04-adapter-workflowtemplate.yaml.tmpl | 4 + deploy/signal/README.md | 62 +++ ...ounded-cloud.io_certificationprofiles.yaml | 390 +++++++++++++++++ ....unbounded-cloud.io_certificationruns.yaml | 392 ++++++++++++++++++ deploy/signal/manifest_test.go | 1 + internal/signal/controller.go | 12 + internal/signal/controller_test.go | 84 ++++ internal/signaladapter/adapter.go | 193 +++++++-- internal/signaladapter/adapter_test.go | 193 +++++++++ internal/signalworkload/policy.go | 233 +++++++++++ internal/signalworkload/policy_test.go | 150 +++++++ 15 files changed, 1771 insertions(+), 51 deletions(-) create mode 100644 internal/signalworkload/policy.go create mode 100644 internal/signalworkload/policy_test.go diff --git a/api/signal/v1alpha1/certificationprofile_types.go b/api/signal/v1alpha1/certificationprofile_types.go index 4666975ccf..bdbf4292a3 100644 --- a/api/signal/v1alpha1/certificationprofile_types.go +++ b/api/signal/v1alpha1/certificationprofile_types.go @@ -53,12 +53,32 @@ type ParticipantPolicy struct { RequiredResources corev1.ResourceList `json:"requiredResources,omitempty"` } +type WorkloadVolume struct { + Name string `json:"name"` + MountPath string `json:"mountPath"` + ReadOnly bool `json:"readOnly,omitempty"` + + HostPath *corev1.HostPathVolumeSource `json:"hostPath,omitempty"` + EmptyDir *corev1.EmptyDirVolumeSource `json:"emptyDir,omitempty"` + PersistentVolumeClaim *corev1.PersistentVolumeClaimVolumeSource `json:"persistentVolumeClaim,omitempty"` +} + +type WorkloadPolicy struct { + Resources corev1.ResourceRequirements `json:"resources,omitempty"` + SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"` + WorkingDir string `json:"workingDir,omitempty"` + Results *corev1.EmptyDirVolumeSource `json:"results,omitempty"` + // +kubebuilder:validation:MaxItems=16 + Volumes []WorkloadVolume `json:"volumes,omitempty"` +} + type CertificationProfileSpec struct { - Participants ParticipantPolicy `json:"participants"` - Script ScriptReference `json:"script"` - Workflow WorkflowReference `json:"workflow"` - Backend ExecutionBackend `json:"backend"` - RunnerImage string `json:"runnerImage"` + Participants ParticipantPolicy `json:"participants"` + Script ScriptReference `json:"script"` + Workflow WorkflowReference `json:"workflow"` + Backend ExecutionBackend `json:"backend"` + RunnerImage string `json:"runnerImage"` + WorkloadPolicy WorkloadPolicy `json:"workloadPolicy,omitempty"` // +kubebuilder:validation:Enum=FullCertification;QuickCertification;Preflight;TargetedVerification;Diagnostic Purpose CertificationPurpose `json:"purpose,omitempty"` StagePlan []CertificationStage `json:"stagePlan,omitempty"` diff --git a/api/signal/v1alpha1/certificationrun_types.go b/api/signal/v1alpha1/certificationrun_types.go index 21a4be518c..e801a725d5 100644 --- a/api/signal/v1alpha1/certificationrun_types.go +++ b/api/signal/v1alpha1/certificationrun_types.go @@ -67,6 +67,7 @@ type ScriptSnapshot struct { // +kubebuilder:validation:XValidation:rule="self.workflow == oldSelf.workflow",message="workflow is immutable" // +kubebuilder:validation:XValidation:rule="self.backend == oldSelf.backend",message="backend is immutable" // +kubebuilder:validation:XValidation:rule="self.runnerImage == oldSelf.runnerImage",message="runnerImage is immutable" +// +kubebuilder:validation:XValidation:rule="self.workloadPolicy == oldSelf.workloadPolicy",message="workloadPolicy is immutable" // +kubebuilder:validation:XValidation:rule="self.parameters == oldSelf.parameters",message="parameters are immutable" // +kubebuilder:validation:XValidation:rule="self.evidencePath == oldSelf.evidencePath",message="evidencePath is immutable" // +kubebuilder:validation:XValidation:rule="self.deadline == oldSelf.deadline",message="deadline is immutable" @@ -77,11 +78,12 @@ type CertificationRunSpec struct { Profile PinnedProfile `json:"profile"` // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=64 - Participants []Participant `json:"participants"` - Script ScriptSnapshot `json:"script"` - Workflow WorkflowReference `json:"workflow"` - Backend ExecutionBackend `json:"backend"` - RunnerImage string `json:"runnerImage"` + Participants []Participant `json:"participants"` + Script ScriptSnapshot `json:"script"` + Workflow WorkflowReference `json:"workflow"` + Backend ExecutionBackend `json:"backend"` + RunnerImage string `json:"runnerImage"` + WorkloadPolicy WorkloadPolicy `json:"workloadPolicy,omitempty"` // +kubebuilder:validation:Enum=FullCertification;QuickCertification;Preflight;TargetedVerification;Diagnostic // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="purpose is immutable" Purpose CertificationPurpose `json:"purpose,omitempty"` diff --git a/api/signal/v1alpha1/zz_generated.deepcopy.go b/api/signal/v1alpha1/zz_generated.deepcopy.go index 6775dfde0e..71d26851ad 100644 --- a/api/signal/v1alpha1/zz_generated.deepcopy.go +++ b/api/signal/v1alpha1/zz_generated.deepcopy.go @@ -99,6 +99,7 @@ func (in *CertificationProfileSpec) DeepCopyInto(out *CertificationProfileSpec) out.Script = in.Script out.Workflow = in.Workflow out.Backend = in.Backend + in.WorkloadPolicy.DeepCopyInto(&out.WorkloadPolicy) if in.StagePlan != nil { in, out := &in.StagePlan, &out.StagePlan *out = make([]CertificationStage, len(*in)) @@ -238,6 +239,7 @@ func (in *CertificationRunSpec) DeepCopyInto(out *CertificationRunSpec) { out.Script = in.Script out.Workflow = in.Workflow out.Backend = in.Backend + in.WorkloadPolicy.DeepCopyInto(&out.WorkloadPolicy) if in.StagePlan != nil { in, out := &in.StagePlan, &out.StagePlan *out = make([]CertificationStage, len(*in)) @@ -569,3 +571,66 @@ func (in *WorkflowStatus) DeepCopy() *WorkflowStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkloadPolicy) DeepCopyInto(out *WorkloadPolicy) { + *out = *in + in.Resources.DeepCopyInto(&out.Resources) + if in.SecurityContext != nil { + in, out := &in.SecurityContext, &out.SecurityContext + *out = new(v1.SecurityContext) + (*in).DeepCopyInto(*out) + } + if in.Results != nil { + in, out := &in.Results, &out.Results + *out = new(v1.EmptyDirVolumeSource) + (*in).DeepCopyInto(*out) + } + if in.Volumes != nil { + in, out := &in.Volumes, &out.Volumes + *out = make([]WorkloadVolume, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadPolicy. +func (in *WorkloadPolicy) DeepCopy() *WorkloadPolicy { + if in == nil { + return nil + } + out := new(WorkloadPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkloadVolume) DeepCopyInto(out *WorkloadVolume) { + *out = *in + if in.HostPath != nil { + in, out := &in.HostPath, &out.HostPath + *out = new(v1.HostPathVolumeSource) + (*in).DeepCopyInto(*out) + } + if in.EmptyDir != nil { + in, out := &in.EmptyDir, &out.EmptyDir + *out = new(v1.EmptyDirVolumeSource) + (*in).DeepCopyInto(*out) + } + if in.PersistentVolumeClaim != nil { + in, out := &in.PersistentVolumeClaim, &out.PersistentVolumeClaim + *out = new(v1.PersistentVolumeClaimVolumeSource) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadVolume. +func (in *WorkloadVolume) DeepCopy() *WorkloadVolume { + if in == nil { + return nil + } + out := new(WorkloadVolume) + in.DeepCopyInto(out) + return out +} diff --git a/cmd/project-signal-adapter/main.go b/cmd/project-signal-adapter/main.go index 7b977d39dc..933bc71be7 100644 --- a/cmd/project-signal-adapter/main.go +++ b/cmd/project-signal-adapter/main.go @@ -63,6 +63,7 @@ func main() { flags.StringVar(&cfg.Queue, "queue", "", "Kueue LocalQueue name") flags.StringVar(&cfg.TolerationsJSON, "tolerations", "[]", "JSON-encoded Pod tolerations") flags.StringVar(&cfg.ScriptEnvironmentJSON, "script-environment", "{}", "JSON-encoded script environment") + flags.StringVar(&cfg.WorkloadPolicyJSON, "workload-policy", "{}", "JSON-encoded participant workload policy") flags.StringVar(&cfg.GPUResourceCount, "gpu-resource-count", "0", "GPU resource limit per participant") flags.BoolVar(&cfg.Privileged, "privileged", false, "Run the participant workload with a privileged security context") flags.IntVar(&cfg.Attempt, "attempt", 0, "One-based Workflow attempt number") diff --git a/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl b/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl index 6d0eaaece9..fee086d32a 100644 --- a/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl +++ b/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl @@ -25,6 +25,8 @@ spec: value: "[]" - name: script-environment value: "{}" + - name: workload-policy + value: "{}" - name: mpi-payload-mode value: "" - name: expected-gpu-product-regex @@ -80,6 +82,8 @@ spec: - "{{`{{workflow.parameters.tolerations}}`}}" - --script-environment - "{{`{{workflow.parameters.script-environment}}`}}" + - --workload-policy + - "{{`{{workflow.parameters.workload-policy}}`}}" - --gpu-resource-count - "{{`{{workflow.parameters.gpu-resource-count}}`}}" - "--privileged={{`{{workflow.parameters.privileged}}`}}" diff --git a/deploy/signal/README.md b/deploy/signal/README.md index beb15185a9..6dc34c7f99 100644 --- a/deploy/signal/README.md +++ b/deploy/signal/README.md @@ -75,6 +75,7 @@ The referenced WorkflowTemplate must accept these string parameters: | `runner-image` | Digest-pinned execution image | | `tolerations` | JSON-encoded toleration list | | `script-environment` | JSON-encoded, validated environment map | +| `workload-policy` | JSON-encoded immutable participant-container resources, security context, and bounded volume policy | | `mpi-payload-mode` | Optional operator-defined execution mode | | `expected-gpu-product-regex` | Optional operator policy | | `expected-gpu-count` | Optional operator policy | @@ -161,6 +162,7 @@ digest-pinned runner image. To use the included adapter, set backend: apiVersion: v1 kind: Pod + queue: ``` or an explicit Kueue queue: @@ -172,6 +174,59 @@ backend: queue: ``` +When a Pod backend supplies a queue, the adapter writes the standard +`kueue.x-k8s.io/queue-name` label. MPIJob backends require a queue; Pod backends +may omit it only when the operator intentionally runs them outside Kueue. + +Profiles may set `spec.workloadPolicy` when a diagnostic needs explicit +participant-container policy. The controller validates and snapshots this +policy into the immutable CertificationRun before reserving Nodes. The included +adapter applies it to the ordinary Pod container or the MPIJob worker only; the +unpinned MPI launcher does not receive participant resources, host devices, or +extra mounts. + +```yaml +workloadPolicy: + resources: + requests: + cpu: "8" + memory: 32Gi + ephemeral-storage: 20Gi + limits: + cpu: "8" + memory: 32Gi + ephemeral-storage: 40Gi + securityContext: + privileged: false + capabilities: + add: [IPC_LOCK] + workingDir: /opt/diagnostic + results: + sizeLimit: 2Gi + volumes: + - name: work + mountPath: /work + emptyDir: + sizeLimit: 16Gi + - name: host-proc + mountPath: /host/proc + readOnly: true + hostPath: + path: /proc + type: Directory +``` + +Each extra volume must select exactly one `hostPath`, `emptyDir`, or +`persistentVolumeClaim` source. Names and mount paths must be unique; +`scripts`, `results`, `/scripts`, and `/results` are reserved. Paths must be +clean and absolute, resource quantities must be nonnegative, and limits cannot +be below requests. The policy is bounded to 16 volumes and 16 KiB of serialized +input. GPU requests and limits remain controlled by `gpu-resource-count`, which +sets equal requests and limits on the participant container. Kubernetes +admission and operator policy still decide whether a capability, host path, +device, PVC, or privileged context is allowed; technical rendering is not +authorization. + `gpu-resource-count` remains an operator-supplied profile parameter. It may be zero for Pod diagnostics and must be positive for MPIJobs. The MPIJob policy is fail-closed: `backoffLimit: 0`, `cleanPodPolicy: All`, a one-hour post-finish @@ -185,6 +240,13 @@ parameters `backend-backoff-limit` and script environment and remain explicit per-profile policy rather than cluster defaults. +A `CertificationProfile` and `CertificationRun` each select one immutable +script key and one backend. `purpose: FullCertification` and `stagePlan` +describe evidence; they do not execute a multi-key suite. Operators that define +a full suite must create one run per script and admit the next run only after +the previous run is terminal and its backend cleanup has completed. The +upstream controller does not synthesize a cross-run verdict. + ## Results and evidence Workflow infrastructure state and certification verdicts are separate: diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml index ab5b7120fe..31d309fc46 100644 --- a/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml @@ -281,6 +281,396 @@ spec: required: - name type: object + workloadPolicy: + properties: + resources: + description: ResourceRequirements describes the compute resource + requirements. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + results: + description: |- + Represents an empty directory for a pod. + Empty directory volumes support ownership management and SELinux relabeling. + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + mode: + description: |- + mode specifies the permission bits for the emptyDir directory, in numeric + notation (e.g., 0755, 01777). Must be a value between 0000 and 01777. + If not specified, defaults to 0777. + This might be in conflict with other options that affect the file + mode, like fsGroup. If fsGroup is specified, the fsGroup permissions + will override the mode specified here. + This field has no effect on Windows. + This field is alpha and requires EmptyDirVolumeMode featuregate to be enabled. + format: int32 + type: integer + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + securityContext: + description: |- + SecurityContext holds security configuration that will be applied to a container. + Some fields are present in both SecurityContext and PodSecurityContext. When both + are set, the values in SecurityContext take precedence. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + volumes: + items: + properties: + emptyDir: + description: |- + Represents an empty directory for a pod. + Empty directory volumes support ownership management and SELinux relabeling. + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + mode: + description: |- + mode specifies the permission bits for the emptyDir directory, in numeric + notation (e.g., 0755, 01777). Must be a value between 0000 and 01777. + If not specified, defaults to 0777. + This might be in conflict with other options that affect the file + mode, like fsGroup. If fsGroup is specified, the fsGroup permissions + will override the mode specified here. + This field has no effect on Windows. + This field is alpha and requires EmptyDirVolumeMode featuregate to be enabled. + format: int32 + type: integer + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + hostPath: + description: |- + Represents a host path mapped into a pod. + Host path volumes do not support ownership management or SELinux relabeling. + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + mountPath: + type: string + name: + type: string + persistentVolumeClaim: + description: |- + PersistentVolumeClaimVolumeSource references the user's PVC in the same namespace. + This volume finds the bound PV and mounts that volume for the pod. A + PersistentVolumeClaimVolumeSource is, essentially, a wrapper around another + type of volume that is owned by someone else (the system). + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + readOnly: + type: boolean + required: + - mountPath + - name + type: object + maxItems: 16 + type: array + workingDir: + type: string + type: object required: - backend - deadline diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml index a95227c6d1..af1646aa25 100644 --- a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml @@ -244,6 +244,396 @@ spec: required: - name type: object + workloadPolicy: + properties: + resources: + description: ResourceRequirements describes the compute resource + requirements. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + results: + description: |- + Represents an empty directory for a pod. + Empty directory volumes support ownership management and SELinux relabeling. + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + mode: + description: |- + mode specifies the permission bits for the emptyDir directory, in numeric + notation (e.g., 0755, 01777). Must be a value between 0000 and 01777. + If not specified, defaults to 0777. + This might be in conflict with other options that affect the file + mode, like fsGroup. If fsGroup is specified, the fsGroup permissions + will override the mode specified here. + This field has no effect on Windows. + This field is alpha and requires EmptyDirVolumeMode featuregate to be enabled. + format: int32 + type: integer + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + securityContext: + description: |- + SecurityContext holds security configuration that will be applied to a container. + Some fields are present in both SecurityContext and PodSecurityContext. When both + are set, the values in SecurityContext take precedence. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + volumes: + items: + properties: + emptyDir: + description: |- + Represents an empty directory for a pod. + Empty directory volumes support ownership management and SELinux relabeling. + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + mode: + description: |- + mode specifies the permission bits for the emptyDir directory, in numeric + notation (e.g., 0755, 01777). Must be a value between 0000 and 01777. + If not specified, defaults to 0777. + This might be in conflict with other options that affect the file + mode, like fsGroup. If fsGroup is specified, the fsGroup permissions + will override the mode specified here. + This field has no effect on Windows. + This field is alpha and requires EmptyDirVolumeMode featuregate to be enabled. + format: int32 + type: integer + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + hostPath: + description: |- + Represents a host path mapped into a pod. + Host path volumes do not support ownership management or SELinux relabeling. + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + mountPath: + type: string + name: + type: string + persistentVolumeClaim: + description: |- + PersistentVolumeClaimVolumeSource references the user's PVC in the same namespace. + This volume finds the bound PV and mounts that volume for the pod. A + PersistentVolumeClaimVolumeSource is, essentially, a wrapper around another + type of volume that is owned by someone else (the system). + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + readOnly: + type: boolean + required: + - mountPath + - name + type: object + maxItems: 16 + type: array + workingDir: + type: string + type: object required: - backend - deadline @@ -269,6 +659,8 @@ spec: rule: self.backend == oldSelf.backend - message: runnerImage is immutable rule: self.runnerImage == oldSelf.runnerImage + - message: workloadPolicy is immutable + rule: self.workloadPolicy == oldSelf.workloadPolicy - message: parameters are immutable rule: self.parameters == oldSelf.parameters - message: evidencePath is immutable diff --git a/deploy/signal/manifest_test.go b/deploy/signal/manifest_test.go index 0fb4cbb6c3..b8ba6e5d9e 100644 --- a/deploy/signal/manifest_test.go +++ b/deploy/signal/manifest_test.go @@ -65,6 +65,7 @@ func TestAdapterManifestIsGenericAndParticipantAware(t *testing.T) { "backend-api-version", "backend-kind", "queue", + "workload-policy", } { if !strings.Contains(manifest, required) { t.Fatalf("adapter WorkflowTemplate does not contain %q", required) diff --git a/internal/signal/controller.go b/internal/signal/controller.go index d91434d75a..13d7675d11 100644 --- a/internal/signal/controller.go +++ b/internal/signal/controller.go @@ -37,6 +37,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/recorder" signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" + "github.com/Azure/unbounded/internal/signalworkload" ) const ( @@ -232,6 +233,10 @@ func (r *AdmissionReconciler) admitNodeWithTriggerLocked( return fmt.Errorf("profile %s parameters: %w", profile.Name, err) } + if err := signalworkload.Validate(profile.Spec.WorkloadPolicy); err != nil { + return fmt.Errorf("profile %s workload policy: %w", profile.Name, err) + } + participants, err := r.selectParticipants(ctx, node, &profile) if err != nil { return err @@ -310,6 +315,7 @@ func (r *AdmissionReconciler) admitNodeWithTriggerLocked( Workflow: profile.Spec.Workflow, Backend: profile.Spec.Backend, RunnerImage: profile.Spec.RunnerImage, + WorkloadPolicy: *profile.Spec.WorkloadPolicy.DeepCopy(), Purpose: profile.Spec.Purpose, StagePlan: append([]signalv1.CertificationStage(nil), profile.Spec.StagePlan...), Parameters: parameters, @@ -789,6 +795,11 @@ func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured return nil, fmt.Errorf("encode script environment: %w", err) } + workloadPolicyJSON, err := signalworkload.Encode(run.Spec.WorkloadPolicy) + if err != nil { + return nil, fmt.Errorf("encode workload policy: %w", err) + } + participantsJSON, err := json.Marshal(run.Spec.Participants) if err != nil { return nil, fmt.Errorf("encode Workflow participants: %w", err) @@ -836,6 +847,7 @@ func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured map[string]any{"name": "runner-image", "value": run.Spec.RunnerImage}, map[string]any{"name": "tolerations", "value": string(tolerationsJSON)}, map[string]any{"name": "script-environment", "value": string(scriptEnvironmentJSON)}, + map[string]any{"name": "workload-policy", "value": workloadPolicyJSON}, map[string]any{"name": "mpi-payload-mode", "value": run.Spec.Parameters["mpi-payload-mode"]}, map[string]any{"name": "expected-gpu-product-regex", "value": run.Spec.Parameters["expected-gpu-product-regex"]}, map[string]any{"name": "expected-gpu-count", "value": run.Spec.Parameters["expected-gpu-count"]}, diff --git a/internal/signal/controller_test.go b/internal/signal/controller_test.go index 0e25380400..3f796d1727 100644 --- a/internal/signal/controller_test.go +++ b/internal/signal/controller_test.go @@ -34,6 +34,17 @@ func TestAdmissionCreatesImmutableSnapshotAndPreventsDuplicate(t *testing.T) { node := readyNode("gpu-0", "example-gpu") node.Labels[RequestLabel] = "gpu-smoke" profile := testProfile("gpu-smoke", "example-gpu") + profile.Spec.WorkloadPolicy = signalv1.WorkloadPolicy{ + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("4")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("8")}, + }, + Volumes: []signalv1.WorkloadVolume{{ + Name: "work", + MountPath: "/work", + EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: quantity("8Gi")}, + }}, + } const payload = "test payload\n" @@ -75,6 +86,12 @@ func TestAdmissionCreatesImmutableSnapshotAndPreventsDuplicate(t *testing.T) { t.Fatalf("purpose = %q, want %q", run.Spec.Purpose, signalv1.PurposeQuickCertification) } + if run.Spec.WorkloadPolicy.Resources.Limits.Cpu().String() != "8" || + len(run.Spec.WorkloadPolicy.Volumes) != 1 || + run.Spec.WorkloadPolicy.Volumes[0].EmptyDir.SizeLimit.String() != "8Gi" { + t.Fatalf("workload policy = %#v, want immutable profile policy", run.Spec.WorkloadPolicy) + } + if run.Spec.Parameters["observed-gpu-capacity"] != "8" || run.Spec.Parameters["observed-gpu-allocatable"] != "8" { t.Fatalf("observed GPU inventory = %#v", run.Spec.Parameters) @@ -119,6 +136,7 @@ func TestAdmissionRejectsMutableRunnerImageBeforeReservation(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, Data: map[string]string{"payload.txt": "test payload\n"}, } + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node, profile, source).Build() reconciler := &AdmissionReconciler{Client: kube, Namespace: "signal", ScriptNamespace: "signal"} @@ -140,6 +158,43 @@ func TestAdmissionRejectsMutableRunnerImageBeforeReservation(t *testing.T) { } } +func TestAdmissionRejectsInvalidWorkloadPolicyBeforeReservation(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + node := readyNode("gpu-0", "example-gpu") + node.Labels[RequestLabel] = "gpu-smoke" + profile := testProfile("gpu-smoke", "example-gpu") + profile.Spec.WorkloadPolicy.Volumes = []signalv1.WorkloadVolume{{ + Name: "replacement", + MountPath: "/results", + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }} + source := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, + Data: map[string]string{"payload.txt": "test payload\n"}, + } + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node, profile, source).Build() + reconciler := &AdmissionReconciler{Client: kube, Namespace: "signal", ScriptNamespace: "signal"} + + _, err := reconciler.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}, + ) + if err == nil || !strings.Contains(err.Error(), "reserved or existing mountPath") { + t.Fatalf("Reconcile error = %v", err) + } + + var got corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &got); err != nil { + t.Fatal(err) + } + + if got.Annotations[ReservationLabel] != "" { + t.Fatalf("node was reserved after rejected workload policy: %#v", got.Annotations) + } +} + func TestParticipantSelectionIsDeterministicAndSkipsLockedNodes(t *testing.T) { t.Parallel() @@ -552,6 +607,17 @@ func TestWorkflowPinsAttemptInputs(t *testing.T) { Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule, }} + run.Spec.WorkloadPolicy = signalv1.WorkloadPolicy{ + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("4")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("8")}, + }, + Volumes: []signalv1.WorkloadVolume{{ + Name: "work", + MountPath: "/work", + EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: quantity("8Gi")}, + }}, + } workflow, err := buildWorkflow(run, 2) if err != nil { @@ -632,6 +698,18 @@ func TestWorkflowPinsAttemptInputs(t *testing.T) { if len(tolerations) != 1 { t.Fatalf("workflow tolerations = %#v, want one deduplicated GPU toleration", tolerations) } + + var policy signalv1.WorkloadPolicy + if err := json.Unmarshal([]byte(values["workload-policy"]), &policy); err != nil { + t.Fatalf("decode workload policy: %v", err) + } + + if policy.Resources.Requests.Cpu().String() != "4" || + policy.Resources.Limits.Cpu().String() != "8" || + len(policy.Volumes) != 1 || + policy.Volumes[0].EmptyDir.SizeLimit.String() != "8Gi" { + t.Fatalf("workflow workload policy = %#v", policy) + } } func TestBuildWorkflowRejectsParameterThatCannotBecomeEnvironmentVariable(t *testing.T) { @@ -1416,3 +1494,9 @@ func workflowArgument(workflow *unstructured.Unstructured, name string) string { func ptrTime(value metav1.Time) *metav1.Time { return &value } + +func quantity(value string) *resource.Quantity { + result := resource.MustParse(value) + + return &result +} diff --git a/internal/signaladapter/adapter.go b/internal/signaladapter/adapter.go index 2d87582106..66f7c70625 100644 --- a/internal/signaladapter/adapter.go +++ b/internal/signaladapter/adapter.go @@ -23,6 +23,7 @@ import ( "k8s.io/apimachinery/pkg/util/validation" signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" + "github.com/Azure/unbounded/internal/signalworkload" ) const ( @@ -72,6 +73,7 @@ type Config struct { Queue string TolerationsJSON string ScriptEnvironmentJSON string + WorkloadPolicyJSON string GPUResourceCount string Privileged bool Attempt int @@ -231,10 +233,6 @@ func ValidateConfig(config Config) ([]signalv1.Participant, []corev1.Toleration, if config.Queue == "" { return nil, nil, nil, errors.New("queue is required for MPIJob execution") } - - if errors := validation.IsDNS1123Subdomain(config.Queue); len(errors) > 0 { - return nil, nil, nil, fmt.Errorf("queue is invalid: %s", strings.Join(errors, ", ")) - } default: return nil, nil, nil, fmt.Errorf( "unsupported backend %s %s", @@ -243,6 +241,12 @@ func ValidateConfig(config Config) ([]signalv1.Participant, []corev1.Toleration, ) } + if config.Queue != "" { + if errors := validation.IsDNS1123Subdomain(config.Queue); len(errors) > 0 { + return nil, nil, nil, fmt.Errorf("queue is invalid: %s", strings.Join(errors, ", ")) + } + } + if config.BackoffLimit < 0 { return nil, nil, nil, errors.New("backoff-limit must not be negative") } @@ -255,6 +259,10 @@ func ValidateConfig(config Config) ([]signalv1.Participant, []corev1.Toleration, return nil, nil, nil, errors.New("attempt must be positive") } + if _, err := signalworkload.Decode(config.WorkloadPolicyJSON); err != nil { + return nil, nil, nil, err + } + participants, err := ParseParticipants( config.ParticipantsJSON, config.ParticipantCount, @@ -339,6 +347,20 @@ func RenderResources( return nil, errors.New("gpu-resource-count must be positive for MPIJob execution") } + policy, err := signalworkload.Decode(config.WorkloadPolicyJSON) + if err != nil { + return nil, err + } + + gpuResource := corev1.ResourceName("nvidia.com/gpu") + if _, exists := policy.Resources.Requests[gpuResource]; exists { + return nil, errors.New("workload policy must not set nvidia.com/gpu requests; use gpu-resource-count") + } + + if _, exists := policy.Resources.Limits[gpuResource]; exists { + return nil, errors.New("workload policy must not set nvidia.com/gpu limits; use gpu-resource-count") + } + var policyEnvironment map[string]string if err := decodeJSONObject(config.ScriptEnvironmentJSON, &policyEnvironment); err != nil { return nil, fmt.Errorf("decode adapter policy: %w", err) @@ -363,9 +385,27 @@ func RenderResources( switch config.BackendKind { case "Pod": - object = ordinaryPod(config, participant, tolerations, environment, workflow, activeDeadlineSeconds, gpuCount) + object = ordinaryPod( + config, + participant, + tolerations, + environment, + policy, + workflow, + activeDeadlineSeconds, + gpuCount, + ) case "MPIJob": - object = mpiJob(config, participant, tolerations, environment, workflow, activeDeadlineSeconds, gpuCount) + object = mpiJob( + config, + participant, + tolerations, + environment, + policy, + workflow, + activeDeadlineSeconds, + gpuCount, + ) default: return nil, fmt.Errorf("unsupported backend kind %q", config.BackendKind) } @@ -600,12 +640,28 @@ func ordinaryPod( participant signalv1.Participant, tolerations []corev1.Toleration, environment []corev1.EnvVar, + policy signalv1.WorkloadPolicy, workflow metav1.OwnerReference, activeDeadlineSeconds int64, gpuCount int64, ) *corev1.Pod { labels := resourceLabels(config, participant) + if config.Queue != "" { + labels = withLabel(labels, "kueue.x-k8s.io/queue-name", config.Queue) + } + env := workloadEnvironment(config, participant, environment) + container := workloadContainer( + config, + policy, + corev1.Container{ + Name: "main", + Image: config.RunnerImage, + Command: resultCommand(), + Env: env, + }, + gpuCount, + ) pod := &corev1.Pod{ TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Pod"}, @@ -621,20 +677,8 @@ func ordinaryPod( Tolerations: tolerations, ActiveDeadlineSeconds: &activeDeadlineSeconds, AutomountServiceAccountToken: ptr(false), - Containers: []corev1.Container{{ - Name: "main", - Image: config.RunnerImage, - Command: resultCommand(), - Env: env, - SecurityContext: &corev1.SecurityContext{ - Privileged: ptr(config.Privileged), - }, - Resources: corev1.ResourceRequirements{ - Limits: gpuResources(gpuCount), - }, - VolumeMounts: resultVolumeMounts(), - }}, - Volumes: resultVolumes(config.ScriptConfigMap), + Containers: []corev1.Container{container}, + Volumes: workloadVolumes(config.ScriptConfigMap, policy), }, } @@ -646,6 +690,7 @@ func mpiJob( participant signalv1.Participant, tolerations []corev1.Toleration, environment []corev1.EnvVar, + policy signalv1.WorkloadPolicy, workflow metav1.OwnerReference, activeDeadlineSeconds int64, gpuCount int64, @@ -654,6 +699,17 @@ func mpiJob( jobName := resourceName(config.RunName, participant.Index) launcherEnv := workloadEnvironment(config, participant, environment) workerEnv := workloadEnvironment(config, participant, environment) + worker := workloadContainer( + config, + policy, + corev1.Container{ + Name: "worker", + Image: config.RunnerImage, + Command: resultCommand("mpi-worker"), + Env: workerEnv, + }, + gpuCount, + ) object := map[string]any{ "apiVersion": "kubeflow.org/v2beta1", @@ -698,18 +754,9 @@ func mpiJob( labels, map[string]string{corev1.LabelHostname: participant.NodeName}, tolerations, - corev1.Container{ - Name: "worker", - Image: config.RunnerImage, - Command: resultCommand("mpi-worker"), - Env: workerEnv, - Resources: corev1.ResourceRequirements{Limits: gpuResources(gpuCount)}, - SecurityContext: &corev1.SecurityContext{ - Privileged: ptr(config.Privileged), - }, - VolumeMounts: resultVolumeMounts(), - }, + worker, config.ScriptConfigMap, + policy, ), }, }, @@ -725,7 +772,13 @@ func podTemplate( tolerations []corev1.Toleration, container corev1.Container, scriptConfigMap string, + policies ...signalv1.WorkloadPolicy, ) map[string]any { + policy := signalv1.WorkloadPolicy{} + if len(policies) > 0 { + policy = policies[0] + } + template := corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{Labels: labels}, Spec: corev1.PodSpec{ @@ -734,7 +787,7 @@ func podTemplate( Tolerations: tolerations, AutomountServiceAccountToken: ptr(false), Containers: []corev1.Container{container}, - Volumes: resultVolumes(scriptConfigMap), + Volumes: workloadVolumes(scriptConfigMap, policy), }, } @@ -782,20 +835,78 @@ func resultVolumes(scriptConfigMap string) []corev1.Volume { } } -func resultVolumeMounts() []corev1.VolumeMount { - return []corev1.VolumeMount{ - {Name: "scripts", MountPath: "/scripts", ReadOnly: true}, - {Name: "results", MountPath: "/results"}, +func workloadContainer( + config Config, + policy signalv1.WorkloadPolicy, + container corev1.Container, + gpuCount int64, +) corev1.Container { + container.Resources = *policy.Resources.DeepCopy() + if gpuCount > 0 { + if container.Resources.Requests == nil { + container.Resources.Requests = corev1.ResourceList{} + } + + if container.Resources.Limits == nil { + container.Resources.Limits = corev1.ResourceList{} + } + + gpu := *resourceQuantity(gpuCount) + container.Resources.Requests[corev1.ResourceName("nvidia.com/gpu")] = gpu + container.Resources.Limits[corev1.ResourceName("nvidia.com/gpu")] = gpu + } + + if policy.SecurityContext != nil { + container.SecurityContext = policy.SecurityContext.DeepCopy() + } else if config.Privileged { + container.SecurityContext = &corev1.SecurityContext{Privileged: ptr(true)} } + + container.WorkingDir = policy.WorkingDir + container.VolumeMounts = append(resultVolumeMounts(), workloadVolumeMounts(policy)...) + + return container } -func gpuResources(count int64) corev1.ResourceList { - if count == 0 { - return nil +func workloadVolumes(scriptConfigMap string, policy signalv1.WorkloadPolicy) []corev1.Volume { + volumes := resultVolumes(scriptConfigMap) + if policy.Results != nil { + volumes[1].EmptyDir = policy.Results.DeepCopy() + } + + for index := range policy.Volumes { + volume := &policy.Volumes[index] + volumes = append(volumes, corev1.Volume{ + Name: volume.Name, + VolumeSource: corev1.VolumeSource{ + HostPath: volume.HostPath.DeepCopy(), + EmptyDir: volume.EmptyDir.DeepCopy(), + PersistentVolumeClaim: volume.PersistentVolumeClaim.DeepCopy(), + }, + }) } - return corev1.ResourceList{ - corev1.ResourceName("nvidia.com/gpu"): *resourceQuantity(count), + return volumes +} + +func workloadVolumeMounts(policy signalv1.WorkloadPolicy) []corev1.VolumeMount { + mounts := make([]corev1.VolumeMount, 0, len(policy.Volumes)) + for index := range policy.Volumes { + volume := &policy.Volumes[index] + mounts = append(mounts, corev1.VolumeMount{ + Name: volume.Name, + MountPath: volume.MountPath, + ReadOnly: volume.ReadOnly, + }) + } + + return mounts +} + +func resultVolumeMounts() []corev1.VolumeMount { + return []corev1.VolumeMount{ + {Name: "scripts", MountPath: "/scripts", ReadOnly: true}, + {Name: "results", MountPath: "/results"}, } } diff --git a/internal/signaladapter/adapter_test.go b/internal/signaladapter/adapter_test.go index d568e8daa4..44b0ba3e6e 100644 --- a/internal/signaladapter/adapter_test.go +++ b/internal/signaladapter/adapter_test.go @@ -10,8 +10,10 @@ import ( "testing" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" @@ -73,6 +75,29 @@ func TestRenderOrdinaryPodsPerParticipant(t *testing.T) { } } +func TestRenderOrdinaryPodWithKueueQueue(t *testing.T) { + t.Parallel() + + config := testConfig() + config.BackendAPIVersion = "v1" + config.BackendKind = "Pod" + config.Queue = "certification" + + participants, tolerations, environment, err := ValidateConfig(config) + if err != nil { + t.Fatal(err) + } + + resources, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner(), 900) + if err != nil { + t.Fatal(err) + } + + if queue := resources[0].GetLabels()["kueue.x-k8s.io/queue-name"]; queue != "certification" { + t.Fatalf("Pod queue label = %q", queue) + } +} + func TestRenderMPIJobsPerParticipant(t *testing.T) { t.Parallel() @@ -224,6 +249,167 @@ func TestRenderMPIJobUsesExplicitLifecycleOverrides(t *testing.T) { } } +func TestRenderWorkloadPolicyOnParticipantContainers(t *testing.T) { + t.Parallel() + + policy := signalv1.WorkloadPolicy{ + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("8"), + corev1.ResourceMemory: resource.MustParse("32Gi"), + corev1.ResourceEphemeralStorage: resource.MustParse("10Gi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("8"), + corev1.ResourceMemory: resource.MustParse("32Gi"), + corev1.ResourceEphemeralStorage: resource.MustParse("20Gi"), + }, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr(int64(0)), + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"IPC_LOCK"}, + }, + }, + WorkingDir: "/opt/diagnostic", + Results: &corev1.EmptyDirVolumeSource{SizeLimit: quantity("2Gi")}, + Volumes: []signalv1.WorkloadVolume{ + { + Name: "host-proc", + MountPath: "/host/proc", + ReadOnly: true, + HostPath: &corev1.HostPathVolumeSource{Path: "/proc"}, + }, + { + Name: "work", + MountPath: "/work", + EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: quantity("8Gi")}, + }, + }, + } + + policyJSON, err := json.Marshal(policy) + if err != nil { + t.Fatal(err) + } + + config := testConfig() + config.BackendAPIVersion = "v1" + config.BackendKind = "Pod" + config.Queue = "" + config.GPUResourceCount = "8" + config.WorkloadPolicyJSON = string(policyJSON) + + participants, tolerations, environment, err := ValidateConfig(config) + if err != nil { + t.Fatal(err) + } + + resources, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner(), 900) + if err != nil { + t.Fatal(err) + } + + var pod corev1.Pod + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(resources[0].Object, &pod); err != nil { + t.Fatal(err) + } + + container := pod.Spec.Containers[0] + + gpuLimit := container.Resources.Limits[corev1.ResourceName("nvidia.com/gpu")] + if container.WorkingDir != "/opt/diagnostic" || + container.Resources.Requests.Cpu().String() != "8" || + container.Resources.Limits.Memory().String() != "32Gi" || + gpuLimit.String() != "8" || + container.SecurityContext == nil || + container.SecurityContext.RunAsUser == nil || + *container.SecurityContext.RunAsUser != 0 || + len(container.SecurityContext.Capabilities.Add) != 1 { + t.Fatalf("participant container = %#v", container) + } + + if len(pod.Spec.Volumes) != 4 || + pod.Spec.Volumes[1].EmptyDir.SizeLimit.String() != "2Gi" || + pod.Spec.Volumes[2].HostPath.Path != "/proc" || + pod.Spec.Volumes[3].EmptyDir.SizeLimit.String() != "8Gi" { + t.Fatalf("participant volumes = %#v", pod.Spec.Volumes) + } +} + +func TestMPIWorkloadPolicyDoesNotLeakOntoLauncher(t *testing.T) { + t.Parallel() + + policyJSON := `{ + "resources":{"requests":{"cpu":"4"},"limits":{"cpu":"4"}}, + "volumes":[{"name":"host-device","mountPath":"/dev/infiniband","hostPath":{"path":"/dev/infiniband"}}] + }` + config := testConfig() + config.WorkloadPolicyJSON = policyJSON + + participants, tolerations, environment, err := ValidateConfig(config) + if err != nil { + t.Fatal(err) + } + + resources, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner(), 900) + if err != nil { + t.Fatal(err) + } + + content, err := json.Marshal(resources[0].Object) + if err != nil { + t.Fatal(err) + } + + var job struct { + Spec struct { + MPIReplicaSpecs map[string]struct { + Template corev1.PodTemplateSpec `json:"template"` + } `json:"mpiReplicaSpecs"` + } `json:"spec"` + } + if err := json.Unmarshal(content, &job); err != nil { + t.Fatal(err) + } + + launcher := job.Spec.MPIReplicaSpecs["Launcher"].Template.Spec + worker := job.Spec.MPIReplicaSpecs["Worker"].Template.Spec + + if len(launcher.Volumes) != 2 || len(launcher.Containers[0].VolumeMounts) != 2 { + t.Fatalf("launcher received participant policy: %#v", launcher) + } + + if len(worker.Volumes) != 3 || + worker.Volumes[2].HostPath.Path != "/dev/infiniband" || + worker.Containers[0].Resources.Requests.Cpu().String() != "4" { + t.Fatalf("worker policy = %#v", worker) + } +} + +func TestRenderRejectsGPUResourceInWorkloadPolicy(t *testing.T) { + t.Parallel() + + config := testConfig() + config.WorkloadPolicyJSON = `{"resources":{"limits":{"nvidia.com/gpu":"2"}}}` + + participants, tolerations, environment, err := ValidateConfig(config) + if err != nil { + t.Fatal(err) + } + + if _, err := RenderResources( + config, + participants, + tolerations, + environment, + testWorkflowOwner(), + 900, + ); err == nil || !strings.Contains(err.Error(), "gpu-resource-count") { + t.Fatalf("RenderResources error = %v", err) + } +} + func TestParseParticipantsRejectsMalformedIdentity(t *testing.T) { t.Parallel() @@ -420,6 +606,7 @@ func TestBackendResourceRequiresMatchingParticipantLabel(t *testing.T) { testParticipants()[0], nil, nil, + signalv1.WorkloadPolicy{}, testWorkflowOwner(), 900, 0, @@ -553,3 +740,9 @@ func testWorkflowOwner() metav1.OwnerReference { Controller: &controller, } } + +func quantity(value string) *resource.Quantity { + result := resource.MustParse(value) + + return &result +} diff --git a/internal/signalworkload/policy.go b/internal/signalworkload/policy.go new file mode 100644 index 0000000000..70e83893fe --- /dev/null +++ b/internal/signalworkload/policy.go @@ -0,0 +1,233 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signalworkload + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "path" + "regexp" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/validation" + + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" +) + +const MaxPolicyBytes = 16 * 1024 + +var capabilityPattern = regexp.MustCompile(`^[A-Z][A-Z0-9_]*$`) + +func Encode(policy signalv1.WorkloadPolicy) (string, error) { + if err := Validate(policy); err != nil { + return "", err + } + + content, err := json.Marshal(policy) + if err != nil { + return "", fmt.Errorf("encode workload policy: %w", err) + } + + if len(content) > MaxPolicyBytes { + return "", fmt.Errorf("workload policy exceeds %d bytes", MaxPolicyBytes) + } + + return string(content), nil +} + +func Decode(content string) (signalv1.WorkloadPolicy, error) { + var policy signalv1.WorkloadPolicy + + if strings.TrimSpace(content) == "" { + content = "{}" + } + + decoder := json.NewDecoder(strings.NewReader(content)) + decoder.DisallowUnknownFields() + + if err := decoder.Decode(&policy); err != nil { + return policy, fmt.Errorf("decode workload policy: %w", err) + } + + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + if err == nil { + return policy, errors.New("workload policy contains trailing JSON values") + } + + return policy, fmt.Errorf("decode trailing workload policy value: %w", err) + } + + if err := Validate(policy); err != nil { + return policy, err + } + + return policy, nil +} + +func Validate(policy signalv1.WorkloadPolicy) error { + if err := validateResources(policy.Resources); err != nil { + return err + } + + if err := validateSecurityContext(policy.SecurityContext); err != nil { + return err + } + + if policy.WorkingDir != "" { + if err := validateAbsolutePath("workingDir", policy.WorkingDir); err != nil { + return err + } + } + + if len(policy.Volumes) > 16 { + return errors.New("workload policy must not contain more than 16 volumes") + } + + names := map[string]struct{}{"scripts": {}, "results": {}} + mountPaths := map[string]struct{}{"/scripts": {}, "/results": {}} + + for index := range policy.Volumes { + volume := &policy.Volumes[index] + if errors := validation.IsDNS1123Label(volume.Name); len(errors) > 0 { + return fmt.Errorf("workload policy volume %d name is invalid: %s", index, strings.Join(errors, ", ")) + } + + if _, exists := names[volume.Name]; exists { + return fmt.Errorf("workload policy volume %d duplicates reserved or existing name %q", index, volume.Name) + } + + names[volume.Name] = struct{}{} + + if err := validateAbsolutePath(fmt.Sprintf("workload policy volume %d mountPath", index), volume.MountPath); err != nil { + return err + } + + if _, exists := mountPaths[volume.MountPath]; exists { + return fmt.Errorf( + "workload policy volume %d duplicates reserved or existing mountPath %q", + index, + volume.MountPath, + ) + } + + mountPaths[volume.MountPath] = struct{}{} + + sources := 0 + if volume.HostPath != nil { + sources++ + + if err := validateAbsolutePath(fmt.Sprintf("workload policy volume %d hostPath", index), volume.HostPath.Path); err != nil { + return err + } + } + + if volume.EmptyDir != nil { + sources++ + + if err := validateQuantity(fmt.Sprintf("workload policy volume %d emptyDir sizeLimit", index), volume.EmptyDir.SizeLimit); err != nil { + return err + } + } + + if volume.PersistentVolumeClaim != nil { + sources++ + + if errors := validation.IsDNS1123Subdomain(volume.PersistentVolumeClaim.ClaimName); len(errors) > 0 { + return fmt.Errorf( + "workload policy volume %d persistentVolumeClaim claimName is invalid: %s", + index, + strings.Join(errors, ", "), + ) + } + } + + if sources != 1 { + return fmt.Errorf("workload policy volume %d must define exactly one source", index) + } + } + + return validateQuantity("workload policy results sizeLimit", sizeLimit(policy.Results)) +} + +func validateResources(resources corev1.ResourceRequirements) error { + for name, quantity := range resources.Requests { + if quantity.Sign() < 0 { + return fmt.Errorf("workload policy resource request %q must not be negative", name) + } + } + + for name, quantity := range resources.Limits { + if quantity.Sign() < 0 { + return fmt.Errorf("workload policy resource limit %q must not be negative", name) + } + + request, exists := resources.Requests[name] + if exists && quantity.Cmp(request) < 0 { + return fmt.Errorf("workload policy resource limit %q must not be less than its request", name) + } + } + + return nil +} + +func validateSecurityContext(context *corev1.SecurityContext) error { + if context == nil || context.Capabilities == nil { + return nil + } + + seen := map[corev1.Capability]string{} + + for listName, capabilities := range map[string][]corev1.Capability{ + "add": context.Capabilities.Add, + "drop": context.Capabilities.Drop, + } { + for _, capability := range capabilities { + if !capabilityPattern.MatchString(string(capability)) { + return fmt.Errorf("workload policy capability %q is invalid", capability) + } + + if previous, exists := seen[capability]; exists { + return fmt.Errorf( + "workload policy capability %q appears in both or repeatedly within %s and %s", + capability, + previous, + listName, + ) + } + + seen[capability] = listName + } + } + + return nil +} + +func validateAbsolutePath(name, value string) error { + if value == "" || !strings.HasPrefix(value, "/") || path.Clean(value) != value { + return fmt.Errorf("%s must be a clean absolute path", name) + } + + return nil +} + +func validateQuantity(name string, quantity *resource.Quantity) error { + if quantity != nil && quantity.Sign() < 0 { + return fmt.Errorf("%s must not be negative", name) + } + + return nil +} + +func sizeLimit(source *corev1.EmptyDirVolumeSource) *resource.Quantity { + if source == nil { + return nil + } + + return source.SizeLimit +} diff --git a/internal/signalworkload/policy_test.go b/internal/signalworkload/policy_test.go new file mode 100644 index 0000000000..48bf60648e --- /dev/null +++ b/internal/signalworkload/policy_test.go @@ -0,0 +1,150 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signalworkload + +import ( + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" +) + +func TestPolicyRoundTripPreservesBoundedExecutionSettings(t *testing.T) { + t.Parallel() + + policy := signalv1.WorkloadPolicy{ + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("8"), + corev1.ResourceMemory: resource.MustParse("32Gi"), + corev1.ResourceEphemeralStorage: resource.MustParse("20Gi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("8"), + corev1.ResourceMemory: resource.MustParse("32Gi"), + corev1.ResourceEphemeralStorage: resource.MustParse("40Gi"), + }, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr(false), + RunAsUser: ptr(int64(0)), + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"IPC_LOCK"}, + Drop: []corev1.Capability{"NET_RAW"}, + }, + }, + WorkingDir: "/opt/diagnostic", + Results: &corev1.EmptyDirVolumeSource{SizeLimit: quantity("2Gi")}, + Volumes: []signalv1.WorkloadVolume{ + { + Name: "host-proc", + MountPath: "/host/proc", + ReadOnly: true, + HostPath: &corev1.HostPathVolumeSource{Path: "/proc"}, + }, + { + Name: "work", + MountPath: "/work", + EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: quantity("8Gi")}, + }, + { + Name: "scratch", + MountPath: "/scratch", + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "diagnostic-scratch"}, + }, + }, + } + + content, err := Encode(policy) + if err != nil { + t.Fatal(err) + } + + got, err := Decode(content) + if err != nil { + t.Fatal(err) + } + + if got.WorkingDir != policy.WorkingDir || + got.Resources.Limits.Cpu().Cmp(*policy.Resources.Limits.Cpu()) != 0 || + len(got.Volumes) != 3 || + got.Volumes[0].HostPath.Path != "/proc" || + got.Volumes[1].EmptyDir.SizeLimit.Cmp(*quantity("8Gi")) != 0 || + got.Volumes[2].PersistentVolumeClaim.ClaimName != "diagnostic-scratch" { + t.Fatalf("decoded policy = %#v", got) + } +} + +func TestValidateRejectsUnsafeOrAmbiguousPolicy(t *testing.T) { + t.Parallel() + + tests := map[string]signalv1.WorkloadPolicy{ + "reserved mount": { + Volumes: []signalv1.WorkloadVolume{{ + Name: "replacement", + MountPath: "/results", + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }}, + }, + "multiple sources": { + Volumes: []signalv1.WorkloadVolume{{ + Name: "scratch", + MountPath: "/scratch", + HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/scratch"}, + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }}, + }, + "relative host path": { + Volumes: []signalv1.WorkloadVolume{{ + Name: "scratch", + MountPath: "/scratch", + HostPath: &corev1.HostPathVolumeSource{Path: "scratch"}, + }}, + }, + "limit below request": { + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("8")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("4")}, + }, + }, + "capability overlap": { + SecurityContext: &corev1.SecurityContext{Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"SYS_ADMIN"}, + Drop: []corev1.Capability{"SYS_ADMIN"}, + }}, + }, + } + + for name, policy := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + if err := Validate(policy); err == nil { + t.Fatal("Validate accepted invalid policy") + } + }) + } +} + +func TestDecodeRejectsOversizedPolicy(t *testing.T) { + t.Parallel() + + policy := signalv1.WorkloadPolicy{WorkingDir: "/" + strings.Repeat("a", MaxPolicyBytes)} + if _, err := Encode(policy); err == nil { + t.Fatal("Encode accepted oversized policy") + } +} + +func quantity(value string) *resource.Quantity { + result := resource.MustParse(value) + + return &result +} + +func ptr[T any](value T) *T { + return &value +} From 45c74250275b53d18f2f24b5cdea6f8cfda3b158 Mon Sep 17 00:00:00 2001 From: chokevin Date: Wed, 21 Oct 2026 16:09:42 -0700 Subject: [PATCH 07/36] signal: support participant host networking --- .../v1alpha1/certificationprofile_types.go | 1 + deploy/signal/README.md | 8 ++++++ ...ounded-cloud.io_certificationprofiles.yaml | 2 ++ ....unbounded-cloud.io_certificationruns.yaml | 2 ++ internal/signal/controller_test.go | 8 ++++-- internal/signaladapter/adapter.go | 11 ++++++++ internal/signaladapter/adapter_test.go | 28 +++++++++++++++---- internal/signalworkload/policy_test.go | 23 +++++++++++++-- 8 files changed, 74 insertions(+), 9 deletions(-) diff --git a/api/signal/v1alpha1/certificationprofile_types.go b/api/signal/v1alpha1/certificationprofile_types.go index bdbf4292a3..378496f58c 100644 --- a/api/signal/v1alpha1/certificationprofile_types.go +++ b/api/signal/v1alpha1/certificationprofile_types.go @@ -66,6 +66,7 @@ type WorkloadVolume struct { type WorkloadPolicy struct { Resources corev1.ResourceRequirements `json:"resources,omitempty"` SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"` + HostNetwork bool `json:"hostNetwork,omitempty"` WorkingDir string `json:"workingDir,omitempty"` Results *corev1.EmptyDirVolumeSource `json:"results,omitempty"` // +kubebuilder:validation:MaxItems=16 diff --git a/deploy/signal/README.md b/deploy/signal/README.md index 6dc34c7f99..6c9512574a 100644 --- a/deploy/signal/README.md +++ b/deploy/signal/README.md @@ -200,6 +200,7 @@ workloadPolicy: privileged: false capabilities: add: [IPC_LOCK] + hostNetwork: true workingDir: /opt/diagnostic results: sizeLimit: 2Gi @@ -227,6 +228,13 @@ admission and operator policy still decide whether a capability, host path, device, PVC, or privileged context is allowed; technical rendering is not authorization. +`hostNetwork: true` is an explicit participant policy for diagnostics that must +observe the selected Node's network namespace, such as physical NIC inventory. +The adapter sets `dnsPolicy: ClusterFirstWithHostNet` with it. For MPIJobs this +applies only to the Node-pinned worker; the unpinned launcher remains outside +the host network namespace. Omitting the field keeps Kubernetes' default Pod +networking. + `gpu-resource-count` remains an operator-supplied profile parameter. It may be zero for Pod diagnostics and must be positive for MPIJobs. The MPIJob policy is fail-closed: `backoffLimit: 0`, `cleanPodPolicy: All`, a one-hour post-finish diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml index 31d309fc46..26906b78e7 100644 --- a/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml @@ -283,6 +283,8 @@ spec: type: object workloadPolicy: properties: + hostNetwork: + type: boolean resources: description: ResourceRequirements describes the compute resource requirements. diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml index af1646aa25..17aeb1e41e 100644 --- a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml @@ -246,6 +246,8 @@ spec: type: object workloadPolicy: properties: + hostNetwork: + type: boolean resources: description: ResourceRequirements describes the compute resource requirements. diff --git a/internal/signal/controller_test.go b/internal/signal/controller_test.go index 3f796d1727..739911710b 100644 --- a/internal/signal/controller_test.go +++ b/internal/signal/controller_test.go @@ -39,6 +39,7 @@ func TestAdmissionCreatesImmutableSnapshotAndPreventsDuplicate(t *testing.T) { Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("4")}, Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("8")}, }, + HostNetwork: true, Volumes: []signalv1.WorkloadVolume{{ Name: "work", MountPath: "/work", @@ -86,7 +87,8 @@ func TestAdmissionCreatesImmutableSnapshotAndPreventsDuplicate(t *testing.T) { t.Fatalf("purpose = %q, want %q", run.Spec.Purpose, signalv1.PurposeQuickCertification) } - if run.Spec.WorkloadPolicy.Resources.Limits.Cpu().String() != "8" || + if !run.Spec.WorkloadPolicy.HostNetwork || + run.Spec.WorkloadPolicy.Resources.Limits.Cpu().String() != "8" || len(run.Spec.WorkloadPolicy.Volumes) != 1 || run.Spec.WorkloadPolicy.Volumes[0].EmptyDir.SizeLimit.String() != "8Gi" { t.Fatalf("workload policy = %#v, want immutable profile policy", run.Spec.WorkloadPolicy) @@ -612,6 +614,7 @@ func TestWorkflowPinsAttemptInputs(t *testing.T) { Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("4")}, Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("8")}, }, + HostNetwork: true, Volumes: []signalv1.WorkloadVolume{{ Name: "work", MountPath: "/work", @@ -704,7 +707,8 @@ func TestWorkflowPinsAttemptInputs(t *testing.T) { t.Fatalf("decode workload policy: %v", err) } - if policy.Resources.Requests.Cpu().String() != "4" || + if !policy.HostNetwork || + policy.Resources.Requests.Cpu().String() != "4" || policy.Resources.Limits.Cpu().String() != "8" || len(policy.Volumes) != 1 || policy.Volumes[0].EmptyDir.SizeLimit.String() != "8Gi" { diff --git a/internal/signaladapter/adapter.go b/internal/signaladapter/adapter.go index 66f7c70625..03488df17d 100644 --- a/internal/signaladapter/adapter.go +++ b/internal/signaladapter/adapter.go @@ -681,6 +681,7 @@ func ordinaryPod( Volumes: workloadVolumes(config.ScriptConfigMap, policy), }, } + applyPodPolicy(&pod.Spec, policy) return pod } @@ -790,6 +791,7 @@ func podTemplate( Volumes: workloadVolumes(scriptConfigMap, policy), }, } + applyPodPolicy(&template.Spec, policy) content, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&template) if err != nil { @@ -799,6 +801,15 @@ func podTemplate( return content } +func applyPodPolicy(spec *corev1.PodSpec, policy signalv1.WorkloadPolicy) { + if !policy.HostNetwork { + return + } + + spec.HostNetwork = true + spec.DNSPolicy = corev1.DNSClusterFirstWithHostNet +} + func workloadEnvironment( config Config, participant signalv1.Participant, diff --git a/internal/signaladapter/adapter_test.go b/internal/signaladapter/adapter_test.go index 44b0ba3e6e..c83c16d03a 100644 --- a/internal/signaladapter/adapter_test.go +++ b/internal/signaladapter/adapter_test.go @@ -72,6 +72,15 @@ func TestRenderOrdinaryPodsPerParticipant(t *testing.T) { if automount { t.Fatalf("resource %d automounts a service account token", index) } + + hostNetwork, _, err := unstructured.NestedBool(resource.Object, "spec", "hostNetwork") + if err != nil { + t.Fatal(err) + } + + if hostNetwork { + t.Fatalf("resource %d unexpectedly uses the host network", index) + } } } @@ -271,8 +280,9 @@ func TestRenderWorkloadPolicyOnParticipantContainers(t *testing.T) { Add: []corev1.Capability{"IPC_LOCK"}, }, }, - WorkingDir: "/opt/diagnostic", - Results: &corev1.EmptyDirVolumeSource{SizeLimit: quantity("2Gi")}, + HostNetwork: true, + WorkingDir: "/opt/diagnostic", + Results: &corev1.EmptyDirVolumeSource{SizeLimit: quantity("2Gi")}, Volumes: []signalv1.WorkloadVolume{ { Name: "host-proc", @@ -318,7 +328,9 @@ func TestRenderWorkloadPolicyOnParticipantContainers(t *testing.T) { container := pod.Spec.Containers[0] gpuLimit := container.Resources.Limits[corev1.ResourceName("nvidia.com/gpu")] - if container.WorkingDir != "/opt/diagnostic" || + if !pod.Spec.HostNetwork || + pod.Spec.DNSPolicy != corev1.DNSClusterFirstWithHostNet || + container.WorkingDir != "/opt/diagnostic" || container.Resources.Requests.Cpu().String() != "8" || container.Resources.Limits.Memory().String() != "32Gi" || gpuLimit.String() != "8" || @@ -341,6 +353,7 @@ func TestMPIWorkloadPolicyDoesNotLeakOntoLauncher(t *testing.T) { t.Parallel() policyJSON := `{ + "hostNetwork":true, "resources":{"requests":{"cpu":"4"},"limits":{"cpu":"4"}}, "volumes":[{"name":"host-device","mountPath":"/dev/infiniband","hostPath":{"path":"/dev/infiniband"}}] }` @@ -376,11 +389,16 @@ func TestMPIWorkloadPolicyDoesNotLeakOntoLauncher(t *testing.T) { launcher := job.Spec.MPIReplicaSpecs["Launcher"].Template.Spec worker := job.Spec.MPIReplicaSpecs["Worker"].Template.Spec - if len(launcher.Volumes) != 2 || len(launcher.Containers[0].VolumeMounts) != 2 { + if launcher.HostNetwork || + launcher.DNSPolicy == corev1.DNSClusterFirstWithHostNet || + len(launcher.Volumes) != 2 || + len(launcher.Containers[0].VolumeMounts) != 2 { t.Fatalf("launcher received participant policy: %#v", launcher) } - if len(worker.Volumes) != 3 || + if !worker.HostNetwork || + worker.DNSPolicy != corev1.DNSClusterFirstWithHostNet || + len(worker.Volumes) != 3 || worker.Volumes[2].HostPath.Path != "/dev/infiniband" || worker.Containers[0].Resources.Requests.Cpu().String() != "4" { t.Fatalf("worker policy = %#v", worker) diff --git a/internal/signalworkload/policy_test.go b/internal/signalworkload/policy_test.go index 48bf60648e..231801adf5 100644 --- a/internal/signalworkload/policy_test.go +++ b/internal/signalworkload/policy_test.go @@ -37,8 +37,9 @@ func TestPolicyRoundTripPreservesBoundedExecutionSettings(t *testing.T) { Drop: []corev1.Capability{"NET_RAW"}, }, }, - WorkingDir: "/opt/diagnostic", - Results: &corev1.EmptyDirVolumeSource{SizeLimit: quantity("2Gi")}, + WorkingDir: "/opt/diagnostic", + HostNetwork: true, + Results: &corev1.EmptyDirVolumeSource{SizeLimit: quantity("2Gi")}, Volumes: []signalv1.WorkloadVolume{ { Name: "host-proc", @@ -70,6 +71,7 @@ func TestPolicyRoundTripPreservesBoundedExecutionSettings(t *testing.T) { } if got.WorkingDir != policy.WorkingDir || + !got.HostNetwork || got.Resources.Limits.Cpu().Cmp(*policy.Resources.Limits.Cpu()) != 0 || len(got.Volumes) != 3 || got.Volumes[0].HostPath.Path != "/proc" || @@ -139,6 +141,23 @@ func TestDecodeRejectsOversizedPolicy(t *testing.T) { } } +func TestDecodeRejectsMalformedHostNetworkPolicy(t *testing.T) { + t.Parallel() + + for name, content := range map[string]string{ + "wrong type": `{"hostNetwork":"true"}`, + "unknown field": `{"hostNetworking":true}`, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + if _, err := Decode(content); err == nil { + t.Fatal("Decode accepted malformed host network policy") + } + }) + } +} + func quantity(value string) *resource.Quantity { result := resource.MustParse(value) From 3406e82258971c3b4049ab70a606b50a1772049a Mon Sep 17 00:00:00 2001 From: chokevin Date: Wed, 21 Oct 2026 18:23:53 -0700 Subject: [PATCH 08/36] fix(signal): align script digest contract --- api/signal/v1alpha1/certificationrun_types.go | 8 +++- api/signal/v1alpha1/digest.go | 16 ++++++++ api/signal/v1alpha1/digest_test.go | 36 +++++++++++++++++ cmd/project-signal-adapter/main.go | 7 +++- deploy/signal/README.md | 2 +- ....unbounded-cloud.io_certificationruns.yaml | 6 +++ deploy/signal/manifest_test.go | 14 +++++++ internal/signal/controller.go | 6 ++- internal/signal/evidence.go | 5 +-- internal/signal/evidence_test.go | 5 +-- internal/signal/projection_test.go | 8 ++-- internal/signaladapter/adapter.go | 11 +++-- internal/signaladapter/adapter_test.go | 40 ++++++++++++++++++- 13 files changed, 141 insertions(+), 23 deletions(-) create mode 100644 api/signal/v1alpha1/digest.go create mode 100644 api/signal/v1alpha1/digest_test.go diff --git a/api/signal/v1alpha1/certificationrun_types.go b/api/signal/v1alpha1/certificationrun_types.go index e801a725d5..6e95888088 100644 --- a/api/signal/v1alpha1/certificationrun_types.go +++ b/api/signal/v1alpha1/certificationrun_types.go @@ -58,7 +58,9 @@ type PinnedProfile struct { type ScriptSnapshot struct { ConfigMapName string `json:"configMapName"` Key string `json:"key"` - Digest string `json:"digest"` + // Digest is the algorithm-qualified SHA-256 digest of the selected script bytes. + // +kubebuilder:validation:Pattern=`^sha256:[0-9a-f]{64}$` + Digest string `json:"digest"` } // +kubebuilder:validation:XValidation:rule="self.profile == oldSelf.profile",message="profile is immutable" @@ -109,7 +111,9 @@ type WorkflowStatus struct { } type EvidenceStatus struct { - URI string `json:"uri,omitempty"` + URI string `json:"uri,omitempty"` + // Digest is the algorithm-qualified SHA-256 digest of the evidence bytes. + // +kubebuilder:validation:Pattern=`^sha256:[0-9a-f]{64}$` Digest string `json:"digest,omitempty"` } diff --git a/api/signal/v1alpha1/digest.go b/api/signal/v1alpha1/digest.go new file mode 100644 index 0000000000..981dc0cba4 --- /dev/null +++ b/api/signal/v1alpha1/digest.go @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import "regexp" + +// SHA256DigestPattern is the canonical Project Signal digest representation. +const SHA256DigestPattern = `^sha256:[0-9a-f]{64}$` + +var sha256DigestPattern = regexp.MustCompile(SHA256DigestPattern) + +// IsSHA256Digest reports whether value is an algorithm-qualified SHA-256 digest. +func IsSHA256Digest(value string) bool { + return sha256DigestPattern.MatchString(value) +} diff --git a/api/signal/v1alpha1/digest_test.go b/api/signal/v1alpha1/digest_test.go new file mode 100644 index 0000000000..e0e1559325 --- /dev/null +++ b/api/signal/v1alpha1/digest_test.go @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + "strings" + "testing" +) + +func TestIsSHA256Digest(t *testing.T) { + t.Parallel() + + valid := "sha256:" + strings.Repeat("3c", 32) + if !IsSHA256Digest(valid) { + t.Fatalf("IsSHA256Digest(%q) = false", valid) + } + + for name, value := range map[string]string{ + "bare hex": strings.Repeat("3c", 32), + "uppercase hex": "sha256:" + strings.Repeat("3C", 32), + "wrong algorithm": "sha512:" + strings.Repeat("3c", 32), + "short": "sha256:" + strings.Repeat("3c", 31), + "long": "sha256:" + strings.Repeat("3c", 33), + "leading space": " " + valid, + "trailing newline": valid + "\n", + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + if IsSHA256Digest(value) { + t.Fatalf("IsSHA256Digest(%q) = true", value) + } + }) + } +} diff --git a/cmd/project-signal-adapter/main.go b/cmd/project-signal-adapter/main.go index 933bc71be7..5950df9bb7 100644 --- a/cmd/project-signal-adapter/main.go +++ b/cmd/project-signal-adapter/main.go @@ -56,7 +56,12 @@ func main() { flags.StringVar(&cfg.LegacyParticipants, "participants", "", "Compatibility comma-separated Node names") flags.StringVar(&cfg.ScriptConfigMap, "script-configmap", "", "Immutable script ConfigMap") flags.StringVar(&cfg.ScriptKey, "script-key", "", "Script ConfigMap key") - flags.StringVar(&cfg.ScriptDigest, "script-digest", "", "SHA-256 digest of the script bytes") + flags.StringVar( + &cfg.ScriptDigest, + "script-digest", + "", + "algorithm-qualified SHA-256 digest of the script bytes (sha256:<64 lowercase hex>)", + ) flags.StringVar(&cfg.RunnerImage, "runner-image", "", "Digest-pinned runner image") flags.StringVar(&cfg.BackendAPIVersion, "backend-api-version", "", "Backend API version") flags.StringVar(&cfg.BackendKind, "backend-kind", "", "Backend kind") diff --git a/deploy/signal/README.md b/deploy/signal/README.md index 6c9512574a..42e70a7a4a 100644 --- a/deploy/signal/README.md +++ b/deploy/signal/README.md @@ -68,7 +68,7 @@ The referenced WorkflowTemplate must accept these string parameters: | `request-node-name` | Node that triggered the grouped run | | `script-configmap` | Immutable run-owned ConfigMap | | `script-key` | Selected ConfigMap key | -| `script-digest` | SHA-256 of the selected script bytes | +| `script-digest` | Algorithm-qualified SHA-256 of the selected script bytes (`sha256:<64 lowercase hex>`) | | `backend-api-version` | Backend API version from the profile | | `backend-kind` | Backend kind from the profile | | `queue` | Optional backend queue | diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml index 17aeb1e41e..625996d350 100644 --- a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml @@ -163,6 +163,9 @@ spec: configMapName: type: string digest: + description: Digest is the algorithm-qualified SHA-256 digest + of the selected script bytes. + pattern: ^sha256:[0-9a-f]{64}$ type: string key: type: string @@ -743,6 +746,9 @@ spec: evidence: properties: digest: + description: Digest is the algorithm-qualified SHA-256 digest + of the evidence bytes. + pattern: ^sha256:[0-9a-f]{64}$ type: string uri: type: string diff --git a/deploy/signal/manifest_test.go b/deploy/signal/manifest_test.go index b8ba6e5d9e..b58730d959 100644 --- a/deploy/signal/manifest_test.go +++ b/deploy/signal/manifest_test.go @@ -83,6 +83,20 @@ func TestAdapterManifestIsGenericAndParticipantAware(t *testing.T) { } } +func TestCertificationRunCRDRequiresCanonicalDigests(t *testing.T) { + t.Parallel() + + content, err := os.ReadFile("crd/signal.unbounded-cloud.io_certificationruns.yaml") + if err != nil { + t.Fatal(err) + } + + const digestPattern = "pattern: ^sha256:[0-9a-f]{64}$" + if count := strings.Count(string(content), digestPattern); count != 2 { + t.Fatalf("canonical digest patterns = %d, want 2", count) + } +} + func TestUpstreamDoesNotPublishCertificationPackage(t *testing.T) { t.Parallel() diff --git a/internal/signal/controller.go b/internal/signal/controller.go index 13d7675d11..7b9bb7be60 100644 --- a/internal/signal/controller.go +++ b/internal/signal/controller.go @@ -1499,7 +1499,11 @@ func workflowMessage(workflow *unstructured.Unstructured) string { } func digestString(content string) string { - sum := sha256.Sum256([]byte(content)) + return digestBytes([]byte(content)) +} + +func digestBytes(content []byte) string { + sum := sha256.Sum256(content) return "sha256:" + hex.EncodeToString(sum[:]) } diff --git a/internal/signal/evidence.go b/internal/signal/evidence.go index dec8e50152..6a6524002b 100644 --- a/internal/signal/evidence.go +++ b/internal/signal/evidence.go @@ -7,8 +7,6 @@ import ( "bytes" "compress/gzip" "context" - "crypto/sha256" - "encoding/hex" "fmt" "net/url" "strings" @@ -46,8 +44,7 @@ func (s *KubernetesEvidenceStore) Put(ctx context.Context, path string, data []b return "", "", fmt.Errorf("evidence path is required") } - sum := sha256.Sum256(data) - digest := "sha256:" + hex.EncodeToString(sum[:]) + digest := digestBytes(data) compressed, err := compressEvidence(data) if err != nil { diff --git a/internal/signal/evidence_test.go b/internal/signal/evidence_test.go index b24f269dea..8c7915f986 100644 --- a/internal/signal/evidence_test.go +++ b/internal/signal/evidence_test.go @@ -8,7 +8,6 @@ import ( "compress/gzip" "context" "io" - "strings" "testing" corev1 "k8s.io/api/core/v1" @@ -41,8 +40,8 @@ func TestKubernetesEvidenceStoreReturnsQueryableRunURIAndDigest(t *testing.T) { t.Fatalf("unexpected URI %q", uri) } - if !strings.HasPrefix(digest, "sha256:") || len(digest) != len("sha256:")+64 { - t.Fatalf("unexpected digest %q", digest) + if digest != digestBytes(data) { + t.Fatalf("digest = %q, want %q", digest, digestBytes(data)) } var evidence corev1.ConfigMap diff --git a/internal/signal/projection_test.go b/internal/signal/projection_test.go index 6af9abcb6b..928b5de346 100644 --- a/internal/signal/projection_test.go +++ b/internal/signal/projection_test.go @@ -41,7 +41,7 @@ func TestNodeProjectionTransitionsAndRetainsTerminalSummary(t *testing.T) { run.Status.ExecutionState = "Completed" run.Status.Verdict = "Pass" run.Status.Reason = "Healthy" - run.Status.Evidence.Digest = "sha256:evidence" + run.Status.Evidence.Digest = "sha256:" + strings.Repeat("e", 64) run.Status.FinishedAt = ptrTime(metav1.NewTime(time.Date(2026, 10, 11, 5, 0, 0, 0, time.UTC))) if err := projectTerminal(context.Background(), kube, run); err != nil { @@ -102,7 +102,7 @@ func TestNodeProjectionDoesNotRegressAfterRestartOrOlderRetry(t *testing.T) { newer := testRun("newer") newer.Status.Phase = signalv1.RunPhaseSucceeded newer.Status.FinishedAt = ptrTime(metav1.NewTime(time.Date(2026, 10, 11, 6, 0, 0, 0, time.UTC))) - newer.Status.Evidence.Digest = "sha256:newer" + newer.Status.Evidence.Digest = "sha256:" + strings.Repeat("a", 64) if err := projectTerminal(context.Background(), kube, newer); err != nil { t.Fatal(err) @@ -111,7 +111,7 @@ func TestNodeProjectionDoesNotRegressAfterRestartOrOlderRetry(t *testing.T) { older := testRun("older") older.Status.Phase = signalv1.RunPhaseFailed older.Status.FinishedAt = ptrTime(metav1.NewTime(time.Date(2026, 10, 11, 5, 0, 0, 0, time.UTC))) - older.Status.Evidence.Digest = "sha256:older" + older.Status.Evidence.Digest = "sha256:" + strings.Repeat("b", 64) if err := projectTerminal(context.Background(), kube, older); err != nil { t.Fatal(err) @@ -124,7 +124,7 @@ func TestNodeProjectionDoesNotRegressAfterRestartOrOlderRetry(t *testing.T) { if got.Labels[StateLabel] != "passed" || got.Annotations[LastRunAnnotation] != newer.Name || - got.Annotations[EvidenceDigestAnnotation] != "sha256:newer" { + got.Annotations[EvidenceDigestAnnotation] != "sha256:"+strings.Repeat("a", 64) { t.Fatalf("projection regressed: labels=%v annotations=%v", got.Labels, got.Annotations) } } diff --git a/internal/signaladapter/adapter.go b/internal/signaladapter/adapter.go index 03488df17d..ec7d7c77e0 100644 --- a/internal/signaladapter/adapter.go +++ b/internal/signaladapter/adapter.go @@ -33,10 +33,7 @@ const ( resultPrefix = "PROJECT_SIGNAL_RESULT_V1 " ) -var ( - runnerImagePattern = regexp.MustCompile(`^[^@\s]+@sha256:[0-9a-f]{64}$`) - scriptDigestPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) -) +var runnerImagePattern = regexp.MustCompile(`^[^@\s]+@sha256:[0-9a-f]{64}$`) var reservedEnvironmentNames = map[string]struct{}{ "BASH_ENV": {}, @@ -223,8 +220,10 @@ func ValidateConfig(config Config) ([]signalv1.Participant, []corev1.Toleration, return nil, nil, nil, errors.New("runner-image must be an immutable sha256 digest reference") } - if !scriptDigestPattern.MatchString(config.ScriptDigest) { - return nil, nil, nil, errors.New("script-digest must contain 64 lowercase hexadecimal characters") + if !signalv1.IsSHA256Digest(config.ScriptDigest) { + return nil, nil, nil, errors.New( + "script-digest must be an algorithm-qualified SHA-256 digest (sha256:<64 lowercase hexadecimal characters>)", + ) } switch { diff --git a/internal/signaladapter/adapter_test.go b/internal/signaladapter/adapter_test.go index c83c16d03a..f71c50fdb2 100644 --- a/internal/signaladapter/adapter_test.go +++ b/internal/signaladapter/adapter_test.go @@ -503,6 +503,44 @@ func TestValidateConfigRequiresDigestAndMPIQueue(t *testing.T) { } } +func TestValidateConfigAcceptsControllerScriptDigest(t *testing.T) { + t.Parallel() + + config := testConfig() + config.ScriptDigest = "sha256:3c11c8ff7b1182a1215a3ee4a0512901116c45646669b08a46710795120601a8" + + if _, _, _, err := ValidateConfig(config); err != nil { + t.Fatalf("ValidateConfig rejected controller script digest: %v", err) + } +} + +func TestValidateConfigRejectsMalformedScriptDigest(t *testing.T) { + t.Parallel() + + valid := "sha256:" + strings.Repeat("b", 64) + for name, digest := range map[string]string{ + "bare hex": strings.Repeat("b", 64), + "uppercase hex": "sha256:" + strings.Repeat("B", 64), + "wrong algorithm": "sha512:" + strings.Repeat("b", 64), + "short": "sha256:" + strings.Repeat("b", 63), + "long": "sha256:" + strings.Repeat("b", 65), + "leading space": " " + valid, + "trailing newline": valid + "\n", + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + config := testConfig() + config.ScriptDigest = digest + + if _, _, _, err := ValidateConfig(config); err == nil || + !strings.Contains(err.Error(), "algorithm-qualified SHA-256 digest") { + t.Fatalf("ValidateConfig script digest error = %v", err) + } + }) + } +} + func TestValidateConfigRejectsReservedEnvironment(t *testing.T) { t.Parallel() @@ -705,7 +743,7 @@ func testConfig() Config { LegacyParticipants: "node-a,node-b", ScriptConfigMap: "run-example-scripts", ScriptKey: "check.sh", - ScriptDigest: strings.Repeat("b", 64), + ScriptDigest: "sha256:" + strings.Repeat("b", 64), RunnerImage: "example.invalid/runner@sha256:" + strings.Repeat("a", 64), BackendAPIVersion: "kubeflow.org/v2beta1", BackendKind: "MPIJob", From 5b117beffd65e6d45c15892f163bb64096b194e9 Mon Sep 17 00:00:00 2001 From: chokevin Date: Wed, 21 Oct 2026 20:22:07 -0700 Subject: [PATCH 09/36] fix(signal): request Kueue topology --- .../v1alpha1/certificationprofile_types.go | 10 ++++ api/signal/v1alpha1/zz_generated.deepcopy.go | 20 +++++++ deploy/signal/README.md | 10 ++++ ...ounded-cloud.io_certificationprofiles.yaml | 13 ++++ ....unbounded-cloud.io_certificationruns.yaml | 13 ++++ deploy/signal/manifest_test.go | 26 ++++++++ internal/signal/controller.go | 5 ++ internal/signal/controller_test.go | 59 ++++++++++++++++++- internal/signaladapter/adapter.go | 22 ++++++- internal/signaladapter/adapter_test.go | 38 ++++++++++-- internal/signalworkload/policy.go | 9 +++ internal/signalworkload/policy_test.go | 31 +++++++++- 12 files changed, 247 insertions(+), 9 deletions(-) diff --git a/api/signal/v1alpha1/certificationprofile_types.go b/api/signal/v1alpha1/certificationprofile_types.go index 378496f58c..fa2aadf827 100644 --- a/api/signal/v1alpha1/certificationprofile_types.go +++ b/api/signal/v1alpha1/certificationprofile_types.go @@ -63,10 +63,20 @@ type WorkloadVolume struct { PersistentVolumeClaim *corev1.PersistentVolumeClaimVolumeSource `json:"persistentVolumeClaim,omitempty"` } +type KueueTopologyPolicy struct { + // RequiredLevel is the Node label key for the minimum topology domain in which + // Kueue must place the participant PodSet. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=316 + // +kubebuilder:validation:Pattern=`^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$` + RequiredLevel string `json:"requiredLevel"` +} + type WorkloadPolicy struct { Resources corev1.ResourceRequirements `json:"resources,omitempty"` SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"` HostNetwork bool `json:"hostNetwork,omitempty"` + KueueTopology *KueueTopologyPolicy `json:"kueueTopology,omitempty"` WorkingDir string `json:"workingDir,omitempty"` Results *corev1.EmptyDirVolumeSource `json:"results,omitempty"` // +kubebuilder:validation:MaxItems=16 diff --git a/api/signal/v1alpha1/zz_generated.deepcopy.go b/api/signal/v1alpha1/zz_generated.deepcopy.go index 71d26851ad..e612e94a5e 100644 --- a/api/signal/v1alpha1/zz_generated.deepcopy.go +++ b/api/signal/v1alpha1/zz_generated.deepcopy.go @@ -366,6 +366,21 @@ func (in *ExecutionBackend) DeepCopy() *ExecutionBackend { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KueueTopologyPolicy) DeepCopyInto(out *KueueTopologyPolicy) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KueueTopologyPolicy. +func (in *KueueTopologyPolicy) DeepCopy() *KueueTopologyPolicy { + if in == nil { + return nil + } + out := new(KueueTopologyPolicy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Participant) DeepCopyInto(out *Participant) { *out = *in @@ -581,6 +596,11 @@ func (in *WorkloadPolicy) DeepCopyInto(out *WorkloadPolicy) { *out = new(v1.SecurityContext) (*in).DeepCopyInto(*out) } + if in.KueueTopology != nil { + in, out := &in.KueueTopology, &out.KueueTopology + *out = new(KueueTopologyPolicy) + **out = **in + } if in.Results != nil { in, out := &in.Results, &out.Results *out = new(v1.EmptyDirVolumeSource) diff --git a/deploy/signal/README.md b/deploy/signal/README.md index 42e70a7a4a..f9d7f61b8f 100644 --- a/deploy/signal/README.md +++ b/deploy/signal/README.md @@ -187,6 +187,8 @@ extra mounts. ```yaml workloadPolicy: + kueueTopology: + requiredLevel: kubernetes.io/hostname resources: requests: cpu: "8" @@ -228,6 +230,14 @@ admission and operator policy still decide whether a capability, host path, device, PVC, or privileged context is allowed; technical rendering is not authorization. +`kueueTopology.requiredLevel` is an explicit Kueue Topology Aware Scheduling +request. It must be a valid Kubernetes label key and requires a nonempty backend +queue. The adapter writes +`kueue.x-k8s.io/podset-required-topology: ` on ordinary +participant Pods and MPIJob worker Pod templates only. It never annotates the +unpinned MPI launcher, does not accept arbitrary annotations, and does not +choose a topology level when the field is omitted. + `hostNetwork: true` is an explicit participant policy for diagnostics that must observe the selected Node's network namespace, such as physical NIC inventory. The adapter sets `dnsPolicy: ClusterFirstWithHostNet` with it. For MPIJobs this diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml index 26906b78e7..490d511384 100644 --- a/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml @@ -285,6 +285,19 @@ spec: properties: hostNetwork: type: boolean + kueueTopology: + properties: + requiredLevel: + description: |- + RequiredLevel is the Node label key for the minimum topology domain in which + Kueue must place the participant PodSet. + maxLength: 316 + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - requiredLevel + type: object resources: description: ResourceRequirements describes the compute resource requirements. diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml index 625996d350..0347c30c6e 100644 --- a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml @@ -251,6 +251,19 @@ spec: properties: hostNetwork: type: boolean + kueueTopology: + properties: + requiredLevel: + description: |- + RequiredLevel is the Node label key for the minimum topology domain in which + Kueue must place the participant PodSet. + maxLength: 316 + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - requiredLevel + type: object resources: description: ResourceRequirements describes the compute resource requirements. diff --git a/deploy/signal/manifest_test.go b/deploy/signal/manifest_test.go index b58730d959..9420c8fdaa 100644 --- a/deploy/signal/manifest_test.go +++ b/deploy/signal/manifest_test.go @@ -97,6 +97,32 @@ func TestCertificationRunCRDRequiresCanonicalDigests(t *testing.T) { } } +func TestSignalCRDsExposeBoundedKueueTopologyPolicy(t *testing.T) { + t.Parallel() + + for _, path := range []string{ + "crd/signal.unbounded-cloud.io_certificationprofiles.yaml", + "crd/signal.unbounded-cloud.io_certificationruns.yaml", + } { + content, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + + manifest := string(content) + for _, required := range []string{ + "kueueTopology:", + "requiredLevel:", + "maxLength: 316", + "minLength: 1", + } { + if !strings.Contains(manifest, required) { + t.Fatalf("%s does not contain %q", path, required) + } + } + } +} + func TestUpstreamDoesNotPublishCertificationPackage(t *testing.T) { t.Parallel() diff --git a/internal/signal/controller.go b/internal/signal/controller.go index 7b9bb7be60..6d5c0b228a 100644 --- a/internal/signal/controller.go +++ b/internal/signal/controller.go @@ -237,6 +237,10 @@ func (r *AdmissionReconciler) admitNodeWithTriggerLocked( return fmt.Errorf("profile %s workload policy: %w", profile.Name, err) } + if profile.Spec.WorkloadPolicy.KueueTopology != nil && profile.Spec.Backend.Queue == "" { + return fmt.Errorf("profile %s workload policy kueueTopology requires a backend queue", profile.Name) + } + participants, err := r.selectParticipants(ctx, node, &profile) if err != nil { return err @@ -1075,6 +1079,7 @@ func (r *RunReconciler) evidenceDocument( "workflowOutputs": run.Status.Workflow.Outputs, "backend": run.Spec.Backend, "runnerImage": run.Spec.RunnerImage, + "workloadPolicy": run.Spec.WorkloadPolicy, "purpose": run.Spec.Purpose, "stagePlan": run.Spec.StagePlan, "parameters": run.Spec.Parameters, diff --git a/internal/signal/controller_test.go b/internal/signal/controller_test.go index 739911710b..212018c754 100644 --- a/internal/signal/controller_test.go +++ b/internal/signal/controller_test.go @@ -8,6 +8,7 @@ import ( "encoding/json" "errors" "fmt" + "reflect" "strings" "testing" "time" @@ -34,12 +35,16 @@ func TestAdmissionCreatesImmutableSnapshotAndPreventsDuplicate(t *testing.T) { node := readyNode("gpu-0", "example-gpu") node.Labels[RequestLabel] = "gpu-smoke" profile := testProfile("gpu-smoke", "example-gpu") + profile.Spec.Backend.Queue = "certification" profile.Spec.WorkloadPolicy = signalv1.WorkloadPolicy{ Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("4")}, Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("8")}, }, HostNetwork: true, + KueueTopology: &signalv1.KueueTopologyPolicy{ + RequiredLevel: corev1.LabelHostname, + }, Volumes: []signalv1.WorkloadVolume{{ Name: "work", MountPath: "/work", @@ -88,6 +93,8 @@ func TestAdmissionCreatesImmutableSnapshotAndPreventsDuplicate(t *testing.T) { } if !run.Spec.WorkloadPolicy.HostNetwork || + run.Spec.WorkloadPolicy.KueueTopology == nil || + run.Spec.WorkloadPolicy.KueueTopology.RequiredLevel != corev1.LabelHostname || run.Spec.WorkloadPolicy.Resources.Limits.Cpu().String() != "8" || len(run.Spec.WorkloadPolicy.Volumes) != 1 || run.Spec.WorkloadPolicy.Volumes[0].EmptyDir.SizeLimit.String() != "8Gi" { @@ -197,6 +204,42 @@ func TestAdmissionRejectsInvalidWorkloadPolicyBeforeReservation(t *testing.T) { } } +func TestAdmissionRejectsKueueTopologyWithoutQueueBeforeReservation(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + node := readyNode("gpu-0", "example-gpu") + node.Labels[RequestLabel] = "gpu-smoke" + profile := testProfile("gpu-smoke", "example-gpu") + profile.Spec.Backend = signalv1.ExecutionBackend{APIVersion: "v1", Kind: "Pod"} + profile.Spec.WorkloadPolicy.KueueTopology = &signalv1.KueueTopologyPolicy{ + RequiredLevel: corev1.LabelHostname, + } + source := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, + Data: map[string]string{"payload.txt": "test payload\n"}, + } + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node, profile, source).Build() + reconciler := &AdmissionReconciler{Client: kube, Namespace: "signal", ScriptNamespace: "signal"} + + _, err := reconciler.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}, + ) + if err == nil || !strings.Contains(err.Error(), "kueueTopology requires a backend queue") { + t.Fatalf("Reconcile error = %v", err) + } + + var got corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &got); err != nil { + t.Fatal(err) + } + + if got.Annotations[ReservationLabel] != "" { + t.Fatalf("node was reserved after rejected topology policy: %#v", got.Annotations) + } +} + func TestParticipantSelectionIsDeterministicAndSkipsLockedNodes(t *testing.T) { t.Parallel() @@ -615,6 +658,9 @@ func TestWorkflowPinsAttemptInputs(t *testing.T) { Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("8")}, }, HostNetwork: true, + KueueTopology: &signalv1.KueueTopologyPolicy{ + RequiredLevel: corev1.LabelHostname, + }, Volumes: []signalv1.WorkloadVolume{{ Name: "work", MountPath: "/work", @@ -708,6 +754,8 @@ func TestWorkflowPinsAttemptInputs(t *testing.T) { } if !policy.HostNetwork || + policy.KueueTopology == nil || + policy.KueueTopology.RequiredLevel != corev1.LabelHostname || policy.Resources.Requests.Cpu().String() != "4" || policy.Resources.Limits.Cpu().String() != "8" || len(policy.Volumes) != 1 || @@ -1000,6 +1048,10 @@ func TestTerminalVerdictPersistsEvidenceAndCleanupReleasesNode(t *testing.T) { scheme := testScheme(t) run := testRun("run-pass") + run.Spec.Backend.Queue = "certification" + run.Spec.WorkloadPolicy.KueueTopology = &signalv1.KueueTopologyPolicy{ + RequiredLevel: corev1.LabelHostname, + } run.Status = signalv1.CertificationRunStatus{ Phase: signalv1.RunPhaseRunning, Workflow: signalv1.WorkflowStatus{Name: "run-pass-a1", Attempt: 1}, @@ -1049,7 +1101,8 @@ func TestTerminalVerdictPersistsEvidenceAndCleanupReleasesNode(t *testing.T) { } var evidence struct { - Workflow signalv1.WorkflowStatus `json:"workflow"` + Workflow signalv1.WorkflowStatus `json:"workflow"` + WorkloadPolicy signalv1.WorkloadPolicy `json:"workloadPolicy"` } if err := json.Unmarshal(store.data, &evidence); err != nil { t.Fatalf("decode evidence: %v", err) @@ -1060,6 +1113,10 @@ func TestTerminalVerdictPersistsEvidenceAndCleanupReleasesNode(t *testing.T) { t.Fatalf("evidence workflow = %#v", evidence.Workflow) } + if !reflect.DeepEqual(evidence.WorkloadPolicy, run.Spec.WorkloadPolicy) { + t.Fatalf("evidence workload policy = %#v, want %#v", evidence.WorkloadPolicy, run.Spec.WorkloadPolicy) + } + if !got.Status.Cleanup.ReservationsReleased || !got.Status.Cleanup.SnapshotDeleted { t.Fatalf("cleanup status = %#v", got.Status.Cleanup) } diff --git a/internal/signaladapter/adapter.go b/internal/signaladapter/adapter.go index ec7d7c77e0..dd6eb92715 100644 --- a/internal/signaladapter/adapter.go +++ b/internal/signaladapter/adapter.go @@ -31,6 +31,7 @@ const ( MaxMeasurementsBytes = 16 * 1024 MaxBackendOutputBytes = 16 * 1024 resultPrefix = "PROJECT_SIGNAL_RESULT_V1 " + requiredTopologyKey = "kueue.x-k8s.io/podset-required-topology" ) var runnerImagePattern = regexp.MustCompile(`^[^@\s]+@sha256:[0-9a-f]{64}$`) @@ -258,10 +259,15 @@ func ValidateConfig(config Config) ([]signalv1.Participant, []corev1.Toleration, return nil, nil, nil, errors.New("attempt must be positive") } - if _, err := signalworkload.Decode(config.WorkloadPolicyJSON); err != nil { + policy, err := signalworkload.Decode(config.WorkloadPolicyJSON) + if err != nil { return nil, nil, nil, err } + if policy.KueueTopology != nil && config.Queue == "" { + return nil, nil, nil, errors.New("workload policy kueueTopology requires a queue") + } + participants, err := ParseParticipants( config.ParticipantsJSON, config.ParticipantCount, @@ -668,6 +674,7 @@ func ordinaryPod( Name: resourceName(config.RunName, participant.Index), Namespace: config.Namespace, Labels: labels, + Annotations: workloadAnnotations(policy), OwnerReferences: []metav1.OwnerReference{workflow}, }, Spec: corev1.PodSpec{ @@ -780,7 +787,10 @@ func podTemplate( } template := corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{Labels: labels}, + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + Annotations: workloadAnnotations(policy), + }, Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyNever, NodeSelector: nodeSelector, @@ -800,6 +810,14 @@ func podTemplate( return content } +func workloadAnnotations(policy signalv1.WorkloadPolicy) map[string]string { + if policy.KueueTopology == nil { + return nil + } + + return map[string]string{requiredTopologyKey: policy.KueueTopology.RequiredLevel} +} + func applyPodPolicy(spec *corev1.PodSpec, policy signalv1.WorkloadPolicy) { if !policy.HostNetwork { return diff --git a/internal/signaladapter/adapter_test.go b/internal/signaladapter/adapter_test.go index f71c50fdb2..dbf18e7c1a 100644 --- a/internal/signaladapter/adapter_test.go +++ b/internal/signaladapter/adapter_test.go @@ -81,6 +81,10 @@ func TestRenderOrdinaryPodsPerParticipant(t *testing.T) { if hostNetwork { t.Fatalf("resource %d unexpectedly uses the host network", index) } + + if resource.GetAnnotations()[requiredTopologyKey] != "" { + t.Fatalf("resource %d unexpectedly requests Kueue topology", index) + } } } @@ -281,8 +285,11 @@ func TestRenderWorkloadPolicyOnParticipantContainers(t *testing.T) { }, }, HostNetwork: true, - WorkingDir: "/opt/diagnostic", - Results: &corev1.EmptyDirVolumeSource{SizeLimit: quantity("2Gi")}, + KueueTopology: &signalv1.KueueTopologyPolicy{ + RequiredLevel: corev1.LabelHostname, + }, + WorkingDir: "/opt/diagnostic", + Results: &corev1.EmptyDirVolumeSource{SizeLimit: quantity("2Gi")}, Volumes: []signalv1.WorkloadVolume{ { Name: "host-proc", @@ -306,7 +313,7 @@ func TestRenderWorkloadPolicyOnParticipantContainers(t *testing.T) { config := testConfig() config.BackendAPIVersion = "v1" config.BackendKind = "Pod" - config.Queue = "" + config.Queue = "certification" config.GPUResourceCount = "8" config.WorkloadPolicyJSON = string(policyJSON) @@ -330,6 +337,7 @@ func TestRenderWorkloadPolicyOnParticipantContainers(t *testing.T) { gpuLimit := container.Resources.Limits[corev1.ResourceName("nvidia.com/gpu")] if !pod.Spec.HostNetwork || pod.Spec.DNSPolicy != corev1.DNSClusterFirstWithHostNet || + pod.Annotations[requiredTopologyKey] != corev1.LabelHostname || container.WorkingDir != "/opt/diagnostic" || container.Resources.Requests.Cpu().String() != "8" || container.Resources.Limits.Memory().String() != "32Gi" || @@ -354,6 +362,7 @@ func TestMPIWorkloadPolicyDoesNotLeakOntoLauncher(t *testing.T) { policyJSON := `{ "hostNetwork":true, + "kueueTopology":{"requiredLevel":"kubernetes.io/hostname"}, "resources":{"requests":{"cpu":"4"},"limits":{"cpu":"4"}}, "volumes":[{"name":"host-device","mountPath":"/dev/infiniband","hostPath":{"path":"/dev/infiniband"}}] }` @@ -386,11 +395,14 @@ func TestMPIWorkloadPolicyDoesNotLeakOntoLauncher(t *testing.T) { t.Fatal(err) } - launcher := job.Spec.MPIReplicaSpecs["Launcher"].Template.Spec - worker := job.Spec.MPIReplicaSpecs["Worker"].Template.Spec + launcherTemplate := job.Spec.MPIReplicaSpecs["Launcher"].Template + workerTemplate := job.Spec.MPIReplicaSpecs["Worker"].Template + launcher := launcherTemplate.Spec + worker := workerTemplate.Spec if launcher.HostNetwork || launcher.DNSPolicy == corev1.DNSClusterFirstWithHostNet || + launcherTemplate.Annotations[requiredTopologyKey] != "" || len(launcher.Volumes) != 2 || len(launcher.Containers[0].VolumeMounts) != 2 { t.Fatalf("launcher received participant policy: %#v", launcher) @@ -398,6 +410,7 @@ func TestMPIWorkloadPolicyDoesNotLeakOntoLauncher(t *testing.T) { if !worker.HostNetwork || worker.DNSPolicy != corev1.DNSClusterFirstWithHostNet || + workerTemplate.Annotations[requiredTopologyKey] != corev1.LabelHostname || len(worker.Volumes) != 3 || worker.Volumes[2].HostPath.Path != "/dev/infiniband" || worker.Containers[0].Resources.Requests.Cpu().String() != "4" { @@ -405,6 +418,21 @@ func TestMPIWorkloadPolicyDoesNotLeakOntoLauncher(t *testing.T) { } } +func TestValidateConfigRejectsKueueTopologyWithoutQueue(t *testing.T) { + t.Parallel() + + config := testConfig() + config.BackendAPIVersion = "v1" + config.BackendKind = "Pod" + config.Queue = "" + config.WorkloadPolicyJSON = `{"kueueTopology":{"requiredLevel":"kubernetes.io/hostname"}}` + + if _, _, _, err := ValidateConfig(config); err == nil || + !strings.Contains(err.Error(), "kueueTopology requires a queue") { + t.Fatalf("ValidateConfig topology error = %v", err) + } +} + func TestRenderRejectsGPUResourceInWorkloadPolicy(t *testing.T) { t.Parallel() diff --git a/internal/signalworkload/policy.go b/internal/signalworkload/policy.go index 70e83893fe..41adfa9d9e 100644 --- a/internal/signalworkload/policy.go +++ b/internal/signalworkload/policy.go @@ -79,6 +79,15 @@ func Validate(policy signalv1.WorkloadPolicy) error { return err } + if policy.KueueTopology != nil { + if errors := validation.IsQualifiedName(policy.KueueTopology.RequiredLevel); len(errors) > 0 { + return fmt.Errorf( + "workload policy kueueTopology requiredLevel is invalid: %s", + strings.Join(errors, ", "), + ) + } + } + if policy.WorkingDir != "" { if err := validateAbsolutePath("workingDir", policy.WorkingDir); err != nil { return err diff --git a/internal/signalworkload/policy_test.go b/internal/signalworkload/policy_test.go index 231801adf5..1ca48a0346 100644 --- a/internal/signalworkload/policy_test.go +++ b/internal/signalworkload/policy_test.go @@ -39,7 +39,10 @@ func TestPolicyRoundTripPreservesBoundedExecutionSettings(t *testing.T) { }, WorkingDir: "/opt/diagnostic", HostNetwork: true, - Results: &corev1.EmptyDirVolumeSource{SizeLimit: quantity("2Gi")}, + KueueTopology: &signalv1.KueueTopologyPolicy{ + RequiredLevel: corev1.LabelHostname, + }, + Results: &corev1.EmptyDirVolumeSource{SizeLimit: quantity("2Gi")}, Volumes: []signalv1.WorkloadVolume{ { Name: "host-proc", @@ -72,6 +75,8 @@ func TestPolicyRoundTripPreservesBoundedExecutionSettings(t *testing.T) { if got.WorkingDir != policy.WorkingDir || !got.HostNetwork || + got.KueueTopology == nil || + got.KueueTopology.RequiredLevel != corev1.LabelHostname || got.Resources.Limits.Cpu().Cmp(*policy.Resources.Limits.Cpu()) != 0 || len(got.Volumes) != 3 || got.Volumes[0].HostPath.Path != "/proc" || @@ -119,6 +124,12 @@ func TestValidateRejectsUnsafeOrAmbiguousPolicy(t *testing.T) { Drop: []corev1.Capability{"SYS_ADMIN"}, }}, }, + "empty topology level": { + KueueTopology: &signalv1.KueueTopologyPolicy{}, + }, + "invalid topology level": { + KueueTopology: &signalv1.KueueTopologyPolicy{RequiredLevel: "not a label key"}, + }, } for name, policy := range tests { @@ -158,6 +169,24 @@ func TestDecodeRejectsMalformedHostNetworkPolicy(t *testing.T) { } } +func TestDecodeRejectsMalformedKueueTopologyPolicy(t *testing.T) { + t.Parallel() + + for name, content := range map[string]string{ + "wrong type": `{"kueueTopology":"kubernetes.io/hostname"}`, + "unknown field": `{"kueueTopology":{"requiredTopology":"kubernetes.io/hostname"}}`, + "empty object": `{"kueueTopology":{}}`, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + if _, err := Decode(content); err == nil { + t.Fatal("Decode accepted malformed Kueue topology policy") + } + }) + } +} + func quantity(value string) *resource.Quantity { result := resource.MustParse(value) From 05d347274eef61dea3b8dad343e12fad8f27b46f Mon Sep 17 00:00:00 2001 From: chokevin Date: Wed, 21 Oct 2026 22:25:02 -0700 Subject: [PATCH 10/36] fix(signal): preserve diagnostic results --- deploy/signal/README.md | 12 ++- internal/signaladapter/adapter.go | 94 +++++++++++++++---- internal/signaladapter/adapter_test.go | 108 ++++++++++++++++++++- internal/signaladapter/runner.go | 71 ++++++++++---- internal/signaladapter/runner_test.go | 125 +++++++++++++++++++++++++ 5 files changed, 367 insertions(+), 43 deletions(-) diff --git a/deploy/signal/README.md b/deploy/signal/README.md index f9d7f61b8f..ad20d77262 100644 --- a/deploy/signal/README.md +++ b/deploy/signal/README.md @@ -121,10 +121,14 @@ execution state, and foreground-deletes every created workload. The selected script is invoked with no arguments for a Pod backend, `mpi-launcher ` in an MPIJob launcher, and `mpi-worker` in its worker. Scripts write `verdict`, `reason`, and optional `measurements.txt` under -`/results`. A nonzero process exit remains an execution error and cannot be -converted into a diagnostic `Fail`. Missing results, partial backend execution, -identity gaps, malformed output, deadline interruption, and cleanup failure are -all `Inconclusive`. +`/results`. The script exit code must match the result verdict: `Pass` exits 0, +`Fail` exits 1, and `Inconclusive` exits 2. A valid bounded result remains +authoritative when its verdict-associated exit code makes the backend Pod or +MPIJob terminally unsuccessful. The adapter removes reserved result files +before each invocation and rejects missing files, malformed values, stale or +contradictory result/exit combinations, and any other exit code. Partial +backend execution, identity gaps, deadline interruption, and cleanup failure +also remain `Inconclusive`. Other backend kinds remain operator-supplied and are not installed or granted permissions by the included adapter. diff --git a/internal/signaladapter/adapter.go b/internal/signaladapter/adapter.go index dd6eb92715..061628284b 100644 --- a/internal/signaladapter/adapter.go +++ b/internal/signaladapter/adapter.go @@ -28,9 +28,10 @@ import ( const ( MaxParticipants = 64 + MaxReasonBytes = 4 * 1024 MaxMeasurementsBytes = 16 * 1024 MaxBackendOutputBytes = 16 * 1024 - resultPrefix = "PROJECT_SIGNAL_RESULT_V1 " + resultPrefix = "PROJECT_SIGNAL_RESULT_V2 " requiredTopologyKey = "kueue.x-k8s.io/podset-required-topology" ) @@ -85,6 +86,7 @@ type Result struct { Verdict string `json:"verdict"` Reason string `json:"reason"` Measurements string `json:"measurements"` + ExitCode int `json:"-"` } type AggregateOutput struct { @@ -491,6 +493,10 @@ func Aggregate( return AggregateOutput{}, fmt.Errorf("results[%d] requires executionState and reason", index) } + if len(result.Reason) > MaxReasonBytes { + return AggregateOutput{}, fmt.Errorf("results[%d].reason exceeds %d bytes", index, MaxReasonBytes) + } + switch result.Verdict { case "Pass": case "Fail": @@ -573,12 +579,13 @@ func ParseResultLog(logContent string, participantIndex int) (Result, error) { } fields := strings.Fields(strings.TrimPrefix(line, resultPrefix)) - if len(fields) != 3 { - return Result{}, errors.New("result record must contain verdict, reason, and measurements") + if len(fields) != 4 { + return Result{}, errors.New("result record must contain verdict, reason, measurements, and exit code") } - values := make([]string, len(fields)) - for fieldIndex, field := range fields { + values := make([]string, 3) + + for fieldIndex, field := range fields[:3] { if field == "-" { continue } @@ -591,18 +598,67 @@ func ParseResultLog(logContent string, participantIndex int) (Result, error) { values[fieldIndex] = string(decoded) } - return Result{ + exitCode, err := strconv.Atoi(fields[3]) + if err != nil || exitCode < 0 || exitCode > 255 { + return Result{}, errors.New("result exit code must be an integer between 0 and 255") + } + + result := Result{ ParticipantIndex: participantIndex, ExecutionState: "Succeeded", Verdict: values[0], Reason: values[1], Measurements: values[2], - }, nil + ExitCode: exitCode, + } + if err := validateResult(result); err != nil { + return Result{}, err + } + + return result, nil } return Result{}, errors.New("workload log does not contain a Project Signal result record") } +func validateResult(result Result) error { + if result.Reason == "" { + return errors.New("result reason must not be empty") + } + + if len(result.Reason) > MaxReasonBytes { + return fmt.Errorf("result reason exceeds %d bytes", MaxReasonBytes) + } + + if len(result.Measurements) > MaxMeasurementsBytes { + return fmt.Errorf("result measurements exceeds %d bytes", MaxMeasurementsBytes) + } + + var expectedExitCode int + + switch result.Verdict { + case "Pass": + expectedExitCode = 0 + case "Fail": + expectedExitCode = 1 + case "Inconclusive": + expectedExitCode = 2 + default: + return fmt.Errorf("result verdict %q is invalid", result.Verdict) + } + + if result.ExitCode != expectedExitCode { + return fmt.Errorf( + "result verdict %s requires exit code %d, got %d", + result.Verdict, + expectedExitCode, + result.ExitCode, + ) + } + + return nil +} + func resultCommand(arguments ...string) []string { quotedArguments := make([]string, len(arguments)) for index, argument := range arguments { @@ -616,25 +672,25 @@ func resultCommand(arguments ...string) []string { set -u mkdir -p /results chmod 0755 /results -printf '%%s' Inconclusive > /results/verdict -printf '%%s' ScriptDidNotProduceReason > /results/reason -: > /results/measurements.txt +rm -f /results/verdict /results/reason /results/measurements.txt set +e /bin/bash "/scripts/${PROJECT_SIGNAL_SCRIPT_KEY}" %s status=$? set -e -if [[ $status -ne 0 ]]; then - [[ -s /results/verdict ]] || printf '%%s' Inconclusive > /results/verdict - [[ -s /results/reason ]] || printf 'ScriptExit%%d' "$status" > /results/reason - printf '\nscript_exit_code=%%d\n' "$status" >> /results/measurements.txt -fi -verdict="$(base64 < /results/verdict | tr -d '\n')" -reason="$(base64 < /results/reason | tr -d '\n')" -measurements="$(base64 < /results/measurements.txt | tr -d '\n')" +encode_result_file() { + if [[ -f "$1" ]]; then + base64 < "$1" | tr -d '\n' + else + printf '%%s' - + fi +} +verdict="$(encode_result_file /results/verdict)" +reason="$(encode_result_file /results/reason)" +measurements="$(encode_result_file /results/measurements.txt)" [[ -n "${verdict}" ]] || verdict=- [[ -n "${reason}" ]] || reason=- [[ -n "${measurements}" ]] || measurements=- -printf '%s%%s %%s %%s\n' "${verdict}" "${reason}" "${measurements}" +printf '%s%%s %%s %%s %%d\n' "${verdict}" "${reason}" "${measurements}" "$status" exit "$status" `, strings.Join(quotedArguments, " "), resultPrefix), } diff --git a/internal/signaladapter/adapter_test.go b/internal/signaladapter/adapter_test.go index dbf18e7c1a..5051429f67 100644 --- a/internal/signaladapter/adapter_test.go +++ b/internal/signaladapter/adapter_test.go @@ -722,7 +722,7 @@ func TestParseResultLogUsesLastCompleteRecord(t *testing.T) { logContent := "noise\n" + resultPrefix + encode("Pass") + " " + encode("Healthy") + " " + - encode("temperature=42") + "\n" + encode("temperature=42") + " 0\n" result, err := ParseResultLog(logContent, 1) if err != nil { @@ -742,7 +742,7 @@ func TestParseResultLogAllowsEmptyMeasurements(t *testing.T) { encode := func(value string) string { return base64.StdEncoding.EncodeToString([]byte(value)) } - logContent := resultPrefix + encode("Pass") + " " + encode("Healthy") + " -\n" + logContent := resultPrefix + encode("Pass") + " " + encode("Healthy") + " - 0\n" result, err := ParseResultLog(logContent, 0) if err != nil { @@ -754,6 +754,110 @@ func TestParseResultLogAllowsEmptyMeasurements(t *testing.T) { } } +func TestParseResultLogAcceptsInconclusiveExitTwo(t *testing.T) { + t.Parallel() + + encode := func(value string) string { + return base64.StdEncoding.EncodeToString([]byte(value)) + } + logContent := resultPrefix + + encode("Inconclusive") + " " + + encode("DiagnosticUnavailable") + " " + + encode("device_state=unknown") + " 2\n" + + result, err := ParseResultLog(logContent, 0) + if err != nil { + t.Fatal(err) + } + + if result.ExecutionState != "Succeeded" || + result.Verdict != "Inconclusive" || + result.ExitCode != 2 { + t.Fatalf("result = %#v", result) + } +} + +func TestParseResultLogRejectsContradictoryVerdictExitCode(t *testing.T) { + t.Parallel() + + encode := func(value string) string { + return base64.StdEncoding.EncodeToString([]byte(value)) + } + + for name, test := range map[string]struct { + verdict string + exitCode string + }{ + "Pass with diagnostic failure": {verdict: "Pass", exitCode: "1"}, + "Fail with successful process": {verdict: "Fail", exitCode: "0"}, + "Inconclusive with successful process": {verdict: "Inconclusive", exitCode: "0"}, + "unexpected exit code": {verdict: "Inconclusive", exitCode: "3"}, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + logContent := resultPrefix + + encode(test.verdict) + " " + + encode("SyntheticReason") + " - " + + test.exitCode + "\n" + if _, err := ParseResultLog(logContent, 0); err == nil { + t.Fatal("ParseResultLog accepted contradictory result") + } + }) + } +} + +func TestParseResultLogRejectsMalformedAndMissingResults(t *testing.T) { + t.Parallel() + + encode := func(value string) string { + return base64.StdEncoding.EncodeToString([]byte(value)) + } + + for name, logContent := range map[string]string{ + "malformed verdict": resultPrefix + "%%% " + encode("Reason") + " - 2\n", + "missing verdict": resultPrefix + "- " + encode("Reason") + " - 2\n", + "missing reason": resultPrefix + encode("Inconclusive") + " - - 2\n", + "missing record": "diagnostic output only\n", + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + if _, err := ParseResultLog(logContent, 0); err == nil { + t.Fatal("ParseResultLog accepted invalid result") + } + }) + } +} + +func TestResultCommandClearsReservedFilesAndReportsExitCode(t *testing.T) { + t.Parallel() + + command := resultCommand() + if len(command) != 3 { + t.Fatalf("resultCommand() = %#v", command) + } + + for _, required := range []string{ + "rm -f /results/verdict /results/reason /results/measurements.txt", + `printf 'PROJECT_SIGNAL_RESULT_V2 %s %s %s %d\n'`, + `exit "$status"`, + } { + if !strings.Contains(command[2], required) { + t.Fatalf("result command does not contain %q:\n%s", required, command[2]) + } + } + + for _, forbidden := range []string{ + "ScriptDidNotProduceReason", + "script_exit_code=", + } { + if strings.Contains(command[2], forbidden) { + t.Fatalf("result command still contains fallback %q:\n%s", forbidden, command[2]) + } + } +} + func testConfig() Config { participantsJSON, err := json.Marshal(testParticipants()) if err != nil { diff --git a/internal/signaladapter/runner.go b/internal/signaladapter/runner.go index cfcd347568..cadabce71b 100644 --- a/internal/signaladapter/runner.go +++ b/internal/signaladapter/runner.go @@ -218,29 +218,18 @@ func (r *Runner) waitForResult(ctx context.Context, config Config, backend creat } if terminal { - if !succeeded { - return result - } - if capturedResult != nil { - return *capturedResult + return completedResult(*capturedResult, succeeded) } logContent, logErr := r.resultLog(ctx, config, backend) - if logErr != nil { - result.Reason = "BackendResultUnavailable: " + logErr.Error() - - return result - } - parsed, parseErr := ParseResultLog(logContent, int(backend.participantIndex)) - if parseErr != nil { - result.Reason = "BackendResultInvalid: " + parseErr.Error() - - return result - } - - return parsed + return backendCompletionResult( + logContent, + logErr, + int(backend.participantIndex), + succeeded, + ) } select { @@ -253,6 +242,52 @@ func (r *Runner) waitForResult(ctx context.Context, config Config, backend creat } } +func backendCompletionResult( + logContent string, + logErr error, + participantIndex int, + backendSucceeded bool, +) Result { + result := Result{ + ParticipantIndex: participantIndex, + ExecutionState: "Error", + Verdict: "Inconclusive", + Reason: "BackendExecutionFailed", + } + + if logErr != nil { + if backendSucceeded { + result.Reason = "BackendResultUnavailable: " + logErr.Error() + } + + return result + } + + parsed, err := ParseResultLog(logContent, participantIndex) + if err != nil { + if backendSucceeded { + result.Reason = "BackendResultInvalid: " + err.Error() + } + + return result + } + + return completedResult(parsed, backendSucceeded) +} + +func completedResult(result Result, backendSucceeded bool) Result { + if (result.ExitCode == 0) != backendSucceeded { + return Result{ + ParticipantIndex: result.ParticipantIndex, + ExecutionState: "Error", + Verdict: "Inconclusive", + Reason: "BackendResultInvalid: result exit code contradicts backend terminal state", + } + } + + return result +} + func backendState(resource *unstructured.Unstructured) (bool, bool, error) { switch resource.GetKind() { case "Pod": diff --git a/internal/signaladapter/runner_test.go b/internal/signaladapter/runner_test.go index f3dd8e0f74..834e5f6102 100644 --- a/internal/signaladapter/runner_test.go +++ b/internal/signaladapter/runner_test.go @@ -5,6 +5,9 @@ package signaladapter import ( "context" + "encoding/base64" + "fmt" + "strings" "testing" "time" @@ -15,6 +18,128 @@ import ( dynamicfake "k8s.io/client-go/dynamic/fake" ) +func TestCompletedResultAcceptsVerdictAssociatedTerminalState(t *testing.T) { + t.Parallel() + + for name, test := range map[string]struct { + result Result + backendSucceeded bool + }{ + "Pass": { + result: Result{ParticipantIndex: 0, ExecutionState: "Succeeded", Verdict: "Pass", Reason: "Healthy"}, + backendSucceeded: true, + }, + "Fail": { + result: Result{ + ParticipantIndex: 0, + ExecutionState: "Succeeded", + Verdict: "Fail", + Reason: "DiagnosticFailed", + ExitCode: 1, + }, + }, + "Inconclusive": { + result: Result{ + ParticipantIndex: 0, + ExecutionState: "Succeeded", + Verdict: "Inconclusive", + Reason: "DiagnosticUnavailable", + ExitCode: 2, + }, + }, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + result := completedResult(test.result, test.backendSucceeded) + if result.ExecutionState != "Succeeded" || result.Verdict != test.result.Verdict { + t.Fatalf("completedResult() = %#v", result) + } + }) + } +} + +func TestBackendCompletionResultKeepsValidInconclusiveFromFailedBackend(t *testing.T) { + t.Parallel() + + logContent := encodedResultRecord("Inconclusive", "DiagnosticUnavailable", "device_state=unknown", 2) + + result := backendCompletionResult(logContent, nil, 0, false) + if result.ExecutionState != "Succeeded" || + result.Verdict != "Inconclusive" || + result.Reason != "DiagnosticUnavailable" { + t.Fatalf("backendCompletionResult() = %#v", result) + } +} + +func TestBackendCompletionResultFailsClosedWithoutValidResult(t *testing.T) { + t.Parallel() + + for name, logContent := range map[string]string{ + "malformed result": resultPrefix + "%%% bm90LXVzZWQ= - 2\n", + "missing result": "diagnostic output only\n", + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + result := backendCompletionResult(logContent, nil, 0, false) + if result.ExecutionState != "Error" || + result.Verdict != "Inconclusive" || + result.Reason != "BackendExecutionFailed" { + t.Fatalf("backendCompletionResult() = %#v", result) + } + }) + } +} + +func TestCompletedResultRejectsTerminalStateMismatch(t *testing.T) { + t.Parallel() + + for name, test := range map[string]struct { + exitCode int + backendSucceeded bool + }{ + "zero exit with failed backend": {backendSucceeded: false}, + "nonzero exit with succeeded backend": {exitCode: 2, backendSucceeded: true}, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + result := completedResult(Result{ + ParticipantIndex: 1, + ExecutionState: "Succeeded", + Verdict: "Inconclusive", + Reason: "SyntheticReason", + ExitCode: test.exitCode, + }, test.backendSucceeded) + if result.ExecutionState != "Error" || + result.Verdict != "Inconclusive" || + !strings.Contains(result.Reason, "contradicts backend terminal state") { + t.Fatalf("completedResult() = %#v", result) + } + }) + } +} + +func encodedResultRecord(verdict, reason, measurements string, exitCode int) string { + encode := func(value string) string { + if value == "" { + return "-" + } + + return base64.StdEncoding.EncodeToString([]byte(value)) + } + + return fmt.Sprintf( + "%s%s %s %s %d\n", + resultPrefix, + encode(verdict), + encode(reason), + encode(measurements), + exitCode, + ) +} + func TestCleanupDeletesCreatedBackends(t *testing.T) { t.Parallel() From 82e6c22404a8b7cd20baef570a70007018882ecc Mon Sep 17 00:00:00 2001 From: chokevin Date: Wed, 21 Oct 2026 22:51:55 -0700 Subject: [PATCH 11/36] feat(signal): add runtime execution contract --- .../v1alpha1/certificationprofile_types.go | 33 +++- api/signal/v1alpha1/certificationrun_types.go | 28 ++- api/signal/v1alpha1/runtime_contract.go | 60 ++++++ api/signal/v1alpha1/runtime_contract_test.go | 98 ++++++++++ api/signal/v1alpha1/zz_generated.deepcopy.go | 27 +++ cmd/project-signal-adapter/main.go | 13 ++ .../04-adapter-workflowtemplate.yaml.tmpl | 10 + deploy/signal/README.md | 32 +++- ...ounded-cloud.io_certificationprofiles.yaml | 26 +++ ....unbounded-cloud.io_certificationruns.yaml | 29 +++ deploy/signal/manifest_test.go | 31 ++++ internal/signal/controller.go | 108 +++++++---- internal/signal/controller_test.go | 143 ++++++++++++--- internal/signaladapter/adapter.go | 93 ++++++---- internal/signaladapter/adapter_test.go | 173 +++++++++++++++--- internal/signaladapter/runner.go | 18 +- 16 files changed, 779 insertions(+), 143 deletions(-) create mode 100644 api/signal/v1alpha1/runtime_contract.go create mode 100644 api/signal/v1alpha1/runtime_contract_test.go diff --git a/api/signal/v1alpha1/certificationprofile_types.go b/api/signal/v1alpha1/certificationprofile_types.go index fa2aadf827..49ec5fbe72 100644 --- a/api/signal/v1alpha1/certificationprofile_types.go +++ b/api/signal/v1alpha1/certificationprofile_types.go @@ -9,8 +9,9 @@ import ( ) type ScriptReference struct { - Name string `json:"name"` - Key string `json:"key"` + Name string `json:"name"` + Key string `json:"key"` + RequiresRuntimeStage bool `json:"requiresRuntimeStage,omitempty"` } type WorkflowReference struct { @@ -37,6 +38,13 @@ type CertificationStage struct { Conditional bool `json:"conditional,omitempty"` } +type RuntimeStage struct { + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` + Name string `json:"name"` +} + type ExecutionBackend struct { APIVersion string `json:"apiVersion"` Kind string `json:"kind"` @@ -83,6 +91,9 @@ type WorkloadPolicy struct { Volumes []WorkloadVolume `json:"volumes,omitempty"` } +// +kubebuilder:validation:XValidation:rule="duration(self.backendActiveDeadline) > duration('0s')",message="backendActiveDeadline must be positive" +// +kubebuilder:validation:XValidation:rule="duration(self.deadline) - duration(self.backendActiveDeadline) >= duration('60s')",message="backendActiveDeadline must leave at least 60 seconds before deadline" +// +kubebuilder:validation:XValidation:rule="!has(self.script.requiresRuntimeStage) || !self.script.requiresRuntimeStage || has(self.runtimeStage)",message="runtimeStage is required when the script requires a runtime stage" type CertificationProfileSpec struct { Participants ParticipantPolicy `json:"participants"` Script ScriptReference `json:"script"` @@ -91,14 +102,16 @@ type CertificationProfileSpec struct { RunnerImage string `json:"runnerImage"` WorkloadPolicy WorkloadPolicy `json:"workloadPolicy,omitempty"` // +kubebuilder:validation:Enum=FullCertification;QuickCertification;Preflight;TargetedVerification;Diagnostic - Purpose CertificationPurpose `json:"purpose,omitempty"` - StagePlan []CertificationStage `json:"stagePlan,omitempty"` - Parameters map[string]string `json:"parameters,omitempty"` - EvidencePath string `json:"evidencePath"` - Deadline metav1.Duration `json:"deadline"` - MaxAttempts int32 `json:"maxAttempts,omitempty"` - Tolerations []corev1.Toleration `json:"tolerations,omitempty"` - Recurring *RecurringPolicy `json:"recurring,omitempty"` + Purpose CertificationPurpose `json:"purpose,omitempty"` + StagePlan []CertificationStage `json:"stagePlan,omitempty"` + RuntimeStage *RuntimeStage `json:"runtimeStage,omitempty"` + Parameters map[string]string `json:"parameters,omitempty"` + EvidencePath string `json:"evidencePath"` + Deadline metav1.Duration `json:"deadline"` + BackendActiveDeadline metav1.Duration `json:"backendActiveDeadline"` + MaxAttempts int32 `json:"maxAttempts,omitempty"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + Recurring *RecurringPolicy `json:"recurring,omitempty"` } type RecurringPolicy struct { diff --git a/api/signal/v1alpha1/certificationrun_types.go b/api/signal/v1alpha1/certificationrun_types.go index 6e95888088..ae4892b2b2 100644 --- a/api/signal/v1alpha1/certificationrun_types.go +++ b/api/signal/v1alpha1/certificationrun_types.go @@ -56,8 +56,9 @@ type PinnedProfile struct { } type ScriptSnapshot struct { - ConfigMapName string `json:"configMapName"` - Key string `json:"key"` + ConfigMapName string `json:"configMapName"` + Key string `json:"key"` + RequiresRuntimeStage bool `json:"requiresRuntimeStage,omitempty"` // Digest is the algorithm-qualified SHA-256 digest of the selected script bytes. // +kubebuilder:validation:Pattern=`^sha256:[0-9a-f]{64}$` Digest string `json:"digest"` @@ -70,12 +71,17 @@ type ScriptSnapshot struct { // +kubebuilder:validation:XValidation:rule="self.backend == oldSelf.backend",message="backend is immutable" // +kubebuilder:validation:XValidation:rule="self.runnerImage == oldSelf.runnerImage",message="runnerImage is immutable" // +kubebuilder:validation:XValidation:rule="self.workloadPolicy == oldSelf.workloadPolicy",message="workloadPolicy is immutable" +// +kubebuilder:validation:XValidation:rule="self.runtimeStage == oldSelf.runtimeStage",message="runtimeStage is immutable" // +kubebuilder:validation:XValidation:rule="self.parameters == oldSelf.parameters",message="parameters are immutable" // +kubebuilder:validation:XValidation:rule="self.evidencePath == oldSelf.evidencePath",message="evidencePath is immutable" // +kubebuilder:validation:XValidation:rule="self.deadline == oldSelf.deadline",message="deadline is immutable" +// +kubebuilder:validation:XValidation:rule="self.backendActiveDeadline == oldSelf.backendActiveDeadline",message="backendActiveDeadline is immutable" // +kubebuilder:validation:XValidation:rule="self.maxAttempts == oldSelf.maxAttempts",message="maxAttempts is immutable" // +kubebuilder:validation:XValidation:rule="self.tolerations == oldSelf.tolerations",message="tolerations are immutable" // +kubebuilder:validation:XValidation:rule="self.requestNodeName == oldSelf.requestNodeName",message="requestNodeName is immutable" +// +kubebuilder:validation:XValidation:rule="duration(self.backendActiveDeadline) > duration('0s')",message="backendActiveDeadline must be positive" +// +kubebuilder:validation:XValidation:rule="duration(self.deadline) - duration(self.backendActiveDeadline) >= duration('60s')",message="backendActiveDeadline must leave at least 60 seconds before deadline" +// +kubebuilder:validation:XValidation:rule="!has(self.script.requiresRuntimeStage) || !self.script.requiresRuntimeStage || has(self.runtimeStage)",message="runtimeStage is required when the script requires a runtime stage" type CertificationRunSpec struct { Profile PinnedProfile `json:"profile"` // +kubebuilder:validation:MinItems=1 @@ -90,14 +96,16 @@ type CertificationRunSpec struct { // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="purpose is immutable" Purpose CertificationPurpose `json:"purpose,omitempty"` // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="stagePlan is immutable" - StagePlan []CertificationStage `json:"stagePlan,omitempty"` - Parameters map[string]string `json:"parameters,omitempty"` - EvidencePath string `json:"evidencePath"` - Deadline metav1.Duration `json:"deadline"` - MaxAttempts int32 `json:"maxAttempts"` - Tolerations []corev1.Toleration `json:"tolerations,omitempty"` - RequestNodeName string `json:"requestNodeName"` - Cancel bool `json:"cancel,omitempty"` + StagePlan []CertificationStage `json:"stagePlan,omitempty"` + RuntimeStage *RuntimeStage `json:"runtimeStage,omitempty"` + Parameters map[string]string `json:"parameters,omitempty"` + EvidencePath string `json:"evidencePath"` + Deadline metav1.Duration `json:"deadline"` + BackendActiveDeadline metav1.Duration `json:"backendActiveDeadline"` + MaxAttempts int32 `json:"maxAttempts"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + RequestNodeName string `json:"requestNodeName"` + Cancel bool `json:"cancel,omitempty"` } type WorkflowStatus struct { diff --git a/api/signal/v1alpha1/runtime_contract.go b/api/signal/v1alpha1/runtime_contract.go new file mode 100644 index 0000000000..9b7af0c5b6 --- /dev/null +++ b/api/signal/v1alpha1/runtime_contract.go @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + "errors" + "fmt" + "time" + + "k8s.io/apimachinery/pkg/util/validation" +) + +const MinimumBackendDeadlineReserve = time.Minute + +func ValidateRuntimeContract( + scriptRequiresStage bool, + stage *RuntimeStage, + backendDeadline time.Duration, + outerDeadline time.Duration, +) error { + if err := ValidateRuntimeStage(scriptRequiresStage, stage); err != nil { + return err + } + + return ValidateExecutionDeadlines(backendDeadline, outerDeadline) +} + +func ValidateRuntimeStage(scriptRequiresStage bool, stage *RuntimeStage) error { + if scriptRequiresStage && stage == nil { + return errors.New("runtime stage is required by the script") + } + + if stage != nil { + if errors := validation.IsDNS1123Label(stage.Name); len(errors) > 0 { + return fmt.Errorf("runtime stage name is invalid: %v", errors) + } + } + + return nil +} + +func ValidateExecutionDeadlines(backendDeadline time.Duration, outerDeadline time.Duration) error { + if backendDeadline <= 0 { + return errors.New("backend active deadline must be positive") + } + + if backendDeadline%time.Second != 0 || outerDeadline%time.Second != 0 { + return errors.New("backend and outer deadlines must use whole seconds") + } + + if outerDeadline-backendDeadline < MinimumBackendDeadlineReserve { + return fmt.Errorf( + "backend active deadline must leave at least %s before the outer deadline", + MinimumBackendDeadlineReserve, + ) + } + + return nil +} diff --git a/api/signal/v1alpha1/runtime_contract_test.go b/api/signal/v1alpha1/runtime_contract_test.go new file mode 100644 index 0000000000..ff7c49cf63 --- /dev/null +++ b/api/signal/v1alpha1/runtime_contract_test.go @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + "strings" + "testing" + "time" +) + +func TestValidateRuntimeContractAcceptsExactDeadlineReserve(t *testing.T) { + t.Parallel() + + err := ValidateRuntimeContract( + true, + &RuntimeStage{Name: "collective-bandwidth"}, + 9*time.Minute, + 10*time.Minute, + ) + if err != nil { + t.Fatal(err) + } +} + +func TestValidateRuntimeContractAllowsNonStageScript(t *testing.T) { + t.Parallel() + + if err := ValidateRuntimeContract(false, nil, time.Minute, 2*time.Minute); err != nil { + t.Fatal(err) + } +} + +func TestValidateRuntimeContractRejectsInvalidStageAndDeadlines(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + requiresStage bool + stage *RuntimeStage + backendDeadline time.Duration + outerDeadline time.Duration + want string + }{ + "missing required stage": { + requiresStage: true, + backendDeadline: time.Minute, + outerDeadline: 2 * time.Minute, + want: "runtime stage is required", + }, + "empty stage": { + stage: &RuntimeStage{}, + backendDeadline: time.Minute, + outerDeadline: 2 * time.Minute, + want: "runtime stage name is invalid", + }, + "malformed stage": { + stage: &RuntimeStage{Name: "Collective/Bandwidth"}, + backendDeadline: time.Minute, + outerDeadline: 2 * time.Minute, + want: "runtime stage name is invalid", + }, + "zero backend deadline": { + backendDeadline: 0, + outerDeadline: 2 * time.Minute, + want: "must be positive", + }, + "fractional seconds": { + backendDeadline: time.Minute + time.Millisecond, + outerDeadline: 2 * time.Minute, + want: "whole seconds", + }, + "reserve one second too short": { + backendDeadline: 61 * time.Second, + outerDeadline: 2 * time.Minute, + want: "at least 1m0s", + }, + "backend equals outer": { + backendDeadline: 2 * time.Minute, + outerDeadline: 2 * time.Minute, + want: "at least 1m0s", + }, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + err := ValidateRuntimeContract( + test.requiresStage, + test.stage, + test.backendDeadline, + test.outerDeadline, + ) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("ValidateRuntimeContract() error = %v, want %q", err, test.want) + } + }) + } +} diff --git a/api/signal/v1alpha1/zz_generated.deepcopy.go b/api/signal/v1alpha1/zz_generated.deepcopy.go index e612e94a5e..262813b6f8 100644 --- a/api/signal/v1alpha1/zz_generated.deepcopy.go +++ b/api/signal/v1alpha1/zz_generated.deepcopy.go @@ -105,6 +105,11 @@ func (in *CertificationProfileSpec) DeepCopyInto(out *CertificationProfileSpec) *out = make([]CertificationStage, len(*in)) copy(*out, *in) } + if in.RuntimeStage != nil { + in, out := &in.RuntimeStage, &out.RuntimeStage + *out = new(RuntimeStage) + **out = **in + } if in.Parameters != nil { in, out := &in.Parameters, &out.Parameters *out = make(map[string]string, len(*in)) @@ -113,6 +118,7 @@ func (in *CertificationProfileSpec) DeepCopyInto(out *CertificationProfileSpec) } } out.Deadline = in.Deadline + out.BackendActiveDeadline = in.BackendActiveDeadline if in.Tolerations != nil { in, out := &in.Tolerations, &out.Tolerations *out = make([]v1.Toleration, len(*in)) @@ -245,6 +251,11 @@ func (in *CertificationRunSpec) DeepCopyInto(out *CertificationRunSpec) { *out = make([]CertificationStage, len(*in)) copy(*out, *in) } + if in.RuntimeStage != nil { + in, out := &in.RuntimeStage, &out.RuntimeStage + *out = new(RuntimeStage) + **out = **in + } if in.Parameters != nil { in, out := &in.Parameters, &out.Parameters *out = make(map[string]string, len(*in)) @@ -253,6 +264,7 @@ func (in *CertificationRunSpec) DeepCopyInto(out *CertificationRunSpec) { } } out.Deadline = in.Deadline + out.BackendActiveDeadline = in.BackendActiveDeadline if in.Tolerations != nil { in, out := &in.Tolerations, &out.Tolerations *out = make([]v1.Toleration, len(*in)) @@ -513,6 +525,21 @@ func (in *RecurringPolicy) DeepCopy() *RecurringPolicy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RuntimeStage) DeepCopyInto(out *RuntimeStage) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuntimeStage. +func (in *RuntimeStage) DeepCopy() *RuntimeStage { + if in == nil { + return nil + } + out := new(RuntimeStage) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ScriptReference) DeepCopyInto(out *ScriptReference) { *out = *in diff --git a/cmd/project-signal-adapter/main.go b/cmd/project-signal-adapter/main.go index 5950df9bb7..f1378e87b3 100644 --- a/cmd/project-signal-adapter/main.go +++ b/cmd/project-signal-adapter/main.go @@ -62,9 +62,22 @@ func main() { "", "algorithm-qualified SHA-256 digest of the script bytes (sha256:<64 lowercase hex>)", ) + flags.BoolVar( + &cfg.ScriptRequiresStage, + "script-requires-runtime-stage", + false, + "Require a validated runtime stage name", + ) + flags.StringVar(&cfg.RuntimeStageName, "runtime-stage-name", "", "Runtime stage name for participant containers") flags.StringVar(&cfg.RunnerImage, "runner-image", "", "Digest-pinned runner image") flags.StringVar(&cfg.BackendAPIVersion, "backend-api-version", "", "Backend API version") flags.StringVar(&cfg.BackendKind, "backend-kind", "", "Backend kind") + flags.Int64Var( + &cfg.BackendDeadlineSeconds, + "backend-active-deadline-seconds", + 0, + "Active deadline for participant backend workloads", + ) flags.StringVar(&cfg.Queue, "queue", "", "Kueue LocalQueue name") flags.StringVar(&cfg.TolerationsJSON, "tolerations", "[]", "JSON-encoded Pod tolerations") flags.StringVar(&cfg.ScriptEnvironmentJSON, "script-environment", "{}", "JSON-encoded script environment") diff --git a/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl b/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl index fee086d32a..3dc673d73f 100644 --- a/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl +++ b/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl @@ -17,8 +17,13 @@ spec: - name: script-configmap - name: script-key - name: script-digest + - name: script-requires-runtime-stage + value: "false" + - name: runtime-stage-name + value: "" - name: backend-api-version - name: backend-kind + - name: backend-active-deadline-seconds - name: queue - name: runner-image - name: tolerations @@ -70,12 +75,17 @@ spec: - "{{`{{workflow.parameters.script-key}}`}}" - --script-digest - "{{`{{workflow.parameters.script-digest}}`}}" + - "--script-requires-runtime-stage={{`{{workflow.parameters.script-requires-runtime-stage}}`}}" + - --runtime-stage-name + - "{{`{{workflow.parameters.runtime-stage-name}}`}}" - --runner-image - "{{`{{workflow.parameters.runner-image}}`}}" - --backend-api-version - "{{`{{workflow.parameters.backend-api-version}}`}}" - --backend-kind - "{{`{{workflow.parameters.backend-kind}}`}}" + - --backend-active-deadline-seconds + - "{{`{{workflow.parameters.backend-active-deadline-seconds}}`}}" - --queue - "{{`{{workflow.parameters.queue}}`}}" - --tolerations diff --git a/deploy/signal/README.md b/deploy/signal/README.md index ad20d77262..288aedf5ca 100644 --- a/deploy/signal/README.md +++ b/deploy/signal/README.md @@ -69,8 +69,11 @@ The referenced WorkflowTemplate must accept these string parameters: | `script-configmap` | Immutable run-owned ConfigMap | | `script-key` | Selected ConfigMap key | | `script-digest` | Algorithm-qualified SHA-256 of the selected script bytes (`sha256:<64 lowercase hex>`) | +| `script-requires-runtime-stage` | Whether the selected script requires a runtime stage | +| `runtime-stage-name` | Optional validated runtime stage name | | `backend-api-version` | Backend API version from the profile | | `backend-kind` | Backend kind from the profile | +| `backend-active-deadline-seconds` | Participant backend deadline, shorter than the outer Workflow deadline | | `queue` | Optional backend queue | | `runner-image` | Digest-pinned execution image | | `tolerations` | JSON-encoded toleration list | @@ -234,6 +237,26 @@ admission and operator policy still decide whether a capability, host path, device, PVC, or privileged context is allowed; technical rendering is not authorization. +Stage-aware scripts declare `spec.script.requiresRuntimeStage: true` and require +an explicit `spec.runtimeStage.name`. The name is a lowercase DNS label of at +most 63 characters and is snapshotted into the immutable CertificationRun and +evidence. The adapter exports it as `PROJECT_SIGNAL_STAGE_NAME` only to ordinary +participant Pods and MPIJob workers. MPI launchers never receive it. Scripts +that are not stage-aware leave both fields unset. + +Every profile sets two distinct deadlines: + +```yaml +deadline: 20m +backendActiveDeadline: 19m +``` + +`deadline` is the outer Workflow deadline for backend observation, result and +evidence projection, and cleanup. `backendActiveDeadline` is rendered to the +participant Pod or MPIJob `activeDeadlineSeconds`. It must be positive and +leave at least 60 seconds before the outer deadline; the controller rejects an +invalid ordering before reserving Nodes. + `kueueTopology.requiredLevel` is an explicit Kueue Topology Aware Scheduling request. It must be a valid Kubernetes label key and requires a nonempty backend queue. The adapter writes @@ -264,10 +287,11 @@ defaults. A `CertificationProfile` and `CertificationRun` each select one immutable script key and one backend. `purpose: FullCertification` and `stagePlan` -describe evidence; they do not execute a multi-key suite. Operators that define -a full suite must create one run per script and admit the next run only after -the previous run is terminal and its backend cleanup has completed. The -upstream controller does not synthesize a cross-run verdict. +describe evidence and are distinct from the optional runtime stage passed to a +single script; they do not execute a multi-key suite. Operators that define a +full suite must create one run per script and admit the next run only after the +previous run is terminal and its backend cleanup has completed. The upstream +controller does not synthesize a cross-run verdict. ## Results and evidence diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml index 490d511384..7a05a2a974 100644 --- a/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml @@ -63,6 +63,8 @@ spec: - apiVersion - kind type: object + backendActiveDeadline: + type: string deadline: type: string evidencePath: @@ -201,12 +203,24 @@ spec: type: object runnerImage: type: string + runtimeStage: + properties: + name: + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object script: properties: key: type: string name: type: string + requiresRuntimeStage: + type: boolean required: - key - name @@ -688,6 +702,7 @@ spec: type: object required: - backend + - backendActiveDeadline - deadline - evidencePath - participants @@ -695,6 +710,17 @@ spec: - script - workflow type: object + x-kubernetes-validations: + - message: backendActiveDeadline must be positive + rule: duration(self.backendActiveDeadline) > duration('0s') + - message: backendActiveDeadline must leave at least 60 seconds before + deadline + rule: duration(self.deadline) - duration(self.backendActiveDeadline) + >= duration('60s') + - message: runtimeStage is required when the script requires a runtime + stage + rule: '!has(self.script.requiresRuntimeStage) || !self.script.requiresRuntimeStage + || has(self.runtimeStage)' status: properties: conditions: diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml index 0347c30c6e..30213a6d40 100644 --- a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml @@ -69,6 +69,8 @@ spec: - apiVersion - kind type: object + backendActiveDeadline: + type: string cancel: type: boolean deadline: @@ -158,6 +160,16 @@ spec: type: string runnerImage: type: string + runtimeStage: + properties: + name: + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object script: properties: configMapName: @@ -169,6 +181,8 @@ spec: type: string key: type: string + requiresRuntimeStage: + type: boolean required: - configMapName - digest @@ -654,6 +668,7 @@ spec: type: object required: - backend + - backendActiveDeadline - deadline - evidencePath - maxAttempts @@ -679,18 +694,32 @@ spec: rule: self.runnerImage == oldSelf.runnerImage - message: workloadPolicy is immutable rule: self.workloadPolicy == oldSelf.workloadPolicy + - message: runtimeStage is immutable + rule: self.runtimeStage == oldSelf.runtimeStage - message: parameters are immutable rule: self.parameters == oldSelf.parameters - message: evidencePath is immutable rule: self.evidencePath == oldSelf.evidencePath - message: deadline is immutable rule: self.deadline == oldSelf.deadline + - message: backendActiveDeadline is immutable + rule: self.backendActiveDeadline == oldSelf.backendActiveDeadline - message: maxAttempts is immutable rule: self.maxAttempts == oldSelf.maxAttempts - message: tolerations are immutable rule: self.tolerations == oldSelf.tolerations - message: requestNodeName is immutable rule: self.requestNodeName == oldSelf.requestNodeName + - message: backendActiveDeadline must be positive + rule: duration(self.backendActiveDeadline) > duration('0s') + - message: backendActiveDeadline must leave at least 60 seconds before + deadline + rule: duration(self.deadline) - duration(self.backendActiveDeadline) + >= duration('60s') + - message: runtimeStage is required when the script requires a runtime + stage + rule: '!has(self.script.requiresRuntimeStage) || !self.script.requiresRuntimeStage + || has(self.runtimeStage)' status: properties: cleanup: diff --git a/deploy/signal/manifest_test.go b/deploy/signal/manifest_test.go index 9420c8fdaa..a4ed548146 100644 --- a/deploy/signal/manifest_test.go +++ b/deploy/signal/manifest_test.go @@ -64,7 +64,10 @@ func TestAdapterManifestIsGenericAndParticipantAware(t *testing.T) { "backend-resources", "backend-api-version", "backend-kind", + "backend-active-deadline-seconds", "queue", + "runtime-stage-name", + "script-requires-runtime-stage", "workload-policy", } { if !strings.Contains(manifest, required) { @@ -83,6 +86,34 @@ func TestAdapterManifestIsGenericAndParticipantAware(t *testing.T) { } } +func TestSignalCRDsExposeRuntimeStageAndDeadlineContract(t *testing.T) { + t.Parallel() + + for _, path := range []string{ + "crd/signal.unbounded-cloud.io_certificationprofiles.yaml", + "crd/signal.unbounded-cloud.io_certificationruns.yaml", + } { + content, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + + manifest := string(content) + for _, required := range []string{ + "backendActiveDeadline:", + "runtimeStage:", + "requiresRuntimeStage:", + "backendActiveDeadline must leave at least 60 seconds before", + "runtimeStage is required when the script requires a runtime", + "maxLength: 63", + } { + if !strings.Contains(manifest, required) { + t.Fatalf("%s does not contain %q", path, required) + } + } + } +} + func TestCertificationRunCRDRequiresCanonicalDigests(t *testing.T) { t.Parallel() diff --git a/internal/signal/controller.go b/internal/signal/controller.go index 6d5c0b228a..086d7718a0 100644 --- a/internal/signal/controller.go +++ b/internal/signal/controller.go @@ -225,6 +225,15 @@ func (r *AdmissionReconciler) admitNodeWithTriggerLocked( return fmt.Errorf("profile %s stage plan: %w", profile.Name, err) } + if err := signalv1.ValidateRuntimeContract( + profile.Spec.Script.RequiresRuntimeStage, + profile.Spec.RuntimeStage, + profile.Spec.BackendActiveDeadline.Duration, + profile.Spec.Deadline.Duration, + ); err != nil { + return fmt.Errorf("profile %s runtime contract: %w", profile.Name, err) + } + if !runnerImageDigestPattern.MatchString(profile.Spec.RunnerImage) { return fmt.Errorf("profile %s runner image must use an immutable sha256 digest", profile.Name) } @@ -314,20 +323,27 @@ func (r *AdmissionReconciler) admitNodeWithTriggerLocked( UID: profile.UID, Generation: profile.Generation, }, - Participants: participants, - Script: signalv1.ScriptSnapshot{ConfigMapName: snapshot.Name, Key: profile.Spec.Script.Key, Digest: scriptDigest}, - Workflow: profile.Spec.Workflow, - Backend: profile.Spec.Backend, - RunnerImage: profile.Spec.RunnerImage, - WorkloadPolicy: *profile.Spec.WorkloadPolicy.DeepCopy(), - Purpose: profile.Spec.Purpose, - StagePlan: append([]signalv1.CertificationStage(nil), profile.Spec.StagePlan...), - Parameters: parameters, - EvidencePath: strings.Trim(profile.Spec.EvidencePath, "/") + "/" + runName + ".json", - Deadline: profile.Spec.Deadline, - MaxAttempts: maxAttempts, - Tolerations: profile.Spec.Tolerations, - RequestNodeName: node.Name, + Participants: participants, + Script: signalv1.ScriptSnapshot{ + ConfigMapName: snapshot.Name, + Key: profile.Spec.Script.Key, + RequiresRuntimeStage: profile.Spec.Script.RequiresRuntimeStage, + Digest: scriptDigest, + }, + Workflow: profile.Spec.Workflow, + Backend: profile.Spec.Backend, + RunnerImage: profile.Spec.RunnerImage, + WorkloadPolicy: *profile.Spec.WorkloadPolicy.DeepCopy(), + Purpose: profile.Spec.Purpose, + StagePlan: append([]signalv1.CertificationStage(nil), profile.Spec.StagePlan...), + RuntimeStage: copyRuntimeStage(profile.Spec.RuntimeStage), + Parameters: parameters, + EvidencePath: strings.Trim(profile.Spec.EvidencePath, "/") + "/" + runName + ".json", + Deadline: profile.Spec.Deadline, + BackendActiveDeadline: profile.Spec.BackendActiveDeadline, + MaxAttempts: maxAttempts, + Tolerations: profile.Spec.Tolerations, + RequestNodeName: node.Name, }, } @@ -845,8 +861,14 @@ func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured map[string]any{"name": "script-configmap", "value": run.Spec.Script.ConfigMapName}, map[string]any{"name": "script-key", "value": run.Spec.Script.Key}, map[string]any{"name": "script-digest", "value": run.Spec.Script.Digest}, + map[string]any{"name": "script-requires-runtime-stage", "value": fmt.Sprintf("%t", run.Spec.Script.RequiresRuntimeStage)}, + map[string]any{"name": "runtime-stage-name", "value": runtimeStageName(run.Spec.RuntimeStage)}, map[string]any{"name": "backend-api-version", "value": run.Spec.Backend.APIVersion}, map[string]any{"name": "backend-kind", "value": run.Spec.Backend.Kind}, + map[string]any{ + "name": "backend-active-deadline-seconds", + "value": fmt.Sprintf("%d", int64(run.Spec.BackendActiveDeadline.Seconds())), + }, map[string]any{"name": "queue", "value": run.Spec.Backend.Queue}, map[string]any{"name": "runner-image", "value": run.Spec.RunnerImage}, map[string]any{"name": "tolerations", "value": string(tolerationsJSON)}, @@ -1070,26 +1092,28 @@ func (r *RunReconciler) evidenceDocument( } doc := map[string]any{ - "apiVersion": "signal.unbounded-cloud.io/evidence/v1alpha1", - "runID": string(run.UID), - "runName": run.Name, - "profile": run.Spec.Profile, - "participants": run.Spec.Participants, - "workflow": run.Status.Workflow, - "workflowOutputs": run.Status.Workflow.Outputs, - "backend": run.Spec.Backend, - "runnerImage": run.Spec.RunnerImage, - "workloadPolicy": run.Spec.WorkloadPolicy, - "purpose": run.Spec.Purpose, - "stagePlan": run.Spec.StagePlan, - "parameters": run.Spec.Parameters, - "scriptSnapshot": run.Spec.Script, - "scriptContent": snapshot.Data[run.Spec.Script.Key], - "executionState": executionState, - "verdict": verdict, - "reason": reason, - "startedAt": run.Status.StartedAt, - "finishedAt": run.Status.FinishedAt.UTC().Format(time.RFC3339Nano), + "apiVersion": "signal.unbounded-cloud.io/evidence/v1alpha1", + "runID": string(run.UID), + "runName": run.Name, + "profile": run.Spec.Profile, + "participants": run.Spec.Participants, + "workflow": run.Status.Workflow, + "workflowOutputs": run.Status.Workflow.Outputs, + "backend": run.Spec.Backend, + "runnerImage": run.Spec.RunnerImage, + "workloadPolicy": run.Spec.WorkloadPolicy, + "purpose": run.Spec.Purpose, + "stagePlan": run.Spec.StagePlan, + "runtimeStage": run.Spec.RuntimeStage, + "backendActiveDeadline": run.Spec.BackendActiveDeadline, + "parameters": run.Spec.Parameters, + "scriptSnapshot": run.Spec.Script, + "scriptContent": snapshot.Data[run.Spec.Script.Key], + "executionState": executionState, + "verdict": verdict, + "reason": reason, + "startedAt": run.Status.StartedAt, + "finishedAt": run.Status.FinishedAt.UTC().Format(time.RFC3339Nano), } return json.MarshalIndent(doc, "", " ") @@ -1591,6 +1615,22 @@ func validateStagePlan(plan []signalv1.CertificationStage) error { return nil } +func copyRuntimeStage(stage *signalv1.RuntimeStage) *signalv1.RuntimeStage { + if stage == nil { + return nil + } + + return &signalv1.RuntimeStage{Name: stage.Name} +} + +func runtimeStageName(stage *signalv1.RuntimeStage) string { + if stage == nil { + return "" + } + + return stage.Name +} + func AddToScheme(scheme *runtime.Scheme) error { return signalv1.AddToScheme(scheme) } diff --git a/internal/signal/controller_test.go b/internal/signal/controller_test.go index 212018c754..06a09fae86 100644 --- a/internal/signal/controller_test.go +++ b/internal/signal/controller_test.go @@ -35,6 +35,8 @@ func TestAdmissionCreatesImmutableSnapshotAndPreventsDuplicate(t *testing.T) { node := readyNode("gpu-0", "example-gpu") node.Labels[RequestLabel] = "gpu-smoke" profile := testProfile("gpu-smoke", "example-gpu") + profile.Spec.Script.RequiresRuntimeStage = true + profile.Spec.RuntimeStage = &signalv1.RuntimeStage{Name: "collective-bandwidth"} profile.Spec.Backend.Queue = "certification" profile.Spec.WorkloadPolicy = signalv1.WorkloadPolicy{ Resources: corev1.ResourceRequirements{ @@ -88,6 +90,14 @@ func TestAdmissionCreatesImmutableSnapshotAndPreventsDuplicate(t *testing.T) { t.Fatalf("stage plan = %#v, want immutable profile plan", run.Spec.StagePlan) } + if !run.Spec.Script.RequiresRuntimeStage || + run.Spec.RuntimeStage == nil || + run.Spec.RuntimeStage.Name != "collective-bandwidth" || + run.Spec.BackendActiveDeadline.Duration != 9*time.Minute { + t.Fatalf("runtime contract = script %#v stage %#v backend deadline %s", + run.Spec.Script, run.Spec.RuntimeStage, run.Spec.BackendActiveDeadline.Duration) + } + if run.Spec.Purpose != signalv1.PurposeQuickCertification { t.Fatalf("purpose = %q, want %q", run.Spec.Purpose, signalv1.PurposeQuickCertification) } @@ -133,6 +143,74 @@ func TestAdmissionCreatesImmutableSnapshotAndPreventsDuplicate(t *testing.T) { } } +func TestAdmissionRejectsMissingRequiredRuntimeStageBeforeReservation(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + node := readyNode("gpu-0", "example-gpu") + node.Labels[RequestLabel] = "gpu-smoke" + profile := testProfile("gpu-smoke", "example-gpu") + profile.Spec.Script.RequiresRuntimeStage = true + source := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, + Data: map[string]string{"payload.txt": "test payload\n"}, + } + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node, profile, source).Build() + reconciler := &AdmissionReconciler{Client: kube, Namespace: "signal", ScriptNamespace: "signal"} + + _, err := reconciler.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}, + ) + if err == nil || !strings.Contains(err.Error(), "runtime stage is required") { + t.Fatalf("Reconcile error = %v", err) + } + + var got corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &got); err != nil { + t.Fatal(err) + } + + if got.Annotations[ReservationLabel] != "" { + t.Fatalf("node was reserved after rejected runtime contract: %#v", got.Annotations) + } +} + +func TestAdmissionRejectsInsufficientBackendDeadlineReserveBeforeReservation(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + node := readyNode("gpu-0", "example-gpu") + node.Labels[RequestLabel] = "gpu-smoke" + profile := testProfile("gpu-smoke", "example-gpu") + profile.Spec.BackendActiveDeadline = metav1.Duration{ + Duration: profile.Spec.Deadline.Duration - 59*time.Second, + } + source := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, + Data: map[string]string{"payload.txt": "test payload\n"}, + } + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node, profile, source).Build() + reconciler := &AdmissionReconciler{Client: kube, Namespace: "signal", ScriptNamespace: "signal"} + + _, err := reconciler.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}, + ) + if err == nil || !strings.Contains(err.Error(), "leave at least 1m0s") { + t.Fatalf("Reconcile error = %v", err) + } + + var got corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &got); err != nil { + t.Fatal(err) + } + + if got.Annotations[ReservationLabel] != "" { + t.Fatalf("node was reserved after rejected deadlines: %#v", got.Annotations) + } +} + func TestAdmissionRejectsMutableRunnerImageBeforeReservation(t *testing.T) { t.Parallel() @@ -647,6 +725,8 @@ func TestWorkflowPinsAttemptInputs(t *testing.T) { "health-mode": "snapshot", "mpi-payload-mode": "script", } + run.Spec.Script.RequiresRuntimeStage = true + run.Spec.RuntimeStage = &signalv1.RuntimeStage{Name: "collective-bandwidth"} run.Spec.Tolerations = []corev1.Toleration{{ Key: "nvidia.com/gpu", Operator: corev1.TolerationOpExists, @@ -681,6 +761,10 @@ func TestWorkflowPinsAttemptInputs(t *testing.T) { t.Fatalf("workflow completion TTL = %d, want 21600", got) } + if got, _, _ := unstructured.NestedInt64(workflow.Object, "spec", "activeDeadlineSeconds"); got != 600 { + t.Fatalf("workflow active deadline = %d, want outer deadline 600", got) + } + if got, _, _ := unstructured.NestedString(workflow.Object, "spec", "podGC", "deleteDelayDuration"); got != "2h" { t.Fatalf("workflow pod GC delay = %q, want 2h", got) } @@ -705,6 +789,12 @@ func TestWorkflowPinsAttemptInputs(t *testing.T) { t.Fatalf("workflow values = %#v", values) } + if values["script-requires-runtime-stage"] != "true" || + values["runtime-stage-name"] != "collective-bandwidth" || + values["backend-active-deadline-seconds"] != "540" { + t.Fatalf("workflow runtime contract = %#v", values) + } + if values["participant-count"] != "1" || values["request-node-name"] != run.Spec.RequestNodeName { t.Fatalf("workflow participant metadata = %#v", values) } @@ -1048,6 +1138,7 @@ func TestTerminalVerdictPersistsEvidenceAndCleanupReleasesNode(t *testing.T) { scheme := testScheme(t) run := testRun("run-pass") + run.Spec.RuntimeStage = &signalv1.RuntimeStage{Name: "collective-bandwidth"} run.Spec.Backend.Queue = "certification" run.Spec.WorkloadPolicy.KueueTopology = &signalv1.KueueTopologyPolicy{ RequiredLevel: corev1.LabelHostname, @@ -1101,8 +1192,10 @@ func TestTerminalVerdictPersistsEvidenceAndCleanupReleasesNode(t *testing.T) { } var evidence struct { - Workflow signalv1.WorkflowStatus `json:"workflow"` - WorkloadPolicy signalv1.WorkloadPolicy `json:"workloadPolicy"` + Workflow signalv1.WorkflowStatus `json:"workflow"` + WorkloadPolicy signalv1.WorkloadPolicy `json:"workloadPolicy"` + RuntimeStage *signalv1.RuntimeStage `json:"runtimeStage"` + BackendActiveDeadline metav1.Duration `json:"backendActiveDeadline"` } if err := json.Unmarshal(store.data, &evidence); err != nil { t.Fatalf("decode evidence: %v", err) @@ -1117,6 +1210,12 @@ func TestTerminalVerdictPersistsEvidenceAndCleanupReleasesNode(t *testing.T) { t.Fatalf("evidence workload policy = %#v, want %#v", evidence.WorkloadPolicy, run.Spec.WorkloadPolicy) } + if !reflect.DeepEqual(evidence.RuntimeStage, run.Spec.RuntimeStage) || + evidence.BackendActiveDeadline != run.Spec.BackendActiveDeadline { + t.Fatalf("evidence runtime contract = stage %#v deadline %s", + evidence.RuntimeStage, evidence.BackendActiveDeadline.Duration) + } + if !got.Status.Cleanup.ReservationsReleased || !got.Status.Cleanup.SnapshotDeleted { t.Fatalf("cleanup status = %#v", got.Status.Cleanup) } @@ -1468,15 +1567,16 @@ func testProfile(name, sku string) *signalv1.CertificationProfile { }}, RequiredResources: corev1.ResourceList{corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("1")}, }, - Script: signalv1.ScriptReference{Name: "scripts", Key: "payload.txt"}, - Workflow: signalv1.WorkflowReference{Name: "gpu-cert"}, - Backend: signalv1.ExecutionBackend{APIVersion: "kubeflow.org/v2beta1", Kind: "MPIJob"}, - RunnerImage: "runner@sha256:" + strings.Repeat("a", 64), - Purpose: signalv1.PurposeQuickCertification, - StagePlan: []signalv1.CertificationStage{{ID: "workload", DisplayName: "Workload", Category: "compute", Required: true}}, - EvidencePath: "signal/" + sku, - Deadline: metav1.Duration{Duration: 10 * time.Minute}, - MaxAttempts: 2, + Script: signalv1.ScriptReference{Name: "scripts", Key: "payload.txt"}, + Workflow: signalv1.WorkflowReference{Name: "gpu-cert"}, + Backend: signalv1.ExecutionBackend{APIVersion: "kubeflow.org/v2beta1", Kind: "MPIJob"}, + RunnerImage: "runner@sha256:" + strings.Repeat("a", 64), + Purpose: signalv1.PurposeQuickCertification, + StagePlan: []signalv1.CertificationStage{{ID: "workload", DisplayName: "Workload", Category: "compute", Required: true}}, + EvidencePath: "signal/" + sku, + Deadline: metav1.Duration{Duration: 10 * time.Minute}, + BackendActiveDeadline: metav1.Duration{Duration: 9 * time.Minute}, + MaxAttempts: 2, }, } } @@ -1490,16 +1590,17 @@ func testRun(name string) *signalv1.CertificationRun { Finalizers: []string{RunFinalizer}, }, Spec: signalv1.CertificationRunSpec{ - Profile: signalv1.PinnedProfile{Name: "gpu-profile", UID: "profile-uid", Generation: 3}, - Participants: []signalv1.Participant{{NodeName: "gpu-0", NodeUID: "gpu-0-uid"}}, - Script: signalv1.ScriptSnapshot{ConfigMapName: name + "-scripts", Key: "payload.txt", Digest: digestString("test payload\n")}, - Workflow: signalv1.WorkflowReference{Name: "gpu-cert"}, - Backend: signalv1.ExecutionBackend{APIVersion: "kubeflow.org/v2beta1", Kind: "MPIJob"}, - RunnerImage: "runner@sha256:" + strings.Repeat("a", 64), - EvidencePath: "signal/" + name + ".json", - Deadline: metav1.Duration{Duration: 10 * time.Minute}, - MaxAttempts: 2, - RequestNodeName: "gpu-0", + Profile: signalv1.PinnedProfile{Name: "gpu-profile", UID: "profile-uid", Generation: 3}, + Participants: []signalv1.Participant{{NodeName: "gpu-0", NodeUID: "gpu-0-uid"}}, + Script: signalv1.ScriptSnapshot{ConfigMapName: name + "-scripts", Key: "payload.txt", Digest: digestString("test payload\n")}, + Workflow: signalv1.WorkflowReference{Name: "gpu-cert"}, + Backend: signalv1.ExecutionBackend{APIVersion: "kubeflow.org/v2beta1", Kind: "MPIJob"}, + RunnerImage: "runner@sha256:" + strings.Repeat("a", 64), + EvidencePath: "signal/" + name + ".json", + Deadline: metav1.Duration{Duration: 10 * time.Minute}, + BackendActiveDeadline: metav1.Duration{Duration: 9 * time.Minute}, + MaxAttempts: 2, + RequestNodeName: "gpu-0", }, } } diff --git a/internal/signaladapter/adapter.go b/internal/signaladapter/adapter.go index 061628284b..75ceaf1178 100644 --- a/internal/signaladapter/adapter.go +++ b/internal/signaladapter/adapter.go @@ -55,29 +55,32 @@ var reservedEnvironmentNames = map[string]struct{}{ } type Config struct { - Namespace string - WorkflowName string - RunName string - RunUID types.UID - ParticipantsJSON string - ParticipantCount int - RequestNodeName string - LegacyParticipants string - ScriptConfigMap string - ScriptKey string - ScriptDigest string - RunnerImage string - BackendAPIVersion string - BackendKind string - Queue string - TolerationsJSON string - ScriptEnvironmentJSON string - WorkloadPolicyJSON string - GPUResourceCount string - Privileged bool - Attempt int - BackoffLimit int64 - TTLSeconds int64 + Namespace string + WorkflowName string + RunName string + RunUID types.UID + ParticipantsJSON string + ParticipantCount int + RequestNodeName string + LegacyParticipants string + ScriptConfigMap string + ScriptKey string + ScriptDigest string + ScriptRequiresStage bool + RuntimeStageName string + RunnerImage string + BackendAPIVersion string + BackendKind string + BackendDeadlineSeconds int64 + Queue string + TolerationsJSON string + ScriptEnvironmentJSON string + WorkloadPolicyJSON string + GPUResourceCount string + Privileged bool + Attempt int + BackoffLimit int64 + TTLSeconds int64 } type Result struct { @@ -229,6 +232,19 @@ func ValidateConfig(config Config) ([]signalv1.Participant, []corev1.Toleration, ) } + var runtimeStage *signalv1.RuntimeStage + if config.RuntimeStageName != "" { + runtimeStage = &signalv1.RuntimeStage{Name: config.RuntimeStageName} + } + + if err := signalv1.ValidateRuntimeStage(config.ScriptRequiresStage, runtimeStage); err != nil { + return nil, nil, nil, fmt.Errorf("runtime stage: %w", err) + } + + if config.BackendDeadlineSeconds <= 0 { + return nil, nil, nil, errors.New("backend-active-deadline-seconds must be positive") + } + switch { case config.BackendKind == "Pod" && config.BackendAPIVersion == "v1": case config.BackendKind == "MPIJob" && config.BackendAPIVersion == "kubeflow.org/v2beta1": @@ -341,7 +357,6 @@ func RenderResources( tolerations []corev1.Toleration, environment []corev1.EnvVar, workflow metav1.OwnerReference, - activeDeadlineSeconds int64, ) ([]*unstructured.Unstructured, error) { resources := make([]*unstructured.Unstructured, 0, len(participants)) @@ -399,7 +414,7 @@ func RenderResources( environment, policy, workflow, - activeDeadlineSeconds, + config.BackendDeadlineSeconds, gpuCount, ) case "MPIJob": @@ -410,7 +425,7 @@ func RenderResources( environment, policy, workflow, - activeDeadlineSeconds, + config.BackendDeadlineSeconds, gpuCount, ) default: @@ -703,7 +718,7 @@ func ordinaryPod( environment []corev1.EnvVar, policy signalv1.WorkloadPolicy, workflow metav1.OwnerReference, - activeDeadlineSeconds int64, + backendDeadlineSeconds int64, gpuCount int64, ) *corev1.Pod { labels := resourceLabels(config, participant) @@ -711,7 +726,7 @@ func ordinaryPod( labels = withLabel(labels, "kueue.x-k8s.io/queue-name", config.Queue) } - env := workloadEnvironment(config, participant, environment) + env := workloadEnvironment(config, participant, environment, true) container := workloadContainer( config, policy, @@ -737,7 +752,7 @@ func ordinaryPod( RestartPolicy: corev1.RestartPolicyNever, NodeSelector: map[string]string{corev1.LabelHostname: participant.NodeName}, Tolerations: tolerations, - ActiveDeadlineSeconds: &activeDeadlineSeconds, + ActiveDeadlineSeconds: &backendDeadlineSeconds, AutomountServiceAccountToken: ptr(false), Containers: []corev1.Container{container}, Volumes: workloadVolumes(config.ScriptConfigMap, policy), @@ -755,13 +770,13 @@ func mpiJob( environment []corev1.EnvVar, policy signalv1.WorkloadPolicy, workflow metav1.OwnerReference, - activeDeadlineSeconds int64, + backendDeadlineSeconds int64, gpuCount int64, ) *unstructured.Unstructured { labels := resourceLabels(config, participant) jobName := resourceName(config.RunName, participant.Index) - launcherEnv := workloadEnvironment(config, participant, environment) - workerEnv := workloadEnvironment(config, participant, environment) + launcherEnv := workloadEnvironment(config, participant, environment, false) + workerEnv := workloadEnvironment(config, participant, environment, true) worker := workloadContainer( config, policy, @@ -789,7 +804,7 @@ func mpiJob( "runLauncherAsWorker": false, "slotsPerWorker": gpuCount, "runPolicy": map[string]any{ - "activeDeadlineSeconds": activeDeadlineSeconds, + "activeDeadlineSeconds": backendDeadlineSeconds, "backoffLimit": config.BackoffLimit, "cleanPodPolicy": "All", "ttlSecondsAfterFinished": config.TTLSeconds, @@ -887,10 +902,11 @@ func workloadEnvironment( config Config, participant signalv1.Participant, environment []corev1.EnvVar, + includeRuntimeStage bool, ) []corev1.EnvVar { result := slices.Clone(environment) - return append( + result = append( result, corev1.EnvVar{Name: "PROJECT_SIGNAL_RUN_NAME", Value: config.RunName}, corev1.EnvVar{Name: "PROJECT_SIGNAL_RUN_UID", Value: string(config.RunUID)}, @@ -902,6 +918,15 @@ func workloadEnvironment( corev1.EnvVar{Name: "PROJECT_SIGNAL_ATTEMPT", Value: strconv.Itoa(config.Attempt)}, corev1.EnvVar{Name: "PROJECT_SIGNAL_RESULTS_DIR", Value: "/results"}, ) + + if includeRuntimeStage && config.RuntimeStageName != "" { + result = append(result, corev1.EnvVar{ + Name: "PROJECT_SIGNAL_STAGE_NAME", + Value: config.RuntimeStageName, + }) + } + + return result } func resultVolumes(scriptConfigMap string) []corev1.Volume { diff --git a/internal/signaladapter/adapter_test.go b/internal/signaladapter/adapter_test.go index 5051429f67..369b3b91eb 100644 --- a/internal/signaladapter/adapter_test.go +++ b/internal/signaladapter/adapter_test.go @@ -32,7 +32,7 @@ func TestRenderOrdinaryPodsPerParticipant(t *testing.T) { t.Fatal(err) } - resources, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner(), 900) + resources, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner()) if err != nil { t.Fatal(err) } @@ -85,6 +85,11 @@ func TestRenderOrdinaryPodsPerParticipant(t *testing.T) { if resource.GetAnnotations()[requiredTopologyKey] != "" { t.Fatalf("resource %d unexpectedly requests Kueue topology", index) } + + activeDeadline, _, err := unstructured.NestedInt64(resource.Object, "spec", "activeDeadlineSeconds") + if err != nil || activeDeadline != config.BackendDeadlineSeconds { + t.Fatalf("resource %d active deadline = %d, err = %v", index, activeDeadline, err) + } } } @@ -101,11 +106,93 @@ func TestRenderOrdinaryPodWithKueueQueue(t *testing.T) { t.Fatal(err) } - resources, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner(), 900) + resources, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner()) if err != nil { t.Fatal(err) } + t.Run("runtime stage is participant only", func(t *testing.T) { + t.Parallel() + + config := testConfig() + config.ScriptRequiresStage = true + config.RuntimeStageName = "collective-bandwidth" + + participants, tolerations, environment, err := ValidateConfig(config) + if err != nil { + t.Fatal(err) + } + + config.BackendAPIVersion = "v1" + config.BackendKind = "Pod" + config.Queue = "" + + pods, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner()) + if err != nil { + t.Fatal(err) + } + + podJSON, err := json.Marshal(pods[0].Object) + if err != nil { + t.Fatal(err) + } + + var pod corev1.Pod + if err := json.Unmarshal(podJSON, &pod); err != nil { + t.Fatal(err) + } + + if value := environmentValue(pod.Spec.Containers[0].Env, "PROJECT_SIGNAL_STAGE_NAME"); value != config.RuntimeStageName { + t.Fatalf("Pod runtime stage = %q", value) + } + + config.BackendAPIVersion = "kubeflow.org/v2beta1" + config.BackendKind = "MPIJob" + config.Queue = "certification" + + jobs, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner()) + if err != nil { + t.Fatal(err) + } + + launcher, _, err := unstructured.NestedSlice( + jobs[0].Object, + "spec", + "mpiReplicaSpecs", + "Launcher", + "template", + "spec", + "containers", + ) + if err != nil { + t.Fatal(err) + } + + worker, _, err := unstructured.NestedSlice( + jobs[0].Object, + "spec", + "mpiReplicaSpecs", + "Worker", + "template", + "spec", + "containers", + ) + if err != nil { + t.Fatal(err) + } + + launcherJSON, _ := json.Marshal(launcher) + workerJSON, _ := json.Marshal(worker) + + if strings.Contains(string(launcherJSON), "PROJECT_SIGNAL_STAGE_NAME") { + t.Fatalf("MPI launcher received runtime stage: %s", launcherJSON) + } + + if !strings.Contains(string(workerJSON), `"name":"PROJECT_SIGNAL_STAGE_NAME","value":"collective-bandwidth"`) { + t.Fatalf("MPI worker runtime stage missing: %s", workerJSON) + } + }) + if queue := resources[0].GetLabels()["kueue.x-k8s.io/queue-name"]; queue != "certification" { t.Fatalf("Pod queue label = %q", queue) } @@ -121,7 +208,7 @@ func TestRenderMPIJobsPerParticipant(t *testing.T) { t.Fatal(err) } - resources, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner(), 1200) + resources, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner()) if err != nil { t.Fatal(err) } @@ -150,7 +237,7 @@ func TestRenderMPIJobsPerParticipant(t *testing.T) { t.Fatal(err) } - if activeDeadline != 1200 { + if activeDeadline != config.BackendDeadlineSeconds { t.Fatalf("resource %d active deadline = %d", index, activeDeadline) } @@ -242,7 +329,7 @@ func TestRenderMPIJobUsesExplicitLifecycleOverrides(t *testing.T) { t.Fatal(err) } - resources, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner(), 1200) + resources, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner()) if err != nil { t.Fatal(err) } @@ -322,7 +409,7 @@ func TestRenderWorkloadPolicyOnParticipantContainers(t *testing.T) { t.Fatal(err) } - resources, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner(), 900) + resources, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner()) if err != nil { t.Fatal(err) } @@ -374,7 +461,7 @@ func TestMPIWorkloadPolicyDoesNotLeakOntoLauncher(t *testing.T) { t.Fatal(err) } - resources, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner(), 900) + resources, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner()) if err != nil { t.Fatal(err) } @@ -450,7 +537,6 @@ func TestRenderRejectsGPUResourceInWorkloadPolicy(t *testing.T) { tolerations, environment, testWorkflowOwner(), - 900, ); err == nil || !strings.Contains(err.Error(), "gpu-resource-count") { t.Fatalf("RenderResources error = %v", err) } @@ -531,6 +617,24 @@ func TestValidateConfigRequiresDigestAndMPIQueue(t *testing.T) { } } +func TestValidateConfigRequiresDeclaredRuntimeStage(t *testing.T) { + t.Parallel() + + config := testConfig() + config.ScriptRequiresStage = true + + if _, _, _, err := ValidateConfig(config); err == nil || + !strings.Contains(err.Error(), "runtime stage is required") { + t.Fatalf("ValidateConfig runtime stage error = %v", err) + } + + config.RuntimeStageName = "Collective/Bandwidth" + if _, _, _, err := ValidateConfig(config); err == nil || + !strings.Contains(err.Error(), "runtime stage name is invalid") { + t.Fatalf("ValidateConfig malformed stage error = %v", err) + } +} + func TestValidateConfigAcceptsControllerScriptDigest(t *testing.T) { t.Parallel() @@ -865,27 +969,28 @@ func testConfig() Config { } return Config{ - Namespace: "project-signal", - WorkflowName: "run-example-a1", - RunName: "run-example", - RunUID: types.UID("run-uid"), - ParticipantsJSON: string(participantsJSON), - ParticipantCount: 2, - RequestNodeName: "node-a", - LegacyParticipants: "node-a,node-b", - ScriptConfigMap: "run-example-scripts", - ScriptKey: "check.sh", - ScriptDigest: "sha256:" + strings.Repeat("b", 64), - RunnerImage: "example.invalid/runner@sha256:" + strings.Repeat("a", 64), - BackendAPIVersion: "kubeflow.org/v2beta1", - BackendKind: "MPIJob", - Queue: "certification", - TolerationsJSON: `[{"key":"nvidia.com/gpu","operator":"Exists","effect":"NoSchedule"}]`, - ScriptEnvironmentJSON: `{"CHECK_MODE":"quick"}`, - GPUResourceCount: "1", - Attempt: 1, - BackoffLimit: 0, - TTLSeconds: 3600, + Namespace: "project-signal", + WorkflowName: "run-example-a1", + RunName: "run-example", + RunUID: types.UID("run-uid"), + ParticipantsJSON: string(participantsJSON), + ParticipantCount: 2, + RequestNodeName: "node-a", + LegacyParticipants: "node-a,node-b", + ScriptConfigMap: "run-example-scripts", + ScriptKey: "check.sh", + ScriptDigest: "sha256:" + strings.Repeat("b", 64), + RunnerImage: "example.invalid/runner@sha256:" + strings.Repeat("a", 64), + BackendAPIVersion: "kubeflow.org/v2beta1", + BackendKind: "MPIJob", + BackendDeadlineSeconds: 540, + Queue: "certification", + TolerationsJSON: `[{"key":"nvidia.com/gpu","operator":"Exists","effect":"NoSchedule"}]`, + ScriptEnvironmentJSON: `{"CHECK_MODE":"quick"}`, + GPUResourceCount: "1", + Attempt: 1, + BackoffLimit: 0, + TTLSeconds: 3600, } } @@ -896,6 +1001,16 @@ func testParticipants() []signalv1.Participant { } } +func environmentValue(environment []corev1.EnvVar, name string) string { + for _, variable := range environment { + if variable.Name == name { + return variable.Value + } + } + + return "" +} + func testBackendResources() []signalv1.BackendResourceReference { return []signalv1.BackendResourceReference{ { diff --git a/internal/signaladapter/runner.go b/internal/signaladapter/runner.go index cadabce71b..fa7f9b8f31 100644 --- a/internal/signaladapter/runner.go +++ b/internal/signaladapter/runner.go @@ -73,6 +73,15 @@ func (r *Runner) Execute(ctx context.Context, config Config) (output AggregateOu return AggregateOutput{}, errors.New("owning Workflow requires a positive activeDeadlineSeconds") } + if err := signalv1.ValidateRuntimeContract( + config.ScriptRequiresStage, + runtimeStage(config.RuntimeStageName), + time.Duration(config.BackendDeadlineSeconds)*time.Second, + time.Duration(activeDeadlineSeconds)*time.Second, + ); err != nil { + return AggregateOutput{}, fmt.Errorf("validate backend and Workflow deadlines: %w", err) + } + executionContext, cancelExecution := context.WithTimeout( ctx, time.Duration(activeDeadlineSeconds)*time.Second, @@ -94,7 +103,6 @@ func (r *Runner) Execute(ctx context.Context, config Config) (output AggregateOu tolerations, environment, owner, - activeDeadlineSeconds, ) if err != nil { return AggregateOutput{}, err @@ -178,6 +186,14 @@ func (r *Runner) Execute(ctx context.Context, config Config) (output AggregateOu return output, nil } +func runtimeStage(name string) *signalv1.RuntimeStage { + if name == "" { + return nil + } + + return &signalv1.RuntimeStage{Name: name} +} + func (r *Runner) waitForResult(ctx context.Context, config Config, backend createdResource) Result { result := Result{ ParticipantIndex: int(backend.participantIndex), From f6c11877808a10ae59760149da4676e15e00dd22 Mon Sep 17 00:00:00 2001 From: chokevin Date: Wed, 21 Oct 2026 23:22:08 -0700 Subject: [PATCH 12/36] fix(signal): group deadline parameters --- api/signal/v1alpha1/runtime_contract.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/api/signal/v1alpha1/runtime_contract.go b/api/signal/v1alpha1/runtime_contract.go index 9b7af0c5b6..217eb1da9e 100644 --- a/api/signal/v1alpha1/runtime_contract.go +++ b/api/signal/v1alpha1/runtime_contract.go @@ -16,8 +16,7 @@ const MinimumBackendDeadlineReserve = time.Minute func ValidateRuntimeContract( scriptRequiresStage bool, stage *RuntimeStage, - backendDeadline time.Duration, - outerDeadline time.Duration, + backendDeadline, outerDeadline time.Duration, ) error { if err := ValidateRuntimeStage(scriptRequiresStage, stage); err != nil { return err @@ -40,7 +39,7 @@ func ValidateRuntimeStage(scriptRequiresStage bool, stage *RuntimeStage) error { return nil } -func ValidateExecutionDeadlines(backendDeadline time.Duration, outerDeadline time.Duration) error { +func ValidateExecutionDeadlines(backendDeadline, outerDeadline time.Duration) error { if backendDeadline <= 0 { return errors.New("backend active deadline must be positive") } From 756a449904ab98b452bd84d5ddf819cbe08586f8 Mon Sep 17 00:00:00 2001 From: chokevin Date: Thu, 22 Oct 2026 00:57:35 -0700 Subject: [PATCH 13/36] fix(signal): guard optional immutable fields --- .github/workflows/ci.yaml | 17 +++ Makefile | 13 +- .../v1alpha1/certificationrun_envtest_test.go | 132 ++++++++++++++++++ api/signal/v1alpha1/certificationrun_types.go | 14 +- ....unbounded-cloud.io_certificationruns.yaml | 24 ++-- deploy/signal/manifest_test.go | 30 ++++ 6 files changed, 212 insertions(+), 18 deletions(-) create mode 100644 api/signal/v1alpha1/certificationrun_envtest_test.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5e0288eb77..38959183b7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -391,6 +391,23 @@ jobs: - name: Provision envtest and run controller API-server race tests run: timeout --signal=TERM --kill-after=10s 300s make racer-envtest-ci + signal-envtest: + name: Project Signal API Server Integration Tests + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache-dependency-path: go.sum + + - name: Provision envtest and run Project Signal API-server race tests + run: timeout --signal=TERM --kill-after=10s 300s make signal-envtest-ci + # ---------- Orca Integration Tests ---------- # Spins up Garage and Azurite via testcontainers-go and runs the # orca in-process integration suite (internal/orca/inttest). Docker diff --git a/Makefile b/Makefile index 31e78414f0..60d661cdb2 100644 --- a/Makefile +++ b/Makefile @@ -244,7 +244,7 @@ NET_FRONTEND_CACHE_FILE := $(NET_FRONTEND_DIST_DIR)/.frontend-build-key # Frontend build toggle (dev builds produce unminified output with sourcemaps). REACT_DEV ?= false -.PHONY: all help fmt lint lint-actions test build vulncheck check-deps kubectl-unbounded kubectl-unbounded-build install-tools install-protoc install-helm generate kubectl-unbounded forge relctl relctl-build agent-artifacts-builder agent-artifacts-builder-build orcadev unbounded-agent machina machina-build machina-oci machina-oci-push machina-manifests machine-ops-controller machine-ops-controller-build machine-ops-controller-oci machine-ops-controller-oci-push machine-ops-manifests project-signal-controller project-signal-controller-build project-signal-adapter project-signal-adapter-build signal-manifests image-project-signal-controller-local image-project-signal-adapter-local metalman metalman-build metalman-oci metalman-oci-push unbounded-operator unbounded-operator-build unbounded-operator-manifests playpen-manifests e2e-gantry e2e-playpen e2e-racer gomod docs-serve unbounded-net-controller unbounded-net-controller-build unbounded-net-node unbounded-net-node-build unbounded-net-routeplan-debug unping unping-build unroute unroute-build license-check notice notice-check gantry gantry-build gantry-manifests inventory-manifests +.PHONY: all help fmt lint lint-actions test build vulncheck check-deps kubectl-unbounded kubectl-unbounded-build install-tools install-protoc install-helm generate kubectl-unbounded forge relctl relctl-build agent-artifacts-builder agent-artifacts-builder-build orcadev unbounded-agent machina machina-build machina-oci machina-oci-push machina-manifests machine-ops-controller machine-ops-controller-build machine-ops-controller-oci machine-ops-controller-oci-push machine-ops-manifests project-signal-controller project-signal-controller-build project-signal-adapter project-signal-adapter-build signal-envtest signal-envtest-ci signal-manifests image-project-signal-controller-local image-project-signal-adapter-local metalman metalman-build metalman-oci metalman-oci-push unbounded-operator unbounded-operator-build unbounded-operator-manifests playpen-manifests e2e-gantry e2e-playpen e2e-racer gomod docs-serve unbounded-net-controller unbounded-net-controller-build unbounded-net-node unbounded-net-node-build unbounded-net-routeplan-debug unping unping-build unroute unroute-build license-check notice notice-check gantry gantry-build gantry-manifests inventory-manifests .PHONY: net-frontend net-frontend-clean net-ebpf-build net-ebpf-generate net-ebpf-verify net-manifests gantry-chart-lint gantry-chart-package release-bom release-manifests unbounded-operator-release-manifest .PHONY: image-machina-local image-token-refresher-local image-machine-ops-controller-local image-metalman-local image-unbounded-operator-local image-unbounded-operator-push image-playpen-local image-net-controller-local image-net-node-local image-gantry-local image-gantry-push images-local .PHONY: image-net-controller-push image-net-node-push images-net-all images-net-all-push @@ -305,6 +305,7 @@ help: ## Show this help @echo " machine-ops-controller Build machine-ops-controller" @echo " project-signal-controller Build Project Signal controller" @echo " project-signal-adapter Build Project Signal participant adapter" + @echo " signal-envtest-ci Provision and run Project Signal API-server tests" @echo " signal-manifests Render Project Signal manifests" @echo " metalman | metalman-build Build metalman controller (with/without lint/test)" @echo " unbounded-operator | unbounded-operator-build Build the top-level Site operator" @@ -615,6 +616,16 @@ racer-envtest: ## Run real API-server, manager election, TLS and crash-recovery @mkdir -p tmp/racer-envtest TMPDIR="$(CURDIR)/tmp/racer-envtest" KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" timeout --signal=TERM --kill-after=10s 300s $(GOTEST) -race ./api/racer/v1alpha1 ./internal/racer ./internal/racer/authority ./deploy/racer ./internal/operator/components/racer -run '^TestEnvtest' -count=1 -v -timeout=5m +signal-envtest-ci: $(SETUP_ENVTEST) ## Provision pinned local API-server assets and require Project Signal envtest + @mkdir -p tmp/signal-envtest + @assets=$$(TMPDIR="$(CURDIR)/tmp/signal-envtest" timeout --signal=TERM --kill-after=10s 300s "$(SETUP_ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(CURDIR)/bin/envtest" -p path) && \ + $(MAKE) signal-envtest KUBEBUILDER_ASSETS="$$assets" + +signal-envtest: ## Run Project Signal API-server validation tests + @test -n "$(KUBEBUILDER_ASSETS)" || { echo "Set KUBEBUILDER_ASSETS to repository-local envtest binaries"; exit 1; } + @mkdir -p tmp/signal-envtest + TMPDIR="$(CURDIR)/tmp/signal-envtest" KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" timeout --signal=TERM --kill-after=10s 300s $(GOTEST) -race ./api/signal/v1alpha1 -run '^TestEnvtest' -count=1 -v -timeout=5m + racer-generate: ## Generate Racer deepcopy and CRD artifacts timeout --signal=TERM --kill-after=10s 300s $(GOCMD) generate ./api/racer/v1alpha1 diff --git a/api/signal/v1alpha1/certificationrun_envtest_test.go b/api/signal/v1alpha1/certificationrun_envtest_test.go new file mode 100644 index 0000000000..c4e9fab71e --- /dev/null +++ b/api/signal/v1alpha1/certificationrun_envtest_test.go @@ -0,0 +1,132 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + "os" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +func TestEnvtestCertificationRunOptionalImmutability(t *testing.T) { + assets := os.Getenv("KUBEBUILDER_ASSETS") + if assets == "" { + t.Skip("set KUBEBUILDER_ASSETS for real API validation tests") + } + + environment := &envtest.Environment{ + BinaryAssetsDirectory: assets, + CRDDirectoryPaths: []string{"../../../deploy/signal/crd"}, + ErrorIfCRDPathMissing: true, + AttachControlPlaneOutput: false, + } + restConfig, err := environment.Start() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, environment.Stop()) }) + + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, AddToScheme(scheme)) + + apiClient, err := client.New(restConfig, client.Options{Scheme: scheme}) + require.NoError(t, err) + require.NoError(t, apiClient.Create(t.Context(), &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "signal"}, + })) + + createRun := func(name string, stage *RuntimeStage) *CertificationRun { + t.Helper() + + run := validEnvtestCertificationRun(name) + run.Spec.RuntimeStage = stage + require.NoError(t, apiClient.Create(t.Context(), run)) + + return run + } + + t.Run("omitted runtime stage allows terminal status update", func(t *testing.T) { + run := createRun("runtime-stage-absent", nil) + run.Status.Phase = RunPhaseInconclusive + run.Status.ExecutionState = "Completed" + run.Status.Verdict = "Inconclusive" + run.Status.Reason = "DiagnosticUnavailable" + require.NoError(t, apiClient.Status().Update(t.Context(), run)) + }) + + t.Run("present equal remains valid", func(t *testing.T) { + run := createRun("runtime-stage-equal", &RuntimeStage{Name: "collective-bandwidth"}) + run.Spec.Cancel = true + require.NoError(t, apiClient.Update(t.Context(), run)) + }) + + for name, test := range map[string]struct { + initial *RuntimeStage + updated *RuntimeStage + }{ + "absent to present": { + updated: &RuntimeStage{Name: "collective-bandwidth"}, + }, + "present to absent": { + initial: &RuntimeStage{Name: "collective-bandwidth"}, + }, + "present unequal": { + initial: &RuntimeStage{Name: "collective-bandwidth"}, + updated: &RuntimeStage{Name: "memory-bandwidth"}, + }, + } { + t.Run(name+" is rejected", func(t *testing.T) { + run := createRun("runtime-stage-"+strings.ReplaceAll(name, " ", "-"), test.initial) + run.Spec.RuntimeStage = test.updated + + err := apiClient.Update(t.Context(), run) + require.True(t, apierrors.IsInvalid(err), "%v", err) + require.ErrorContains(t, err, "runtimeStage is immutable") + }) + } +} + +func validEnvtestCertificationRun(name string) *CertificationRun { + return &CertificationRun{ + TypeMeta: metav1.TypeMeta{APIVersion: GroupVersion.String(), Kind: "CertificationRun"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "signal", + }, + Spec: CertificationRunSpec{ + Profile: PinnedProfile{ + Name: "profile", + UID: types.UID("profile-uid"), + Generation: 1, + }, + Participants: []Participant{{ + Index: 0, + NodeName: "node-a", + NodeUID: types.UID("node-a-uid"), + }}, + Script: ScriptSnapshot{ + ConfigMapName: "scripts", + Key: "check.sh", + Digest: "sha256:" + strings.Repeat("a", 64), + }, + Workflow: WorkflowReference{Name: "workflow"}, + Backend: ExecutionBackend{APIVersion: "v1", Kind: "Pod"}, + RunnerImage: "example.invalid/runner@sha256:" + strings.Repeat("b", 64), + EvidencePath: "signal/" + name + ".json", + Deadline: metav1.Duration{Duration: 2 * time.Minute}, + BackendActiveDeadline: metav1.Duration{Duration: time.Minute}, + MaxAttempts: 1, + RequestNodeName: "node-a", + }, + } +} diff --git a/api/signal/v1alpha1/certificationrun_types.go b/api/signal/v1alpha1/certificationrun_types.go index ae4892b2b2..069922d751 100644 --- a/api/signal/v1alpha1/certificationrun_types.go +++ b/api/signal/v1alpha1/certificationrun_types.go @@ -70,14 +70,16 @@ type ScriptSnapshot struct { // +kubebuilder:validation:XValidation:rule="self.workflow == oldSelf.workflow",message="workflow is immutable" // +kubebuilder:validation:XValidation:rule="self.backend == oldSelf.backend",message="backend is immutable" // +kubebuilder:validation:XValidation:rule="self.runnerImage == oldSelf.runnerImage",message="runnerImage is immutable" -// +kubebuilder:validation:XValidation:rule="self.workloadPolicy == oldSelf.workloadPolicy",message="workloadPolicy is immutable" -// +kubebuilder:validation:XValidation:rule="self.runtimeStage == oldSelf.runtimeStage",message="runtimeStage is immutable" -// +kubebuilder:validation:XValidation:rule="self.parameters == oldSelf.parameters",message="parameters are immutable" +// +kubebuilder:validation:XValidation:rule="has(self.workloadPolicy) == has(oldSelf.workloadPolicy) && (!has(self.workloadPolicy) || self.workloadPolicy == oldSelf.workloadPolicy)",message="workloadPolicy is immutable" +// +kubebuilder:validation:XValidation:rule="has(self.runtimeStage) == has(oldSelf.runtimeStage) && (!has(self.runtimeStage) || self.runtimeStage == oldSelf.runtimeStage)",message="runtimeStage is immutable" +// +kubebuilder:validation:XValidation:rule="has(self.parameters) == has(oldSelf.parameters) && (!has(self.parameters) || self.parameters == oldSelf.parameters)",message="parameters are immutable" +// +kubebuilder:validation:XValidation:rule="has(self.purpose) == has(oldSelf.purpose) && (!has(self.purpose) || self.purpose == oldSelf.purpose)",message="purpose is immutable" +// +kubebuilder:validation:XValidation:rule="has(self.stagePlan) == has(oldSelf.stagePlan) && (!has(self.stagePlan) || self.stagePlan == oldSelf.stagePlan)",message="stagePlan is immutable" // +kubebuilder:validation:XValidation:rule="self.evidencePath == oldSelf.evidencePath",message="evidencePath is immutable" // +kubebuilder:validation:XValidation:rule="self.deadline == oldSelf.deadline",message="deadline is immutable" // +kubebuilder:validation:XValidation:rule="self.backendActiveDeadline == oldSelf.backendActiveDeadline",message="backendActiveDeadline is immutable" // +kubebuilder:validation:XValidation:rule="self.maxAttempts == oldSelf.maxAttempts",message="maxAttempts is immutable" -// +kubebuilder:validation:XValidation:rule="self.tolerations == oldSelf.tolerations",message="tolerations are immutable" +// +kubebuilder:validation:XValidation:rule="has(self.tolerations) == has(oldSelf.tolerations) && (!has(self.tolerations) || self.tolerations == oldSelf.tolerations)",message="tolerations are immutable" // +kubebuilder:validation:XValidation:rule="self.requestNodeName == oldSelf.requestNodeName",message="requestNodeName is immutable" // +kubebuilder:validation:XValidation:rule="duration(self.backendActiveDeadline) > duration('0s')",message="backendActiveDeadline must be positive" // +kubebuilder:validation:XValidation:rule="duration(self.deadline) - duration(self.backendActiveDeadline) >= duration('60s')",message="backendActiveDeadline must leave at least 60 seconds before deadline" @@ -93,9 +95,7 @@ type CertificationRunSpec struct { RunnerImage string `json:"runnerImage"` WorkloadPolicy WorkloadPolicy `json:"workloadPolicy,omitempty"` // +kubebuilder:validation:Enum=FullCertification;QuickCertification;Preflight;TargetedVerification;Diagnostic - // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="purpose is immutable" - Purpose CertificationPurpose `json:"purpose,omitempty"` - // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="stagePlan is immutable" + Purpose CertificationPurpose `json:"purpose,omitempty"` StagePlan []CertificationStage `json:"stagePlan,omitempty"` RuntimeStage *RuntimeStage `json:"runtimeStage,omitempty"` Parameters map[string]string `json:"parameters,omitempty"` diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml index 30213a6d40..05d7f0912d 100644 --- a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml @@ -153,9 +153,6 @@ spec: - TargetedVerification - Diagnostic type: string - x-kubernetes-validations: - - message: purpose is immutable - rule: self == oldSelf requestNodeName: type: string runnerImage: @@ -212,9 +209,6 @@ spec: - required type: object type: array - x-kubernetes-validations: - - message: stagePlan is immutable - rule: self == oldSelf tolerations: items: description: |- @@ -693,11 +687,20 @@ spec: - message: runnerImage is immutable rule: self.runnerImage == oldSelf.runnerImage - message: workloadPolicy is immutable - rule: self.workloadPolicy == oldSelf.workloadPolicy + rule: has(self.workloadPolicy) == has(oldSelf.workloadPolicy) && (!has(self.workloadPolicy) + || self.workloadPolicy == oldSelf.workloadPolicy) - message: runtimeStage is immutable - rule: self.runtimeStage == oldSelf.runtimeStage + rule: has(self.runtimeStage) == has(oldSelf.runtimeStage) && (!has(self.runtimeStage) + || self.runtimeStage == oldSelf.runtimeStage) - message: parameters are immutable - rule: self.parameters == oldSelf.parameters + rule: has(self.parameters) == has(oldSelf.parameters) && (!has(self.parameters) + || self.parameters == oldSelf.parameters) + - message: purpose is immutable + rule: has(self.purpose) == has(oldSelf.purpose) && (!has(self.purpose) + || self.purpose == oldSelf.purpose) + - message: stagePlan is immutable + rule: has(self.stagePlan) == has(oldSelf.stagePlan) && (!has(self.stagePlan) + || self.stagePlan == oldSelf.stagePlan) - message: evidencePath is immutable rule: self.evidencePath == oldSelf.evidencePath - message: deadline is immutable @@ -707,7 +710,8 @@ spec: - message: maxAttempts is immutable rule: self.maxAttempts == oldSelf.maxAttempts - message: tolerations are immutable - rule: self.tolerations == oldSelf.tolerations + rule: has(self.tolerations) == has(oldSelf.tolerations) && (!has(self.tolerations) + || self.tolerations == oldSelf.tolerations) - message: requestNodeName is immutable rule: self.requestNodeName == oldSelf.requestNodeName - message: backendActiveDeadline must be positive diff --git a/deploy/signal/manifest_test.go b/deploy/signal/manifest_test.go index a4ed548146..38a66385f0 100644 --- a/deploy/signal/manifest_test.go +++ b/deploy/signal/manifest_test.go @@ -114,6 +114,36 @@ func TestSignalCRDsExposeRuntimeStageAndDeadlineContract(t *testing.T) { } } +func TestCertificationRunCRDUsesPresenceAwareOptionalImmutability(t *testing.T) { + t.Parallel() + + content, err := os.ReadFile("crd/signal.unbounded-cloud.io_certificationruns.yaml") + if err != nil { + t.Fatal(err) + } + + manifest := string(content) + optionalImmutableFields := []string{ + "workloadPolicy", + "runtimeStage", + "parameters", + "purpose", + "stagePlan", + "tolerations", + } + + for _, field := range optionalImmutableFields { + rule := "has(self." + field + ") == has(oldSelf." + field + ")" + if !strings.Contains(manifest, rule) { + t.Fatalf("CertificationRun CRD does not contain presence-aware rule %q", rule) + } + } + + if strings.Contains(manifest, "rule: self.runtimeStage == oldSelf.runtimeStage") { + t.Fatal("CertificationRun CRD still dereferences an omitted runtimeStage") + } +} + func TestCertificationRunCRDRequiresCanonicalDigests(t *testing.T) { t.Parallel() From b517deef3226370f539682e235c55f37b2c9045d Mon Sep 17 00:00:00 2001 From: chokevin Date: Thu, 22 Oct 2026 01:47:32 -0700 Subject: [PATCH 14/36] fix(signal): preserve backend failure through cleanup --- cmd/project-signal-adapter/main.go | 8 +++ cmd/project-signal-adapter/main_test.go | 53 +++++++++++++++++++ .../04-adapter-workflowtemplate.yaml.tmpl | 3 ++ deploy/signal/README.md | 5 ++ deploy/signal/manifest_test.go | 2 + internal/signal/controller.go | 1 + internal/signal/controller_test.go | 3 ++ internal/signaladapter/adapter.go | 21 +++++--- internal/signaladapter/adapter_test.go | 2 +- internal/signaladapter/runner.go | 14 +++-- internal/signaladapter/runner_test.go | 46 ++++++++++++++++ 11 files changed, 147 insertions(+), 11 deletions(-) create mode 100644 cmd/project-signal-adapter/main_test.go diff --git a/cmd/project-signal-adapter/main.go b/cmd/project-signal-adapter/main.go index f1378e87b3..f3ecc5bffb 100644 --- a/cmd/project-signal-adapter/main.go +++ b/cmd/project-signal-adapter/main.go @@ -131,6 +131,13 @@ func writeOutputs(directory string, output signaladapter.AggregateOutput) error return errors.New("output-directory is required") } + if len(output.CleanupFailure) > signaladapter.MaxCleanupFailureBytes { + return fmt.Errorf( + "cleanup failure exceeds %d bytes", + signaladapter.MaxCleanupFailureBytes, + ) + } + if err := os.MkdirAll(directory, 0o750); err != nil { return fmt.Errorf("create output directory: %w", err) } @@ -138,6 +145,7 @@ func writeOutputs(directory string, output signaladapter.AggregateOutput) error outputs := map[string]string{ "verdict": output.Verdict, "reason": output.Reason, + "cleanup-failure": output.CleanupFailure, "measurements": output.Measurements, "backend-resources": output.BackendJSON, } diff --git a/cmd/project-signal-adapter/main_test.go b/cmd/project-signal-adapter/main_test.go new file mode 100644 index 0000000000..8895877501 --- /dev/null +++ b/cmd/project-signal-adapter/main_test.go @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/Azure/unbounded/internal/signaladapter" +) + +func TestWriteOutputsProjectsCleanupFailure(t *testing.T) { + t.Parallel() + + directory := t.TempDir() + + output := signaladapter.AggregateOutput{ + ExecutionState: "Error", + Verdict: "Inconclusive", + Reason: "BackendExecutionFailed", + CleanupFailure: "BackendCleanupFailed", + } + if err := writeOutputs(directory, output); err != nil { + t.Fatal(err) + } + + for name, want := range map[string]string{ + "reason": "BackendExecutionFailed", + "cleanup-failure": "BackendCleanupFailed", + } { + got, err := os.ReadFile(filepath.Join(directory, name)) + if err != nil { + t.Fatal(err) + } + + if string(got) != want { + t.Fatalf("%s = %q, want %q", name, got, want) + } + } +} + +func TestWriteOutputsRejectsOversizedCleanupFailure(t *testing.T) { + t.Parallel() + + err := writeOutputs(t.TempDir(), signaladapter.AggregateOutput{ + CleanupFailure: string(make([]byte, signaladapter.MaxCleanupFailureBytes+1)), + }) + if err == nil { + t.Fatal("writeOutputs accepted oversized cleanup failure") + } +} diff --git a/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl b/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl index 3dc673d73f..52e6adafae 100644 --- a/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl +++ b/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl @@ -122,6 +122,9 @@ spec: - name: reason valueFrom: path: /outputs/reason + - name: cleanup-failure + valueFrom: + path: /outputs/cleanup-failure - name: measurements valueFrom: path: /outputs/measurements diff --git a/deploy/signal/README.md b/deploy/signal/README.md index 288aedf5ca..cf6f691c58 100644 --- a/deploy/signal/README.md +++ b/deploy/signal/README.md @@ -304,6 +304,11 @@ executionState=Error, verdict=Inconclusive, reason= Ambiguous or incomplete workflow execution remains `Inconclusive`. The controller does not infer a successful result from partial evidence. +If backend execution and cleanup both fail, `reason` retains the first causal +backend failure and the bounded `cleanup-failure` Workflow output records +`BackendCleanupFailed` separately. Cleanup failure always forces +`executionState=Error` and `verdict=Inconclusive`, including after an otherwise +successful diagnostic. Cluster-local evidence consists of CertificationRun status, the referenced Workflow and backend status, Events, pod logs, termination messages, and the diff --git a/deploy/signal/manifest_test.go b/deploy/signal/manifest_test.go index 38a66385f0..b58dbf977e 100644 --- a/deploy/signal/manifest_test.go +++ b/deploy/signal/manifest_test.go @@ -62,6 +62,8 @@ func TestAdapterManifestIsGenericAndParticipantAware(t *testing.T) { "participant-count", "request-node-name", "backend-resources", + "cleanup-failure", + "/outputs/cleanup-failure", "backend-api-version", "backend-kind", "backend-active-deadline-seconds", diff --git a/internal/signal/controller.go b/internal/signal/controller.go index 086d7718a0..fcfd73a911 100644 --- a/internal/signal/controller.go +++ b/internal/signal/controller.go @@ -74,6 +74,7 @@ var reservedScriptEnvironmentNames = map[string]struct{}{ var workflowOutputLimits = map[string]int{ "backend-resources": 16 * 1024, + "cleanup-failure": 64, "measurements": 16 * 1024, "reason": 4 * 1024, "verdict": 32, diff --git a/internal/signal/controller_test.go b/internal/signal/controller_test.go index 06a09fae86..66be858a95 100644 --- a/internal/signal/controller_test.go +++ b/internal/signal/controller_test.go @@ -903,6 +903,9 @@ func TestWorkflowParametersRejectMalformedEvidence(t *testing.T) { "oversized": { map[string]any{"name": "measurements", "value": strings.Repeat("x", 16*1024+1)}, }, + "oversized cleanup failure": { + map[string]any{"name": "cleanup-failure", "value": strings.Repeat("x", 65)}, + }, "wrong type": { map[string]any{"name": "verdict", "value": true}, }, diff --git a/internal/signaladapter/adapter.go b/internal/signaladapter/adapter.go index 75ceaf1178..1447e671e1 100644 --- a/internal/signaladapter/adapter.go +++ b/internal/signaladapter/adapter.go @@ -27,12 +27,13 @@ import ( ) const ( - MaxParticipants = 64 - MaxReasonBytes = 4 * 1024 - MaxMeasurementsBytes = 16 * 1024 - MaxBackendOutputBytes = 16 * 1024 - resultPrefix = "PROJECT_SIGNAL_RESULT_V2 " - requiredTopologyKey = "kueue.x-k8s.io/podset-required-topology" + MaxParticipants = 64 + MaxReasonBytes = 4 * 1024 + MaxMeasurementsBytes = 16 * 1024 + MaxBackendOutputBytes = 16 * 1024 + MaxCleanupFailureBytes = 64 + resultPrefix = "PROJECT_SIGNAL_RESULT_V2 " + requiredTopologyKey = "kueue.x-k8s.io/podset-required-topology" ) var runnerImagePattern = regexp.MustCompile(`^[^@\s]+@sha256:[0-9a-f]{64}$`) @@ -96,6 +97,7 @@ type AggregateOutput struct { ExecutionState string `json:"executionState"` Verdict string `json:"verdict"` Reason string `json:"reason"` + CleanupFailure string `json:"cleanupFailure,omitempty"` Measurements string `json:"measurements"` BackendResources []signalv1.BackendResourceReference `json:"-"` BackendJSON string `json:"backend-resources"` @@ -490,6 +492,7 @@ func Aggregate( ordered := make([]Result, participantCount) seen := make([]bool, participantCount) executionFailed := false + executionFailureReason := "" diagnosticFailed := false allPassed := true @@ -525,6 +528,10 @@ func Aggregate( if result.ExecutionState != "Succeeded" { executionFailed = true + + if executionFailureReason == "" { + executionFailureReason = result.Reason + } } if len(result.Measurements) > MaxMeasurementsBytes { @@ -573,7 +580,7 @@ func Aggregate( switch { case executionFailed: output.ExecutionState = "Error" - output.Reason = "ParticipantCoverageIncomplete" + output.Reason = executionFailureReason case diagnosticFailed: output.Verdict = "Fail" output.Reason = "ParticipantCertificationFailed" diff --git a/internal/signaladapter/adapter_test.go b/internal/signaladapter/adapter_test.go index 369b3b91eb..3f6eff87ce 100644 --- a/internal/signaladapter/adapter_test.go +++ b/internal/signaladapter/adapter_test.go @@ -730,7 +730,7 @@ func TestAggregateBackendFailurePrecedesDiagnosticFailure(t *testing.T) { if output.ExecutionState != "Error" || output.Verdict != "Inconclusive" || - output.Reason != "ParticipantCoverageIncomplete" { + output.Reason != "BackendFailed" { t.Fatalf("output = %#v", output) } } diff --git a/internal/signaladapter/runner.go b/internal/signaladapter/runner.go index fa7f9b8f31..2689cdf415 100644 --- a/internal/signaladapter/runner.go +++ b/internal/signaladapter/runner.go @@ -174,9 +174,7 @@ func (r *Runner) Execute(ctx context.Context, config Config) (output AggregateOu defer cancel() if err := r.cleanup(cleanupContext, config.Namespace, created); err != nil { - output.ExecutionState = "Error" - output.Verdict = "Inconclusive" - output.Reason = "BackendCleanupFailed" + recordCleanupFailure(&output) return output, nil } @@ -186,6 +184,16 @@ func (r *Runner) Execute(ctx context.Context, config Config) (output AggregateOu return output, nil } +func recordCleanupFailure(output *AggregateOutput) { + output.ExecutionState = "Error" + output.Verdict = "Inconclusive" + output.CleanupFailure = "BackendCleanupFailed" + + if output.Reason == "" || output.Reason == "AllParticipantsPassed" { + output.Reason = output.CleanupFailure + } +} + func runtimeStage(name string) *signalv1.RuntimeStage { if name == "" { return nil diff --git a/internal/signaladapter/runner_test.go b/internal/signaladapter/runner_test.go index 834e5f6102..709c108030 100644 --- a/internal/signaladapter/runner_test.go +++ b/internal/signaladapter/runner_test.go @@ -92,6 +92,52 @@ func TestBackendCompletionResultFailsClosedWithoutValidResult(t *testing.T) { } } +func TestCleanupFailurePreservesExecutionFailure(t *testing.T) { + t.Parallel() + + output, err := Aggregate(1, testBackendResources()[:1], []Result{{ + ParticipantIndex: 0, + ExecutionState: "Error", + Verdict: "Inconclusive", + Reason: "BackendExecutionFailed", + }}) + if err != nil { + t.Fatal(err) + } + + recordCleanupFailure(&output) + + if output.ExecutionState != "Error" || + output.Verdict != "Inconclusive" || + output.Reason != "BackendExecutionFailed" || + output.CleanupFailure != "BackendCleanupFailed" { + t.Fatalf("output = %#v", output) + } +} + +func TestCleanupFailurePreventsPass(t *testing.T) { + t.Parallel() + + output, err := Aggregate(1, testBackendResources()[:1], []Result{{ + ParticipantIndex: 0, + ExecutionState: "Succeeded", + Verdict: "Pass", + Reason: "Healthy", + }}) + if err != nil { + t.Fatal(err) + } + + recordCleanupFailure(&output) + + if output.ExecutionState != "Error" || + output.Verdict != "Inconclusive" || + output.Reason != "BackendCleanupFailed" || + output.CleanupFailure != "BackendCleanupFailed" { + t.Fatalf("output = %#v", output) + } +} + func TestCompletedResultRejectsTerminalStateMismatch(t *testing.T) { t.Parallel() From 59eb0d57cbe825ae5a5fddb8d6a5df93dd551706 Mon Sep 17 00:00:00 2001 From: chokevin Date: Thu, 22 Oct 2026 01:59:59 -0700 Subject: [PATCH 15/36] fix(signal): fence MPI worker readiness --- deploy/signal/README.md | 11 +++ internal/signaladapter/adapter.go | 89 +++++++++++++++-- internal/signaladapter/adapter_test.go | 129 +++++++++++++++++++++++++ 3 files changed, 223 insertions(+), 6 deletions(-) diff --git a/deploy/signal/README.md b/deploy/signal/README.md index cf6f691c58..77006d5f92 100644 --- a/deploy/signal/README.md +++ b/deploy/signal/README.md @@ -133,6 +133,17 @@ contradictory result/exit combinations, and any other exit code. Partial backend execution, identity gaps, deadline interruption, and cleanup failure also remain `Inconclusive`. +The MPI launcher also applies a compatibility fence for +[`kubeflow/mpi-operator#615`](https://github.com/kubeflow/mpi-operator/issues/615), +whose suspend/resume race is fixed upstream by +[`kubeflow/mpi-operator#868`](https://github.com/kubeflow/mpi-operator/pull/868) +at commit `3a1e56d8accd0d924703d79af3149e3fa8ba1b01`. Before invoking the diagnostic +launcher command, it waits for every host in `/etc/mpi/hostfile` to resolve and +accept TCP port 22. The wait uses Bash only, is capped at 120 seconds, and is +always at least one second shorter than `backendActiveDeadline`. Timeout emits +a bounded, actionable stderr message and fails closed without changing +launcher placement, resources, or participant-only policy isolation. + Other backend kinds remain operator-supplied and are not installed or granted permissions by the included adapter. diff --git a/internal/signaladapter/adapter.go b/internal/signaladapter/adapter.go index 1447e671e1..1cdcd9497d 100644 --- a/internal/signaladapter/adapter.go +++ b/internal/signaladapter/adapter.go @@ -32,6 +32,7 @@ const ( MaxMeasurementsBytes = 16 * 1024 MaxBackendOutputBytes = 16 * 1024 MaxCleanupFailureBytes = 64 + maxMPIWorkerReadyWait = int64(120) resultPrefix = "PROJECT_SIGNAL_RESULT_V2 " requiredTopologyKey = "kueue.x-k8s.io/podset-required-topology" ) @@ -253,6 +254,12 @@ func ValidateConfig(config Config) ([]signalv1.Participant, []corev1.Toleration, if config.Queue == "" { return nil, nil, nil, errors.New("queue is required for MPIJob execution") } + + if config.BackendDeadlineSeconds < 2 { + return nil, nil, nil, errors.New( + "backend-active-deadline-seconds must be at least 2 for MPIJob worker readiness", + ) + } default: return nil, nil, nil, fmt.Errorf( "unsupported backend %s %s", @@ -682,6 +689,10 @@ func validateResult(result Result) error { } func resultCommand(arguments ...string) []string { + return resultCommandWithPrelude("", arguments...) +} + +func resultCommandWithPrelude(prelude string, arguments ...string) []string { quotedArguments := make([]string, len(arguments)) for index, argument := range arguments { quotedArguments[index] = shellQuote(argument) @@ -696,8 +707,14 @@ mkdir -p /results chmod 0755 /results rm -f /results/verdict /results/reason /results/measurements.txt set +e -/bin/bash "/scripts/${PROJECT_SIGNAL_SCRIPT_KEY}" %s -status=$? +%s +prelude_status=$? +if [[ "${prelude_status}" -eq 0 ]]; then + /bin/bash "/scripts/${PROJECT_SIGNAL_SCRIPT_KEY}" %s + status=$? +else + status="${prelude_status}" +fi set -e encode_result_file() { if [[ -f "$1" ]]; then @@ -714,8 +731,63 @@ measurements="$(encode_result_file /results/measurements.txt)" [[ -n "${measurements}" ]] || measurements=- printf '%s%%s %%s %%s %%d\n' "${verdict}" "${reason}" "${measurements}" "$status" exit "$status" -`, strings.Join(quotedArguments, " "), resultPrefix), +`, prelude, strings.Join(quotedArguments, " "), resultPrefix), + } +} + +func mpiWorkerReadinessTimeout(backendDeadlineSeconds int64) int64 { + timeout := backendDeadlineSeconds - 1 + if timeout > maxMPIWorkerReadyWait { + timeout = maxMPIWorkerReadyWait } + + return timeout +} + +func mpiWorkerReadinessScript(hostfile string, port int, timeoutSeconds int64) string { + return fmt.Sprintf(` +wait_for_mpi_workers() { + local hostfile=%s + local port=%d + local timeout_seconds=%d + local deadline=$((SECONDS + timeout_seconds)) + local host ignored all_ready + local -a hosts + while (( SECONDS < deadline )); do + hosts=() + if [[ -r "${hostfile}" ]]; then + while IFS=$' \t' read -r host ignored; do + [[ -n "${host}" ]] || continue + if [[ ! "${host}" =~ ^[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?$ ]]; then + printf 'Project Signal MPI readiness rejected invalid worker host %%q in %%s\n' "${host}" "${hostfile}" >&2 + return 64 + fi + hosts+=("${host}") + done < "${hostfile}" + fi + if (( ${#hosts[@]} > 0 )); then + all_ready=1 + for host in "${hosts[@]}"; do + if exec 3<>"/dev/tcp/${host}/${port}" 2>/dev/null; then + exec 3>&- + exec 3<&- + else + all_ready=0 + break + fi + done + if (( all_ready == 1 )); then + return 0 + fi + fi + sleep 1 + done + printf 'Project Signal MPI readiness timed out after %%ds waiting for every worker in %%s to resolve and accept TCP/%%d\n' \ + "${timeout_seconds}" "${hostfile}" "${port}" >&2 + return 124 +} +wait_for_mpi_workers +`, shellQuote(hostfile), port, timeoutSeconds) } func ordinaryPod( @@ -784,6 +856,7 @@ func mpiJob( jobName := resourceName(config.RunName, participant.Index) launcherEnv := workloadEnvironment(config, participant, environment, false) workerEnv := workloadEnvironment(config, participant, environment, true) + readinessTimeout := mpiWorkerReadinessTimeout(backendDeadlineSeconds) worker := workloadContainer( config, policy, @@ -824,9 +897,13 @@ func mpiJob( nil, nil, corev1.Container{ - Name: "launcher", - Image: config.RunnerImage, - Command: resultCommand("mpi-launcher", strconv.FormatInt(gpuCount, 10)), + Name: "launcher", + Image: config.RunnerImage, + Command: resultCommandWithPrelude( + mpiWorkerReadinessScript("/etc/mpi/hostfile", 22, readinessTimeout), + "mpi-launcher", + strconv.FormatInt(gpuCount, 10), + ), Env: launcherEnv, VolumeMounts: resultVolumeMounts(), }, diff --git a/internal/signaladapter/adapter_test.go b/internal/signaladapter/adapter_test.go index 3f6eff87ce..f6a1cda790 100644 --- a/internal/signaladapter/adapter_test.go +++ b/internal/signaladapter/adapter_test.go @@ -4,10 +4,19 @@ package signaladapter import ( + "bytes" + "context" "encoding/base64" "encoding/json" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "strconv" "strings" "testing" + "time" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -312,9 +321,117 @@ func TestRenderMPIJobsPerParticipant(t *testing.T) { if strings.Contains(string(launcherJSON), "nvidia.com/gpu") { t.Fatalf("resource %d launcher requests GPUs: %s", index, launcherJSON) } + + if !strings.Contains(string(launcherJSON), "/etc/mpi/hostfile") || + !strings.Contains(string(launcherJSON), `/dev/tcp/${host}/${port}`) || + !strings.Contains(string(launcherJSON), "Project Signal MPI readiness timed out") { + t.Fatalf("resource %d launcher readiness fence missing: %s", index, launcherJSON) + } + } +} + +func TestMPIWorkerReadinessTimeoutIsBoundedByBackendDeadline(t *testing.T) { + t.Parallel() + + for _, backendDeadline := range []int64{2, 60, 121, 540} { + timeout := mpiWorkerReadinessTimeout(backendDeadline) + if timeout < 1 || timeout >= backendDeadline || timeout > maxMPIWorkerReadyWait { + t.Fatalf("mpiWorkerReadinessTimeout(%d) = %d", backendDeadline, timeout) + } } } +func TestMPIWorkerReadinessFenceWaitsForWorkers(t *testing.T) { + t.Parallel() + + hostfile := filepath.Join(t.TempDir(), "hostfile") + port := unusedTCPPort(t) + ready := make(chan error, 1) + + go func() { + time.Sleep(100 * time.Millisecond) + + if err := os.WriteFile(hostfile, []byte("127.0.0.1 slots=1\n"), 0o600); err != nil { + ready <- err + + return + } + + time.Sleep(1100 * time.Millisecond) + + listener, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port))) + if err != nil { + ready <- err + + return + } + defer listener.Close() + + connection, err := listener.Accept() + if err == nil { + err = connection.Close() + } + + ready <- err + }() + + ctx, cancel := context.WithTimeout(t.Context(), 8*time.Second) + defer cancel() + + command := exec.CommandContext(ctx, "/bin/bash", "-c", mpiWorkerReadinessScript(hostfile, port, 5)) + + var stderr bytes.Buffer + + command.Stderr = &stderr + + if err := command.Run(); err != nil { + t.Fatalf("readiness fence failed: %v stderr=%q", err, stderr.String()) + } + + if err := <-ready; err != nil { + t.Fatal(err) + } +} + +func TestMPIWorkerReadinessFenceTimesOut(t *testing.T) { + t.Parallel() + + hostfile := filepath.Join(t.TempDir(), "hostfile") + if err := os.WriteFile(hostfile, []byte("127.0.0.1 slots=1\n"), 0o600); err != nil { + t.Fatal(err) + } + + port := unusedTCPPort(t) + command := exec.Command("/bin/bash", "-c", mpiWorkerReadinessScript(hostfile, port, 2)) + + var stderr bytes.Buffer + + command.Stderr = &stderr + + err := command.Run() + if err == nil { + t.Fatal("readiness fence succeeded without a listening worker") + } + + if !strings.Contains(stderr.String(), "timed out after 2s") || + !strings.Contains(stderr.String(), hostfile) || + !strings.Contains(stderr.String(), fmt.Sprintf("TCP/%d", port)) { + t.Fatalf("readiness timeout stderr = %q", stderr.String()) + } +} + +func unusedTCPPort(t *testing.T) int { + t.Helper() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + + return listener.Addr().(*net.TCPAddr).Port +} + func TestRenderMPIJobUsesExplicitLifecycleOverrides(t *testing.T) { t.Parallel() @@ -520,6 +637,18 @@ func TestValidateConfigRejectsKueueTopologyWithoutQueue(t *testing.T) { } } +func TestValidateConfigRejectsMPIWorkerReadinessWithoutTimeBudget(t *testing.T) { + t.Parallel() + + config := testConfig() + config.BackendDeadlineSeconds = 1 + + if _, _, _, err := ValidateConfig(config); err == nil || + !strings.Contains(err.Error(), "at least 2 for MPIJob worker readiness") { + t.Fatalf("ValidateConfig deadline error = %v", err) + } +} + func TestRenderRejectsGPUResourceInWorkloadPolicy(t *testing.T) { t.Parallel() From b8f78f0bd66d9a9ab206b66dfd4f713ad8207007 Mon Sep 17 00:00:00 2001 From: chokevin Date: Thu, 22 Oct 2026 02:11:52 -0700 Subject: [PATCH 16/36] fix(signal): probe MPI workers over SSH --- deploy/signal/README.md | 9 ++-- internal/signaladapter/adapter.go | 30 +++++------ internal/signaladapter/adapter_test.go | 69 ++++++++++++++------------ 3 files changed, 57 insertions(+), 51 deletions(-) diff --git a/deploy/signal/README.md b/deploy/signal/README.md index 77006d5f92..be902af5ca 100644 --- a/deploy/signal/README.md +++ b/deploy/signal/README.md @@ -139,10 +139,11 @@ whose suspend/resume race is fixed upstream by [`kubeflow/mpi-operator#868`](https://github.com/kubeflow/mpi-operator/pull/868) at commit `3a1e56d8accd0d924703d79af3149e3fa8ba1b01`. Before invoking the diagnostic launcher command, it waits for every host in `/etc/mpi/hostfile` to resolve and -accept TCP port 22. The wait uses Bash only, is capped at 120 seconds, and is -always at least one second shorter than `backendActiveDeadline`. Timeout emits -a bounded, actionable stderr message and fails closed without changing -launcher placement, resources, or participant-only policy isolation. +complete a non-interactive SSH `/bin/true`. The wait is capped at 30 seconds +total and is always at least one second shorter than `backendActiveDeadline`. +Timeout emits an actionable stderr message naming the last unready host and +fails closed without changing launcher placement, resources, or +participant-only policy isolation. Other backend kinds remain operator-supplied and are not installed or granted permissions by the included adapter. diff --git a/internal/signaladapter/adapter.go b/internal/signaladapter/adapter.go index 1cdcd9497d..0587c06a5b 100644 --- a/internal/signaladapter/adapter.go +++ b/internal/signaladapter/adapter.go @@ -32,7 +32,7 @@ const ( MaxMeasurementsBytes = 16 * 1024 MaxBackendOutputBytes = 16 * 1024 MaxCleanupFailureBytes = 64 - maxMPIWorkerReadyWait = int64(120) + maxMPIWorkerReadyWait = int64(30) resultPrefix = "PROJECT_SIGNAL_RESULT_V2 " requiredTopologyKey = "kueue.x-k8s.io/podset-required-topology" ) @@ -744,35 +744,35 @@ func mpiWorkerReadinessTimeout(backendDeadlineSeconds int64) int64 { return timeout } -func mpiWorkerReadinessScript(hostfile string, port int, timeoutSeconds int64) string { +func mpiWorkerReadinessScript(hostfile string, timeoutSeconds int64) string { return fmt.Sprintf(` wait_for_mpi_workers() { local hostfile=%s - local port=%d local timeout_seconds=%d local deadline=$((SECONDS + timeout_seconds)) - local host ignored all_ready + local host all_ready last_unready remaining local -a hosts while (( SECONDS < deadline )); do hosts=() if [[ -r "${hostfile}" ]]; then - while IFS=$' \t' read -r host ignored; do - [[ -n "${host}" ]] || continue + while IFS= read -r host; do if [[ ! "${host}" =~ ^[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?$ ]]; then printf 'Project Signal MPI readiness rejected invalid worker host %%q in %%s\n' "${host}" "${hostfile}" >&2 return 64 fi hosts+=("${host}") - done < "${hostfile}" + done < <(/usr/bin/awk 'NF {print $1}' "${hostfile}") fi if (( ${#hosts[@]} > 0 )); then all_ready=1 for host in "${hosts[@]}"; do - if exec 3<>"/dev/tcp/${host}/${port}" 2>/dev/null; then - exec 3>&- - exec 3<&- - else + remaining=$((deadline - SECONDS)) + if (( remaining < 1 )) || ! timeout "${remaining}s" ssh \ + -o BatchMode=yes \ + -o ConnectTimeout=2 \ + -- "${host}" /bin/true /dev/null 2>&1; then all_ready=0 + last_unready="${host}" break fi done @@ -782,12 +782,12 @@ wait_for_mpi_workers() { fi sleep 1 done - printf 'Project Signal MPI readiness timed out after %%ds waiting for every worker in %%s to resolve and accept TCP/%%d\n' \ - "${timeout_seconds}" "${hostfile}" "${port}" >&2 + printf 'Project Signal MPI readiness timed out after %%ds waiting for every worker in %%s to resolve and accept SSH/22; last unready host=%%s\n' \ + "${timeout_seconds}" "${hostfile}" "${last_unready:-unknown}" >&2 return 124 } wait_for_mpi_workers -`, shellQuote(hostfile), port, timeoutSeconds) +`, shellQuote(hostfile), timeoutSeconds) } func ordinaryPod( @@ -900,7 +900,7 @@ func mpiJob( Name: "launcher", Image: config.RunnerImage, Command: resultCommandWithPrelude( - mpiWorkerReadinessScript("/etc/mpi/hostfile", 22, readinessTimeout), + mpiWorkerReadinessScript("/etc/mpi/hostfile", readinessTimeout), "mpi-launcher", strconv.FormatInt(gpuCount, 10), ), diff --git a/internal/signaladapter/adapter_test.go b/internal/signaladapter/adapter_test.go index f6a1cda790..d5bea1fefc 100644 --- a/internal/signaladapter/adapter_test.go +++ b/internal/signaladapter/adapter_test.go @@ -8,12 +8,9 @@ import ( "context" "encoding/base64" "encoding/json" - "fmt" - "net" "os" "os/exec" "path/filepath" - "strconv" "strings" "testing" "time" @@ -323,7 +320,8 @@ func TestRenderMPIJobsPerParticipant(t *testing.T) { } if !strings.Contains(string(launcherJSON), "/etc/mpi/hostfile") || - !strings.Contains(string(launcherJSON), `/dev/tcp/${host}/${port}`) || + !strings.Contains(string(launcherJSON), "BatchMode=yes") || + !strings.Contains(string(launcherJSON), "/bin/true") || !strings.Contains(string(launcherJSON), "Project Signal MPI readiness timed out") { t.Fatalf("resource %d launcher readiness fence missing: %s", index, launcherJSON) } @@ -344,14 +342,16 @@ func TestMPIWorkerReadinessTimeoutIsBoundedByBackendDeadline(t *testing.T) { func TestMPIWorkerReadinessFenceWaitsForWorkers(t *testing.T) { t.Parallel() - hostfile := filepath.Join(t.TempDir(), "hostfile") - port := unusedTCPPort(t) + directory := t.TempDir() + hostfile := filepath.Join(directory, "hostfile") + readyFile := filepath.Join(directory, "ready") + binDirectory := fakeSSH(t, directory) ready := make(chan error, 1) go func() { time.Sleep(100 * time.Millisecond) - if err := os.WriteFile(hostfile, []byte("127.0.0.1 slots=1\n"), 0o600); err != nil { + if err := os.WriteFile(hostfile, []byte("worker-0 slots=1\nworker-1 slots=1\n"), 0o600); err != nil { ready <- err return @@ -359,26 +359,19 @@ func TestMPIWorkerReadinessFenceWaitsForWorkers(t *testing.T) { time.Sleep(1100 * time.Millisecond) - listener, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port))) - if err != nil { - ready <- err - - return - } - defer listener.Close() - - connection, err := listener.Accept() - if err == nil { - err = connection.Close() - } - - ready <- err + ready <- os.WriteFile(readyFile, nil, 0o600) }() ctx, cancel := context.WithTimeout(t.Context(), 8*time.Second) defer cancel() - command := exec.CommandContext(ctx, "/bin/bash", "-c", mpiWorkerReadinessScript(hostfile, port, 5)) + command := exec.CommandContext(ctx, "/bin/bash", "-c", mpiWorkerReadinessScript(hostfile, 5)) + + command.Env = append( + os.Environ(), + "PATH="+binDirectory+":"+os.Getenv("PATH"), + "PROJECT_SIGNAL_TEST_SSH_READY="+readyFile, + ) var stderr bytes.Buffer @@ -396,13 +389,16 @@ func TestMPIWorkerReadinessFenceWaitsForWorkers(t *testing.T) { func TestMPIWorkerReadinessFenceTimesOut(t *testing.T) { t.Parallel() - hostfile := filepath.Join(t.TempDir(), "hostfile") - if err := os.WriteFile(hostfile, []byte("127.0.0.1 slots=1\n"), 0o600); err != nil { + directory := t.TempDir() + + hostfile := filepath.Join(directory, "hostfile") + if err := os.WriteFile(hostfile, []byte("worker-0 slots=1\n"), 0o600); err != nil { t.Fatal(err) } - port := unusedTCPPort(t) - command := exec.Command("/bin/bash", "-c", mpiWorkerReadinessScript(hostfile, port, 2)) + command := exec.Command("/bin/bash", "-c", mpiWorkerReadinessScript(hostfile, 2)) + + command.Env = append(os.Environ(), "PATH="+fakeSSH(t, directory)+":"+os.Getenv("PATH")) var stderr bytes.Buffer @@ -415,21 +411,30 @@ func TestMPIWorkerReadinessFenceTimesOut(t *testing.T) { if !strings.Contains(stderr.String(), "timed out after 2s") || !strings.Contains(stderr.String(), hostfile) || - !strings.Contains(stderr.String(), fmt.Sprintf("TCP/%d", port)) { + !strings.Contains(stderr.String(), "last unready host=worker-0") { t.Fatalf("readiness timeout stderr = %q", stderr.String()) } } -func unusedTCPPort(t *testing.T) int { +func fakeSSH(t *testing.T, directory string) string { t.Helper() - listener, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { + binDirectory := filepath.Join(directory, "bin") + if err := os.Mkdir(binDirectory, 0o750); err != nil { + t.Fatal(err) + } + + content := []byte(`#!/bin/bash +if [[ -n "${PROJECT_SIGNAL_TEST_SSH_READY:-}" && -f "${PROJECT_SIGNAL_TEST_SSH_READY}" ]]; then + exit 0 +fi +exit 255 +`) + if err := os.WriteFile(filepath.Join(binDirectory, "ssh"), content, 0o750); err != nil { t.Fatal(err) } - defer listener.Close() - return listener.Addr().(*net.TCPAddr).Port + return binDirectory } func TestRenderMPIJobUsesExplicitLifecycleOverrides(t *testing.T) { From f4259c83e56309ec6c111ff8fdb3bbfec6c1d0f0 Mon Sep 17 00:00:00 2001 From: chokevin Date: Thu, 22 Oct 2026 04:12:19 -0700 Subject: [PATCH 17/36] fix(signal): harden MPI launcher recovery --- deploy/signal/README.md | 30 ++- internal/signaladapter/adapter.go | 186 ++++++++++++++---- internal/signaladapter/adapter_test.go | 259 +++++++++++++++++++++++-- internal/signaladapter/runner.go | 248 +++++++++++++++++++++-- internal/signaladapter/runner_test.go | 151 ++++++++++++++ 5 files changed, 784 insertions(+), 90 deletions(-) diff --git a/deploy/signal/README.md b/deploy/signal/README.md index be902af5ca..6b1670d8f7 100644 --- a/deploy/signal/README.md +++ b/deploy/signal/README.md @@ -138,12 +138,30 @@ The MPI launcher also applies a compatibility fence for whose suspend/resume race is fixed upstream by [`kubeflow/mpi-operator#868`](https://github.com/kubeflow/mpi-operator/pull/868) at commit `3a1e56d8accd0d924703d79af3149e3fa8ba1b01`. Before invoking the diagnostic -launcher command, it waits for every host in `/etc/mpi/hostfile` to resolve and -complete a non-interactive SSH `/bin/true`. The wait is capped at 30 seconds -total and is always at least one second shorter than `backendActiveDeadline`. -Timeout emits an actionable stderr message naming the last unready host and -fails closed without changing launcher placement, resources, or -participant-only policy isolation. +launcher command, the adapter writes `/tmp/project-signal-hostfile` from the +operator's deterministic worker DNS contract +`-worker-...svc` and the validated +`slotsPerWorker`. It exports that file through +`OMPI_MCA_orte_default_hostfile`, so an empty or stale operator-injected +`/etc/mpi/hostfile` cannot control launcher behavior after resume. + +The launcher waits for every generated host to complete SSH `/bin/true` with +`BatchMode=yes`, `ConnectTimeout=2`, `ConnectionAttempts=1`, +`StrictHostKeyChecking=accept-new`, and a launcher-local +`UserKnownHostsFile=/tmp/project-signal-known-hosts`. SSH stderr remains visible +for diagnosis. The wait is capped at 30 seconds total and is always at least +one second shorter than `backendActiveDeadline`; timeout exits 124 and names +the last unready host. Launcher and worker replica restart policies are both +`Never`, preserving the result prelude and marker. + +The adapter keeps the current launcher log primary and reads the previous +container log defensively. A single valid previous result can recover a +restarted launcher when the current log has no result, identical current and +previous results select the current log, and conflicting results or a malformed +current marker fail closed. Bounded current/previous availability, byte counts, +parse status, and selected-source metadata are included in aggregate +measurements before foreground backend cleanup. This preserves the first +execution failure while keeping any cleanup failure separately visible. Other backend kinds remain operator-supplied and are not installed or granted permissions by the included adapter. diff --git a/internal/signaladapter/adapter.go b/internal/signaladapter/adapter.go index 0587c06a5b..5dd49c90cd 100644 --- a/internal/signaladapter/adapter.go +++ b/internal/signaladapter/adapter.go @@ -33,6 +33,9 @@ const ( MaxBackendOutputBytes = 16 * 1024 MaxCleanupFailureBytes = 64 maxMPIWorkerReadyWait = int64(30) + maxMPIWorkerSlots = int64(1<<31 - 1) + mpiHostfilePath = "/tmp/project-signal-hostfile" + mpiKnownHostsPath = "/tmp/project-signal-known-hosts" resultPrefix = "PROJECT_SIGNAL_RESULT_V2 " requiredTopologyKey = "kueue.x-k8s.io/podset-required-topology" ) @@ -86,12 +89,29 @@ type Config struct { } type Result struct { - ParticipantIndex int `json:"participantIndex"` - ExecutionState string `json:"executionState"` - Verdict string `json:"verdict"` - Reason string `json:"reason"` - Measurements string `json:"measurements"` - ExitCode int `json:"-"` + ParticipantIndex int `json:"participantIndex"` + ExecutionState string `json:"executionState"` + Verdict string `json:"verdict"` + Reason string `json:"reason"` + Measurements string `json:"measurements"` + ResultLog *ResultLogMetadata `json:"resultLog,omitempty"` + ExitCode int `json:"-"` +} + +type ResultLogSourceMetadata struct { + Status string `json:"status"` + Bytes int `json:"bytes,omitempty"` +} + +type ResultLogMetadata struct { + Current ResultLogSourceMetadata `json:"current"` + Previous ResultLogSourceMetadata `json:"previous"` + SelectedSource string `json:"selectedSource,omitempty"` +} + +type mpiWorkerHost struct { + Name string + Slots int64 } type AggregateOutput struct { @@ -427,7 +447,7 @@ func RenderResources( gpuCount, ) case "MPIJob": - object = mpiJob( + object, err = mpiJob( config, participant, tolerations, @@ -437,6 +457,9 @@ func RenderResources( config.BackendDeadlineSeconds, gpuCount, ) + if err != nil { + return nil, fmt.Errorf("render MPIJob for participant %d: %w", participant.Index, err) + } default: return nil, fmt.Errorf("unsupported backend kind %q", config.BackendKind) } @@ -744,41 +767,89 @@ func mpiWorkerReadinessTimeout(backendDeadlineSeconds int64) int64 { return timeout } -func mpiWorkerReadinessScript(hostfile string, timeoutSeconds int64) string { - return fmt.Sprintf(` +func mpiWorkerHosts(namespace, jobName string, workerCount, slots int64) ([]mpiWorkerHost, error) { + if workerCount < 1 || workerCount > MaxParticipants { + return nil, fmt.Errorf("MPI worker count must be between 1 and %d", MaxParticipants) + } + + if slots < 1 || slots > maxMPIWorkerSlots { + return nil, fmt.Errorf("MPI worker slots must be between 1 and %d", maxMPIWorkerSlots) + } + + for field, value := range map[string]string{ + "namespace": namespace, + "job name": jobName, + } { + if errors := validation.IsDNS1123Label(value); len(errors) > 0 { + return nil, fmt.Errorf("MPI %s %q is invalid: %s", field, value, strings.Join(errors, ", ")) + } + } + + hosts := make([]mpiWorkerHost, workerCount) + for index := range workerCount { + workerName := fmt.Sprintf("%s-worker-%d", jobName, index) + if errors := validation.IsDNS1123Label(workerName); len(errors) > 0 { + return nil, fmt.Errorf( + "generated MPI worker name %q is invalid: %s", + workerName, + strings.Join(errors, ", "), + ) + } + + name := fmt.Sprintf( + "%s.%s.%s.svc", + workerName, + jobName, + namespace, + ) + if errors := validation.IsDNS1123Subdomain(name); len(errors) > 0 { + return nil, fmt.Errorf("generated MPI worker host %q is invalid: %s", name, strings.Join(errors, ", ")) + } + + hosts[index] = mpiWorkerHost{Name: name, Slots: slots} + } + + return hosts, nil +} + +func mpiLauncherPrelude(hostfile string, hosts []mpiWorkerHost, timeoutSeconds int64) string { + hostfileLines := make([]string, len(hosts)) + + hostArguments := make([]string, len(hosts)) + for index, host := range hosts { + hostfileLines[index] = fmt.Sprintf("%s slots=%d", host.Name, host.Slots) + hostArguments[index] = shellQuote(host.Name) + } + + return fmt.Sprintf( + ` +umask 077 +printf '%%s\n' %s > %s +export OMPI_MCA_orte_default_hostfile=%s wait_for_mpi_workers() { local hostfile=%s local timeout_seconds=%d local deadline=$((SECONDS + timeout_seconds)) local host all_ready last_unready remaining - local -a hosts + local -a hosts=(%s) while (( SECONDS < deadline )); do - hosts=() - if [[ -r "${hostfile}" ]]; then - while IFS= read -r host; do - if [[ ! "${host}" =~ ^[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?$ ]]; then - printf 'Project Signal MPI readiness rejected invalid worker host %%q in %%s\n' "${host}" "${hostfile}" >&2 - return 64 - fi - hosts+=("${host}") - done < <(/usr/bin/awk 'NF {print $1}' "${hostfile}") - fi - if (( ${#hosts[@]} > 0 )); then - all_ready=1 - for host in "${hosts[@]}"; do - remaining=$((deadline - SECONDS)) - if (( remaining < 1 )) || ! timeout "${remaining}s" ssh \ - -o BatchMode=yes \ - -o ConnectTimeout=2 \ - -- "${host}" /bin/true /dev/null 2>&1; then - all_ready=0 - last_unready="${host}" - break - fi - done - if (( all_ready == 1 )); then - return 0 + all_ready=1 + for host in "${hosts[@]}"; do + remaining=$((deadline - SECONDS)) + if (( remaining < 1 )) || ! timeout "${remaining}s" ssh \ + -o BatchMode=yes \ + -o ConnectTimeout=2 \ + -o ConnectionAttempts=1 \ + -o StrictHostKeyChecking=accept-new \ + -o UserKnownHostsFile=%s \ + -- "${host}" /bin/true /dev/null; then + all_ready=0 + last_unready="${host}" + break fi + done + if (( all_ready == 1 )); then + return 0 fi sleep 1 done @@ -787,7 +858,15 @@ wait_for_mpi_workers() { return 124 } wait_for_mpi_workers -`, shellQuote(hostfile), timeoutSeconds) +`, + strings.Join(quotedShellArguments(hostfileLines), " "), + shellQuote(hostfile), + shellQuote(hostfile), + shellQuote(hostfile), + timeoutSeconds, + strings.Join(hostArguments, " "), + shellQuote(mpiKnownHostsPath), + ) } func ordinaryPod( @@ -851,12 +930,18 @@ func mpiJob( workflow metav1.OwnerReference, backendDeadlineSeconds int64, gpuCount int64, -) *unstructured.Unstructured { +) (*unstructured.Unstructured, error) { labels := resourceLabels(config, participant) - jobName := resourceName(config.RunName, participant.Index) + jobName := resourceNameWithLimit(config.RunName, participant.Index, 63-len("-worker-0")) launcherEnv := workloadEnvironment(config, participant, environment, false) workerEnv := workloadEnvironment(config, participant, environment, true) readinessTimeout := mpiWorkerReadinessTimeout(backendDeadlineSeconds) + + workerHosts, err := mpiWorkerHosts(config.Namespace, jobName, 1, gpuCount) + if err != nil { + return nil, err + } + worker := workloadContainer( config, policy, @@ -891,7 +976,8 @@ func mpiJob( }, "mpiReplicaSpecs": map[string]any{ "Launcher": map[string]any{ - "replicas": int64(1), + "replicas": int64(1), + "restartPolicy": string(corev1.RestartPolicyNever), "template": podTemplate( labels, nil, @@ -900,7 +986,7 @@ func mpiJob( Name: "launcher", Image: config.RunnerImage, Command: resultCommandWithPrelude( - mpiWorkerReadinessScript("/etc/mpi/hostfile", readinessTimeout), + mpiLauncherPrelude(mpiHostfilePath, workerHosts, readinessTimeout), "mpi-launcher", strconv.FormatInt(gpuCount, 10), ), @@ -911,7 +997,8 @@ func mpiJob( ), }, "Worker": map[string]any{ - "replicas": int64(1), + "replicas": int64(1), + "restartPolicy": string(corev1.RestartPolicyNever), "template": podTemplate( labels, map[string]string{corev1.LabelHostname: participant.NodeName}, @@ -925,7 +1012,7 @@ func mpiJob( }, } - return &unstructured.Unstructured{Object: object} + return &unstructured.Unstructured{Object: object}, nil } func podTemplate( @@ -1193,9 +1280,13 @@ func resourceLabels(config Config, participant signalv1.Participant) map[string] } func resourceName(runName string, participantIndex int32) string { + return resourceNameWithLimit(runName, participantIndex, 63) +} + +func resourceNameWithLimit(runName string, participantIndex int32, maximumLength int) string { suffix := fmt.Sprintf("-p%d", participantIndex) - maximumRunLength := 63 - len(suffix) + maximumRunLength := maximumLength - len(suffix) if len(runName) > maximumRunLength { runName = strings.TrimRight(runName[:maximumRunLength], "-.") } @@ -1233,6 +1324,15 @@ func shellQuote(value string) string { return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'" } +func quotedShellArguments(values []string) []string { + quoted := make([]string, len(values)) + for index, value := range values { + quoted[index] = shellQuote(value) + } + + return quoted +} + func ptr[T any](value T) *T { return &value } diff --git a/internal/signaladapter/adapter_test.go b/internal/signaladapter/adapter_test.go index d5bea1fefc..ce35aef1cd 100644 --- a/internal/signaladapter/adapter_test.go +++ b/internal/signaladapter/adapter_test.go @@ -8,6 +8,8 @@ import ( "context" "encoding/base64" "encoding/json" + "errors" + "fmt" "os" "os/exec" "path/filepath" @@ -257,6 +259,21 @@ func TestRenderMPIJobsPerParticipant(t *testing.T) { } for _, replica := range []string{"Launcher", "Worker"} { + replicaRestartPolicy, _, err := unstructured.NestedString( + resource.Object, + "spec", + "mpiReplicaSpecs", + replica, + "restartPolicy", + ) + if err != nil { + t.Fatal(err) + } + + if replicaRestartPolicy != string(corev1.RestartPolicyNever) { + t.Fatalf("resource %d %s replica restart policy = %q", index, replica, replicaRestartPolicy) + } + restartPolicy, _, err := unstructured.NestedString( resource.Object, "spec", @@ -319,12 +336,30 @@ func TestRenderMPIJobsPerParticipant(t *testing.T) { t.Fatalf("resource %d launcher requests GPUs: %s", index, launcherJSON) } - if !strings.Contains(string(launcherJSON), "/etc/mpi/hostfile") || + if strings.Contains(string(launcherJSON), "/etc/mpi/hostfile") || + !strings.Contains(string(launcherJSON), mpiHostfilePath) || + !strings.Contains(string(launcherJSON), "OMPI_MCA_orte_default_hostfile") || !strings.Contains(string(launcherJSON), "BatchMode=yes") || + !strings.Contains(string(launcherJSON), "ConnectionAttempts=1") || + !strings.Contains(string(launcherJSON), "StrictHostKeyChecking=accept-new") || + !strings.Contains(string(launcherJSON), "UserKnownHostsFile=") || + !strings.Contains(string(launcherJSON), mpiKnownHostsPath) || !strings.Contains(string(launcherJSON), "/bin/true") || !strings.Contains(string(launcherJSON), "Project Signal MPI readiness timed out") { t.Fatalf("resource %d launcher readiness fence missing: %s", index, launcherJSON) } + + jobName := resource.GetName() + + expectedHost := fmt.Sprintf( + "%s-worker-0.%s.%s.svc", + jobName, + jobName, + config.Namespace, + ) + if !strings.Contains(string(launcherJSON), expectedHost+" slots="+config.GPUResourceCount) { + t.Fatalf("resource %d launcher hostfile entry missing: %s", index, launcherJSON) + } } } @@ -345,32 +380,30 @@ func TestMPIWorkerReadinessFenceWaitsForWorkers(t *testing.T) { directory := t.TempDir() hostfile := filepath.Join(directory, "hostfile") readyFile := filepath.Join(directory, "ready") + argumentsFile := filepath.Join(directory, "ssh-arguments") binDirectory := fakeSSH(t, directory) ready := make(chan error, 1) + hosts := []mpiWorkerHost{ + {Name: "worker-0.job.namespace.svc", Slots: 2}, + {Name: "worker-1.job.namespace.svc", Slots: 2}, + } go func() { time.Sleep(100 * time.Millisecond) - if err := os.WriteFile(hostfile, []byte("worker-0 slots=1\nworker-1 slots=1\n"), 0o600); err != nil { - ready <- err - - return - } - - time.Sleep(1100 * time.Millisecond) - ready <- os.WriteFile(readyFile, nil, 0o600) }() ctx, cancel := context.WithTimeout(t.Context(), 8*time.Second) defer cancel() - command := exec.CommandContext(ctx, "/bin/bash", "-c", mpiWorkerReadinessScript(hostfile, 5)) + command := exec.CommandContext(ctx, "/bin/bash", "-c", mpiLauncherPrelude(hostfile, hosts, 5)) command.Env = append( os.Environ(), "PATH="+binDirectory+":"+os.Getenv("PATH"), "PROJECT_SIGNAL_TEST_SSH_READY="+readyFile, + "PROJECT_SIGNAL_TEST_SSH_ARGUMENTS="+argumentsFile, ) var stderr bytes.Buffer @@ -384,6 +417,33 @@ func TestMPIWorkerReadinessFenceWaitsForWorkers(t *testing.T) { if err := <-ready; err != nil { t.Fatal(err) } + + hostfileContent, err := os.ReadFile(hostfile) + if err != nil { + t.Fatal(err) + } + + if string(hostfileContent) != "worker-0.job.namespace.svc slots=2\nworker-1.job.namespace.svc slots=2\n" { + t.Fatalf("generated hostfile = %q", hostfileContent) + } + + arguments, err := os.ReadFile(argumentsFile) + if err != nil { + t.Fatal(err) + } + + for _, expected := range []string{ + "-o BatchMode=yes", + "-o ConnectTimeout=2", + "-o ConnectionAttempts=1", + "-o StrictHostKeyChecking=accept-new", + "-o UserKnownHostsFile=" + mpiKnownHostsPath, + "worker-1.job.namespace.svc /bin/true", + } { + if !strings.Contains(string(arguments), expected) { + t.Fatalf("SSH arguments %q do not contain %q", arguments, expected) + } + } } func TestMPIWorkerReadinessFenceTimesOut(t *testing.T) { @@ -392,11 +452,15 @@ func TestMPIWorkerReadinessFenceTimesOut(t *testing.T) { directory := t.TempDir() hostfile := filepath.Join(directory, "hostfile") - if err := os.WriteFile(hostfile, []byte("worker-0 slots=1\n"), 0o600); err != nil { - t.Fatal(err) - } - - command := exec.Command("/bin/bash", "-c", mpiWorkerReadinessScript(hostfile, 2)) + command := exec.Command( + "/bin/bash", + "-c", + mpiLauncherPrelude( + hostfile, + []mpiWorkerHost{{Name: "worker-0.job.namespace.svc", Slots: 1}}, + 2, + ), + ) command.Env = append(os.Environ(), "PATH="+fakeSSH(t, directory)+":"+os.Getenv("PATH")) @@ -409,13 +473,109 @@ func TestMPIWorkerReadinessFenceTimesOut(t *testing.T) { t.Fatal("readiness fence succeeded without a listening worker") } + var exitError *exec.ExitError + if !errors.As(err, &exitError) || exitError.ExitCode() != 124 { + t.Fatalf("readiness fence error = %v, want exit 124", err) + } + if !strings.Contains(stderr.String(), "timed out after 2s") || !strings.Contains(stderr.String(), hostfile) || - !strings.Contains(stderr.String(), "last unready host=worker-0") { + !strings.Contains(stderr.String(), "last unready host=worker-0.job.namespace.svc") || + !strings.Contains(stderr.String(), "simulated SSH connection failure") { t.Fatalf("readiness timeout stderr = %q", stderr.String()) } } +func TestMPILauncherPreludeIgnoresOperatorHostfile(t *testing.T) { + t.Parallel() + + directory := t.TempDir() + + operatorHostfile := filepath.Join(directory, "operator-hostfile") + if err := os.WriteFile(operatorHostfile, []byte("stale-worker slots=99\n"), 0o600); err != nil { + t.Fatal(err) + } + + generatedHostfile := filepath.Join(directory, "generated-hostfile") + readyFile := filepath.Join(directory, "ready") + observedOpenMPIHostfile := filepath.Join(directory, "observed-openmpi-hostfile") + + if err := os.WriteFile(readyFile, nil, 0o600); err != nil { + t.Fatal(err) + } + + command := exec.Command( + "/bin/bash", + "-c", + mpiLauncherPrelude( + generatedHostfile, + []mpiWorkerHost{{Name: "worker-0.job.namespace.svc", Slots: 4}}, + 2, + ), + ) + + command.Env = append( + os.Environ(), + "PATH="+fakeSSH(t, directory)+":"+os.Getenv("PATH"), + "PROJECT_SIGNAL_TEST_SSH_READY="+readyFile, + "PROJECT_SIGNAL_TEST_OMPI_HOSTFILE_OUTPUT="+observedOpenMPIHostfile, + "OMPI_MCA_orte_default_hostfile="+operatorHostfile, + ) + + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("launcher prelude failed: %v output=%q", err, output) + } + + content, err := os.ReadFile(generatedHostfile) + if err != nil { + t.Fatal(err) + } + + if strings.Contains(string(content), "stale-worker") || + string(content) != "worker-0.job.namespace.svc slots=4\n" { + t.Fatalf("generated hostfile = %q", content) + } + + observed, err := os.ReadFile(observedOpenMPIHostfile) + if err != nil { + t.Fatal(err) + } + + if string(observed) != generatedHostfile { + t.Fatalf("OpenMPI hostfile = %q, want %q", observed, generatedHostfile) + } +} + +func TestMPIWorkerHostsRejectsInvalidInputs(t *testing.T) { + t.Parallel() + + for name, test := range map[string]struct { + namespace string + jobName string + workerCount int64 + slots int64 + }{ + "zero workers": {namespace: "namespace", jobName: "job", slots: 1}, + "zero slots": {namespace: "namespace", jobName: "job", workerCount: 1}, + "invalid host": {namespace: "namespace", jobName: "INVALID", workerCount: 1, slots: 1}, + "too many hosts": {namespace: "namespace", jobName: "job", workerCount: MaxParticipants + 1, slots: 1}, + "too many slots": {namespace: "namespace", jobName: "job", workerCount: 1, slots: maxMPIWorkerSlots + 1}, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + if _, err := mpiWorkerHosts( + test.namespace, + test.jobName, + test.workerCount, + test.slots, + ); err == nil { + t.Fatal("mpiWorkerHosts accepted invalid input") + } + }) + } +} + func fakeSSH(t *testing.T, directory string) string { t.Helper() @@ -425,11 +585,32 @@ func fakeSSH(t *testing.T, directory string) string { } content := []byte(`#!/bin/bash -if [[ -n "${PROJECT_SIGNAL_TEST_SSH_READY:-}" && -f "${PROJECT_SIGNAL_TEST_SSH_READY}" ]]; then - exit 0 + if [[ -n "${PROJECT_SIGNAL_TEST_SSH_ARGUMENTS:-}" ]]; then + printf '%s\n' "$*" >> "${PROJECT_SIGNAL_TEST_SSH_ARGUMENTS}" fi -exit 255 -`) + if [[ -n "${PROJECT_SIGNAL_TEST_OMPI_HOSTFILE_OUTPUT:-}" ]]; then + printf '%s' "${OMPI_MCA_orte_default_hostfile:-}" > "${PROJECT_SIGNAL_TEST_OMPI_HOSTFILE_OUTPUT}" + fi + required=( + "-o BatchMode=yes" + "-o ConnectTimeout=2" + "-o ConnectionAttempts=1" + "-o StrictHostKeyChecking=accept-new" + "-o UserKnownHostsFile=/tmp/project-signal-known-hosts" + ) + arguments=" $* " + for option in "${required[@]}"; do + if [[ "${arguments}" != *" ${option} "* ]]; then + printf 'missing SSH option: %s\n' "${option}" >&2 + exit 255 + fi + done + if [[ -n "${PROJECT_SIGNAL_TEST_SSH_READY:-}" && -f "${PROJECT_SIGNAL_TEST_SSH_READY}" ]]; then + exit 0 + fi + printf 'simulated SSH connection failure for %s\n' "$*" >&2 + exit 255 + `) if err := os.WriteFile(filepath.Join(binDirectory, "ssh"), content, 0o750); err != nil { t.Fatal(err) } @@ -885,6 +1066,44 @@ func TestAggregateRetainsDiagnosticFailureWhenBackendsSucceed(t *testing.T) { } } +func TestAggregateBoundsResultLogMetadataAtMaximumParticipants(t *testing.T) { + t.Parallel() + + resources := make([]signalv1.BackendResourceReference, MaxParticipants) + + results := make([]Result, MaxParticipants) + for index := range MaxParticipants { + resources[index] = signalv1.BackendResourceReference{ + APIVersion: "kubeflow.org/v2beta1", + Kind: "MPIJob", + Namespace: "project-signal", + Name: fmt.Sprintf("run-p%d", index), + UID: types.UID(fmt.Sprintf("uid-%d", index)), + ParticipantIndexes: []int32{int32(index)}, + } + results[index] = Result{ + ParticipantIndex: index, + ExecutionState: "Succeeded", + Verdict: "Pass", + Reason: "Healthy", + ResultLog: &ResultLogMetadata{ + Current: ResultLogSourceMetadata{Status: "valid", Bytes: 128}, + Previous: ResultLogSourceMetadata{Status: "unavailable"}, + SelectedSource: "current", + }, + } + } + + output, err := Aggregate(MaxParticipants, resources, results) + if err != nil { + t.Fatal(err) + } + + if len(output.Measurements) > MaxMeasurementsBytes { + t.Fatalf("measurements size = %d", len(output.Measurements)) + } +} + func TestAggregateRejectsDuplicateIdentityAndOversizedMeasurements(t *testing.T) { t.Parallel() diff --git a/internal/signaladapter/runner.go b/internal/signaladapter/runner.go index 2689cdf415..d143bc1973 100644 --- a/internal/signaladapter/runner.go +++ b/internal/signaladapter/runner.go @@ -36,6 +36,7 @@ type Runner struct { Kubernetes kubernetes.Interface PollInterval time.Duration CleanupWait time.Duration + ReadPodLog func(context.Context, string, string, string, bool) (string, error) } type createdResource struct { @@ -44,6 +45,19 @@ type createdResource struct { participantIndex int32 } +const maxResultLogBytes = 64 * 1024 + +type resultLogSource struct { + content string + err error + metadata ResultLogSourceMetadata +} + +type resultLogSnapshot struct { + current resultLogSource + previous resultLogSource +} + func (r *Runner) Execute(ctx context.Context, config Config) (output AggregateOutput, resultErr error) { if r.Dynamic == nil || r.Kubernetes == nil { return AggregateOutput{}, errors.New("kubernetes clients are required") @@ -232,28 +246,39 @@ func (r *Runner) waitForResult(ctx context.Context, config Config, backend creat } if backend.resource.GetKind() == "MPIJob" && capturedResult == nil { - logContent, logErr := r.resultLog(ctx, config, backend) - if logErr == nil { - parsed, parseErr := ParseResultLog(logContent, int(backend.participantIndex)) + current := r.resultLogs(ctx, config, backend, false).current + if current.err == nil { + parsed, parseErr := ParseResultLog(current.content, int(backend.participantIndex)) if parseErr == nil { + parsed.ResultLog = &ResultLogMetadata{ + Current: current.metadata, + Previous: ResultLogSourceMetadata{Status: "not-read"}, + SelectedSource: "current", + } capturedResult = &parsed } } } if terminal { - if capturedResult != nil { - return completedResult(*capturedResult, succeeded) - } + logs := r.resultLogs(ctx, config, backend, backend.resource.GetKind() == "MPIJob") - logContent, logErr := r.resultLog(ctx, config, backend) - - return backendCompletionResult( - logContent, - logErr, + completed, hardFailure := backendCompletionFromLogs( + logs, int(backend.participantIndex), succeeded, ) + if completed.ExecutionState == "Succeeded" || hardFailure || capturedResult == nil { + return completed + } + + capturedResult.ResultLog = &ResultLogMetadata{ + Current: capturedResult.ResultLog.Current, + Previous: logs.previous.metadata, + SelectedSource: "captured-current", + } + + return completedResult(*capturedResult, succeeded) } select { @@ -271,23 +296,47 @@ func backendCompletionResult( logErr error, participantIndex int, backendSucceeded bool, +) Result { + return backendCompletionFromSource( + resultLogSource{ + content: logContent, + err: logErr, + metadata: resultLogSourceMetadata( + logContent, + logErr, + ), + }, + participantIndex, + backendSucceeded, + ) +} + +func backendCompletionFromSource( + source resultLogSource, + participantIndex int, + backendSucceeded bool, ) Result { result := Result{ ParticipantIndex: participantIndex, ExecutionState: "Error", Verdict: "Inconclusive", Reason: "BackendExecutionFailed", + ResultLog: &ResultLogMetadata{ + Current: source.metadata, + Previous: ResultLogSourceMetadata{Status: "not-read"}, + SelectedSource: "", + }, } - if logErr != nil { + if source.err != nil { if backendSucceeded { - result.Reason = "BackendResultUnavailable: " + logErr.Error() + result.Reason = "BackendResultUnavailable: " + source.err.Error() } return result } - parsed, err := ParseResultLog(logContent, participantIndex) + parsed, err := ParseResultLog(source.content, participantIndex) if err != nil { if backendSucceeded { result.Reason = "BackendResultInvalid: " + err.Error() @@ -296,9 +345,90 @@ func backendCompletionResult( return result } + parsed.ResultLog = result.ResultLog + parsed.ResultLog.SelectedSource = "current" + return completedResult(parsed, backendSucceeded) } +func backendCompletionFromLogs( + logs resultLogSnapshot, + participantIndex int, + backendSucceeded bool, +) (Result, bool) { + result := Result{ + ParticipantIndex: participantIndex, + ExecutionState: "Error", + Verdict: "Inconclusive", + Reason: "BackendExecutionFailed", + ResultLog: &ResultLogMetadata{ + Current: logs.current.metadata, + Previous: logs.previous.metadata, + }, + } + + current, currentErr := parseResultLogSource(logs.current, participantIndex) + previous, previousErr := parseResultLogSource(logs.previous, participantIndex) + + if currentErr == nil { + if previousErr == nil && !sameResult(current, previous) { + result.Reason = "BackendResultInvalid: current and previous launcher logs contain conflicting results" + + return result, true + } + + current.ResultLog = result.ResultLog + current.ResultLog.SelectedSource = "current" + + return completedResult(current, backendSucceeded), false + } + + if logs.current.metadata.Status == "invalid" { + if backendSucceeded { + result.Reason = "BackendResultInvalid: " + currentErr.Error() + } + + return result, true + } + + if previousErr == nil { + previous.ResultLog = result.ResultLog + previous.ResultLog.SelectedSource = "previous" + + return completedResult(previous, backendSucceeded), false + } + + hardFailure := logs.previous.metadata.Status == "invalid" + + if backendSucceeded { + switch { + case hardFailure: + result.Reason = "BackendResultInvalid: " + previousErr.Error() + case logs.current.err != nil: + result.Reason = "BackendResultUnavailable: " + logs.current.err.Error() + default: + result.Reason = "BackendResultInvalid: " + currentErr.Error() + } + } + + return result, hardFailure +} + +func parseResultLogSource(source resultLogSource, participantIndex int) (Result, error) { + if source.err != nil { + return Result{}, source.err + } + + return ParseResultLog(source.content, participantIndex) +} + +func sameResult(left, right Result) bool { + return left.Verdict == right.Verdict && + left.Reason == right.Reason && + left.Measurements == right.Measurements && + left.ExitCode == right.ExitCode +} + func completedResult(result Result, backendSucceeded bool) Result { if (result.ExitCode == 0) != backendSucceeded { return Result{ @@ -306,6 +436,7 @@ func completedResult(result Result, backendSucceeded bool) Result { ExecutionState: "Error", Verdict: "Inconclusive", Reason: "BackendResultInvalid: result exit code contradicts backend terminal state", + ResultLog: result.ResultLog, } } @@ -369,11 +500,12 @@ func backendState(resource *unstructured.Unstructured) (bool, bool, error) { } } -func (r *Runner) resultLog( +func (r *Runner) resultLogs( ctx context.Context, config Config, backend createdResource, -) (string, error) { + includePrevious bool, +) resultLogSnapshot { podName := backend.resource.GetName() containerName := "main" @@ -386,7 +518,12 @@ func (r *Runner) resultLog( }, ","), }) if err != nil { - return "", fmt.Errorf("list MPIJob Pods: %w", err) + source := resultLogSourceFromError(fmt.Errorf("list MPIJob Pods: %w", err)) + + return resultLogSnapshot{ + current: source, + previous: resultLogSourceMetadataOnly("not-read"), + } } podName = "" @@ -408,18 +545,46 @@ func (r *Runner) resultLog( } if podName == "" { - return "", errors.New("MPIJob launcher Pod was not found") + source := resultLogSourceFromError(errors.New("MPIJob launcher Pod was not found")) + + return resultLogSnapshot{ + current: source, + previous: resultLogSourceMetadataOnly("not-read"), + } } } - stream, err := r.Kubernetes.CoreV1().Pods(config.Namespace). - GetLogs(podName, &corev1.PodLogOptions{Container: containerName}). + currentContent, currentErr := r.readPodLog(ctx, config.Namespace, podName, containerName, false) + snapshot := resultLogSnapshot{ + current: resultLogSourceWithContent(currentContent, currentErr), + previous: resultLogSourceMetadataOnly("not-read"), + } + + if includePrevious { + previousContent, previousErr := r.readPodLog(ctx, config.Namespace, podName, containerName, true) + snapshot.previous = resultLogSourceWithContent(previousContent, previousErr) + } + + return snapshot +} + +func (r *Runner) readPodLog( + ctx context.Context, + namespace, podName, containerName string, + previous bool, +) (string, error) { + if r.ReadPodLog != nil { + return r.ReadPodLog(ctx, namespace, podName, containerName, previous) + } + + stream, err := r.Kubernetes.CoreV1().Pods(namespace). + GetLogs(podName, &corev1.PodLogOptions{Container: containerName, Previous: previous}). Stream(ctx) if err != nil { return "", fmt.Errorf("open result log: %w", err) } - content, err := io.ReadAll(io.LimitReader(stream, 64*1024)) + content, err := io.ReadAll(io.LimitReader(stream, maxResultLogBytes+1)) if err != nil { closeErr := stream.Close() if closeErr != nil { @@ -436,9 +601,50 @@ func (r *Runner) resultLog( return "", fmt.Errorf("close result log: %w", err) } + if len(content) > maxResultLogBytes { + return "", fmt.Errorf("result log exceeds %d bytes", maxResultLogBytes) + } + return string(content), nil } +func resultLogSourceWithContent(content string, err error) resultLogSource { + return resultLogSource{ + content: content, + err: err, + metadata: resultLogSourceMetadata(content, err), + } +} + +func resultLogSourceFromError(err error) resultLogSource { + return resultLogSourceWithContent("", err) +} + +func resultLogSourceMetadataOnly(status string) resultLogSource { + return resultLogSource{metadata: ResultLogSourceMetadata{Status: status}} +} + +func resultLogSourceMetadata(content string, err error) ResultLogSourceMetadata { + if err != nil { + return ResultLogSourceMetadata{Status: "unavailable"} + } + + status := "missing" + + if strings.Contains(content, resultPrefix) { + if _, parseErr := ParseResultLog(content, 0); parseErr != nil { + status = "invalid" + } else { + status = "valid" + } + } + + return ResultLogSourceMetadata{ + Status: status, + Bytes: len(content), + } +} + func (r *Runner) cleanup( ctx context.Context, namespace string, diff --git a/internal/signaladapter/runner_test.go b/internal/signaladapter/runner_test.go index 709c108030..6f2e2e06e6 100644 --- a/internal/signaladapter/runner_test.go +++ b/internal/signaladapter/runner_test.go @@ -11,11 +11,13 @@ import ( "testing" "time" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" dynamicfake "k8s.io/client-go/dynamic/fake" + kubernetesfake "k8s.io/client-go/kubernetes/fake" ) func TestCompletedResultAcceptsVerdictAssociatedTerminalState(t *testing.T) { @@ -92,6 +94,155 @@ func TestBackendCompletionResultFailsClosedWithoutValidResult(t *testing.T) { } } +func TestBackendCompletionFromLogsRecoversPreviousResult(t *testing.T) { + t.Parallel() + + logs := resultLogSnapshot{ + current: resultLogSourceWithContent("launcher restarted\n", nil), + previous: resultLogSourceWithContent( + encodedResultRecord("Inconclusive", "DiagnosticUnavailable", "state=unknown", 2), + nil, + ), + } + + result, hardFailure := backendCompletionFromLogs(logs, 0, false) + if hardFailure || + result.ExecutionState != "Succeeded" || + result.Verdict != "Inconclusive" || + result.Reason != "DiagnosticUnavailable" || + result.ResultLog == nil || + result.ResultLog.SelectedSource != "previous" || + result.ResultLog.Current.Status != "missing" || + result.ResultLog.Previous.Status != "valid" { + t.Fatalf("backendCompletionFromLogs() = %#v, hardFailure=%t", result, hardFailure) + } +} + +func TestBackendCompletionFromLogsPrefersCurrentResult(t *testing.T) { + t.Parallel() + + record := encodedResultRecord("Pass", "Healthy", "state=ready", 0) + logs := resultLogSnapshot{ + current: resultLogSourceWithContent(record, nil), + previous: resultLogSourceWithContent(record, nil), + } + + result, hardFailure := backendCompletionFromLogs(logs, 0, true) + if hardFailure || + result.ExecutionState != "Succeeded" || + result.Verdict != "Pass" || + result.ResultLog == nil || + result.ResultLog.SelectedSource != "current" { + t.Fatalf("backendCompletionFromLogs() = %#v, hardFailure=%t", result, hardFailure) + } +} + +func TestBackendCompletionFromLogsRejectsConflicts(t *testing.T) { + t.Parallel() + + logs := resultLogSnapshot{ + current: resultLogSourceWithContent( + encodedResultRecord("Pass", "Healthy", "state=ready", 0), + nil, + ), + previous: resultLogSourceWithContent( + encodedResultRecord("Inconclusive", "DiagnosticUnavailable", "state=unknown", 2), + nil, + ), + } + + result, hardFailure := backendCompletionFromLogs(logs, 0, false) + if !hardFailure || + result.ExecutionState != "Error" || + result.Verdict != "Inconclusive" || + !strings.Contains(result.Reason, "conflicting results") || + result.ResultLog == nil || + result.ResultLog.Current.Status != "valid" || + result.ResultLog.Previous.Status != "valid" { + t.Fatalf("backendCompletionFromLogs() = %#v, hardFailure=%t", result, hardFailure) + } +} + +func TestBackendCompletionFromLogsRejectsMalformedCurrentBeforePreviousFallback(t *testing.T) { + t.Parallel() + + logs := resultLogSnapshot{ + current: resultLogSourceWithContent(resultPrefix+"%%% bm90LXVzZWQ= - 2\n", nil), + previous: resultLogSourceWithContent( + encodedResultRecord("Inconclusive", "DiagnosticUnavailable", "state=unknown", 2), + nil, + ), + } + + result, hardFailure := backendCompletionFromLogs(logs, 0, false) + if !hardFailure || + result.ExecutionState != "Error" || + result.Reason != "BackendExecutionFailed" || + result.ResultLog == nil || + result.ResultLog.Current.Status != "invalid" { + t.Fatalf("backendCompletionFromLogs() = %#v, hardFailure=%t", result, hardFailure) + } +} + +func TestResultLogsReadsCurrentAndPreviousLauncherLogs(t *testing.T) { + t.Parallel() + + job := &unstructured.Unstructured{} + job.SetAPIVersion("kubeflow.org/v2beta1") + job.SetKind("MPIJob") + job.SetName("run-p0") + job.SetNamespace("project-signal") + job.SetUID("job-uid") + + launcher := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "run-p0-launcher", + Namespace: "project-signal", + Labels: map[string]string{ + "signal.unbounded-cloud.io/run": "run", + "signal.unbounded-cloud.io/participant-index": "0", + }, + OwnerReferences: []metav1.OwnerReference{{UID: "job-uid"}}, + }, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "launcher"}}}, + } + + var previousValues []bool + + runner := &Runner{ + Kubernetes: kubernetesfake.NewSimpleClientset(launcher), + ReadPodLog: func( + _ context.Context, + namespace, podName, containerName string, + previous bool, + ) (string, error) { + if namespace != launcher.Namespace || + podName != launcher.Name || + containerName != "launcher" { + t.Fatalf("read log identity = %s/%s %s", namespace, podName, containerName) + } + + previousValues = append(previousValues, previous) + + return encodedResultRecord("Pass", "Healthy", "", 0), nil + }, + } + + logs := runner.resultLogs( + t.Context(), + Config{Namespace: "project-signal", RunName: "run"}, + createdResource{resource: job, participantIndex: 0}, + true, + ) + if len(previousValues) != 2 || previousValues[0] || !previousValues[1] { + t.Fatalf("previous log reads = %#v", previousValues) + } + + if logs.current.metadata.Status != "valid" || logs.previous.metadata.Status != "valid" { + t.Fatalf("result log metadata = %#v", logs) + } +} + func TestCleanupFailurePreservesExecutionFailure(t *testing.T) { t.Parallel() From 4bbda32758f9dcfc7e01d5b63100202a62d0ac70 Mon Sep 17 00:00:00 2001 From: chokevin Date: Thu, 22 Oct 2026 07:44:29 -0700 Subject: [PATCH 18/36] fix(signal): preserve pre-cleanup backend evidence --- cmd/project-signal-adapter/main.go | 28 +- cmd/project-signal-adapter/main_test.go | 25 +- .../03-adapter-serviceaccount-rbac.yaml.tmpl | 3 + .../04-adapter-workflowtemplate.yaml.tmpl | 3 + deploy/signal/README.md | 19 + deploy/signal/manifest_test.go | 24 + internal/signal/controller.go | 55 ++ internal/signal/controller_test.go | 70 +- internal/signaladapter/adapter.go | 29 +- internal/signaladapter/adapter_test.go | 2 + internal/signaladapter/backend_evidence.go | 780 ++++++++++++++++++ .../signaladapter/backend_evidence_test.go | 481 +++++++++++ internal/signaladapter/runner.go | 95 ++- internal/signaladapter/runner_test.go | 13 +- 14 files changed, 1584 insertions(+), 43 deletions(-) create mode 100644 internal/signaladapter/backend_evidence.go create mode 100644 internal/signaladapter/backend_evidence_test.go diff --git a/cmd/project-signal-adapter/main.go b/cmd/project-signal-adapter/main.go index f3ecc5bffb..4e68dfcf73 100644 --- a/cmd/project-signal-adapter/main.go +++ b/cmd/project-signal-adapter/main.go @@ -116,6 +116,17 @@ func run(ctx context.Context, cfg config) error { runner := &signaladapter.Runner{ Dynamic: dynamicClient, Kubernetes: kubernetesClient, + PersistBackendEvidence: func(content string) error { + if err := signaladapter.ValidateBackendEvidence(content); err != nil { + return err + } + + if err := os.MkdirAll(cfg.outputDirectory, 0o750); err != nil { + return fmt.Errorf("create output directory: %w", err) + } + + return writeOutput(cfg.outputDirectory, "backend-evidence", content) + }, } output, err := runner.Execute(ctx, cfg.Config) @@ -138,6 +149,10 @@ func writeOutputs(directory string, output signaladapter.AggregateOutput) error ) } + if err := signaladapter.ValidateBackendEvidence(output.BackendEvidence); err != nil { + return err + } + if err := os.MkdirAll(directory, 0o750); err != nil { return fmt.Errorf("create output directory: %w", err) } @@ -148,10 +163,11 @@ func writeOutputs(directory string, output signaladapter.AggregateOutput) error "cleanup-failure": output.CleanupFailure, "measurements": output.Measurements, "backend-resources": output.BackendJSON, + "backend-evidence": output.BackendEvidence, } for name, value := range outputs { - if err := os.WriteFile(filepath.Join(directory, name), []byte(value), 0o600); err != nil { - return fmt.Errorf("write %s output: %w", name, err) + if err := writeOutput(directory, name, value); err != nil { + return err } } @@ -165,6 +181,14 @@ func writeOutputs(directory string, output signaladapter.AggregateOutput) error return nil } +func writeOutput(directory, name, value string) error { + if err := os.WriteFile(filepath.Join(directory, name), []byte(value), 0o600); err != nil { + return fmt.Errorf("write %s output: %w", name, err) + } + + return nil +} + type uidValue struct { target *types.UID } diff --git a/cmd/project-signal-adapter/main_test.go b/cmd/project-signal-adapter/main_test.go index 8895877501..6d7e2dd1a9 100644 --- a/cmd/project-signal-adapter/main_test.go +++ b/cmd/project-signal-adapter/main_test.go @@ -17,18 +17,20 @@ func TestWriteOutputsProjectsCleanupFailure(t *testing.T) { directory := t.TempDir() output := signaladapter.AggregateOutput{ - ExecutionState: "Error", - Verdict: "Inconclusive", - Reason: "BackendExecutionFailed", - CleanupFailure: "BackendCleanupFailed", + ExecutionState: "Error", + Verdict: "Inconclusive", + Reason: "BackendExecutionFailed", + CleanupFailure: "BackendCleanupFailed", + BackendEvidence: `{"version":"v1","complete":true,"participants":[{}]}`, } if err := writeOutputs(directory, output); err != nil { t.Fatal(err) } for name, want := range map[string]string{ - "reason": "BackendExecutionFailed", - "cleanup-failure": "BackendCleanupFailed", + "reason": "BackendExecutionFailed", + "cleanup-failure": "BackendCleanupFailed", + "backend-evidence": `{"version":"v1","complete":true,"participants":[{}]}`, } { got, err := os.ReadFile(filepath.Join(directory, name)) if err != nil { @@ -51,3 +53,14 @@ func TestWriteOutputsRejectsOversizedCleanupFailure(t *testing.T) { t.Fatal("writeOutputs accepted oversized cleanup failure") } } + +func TestWriteOutputsRejectsMalformedBackendEvidence(t *testing.T) { + t.Parallel() + + err := writeOutputs(t.TempDir(), signaladapter.AggregateOutput{ + BackendEvidence: `{"version":"v1","complete":true}`, + }) + if err == nil { + t.Fatal("writeOutputs accepted malformed backend evidence") + } +} diff --git a/deploy/signal/03-adapter-serviceaccount-rbac.yaml.tmpl b/deploy/signal/03-adapter-serviceaccount-rbac.yaml.tmpl index 102402fe25..7a32f5ca0e 100644 --- a/deploy/signal/03-adapter-serviceaccount-rbac.yaml.tmpl +++ b/deploy/signal/03-adapter-serviceaccount-rbac.yaml.tmpl @@ -20,6 +20,9 @@ rules: - apiGroups: [""] resources: ["pods/log"] verbs: ["get"] + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "list"] - apiGroups: ["kubeflow.org"] resources: ["mpijobs"] verbs: ["get", "list", "watch", "create", "delete"] diff --git a/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl b/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl index 52e6adafae..3ab4f44a3f 100644 --- a/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl +++ b/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl @@ -131,3 +131,6 @@ spec: - name: backend-resources valueFrom: path: /outputs/backend-resources + - name: backend-evidence + valueFrom: + path: /outputs/backend-evidence diff --git a/deploy/signal/README.md b/deploy/signal/README.md index 6b1670d8f7..46fba22e3e 100644 --- a/deploy/signal/README.md +++ b/deploy/signal/README.md @@ -96,6 +96,7 @@ The Workflow must publish only these output parameters: | `reason` | 4 KiB | Bounded diagnostic reason | | `measurements` | 16 KiB | Bounded measurement summary | | `backend-resources` | 16 KiB | Optional JSON array of backend workload identities and covered participant indexes; required for a multi-Node `Pass` | +| `backend-evidence` | 256 KiB | Required redacted pre-cleanup backend object, container state, and bounded current/previous log projection | Unexpected, duplicate, malformed, or oversized outputs are rejected and the run remains fail-closed. Each `backend-resources` entry must contain @@ -103,6 +104,24 @@ run remains fail-closed. Each `backend-resources` entry must contain that the workload executed. `namespace` is optional. A multi-Node successful result must cover every admitted participant. +Before foreground deletion, the adapter captures and writes a typed v1 +projection for every participant backend to the Argo output path. The +projection includes Pod or MPIJob state and +conditions, launcher Job conditions, participant container exit state and +restart count, and bounded current and previous container log tails. Generated +participant commands emit lifecycle markers for result-directory preparation +and final verdict, reason, and measurements file presence. The projection +records those markers without retaining the result payload itself. + +Each retained log tail is limited to 512 bytes and redacts bearer credentials, +secret-like assignments, URL user information, private-key blocks, and complete +result-marker payloads. Object reason and message fields are also bounded. If a +required object or current log cannot be captured, a restarted container's +previous log is unavailable, or the projection must omit content to fit the +256 KiB output limit, `complete` is false and the run cannot pass. Capture and +cleanup failures remain explicit, while an earlier backend execution failure +continues to be the causal result reason. + ## Multi-Node execution Participant admission is deterministic and atomic. The requesting Node is diff --git a/deploy/signal/manifest_test.go b/deploy/signal/manifest_test.go index b58dbf977e..ff9d34952d 100644 --- a/deploy/signal/manifest_test.go +++ b/deploy/signal/manifest_test.go @@ -62,6 +62,8 @@ func TestAdapterManifestIsGenericAndParticipantAware(t *testing.T) { "participant-count", "request-node-name", "backend-resources", + "backend-evidence", + "/outputs/backend-evidence", "cleanup-failure", "/outputs/cleanup-failure", "backend-api-version", @@ -88,6 +90,28 @@ func TestAdapterManifestIsGenericAndParticipantAware(t *testing.T) { } } +func TestAdapterRBACAllowsBackendEvidenceCapture(t *testing.T) { + t.Parallel() + + content, err := os.ReadFile("03-adapter-serviceaccount-rbac.yaml.tmpl") + if err != nil { + t.Fatal(err) + } + + manifest := string(content) + for _, required := range []string{ + `resources: ["pods"]`, + `resources: ["pods/log"]`, + `apiGroups: ["batch"]`, + `resources: ["jobs"]`, + `verbs: ["get", "list"]`, + } { + if !strings.Contains(manifest, required) { + t.Fatalf("adapter RBAC does not contain %q", required) + } + } +} + func TestSignalCRDsExposeRuntimeStageAndDeadlineContract(t *testing.T) { t.Parallel() diff --git a/internal/signal/controller.go b/internal/signal/controller.go index fcfd73a911..2e629cfdd9 100644 --- a/internal/signal/controller.go +++ b/internal/signal/controller.go @@ -74,6 +74,7 @@ var reservedScriptEnvironmentNames = map[string]struct{}{ var workflowOutputLimits = map[string]int{ "backend-resources": 16 * 1024, + "backend-evidence": 256 * 1024, "cleanup-failure": 64, "measurements": 16 * 1024, "reason": 4 * 1024, @@ -983,6 +984,18 @@ func (r *RunReconciler) finish( } else { run.Status.Workflow.Outputs = outputs + evidenceComplete, evidenceErr := backendEvidenceComplete( + outputs["backend-evidence"], + len(run.Spec.Participants), + ) + if evidenceErr != nil { + run.Status.Verdict = "Inconclusive" + finalReason = "workflow backend evidence rejected: " + evidenceErr.Error() + } else if run.Status.Verdict == "Pass" && !evidenceComplete { + run.Status.Verdict = "Inconclusive" + finalReason = "workflow backend evidence capture was incomplete" + } + backendResources, backendErr := backendResourceReferences( outputs["backend-resources"], len(run.Spec.Participants), @@ -1252,6 +1265,48 @@ func workflowParameters(workflow *unstructured.Unstructured) (map[string]string, return outputs, nil } +func backendEvidenceComplete(content string, participantCount int) (bool, error) { + if content == "" { + return false, errors.New("backend-evidence is required") + } + + var evidence struct { + Version string `json:"version"` + Complete bool `json:"complete"` + Participants []any `json:"participants"` + } + + decoder := json.NewDecoder(strings.NewReader(content)) + decoder.DisallowUnknownFields() + + if err := decoder.Decode(&evidence); err != nil { + return false, fmt.Errorf("decode backend-evidence: %w", err) + } + + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + if err == nil { + return false, errors.New("backend-evidence contains trailing JSON values") + } + + return false, fmt.Errorf("decode trailing backend-evidence value: %w", err) + } + + if evidence.Version != "v1" { + return false, fmt.Errorf("backend-evidence version %q is invalid", evidence.Version) + } + + if len(evidence.Participants) != participantCount { + return false, fmt.Errorf( + "backend-evidence contains %d participants, want %d", + len(evidence.Participants), + participantCount, + ) + } + + return evidence.Complete, nil +} + func (r *RunReconciler) cancel(ctx context.Context, run *signalv1.CertificationRun) (ctrl.Result, error) { stopped, err := r.ensureWorkflowStopped(ctx, run) if err != nil { diff --git a/internal/signal/controller_test.go b/internal/signal/controller_test.go index 66be858a95..a2ca4353f2 100644 --- a/internal/signal/controller_test.go +++ b/internal/signal/controller_test.go @@ -906,6 +906,9 @@ func TestWorkflowParametersRejectMalformedEvidence(t *testing.T) { "oversized cleanup failure": { map[string]any{"name": "cleanup-failure", "value": strings.Repeat("x", 65)}, }, + "oversized backend evidence": { + map[string]any{"name": "backend-evidence", "value": strings.Repeat("x", 256*1024+1)}, + }, "wrong type": { map[string]any{"name": "verdict", "value": true}, }, @@ -928,6 +931,55 @@ func TestWorkflowParametersRejectMalformedEvidence(t *testing.T) { } } +func TestBackendEvidenceCompleteValidation(t *testing.T) { + t.Parallel() + + for name, test := range map[string]struct { + content string + count int + want bool + wantErr bool + }{ + "complete": { + content: `{"version":"v1","complete":true,"participants":[{}]}`, + count: 1, + want: true, + }, + "incomplete": { + content: `{"version":"v1","complete":false,"participants":[{}]}`, + count: 1, + }, + "missing": { + count: 1, + wantErr: true, + }, + "wrong count": { + content: `{"version":"v1","complete":true,"participants":[{},{}]}`, + count: 1, + wantErr: true, + }, + "malformed": { + content: `{"version":"v1","complete":true,"participants":`, + count: 1, + wantErr: true, + }, + "unknown version": { + content: `{"version":"v2","complete":true,"participants":[{}]}`, + count: 1, + wantErr: true, + }, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + got, err := backendEvidenceComplete(test.content, test.count) + if (err != nil) != test.wantErr || got != test.want { + t.Fatalf("backendEvidenceComplete() = %t, %v", got, err) + } + }) + } +} + func TestBackendResourceReferencesRequireCompleteMultiNodeCoverage(t *testing.T) { t.Parallel() @@ -1190,6 +1242,10 @@ func TestTerminalVerdictPersistsEvidenceAndCleanupReleasesNode(t *testing.T) { t.Fatalf("retained workflow outputs = %#v", got.Status.Workflow.Outputs) } + if got.Status.Workflow.Outputs["backend-evidence"] == "" { + t.Fatalf("backend evidence was not retained: %#v", got.Status.Workflow.Outputs) + } + if got.Status.Evidence.URI == "" || len(store.data) == 0 { t.Fatal("evidence was not persisted") } @@ -1205,7 +1261,8 @@ func TestTerminalVerdictPersistsEvidenceAndCleanupReleasesNode(t *testing.T) { } if evidence.Workflow.Phase != "Succeeded" || - evidence.Workflow.Outputs["measurements"] != "checked=14" { + evidence.Workflow.Outputs["measurements"] != "checked=14" || + evidence.Workflow.Outputs["backend-evidence"] != got.Status.Workflow.Outputs["backend-evidence"] { t.Fatalf("evidence workflow = %#v", evidence.Workflow) } @@ -1612,6 +1669,16 @@ func workflowObject(run *signalv1.CertificationRun, phase, verdict, reason strin const workflowUID = "workflow-uid" run.Status.Workflow.UID = workflowUID + + backendEvidence, err := json.Marshal(map[string]any{ + "version": "v1", + "complete": true, + "participants": make([]map[string]any, len(run.Spec.Participants)), + }) + if err != nil { + panic(err) + } + workflow := &unstructured.Unstructured{Object: map[string]any{ "apiVersion": workflowGVK.GroupVersion().String(), "kind": workflowGVK.Kind, @@ -1637,6 +1704,7 @@ func workflowObject(run *signalv1.CertificationRun, phase, verdict, reason strin map[string]any{"name": "verdict", "value": verdict}, map[string]any{"name": "reason", "value": reason}, map[string]any{"name": "measurements", "value": "checked=14"}, + map[string]any{"name": "backend-evidence", "value": string(backendEvidence)}, }}, }, }} diff --git a/internal/signaladapter/adapter.go b/internal/signaladapter/adapter.go index 5dd49c90cd..f47e710d04 100644 --- a/internal/signaladapter/adapter.go +++ b/internal/signaladapter/adapter.go @@ -27,17 +27,18 @@ import ( ) const ( - MaxParticipants = 64 - MaxReasonBytes = 4 * 1024 - MaxMeasurementsBytes = 16 * 1024 - MaxBackendOutputBytes = 16 * 1024 - MaxCleanupFailureBytes = 64 - maxMPIWorkerReadyWait = int64(30) - maxMPIWorkerSlots = int64(1<<31 - 1) - mpiHostfilePath = "/tmp/project-signal-hostfile" - mpiKnownHostsPath = "/tmp/project-signal-known-hosts" - resultPrefix = "PROJECT_SIGNAL_RESULT_V2 " - requiredTopologyKey = "kueue.x-k8s.io/podset-required-topology" + MaxParticipants = 64 + MaxReasonBytes = 4 * 1024 + MaxMeasurementsBytes = 16 * 1024 + MaxBackendOutputBytes = 16 * 1024 + MaxBackendEvidenceBytes = 256 * 1024 + MaxCleanupFailureBytes = 64 + maxMPIWorkerReadyWait = int64(30) + maxMPIWorkerSlots = int64(1<<31 - 1) + mpiHostfilePath = "/tmp/project-signal-hostfile" + mpiKnownHostsPath = "/tmp/project-signal-known-hosts" + resultPrefix = "PROJECT_SIGNAL_RESULT_V2 " + requiredTopologyKey = "kueue.x-k8s.io/podset-required-topology" ) var runnerImagePattern = regexp.MustCompile(`^[^@\s]+@sha256:[0-9a-f]{64}$`) @@ -122,6 +123,7 @@ type AggregateOutput struct { Measurements string `json:"measurements"` BackendResources []signalv1.BackendResourceReference `json:"-"` BackendJSON string `json:"backend-resources"` + BackendEvidence string `json:"backend-evidence"` } func ParseParticipants( @@ -729,6 +731,7 @@ set -u mkdir -p /results chmod 0755 /results rm -f /results/verdict /results/reason /results/measurements.txt +printf 'PROJECT_SIGNAL_RESULTS_V1 prepared\n' set +e %s prelude_status=$? @@ -752,6 +755,10 @@ measurements="$(encode_result_file /results/measurements.txt)" [[ -n "${verdict}" ]] || verdict=- [[ -n "${reason}" ]] || reason=- [[ -n "${measurements}" ]] || measurements=- +printf 'PROJECT_SIGNAL_RESULTS_V1 verdict=%%s reason=%%s measurements=%%s\n' \ + "$([[ -f /results/verdict ]] && printf present || printf missing)" \ + "$([[ -f /results/reason ]] && printf present || printf missing)" \ + "$([[ -f /results/measurements.txt ]] && printf present || printf missing)" printf '%s%%s %%s %%s %%d\n' "${verdict}" "${reason}" "${measurements}" "$status" exit "$status" `, prelude, strings.Join(quotedArguments, " "), resultPrefix), diff --git a/internal/signaladapter/adapter_test.go b/internal/signaladapter/adapter_test.go index ce35aef1cd..a068a75461 100644 --- a/internal/signaladapter/adapter_test.go +++ b/internal/signaladapter/adapter_test.go @@ -1297,6 +1297,8 @@ func TestResultCommandClearsReservedFilesAndReportsExitCode(t *testing.T) { for _, required := range []string{ "rm -f /results/verdict /results/reason /results/measurements.txt", + `printf 'PROJECT_SIGNAL_RESULTS_V1 prepared\n'`, + `printf 'PROJECT_SIGNAL_RESULTS_V1 verdict=%s reason=%s measurements=%s\n'`, `printf 'PROJECT_SIGNAL_RESULT_V2 %s %s %s %d\n'`, `exit "$status"`, } { diff --git a/internal/signaladapter/backend_evidence.go b/internal/signaladapter/backend_evidence.go new file mode 100644 index 0000000000..adb3b7a0a5 --- /dev/null +++ b/internal/signaladapter/backend_evidence.go @@ -0,0 +1,780 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signaladapter + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "regexp" + "slices" + "strconv" + "strings" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" +) + +const ( + backendEvidenceVersion = "v1" + maxEvidenceLogBytes = 512 + maxEvidenceTextBytes = 512 + resultsStatusPrefix = "PROJECT_SIGNAL_RESULTS_V1 " +) + +var ( + sensitiveAssignmentPattern = regexp.MustCompile( + `(?i)((?:token|password|passwd|secret|credential|api[_-]?key)\s*[=:]\s*)([^\s]+)`, + ) + bearerPattern = regexp.MustCompile(`(?i)(authorization:\s*bearer\s+)([^\s]+)`) + urlUserPattern = regexp.MustCompile(`(https?://)[^/@\s]+@`) + resultsFinalPattern = regexp.MustCompile( + `^verdict=(?:present|missing) reason=(?:present|missing) measurements=(?:present|missing)$`, + ) +) + +type BackendEvidence struct { + Version string `json:"version"` + Complete bool `json:"complete"` + Participants []ParticipantBackendEvidence `json:"participants"` + CaptureError string `json:"captureError,omitempty"` +} + +type ParticipantBackendEvidence struct { + ParticipantIndex int `json:"participantIndex"` + Resource BackendObjectEvidence `json:"resource"` + LauncherJob *JobEvidence `json:"launcherJob,omitempty"` + Pods []PodEvidence `json:"pods"` + CaptureError string `json:"captureError,omitempty"` +} + +type BackendObjectEvidence struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Namespace string `json:"namespace,omitempty"` + Name string `json:"name"` + UID types.UID `json:"uid"` + Phase string `json:"phase,omitempty"` + Reason string `json:"reason,omitempty"` + Message string `json:"message,omitempty"` + Conditions []ConditionEvidence `json:"conditions,omitempty"` + ReplicaStatuses map[string]ReplicaStatusEvidence `json:"replicaStatuses,omitempty"` +} + +type ReplicaStatusEvidence struct { + Active int64 `json:"active,omitempty"` + Succeeded int64 `json:"succeeded,omitempty"` + Failed int64 `json:"failed,omitempty"` +} + +type ConditionEvidence struct { + Type string `json:"type"` + Status string `json:"status"` + Reason string `json:"reason,omitempty"` + Message string `json:"message,omitempty"` +} + +type JobEvidence struct { + Name string `json:"name"` + UID types.UID `json:"uid"` + Conditions []ConditionEvidence `json:"conditions,omitempty"` +} + +type PodEvidence struct { + Name string `json:"name"` + UID types.UID `json:"uid"` + Role string `json:"role"` + Phase corev1.PodPhase `json:"phase,omitempty"` + Reason string `json:"reason,omitempty"` + Message string `json:"message,omitempty"` + Containers []ContainerEvidence `json:"containers"` + Conditions []ConditionEvidence `json:"conditions,omitempty"` +} + +type ContainerEvidence struct { + Name string `json:"name"` + RestartCount int32 `json:"restartCount"` + State ContainerStateEvidence `json:"state"` + LastState ContainerStateEvidence `json:"lastState"` + CurrentLog CapturedLogEvidence `json:"currentLog"` + PreviousLog CapturedLogEvidence `json:"previousLog"` +} + +type ContainerStateEvidence struct { + State string `json:"state,omitempty"` + ExitCode int32 `json:"exitCode,omitempty"` + Signal int32 `json:"signal,omitempty"` + Reason string `json:"reason,omitempty"` + Message string `json:"message,omitempty"` + FinishedAt string `json:"finishedAt,omitempty"` +} + +type CapturedLogEvidence struct { + Status string `json:"status"` + Bytes int `json:"bytes,omitempty"` + CapturedBytes int `json:"capturedBytes,omitempty"` + Truncated bool `json:"truncated,omitempty"` + Content string `json:"content,omitempty"` + ResultMarkerStatus string `json:"resultMarkerStatus,omitempty"` + ResultDirectoryStatus string `json:"resultDirectoryStatus,omitempty"` + Error string `json:"error,omitempty"` +} + +func (r *Runner) captureBackendEvidence( + ctx context.Context, + config Config, + backends []createdResource, +) (string, bool, error) { + evidence := BackendEvidence{ + Version: backendEvidenceVersion, + Complete: true, + Participants: make([]ParticipantBackendEvidence, len(backends)), + } + + for index := range backends { + participant, complete := r.captureParticipantEvidence(ctx, config, backends[index]) + evidence.Participants[index] = participant + evidence.Complete = evidence.Complete && complete + } + + content, complete, err := marshalBackendEvidence(evidence) + if err != nil { + return "", false, err + } + + return string(content), complete, nil +} + +func marshalBackendEvidence(evidence BackendEvidence) ([]byte, bool, error) { + content, err := json.Marshal(evidence) + if err != nil { + return nil, false, fmt.Errorf("encode backend evidence: %w", err) + } + + if len(content) <= MaxBackendEvidenceBytes { + return content, evidence.Complete, nil + } + + evidence.Complete = false + for index := range evidence.Participants { + participant := &evidence.Participants[index] + + participant.CaptureError = "container log content omitted because backend evidence exceeded the size limit" + for podIndex := range participant.Pods { + pod := &participant.Pods[podIndex] + for containerIndex := range pod.Containers { + container := &pod.Containers[containerIndex] + omitLogContent(&container.CurrentLog) + omitLogContent(&container.PreviousLog) + } + } + } + + content, err = json.Marshal(evidence) + if err != nil { + return nil, false, fmt.Errorf("encode bounded backend evidence: %w", err) + } + + if len(content) <= MaxBackendEvidenceBytes { + return content, false, nil + } + + for index := range evidence.Participants { + participant := &evidence.Participants[index] + participant.Resource = BackendObjectEvidence{ + APIVersion: participant.Resource.APIVersion, + Kind: participant.Resource.Kind, + Namespace: participant.Resource.Namespace, + Name: participant.Resource.Name, + UID: participant.Resource.UID, + } + participant.LauncherJob = nil + participant.Pods = nil + participant.CaptureError = "backend evidence projection reduced because it exceeded the size limit" + } + + content, err = json.Marshal(evidence) + if err != nil { + return nil, false, fmt.Errorf("encode reduced backend evidence: %w", err) + } + + if len(content) > MaxBackendEvidenceBytes { + return nil, false, fmt.Errorf("reduced backend evidence exceeds %d bytes", MaxBackendEvidenceBytes) + } + + return content, false, nil +} + +func omitLogContent(logEvidence *CapturedLogEvidence) { + if logEvidence.Content == "" { + return + } + + logEvidence.Status = "unavailable" + logEvidence.Content = "" + logEvidence.CapturedBytes = 0 + logEvidence.Error = "content omitted because backend evidence exceeded the size limit" +} + +func markBackendEvidenceIncomplete(content, captureError string) (string, error) { + var evidence BackendEvidence + + if err := json.Unmarshal([]byte(content), &evidence); err != nil { + return "", fmt.Errorf("decode backend evidence after persistence failure: %w", err) + } + + evidence.Complete = false + evidence.CaptureError = boundedRedactedText(captureError, maxEvidenceTextBytes) + + bounded, _, err := marshalBackendEvidence(evidence) + if err != nil { + return "", err + } + + return string(bounded), nil +} + +func (r *Runner) captureParticipantEvidence( + ctx context.Context, + config Config, + backend createdResource, +) (ParticipantBackendEvidence, bool) { + participant := ParticipantBackendEvidence{ + ParticipantIndex: int(backend.participantIndex), + Resource: BackendObjectEvidence{ + APIVersion: backend.resource.GetAPIVersion(), + Kind: backend.resource.GetKind(), + Namespace: backend.resource.GetNamespace(), + Name: backend.resource.GetName(), + UID: backend.resource.GetUID(), + }, + } + complete := true + + object, err := r.Dynamic.Resource(backend.gvr).Namespace(config.Namespace). + Get(ctx, backend.resource.GetName(), metav1.GetOptions{}) + if err != nil { + participant.CaptureError = boundedRedactedText("get backend object: "+err.Error(), maxEvidenceTextBytes) + complete = false + } else { + projected, projectErr := projectBackendObject(object) + if projectErr != nil { + participant.CaptureError = joinCaptureError( + participant.CaptureError, + "project backend object: "+projectErr.Error(), + ) + complete = false + } else { + participant.Resource = projected + } + } + + switch backend.resource.GetKind() { + case "Pod": + pod, podErr := r.Kubernetes.CoreV1().Pods(config.Namespace). + Get(ctx, backend.resource.GetName(), metav1.GetOptions{}) + if podErr != nil { + participant.CaptureError = joinCaptureError(participant.CaptureError, "get participant Pod: "+podErr.Error()) + + return participant, false + } + + podEvidence, podComplete := r.capturePodEvidence(ctx, config.Namespace, pod, "participant") + participant.Pods = []PodEvidence{podEvidence} + complete = complete && podComplete + case "MPIJob": + jobs, jobErr := r.Kubernetes.BatchV1().Jobs(config.Namespace).List(ctx, metav1.ListOptions{}) + if jobErr != nil { + participant.CaptureError = joinCaptureError(participant.CaptureError, "list launcher Jobs: "+jobErr.Error()) + complete = false + } + + jobUIDs := map[types.UID]struct{}{} + + if jobErr == nil { + for index := range jobs.Items { + job := &jobs.Items[index] + if !ownedBy(job.OwnerReferences, backend.resource.GetUID()) { + continue + } + + jobUIDs[job.UID] = struct{}{} + projected := projectJob(job) + participant.LauncherJob = &projected + } + } + + pods, podErr := r.Kubernetes.CoreV1().Pods(config.Namespace).List(ctx, metav1.ListOptions{ + LabelSelector: participantLabelSelector(config.RunName, backend.participantIndex), + }) + if podErr != nil { + participant.CaptureError = joinCaptureError(participant.CaptureError, "list MPI Pods: "+podErr.Error()) + + return participant, false + } + + for index := range pods.Items { + pod := &pods.Items[index] + + role := mpiPodRole(pod, backend.resource.GetUID(), jobUIDs) + if role == "" { + continue + } + + podEvidence, podComplete := r.capturePodEvidence(ctx, config.Namespace, pod, role) + participant.Pods = append(participant.Pods, podEvidence) + complete = complete && podComplete + } + + slices.SortFunc(participant.Pods, func(left, right PodEvidence) int { + return strings.Compare(left.Name, right.Name) + }) + + if participant.LauncherJob == nil { + participant.CaptureError = joinCaptureError(participant.CaptureError, "MPI launcher Job was not found") + complete = false + } + + if !hasPodRole(participant.Pods, "launcher") || !hasPodRole(participant.Pods, "worker") { + participant.CaptureError = joinCaptureError( + participant.CaptureError, + "MPI launcher and worker Pods were not both found", + ) + complete = false + } + default: + participant.CaptureError = joinCaptureError( + participant.CaptureError, + fmt.Sprintf("unsupported backend kind %q", backend.resource.GetKind()), + ) + complete = false + } + + return participant, complete +} + +func (r *Runner) capturePodEvidence( + ctx context.Context, + namespace string, + pod *corev1.Pod, + role string, +) (PodEvidence, bool) { + evidence := PodEvidence{ + Name: pod.Name, + UID: pod.UID, + Role: role, + Phase: pod.Status.Phase, + Reason: boundedRedactedText(pod.Status.Reason, maxEvidenceTextBytes), + Message: boundedRedactedText(pod.Status.Message, maxEvidenceTextBytes), + Conditions: projectPodConditions(pod.Status.Conditions), + } + complete := true + + statuses := append(slices.Clone(pod.Status.InitContainerStatuses), pod.Status.ContainerStatuses...) + for _, status := range statuses { + if !capturedContainer(role, status.Name) { + continue + } + + container := ContainerEvidence{ + Name: status.Name, + RestartCount: status.RestartCount, + State: projectContainerState(status.State), + LastState: projectContainerState(status.LastTerminationState), + } + + current, currentErr := r.readPodLog(ctx, namespace, pod.Name, status.Name, false) + + container.CurrentLog = capturedLog(current, currentErr) + if currentErr != nil { + complete = false + } + + if status.RestartCount > 0 || status.LastTerminationState.Terminated != nil { + previous, previousErr := r.readPodLog(ctx, namespace, pod.Name, status.Name, true) + + container.PreviousLog = capturedLog(previous, previousErr) + if previousErr != nil { + complete = false + } + } else { + container.PreviousLog = CapturedLogEvidence{Status: "not-applicable"} + } + + evidence.Containers = append(evidence.Containers, container) + } + + if len(evidence.Containers) == 0 { + complete = false + } + + return evidence, complete +} + +func projectBackendObject(object *unstructured.Unstructured) (BackendObjectEvidence, error) { + evidence := BackendObjectEvidence{ + APIVersion: object.GetAPIVersion(), + Kind: object.GetKind(), + Namespace: object.GetNamespace(), + Name: object.GetName(), + UID: object.GetUID(), + } + + var err error + + evidence.Phase, _, err = unstructured.NestedString(object.Object, "status", "phase") + if err != nil { + return evidence, fmt.Errorf("read status.phase: %w", err) + } + + evidence.Reason, _, err = unstructured.NestedString(object.Object, "status", "reason") + if err != nil { + return evidence, fmt.Errorf("read status.reason: %w", err) + } + + evidence.Message, _, err = unstructured.NestedString(object.Object, "status", "message") + if err != nil { + return evidence, fmt.Errorf("read status.message: %w", err) + } + + evidence.Reason = boundedRedactedText(evidence.Reason, maxEvidenceTextBytes) + evidence.Message = boundedRedactedText(evidence.Message, maxEvidenceTextBytes) + + evidence.Conditions, err = projectUnstructuredConditions(object) + if err != nil { + return evidence, err + } + + statuses, found, err := unstructured.NestedMap(object.Object, "status", "replicaStatuses") + if err != nil { + return evidence, fmt.Errorf("read status.replicaStatuses: %w", err) + } + + if found { + evidence.ReplicaStatuses = make(map[string]ReplicaStatusEvidence, len(statuses)) + for name := range statuses { + active, countErr := nestedReplicaCount(statuses, name, "active") + if countErr != nil { + return evidence, countErr + } + + succeeded, countErr := nestedReplicaCount(statuses, name, "succeeded") + if countErr != nil { + return evidence, countErr + } + + failed, countErr := nestedReplicaCount(statuses, name, "failed") + if countErr != nil { + return evidence, countErr + } + + evidence.ReplicaStatuses[name] = ReplicaStatusEvidence{ + Active: active, + Succeeded: succeeded, + Failed: failed, + } + } + } + + return evidence, nil +} + +func nestedReplicaCount(statuses map[string]any, replica, field string) (int64, error) { + value, _, err := unstructured.NestedInt64(statuses, replica, field) + if err != nil { + return 0, fmt.Errorf("read status.replicaStatuses.%s.%s: %w", replica, field, err) + } + + return value, nil +} + +func projectUnstructuredConditions(object *unstructured.Unstructured) ([]ConditionEvidence, error) { + conditions, found, err := unstructured.NestedSlice(object.Object, "status", "conditions") + if err != nil { + return nil, fmt.Errorf("read status.conditions: %w", err) + } + + if !found { + return nil, nil + } + + projected := make([]ConditionEvidence, 0, len(conditions)) + for _, raw := range conditions { + condition, ok := raw.(map[string]any) + if !ok { + continue + } + + projected = append(projected, ConditionEvidence{ + Type: stringValue(condition["type"]), + Status: stringValue(condition["status"]), + Reason: boundedRedactedText(stringValue(condition["reason"]), maxEvidenceTextBytes), + Message: boundedRedactedText(stringValue(condition["message"]), maxEvidenceTextBytes), + }) + } + + return projected, nil +} + +func projectJob(job *batchv1.Job) JobEvidence { + conditions := make([]ConditionEvidence, 0, len(job.Status.Conditions)) + for _, condition := range job.Status.Conditions { + conditions = append(conditions, ConditionEvidence{ + Type: string(condition.Type), + Status: string(condition.Status), + Reason: boundedRedactedText(condition.Reason, maxEvidenceTextBytes), + Message: boundedRedactedText(condition.Message, maxEvidenceTextBytes), + }) + } + + return JobEvidence{Name: job.Name, UID: job.UID, Conditions: conditions} +} + +func projectPodConditions(conditions []corev1.PodCondition) []ConditionEvidence { + projected := make([]ConditionEvidence, 0, len(conditions)) + for _, condition := range conditions { + projected = append(projected, ConditionEvidence{ + Type: string(condition.Type), + Status: string(condition.Status), + Reason: boundedRedactedText(condition.Reason, maxEvidenceTextBytes), + Message: boundedRedactedText(condition.Message, maxEvidenceTextBytes), + }) + } + + return projected +} + +func projectContainerState(state corev1.ContainerState) ContainerStateEvidence { + switch { + case state.Terminated != nil: + return ContainerStateEvidence{ + State: "terminated", + ExitCode: state.Terminated.ExitCode, + Signal: state.Terminated.Signal, + Reason: boundedRedactedText(state.Terminated.Reason, maxEvidenceTextBytes), + Message: boundedRedactedText(state.Terminated.Message, maxEvidenceTextBytes), + FinishedAt: state.Terminated.FinishedAt.UTC().Format("2006-01-02T15:04:05.999999999Z07:00"), + } + case state.Waiting != nil: + return ContainerStateEvidence{ + State: "waiting", + Reason: boundedRedactedText(state.Waiting.Reason, maxEvidenceTextBytes), + Message: boundedRedactedText(state.Waiting.Message, maxEvidenceTextBytes), + } + case state.Running != nil: + return ContainerStateEvidence{State: "running"} + default: + return ContainerStateEvidence{} + } +} + +func capturedLog(content string, err error) CapturedLogEvidence { + if err != nil { + return CapturedLogEvidence{ + Status: "unavailable", + Error: boundedRedactedText(err.Error(), maxEvidenceTextBytes), + } + } + + redacted := redactLog(content) + + truncated := len(redacted) > maxEvidenceLogBytes + if truncated { + redacted = redacted[len(redacted)-maxEvidenceLogBytes:] + } + + return CapturedLogEvidence{ + Status: "captured", + Bytes: len(content), + CapturedBytes: len(redacted), + Truncated: truncated, + Content: redacted, + ResultMarkerStatus: resultLogSourceMetadata(content, nil).Status, + ResultDirectoryStatus: resultDirectoryStatus(content), + } +} + +func redactLog(content string) string { + lines := strings.Split(content, "\n") + inPrivateKey := false + + for index, line := range lines { + if strings.Contains(line, "BEGIN ") && strings.Contains(line, "PRIVATE KEY") { + inPrivateKey = true + lines[index] = "[REDACTED PRIVATE KEY]" + + continue + } + + if inPrivateKey { + lines[index] = "[REDACTED PRIVATE KEY]" + + if strings.Contains(line, "END ") && strings.Contains(line, "PRIVATE KEY") { + inPrivateKey = false + } + + continue + } + + if strings.HasPrefix(line, resultPrefix) { + lines[index] = resultPrefix + "[REDACTED]" + + continue + } + + line = bearerPattern.ReplaceAllString(line, `${1}[REDACTED]`) + line = sensitiveAssignmentPattern.ReplaceAllString(line, `${1}[REDACTED]`) + lines[index] = urlUserPattern.ReplaceAllString(line, `${1}[REDACTED]@`) + } + + return strings.Join(lines, "\n") +} + +func resultDirectoryStatus(content string) string { + status := "missing" + + for _, line := range strings.Split(content, "\n") { + if strings.HasPrefix(line, resultsStatusPrefix) { + value := strings.TrimPrefix(line, resultsStatusPrefix) + switch { + case value == "prepared", resultsFinalPattern.MatchString(value): + status = value + default: + status = "invalid" + } + } + } + + return status +} + +func boundedRedactedText(value string, limit int) string { + value = redactLog(value) + if len(value) <= limit { + return value + } + + return value[:limit] +} + +func participantLabelSelector(runName string, participantIndex int32) string { + return strings.Join([]string{ + "signal.unbounded-cloud.io/run=" + runName, + "signal.unbounded-cloud.io/participant-index=" + strconv.FormatInt(int64(participantIndex), 10), + }, ",") +} + +func mpiPodRole(pod *corev1.Pod, mpiJobUID types.UID, jobUIDs map[types.UID]struct{}) string { + if ownedBy(pod.OwnerReferences, mpiJobUID) && hasContainer(pod, "worker") { + return "worker" + } + + for _, reference := range pod.OwnerReferences { + if _, exists := jobUIDs[reference.UID]; exists && hasContainer(pod, "launcher") { + return "launcher" + } + } + + return "" +} + +func hasContainer(pod *corev1.Pod, name string) bool { + return slices.ContainsFunc(pod.Spec.Containers, func(container corev1.Container) bool { + return container.Name == name + }) +} + +func capturedContainer(role, name string) bool { + switch role { + case "participant": + return name == "main" + case "launcher": + return name == "launcher" + case "worker": + return name == "worker" + default: + return false + } +} + +func hasPodRole(pods []PodEvidence, role string) bool { + return slices.ContainsFunc(pods, func(pod PodEvidence) bool { + return pod.Role == role + }) +} + +func joinCaptureError(current, next string) string { + next = boundedRedactedText(next, maxEvidenceTextBytes) + if current == "" { + return next + } + + return boundedRedactedText(current+"; "+next, maxEvidenceTextBytes) +} + +func stringValue(value any) string { + text, ok := value.(string) + if !ok { + return "" + } + + return text +} + +func recordEvidenceCaptureFailure(output *AggregateOutput) { + output.ExecutionState = "Error" + output.Verdict = "Inconclusive" + + if output.Reason == "" || output.Reason == "AllParticipantsPassed" || + output.Reason == "ParticipantCertificationFailed" || + output.Reason == "ParticipantCertificationInconclusive" { + output.Reason = "BackendEvidenceCaptureFailed" + } +} + +func ValidateBackendEvidence(content string) error { + if content == "" { + return errors.New("backend evidence is required") + } + + if len(content) > MaxBackendEvidenceBytes { + return fmt.Errorf("backend evidence exceeds %d bytes", MaxBackendEvidenceBytes) + } + + var evidence BackendEvidence + + decoder := json.NewDecoder(strings.NewReader(content)) + decoder.DisallowUnknownFields() + + if err := decoder.Decode(&evidence); err != nil { + return fmt.Errorf("decode backend evidence: %w", err) + } + + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + if err == nil { + return errors.New("backend evidence contains trailing JSON values") + } + + return fmt.Errorf("decode trailing backend evidence value: %w", err) + } + + if evidence.Version != backendEvidenceVersion { + return fmt.Errorf("backend evidence version %q is invalid", evidence.Version) + } + + if len(evidence.Participants) < 1 || len(evidence.Participants) > MaxParticipants { + return fmt.Errorf( + "backend evidence must contain between 1 and %d participants", + MaxParticipants, + ) + } + + return nil +} diff --git a/internal/signaladapter/backend_evidence_test.go b/internal/signaladapter/backend_evidence_test.go new file mode 100644 index 0000000000..67f279f548 --- /dev/null +++ b/internal/signaladapter/backend_evidence_test.go @@ -0,0 +1,481 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signaladapter + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubernetesfake "k8s.io/client-go/kubernetes/fake" +) + +func TestCaptureBackendEvidencePreservesBackoffFailureAndPreviousInvalidResult(t *testing.T) { + t.Parallel() + + const ( + namespace = "project-signal" + mpiJobUID = types.UID("mpi-job-uid") + launcherJobUID = types.UID("launcher-job-uid") + ) + + mpiJob := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "kubeflow.org/v2beta1", + "kind": "MPIJob", + "metadata": map[string]any{ + "name": "run-p0", + "namespace": namespace, + "uid": string(mpiJobUID), + }, + "status": map[string]any{ + "conditions": []any{map[string]any{ + "type": "Failed", + "status": "True", + "reason": "LauncherJobFailed", + "message": "launcher failed token=secret-value", + }}, + "replicaStatuses": map[string]any{ + "Launcher": map[string]any{"failed": int64(1)}, + "Worker": map[string]any{"active": int64(1)}, + }, + }, + }} + launcherJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "run-p0-launcher", + Namespace: namespace, + UID: launcherJobUID, + OwnerReferences: []metav1.OwnerReference{{UID: mpiJobUID}}, + }, + Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{ + Type: batchv1.JobFailed, + Status: corev1.ConditionTrue, + Reason: batchv1.JobReasonBackoffLimitExceeded, + Message: "Job has reached the specified backoff limit", + }}}, + } + launcher := evidenceTestPod( + "run-p0-launcher-pod", + "launcher-pod-uid", + "launcher", + []metav1.OwnerReference{{UID: launcherJobUID}}, + 1, + ) + worker := evidenceTestPod( + "run-p0-worker-0", + "worker-pod-uid", + "worker", + []metav1.OwnerReference{{UID: mpiJobUID}}, + 0, + ) + + dynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds( + runtime.NewScheme(), + map[schema.GroupVersionResource]string{mpiJobGVR: "MPIJobList"}, + mpiJob, + ) + runner := &Runner{ + Dynamic: dynamicClient, + Kubernetes: kubernetesfake.NewSimpleClientset(launcherJob, launcher, worker), + ReadPodLog: func( + _ context.Context, + _, podName, containerName string, + previous bool, + ) (string, error) { + switch { + case podName == launcher.Name && containerName == "launcher" && !previous: + return "current launcher log password=hunter2\n", nil + case podName == launcher.Name && containerName == "launcher" && previous: + return resultsStatusPrefix + + "verdict=present reason=present measurements=missing\n" + + resultPrefix + "%%% bm90LXVzZWQ= - 2\n", nil + case podName == worker.Name && containerName == "worker" && !previous: + return resultsStatusPrefix + "prepared\nworker ready\n", nil + default: + return "", errors.New("unexpected log request") + } + }, + } + + content, complete, err := runner.captureBackendEvidence( + t.Context(), + Config{Namespace: namespace, RunName: "run"}, + []createdResource{{ + gvr: mpiJobGVR, + resource: mpiJob, + participantIndex: 0, + }}, + ) + if err != nil { + t.Fatal(err) + } + + if !complete { + t.Fatalf("captureBackendEvidence complete = false, content=%s", content) + } + + var evidence BackendEvidence + if err := json.Unmarshal([]byte(content), &evidence); err != nil { + t.Fatal(err) + } + + participant := evidence.Participants[0] + if participant.LauncherJob == nil || + len(participant.LauncherJob.Conditions) != 1 || + participant.LauncherJob.Conditions[0].Reason != batchv1.JobReasonBackoffLimitExceeded { + t.Fatalf("launcher Job evidence = %#v", participant.LauncherJob) + } + + launcherEvidence := podEvidenceByRole(t, participant.Pods, "launcher") + + container := launcherEvidence.Containers[0] + if container.RestartCount != 1 || + container.State.ExitCode != 1 || + container.PreviousLog.ResultMarkerStatus != "invalid" || + container.PreviousLog.ResultDirectoryStatus != + "verdict=present reason=present measurements=missing" { + t.Fatalf("launcher container evidence = %#v", container) + } + + if strings.Contains(container.CurrentLog.Content, "hunter2") || + !strings.Contains(container.CurrentLog.Content, "password=[REDACTED]") { + t.Fatalf("current log was not redacted: %q", container.CurrentLog.Content) + } + + result, hardFailure := backendCompletionFromLogs( + resultLogSnapshot{ + current: resultLogSourceWithContent("current launcher log\n", nil), + previous: resultLogSourceWithContent( + resultPrefix+"%%% bm90LXVzZWQ= - 2\n", + nil, + ), + }, + 0, + false, + ) + if !hardFailure { + t.Fatal("invalid previous result did not fail closed") + } + + output, err := Aggregate(1, testBackendResources()[:1], []Result{result}) + if err != nil { + t.Fatal(err) + } + + output.BackendEvidence = content + recordCleanupFailure(&output) + + if output.Reason != "BackendExecutionFailed" || + output.CleanupFailure != "BackendCleanupFailed" || + output.Verdict != "Inconclusive" { + t.Fatalf("output = %#v", output) + } +} + +func TestCaptureBackendEvidenceProjectsParticipantPod(t *testing.T) { + t.Parallel() + + backend := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": map[string]any{ + "name": "run-p0", + "namespace": "project-signal", + "uid": "pod-uid", + }, + "status": map[string]any{"phase": "Succeeded"}, + }} + pod := evidenceTestPod("run-p0", "pod-uid", "main", nil, 0) + pod.Status.Phase = corev1.PodSucceeded + pod.Status.ContainerStatuses[0].State.Terminated.ExitCode = 0 + pod.Status.ContainerStatuses[0].State.Terminated.Reason = "Completed" + + runner := &Runner{ + Dynamic: dynamicfake.NewSimpleDynamicClientWithCustomListKinds( + runtime.NewScheme(), + map[schema.GroupVersionResource]string{podGVR: "PodList"}, + backend, + ), + Kubernetes: kubernetesfake.NewSimpleClientset(pod), + ReadPodLog: func( + _ context.Context, + _, _, _ string, + previous bool, + ) (string, error) { + if previous { + t.Fatal("read previous log without a restart") + } + + return resultsStatusPrefix + "prepared\n", nil + }, + } + + content, complete, err := runner.captureBackendEvidence( + context.Background(), + Config{Namespace: "project-signal", RunName: "run"}, + []createdResource{{ + gvr: podGVR, + resource: backend, + participantIndex: 0, + }}, + ) + if err != nil { + t.Fatal(err) + } + + if !complete { + t.Fatalf("captureBackendEvidence complete = false, content=%s", content) + } + + var evidence BackendEvidence + if err := json.Unmarshal([]byte(content), &evidence); err != nil { + t.Fatal(err) + } + + participant := evidence.Participants[0] + if participant.Resource.Phase != "Succeeded" || + len(participant.Pods) != 1 || + participant.Pods[0].Role != "participant" || + participant.Pods[0].Containers[0].PreviousLog.Status != "not-applicable" { + t.Fatalf("participant evidence = %#v", participant) + } +} + +func TestCaptureBackendEvidenceFailurePreventsSuccess(t *testing.T) { + t.Parallel() + + output, err := Aggregate(1, testBackendResources()[:1], []Result{{ + ParticipantIndex: 0, + ExecutionState: "Succeeded", + Verdict: "Pass", + Reason: "Healthy", + }}) + if err != nil { + t.Fatal(err) + } + + recordEvidenceCaptureFailure(&output) + + if output.ExecutionState != "Error" || + output.Verdict != "Inconclusive" || + output.Reason != "BackendEvidenceCaptureFailed" { + t.Fatalf("output = %#v", output) + } +} + +func TestPersistBackendEvidenceFailurePreventsSuccess(t *testing.T) { + t.Parallel() + + output, err := Aggregate(1, testBackendResources()[:1], []Result{{ + ParticipantIndex: 0, + ExecutionState: "Succeeded", + Verdict: "Pass", + Reason: "Healthy", + }}) + if err != nil { + t.Fatal(err) + } + + output.BackendEvidence = `{"version":"v1","complete":true,"participants":[{}]}` + runner := &Runner{ + PersistBackendEvidence: func(string) error { + return errors.New("write failed") + }, + } + + if err := runner.persistEvidenceBeforeCleanup(&output); err != nil { + t.Fatal(err) + } + + var evidence BackendEvidence + if err := json.Unmarshal([]byte(output.BackendEvidence), &evidence); err != nil { + t.Fatal(err) + } + + if output.ExecutionState != "Error" || + output.Verdict != "Inconclusive" || + output.Reason != "BackendEvidenceCaptureFailed" || + evidence.Complete || + !strings.Contains(evidence.CaptureError, "persist backend evidence before cleanup") { + t.Fatalf("output = %#v, evidence = %#v", output, evidence) + } +} + +func TestCapturedLogIsBoundedAndRedacted(t *testing.T) { + t.Parallel() + + content := strings.Repeat("x", maxEvidenceLogBytes+64) + + "\nAuthorization: Bearer secret-token\n" + + "https://user:password@example.test/path\n" + captured := capturedLog(content, nil) + + if captured.CapturedBytes > maxEvidenceLogBytes || + !captured.Truncated || + strings.Contains(captured.Content, "secret-token") || + strings.Contains(captured.Content, "user:password") { + t.Fatalf("captured log = %#v", captured) + } +} + +func TestCapturedLogRedactsSensitiveValuesInRetainedTail(t *testing.T) { + t.Parallel() + + content := strings.Repeat("x", maxEvidenceLogBytes+64) + + "\nAuthorization: " + "Bearer " + "opaque-value\n" + + "https://" + "alice:credential-value@" + "example.test/path\n" + captured := capturedLog(content, nil) + + if strings.Contains(captured.Content, "opaque-value") || + strings.Contains(captured.Content, "alice:credential-value") { + t.Fatalf("captured log = %#v", captured) + } +} + +func TestResultDirectoryStatusRejectsUnstructuredMarker(t *testing.T) { + t.Parallel() + + const value = "do-not-retain" + + status := resultDirectoryStatus(resultsStatusPrefix + value) + if status != "invalid" || strings.Contains(status, value) { + t.Fatalf("resultDirectoryStatus() = %q", status) + } +} + +func TestMarshalBackendEvidenceBoundsOversizedProjection(t *testing.T) { + t.Parallel() + + evidence := BackendEvidence{ + Version: backendEvidenceVersion, + Complete: true, + Participants: make([]ParticipantBackendEvidence, 64), + } + for participantIndex := range evidence.Participants { + participant := &evidence.Participants[participantIndex] + participant.ParticipantIndex = participantIndex + participant.Resource = BackendObjectEvidence{ + APIVersion: "kubeflow.org/v2beta1", + Kind: "MPIJob", + Namespace: "project-signal", + Name: fmt.Sprintf("run-p%d", participantIndex), + UID: types.UID(fmt.Sprintf("mpi-%d", participantIndex)), + Message: strings.Repeat("m", maxEvidenceTextBytes), + } + + participant.Pods = make([]PodEvidence, 4) + for podIndex := range participant.Pods { + pod := &participant.Pods[podIndex] + pod.Name = fmt.Sprintf("run-p%d-%d", participantIndex, podIndex) + pod.UID = types.UID(fmt.Sprintf("pod-%d-%d", participantIndex, podIndex)) + pod.Containers = []ContainerEvidence{{ + Name: "participant", + CurrentLog: CapturedLogEvidence{ + Status: "captured", + Bytes: maxEvidenceLogBytes, + CapturedBytes: maxEvidenceLogBytes, + Content: strings.Repeat("x", maxEvidenceLogBytes), + }, + PreviousLog: CapturedLogEvidence{ + Status: "captured", + Bytes: maxEvidenceLogBytes, + CapturedBytes: maxEvidenceLogBytes, + Content: strings.Repeat("y", maxEvidenceLogBytes), + }, + }} + } + } + + content, complete, err := marshalBackendEvidence(evidence) + if err != nil { + t.Fatal(err) + } + + if complete { + t.Fatal("oversized evidence remained complete") + } + + if len(content) > MaxBackendEvidenceBytes { + t.Fatalf("backend evidence size = %d, want <= %d", len(content), MaxBackendEvidenceBytes) + } + + if err := ValidateBackendEvidence(string(content)); err != nil { + t.Fatalf("ValidateBackendEvidence() error = %v", err) + } + + var bounded BackendEvidence + if err := json.Unmarshal(content, &bounded); err != nil { + t.Fatal(err) + } + + if bounded.Complete || bounded.Participants[0].CaptureError == "" { + t.Fatalf("bounded evidence = %#v", bounded) + } +} + +func evidenceTestPod( + name string, + uid types.UID, + containerName string, + owners []metav1.OwnerReference, + restarts int32, +) *corev1.Pod { + lastState := corev1.ContainerState{} + if restarts > 0 { + lastState.Terminated = &corev1.ContainerStateTerminated{ + ExitCode: 1, + Reason: "Error", + } + } + + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "project-signal", + UID: uid, + OwnerReferences: owners, + Labels: map[string]string{ + "signal.unbounded-cloud.io/run": "run", + "signal.unbounded-cloud.io/participant-index": "0", + }, + }, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: containerName}}}, + Status: corev1.PodStatus{ + Phase: corev1.PodFailed, + ContainerStatuses: []corev1.ContainerStatus{{ + Name: containerName, + RestartCount: restarts, + State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ + ExitCode: 1, + Reason: "Error", + }}, + LastTerminationState: lastState, + }}, + }, + } +} + +func podEvidenceByRole(t *testing.T, pods []PodEvidence, role string) PodEvidence { + t.Helper() + + for _, pod := range pods { + if pod.Role == role { + return pod + } + } + + t.Fatalf("pod role %q was not captured: %#v", role, pods) + + return PodEvidence{} +} diff --git a/internal/signaladapter/runner.go b/internal/signaladapter/runner.go index d143bc1973..f6a5609798 100644 --- a/internal/signaladapter/runner.go +++ b/internal/signaladapter/runner.go @@ -8,7 +8,6 @@ import ( "errors" "fmt" "io" - "strconv" "strings" "sync" "time" @@ -32,11 +31,12 @@ var ( ) type Runner struct { - Dynamic dynamic.Interface - Kubernetes kubernetes.Interface - PollInterval time.Duration - CleanupWait time.Duration - ReadPodLog func(context.Context, string, string, string, bool) (string, error) + Dynamic dynamic.Interface + Kubernetes kubernetes.Interface + PollInterval time.Duration + CleanupWait time.Duration + ReadPodLog func(context.Context, string, string, string, bool) (string, error) + PersistBackendEvidence func(string) error } type createdResource struct { @@ -184,6 +184,28 @@ func (r *Runner) Execute(ctx context.Context, config Config) (output AggregateOu return AggregateOutput{}, err } + captureContext, cancelCapture := context.WithTimeout(context.Background(), r.cleanupWait()) + defer cancelCapture() + + var evidenceComplete bool + + output.BackendEvidence, evidenceComplete, err = r.captureBackendEvidence( + captureContext, + config, + created, + ) + if err != nil { + return AggregateOutput{}, err + } + + if !evidenceComplete { + recordEvidenceCaptureFailure(&output) + } + + if err := r.persistEvidenceBeforeCleanup(&output); err != nil { + return AggregateOutput{}, err + } + cleanupContext, cancel := context.WithTimeout(context.Background(), r.cleanupWait()) defer cancel() @@ -198,6 +220,28 @@ func (r *Runner) Execute(ctx context.Context, config Config) (output AggregateOu return output, nil } +func (r *Runner) persistEvidenceBeforeCleanup(output *AggregateOutput) error { + if r.PersistBackendEvidence == nil { + return nil + } + + if err := r.PersistBackendEvidence(output.BackendEvidence); err != nil { + var markErr error + + output.BackendEvidence, markErr = markBackendEvidenceIncomplete( + output.BackendEvidence, + "persist backend evidence before cleanup: "+err.Error(), + ) + if markErr != nil { + return markErr + } + + recordEvidenceCaptureFailure(output) + } + + return nil +} + func recordCleanupFailure(output *AggregateOutput) { output.ExecutionState = "Error" output.Verdict = "Inconclusive" @@ -510,12 +554,27 @@ func (r *Runner) resultLogs( containerName := "main" if backend.resource.GetKind() == "MPIJob" { + jobs, err := r.Kubernetes.BatchV1().Jobs(config.Namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + source := resultLogSourceFromError(fmt.Errorf("list MPIJob launcher Jobs: %w", err)) + + return resultLogSnapshot{ + current: source, + previous: resultLogSourceMetadataOnly("not-read"), + } + } + + jobUIDs := make(map[types.UID]struct{}) + + for index := range jobs.Items { + job := &jobs.Items[index] + if ownedBy(job.OwnerReferences, backend.resource.GetUID()) { + jobUIDs[job.UID] = struct{}{} + } + } + pods, err := r.Kubernetes.CoreV1().Pods(config.Namespace).List(ctx, metav1.ListOptions{ - LabelSelector: strings.Join([]string{ - "signal.unbounded-cloud.io/run=" + config.RunName, - "signal.unbounded-cloud.io/participant-index=" + - strconv.FormatInt(int64(backend.participantIndex), 10), - }, ","), + LabelSelector: participantLabelSelector(config.RunName, backend.participantIndex), }) if err != nil { source := resultLogSourceFromError(fmt.Errorf("list MPIJob Pods: %w", err)) @@ -530,17 +589,11 @@ func (r *Runner) resultLogs( for index := range pods.Items { pod := &pods.Items[index] - if !ownedBy(pod.OwnerReferences, backend.resource.GetUID()) { - continue - } - - for _, container := range pod.Spec.Containers { - if container.Name == "launcher" { - podName = pod.Name - containerName = container.Name + if mpiPodRole(pod, backend.resource.GetUID(), jobUIDs) == "launcher" { + podName = pod.Name + containerName = "launcher" - break - } + break } } diff --git a/internal/signaladapter/runner_test.go b/internal/signaladapter/runner_test.go index 6f2e2e06e6..6ec2d68f5a 100644 --- a/internal/signaladapter/runner_test.go +++ b/internal/signaladapter/runner_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -202,15 +203,23 @@ func TestResultLogsReadsCurrentAndPreviousLauncherLogs(t *testing.T) { "signal.unbounded-cloud.io/run": "run", "signal.unbounded-cloud.io/participant-index": "0", }, - OwnerReferences: []metav1.OwnerReference{{UID: "job-uid"}}, + OwnerReferences: []metav1.OwnerReference{{UID: "launcher-job-uid"}}, }, Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "launcher"}}}, } + launcherJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "run-p0-launcher", + Namespace: "project-signal", + UID: "launcher-job-uid", + OwnerReferences: []metav1.OwnerReference{{UID: "job-uid"}}, + }, + } var previousValues []bool runner := &Runner{ - Kubernetes: kubernetesfake.NewSimpleClientset(launcher), + Kubernetes: kubernetesfake.NewSimpleClientset(launcher, launcherJob), ReadPodLog: func( _ context.Context, namespace, podName, containerName string, From 8bae85034c656bab96ba3e17f0830260ee104721 Mon Sep 17 00:00:00 2001 From: chokevin Date: Thu, 22 Oct 2026 07:58:36 -0700 Subject: [PATCH 19/36] docs(signal): define evidence crash boundary --- deploy/signal/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/deploy/signal/README.md b/deploy/signal/README.md index 46fba22e3e..71afbfa5bc 100644 --- a/deploy/signal/README.md +++ b/deploy/signal/README.md @@ -122,6 +122,14 @@ previous log is unavailable, or the projection must omit content to fit the cleanup failures remain explicit, while an earlier backend execution failure continues to be the causal result reason. +The adapter cannot observe or serialize its own state if the adapter container +or Node terminates before this pre-cleanup write. In that boundary the +`backend-evidence` output is missing and the controller rejects the Workflow +outputs as Inconclusive; the backend cause remains unknown rather than being +inferred. A termination after the write may leave the evidence file available +to Argo, but collection still depends on the Workflow executor surviving long +enough to publish output parameters. + ## Multi-Node execution Participant admission is deterministic and atomic. The requesting Node is From 6d81f367e407ffcf7058ec646bc118e4a523262b Mon Sep 17 00:00:00 2001 From: chokevin Date: Thu, 22 Oct 2026 10:40:18 -0700 Subject: [PATCH 20/36] fix(signal): recover canonical workflow outputs --- deploy/signal/README.md | 15 ++- internal/signal/controller.go | 140 ++++++++++++------- internal/signal/controller_test.go | 179 ++++++++++++++++++++++++- internal/signaladapter/adapter.go | 10 ++ internal/signaladapter/adapter_test.go | 64 +++++++++ 5 files changed, 353 insertions(+), 55 deletions(-) diff --git a/deploy/signal/README.md b/deploy/signal/README.md index 71afbfa5bc..d77c8e6e27 100644 --- a/deploy/signal/README.md +++ b/deploy/signal/README.md @@ -102,7 +102,11 @@ Unexpected, duplicate, malformed, or oversized outputs are rejected and the run remains fail-closed. Each `backend-resources` entry must contain `apiVersion`, `kind`, `name`, `uid`, and the zero-based `participantIndexes` that the workload executed. `namespace` is optional. A multi-Node successful -result must cover every admitted participant. +result must cover every admitted participant. The controller prefers present +top-level Workflow outputs. When Argo leaves those outputs absent, it accepts +outputs only from the unique Workflow entrypoint node whose name exactly +matches the Workflow name; missing, malformed, duplicate, or ambiguous +entrypoint identity and outputs fail closed. Before foreground deletion, the adapter captures and writes a typed v1 projection for every participant backend to the Argo output path. The @@ -155,10 +159,11 @@ Scripts write `verdict`, `reason`, and optional `measurements.txt` under `Fail` exits 1, and `Inconclusive` exits 2. A valid bounded result remains authoritative when its verdict-associated exit code makes the backend Pod or MPIJob terminally unsuccessful. The adapter removes reserved result files -before each invocation and rejects missing files, malformed values, stale or -contradictory result/exit combinations, and any other exit code. Partial -backend execution, identity gaps, deadline interruption, and cleanup failure -also remain `Inconclusive`. +before each invocation and removes trailing newline bytes from repository-owned +result files before encoding them. It rejects missing files, malformed values, +stale or contradictory result/exit combinations, and any other exit code. +Partial backend execution, identity gaps, deadline interruption, and cleanup +failure also remain `Inconclusive`. The MPI launcher also applies a compatibility fence for [`kubeflow/mpi-operator#615`](https://github.com/kubeflow/mpi-operator/issues/615), diff --git a/internal/signal/controller.go b/internal/signal/controller.go index 2e629cfdd9..0ff36b5dcc 100644 --- a/internal/signal/controller.go +++ b/internal/signal/controller.go @@ -1225,13 +1225,17 @@ func backendResourceReferences( func workflowParameters(workflow *unstructured.Unstructured) (map[string]string, error) { parameters, found, err := unstructured.NestedSlice(workflow.Object, "status", "outputs", "parameters") if err != nil { - return nil, fmt.Errorf("read parameters: %w", err) + return nil, fmt.Errorf("read top-level parameters: %w", err) } - if !found { - return nil, nil + if found { + return decodeWorkflowParameters(parameters) } + return entrypointWorkflowParameters(workflow) +} + +func decodeWorkflowParameters(parameters []any) (map[string]string, error) { outputs := make(map[string]string, len(parameters)) for _, raw := range parameters { parameter, ok := raw.(map[string]any) @@ -1265,6 +1269,88 @@ func workflowParameters(workflow *unstructured.Unstructured) (map[string]string, return outputs, nil } +func entrypointWorkflowParameters(workflow *unstructured.Unstructured) (map[string]string, error) { + workflowName := workflow.GetName() + if workflowName == "" { + return nil, errors.New("workflow name is required to identify entrypoint outputs") + } + + nodes, found, err := unstructured.NestedMap(workflow.Object, "status", "nodes") + if err != nil { + return nil, fmt.Errorf("read workflow nodes: %w", err) + } + + if !found { + return nil, errors.New("top-level outputs are absent and workflow nodes are missing") + } + + var entrypoint map[string]any + + entrypointCount := 0 + rootKeyMismatch := "" + + for key, raw := range nodes { + node, ok := raw.(map[string]any) + if !ok { + if key == workflowName { + return nil, fmt.Errorf("entrypoint node %q is not an object", key) + } + + continue + } + + nodeName, nameOK := node["name"].(string) + if key == workflowName && !nameOK { + return nil, fmt.Errorf("entrypoint node %q requires a string name", key) + } + + if key == workflowName && nodeName != workflowName { + rootKeyMismatch = nodeName + } + + if nameOK && nodeName == workflowName { + entrypoint = node + entrypointCount++ + } + } + + if rootKeyMismatch != "" && entrypointCount > 0 { + return nil, fmt.Errorf( + "entrypoint identity is ambiguous: node key %q names %q while another node names the workflow", + workflowName, + rootKeyMismatch, + ) + } + + if rootKeyMismatch != "" { + return nil, fmt.Errorf( + "entrypoint node %q has mismatched name %q", + workflowName, + rootKeyMismatch, + ) + } + + if entrypointCount == 0 { + return nil, fmt.Errorf("entrypoint node named %q was not found", workflowName) + } + + if entrypointCount > 1 { + return nil, fmt.Errorf("entrypoint node named %q is duplicated", workflowName) + } + + outputs, ok := entrypoint["outputs"].(map[string]any) + if !ok { + return nil, fmt.Errorf("entrypoint node %q outputs are missing or malformed", workflowName) + } + + parameters, ok := outputs["parameters"].([]any) + if !ok { + return nil, fmt.Errorf("entrypoint node %q parameters are missing or malformed", workflowName) + } + + return decodeWorkflowParameters(parameters) +} + func backendEvidenceComplete(content string, participantCount int) (bool, error) { if content == "" { return false, errors.New("backend-evidence is required") @@ -1518,56 +1604,12 @@ func participantNodeUID(run *signalv1.CertificationRun, nodeName string) types.U } func workflowParameter(workflow *unstructured.Unstructured, name string) string { - parameters, _, err := unstructured.NestedSlice(workflow.Object, "status", "outputs", "parameters") - if err == nil { - if value := parameterValue(parameters, name); value != "" { - return value - } - } - - nodes, _, err := unstructured.NestedMap(workflow.Object, "status", "nodes") + outputs, err := workflowParameters(workflow) if err != nil { return "" } - for _, raw := range nodes { - node, ok := raw.(map[string]any) - if !ok { - continue - } - - outputs, ok := node["outputs"].(map[string]any) - if !ok { - continue - } - - parameters, ok := outputs["parameters"].([]any) - if ok { - if value := parameterValue(parameters, name); value != "" { - return value - } - } - } - - return "" -} - -func parameterValue(parameters []any, name string) string { - for _, raw := range parameters { - parameter, ok := raw.(map[string]any) - if !ok || parameter["name"] != name { - continue - } - - value, ok := parameter["value"].(string) - if !ok { - return "" - } - - return value - } - - return "" + return outputs[name] } func workflowMessage(workflow *unstructured.Unstructured) string { diff --git a/internal/signal/controller_test.go b/internal/signal/controller_test.go index a2ca4353f2..96f42c299e 100644 --- a/internal/signal/controller_test.go +++ b/internal/signal/controller_test.go @@ -1157,9 +1157,11 @@ func TestWorkflowParameterFallsBackToEntrypointNodeOutputs(t *testing.T) { t.Parallel() workflow := &unstructured.Unstructured{Object: map[string]any{ + "metadata": map[string]any{"name": "run-a1"}, "status": map[string]any{ "nodes": map[string]any{ - "root": map[string]any{ + "run-a1": map[string]any{ + "name": "run-a1", "outputs": map[string]any{ "parameters": []any{map[string]any{"name": "verdict", "value": "Pass"}}, }, @@ -1173,6 +1175,165 @@ func TestWorkflowParameterFallsBackToEntrypointNodeOutputs(t *testing.T) { } } +func TestWorkflowParametersPreferPresentTopLevelOutputs(t *testing.T) { + t.Parallel() + + workflow := workflowWithEntrypointOutputs( + []any{map[string]any{"name": "verdict", "value": "NodePass"}}, + ) + if err := unstructured.SetNestedSlice( + workflow.Object, + []any{map[string]any{"name": "verdict", "value": "TopLevelPass"}}, + "status", + "outputs", + "parameters", + ); err != nil { + t.Fatal(err) + } + + outputs, err := workflowParameters(workflow) + if err != nil { + t.Fatal(err) + } + + if outputs["verdict"] != "TopLevelPass" { + t.Fatalf("verdict = %q", outputs["verdict"]) + } +} + +func TestWorkflowParametersFallBackToAuthoritativeEntrypoint(t *testing.T) { + t.Parallel() + + workflow := workflowWithEntrypointOutputs( + []any{ + map[string]any{"name": "verdict", "value": "Pass"}, + map[string]any{"name": "reason", "value": "Healthy"}, + }, + ) + + outputs, err := workflowParameters(workflow) + if err != nil { + t.Fatal(err) + } + + if outputs["verdict"] != "Pass" || outputs["reason"] != "Healthy" { + t.Fatalf("outputs = %#v", outputs) + } +} + +func TestWorkflowParametersRejectInvalidEntrypointFallback(t *testing.T) { + t.Parallel() + + for name, mutate := range map[string]func(*unstructured.Unstructured){ + "missing entrypoint": func(workflow *unstructured.Unstructured) { + workflow.Object["status"].(map[string]any)["nodes"] = map[string]any{ + "child": map[string]any{"name": "run-a1.child"}, + } + }, + "malformed entrypoint identity": func(workflow *unstructured.Unstructured) { + workflow.Object["status"].(map[string]any)["nodes"] = map[string]any{ + "run-a1": map[string]any{"name": true}, + } + }, + "duplicate entrypoint": func(workflow *unstructured.Unstructured) { + nodes := workflow.Object["status"].(map[string]any)["nodes"].(map[string]any) + nodes["duplicate"] = map[string]any{ + "name": "run-a1", + "outputs": map[string]any{"parameters": []any{}}, + } + }, + "ambiguous entrypoint": func(workflow *unstructured.Unstructured) { + workflow.Object["status"].(map[string]any)["nodes"] = map[string]any{ + "run-a1": map[string]any{"name": "run-a1.child"}, + "other": map[string]any{ + "name": "run-a1", + "outputs": map[string]any{"parameters": []any{}}, + }, + } + }, + "malformed outputs": func(workflow *unstructured.Unstructured) { + nodes := workflow.Object["status"].(map[string]any)["nodes"].(map[string]any) + nodes["run-a1"].(map[string]any)["outputs"] = map[string]any{ + "parameters": "not-a-list", + } + }, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + workflow := workflowWithEntrypointOutputs([]any{}) + mutate(workflow) + + if _, err := workflowParameters(workflow); err == nil { + t.Fatal("workflowParameters accepted invalid entrypoint fallback") + } + }) + } +} + +func TestTerminalVerdictUsesEntrypointOutputsWhenTopLevelAbsent(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + run := testRun("run-entrypoint-output") + run.Status = signalv1.CertificationRunStatus{ + Phase: signalv1.RunPhaseRunning, + Workflow: signalv1.WorkflowStatus{Name: "run-entrypoint-output-a1", Attempt: 1}, + StartedAt: ptrTime(metav1.NewTime(time.Unix(100, 0))), + } + snapshot := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Script.ConfigMapName, Namespace: run.Namespace}, + Data: map[string]string{run.Spec.Script.Key: "test payload\n"}, + } + workflow := workflowObject(run, "Succeeded", "Pass", "Healthy") + + parameters, found, err := unstructured.NestedSlice( + workflow.Object, + "status", + "outputs", + "parameters", + ) + if err != nil || !found { + t.Fatalf("read top-level outputs: found=%t, err=%v", found, err) + } + + delete(workflow.Object["status"].(map[string]any), "outputs") + workflow.Object["status"].(map[string]any)["nodes"] = map[string]any{ + workflow.GetName(): map[string]any{ + "name": workflow.GetName(), + "outputs": map[string]any{ + "parameters": parameters, + }, + }, + } + + store := &memoryEvidenceStore{} + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationRun{}). + WithObjects(run, snapshot, workflow). + Build() + r := &RunReconciler{Client: kube, Evidence: store, Now: func() time.Time { return time.Unix(200, 0) }} + + if _, err := r.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}, + ); err != nil { + t.Fatalf("Reconcile: %v", err) + } + + var got signalv1.CertificationRun + if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { + t.Fatal(err) + } + + if got.Status.Phase != signalv1.RunPhaseSucceeded || + got.Status.Verdict != "Pass" || + got.Status.Workflow.Outputs["reason"] != "Healthy" { + t.Fatalf("status = %#v", got.Status) + } +} + func TestValidateStagePlanRejectsInvalidAndDuplicateStages(t *testing.T) { t.Parallel() @@ -1712,6 +1873,22 @@ func workflowObject(run *signalv1.CertificationRun, phase, verdict, reason strin return workflow } +func workflowWithEntrypointOutputs(parameters []any) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "metadata": map[string]any{"name": "run-a1"}, + "status": map[string]any{ + "nodes": map[string]any{ + "run-a1": map[string]any{ + "name": "run-a1", + "outputs": map[string]any{ + "parameters": parameters, + }, + }, + }, + }, + }} +} + func workflowArgument(workflow *unstructured.Unstructured, name string) string { parameters, _, _ := unstructured.NestedSlice(workflow.Object, "spec", "arguments", "parameters") for _, raw := range parameters { diff --git a/internal/signaladapter/adapter.go b/internal/signaladapter/adapter.go index f47e710d04..61df4a5bff 100644 --- a/internal/signaladapter/adapter.go +++ b/internal/signaladapter/adapter.go @@ -742,6 +742,16 @@ else status="${prelude_status}" fi set -e +canonicalize_result_file() { + if [[ -f "$1" ]]; then + local value + value="$(<"$1")" + printf '%%s' "${value}" > "$1" + fi +} +canonicalize_result_file /results/verdict +canonicalize_result_file /results/reason +canonicalize_result_file /results/measurements.txt encode_result_file() { if [[ -f "$1" ]]; then base64 < "$1" | tr -d '\n' diff --git a/internal/signaladapter/adapter_test.go b/internal/signaladapter/adapter_test.go index a068a75461..fc296c8f25 100644 --- a/internal/signaladapter/adapter_test.go +++ b/internal/signaladapter/adapter_test.go @@ -1298,6 +1298,9 @@ func TestResultCommandClearsReservedFilesAndReportsExitCode(t *testing.T) { for _, required := range []string{ "rm -f /results/verdict /results/reason /results/measurements.txt", `printf 'PROJECT_SIGNAL_RESULTS_V1 prepared\n'`, + "canonicalize_result_file /results/verdict", + "canonicalize_result_file /results/reason", + "canonicalize_result_file /results/measurements.txt", `printf 'PROJECT_SIGNAL_RESULTS_V1 verdict=%s reason=%s measurements=%s\n'`, `printf 'PROJECT_SIGNAL_RESULT_V2 %s %s %s %d\n'`, `exit "$status"`, @@ -1317,6 +1320,67 @@ func TestResultCommandClearsReservedFilesAndReportsExitCode(t *testing.T) { } } +func TestResultCommandCanonicalizesTrailingNewlines(t *testing.T) { + t.Parallel() + + directory := t.TempDir() + resultsDirectory := filepath.Join(directory, "results") + + scriptsDirectory := filepath.Join(directory, "scripts") + if err := os.MkdirAll(scriptsDirectory, 0o750); err != nil { + t.Fatal(err) + } + + script := filepath.Join(scriptsDirectory, "diagnostic.sh") + if err := os.WriteFile(script, []byte(`#!/bin/bash +printf 'Pass\n' > "${PROJECT_SIGNAL_RESULTS_DIR}/verdict" +printf 'Healthy\n' > "${PROJECT_SIGNAL_RESULTS_DIR}/reason" +printf 'temperature=42\n' > "${PROJECT_SIGNAL_RESULTS_DIR}/measurements.txt" +`), 0o750); err != nil { + t.Fatal(err) + } + + command := resultCommand() + commandText := strings.ReplaceAll(command[2], "/results", resultsDirectory) + commandText = strings.ReplaceAll(commandText, "/scripts", scriptsDirectory) + process := exec.Command(command[0], command[1], commandText) + + process.Env = append( + os.Environ(), + "PROJECT_SIGNAL_SCRIPT_KEY=diagnostic.sh", + "PROJECT_SIGNAL_RESULTS_DIR="+resultsDirectory, + ) + + output, err := process.CombinedOutput() + if err != nil { + t.Fatalf("result command: %v\n%s", err, output) + } + + result, err := ParseResultLog(string(output), 0) + if err != nil { + t.Fatalf("ParseResultLog: %v\n%s", err, output) + } + + if result.Verdict != "Pass" || result.Reason != "Healthy" || result.Measurements != "temperature=42" { + t.Fatalf("result = %#v", result) + } + + for name, want := range map[string]string{ + "verdict": "Pass", + "reason": "Healthy", + "measurements.txt": "temperature=42", + } { + content, err := os.ReadFile(filepath.Join(resultsDirectory, name)) + if err != nil { + t.Fatal(err) + } + + if string(content) != want { + t.Fatalf("%s = %q, want %q", name, content, want) + } + } +} + func testConfig() Config { participantsJSON, err := json.Marshal(testParticipants()) if err != nil { From fcddc200d343a3accf2ac4cc55c9a05065c7eddc Mon Sep 17 00:00:00 2001 From: chokevin Date: Thu, 22 Oct 2026 11:24:47 -0700 Subject: [PATCH 21/36] feat(signal): group certification checks in one run --- .../v1alpha1/certificationprofile_types.go | 137 +- .../v1alpha1/certificationrun_envtest_test.go | 73 +- api/signal/v1alpha1/certificationrun_types.go | 166 +- api/signal/v1alpha1/zz_generated.deepcopy.go | 328 ++- deploy/signal/README.md | 120 +- ...ounded-cloud.io_certificationprofiles.yaml | 1140 +++++---- ....unbounded-cloud.io_certificationruns.yaml | 1276 ++++++---- deploy/signal/manifest_test.go | 62 +- internal/signal/controller.go | 2130 +++++++++++++++-- internal/signal/controller_test.go | 195 +- internal/signal/evidence.go | 214 +- internal/signal/evidence_test.go | 86 + internal/signal/grouped_controller_test.go | 854 +++++++ internal/signal/projection.go | 18 +- internal/signal/projection_test.go | 2 +- 15 files changed, 5413 insertions(+), 1388 deletions(-) create mode 100644 internal/signal/grouped_controller_test.go diff --git a/api/signal/v1alpha1/certificationprofile_types.go b/api/signal/v1alpha1/certificationprofile_types.go index 49ec5fbe72..d7a38d9dd9 100644 --- a/api/signal/v1alpha1/certificationprofile_types.go +++ b/api/signal/v1alpha1/certificationprofile_types.go @@ -12,6 +12,9 @@ type ScriptReference struct { Name string `json:"name"` Key string `json:"key"` RequiresRuntimeStage bool `json:"requiresRuntimeStage,omitempty"` + // Digest is the approved algorithm-qualified SHA-256 digest of the script bytes. + // +kubebuilder:validation:Pattern=`^sha256:[0-9a-f]{64}$` + Digest string `json:"digest"` } type WorkflowReference struct { @@ -38,6 +41,87 @@ type CertificationStage struct { Conditional bool `json:"conditional,omitempty"` } +type CheckID string + +type ResultBounds struct { + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=4096 + ReasonBytes int32 `json:"reasonBytes"` + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=16384 + MeasurementsBytes int32 `json:"measurementsBytes"` + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=16384 + BackendResourcesBytes int32 `json:"backendResourcesBytes"` + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65536 + BackendEvidenceBytes int32 `json:"backendEvidenceBytes"` +} + +type ParticipantRoleConstraint struct { + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` + Name string `json:"name"` + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=64 + Count int32 `json:"count"` + // Selector further constrains participants assigned to this role. + Selector metav1.LabelSelector `json:"selector,omitempty"` +} + +type ParticipantTopologyConstraint struct { + // Role names the participant role whose selected Nodes are evaluated. + Role string `json:"role"` + // LabelKey is the Node label used to identify topology domains. + LabelKey string `json:"labelKey"` + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=64 + MinDistinctValues int32 `json:"minDistinctValues"` +} + +type CheckParticipantPolicy struct { + // +kubebuilder:validation:MaxItems=16 + RequiredRoles []string `json:"requiredRoles,omitempty"` + // +kubebuilder:validation:MaxItems=16 + ExcludedRoles []string `json:"excludedRoles,omitempty"` +} + +// +kubebuilder:validation:XValidation:rule="self.maxAttempts == 1",message="maxAttempts must be exactly 1" +// +kubebuilder:validation:XValidation:rule="duration(self.backendActiveDeadline) > duration('0s')",message="backendActiveDeadline must be positive" +// +kubebuilder:validation:XValidation:rule="!has(self.script.requiresRuntimeStage) || !self.script.requiresRuntimeStage || has(self.runtimeStage)",message="runtimeStage is required when the script requires a runtime stage" +type CertificationCheck struct { + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + // +kubebuilder:validation:Pattern=`^R(0[1-9]|1[0-5])$` + ID CheckID `json:"id"` + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=14 + Order int32 `json:"order"` + Script ScriptReference `json:"script"` + RunnerImage string `json:"runnerImage"` + Backend ExecutionBackend `json:"backend"` + WorkloadPolicy WorkloadPolicy `json:"workloadPolicy,omitempty"` + RuntimeStage *RuntimeStage `json:"runtimeStage,omitempty"` + Parameters map[string]string `json:"parameters,omitempty"` + BackendActiveDeadline metav1.Duration `json:"backendActiveDeadline"` + ResultBounds ResultBounds `json:"resultBounds"` + ParticipantPolicy CheckParticipantPolicy `json:"participantPolicy,omitempty"` + MaxAttempts int32 `json:"maxAttempts"` +} + +type SuiteFailurePolicy string + +const ( + SuiteFailurePolicyStop SuiteFailurePolicy = "Stop" + SuiteFailurePolicyContinue SuiteFailurePolicy = "Continue" +) + +type SuiteExecutionPolicy struct { + // +kubebuilder:validation:Enum=Stop;Continue + OnFailure SuiteFailurePolicy `json:"onFailure"` +} + type RuntimeStage struct { // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:MaxLength=63 @@ -91,27 +175,40 @@ type WorkloadPolicy struct { Volumes []WorkloadVolume `json:"volumes,omitempty"` } -// +kubebuilder:validation:XValidation:rule="duration(self.backendActiveDeadline) > duration('0s')",message="backendActiveDeadline must be positive" -// +kubebuilder:validation:XValidation:rule="duration(self.deadline) - duration(self.backendActiveDeadline) >= duration('60s')",message="backendActiveDeadline must leave at least 60 seconds before deadline" -// +kubebuilder:validation:XValidation:rule="!has(self.script.requiresRuntimeStage) || !self.script.requiresRuntimeStage || has(self.runtimeStage)",message="runtimeStage is required when the script requires a runtime stage" +// +kubebuilder:validation:XValidation:rule="self.checks.all(c, self.checks.exists_one(x, x.id == c.id))",message="checks must contain unique IDs" +// +kubebuilder:validation:XValidation:rule="self.checks.map(c, c.order) == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]",message="checks must use immutable contiguous order 0 through 14" +// +kubebuilder:validation:XValidation:rule="self.checks.map(c, c.id) == ['R01', 'R02', 'R03', 'R04', 'R05', 'R06', 'R07', 'R08', 'R09', 'R10', 'R11', 'R12', 'R13', 'R14', 'R15']",message="checks must use ordered IDs R01 through R15" +// +kubebuilder:validation:XValidation:rule="self.checks.all(c, duration(self.deadline) - duration(c.backendActiveDeadline) >= duration('60s'))",message="every backendActiveDeadline must leave at least 60 seconds before deadline" +// +kubebuilder:validation:XValidation:rule="!has(self.recurring) || !self.recurring.enabled",message="recurring certification is not supported for grouped suites" type CertificationProfileSpec struct { - Participants ParticipantPolicy `json:"participants"` - Script ScriptReference `json:"script"` - Workflow WorkflowReference `json:"workflow"` - Backend ExecutionBackend `json:"backend"` - RunnerImage string `json:"runnerImage"` - WorkloadPolicy WorkloadPolicy `json:"workloadPolicy,omitempty"` + Participants ParticipantPolicy `json:"participants"` + // +kubebuilder:validation:MaxItems=16 + ParticipantRoles []ParticipantRoleConstraint `json:"participantRoles,omitempty"` + // +kubebuilder:validation:MaxItems=16 + TopologyConstraints []ParticipantTopologyConstraint `json:"topologyConstraints,omitempty"` + Workflow WorkflowReference `json:"workflow"` + // +kubebuilder:validation:MinItems=15 + // +kubebuilder:validation:MaxItems=15 + Checks []CertificationCheck `json:"checks"` + ExecutionPolicy SuiteExecutionPolicy `json:"executionPolicy"` // +kubebuilder:validation:Enum=FullCertification;QuickCertification;Preflight;TargetedVerification;Diagnostic - Purpose CertificationPurpose `json:"purpose,omitempty"` - StagePlan []CertificationStage `json:"stagePlan,omitempty"` - RuntimeStage *RuntimeStage `json:"runtimeStage,omitempty"` - Parameters map[string]string `json:"parameters,omitempty"` - EvidencePath string `json:"evidencePath"` - Deadline metav1.Duration `json:"deadline"` - BackendActiveDeadline metav1.Duration `json:"backendActiveDeadline"` - MaxAttempts int32 `json:"maxAttempts,omitempty"` - Tolerations []corev1.Toleration `json:"tolerations,omitempty"` - Recurring *RecurringPolicy `json:"recurring,omitempty"` + Purpose CertificationPurpose `json:"purpose,omitempty"` + EvidencePath string `json:"evidencePath"` + Deadline metav1.Duration `json:"deadline"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + Recurring *RecurringPolicy `json:"recurring,omitempty"` + + // Deprecated Go-only aliases retained for unit fixtures. They are not part + // of the v1alpha1 wire contract. + Script ScriptReference `json:"-"` + Backend ExecutionBackend `json:"-"` + RunnerImage string `json:"-"` + WorkloadPolicy WorkloadPolicy `json:"-"` + StagePlan []CertificationStage `json:"-"` + RuntimeStage *RuntimeStage `json:"-"` + Parameters map[string]string `json:"-"` + BackendActiveDeadline metav1.Duration `json:"-"` + MaxAttempts int32 `json:"-"` } type RecurringPolicy struct { @@ -151,7 +248,7 @@ type CertificationProfileStatus struct { // +kubebuilder:object:root=true // +kubebuilder:resource:scope=Cluster,shortName=certprofile // +kubebuilder:subresource:status -// +kubebuilder:printcolumn:name="Backend",type=string,JSONPath=`.spec.backend.kind` +// +kubebuilder:printcolumn:name="Checks",type=integer,JSONPath=`.spec.checks.size()` // +kubebuilder:printcolumn:name="Participants",type=integer,JSONPath=`.spec.participants.count` // +kubebuilder:printcolumn:name="Recurring",type=boolean,JSONPath=`.spec.recurring.enabled` type CertificationProfile struct { diff --git a/api/signal/v1alpha1/certificationrun_envtest_test.go b/api/signal/v1alpha1/certificationrun_envtest_test.go index c4e9fab71e..3268185d71 100644 --- a/api/signal/v1alpha1/certificationrun_envtest_test.go +++ b/api/signal/v1alpha1/certificationrun_envtest_test.go @@ -4,6 +4,7 @@ package v1alpha1 import ( + "fmt" "os" "strings" "testing" @@ -49,7 +50,8 @@ func TestEnvtestCertificationRunOptionalImmutability(t *testing.T) { t.Helper() run := validEnvtestCertificationRun(name) - run.Spec.RuntimeStage = stage + run.Spec.Checks[0].RuntimeStage = stage + run.Spec.Checks[0].Script.RequiresRuntimeStage = stage != nil require.NoError(t, apiClient.Create(t.Context(), run)) return run @@ -87,11 +89,42 @@ func TestEnvtestCertificationRunOptionalImmutability(t *testing.T) { } { t.Run(name+" is rejected", func(t *testing.T) { run := createRun("runtime-stage-"+strings.ReplaceAll(name, " ", "-"), test.initial) - run.Spec.RuntimeStage = test.updated + run.Spec.Checks[0].RuntimeStage = test.updated + run.Spec.Checks[0].Script.RequiresRuntimeStage = test.updated != nil err := apiClient.Update(t.Context(), run) require.True(t, apierrors.IsInvalid(err), "%v", err) - require.ErrorContains(t, err, "runtimeStage is immutable") + require.ErrorContains(t, err, "checks are immutable") + }) + } + + for name, mutate := range map[string]func(*CertificationRun){ + "fourteen checks": func(run *CertificationRun) { + run.Spec.Checks = run.Spec.Checks[:14] + }, + "duplicate check id": func(run *CertificationRun) { + run.Spec.Checks[1].ID = run.Spec.Checks[0].ID + }, + "wrong ordered check id": func(run *CertificationRun) { + run.Spec.Checks[4].ID = "R15" + run.Spec.Checks[14].ID = "R05" + }, + "noncontiguous order": func(run *CertificationRun) { + run.Spec.Checks[7].Order = 8 + }, + "retry enabled": func(run *CertificationRun) { + run.Spec.Checks[2].MaxAttempts = 2 + }, + "insufficient cleanup reserve": func(run *CertificationRun) { + run.Spec.Checks[3].BackendActiveDeadline = metav1.Duration{Duration: 61 * time.Second} + }, + } { + t.Run(name+" is rejected", func(t *testing.T) { + run := validEnvtestCertificationRun("invalid-" + strings.ReplaceAll(name, " ", "-")) + mutate(run) + + err := apiClient.Create(t.Context(), run) + require.True(t, apierrors.IsInvalid(err), "%v", err) }) } } @@ -114,19 +147,39 @@ func validEnvtestCertificationRun(name string) *CertificationRun { NodeName: "node-a", NodeUID: types.UID("node-a-uid"), }}, + Checks: validEnvtestChecks(), + ExecutionPolicy: SuiteExecutionPolicy{OnFailure: SuiteFailurePolicyStop}, + Workflow: WorkflowReference{Name: "workflow"}, + EvidencePath: "signal/" + name + ".json", + Deadline: metav1.Duration{Duration: 2 * time.Minute}, + RequestNodeName: "node-a", + }, + } +} + +func validEnvtestChecks() []CertificationCheckSnapshot { + checks := make([]CertificationCheckSnapshot, 15) + for index := range checks { + checks[index] = CertificationCheckSnapshot{ + ID: CheckID(fmt.Sprintf("R%02d", index+1)), + Order: int32(index), Script: ScriptSnapshot{ ConfigMapName: "scripts", Key: "check.sh", Digest: "sha256:" + strings.Repeat("a", 64), }, - Workflow: WorkflowReference{Name: "workflow"}, - Backend: ExecutionBackend{APIVersion: "v1", Kind: "Pod"}, RunnerImage: "example.invalid/runner@sha256:" + strings.Repeat("b", 64), - EvidencePath: "signal/" + name + ".json", - Deadline: metav1.Duration{Duration: 2 * time.Minute}, + Backend: ExecutionBackend{APIVersion: "v1", Kind: "Pod"}, BackendActiveDeadline: metav1.Duration{Duration: time.Minute}, - MaxAttempts: 1, - RequestNodeName: "node-a", - }, + ResultBounds: ResultBounds{ + ReasonBytes: 4096, + MeasurementsBytes: 16384, + BackendResourcesBytes: 16384, + BackendEvidenceBytes: 65536, + }, + MaxAttempts: 1, + } } + + return checks } diff --git a/api/signal/v1alpha1/certificationrun_types.go b/api/signal/v1alpha1/certificationrun_types.go index 069922d751..196127329c 100644 --- a/api/signal/v1alpha1/certificationrun_types.go +++ b/api/signal/v1alpha1/certificationrun_types.go @@ -36,6 +36,8 @@ type Participant struct { NodeUID types.UID `json:"nodeUID"` Capacity corev1.ResourceList `json:"capacity,omitempty"` Allocatable corev1.ResourceList `json:"allocatable,omitempty"` + // +kubebuilder:validation:MaxItems=16 + Roles []string `json:"roles,omitempty"` } type BackendResourceReference struct { @@ -64,48 +66,85 @@ type ScriptSnapshot struct { Digest string `json:"digest"` } +type LegacyScriptSnapshot struct { + ConfigMapName string `json:"configMapName,omitempty"` + Key string `json:"key,omitempty"` + RequiresRuntimeStage bool `json:"requiresRuntimeStage,omitempty"` + Digest string `json:"digest,omitempty"` +} + +// +kubebuilder:validation:XValidation:rule="self.maxAttempts == 1",message="maxAttempts must be exactly 1" +// +kubebuilder:validation:XValidation:rule="duration(self.backendActiveDeadline) > duration('0s')",message="backendActiveDeadline must be positive" +// +kubebuilder:validation:XValidation:rule="!has(self.script.requiresRuntimeStage) || !self.script.requiresRuntimeStage || has(self.runtimeStage)",message="runtimeStage is required when the script requires a runtime stage" +type CertificationCheckSnapshot struct { + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + // +kubebuilder:validation:Pattern=`^R(0[1-9]|1[0-5])$` + ID CheckID `json:"id"` + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=14 + Order int32 `json:"order"` + Script ScriptSnapshot `json:"script"` + RunnerImage string `json:"runnerImage"` + Backend ExecutionBackend `json:"backend"` + WorkloadPolicy WorkloadPolicy `json:"workloadPolicy,omitempty"` + RuntimeStage *RuntimeStage `json:"runtimeStage,omitempty"` + Parameters map[string]string `json:"parameters,omitempty"` + BackendActiveDeadline metav1.Duration `json:"backendActiveDeadline"` + ResultBounds ResultBounds `json:"resultBounds"` + ParticipantPolicy CheckParticipantPolicy `json:"participantPolicy,omitempty"` + MaxAttempts int32 `json:"maxAttempts"` +} + // +kubebuilder:validation:XValidation:rule="self.profile == oldSelf.profile",message="profile is immutable" // +kubebuilder:validation:XValidation:rule="self.participants == oldSelf.participants",message="participants are immutable" -// +kubebuilder:validation:XValidation:rule="self.script == oldSelf.script",message="script is immutable" +// +kubebuilder:validation:XValidation:rule="has(self.participantRoles) == has(oldSelf.participantRoles) && (!has(self.participantRoles) || self.participantRoles == oldSelf.participantRoles)",message="participantRoles are immutable" +// +kubebuilder:validation:XValidation:rule="has(self.topologyConstraints) == has(oldSelf.topologyConstraints) && (!has(self.topologyConstraints) || self.topologyConstraints == oldSelf.topologyConstraints)",message="topologyConstraints are immutable" +// +kubebuilder:validation:XValidation:rule="self.checks == oldSelf.checks",message="checks are immutable" +// +kubebuilder:validation:XValidation:rule="self.executionPolicy == oldSelf.executionPolicy",message="executionPolicy is immutable" // +kubebuilder:validation:XValidation:rule="self.workflow == oldSelf.workflow",message="workflow is immutable" -// +kubebuilder:validation:XValidation:rule="self.backend == oldSelf.backend",message="backend is immutable" -// +kubebuilder:validation:XValidation:rule="self.runnerImage == oldSelf.runnerImage",message="runnerImage is immutable" -// +kubebuilder:validation:XValidation:rule="has(self.workloadPolicy) == has(oldSelf.workloadPolicy) && (!has(self.workloadPolicy) || self.workloadPolicy == oldSelf.workloadPolicy)",message="workloadPolicy is immutable" -// +kubebuilder:validation:XValidation:rule="has(self.runtimeStage) == has(oldSelf.runtimeStage) && (!has(self.runtimeStage) || self.runtimeStage == oldSelf.runtimeStage)",message="runtimeStage is immutable" -// +kubebuilder:validation:XValidation:rule="has(self.parameters) == has(oldSelf.parameters) && (!has(self.parameters) || self.parameters == oldSelf.parameters)",message="parameters are immutable" // +kubebuilder:validation:XValidation:rule="has(self.purpose) == has(oldSelf.purpose) && (!has(self.purpose) || self.purpose == oldSelf.purpose)",message="purpose is immutable" -// +kubebuilder:validation:XValidation:rule="has(self.stagePlan) == has(oldSelf.stagePlan) && (!has(self.stagePlan) || self.stagePlan == oldSelf.stagePlan)",message="stagePlan is immutable" // +kubebuilder:validation:XValidation:rule="self.evidencePath == oldSelf.evidencePath",message="evidencePath is immutable" // +kubebuilder:validation:XValidation:rule="self.deadline == oldSelf.deadline",message="deadline is immutable" -// +kubebuilder:validation:XValidation:rule="self.backendActiveDeadline == oldSelf.backendActiveDeadline",message="backendActiveDeadline is immutable" -// +kubebuilder:validation:XValidation:rule="self.maxAttempts == oldSelf.maxAttempts",message="maxAttempts is immutable" // +kubebuilder:validation:XValidation:rule="has(self.tolerations) == has(oldSelf.tolerations) && (!has(self.tolerations) || self.tolerations == oldSelf.tolerations)",message="tolerations are immutable" // +kubebuilder:validation:XValidation:rule="self.requestNodeName == oldSelf.requestNodeName",message="requestNodeName is immutable" -// +kubebuilder:validation:XValidation:rule="duration(self.backendActiveDeadline) > duration('0s')",message="backendActiveDeadline must be positive" -// +kubebuilder:validation:XValidation:rule="duration(self.deadline) - duration(self.backendActiveDeadline) >= duration('60s')",message="backendActiveDeadline must leave at least 60 seconds before deadline" -// +kubebuilder:validation:XValidation:rule="!has(self.script.requiresRuntimeStage) || !self.script.requiresRuntimeStage || has(self.runtimeStage)",message="runtimeStage is required when the script requires a runtime stage" +// +kubebuilder:validation:XValidation:rule="self.checks.all(c, self.checks.exists_one(x, x.id == c.id))",message="checks must contain unique IDs" +// +kubebuilder:validation:XValidation:rule="self.checks.map(c, c.order) == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]",message="checks must use immutable contiguous order 0 through 14" +// +kubebuilder:validation:XValidation:rule="self.checks.map(c, c.id) == ['R01', 'R02', 'R03', 'R04', 'R05', 'R06', 'R07', 'R08', 'R09', 'R10', 'R11', 'R12', 'R13', 'R14', 'R15']",message="checks must use ordered IDs R01 through R15" +// +kubebuilder:validation:XValidation:rule="self.checks.all(c, duration(self.deadline) - duration(c.backendActiveDeadline) >= duration('60s'))",message="every backendActiveDeadline must leave at least 60 seconds before deadline" type CertificationRunSpec struct { Profile PinnedProfile `json:"profile"` // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=64 - Participants []Participant `json:"participants"` - Script ScriptSnapshot `json:"script"` - Workflow WorkflowReference `json:"workflow"` - Backend ExecutionBackend `json:"backend"` - RunnerImage string `json:"runnerImage"` - WorkloadPolicy WorkloadPolicy `json:"workloadPolicy,omitempty"` + Participants []Participant `json:"participants"` + // +kubebuilder:validation:MaxItems=16 + ParticipantRoles []ParticipantRoleConstraint `json:"participantRoles,omitempty"` + // +kubebuilder:validation:MaxItems=16 + TopologyConstraints []ParticipantTopologyConstraint `json:"topologyConstraints,omitempty"` + // +kubebuilder:validation:MinItems=15 + // +kubebuilder:validation:MaxItems=15 + Checks []CertificationCheckSnapshot `json:"checks"` + ExecutionPolicy SuiteExecutionPolicy `json:"executionPolicy"` + Workflow WorkflowReference `json:"workflow"` // +kubebuilder:validation:Enum=FullCertification;QuickCertification;Preflight;TargetedVerification;Diagnostic - Purpose CertificationPurpose `json:"purpose,omitempty"` - StagePlan []CertificationStage `json:"stagePlan,omitempty"` - RuntimeStage *RuntimeStage `json:"runtimeStage,omitempty"` - Parameters map[string]string `json:"parameters,omitempty"` - EvidencePath string `json:"evidencePath"` - Deadline metav1.Duration `json:"deadline"` - BackendActiveDeadline metav1.Duration `json:"backendActiveDeadline"` - MaxAttempts int32 `json:"maxAttempts"` - Tolerations []corev1.Toleration `json:"tolerations,omitempty"` - RequestNodeName string `json:"requestNodeName"` - Cancel bool `json:"cancel,omitempty"` + Purpose CertificationPurpose `json:"purpose,omitempty"` + EvidencePath string `json:"evidencePath"` + Deadline metav1.Duration `json:"deadline"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + RequestNodeName string `json:"requestNodeName"` + Cancel bool `json:"cancel,omitempty"` + + // Deprecated Go-only aliases retained for unit fixtures. They are not part + // of the v1alpha1 wire contract. + Script LegacyScriptSnapshot `json:"-"` + Backend ExecutionBackend `json:"-"` + RunnerImage string `json:"-"` + WorkloadPolicy WorkloadPolicy `json:"-"` + StagePlan []CertificationStage `json:"-"` + RuntimeStage *RuntimeStage `json:"-"` + Parameters map[string]string `json:"-"` + BackendActiveDeadline metav1.Duration `json:"-"` + MaxAttempts int32 `json:"-"` } type WorkflowStatus struct { @@ -130,17 +169,66 @@ type CleanupStatus struct { SnapshotDeleted bool `json:"snapshotDeleted,omitempty"` } +type CheckPhase string + +const ( + CheckPhasePending CheckPhase = "Pending" + CheckPhaseRunning CheckPhase = "Running" + CheckPhaseSucceeded CheckPhase = "Succeeded" + CheckPhaseFailed CheckPhase = "Failed" + CheckPhaseInconclusive CheckPhase = "Inconclusive" + CheckPhaseSkipped CheckPhase = "Skipped" +) + +type BackendEvidenceStatus struct { + Version string `json:"version,omitempty"` + Complete bool `json:"complete,omitempty"` + // Digest identifies the complete backend-evidence v1 document retained in + // the durable evidence artifact. + // +kubebuilder:validation:Pattern=`^sha256:[0-9a-f]{64}$` + Digest string `json:"digest,omitempty"` + // Content is populated only while constructing the durable evidence + // artifact and is not persisted in CertificationRun status. + Content string `json:"content,omitempty"` +} + +type CheckCleanupStatus struct { + Complete bool `json:"complete,omitempty"` + Reason string `json:"reason,omitempty"` +} + +type CheckStatus struct { + ID CheckID `json:"id"` + Order int32 `json:"order"` + WorkflowNodeName string `json:"workflowNodeName,omitempty"` + Phase CheckPhase `json:"phase"` + Verdict string `json:"verdict,omitempty"` + Reason string `json:"reason,omitempty"` + Measurements string `json:"measurements,omitempty"` + BackendResources []BackendResourceReference `json:"backendResources,omitempty"` + BackendEvidence BackendEvidenceStatus `json:"backendEvidence,omitempty"` + StartedAt *metav1.Time `json:"startedAt,omitempty"` + FinishedAt *metav1.Time `json:"finishedAt,omitempty"` + Duration metav1.Duration `json:"duration,omitempty"` + Attempt int32 `json:"attempt"` + RetryCount int32 `json:"retryCount"` + Cleanup CheckCleanupStatus `json:"cleanup,omitempty"` + NotRun bool `json:"notRun,omitempty"` +} + type CertificationRunStatus struct { - Phase RunPhase `json:"phase,omitempty"` - ExecutionState string `json:"executionState,omitempty"` - Verdict string `json:"verdict,omitempty"` - Reason string `json:"reason,omitempty"` - Workflow WorkflowStatus `json:"workflow,omitempty"` - Evidence EvidenceStatus `json:"evidence,omitempty"` - Cleanup CleanupStatus `json:"cleanup,omitempty"` - StartedAt *metav1.Time `json:"startedAt,omitempty"` - FinishedAt *metav1.Time `json:"finishedAt,omitempty"` - Conditions []metav1.Condition `json:"conditions,omitempty"` + Phase RunPhase `json:"phase,omitempty"` + ExecutionState string `json:"executionState,omitempty"` + Verdict string `json:"verdict,omitempty"` + Reason string `json:"reason,omitempty"` + Workflow WorkflowStatus `json:"workflow,omitempty"` + Evidence EvidenceStatus `json:"evidence,omitempty"` + Cleanup CleanupStatus `json:"cleanup,omitempty"` + // +kubebuilder:validation:MaxItems=15 + Checks []CheckStatus `json:"checks,omitempty"` + StartedAt *metav1.Time `json:"startedAt,omitempty"` + FinishedAt *metav1.Time `json:"finishedAt,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` } // +kubebuilder:object:root=true diff --git a/api/signal/v1alpha1/zz_generated.deepcopy.go b/api/signal/v1alpha1/zz_generated.deepcopy.go index 262813b6f8..907981bee4 100644 --- a/api/signal/v1alpha1/zz_generated.deepcopy.go +++ b/api/signal/v1alpha1/zz_generated.deepcopy.go @@ -13,6 +13,21 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackendEvidenceStatus) DeepCopyInto(out *BackendEvidenceStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackendEvidenceStatus. +func (in *BackendEvidenceStatus) DeepCopy() *BackendEvidenceStatus { + if in == nil { + return nil + } + out := new(BackendEvidenceStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackendResourceReference) DeepCopyInto(out *BackendResourceReference) { *out = *in @@ -33,6 +48,72 @@ func (in *BackendResourceReference) DeepCopy() *BackendResourceReference { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationCheck) DeepCopyInto(out *CertificationCheck) { + *out = *in + out.Script = in.Script + out.Backend = in.Backend + in.WorkloadPolicy.DeepCopyInto(&out.WorkloadPolicy) + if in.RuntimeStage != nil { + in, out := &in.RuntimeStage, &out.RuntimeStage + *out = new(RuntimeStage) + **out = **in + } + if in.Parameters != nil { + in, out := &in.Parameters, &out.Parameters + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + out.BackendActiveDeadline = in.BackendActiveDeadline + out.ResultBounds = in.ResultBounds + in.ParticipantPolicy.DeepCopyInto(&out.ParticipantPolicy) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationCheck. +func (in *CertificationCheck) DeepCopy() *CertificationCheck { + if in == nil { + return nil + } + out := new(CertificationCheck) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationCheckSnapshot) DeepCopyInto(out *CertificationCheckSnapshot) { + *out = *in + out.Script = in.Script + out.Backend = in.Backend + in.WorkloadPolicy.DeepCopyInto(&out.WorkloadPolicy) + if in.RuntimeStage != nil { + in, out := &in.RuntimeStage, &out.RuntimeStage + *out = new(RuntimeStage) + **out = **in + } + if in.Parameters != nil { + in, out := &in.Parameters, &out.Parameters + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + out.BackendActiveDeadline = in.BackendActiveDeadline + out.ResultBounds = in.ResultBounds + in.ParticipantPolicy.DeepCopyInto(&out.ParticipantPolicy) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationCheckSnapshot. +func (in *CertificationCheckSnapshot) DeepCopy() *CertificationCheckSnapshot { + if in == nil { + return nil + } + out := new(CertificationCheckSnapshot) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CertificationProfile) DeepCopyInto(out *CertificationProfile) { *out = *in @@ -96,8 +177,41 @@ func (in *CertificationProfileList) DeepCopyObject() runtime.Object { func (in *CertificationProfileSpec) DeepCopyInto(out *CertificationProfileSpec) { *out = *in in.Participants.DeepCopyInto(&out.Participants) - out.Script = in.Script + if in.ParticipantRoles != nil { + in, out := &in.ParticipantRoles, &out.ParticipantRoles + *out = make([]ParticipantRoleConstraint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.TopologyConstraints != nil { + in, out := &in.TopologyConstraints, &out.TopologyConstraints + *out = make([]ParticipantTopologyConstraint, len(*in)) + copy(*out, *in) + } out.Workflow = in.Workflow + if in.Checks != nil { + in, out := &in.Checks, &out.Checks + *out = make([]CertificationCheck, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + out.ExecutionPolicy = in.ExecutionPolicy + out.Deadline = in.Deadline + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]v1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Recurring != nil { + in, out := &in.Recurring, &out.Recurring + *out = new(RecurringPolicy) + (*in).DeepCopyInto(*out) + } + out.Script = in.Script out.Backend = in.Backend in.WorkloadPolicy.DeepCopyInto(&out.WorkloadPolicy) if in.StagePlan != nil { @@ -117,20 +231,7 @@ func (in *CertificationProfileSpec) DeepCopyInto(out *CertificationProfileSpec) (*out)[key] = val } } - out.Deadline = in.Deadline out.BackendActiveDeadline = in.BackendActiveDeadline - if in.Tolerations != nil { - in, out := &in.Tolerations, &out.Tolerations - *out = make([]v1.Toleration, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.Recurring != nil { - in, out := &in.Recurring, &out.Recurring - *out = new(RecurringPolicy) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationProfileSpec. @@ -242,8 +343,36 @@ func (in *CertificationRunSpec) DeepCopyInto(out *CertificationRunSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - out.Script = in.Script + if in.ParticipantRoles != nil { + in, out := &in.ParticipantRoles, &out.ParticipantRoles + *out = make([]ParticipantRoleConstraint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.TopologyConstraints != nil { + in, out := &in.TopologyConstraints, &out.TopologyConstraints + *out = make([]ParticipantTopologyConstraint, len(*in)) + copy(*out, *in) + } + if in.Checks != nil { + in, out := &in.Checks, &out.Checks + *out = make([]CertificationCheckSnapshot, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + out.ExecutionPolicy = in.ExecutionPolicy out.Workflow = in.Workflow + out.Deadline = in.Deadline + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]v1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + out.Script = in.Script out.Backend = in.Backend in.WorkloadPolicy.DeepCopyInto(&out.WorkloadPolicy) if in.StagePlan != nil { @@ -263,15 +392,7 @@ func (in *CertificationRunSpec) DeepCopyInto(out *CertificationRunSpec) { (*out)[key] = val } } - out.Deadline = in.Deadline out.BackendActiveDeadline = in.BackendActiveDeadline - if in.Tolerations != nil { - in, out := &in.Tolerations, &out.Tolerations - *out = make([]v1.Toleration, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationRunSpec. @@ -290,6 +411,13 @@ func (in *CertificationRunStatus) DeepCopyInto(out *CertificationRunStatus) { in.Workflow.DeepCopyInto(&out.Workflow) out.Evidence = in.Evidence out.Cleanup = in.Cleanup + if in.Checks != nil { + in, out := &in.Checks, &out.Checks + *out = make([]CheckStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.StartedAt != nil { in, out := &in.StartedAt, &out.StartedAt *out = (*in).DeepCopy() @@ -333,6 +461,79 @@ func (in *CertificationStage) DeepCopy() *CertificationStage { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CheckCleanupStatus) DeepCopyInto(out *CheckCleanupStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CheckCleanupStatus. +func (in *CheckCleanupStatus) DeepCopy() *CheckCleanupStatus { + if in == nil { + return nil + } + out := new(CheckCleanupStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CheckParticipantPolicy) DeepCopyInto(out *CheckParticipantPolicy) { + *out = *in + if in.RequiredRoles != nil { + in, out := &in.RequiredRoles, &out.RequiredRoles + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ExcludedRoles != nil { + in, out := &in.ExcludedRoles, &out.ExcludedRoles + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CheckParticipantPolicy. +func (in *CheckParticipantPolicy) DeepCopy() *CheckParticipantPolicy { + if in == nil { + return nil + } + out := new(CheckParticipantPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CheckStatus) DeepCopyInto(out *CheckStatus) { + *out = *in + if in.BackendResources != nil { + in, out := &in.BackendResources, &out.BackendResources + *out = make([]BackendResourceReference, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + out.BackendEvidence = in.BackendEvidence + if in.StartedAt != nil { + in, out := &in.StartedAt, &out.StartedAt + *out = (*in).DeepCopy() + } + if in.FinishedAt != nil { + in, out := &in.FinishedAt, &out.FinishedAt + *out = (*in).DeepCopy() + } + out.Duration = in.Duration + out.Cleanup = in.Cleanup +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CheckStatus. +func (in *CheckStatus) DeepCopy() *CheckStatus { + if in == nil { + return nil + } + out := new(CheckStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CleanupStatus) DeepCopyInto(out *CleanupStatus) { *out = *in @@ -393,6 +594,21 @@ func (in *KueueTopologyPolicy) DeepCopy() *KueueTopologyPolicy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LegacyScriptSnapshot) DeepCopyInto(out *LegacyScriptSnapshot) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LegacyScriptSnapshot. +func (in *LegacyScriptSnapshot) DeepCopy() *LegacyScriptSnapshot { + if in == nil { + return nil + } + out := new(LegacyScriptSnapshot) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Participant) DeepCopyInto(out *Participant) { *out = *in @@ -410,6 +626,11 @@ func (in *Participant) DeepCopyInto(out *Participant) { (*out)[key] = val.DeepCopy() } } + if in.Roles != nil { + in, out := &in.Roles, &out.Roles + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Participant. @@ -445,6 +666,37 @@ func (in *ParticipantPolicy) DeepCopy() *ParticipantPolicy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ParticipantRoleConstraint) DeepCopyInto(out *ParticipantRoleConstraint) { + *out = *in + in.Selector.DeepCopyInto(&out.Selector) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ParticipantRoleConstraint. +func (in *ParticipantRoleConstraint) DeepCopy() *ParticipantRoleConstraint { + if in == nil { + return nil + } + out := new(ParticipantRoleConstraint) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ParticipantTopologyConstraint) DeepCopyInto(out *ParticipantTopologyConstraint) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ParticipantTopologyConstraint. +func (in *ParticipantTopologyConstraint) DeepCopy() *ParticipantTopologyConstraint { + if in == nil { + return nil + } + out := new(ParticipantTopologyConstraint) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PinnedProfile) DeepCopyInto(out *PinnedProfile) { *out = *in @@ -525,6 +777,21 @@ func (in *RecurringPolicy) DeepCopy() *RecurringPolicy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResultBounds) DeepCopyInto(out *ResultBounds) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResultBounds. +func (in *ResultBounds) DeepCopy() *ResultBounds { + if in == nil { + return nil + } + out := new(ResultBounds) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RuntimeStage) DeepCopyInto(out *RuntimeStage) { *out = *in @@ -570,6 +837,21 @@ func (in *ScriptSnapshot) DeepCopy() *ScriptSnapshot { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SuiteExecutionPolicy) DeepCopyInto(out *SuiteExecutionPolicy) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SuiteExecutionPolicy. +func (in *SuiteExecutionPolicy) DeepCopy() *SuiteExecutionPolicy { + if in == nil { + return nil + } + out := new(SuiteExecutionPolicy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WorkflowReference) DeepCopyInto(out *WorkflowReference) { *out = *in diff --git a/deploy/signal/README.md b/deploy/signal/README.md index d77c8e6e27..24af8e79c8 100644 --- a/deploy/signal/README.md +++ b/deploy/signal/README.md @@ -10,27 +10,27 @@ or environment-specific queue configuration. ## Architecture ```text -approved Node or recurring policy +approved Node request | v CertificationProfile | v - reserve Nodes and snapshot the selected ConfigMap key + reserve Nodes and snapshot the ordered 15-check script plan | v immutable CertificationRun input + SHA-256 digest | v - operator-supplied or generic adapter WorkflowTemplate + one Argo Workflow with a serial, visible 15-task DAG | v - typed result -> retained evidence -> Node projection -> cleanup + per-check typed results -> aggregate evidence -> Node projection -> cleanup ``` -The controller snapshots one selected script key from a source ConfigMap into -a run-owned immutable ConfigMap. Updating the source affects only later runs. -The controller does not contain or publish certification script bodies. +The controller snapshots all 15 selected script keys into one run-owned +immutable ConfigMap. Updating a source affects only later runs. The controller +does not contain or publish certification script bodies. ## Prerequisites @@ -96,7 +96,7 @@ The Workflow must publish only these output parameters: | `reason` | 4 KiB | Bounded diagnostic reason | | `measurements` | 16 KiB | Bounded measurement summary | | `backend-resources` | 16 KiB | Optional JSON array of backend workload identities and covered participant indexes; required for a multi-Node `Pass` | -| `backend-evidence` | 256 KiB | Required redacted pre-cleanup backend object, container state, and bounded current/previous log projection | +| `backend-evidence` | 64 KiB per check | Required redacted pre-cleanup backend object, container state, and bounded current/previous log projection | Unexpected, duplicate, malformed, or oversized outputs are rejected and the run remains fail-closed. Each `backend-resources` entry must contain @@ -223,31 +223,50 @@ kubectl apply -f deploy/signal/rendered/ ``` The upstream manifests install no profiles. Operators must create a -CertificationProfile that references their own script ConfigMap and -digest-pinned runner image. To use the included adapter, set -`spec.workflow.name: project-signal-participant-adapter` and select either: +CertificationProfile with exactly 15 checks that reference operator-owned +script ConfigMaps and digest-pinned runner images. To use the included adapter, +set `spec.workflow.name: project-signal-participant-adapter`. Each +`spec.checks[]` entry selects either: ```yaml -backend: - apiVersion: v1 - kind: Pod - queue: +checks: + - id: + order: 0 + script: + name: + key: + digest: sha256: + runnerImage: @sha256: + backend: + apiVersion: v1 + kind: Pod + queue: + backendActiveDeadline: 19m + resultBounds: + reasonBytes: 4096 + measurementsBytes: 16384 + backendResourcesBytes: 16384 + backendEvidenceBytes: 65536 + maxAttempts: 1 ``` or an explicit Kueue queue: ```yaml -backend: - apiVersion: kubeflow.org/v2beta1 - kind: MPIJob - queue: +checks: + - id: + order: <0-14> + backend: + apiVersion: kubeflow.org/v2beta1 + kind: MPIJob + queue: ``` When a Pod backend supplies a queue, the adapter writes the standard `kueue.x-k8s.io/queue-name` label. MPIJob backends require a queue; Pod backends may omit it only when the operator intentionally runs them outside Kueue. -Profiles may set `spec.workloadPolicy` when a diagnostic needs explicit +Checks may set `spec.checks[].workloadPolicy` when a diagnostic needs explicit participant-container policy. The controller validates and snapshots this policy into the immutable CertificationRun before reserving Nodes. The included adapter applies it to the ordinary Pod container or the MPIJob worker only; the @@ -299,9 +318,10 @@ admission and operator policy still decide whether a capability, host path, device, PVC, or privileged context is allowed; technical rendering is not authorization. -Stage-aware scripts declare `spec.script.requiresRuntimeStage: true` and require -an explicit `spec.runtimeStage.name`. The name is a lowercase DNS label of at -most 63 characters and is snapshotted into the immutable CertificationRun and +Stage-aware scripts declare +`spec.checks[].script.requiresRuntimeStage: true` and require an explicit +`spec.checks[].runtimeStage.name`. The name is a lowercase DNS label of at most +63 characters and is snapshotted into the immutable CertificationRun and evidence. The adapter exports it as `PROJECT_SIGNAL_STAGE_NAME` only to ordinary participant Pods and MPIJob workers. MPI launchers never receive it. Scripts that are not stage-aware leave both fields unset. @@ -310,14 +330,15 @@ Every profile sets two distinct deadlines: ```yaml deadline: 20m -backendActiveDeadline: 19m +checks: + - backendActiveDeadline: 19m ``` `deadline` is the outer Workflow deadline for backend observation, result and -evidence projection, and cleanup. `backendActiveDeadline` is rendered to the -participant Pod or MPIJob `activeDeadlineSeconds`. It must be positive and -leave at least 60 seconds before the outer deadline; the controller rejects an -invalid ordering before reserving Nodes. +evidence projection, and cleanup. Each check's `backendActiveDeadline` is +rendered to its participant Pod or MPIJob `activeDeadlineSeconds`. It must be +positive and leave at least 60 seconds before the outer deadline; the +controller rejects an invalid ordering before reserving Nodes. `kueueTopology.requiredLevel` is an explicit Kueue Topology Aware Scheduling request. It must be a valid Kubernetes label key and requires a nonempty backend @@ -347,13 +368,24 @@ parameters `backend-backoff-limit` and script environment and remain explicit per-profile policy rather than cluster defaults. -A `CertificationProfile` and `CertificationRun` each select one immutable -script key and one backend. `purpose: FullCertification` and `stagePlan` -describe evidence and are distinct from the optional runtime stage passed to a -single script; they do not execute a multi-key suite. Operators that define a -full suite must create one run per script and admit the next run only after the -previous run is terminal and its backend cleanup has completed. The upstream -controller does not synthesize a cross-run verdict. +A `CertificationProfile` defines exactly 15 ordered checks. Admission snapshots +the complete plan and selected participants once into one immutable +`CertificationRun`. Each check binds its script identity and digest, runner, +backend, workload and topology policy, runtime inputs, deadline, result bounds, +participant-role inclusion or exclusion, and `maxAttempts: 1`. + +The controller creates one Argo Workflow with an inline DAG entrypoint. Every +task invokes the configured adapter `certification` template through +`templateRef`; dependencies make execution serial and visible in +`status.nodes`. `executionPolicy.onFailure` is immutable and is either `Stop` +or `Continue`. A stopped task is recorded as skipped/not-run, never inferred as +a diagnostic result. Grouped suites cannot enable recurring admission. + +Participant roles and topology constraints are generic typed policy. Profiles +can, for example, bind one role to one participant, another role to three +participants, require the latter to span at least two values of an operator +chosen topology label, and exclude a role from a specific check without +publishing environment-specific names or label values upstream. ## Results and evidence @@ -365,16 +397,22 @@ executionState=Error, verdict=Inconclusive, reason= ``` Ambiguous or incomplete workflow execution remains `Inconclusive`. The -controller does not infer a successful result from partial evidence. +controller identifies each check only by its deterministic DAG task-node name +and rejects missing, duplicate, malformed, partial, or unexpected outputs. An +aggregate `Pass` requires all 15 checks, exact participant and backend +coverage, complete v1 backend evidence, and successful backend cleanup. If backend execution and cleanup both fail, `reason` retains the first causal backend failure and the bounded `cleanup-failure` Workflow output records `BackendCleanupFailed` separately. Cleanup failure always forces `executionState=Error` and `verdict=Inconclusive`, including after an otherwise successful diagnostic. -Cluster-local evidence consists of CertificationRun status, the referenced -Workflow and backend status, Events, pod logs, termination messages, and the -SHA-256 digest of the controller's terminal evidence document. +CertificationRun status retains bounded measurements, backend identities, and +the SHA-256 digest and completeness of each check's backend-evidence v1 +document. Full backend-evidence content and script snapshots are retained only +in the immutable terminal evidence artifact. Small artifacts use one +ConfigMap; larger artifacts use an immutable manifest plus bounded immutable +chunks so valid grouped evidence cannot exceed one Kubernetes object limit. ## Node metadata @@ -406,8 +444,8 @@ kubectl -n project-signal patch certificationrun \ --type=merge -p '{"spec":{"cancel":true}}' ``` -Before removing the controller, disable recurring policies and wait for active -runs to become terminal. Argo Workflows and any workload backend may be shared +Before removing the controller, wait for active grouped runs to become +terminal. Argo Workflows and any workload backend may be shared cluster services and must not be removed as part of Project Signal cleanup. ## Non-goals diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml index 7a05a2a974..d8c5d91450 100644 --- a/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml @@ -19,9 +19,9 @@ spec: scope: Cluster versions: - additionalPrinterColumns: - - jsonPath: .spec.backend.kind - name: Backend - type: string + - jsonPath: .spec.checks.size() + name: Checks + type: integer - jsonPath: .spec.participants.count name: Participants type: integer @@ -51,31 +51,619 @@ spec: type: object spec: properties: - backend: - properties: - apiVersion: - type: string - kind: - type: string - queue: - type: string - required: - - apiVersion - - kind - type: object - backendActiveDeadline: - type: string + checks: + items: + properties: + backend: + properties: + apiVersion: + type: string + kind: + type: string + queue: + type: string + required: + - apiVersion + - kind + type: object + backendActiveDeadline: + type: string + id: + maxLength: 63 + minLength: 1 + pattern: ^R(0[1-9]|1[0-5])$ + type: string + maxAttempts: + format: int32 + type: integer + order: + format: int32 + maximum: 14 + minimum: 0 + type: integer + parameters: + additionalProperties: + type: string + type: object + participantPolicy: + properties: + excludedRoles: + items: + type: string + maxItems: 16 + type: array + requiredRoles: + items: + type: string + maxItems: 16 + type: array + type: object + resultBounds: + properties: + backendEvidenceBytes: + format: int32 + maximum: 65536 + minimum: 1 + type: integer + backendResourcesBytes: + format: int32 + maximum: 16384 + minimum: 1 + type: integer + measurementsBytes: + format: int32 + maximum: 16384 + minimum: 1 + type: integer + reasonBytes: + format: int32 + maximum: 4096 + minimum: 1 + type: integer + required: + - backendEvidenceBytes + - backendResourcesBytes + - measurementsBytes + - reasonBytes + type: object + runnerImage: + type: string + runtimeStage: + properties: + name: + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + script: + properties: + digest: + description: Digest is the approved algorithm-qualified + SHA-256 digest of the script bytes. + pattern: ^sha256:[0-9a-f]{64}$ + type: string + key: + type: string + name: + type: string + requiresRuntimeStage: + type: boolean + required: + - digest + - key + - name + type: object + workloadPolicy: + properties: + hostNetwork: + type: boolean + kueueTopology: + properties: + requiredLevel: + description: |- + RequiredLevel is the Node label key for the minimum topology domain in which + Kueue must place the participant PodSet. + maxLength: 316 + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - requiredLevel + type: object + resources: + description: ResourceRequirements describes the compute + resource requirements. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in + PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + results: + description: |- + Represents an empty directory for a pod. + Empty directory volumes support ownership management and SELinux relabeling. + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + mode: + description: |- + mode specifies the permission bits for the emptyDir directory, in numeric + notation (e.g., 0755, 01777). Must be a value between 0000 and 01777. + If not specified, defaults to 0777. + This might be in conflict with other options that affect the file + mode, like fsGroup. If fsGroup is specified, the fsGroup permissions + will override the mode specified here. + This field has no effect on Windows. + This field is alpha and requires EmptyDirVolumeMode featuregate to be enabled. + format: int32 + type: integer + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + securityContext: + description: |- + SecurityContext holds security configuration that will be applied to a container. + Some fields are present in both SecurityContext and PodSecurityContext. When both + are set, the values in SecurityContext take precedence. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + volumes: + items: + properties: + emptyDir: + description: |- + Represents an empty directory for a pod. + Empty directory volumes support ownership management and SELinux relabeling. + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + mode: + description: |- + mode specifies the permission bits for the emptyDir directory, in numeric + notation (e.g., 0755, 01777). Must be a value between 0000 and 01777. + If not specified, defaults to 0777. + This might be in conflict with other options that affect the file + mode, like fsGroup. If fsGroup is specified, the fsGroup permissions + will override the mode specified here. + This field has no effect on Windows. + This field is alpha and requires EmptyDirVolumeMode featuregate to be enabled. + format: int32 + type: integer + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + hostPath: + description: |- + Represents a host path mapped into a pod. + Host path volumes do not support ownership management or SELinux relabeling. + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + mountPath: + type: string + name: + type: string + persistentVolumeClaim: + description: |- + PersistentVolumeClaimVolumeSource references the user's PVC in the same namespace. + This volume finds the bound PV and mounts that volume for the pod. A + PersistentVolumeClaimVolumeSource is, essentially, a wrapper around another + type of volume that is owned by someone else (the system). + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + readOnly: + type: boolean + required: + - mountPath + - name + type: object + maxItems: 16 + type: array + workingDir: + type: string + type: object + required: + - backend + - backendActiveDeadline + - id + - maxAttempts + - order + - resultBounds + - runnerImage + - script + type: object + x-kubernetes-validations: + - message: maxAttempts must be exactly 1 + rule: self.maxAttempts == 1 + - message: backendActiveDeadline must be positive + rule: duration(self.backendActiveDeadline) > duration('0s') + - message: runtimeStage is required when the script requires a runtime + stage + rule: '!has(self.script.requiresRuntimeStage) || !self.script.requiresRuntimeStage + || has(self.runtimeStage)' + maxItems: 15 + minItems: 15 + type: array deadline: type: string evidencePath: type: string - maxAttempts: - format: int32 - type: integer - parameters: - additionalProperties: - type: string + executionPolicy: + properties: + onFailure: + enum: + - Stop + - Continue + type: string + required: + - onFailure type: object + participantRoles: + items: + properties: + count: + format: int32 + maximum: 64 + minimum: 1 + type: integer + name: + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + selector: + description: Selector further constrains participants assigned + to this role. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - count + - name + type: object + maxItems: 16 + type: array participants: properties: count: @@ -201,54 +789,6 @@ spec: - nodeNames - validFor type: object - runnerImage: - type: string - runtimeStage: - properties: - name: - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - required: - - name - type: object - script: - properties: - key: - type: string - name: - type: string - requiresRuntimeStage: - type: boolean - required: - - key - - name - type: object - stagePlan: - items: - properties: - category: - type: string - conditional: - type: boolean - description: - type: string - displayName: - type: string - expectedDuration: - type: string - id: - type: string - required: - type: boolean - required: - - category - - displayName - - id - - required - type: object - type: array tolerations: items: description: |- @@ -288,6 +828,29 @@ spec: type: string type: object type: array + topologyConstraints: + items: + properties: + labelKey: + description: LabelKey is the Node label used to identify topology + domains. + type: string + minDistinctValues: + format: int32 + maximum: 64 + minimum: 1 + type: integer + role: + description: Role names the participant role whose selected + Nodes are evaluated. + type: string + required: + - labelKey + - minDistinctValues + - role + type: object + maxItems: 16 + type: array workflow: properties: name: @@ -295,432 +858,29 @@ spec: required: - name type: object - workloadPolicy: - properties: - hostNetwork: - type: boolean - kueueTopology: - properties: - requiredLevel: - description: |- - RequiredLevel is the Node label key for the minimum topology domain in which - Kueue must place the participant PodSet. - maxLength: 316 - minLength: 1 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - requiredLevel - type: object - resources: - description: ResourceRequirements describes the compute resource - requirements. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This field depends on the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - results: - description: |- - Represents an empty directory for a pod. - Empty directory volumes support ownership management and SELinux relabeling. - properties: - medium: - description: |- - medium represents what type of storage medium should back this directory. - The default is "" which means to use the node's default medium. - Must be an empty string (default) or Memory. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - type: string - mode: - description: |- - mode specifies the permission bits for the emptyDir directory, in numeric - notation (e.g., 0755, 01777). Must be a value between 0000 and 01777. - If not specified, defaults to 0777. - This might be in conflict with other options that affect the file - mode, like fsGroup. If fsGroup is specified, the fsGroup permissions - will override the mode specified here. - This field has no effect on Windows. - This field is alpha and requires EmptyDirVolumeMode featuregate to be enabled. - format: int32 - type: integer - sizeLimit: - anyOf: - - type: integer - - type: string - description: |- - sizeLimit is the total amount of local storage required for this EmptyDir volume. - The size limit is also applicable for memory medium. - The maximum usage on memory medium EmptyDir would be the minimum value between - the SizeLimit specified here and the sum of memory limits of all containers in a pod. - The default is nil which means that the limit is undefined. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - securityContext: - description: |- - SecurityContext holds security configuration that will be applied to a container. - Some fields are present in both SecurityContext and PodSecurityContext. When both - are set, the values in SecurityContext take precedence. - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities - type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities - type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies - to the container. - type: string - role: - description: Role is a SELinux role label that applies - to the container. - type: string - type: - description: Type is a SELinux type label that applies - to the container. - type: string - user: - description: User is a SELinux user label that applies - to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the - GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - volumes: - items: - properties: - emptyDir: - description: |- - Represents an empty directory for a pod. - Empty directory volumes support ownership management and SELinux relabeling. - properties: - medium: - description: |- - medium represents what type of storage medium should back this directory. - The default is "" which means to use the node's default medium. - Must be an empty string (default) or Memory. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - type: string - mode: - description: |- - mode specifies the permission bits for the emptyDir directory, in numeric - notation (e.g., 0755, 01777). Must be a value between 0000 and 01777. - If not specified, defaults to 0777. - This might be in conflict with other options that affect the file - mode, like fsGroup. If fsGroup is specified, the fsGroup permissions - will override the mode specified here. - This field has no effect on Windows. - This field is alpha and requires EmptyDirVolumeMode featuregate to be enabled. - format: int32 - type: integer - sizeLimit: - anyOf: - - type: integer - - type: string - description: |- - sizeLimit is the total amount of local storage required for this EmptyDir volume. - The size limit is also applicable for memory medium. - The maximum usage on memory medium EmptyDir would be the minimum value between - the SizeLimit specified here and the sum of memory limits of all containers in a pod. - The default is nil which means that the limit is undefined. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - hostPath: - description: |- - Represents a host path mapped into a pod. - Host path volumes do not support ownership management or SELinux relabeling. - properties: - path: - description: |- - path of the directory on the host. - If the path is a symlink, it will follow the link to the real path. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - type: - description: |- - type for HostPath Volume - Defaults to "" - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - required: - - path - type: object - mountPath: - type: string - name: - type: string - persistentVolumeClaim: - description: |- - PersistentVolumeClaimVolumeSource references the user's PVC in the same namespace. - This volume finds the bound PV and mounts that volume for the pod. A - PersistentVolumeClaimVolumeSource is, essentially, a wrapper around another - type of volume that is owned by someone else (the system). - properties: - claimName: - description: |- - claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - type: string - readOnly: - description: |- - readOnly Will force the ReadOnly setting in VolumeMounts. - Default false. - type: boolean - required: - - claimName - type: object - readOnly: - type: boolean - required: - - mountPath - - name - type: object - maxItems: 16 - type: array - workingDir: - type: string - type: object required: - - backend - - backendActiveDeadline + - checks - deadline - evidencePath + - executionPolicy - participants - - runnerImage - - script - workflow type: object x-kubernetes-validations: - - message: backendActiveDeadline must be positive - rule: duration(self.backendActiveDeadline) > duration('0s') - - message: backendActiveDeadline must leave at least 60 seconds before - deadline - rule: duration(self.deadline) - duration(self.backendActiveDeadline) - >= duration('60s') - - message: runtimeStage is required when the script requires a runtime - stage - rule: '!has(self.script.requiresRuntimeStage) || !self.script.requiresRuntimeStage - || has(self.runtimeStage)' + - message: checks must contain unique IDs + rule: self.checks.all(c, self.checks.exists_one(x, x.id == c.id)) + - message: checks must use immutable contiguous order 0 through 14 + rule: self.checks.map(c, c.order) == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14] + - message: checks must use ordered IDs R01 through R15 + rule: self.checks.map(c, c.id) == ['R01', 'R02', 'R03', 'R04', 'R05', + 'R06', 'R07', 'R08', 'R09', 'R10', 'R11', 'R12', 'R13', 'R14', 'R15'] + - message: every backendActiveDeadline must leave at least 60 seconds + before deadline + rule: self.checks.all(c, duration(self.deadline) - duration(c.backendActiveDeadline) + >= duration('60s')) + - message: recurring certification is not supported for grouped suites + rule: '!has(self.recurring) || !self.recurring.enabled' status: properties: conditions: diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml index 05d7f0912d..b097c308d6 100644 --- a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml @@ -57,33 +57,621 @@ spec: type: object spec: properties: - backend: - properties: - apiVersion: - type: string - kind: - type: string - queue: - type: string - required: - - apiVersion - - kind - type: object - backendActiveDeadline: - type: string cancel: type: boolean + checks: + items: + properties: + backend: + properties: + apiVersion: + type: string + kind: + type: string + queue: + type: string + required: + - apiVersion + - kind + type: object + backendActiveDeadline: + type: string + id: + maxLength: 63 + minLength: 1 + pattern: ^R(0[1-9]|1[0-5])$ + type: string + maxAttempts: + format: int32 + type: integer + order: + format: int32 + maximum: 14 + minimum: 0 + type: integer + parameters: + additionalProperties: + type: string + type: object + participantPolicy: + properties: + excludedRoles: + items: + type: string + maxItems: 16 + type: array + requiredRoles: + items: + type: string + maxItems: 16 + type: array + type: object + resultBounds: + properties: + backendEvidenceBytes: + format: int32 + maximum: 65536 + minimum: 1 + type: integer + backendResourcesBytes: + format: int32 + maximum: 16384 + minimum: 1 + type: integer + measurementsBytes: + format: int32 + maximum: 16384 + minimum: 1 + type: integer + reasonBytes: + format: int32 + maximum: 4096 + minimum: 1 + type: integer + required: + - backendEvidenceBytes + - backendResourcesBytes + - measurementsBytes + - reasonBytes + type: object + runnerImage: + type: string + runtimeStage: + properties: + name: + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + script: + properties: + configMapName: + type: string + digest: + description: Digest is the algorithm-qualified SHA-256 digest + of the selected script bytes. + pattern: ^sha256:[0-9a-f]{64}$ + type: string + key: + type: string + requiresRuntimeStage: + type: boolean + required: + - configMapName + - digest + - key + type: object + workloadPolicy: + properties: + hostNetwork: + type: boolean + kueueTopology: + properties: + requiredLevel: + description: |- + RequiredLevel is the Node label key for the minimum topology domain in which + Kueue must place the participant PodSet. + maxLength: 316 + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - requiredLevel + type: object + resources: + description: ResourceRequirements describes the compute + resource requirements. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in + PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + results: + description: |- + Represents an empty directory for a pod. + Empty directory volumes support ownership management and SELinux relabeling. + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + mode: + description: |- + mode specifies the permission bits for the emptyDir directory, in numeric + notation (e.g., 0755, 01777). Must be a value between 0000 and 01777. + If not specified, defaults to 0777. + This might be in conflict with other options that affect the file + mode, like fsGroup. If fsGroup is specified, the fsGroup permissions + will override the mode specified here. + This field has no effect on Windows. + This field is alpha and requires EmptyDirVolumeMode featuregate to be enabled. + format: int32 + type: integer + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + securityContext: + description: |- + SecurityContext holds security configuration that will be applied to a container. + Some fields are present in both SecurityContext and PodSecurityContext. When both + are set, the values in SecurityContext take precedence. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + volumes: + items: + properties: + emptyDir: + description: |- + Represents an empty directory for a pod. + Empty directory volumes support ownership management and SELinux relabeling. + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + mode: + description: |- + mode specifies the permission bits for the emptyDir directory, in numeric + notation (e.g., 0755, 01777). Must be a value between 0000 and 01777. + If not specified, defaults to 0777. + This might be in conflict with other options that affect the file + mode, like fsGroup. If fsGroup is specified, the fsGroup permissions + will override the mode specified here. + This field has no effect on Windows. + This field is alpha and requires EmptyDirVolumeMode featuregate to be enabled. + format: int32 + type: integer + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + hostPath: + description: |- + Represents a host path mapped into a pod. + Host path volumes do not support ownership management or SELinux relabeling. + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + mountPath: + type: string + name: + type: string + persistentVolumeClaim: + description: |- + PersistentVolumeClaimVolumeSource references the user's PVC in the same namespace. + This volume finds the bound PV and mounts that volume for the pod. A + PersistentVolumeClaimVolumeSource is, essentially, a wrapper around another + type of volume that is owned by someone else (the system). + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + readOnly: + type: boolean + required: + - mountPath + - name + type: object + maxItems: 16 + type: array + workingDir: + type: string + type: object + required: + - backend + - backendActiveDeadline + - id + - maxAttempts + - order + - resultBounds + - runnerImage + - script + type: object + x-kubernetes-validations: + - message: maxAttempts must be exactly 1 + rule: self.maxAttempts == 1 + - message: backendActiveDeadline must be positive + rule: duration(self.backendActiveDeadline) > duration('0s') + - message: runtimeStage is required when the script requires a runtime + stage + rule: '!has(self.script.requiresRuntimeStage) || !self.script.requiresRuntimeStage + || has(self.runtimeStage)' + maxItems: 15 + minItems: 15 + type: array deadline: type: string evidencePath: type: string - maxAttempts: - format: int32 - type: integer - parameters: - additionalProperties: - type: string + executionPolicy: + properties: + onFailure: + enum: + - Stop + - Continue + type: string + required: + - onFailure type: object + participantRoles: + items: + properties: + count: + format: int32 + maximum: 64 + minimum: 1 + type: integer + name: + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + selector: + description: Selector further constrains participants assigned + to this role. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - count + - name + type: object + maxItems: 16 + type: array participants: items: properties: @@ -119,6 +707,11 @@ spec: don't ONLY use UUIDs, this is an alias to string. Being a type captures intent and helps make sure that UIDs and names do not get conflated. type: string + roles: + items: + type: string + maxItems: 16 + type: array required: - index - nodeName @@ -155,60 +748,6 @@ spec: type: string requestNodeName: type: string - runnerImage: - type: string - runtimeStage: - properties: - name: - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - required: - - name - type: object - script: - properties: - configMapName: - type: string - digest: - description: Digest is the algorithm-qualified SHA-256 digest - of the selected script bytes. - pattern: ^sha256:[0-9a-f]{64}$ - type: string - key: - type: string - requiresRuntimeStage: - type: boolean - required: - - configMapName - - digest - - key - type: object - stagePlan: - items: - properties: - category: - type: string - conditional: - type: boolean - description: - type: string - displayName: - type: string - expectedDuration: - type: string - id: - type: string - required: - type: boolean - required: - - category - - displayName - - id - - required - type: object - type: array tolerations: items: description: |- @@ -248,6 +787,29 @@ spec: type: string type: object type: array + topologyConstraints: + items: + properties: + labelKey: + description: LabelKey is the Node label used to identify topology + domains. + type: string + minDistinctValues: + format: int32 + maximum: 64 + minimum: 1 + type: integer + role: + description: Role names the participant role whose selected + Nodes are evaluated. + type: string + required: + - labelKey + - minDistinctValues + - role + type: object + maxItems: 16 + type: array workflow: properties: name: @@ -255,422 +817,14 @@ spec: required: - name type: object - workloadPolicy: - properties: - hostNetwork: - type: boolean - kueueTopology: - properties: - requiredLevel: - description: |- - RequiredLevel is the Node label key for the minimum topology domain in which - Kueue must place the participant PodSet. - maxLength: 316 - minLength: 1 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - requiredLevel - type: object - resources: - description: ResourceRequirements describes the compute resource - requirements. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This field depends on the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - results: - description: |- - Represents an empty directory for a pod. - Empty directory volumes support ownership management and SELinux relabeling. - properties: - medium: - description: |- - medium represents what type of storage medium should back this directory. - The default is "" which means to use the node's default medium. - Must be an empty string (default) or Memory. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - type: string - mode: - description: |- - mode specifies the permission bits for the emptyDir directory, in numeric - notation (e.g., 0755, 01777). Must be a value between 0000 and 01777. - If not specified, defaults to 0777. - This might be in conflict with other options that affect the file - mode, like fsGroup. If fsGroup is specified, the fsGroup permissions - will override the mode specified here. - This field has no effect on Windows. - This field is alpha and requires EmptyDirVolumeMode featuregate to be enabled. - format: int32 - type: integer - sizeLimit: - anyOf: - - type: integer - - type: string - description: |- - sizeLimit is the total amount of local storage required for this EmptyDir volume. - The size limit is also applicable for memory medium. - The maximum usage on memory medium EmptyDir would be the minimum value between - the SizeLimit specified here and the sum of memory limits of all containers in a pod. - The default is nil which means that the limit is undefined. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - securityContext: - description: |- - SecurityContext holds security configuration that will be applied to a container. - Some fields are present in both SecurityContext and PodSecurityContext. When both - are set, the values in SecurityContext take precedence. - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities - type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities - type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies - to the container. - type: string - role: - description: Role is a SELinux role label that applies - to the container. - type: string - type: - description: Type is a SELinux type label that applies - to the container. - type: string - user: - description: User is a SELinux user label that applies - to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the - GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - volumes: - items: - properties: - emptyDir: - description: |- - Represents an empty directory for a pod. - Empty directory volumes support ownership management and SELinux relabeling. - properties: - medium: - description: |- - medium represents what type of storage medium should back this directory. - The default is "" which means to use the node's default medium. - Must be an empty string (default) or Memory. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - type: string - mode: - description: |- - mode specifies the permission bits for the emptyDir directory, in numeric - notation (e.g., 0755, 01777). Must be a value between 0000 and 01777. - If not specified, defaults to 0777. - This might be in conflict with other options that affect the file - mode, like fsGroup. If fsGroup is specified, the fsGroup permissions - will override the mode specified here. - This field has no effect on Windows. - This field is alpha and requires EmptyDirVolumeMode featuregate to be enabled. - format: int32 - type: integer - sizeLimit: - anyOf: - - type: integer - - type: string - description: |- - sizeLimit is the total amount of local storage required for this EmptyDir volume. - The size limit is also applicable for memory medium. - The maximum usage on memory medium EmptyDir would be the minimum value between - the SizeLimit specified here and the sum of memory limits of all containers in a pod. - The default is nil which means that the limit is undefined. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - hostPath: - description: |- - Represents a host path mapped into a pod. - Host path volumes do not support ownership management or SELinux relabeling. - properties: - path: - description: |- - path of the directory on the host. - If the path is a symlink, it will follow the link to the real path. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - type: - description: |- - type for HostPath Volume - Defaults to "" - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - required: - - path - type: object - mountPath: - type: string - name: - type: string - persistentVolumeClaim: - description: |- - PersistentVolumeClaimVolumeSource references the user's PVC in the same namespace. - This volume finds the bound PV and mounts that volume for the pod. A - PersistentVolumeClaimVolumeSource is, essentially, a wrapper around another - type of volume that is owned by someone else (the system). - properties: - claimName: - description: |- - claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - type: string - readOnly: - description: |- - readOnly Will force the ReadOnly setting in VolumeMounts. - Default false. - type: boolean - required: - - claimName - type: object - readOnly: - type: boolean - required: - - mountPath - - name - type: object - maxItems: 16 - type: array - workingDir: - type: string - type: object required: - - backend - - backendActiveDeadline + - checks - deadline - evidencePath - - maxAttempts + - executionPolicy - participants - profile - requestNodeName - - runnerImage - - script - workflow type: object x-kubernetes-validations: @@ -678,54 +832,144 @@ spec: rule: self.profile == oldSelf.profile - message: participants are immutable rule: self.participants == oldSelf.participants - - message: script is immutable - rule: self.script == oldSelf.script + - message: participantRoles are immutable + rule: has(self.participantRoles) == has(oldSelf.participantRoles) && + (!has(self.participantRoles) || self.participantRoles == oldSelf.participantRoles) + - message: topologyConstraints are immutable + rule: has(self.topologyConstraints) == has(oldSelf.topologyConstraints) + && (!has(self.topologyConstraints) || self.topologyConstraints == + oldSelf.topologyConstraints) + - message: checks are immutable + rule: self.checks == oldSelf.checks + - message: executionPolicy is immutable + rule: self.executionPolicy == oldSelf.executionPolicy - message: workflow is immutable rule: self.workflow == oldSelf.workflow - - message: backend is immutable - rule: self.backend == oldSelf.backend - - message: runnerImage is immutable - rule: self.runnerImage == oldSelf.runnerImage - - message: workloadPolicy is immutable - rule: has(self.workloadPolicy) == has(oldSelf.workloadPolicy) && (!has(self.workloadPolicy) - || self.workloadPolicy == oldSelf.workloadPolicy) - - message: runtimeStage is immutable - rule: has(self.runtimeStage) == has(oldSelf.runtimeStage) && (!has(self.runtimeStage) - || self.runtimeStage == oldSelf.runtimeStage) - - message: parameters are immutable - rule: has(self.parameters) == has(oldSelf.parameters) && (!has(self.parameters) - || self.parameters == oldSelf.parameters) - message: purpose is immutable rule: has(self.purpose) == has(oldSelf.purpose) && (!has(self.purpose) || self.purpose == oldSelf.purpose) - - message: stagePlan is immutable - rule: has(self.stagePlan) == has(oldSelf.stagePlan) && (!has(self.stagePlan) - || self.stagePlan == oldSelf.stagePlan) - message: evidencePath is immutable rule: self.evidencePath == oldSelf.evidencePath - message: deadline is immutable rule: self.deadline == oldSelf.deadline - - message: backendActiveDeadline is immutable - rule: self.backendActiveDeadline == oldSelf.backendActiveDeadline - - message: maxAttempts is immutable - rule: self.maxAttempts == oldSelf.maxAttempts - message: tolerations are immutable rule: has(self.tolerations) == has(oldSelf.tolerations) && (!has(self.tolerations) || self.tolerations == oldSelf.tolerations) - message: requestNodeName is immutable rule: self.requestNodeName == oldSelf.requestNodeName - - message: backendActiveDeadline must be positive - rule: duration(self.backendActiveDeadline) > duration('0s') - - message: backendActiveDeadline must leave at least 60 seconds before - deadline - rule: duration(self.deadline) - duration(self.backendActiveDeadline) - >= duration('60s') - - message: runtimeStage is required when the script requires a runtime - stage - rule: '!has(self.script.requiresRuntimeStage) || !self.script.requiresRuntimeStage - || has(self.runtimeStage)' + - message: checks must contain unique IDs + rule: self.checks.all(c, self.checks.exists_one(x, x.id == c.id)) + - message: checks must use immutable contiguous order 0 through 14 + rule: self.checks.map(c, c.order) == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14] + - message: checks must use ordered IDs R01 through R15 + rule: self.checks.map(c, c.id) == ['R01', 'R02', 'R03', 'R04', 'R05', + 'R06', 'R07', 'R08', 'R09', 'R10', 'R11', 'R12', 'R13', 'R14', 'R15'] + - message: every backendActiveDeadline must leave at least 60 seconds + before deadline + rule: self.checks.all(c, duration(self.deadline) - duration(c.backendActiveDeadline) + >= duration('60s')) status: properties: + checks: + items: + properties: + attempt: + format: int32 + type: integer + backendEvidence: + properties: + complete: + type: boolean + content: + description: |- + Content is populated only while constructing the durable evidence + artifact and is not persisted in CertificationRun status. + type: string + digest: + description: |- + Digest identifies the complete backend-evidence v1 document retained in + the durable evidence artifact. + pattern: ^sha256:[0-9a-f]{64}$ + type: string + version: + type: string + type: object + backendResources: + items: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + participantIndexes: + items: + format: int32 + type: integer + maxItems: 64 + minItems: 1 + type: array + uid: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + required: + - apiVersion + - kind + - name + - uid + type: object + type: array + cleanup: + properties: + complete: + type: boolean + reason: + type: string + type: object + duration: + type: string + finishedAt: + format: date-time + type: string + id: + type: string + measurements: + type: string + notRun: + type: boolean + order: + format: int32 + type: integer + phase: + type: string + reason: + type: string + retryCount: + format: int32 + type: integer + startedAt: + format: date-time + type: string + verdict: + type: string + workflowNodeName: + type: string + required: + - attempt + - id + - order + - phase + - retryCount + type: object + maxItems: 15 + type: array cleanup: properties: reservationsReleased: diff --git a/deploy/signal/manifest_test.go b/deploy/signal/manifest_test.go index ff9d34952d..35fe8bcd2c 100644 --- a/deploy/signal/manifest_test.go +++ b/deploy/signal/manifest_test.go @@ -112,7 +112,7 @@ func TestAdapterRBACAllowsBackendEvidenceCapture(t *testing.T) { } } -func TestSignalCRDsExposeRuntimeStageAndDeadlineContract(t *testing.T) { +func TestSignalCRDsExposeGroupedRuntimeAndDeadlineContract(t *testing.T) { t.Parallel() for _, path := range []string{ @@ -126,21 +126,50 @@ func TestSignalCRDsExposeRuntimeStageAndDeadlineContract(t *testing.T) { manifest := string(content) for _, required := range []string{ + "checks:", + "maxItems: 15", + "minItems: 15", + "executionPolicy:", + "participantRoles:", + "topologyConstraints:", + "resultBounds:", "backendActiveDeadline:", "runtimeStage:", "requiresRuntimeStage:", - "backendActiveDeadline must leave at least 60 seconds before", + "every backendActiveDeadline must leave at least 60 seconds", "runtimeStage is required when the script requires a runtime", + "maxAttempts must be exactly 1", + "checks must contain unique IDs", "maxLength: 63", } { if !strings.Contains(manifest, required) { t.Fatalf("%s does not contain %q", path, required) } + + profileContent, err := os.ReadFile("crd/signal.unbounded-cloud.io_certificationprofiles.yaml") + if err != nil { + t.Fatal(err) + } + + if !strings.Contains(string(profileContent), "recurring certification is not supported for grouped suites") { + t.Fatal("CertificationProfile CRD does not reject grouped recurrence") + } + + runContent, err := os.ReadFile("crd/signal.unbounded-cloud.io_certificationruns.yaml") + if err != nil { + t.Fatal(err) + } + + for _, required := range []string{"backendEvidence:", "workflowNodeName:", "retryCount:", "notRun:"} { + if !strings.Contains(string(runContent), required) { + t.Fatalf("CertificationRun CRD does not contain per-check status field %q", required) + } + } } } } -func TestCertificationRunCRDUsesPresenceAwareOptionalImmutability(t *testing.T) { +func TestCertificationRunCRDMakesGroupedSnapshotImmutable(t *testing.T) { t.Parallel() content, err := os.ReadFile("crd/signal.unbounded-cloud.io_certificationruns.yaml") @@ -149,25 +178,20 @@ func TestCertificationRunCRDUsesPresenceAwareOptionalImmutability(t *testing.T) } manifest := string(content) - optionalImmutableFields := []string{ - "workloadPolicy", - "runtimeStage", - "parameters", - "purpose", - "stagePlan", - "tolerations", + requiredRules := []string{ + "checks are immutable", + "participants are immutable", + "participantRoles are immutable", + "topologyConstraints are immutable", + "executionPolicy is immutable", + "workflow is immutable", } - for _, field := range optionalImmutableFields { - rule := "has(self." + field + ") == has(oldSelf." + field + ")" + for _, rule := range requiredRules { if !strings.Contains(manifest, rule) { - t.Fatalf("CertificationRun CRD does not contain presence-aware rule %q", rule) + t.Fatalf("CertificationRun CRD does not contain rule %q", rule) } } - - if strings.Contains(manifest, "rule: self.runtimeStage == oldSelf.runtimeStage") { - t.Fatal("CertificationRun CRD still dereferences an omitted runtimeStage") - } } func TestCertificationRunCRDRequiresCanonicalDigests(t *testing.T) { @@ -179,8 +203,8 @@ func TestCertificationRunCRDRequiresCanonicalDigests(t *testing.T) { } const digestPattern = "pattern: ^sha256:[0-9a-f]{64}$" - if count := strings.Count(string(content), digestPattern); count != 2 { - t.Fatalf("canonical digest patterns = %d, want 2", count) + if count := strings.Count(string(content), digestPattern); count != 3 { + t.Fatalf("canonical digest patterns = %d, want 3", count) } } diff --git a/internal/signal/controller.go b/internal/signal/controller.go index 0ff36b5dcc..da317d0707 100644 --- a/internal/signal/controller.go +++ b/internal/signal/controller.go @@ -11,8 +11,10 @@ import ( "errors" "fmt" "io" + "maps" "reflect" "regexp" + "slices" "sort" "strings" "sync" @@ -81,6 +83,8 @@ var workflowOutputLimits = map[string]int{ "verdict": 32, } +const groupedCheckCount = 15 + var workflowGVK = schema.GroupVersionKind{ Group: "argoproj.io", Version: "v1alpha1", @@ -223,33 +227,41 @@ func (r *AdmissionReconciler) admitNodeWithTriggerLocked( return fmt.Errorf("get CertificationProfile %s: %w", profileName, err) } - if err := validateStagePlan(profile.Spec.StagePlan); err != nil { - return fmt.Errorf("profile %s stage plan: %w", profile.Name, err) + checks := profileChecks(&profile) + if err := validateCheckPlan(checks, len(profile.Spec.Checks) == groupedCheckCount); err != nil { + return fmt.Errorf("profile %s checks: %w", profile.Name, err) } - if err := signalv1.ValidateRuntimeContract( - profile.Spec.Script.RequiresRuntimeStage, - profile.Spec.RuntimeStage, - profile.Spec.BackendActiveDeadline.Duration, - profile.Spec.Deadline.Duration, - ); err != nil { - return fmt.Errorf("profile %s runtime contract: %w", profile.Name, err) + if err := validateParticipantConstraints(&profile, checks); err != nil { + return fmt.Errorf("profile %s participant constraints: %w", profile.Name, err) } - if !runnerImageDigestPattern.MatchString(profile.Spec.RunnerImage) { - return fmt.Errorf("profile %s runner image must use an immutable sha256 digest", profile.Name) - } + for index := range checks { + check := &checks[index] + if err := signalv1.ValidateRuntimeContract( + check.Script.RequiresRuntimeStage, + check.RuntimeStage, + check.BackendActiveDeadline.Duration, + profile.Spec.Deadline.Duration, + ); err != nil { + return fmt.Errorf("profile %s check %s runtime contract: %w", profile.Name, check.ID, err) + } - if _, err := scriptEnvironment(profile.Spec.Parameters); err != nil { - return fmt.Errorf("profile %s parameters: %w", profile.Name, err) - } + if !runnerImageDigestPattern.MatchString(check.RunnerImage) { + return fmt.Errorf("profile %s check %s runner image must use an immutable sha256 digest", profile.Name, check.ID) + } - if err := signalworkload.Validate(profile.Spec.WorkloadPolicy); err != nil { - return fmt.Errorf("profile %s workload policy: %w", profile.Name, err) - } + if _, err := scriptEnvironment(check.Parameters); err != nil { + return fmt.Errorf("profile %s check %s parameters: %w", profile.Name, check.ID, err) + } + + if err := signalworkload.Validate(check.WorkloadPolicy); err != nil { + return fmt.Errorf("profile %s check %s workload policy: %w", profile.Name, check.ID, err) + } - if profile.Spec.WorkloadPolicy.KueueTopology != nil && profile.Spec.Backend.Queue == "" { - return fmt.Errorf("profile %s workload policy kueueTopology requires a backend queue", profile.Name) + if check.WorkloadPolicy.KueueTopology != nil && check.Backend.Queue == "" { + return fmt.Errorf("profile %s check %s workload policy kueueTopology requires a backend queue", profile.Name, check.ID) + } } participants, err := r.selectParticipants(ctx, node, &profile) @@ -271,11 +283,6 @@ func (r *AdmissionReconciler) admitNodeWithTriggerLocked( } }() - parameters := make(map[string]string, len(profile.Spec.Parameters)+2) - for key, value := range profile.Spec.Parameters { - parameters[key] = value - } - if len(participants) == 1 { var participantNode corev1.Node @@ -291,20 +298,22 @@ func (r *AdmissionReconciler) admitNodeWithTriggerLocked( gpuResource := corev1.ResourceName("nvidia.com/gpu") gpuCapacity := participantNode.Status.Capacity[gpuResource] gpuAllocatable := participantNode.Status.Allocatable[gpuResource] - parameters["observed-gpu-capacity"] = gpuCapacity.String() - parameters["observed-gpu-allocatable"] = gpuAllocatable.String() + + for index := range checks { + if checks[index].Parameters == nil { + checks[index].Parameters = map[string]string{} + } + + checks[index].Parameters["observed-gpu-capacity"] = gpuCapacity.String() + checks[index].Parameters["observed-gpu-allocatable"] = gpuAllocatable.String() + } } - snapshot, scriptDigest, err := r.createSnapshot(ctx, runName, &profile) + snapshot, checkSnapshots, err := r.createSnapshot(ctx, runName, checks) if err != nil { return err } - maxAttempts := profile.Spec.MaxAttempts - if maxAttempts < 1 { - maxAttempts = DefaultMaxAttempts - } - run := &signalv1.CertificationRun{ TypeMeta: metav1.TypeMeta{APIVersion: signalv1.GroupVersion.String(), Kind: "CertificationRun"}, ObjectMeta: metav1.ObjectMeta{ @@ -325,29 +334,23 @@ func (r *AdmissionReconciler) admitNodeWithTriggerLocked( UID: profile.UID, Generation: profile.Generation, }, - Participants: participants, - Script: signalv1.ScriptSnapshot{ - ConfigMapName: snapshot.Name, - Key: profile.Spec.Script.Key, - RequiresRuntimeStage: profile.Spec.Script.RequiresRuntimeStage, - Digest: scriptDigest, - }, - Workflow: profile.Spec.Workflow, - Backend: profile.Spec.Backend, - RunnerImage: profile.Spec.RunnerImage, - WorkloadPolicy: *profile.Spec.WorkloadPolicy.DeepCopy(), - Purpose: profile.Spec.Purpose, - StagePlan: append([]signalv1.CertificationStage(nil), profile.Spec.StagePlan...), - RuntimeStage: copyRuntimeStage(profile.Spec.RuntimeStage), - Parameters: parameters, - EvidencePath: strings.Trim(profile.Spec.EvidencePath, "/") + "/" + runName + ".json", - Deadline: profile.Spec.Deadline, - BackendActiveDeadline: profile.Spec.BackendActiveDeadline, - MaxAttempts: maxAttempts, - Tolerations: profile.Spec.Tolerations, - RequestNodeName: node.Name, + Participants: participants, + ParticipantRoles: append([]signalv1.ParticipantRoleConstraint(nil), profile.Spec.ParticipantRoles...), + TopologyConstraints: append([]signalv1.ParticipantTopologyConstraint(nil), profile.Spec.TopologyConstraints...), + Checks: checkSnapshots, + ExecutionPolicy: profile.Spec.ExecutionPolicy, + Workflow: profile.Spec.Workflow, + Purpose: profile.Spec.Purpose, + EvidencePath: strings.Trim(profile.Spec.EvidencePath, "/") + "/" + runName + ".json", + Deadline: profile.Spec.Deadline, + Tolerations: profile.Spec.Tolerations, + RequestNodeName: node.Name, }, } + if len(profile.Spec.Checks) != groupedCheckCount { + setLegacyRunAliases(&run.Spec) + run.Spec.StagePlan = append([]signalv1.CertificationStage(nil), profile.Spec.StagePlan...) + } if err := r.Create(ctx, run); err != nil { deleteErr := r.Delete(context.WithoutCancel(ctx), snapshot) @@ -381,7 +384,7 @@ func (r *AdmissionReconciler) admitNodeWithTriggerLocked( if r.Recorder != nil { r.Recorder.Eventf(run, nil, corev1.EventTypeNormal, "CertificationAdmitted", "AdmitCertification", - "Created CertificationRun %s with immutable script digest %s", run.Name, scriptDigest) + "Created CertificationRun %s with %d immutable checks", run.Name, len(checkSnapshots)) } return nil @@ -434,15 +437,11 @@ func (r *AdmissionReconciler) selectParticipants( sort.Slice(nodes.Items, func(i, j int) bool { return nodes.Items[i].Name < nodes.Items[j].Name }) - selected := []signalv1.Participant{participantSnapshot(requestNode, 0)} + candidates := []corev1.Node{*requestNode.DeepCopy()} for i := range nodes.Items { candidate := &nodes.Items[i] - if int32(len(selected)) == count { - break - } - if candidate.Name == requestNode.Name || !selector.Matches(labels.Set(candidate.Labels)) || !nodeEligible(candidate, profile.Spec.Participants.RequiredResources) || @@ -450,11 +449,27 @@ func (r *AdmissionReconciler) selectParticipants( continue } - selected = append(selected, participantSnapshot(candidate, int32(len(selected)))) + candidates = append(candidates, *candidate.DeepCopy()) + } + + if int32(len(candidates)) < count { + return nil, fmt.Errorf( + "profile %s requires %d eligible unlocked participants, found %d", + profile.Name, + count, + len(candidates), + ) } - if int32(len(selected)) != count { - return nil, fmt.Errorf("profile %s requires %d eligible unlocked participants, found %d", profile.Name, count, len(selected)) + selected, err := selectParticipantPlan( + candidates, + requestNode.Name, + count, + profile.Spec.ParticipantRoles, + profile.Spec.TopologyConstraints, + ) + if err != nil { + return nil, fmt.Errorf("profile %s participant constraints: %w", profile.Name, err) } return selected, nil @@ -496,6 +511,376 @@ func participantSnapshot(node *corev1.Node, index int32) signalv1.Participant { } } +func (r *AdmissionReconciler) assignParticipantRoles( + ctx context.Context, + participants []signalv1.Participant, + roles []signalv1.ParticipantRoleConstraint, + topology []signalv1.ParticipantTopologyConstraint, +) error { + if len(roles) == 0 { + return nil + } + + reader := r.APIReader + if reader == nil { + reader = r.Client + } + + nodes := make([]corev1.Node, len(participants)) + for index := range participants { + if err := reader.Get(ctx, client.ObjectKey{Name: participants[index].NodeName}, &nodes[index]); err != nil { + return fmt.Errorf("read participant %s for role assignment: %w", participants[index].NodeName, err) + } + } + + assignments, err := solveParticipantRoles(nodes, "", int32(len(nodes)), roles, topology) + if err != nil { + return err + } + + for index := range participants { + participants[index].Roles = assignments[index] + } + + return nil +} + +type participantRoleSlot struct { + name string +} + +func selectParticipantPlan( + candidates []corev1.Node, + requestNodeName string, + count int32, + roles []signalv1.ParticipantRoleConstraint, + topology []signalv1.ParticipantTopologyConstraint, +) ([]signalv1.Participant, error) { + assignments, err := solveParticipantRoles( + candidates, + requestNodeName, + count, + roles, + topology, + ) + if err != nil { + return nil, err + } + + selectedIndexes := make([]int, 0, count) + selected := make([]bool, len(candidates)) + + selectedIndexes = append(selectedIndexes, 0) + selected[0] = true + + for index := range candidates { + if selected[index] || len(assignments[index]) == 0 { + continue + } + + selectedIndexes = append(selectedIndexes, index) + selected[index] = true + } + + for index := range candidates { + if int32(len(selectedIndexes)) == count { + break + } + + if selected[index] { + continue + } + + selectedIndexes = append(selectedIndexes, index) + selected[index] = true + } + + if int32(len(selectedIndexes)) != count { + return nil, fmt.Errorf("participant plan selected %d participants, want %d", len(selectedIndexes), count) + } + + participants := make([]signalv1.Participant, 0, count) + for _, candidateIndex := range selectedIndexes { + participant := participantSnapshot(&candidates[candidateIndex], int32(len(participants))) + participant.Roles = slices.Clone(assignments[candidateIndex]) + participants = append(participants, participant) + } + + return participants, nil +} + +func solveParticipantRoles( + candidates []corev1.Node, + requestNodeName string, + participantCount int32, + roles []signalv1.ParticipantRoleConstraint, + topology []signalv1.ParticipantTopologyConstraint, +) ([][]string, error) { + roleNames := make(map[string]struct{}, len(roles)) + roleSelectors := make(map[string]labels.Selector, len(roles)) + slots := make([]participantRoleSlot, 0, participantCount) + + for roleIndex := range roles { + role := &roles[roleIndex] + if _, duplicate := roleNames[role.Name]; duplicate { + return nil, fmt.Errorf("duplicate participant role %q", role.Name) + } + + selector, err := metav1.LabelSelectorAsSelector(&role.Selector) + if err != nil { + return nil, fmt.Errorf("role %s selector: %w", role.Name, err) + } + + roleNames[role.Name] = struct{}{} + roleSelectors[role.Name] = selector + + for range role.Count { + slots = append(slots, participantRoleSlot{name: role.Name}) + } + } + + if int32(len(slots)) > participantCount { + return nil, fmt.Errorf( + "participant roles require %d assignments but participant count is %d", + len(slots), + participantCount, + ) + } + + for index := range topology { + if _, exists := roleNames[topology[index].Role]; !exists { + return nil, fmt.Errorf("topology constraint references unknown role %q", topology[index].Role) + } + } + + assignments := make([][]string, len(candidates)) + assigned := make([]bool, len(candidates)) + slotAssignments := make([]int, len(slots)) + candidateOrders := make(map[string][]int, len(roles)) + + for role, selector := range roleSelectors { + candidateOrders[role] = roleCandidateOrder(candidates, selector, role, topology) + } + + const maximumSearchStates = 1_000_000 + + searchStates := 0 + searchExhausted := false + + var search func(int) bool + + search = func(slotIndex int) bool { + if searchExhausted { + return false + } + + searchStates++ + if searchStates > maximumSearchStates { + searchExhausted = true + + return false + } + + if slotIndex == len(slots) { + requestAssigned := requestNodeName == "" + assignedCount := int32(0) + + for index := range candidates { + if len(assignments[index]) == 0 { + continue + } + + assignedCount++ + + if candidates[index].Name == requestNodeName { + requestAssigned = true + } + } + + if !requestAssigned && assignedCount >= participantCount { + return false + } + + return participantTopologySatisfied(candidates, assignments, topology) + } + + slot := &slots[slotIndex] + + firstCandidate := 0 + if slotIndex > 0 && slots[slotIndex-1].name == slot.name { + firstCandidate = slotAssignments[slotIndex-1] + 1 + } + + order := candidateOrders[slot.name] + for candidatePosition := firstCandidate; candidatePosition < len(order); candidatePosition++ { + candidateIndex := order[candidatePosition] + if assigned[candidateIndex] { + continue + } + + assigned[candidateIndex] = true + assignments[candidateIndex] = []string{slot.name} + slotAssignments[slotIndex] = candidatePosition + + roleComplete := slotIndex+1 == len(slots) || slots[slotIndex+1].name != slot.name + if (!roleComplete || participantRoleTopologySatisfied( + candidates, + assignments, + slot.name, + topology, + )) && search(slotIndex+1) { + return true + } + + assignments[candidateIndex] = nil + assigned[candidateIndex] = false + } + + return false + } + + if !search(0) { + if searchExhausted { + return nil, fmt.Errorf("participant role search exceeded %d states", maximumSearchStates) + } + + return nil, errors.New("eligible participants cannot satisfy role and topology constraints") + } + + return assignments, nil +} + +func roleCandidateOrder( + candidates []corev1.Node, + selector labels.Selector, + role string, + topology []signalv1.ParticipantTopologyConstraint, +) []int { + constraints := make([]signalv1.ParticipantTopologyConstraint, 0) + + for index := range topology { + if topology[index].Role == role { + constraints = append(constraints, topology[index]) + } + } + + if len(constraints) == 0 { + order := make([]int, 0, len(candidates)) + for index := range candidates { + if selector.Matches(labels.Set(candidates[index].Labels)) { + order = append(order, index) + } + } + + return order + } + + groups := map[string][]int{} + + for index := range candidates { + if !selector.Matches(labels.Set(candidates[index].Labels)) { + continue + } + + values := make([]string, len(constraints)) + valid := true + + for constraintIndex := range constraints { + values[constraintIndex] = candidates[index].Labels[constraints[constraintIndex].LabelKey] + if values[constraintIndex] == "" { + valid = false + + break + } + } + + if valid { + key := strings.Join(values, "\x00") + groups[key] = append(groups[key], index) + } + } + + keys := slices.Sorted(maps.Keys(groups)) + order := make([]int, 0, len(candidates)) + + for offset := 0; ; offset++ { + added := false + + for _, key := range keys { + if offset >= len(groups[key]) { + continue + } + + order = append(order, groups[key][offset]) + added = true + } + + if !added { + return order + } + } +} + +func participantRoleTopologySatisfied( + candidates []corev1.Node, + assignments [][]string, + role string, + topology []signalv1.ParticipantTopologyConstraint, +) bool { + for index := range topology { + if topology[index].Role != role { + continue + } + + if !participantTopologyConstraintSatisfied(candidates, assignments, &topology[index]) { + return false + } + } + + return true +} + +func participantTopologySatisfied( + candidates []corev1.Node, + assignments [][]string, + topology []signalv1.ParticipantTopologyConstraint, +) bool { + for constraintIndex := range topology { + if !participantTopologyConstraintSatisfied( + candidates, + assignments, + &topology[constraintIndex], + ) { + return false + } + } + + return true +} + +func participantTopologyConstraintSatisfied( + candidates []corev1.Node, + assignments [][]string, + constraint *signalv1.ParticipantTopologyConstraint, +) bool { + values := map[string]struct{}{} + + for participantIndex := range candidates { + if !slices.Contains(assignments[participantIndex], constraint.Role) { + continue + } + + value := candidates[participantIndex].Labels[constraint.LabelKey] + if value == "" { + return false + } + + values[value] = struct{}{} + } + + return int32(len(values)) >= constraint.MinDistinctValues +} + func nodeEligible(node *corev1.Node, required corev1.ResourceList) bool { if node.Spec.Unschedulable || !nodeReady(node) { return false @@ -604,19 +989,8 @@ func (r *AdmissionReconciler) releaseParticipants(ctx context.Context, participa func (r *AdmissionReconciler) createSnapshot( ctx context.Context, runName string, - profile *signalv1.CertificationProfile, -) (*corev1.ConfigMap, string, error) { - var source corev1.ConfigMap - if err := r.Get(ctx, client.ObjectKey{Namespace: r.ScriptNamespace, Name: profile.Spec.Script.Name}, &source); err != nil { - return nil, "", fmt.Errorf("get source script ConfigMap: %w", err) - } - - content, ok := source.Data[profile.Spec.Script.Key] - if !ok { - return nil, "", fmt.Errorf("source script ConfigMap %s has no key %s", source.Name, profile.Spec.Script.Key) - } - - digest := digestString(content) + checks []signalv1.CertificationCheck, +) (*corev1.ConfigMap, []signalv1.CertificationCheckSnapshot, error) { immutable := true snapshot := &corev1.ConfigMap{ @@ -626,16 +1000,74 @@ func (r *AdmissionReconciler) createSnapshot( Labels: map[string]string{SnapshotRunLabel: runName}, }, Immutable: &immutable, - Data: map[string]string{ - profile.Spec.Script.Key: content, - "sha256": digest, - }, + Data: map[string]string{}, + } + + checkSnapshots := make([]signalv1.CertificationCheckSnapshot, len(checks)) + for index := range checks { + check := &checks[index] + + var source corev1.ConfigMap + if err := r.Get( + ctx, + client.ObjectKey{Namespace: r.ScriptNamespace, Name: check.Script.Name}, + &source, + ); err != nil { + return nil, nil, fmt.Errorf("get source script ConfigMap for check %s: %w", check.ID, err) + } + + content, ok := source.Data[check.Script.Key] + if !ok { + return nil, nil, fmt.Errorf( + "source script ConfigMap %s has no key %s for check %s", + source.Name, + check.Script.Key, + check.ID, + ) + } + + digest := digestString(content) + if check.Script.Digest != "" && digest != check.Script.Digest { + return nil, nil, fmt.Errorf( + "source script digest for check %s is %s, want %s", + check.ID, + digest, + check.Script.Digest, + ) + } + + key := fmt.Sprintf("%02d-%s", check.Order, sanitizeTaskName(string(check.ID))) + if len(checks) == 1 && check.ID == "legacy" { + key = check.Script.Key + } + + snapshot.Data[key] = content + checkSnapshots[index] = signalv1.CertificationCheckSnapshot{ + ID: check.ID, + Order: check.Order, + Script: signalv1.ScriptSnapshot{ + ConfigMapName: snapshot.Name, + Key: key, + RequiresRuntimeStage: check.Script.RequiresRuntimeStage, + Digest: digest, + }, + RunnerImage: check.RunnerImage, + Backend: check.Backend, + WorkloadPolicy: *check.WorkloadPolicy.DeepCopy(), + RuntimeStage: copyRuntimeStage(check.RuntimeStage), + Parameters: maps.Clone(check.Parameters), + BackendActiveDeadline: check.BackendActiveDeadline, + ResultBounds: check.ResultBounds, + ParticipantPolicy: check.ParticipantPolicy, + MaxAttempts: check.MaxAttempts, + } } + if err := r.Create(ctx, snapshot); err != nil { - return nil, "", fmt.Errorf("create immutable script snapshot: %w", err) + return nil, nil, fmt.Errorf("create immutable script snapshot: %w", err) } - return snapshot, digest, nil + return snapshot, checkSnapshots, nil } func (r *RunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -652,7 +1084,14 @@ func (r *RunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R return r.cleanup(ctx, &run) } - if run.Status.FinishedAt != nil { + if isGroupedRun(&run) && run.Status.Evidence.URI == "" { + result, recovered, err := r.resumeGroupedEvidence(ctx, &run) + if err != nil || recovered { + return result, err + } + } + + if run.Status.FinishedAt != nil && !isGroupedRun(&run) { return r.finish( ctx, &run, @@ -681,6 +1120,16 @@ func (r *RunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R if err := reader.Get(ctx, client.ObjectKey{Namespace: run.Namespace, Name: run.Status.Workflow.Name}, workflow); err != nil { if apierrors.IsNotFound(err) { + if isGroupedRun(&run) { + workflow.SetName(run.Status.Workflow.Name) + workflow.Object["status"] = map[string]any{ + "phase": "Error", + "message": "workflow disappeared before terminal status", + } + + return r.finishGrouped(ctx, &run, workflow, "Error") + } + return r.retryOrFinish(ctx, &run, "Error", "WorkflowMissing", "workflow disappeared before terminal status") } @@ -707,11 +1156,20 @@ func (r *RunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R return ctrl.Result{}, err } - if run.Status.Phase != signalv1.RunPhaseRunning || run.Status.Workflow.Phase != phase { + checkStatuses := run.Status.Checks + if isGroupedRun(&run) { + checkStatuses = runningCheckStatuses(workflow, &run) + } + + if run.Status.Phase != signalv1.RunPhaseRunning || + run.Status.Workflow.Phase != phase || + !reflect.DeepEqual(run.Status.Checks, checkStatuses) { before := run.DeepCopy() run.Status.Phase = signalv1.RunPhaseRunning run.Status.Workflow.Phase = phase + + run.Status.Checks = checkStatuses if run.Status.StartedAt == nil { now := metav1.NewTime(r.now()) run.Status.StartedAt = &now @@ -722,6 +1180,10 @@ func (r *RunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R return ctrl.Result{RequeueAfter: 15 * time.Second}, nil case "Succeeded": + if isGroupedRun(&run) { + return r.finishGrouped(ctx, &run, workflow, phase) + } + verdict := workflowParameter(workflow, "verdict") reason := workflowParameter(workflow, "reason") @@ -732,6 +1194,10 @@ func (r *RunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R return r.finish(ctx, &run, workflow, "Completed", verdict, reason) case "Failed", "Error": + if isGroupedRun(&run) { + return r.finishGrouped(ctx, &run, workflow, phase) + } + return r.retryOrFinish(ctx, &run, phase, "Workflow"+phase, workflowMessage(workflow)) default: return ctrl.Result{}, fmt.Errorf("unsupported Workflow phase %q", phase) @@ -773,6 +1239,10 @@ func (r *RunReconciler) startAttempt(ctx context.Context, run *signalv1.Certific Attempt: attempt, Phase: "Pending", } + if isGroupedRun(run) { + run.Status.Checks = initialCheckStatuses(run.Spec.Checks) + } + if run.Status.StartedAt == nil { now := metav1.NewTime(r.now()) run.Status.StartedAt = &now @@ -791,9 +1261,13 @@ func (r *RunReconciler) startAttempt(ctx context.Context, run *signalv1.Certific } func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured.Unstructured, error) { - participantNames := make([]string, 0, len(run.Spec.Participants)) - for _, participant := range run.Spec.Participants { - participantNames = append(participantNames, participant.NodeName) + if !isGroupedRun(run) { + return buildLegacyWorkflow(run, attempt) + } + + checks := runChecks(run) + if err := validateRunCheckPlan(checks, isGroupedRun(run)); err != nil { + return nil, err } tolerations := uniqueTolerations(append([]corev1.Toleration{{ @@ -807,17 +1281,158 @@ func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured return nil, fmt.Errorf("encode Workflow tolerations: %w", err) } - scriptEnvironment, err := scriptEnvironment(run.Spec.Parameters) + name := fmt.Sprintf("%s-a%d", run.Name, attempt) + tasks := make([]any, 0, len(checks)) + + for index := range checks { + check := &checks[index] + + checkParticipants, _, participantErr := participantsForCheck( + run.Spec.Participants, + check.ParticipantPolicy, + ) + if participantErr != nil { + return nil, fmt.Errorf("check %s participants: %w", check.ID, participantErr) + } + + participantNames := make([]string, 0, len(checkParticipants)) + for participantIndex := range checkParticipants { + participantNames = append(participantNames, checkParticipants[participantIndex].NodeName) + } + + participantsJSON, marshalErr := json.Marshal(checkParticipants) + if marshalErr != nil { + return nil, fmt.Errorf("encode check %s participants: %w", check.ID, marshalErr) + } + + environment, environmentErr := scriptEnvironment(check.Parameters) + if environmentErr != nil { + return nil, fmt.Errorf("check %s: %w", check.ID, environmentErr) + } + + environmentJSON, marshalErr := json.Marshal(environment) + if marshalErr != nil { + return nil, fmt.Errorf("encode check %s script environment: %w", check.ID, marshalErr) + } + + workloadPolicyJSON, encodeErr := signalworkload.Encode(check.WorkloadPolicy) + if encodeErr != nil { + return nil, fmt.Errorf("encode check %s workload policy: %w", check.ID, encodeErr) + } + + taskName := checkTaskName(check) + task := map[string]any{ + "name": taskName, + "templateRef": map[string]any{ + "name": run.Spec.Workflow.Name, + "template": "certification", + }, + "arguments": map[string]any{ + "parameters": checkWorkflowParameters( + run, + check, + attempt, + checkParticipants, + participantNames, + string(participantsJSON), + string(tolerationsJSON), + string(environmentJSON), + workloadPolicyJSON, + ), + }, + } + + if index > 0 { + previous := checkTaskName(&checks[index-1]) + if run.Spec.ExecutionPolicy.OnFailure == signalv1.SuiteFailurePolicyContinue { + task["depends"] = previous + ".Completed" + } else { + task["depends"] = previous + ".Succeeded" + task["when"] = fmt.Sprintf( + "{{tasks.%s.outputs.parameters.verdict}} == Pass", + previous, + ) + } + } + + tasks = append(tasks, task) + } + + workflow := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": workflowGVK.GroupVersion().String(), + "kind": workflowGVK.Kind, + "metadata": map[string]any{ + "name": name, + "namespace": run.Namespace, + "labels": map[string]any{ + SnapshotRunLabel: run.Name, + }, + }, + "spec": map[string]any{ + "activeDeadlineSeconds": int64(run.Spec.Deadline.Seconds()), + "ttlStrategy": map[string]any{ + "secondsAfterCompletion": int64(6 * time.Hour / time.Second), + "secondsAfterSuccess": int64(6 * time.Hour / time.Second), + "secondsAfterFailure": int64(6 * time.Hour / time.Second), + }, + "podGC": map[string]any{ + "strategy": "OnWorkflowCompletion", + "deleteDelayDuration": "2h", + }, + "entrypoint": "certification-suite", + "templates": []any{ + map[string]any{ + "name": "certification-suite", + "dag": map[string]any{ + "failFast": false, + "tasks": tasks, + }, + }, + }, + }, + }} + workflow.SetOwnerReferences([]metav1.OwnerReference{ + *metav1.NewControllerRef(run, signalv1.GroupVersion.WithKind("CertificationRun")), + }) + + return workflow, nil +} + +func buildLegacyWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured.Unstructured, error) { + checks := runChecks(run) + if len(checks) != 1 { + return nil, errors.New("legacy workflow requires exactly one check") + } + + check := &checks[0] + + participantNames := make([]string, 0, len(run.Spec.Participants)) + for _, participant := range run.Spec.Participants { + participantNames = append(participantNames, participant.NodeName) + } + + tolerations := uniqueTolerations(append([]corev1.Toleration{{ + Key: "nvidia.com/gpu", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }}, run.Spec.Tolerations...)) + + tolerationsJSON, err := json.Marshal(tolerations) + if err != nil { + return nil, fmt.Errorf("encode Workflow tolerations: %w", err) + } + + environment, err := scriptEnvironment(check.Parameters) if err != nil { return nil, err } - scriptEnvironmentJSON, err := json.Marshal(scriptEnvironment) + environmentJSON, err := json.Marshal(environment) if err != nil { return nil, fmt.Errorf("encode script environment: %w", err) } - workloadPolicyJSON, err := signalworkload.Encode(run.Spec.WorkloadPolicy) + workloadPolicyJSON, err := signalworkload.Encode(check.WorkloadPolicy) if err != nil { return nil, fmt.Errorf("encode workload policy: %w", err) } @@ -853,38 +1468,17 @@ func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured "name": run.Spec.Workflow.Name, }, "arguments": map[string]any{ - "parameters": []any{ - map[string]any{"name": "run-name", "value": run.Name}, - map[string]any{"name": "run-uid", "value": string(run.UID)}, - map[string]any{"name": "participants", "value": strings.Join(participantNames, ",")}, - map[string]any{"name": "participants-json", "value": string(participantsJSON)}, - map[string]any{"name": "participant-count", "value": fmt.Sprintf("%d", len(run.Spec.Participants))}, - map[string]any{"name": "request-node-name", "value": run.Spec.RequestNodeName}, - map[string]any{"name": "script-configmap", "value": run.Spec.Script.ConfigMapName}, - map[string]any{"name": "script-key", "value": run.Spec.Script.Key}, - map[string]any{"name": "script-digest", "value": run.Spec.Script.Digest}, - map[string]any{"name": "script-requires-runtime-stage", "value": fmt.Sprintf("%t", run.Spec.Script.RequiresRuntimeStage)}, - map[string]any{"name": "runtime-stage-name", "value": runtimeStageName(run.Spec.RuntimeStage)}, - map[string]any{"name": "backend-api-version", "value": run.Spec.Backend.APIVersion}, - map[string]any{"name": "backend-kind", "value": run.Spec.Backend.Kind}, - map[string]any{ - "name": "backend-active-deadline-seconds", - "value": fmt.Sprintf("%d", int64(run.Spec.BackendActiveDeadline.Seconds())), - }, - map[string]any{"name": "queue", "value": run.Spec.Backend.Queue}, - map[string]any{"name": "runner-image", "value": run.Spec.RunnerImage}, - map[string]any{"name": "tolerations", "value": string(tolerationsJSON)}, - map[string]any{"name": "script-environment", "value": string(scriptEnvironmentJSON)}, - map[string]any{"name": "workload-policy", "value": workloadPolicyJSON}, - map[string]any{"name": "mpi-payload-mode", "value": run.Spec.Parameters["mpi-payload-mode"]}, - map[string]any{"name": "expected-gpu-product-regex", "value": run.Spec.Parameters["expected-gpu-product-regex"]}, - map[string]any{"name": "expected-gpu-count", "value": run.Spec.Parameters["expected-gpu-count"]}, - map[string]any{"name": "observed-gpu-capacity", "value": run.Spec.Parameters["observed-gpu-capacity"]}, - map[string]any{"name": "observed-gpu-allocatable", "value": run.Spec.Parameters["observed-gpu-allocatable"]}, - map[string]any{"name": "gpu-resource-count", "value": run.Spec.Parameters["gpu-resource-count"]}, - map[string]any{"name": "privileged", "value": run.Spec.Parameters["privileged"]}, - map[string]any{"name": "attempt", "value": fmt.Sprintf("%d", attempt)}, - }, + "parameters": checkWorkflowParameters( + run, + check, + attempt, + run.Spec.Participants, + participantNames, + string(participantsJSON), + string(tolerationsJSON), + string(environmentJSON), + workloadPolicyJSON, + ), }, }, }} @@ -895,6 +1489,54 @@ func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured return workflow, nil } +func checkWorkflowParameters( + run *signalv1.CertificationRun, + check *signalv1.CertificationCheckSnapshot, + attempt int32, + participants []signalv1.Participant, + participantNames []string, + participantsJSON string, + tolerationsJSON string, + scriptEnvironmentJSON string, + workloadPolicyJSON string, +) []any { + return []any{ + map[string]any{"name": "run-name", "value": checkRunName(run.Name, check.ID)}, + map[string]any{"name": "run-uid", "value": string(run.UID)}, + map[string]any{"name": "participants", "value": strings.Join(participantNames, ",")}, + map[string]any{"name": "participants-json", "value": participantsJSON}, + map[string]any{"name": "participant-count", "value": fmt.Sprintf("%d", len(participants))}, + map[string]any{"name": "request-node-name", "value": run.Spec.RequestNodeName}, + map[string]any{"name": "script-configmap", "value": check.Script.ConfigMapName}, + map[string]any{"name": "script-key", "value": check.Script.Key}, + map[string]any{"name": "script-digest", "value": check.Script.Digest}, + map[string]any{ + "name": "script-requires-runtime-stage", + "value": fmt.Sprintf("%t", check.Script.RequiresRuntimeStage), + }, + map[string]any{"name": "runtime-stage-name", "value": runtimeStageName(check.RuntimeStage)}, + map[string]any{"name": "backend-api-version", "value": check.Backend.APIVersion}, + map[string]any{"name": "backend-kind", "value": check.Backend.Kind}, + map[string]any{ + "name": "backend-active-deadline-seconds", + "value": fmt.Sprintf("%d", int64(check.BackendActiveDeadline.Seconds())), + }, + map[string]any{"name": "queue", "value": check.Backend.Queue}, + map[string]any{"name": "runner-image", "value": check.RunnerImage}, + map[string]any{"name": "tolerations", "value": tolerationsJSON}, + map[string]any{"name": "script-environment", "value": scriptEnvironmentJSON}, + map[string]any{"name": "workload-policy", "value": workloadPolicyJSON}, + map[string]any{"name": "mpi-payload-mode", "value": check.Parameters["mpi-payload-mode"]}, + map[string]any{"name": "expected-gpu-product-regex", "value": check.Parameters["expected-gpu-product-regex"]}, + map[string]any{"name": "expected-gpu-count", "value": check.Parameters["expected-gpu-count"]}, + map[string]any{"name": "observed-gpu-capacity", "value": check.Parameters["observed-gpu-capacity"]}, + map[string]any{"name": "observed-gpu-allocatable", "value": check.Parameters["observed-gpu-allocatable"]}, + map[string]any{"name": "gpu-resource-count", "value": check.Parameters["gpu-resource-count"]}, + map[string]any{"name": "privileged", "value": check.Parameters["privileged"]}, + map[string]any{"name": "attempt", "value": fmt.Sprintf("%d", attempt)}, + } +} + func validEnvironmentName(value string) bool { if value == "" || (value[0] != '_' && (value[0] < 'A' || value[0] > 'Z')) { return false @@ -955,13 +1597,563 @@ func (r *RunReconciler) retryOrFinish( reason string, message string, ) (ctrl.Result, error) { - if run.Status.Workflow.Attempt < run.Spec.MaxAttempts { + maxAttempts := run.Spec.MaxAttempts + if checks := runChecks(run); len(checks) == 1 { + maxAttempts = checks[0].MaxAttempts + } + + if run.Status.Workflow.Attempt < maxAttempts { return r.startAttempt(ctx, run, run.Status.Workflow.Attempt+1) } return r.finish(ctx, run, nil, executionState, "Inconclusive", reason+": "+message) } +func (r *RunReconciler) finishGrouped( + ctx context.Context, + run *signalv1.CertificationRun, + workflow *unstructured.Unstructured, + workflowPhase string, +) (ctrl.Result, error) { + checks, verdict, reason := terminalCheckStatuses(workflow, run) + executionState := "Completed" + + if workflowPhase != "Succeeded" { + executionState = workflowPhase + verdict = "Inconclusive" + + if reason == "" { + reason = "Workflow" + workflowPhase + ": " + workflowMessage(workflow) + } + } + + evidenceRun := run.DeepCopy() + evidenceRun.Status.Checks = checks + evidenceRun.Status.ExecutionState = executionState + evidenceRun.Status.Verdict = verdict + evidenceRun.Status.Reason = reason + evidenceRun.Status.Workflow.Phase = workflowPhase + + finishedAt := groupedFinishedAt(workflow, run) + evidenceRun.Status.FinishedAt = &finishedAt + + evidence, err := r.evidenceDocument( + ctx, + evidenceRun, + executionState, + verdict, + reason, + ) + if err != nil { + return ctrl.Result{}, err + } + + uri, digest, err := r.Evidence.Put(ctx, run.Spec.EvidencePath, evidence) + if err != nil { + return ctrl.Result{}, err + } + + before := run.DeepCopy() + run.Status = evidenceRun.Status + run.Status.Checks = compactCheckStatuses(checks) + run.Status.Evidence = signalv1.EvidenceStatus{URI: uri, Digest: digest} + run.Status.Phase = runPhaseForVerdict(verdict) + + if err := r.Status().Patch(ctx, run, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, err + } + + if err := projectTerminal(ctx, r.Client, run); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{RequeueAfter: time.Millisecond}, nil +} + +func (r *RunReconciler) resumeGroupedEvidence( + ctx context.Context, + run *signalv1.CertificationRun, +) (ctrl.Result, bool, error) { + data, uri, digest, found, err := r.Evidence.Get(ctx, run.Spec.EvidencePath) + if err != nil || !found { + return ctrl.Result{}, false, err + } + + var document struct { + RunID string `json:"runID"` + RunName string `json:"runName"` + Workflow signalv1.WorkflowStatus `json:"workflow"` + CheckStatus []signalv1.CheckStatus `json:"checkStatus"` + ExecutionState string `json:"executionState"` + Verdict string `json:"verdict"` + Reason string `json:"reason"` + FinishedAt string `json:"finishedAt"` + } + if err := json.Unmarshal(data, &document); err != nil { + return ctrl.Result{}, false, fmt.Errorf("decode persisted grouped evidence: %w", err) + } + + if document.RunName != run.Name || + (document.RunID != "" && run.UID != "" && document.RunID != string(run.UID)) { + return ctrl.Result{}, false, fmt.Errorf("persisted grouped evidence identity does not match run %s", run.Name) + } + + if len(document.CheckStatus) != len(run.Spec.Checks) { + return ctrl.Result{}, false, fmt.Errorf( + "persisted grouped evidence contains %d check statuses, want %d", + len(document.CheckStatus), + len(run.Spec.Checks), + ) + } + + for index := range run.Spec.Checks { + if document.CheckStatus[index].ID != run.Spec.Checks[index].ID || + document.CheckStatus[index].Order != run.Spec.Checks[index].Order { + return ctrl.Result{}, false, fmt.Errorf( + "persisted grouped evidence check %d identity does not match run", + index, + ) + } + } + + finishedAt, err := time.Parse(time.RFC3339Nano, document.FinishedAt) + if err != nil { + return ctrl.Result{}, false, fmt.Errorf("parse persisted grouped evidence finishedAt: %w", err) + } + + before := run.DeepCopy() + run.Status.Checks = compactCheckStatuses(document.CheckStatus) + run.Status.ExecutionState = document.ExecutionState + run.Status.Verdict = document.Verdict + run.Status.Reason = document.Reason + run.Status.Workflow = document.Workflow + run.Status.Evidence = signalv1.EvidenceStatus{URI: uri, Digest: digest} + run.Status.Phase = runPhaseForVerdict(document.Verdict) + + value := metav1.NewTime(finishedAt) + run.Status.FinishedAt = &value + + if err := r.Status().Patch(ctx, run, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, true, err + } + + if err := projectTerminal(ctx, r.Client, run); err != nil { + return ctrl.Result{}, true, err + } + + return ctrl.Result{RequeueAfter: time.Millisecond}, true, nil +} + +func runPhaseForVerdict(verdict string) signalv1.RunPhase { + switch verdict { + case "Pass": + return signalv1.RunPhaseSucceeded + case "Fail": + return signalv1.RunPhaseFailed + default: + return signalv1.RunPhaseInconclusive + } +} + +func groupedFinishedAt( + workflow *unstructured.Unstructured, + run *signalv1.CertificationRun, +) metav1.Time { + if value, found, err := unstructured.NestedString(workflow.Object, "status", "finishedAt"); err == nil && found { + if parsed, parseErr := time.Parse(time.RFC3339, value); parseErr == nil { + return metav1.NewTime(parsed) + } + } + + nodes, found, err := unstructured.NestedMap(workflow.Object, "status", "nodes") + if err == nil && found { + var latest time.Time + + for index := range run.Spec.Checks { + node, count := exactCheckNode(nodes, workflow.GetName(), &run.Spec.Checks[index]) + if count != 1 { + continue + } + + parsed, parseErr := time.Parse(time.RFC3339, nodeString(node, "finishedAt")) + if parseErr == nil && parsed.After(latest) { + latest = parsed + } + } + + if !latest.IsZero() { + return metav1.NewTime(latest) + } + } + + if run.Status.StartedAt != nil { + return *run.Status.StartedAt.DeepCopy() + } + + if !run.CreationTimestamp.IsZero() { + return run.CreationTimestamp + } + + return metav1.NewTime(time.Unix(0, 0).UTC()) +} + +func initialCheckStatuses(checks []signalv1.CertificationCheckSnapshot) []signalv1.CheckStatus { + statuses := make([]signalv1.CheckStatus, len(checks)) + for index := range checks { + statuses[index] = signalv1.CheckStatus{ + ID: checks[index].ID, + Order: checks[index].Order, + Phase: signalv1.CheckPhasePending, + Attempt: 1, + RetryCount: 0, + } + } + + return statuses +} + +func compactCheckStatuses(statuses []signalv1.CheckStatus) []signalv1.CheckStatus { + compacted := make([]signalv1.CheckStatus, len(statuses)) + for index := range statuses { + compacted[index] = *statuses[index].DeepCopy() + compacted[index].BackendEvidence.Content = "" + } + + return compacted +} + +func contentDigest(content string) string { + sum := sha256.Sum256([]byte(content)) + + return fmt.Sprintf("sha256:%x", sum) +} + +func runningCheckStatuses( + workflow *unstructured.Unstructured, + run *signalv1.CertificationRun, +) []signalv1.CheckStatus { + statuses := initialCheckStatuses(run.Spec.Checks) + + nodes, found, err := unstructured.NestedMap(workflow.Object, "status", "nodes") + if err != nil || !found { + return statuses + } + + for index := range run.Spec.Checks { + node, count := exactCheckNode(nodes, workflow.GetName(), &run.Spec.Checks[index]) + if count != 1 { + continue + } + + statuses[index].WorkflowNodeName = nodeString(node, "name") + phase := nodeString(node, "phase") + + switch phase { + case "Running": + statuses[index].Phase = signalv1.CheckPhaseRunning + case "Succeeded": + statuses[index].Phase = signalv1.CheckPhaseSucceeded + case "Failed", "Error": + statuses[index].Phase = signalv1.CheckPhaseInconclusive + case "Omitted", "Skipped": + statuses[index].Phase = signalv1.CheckPhaseSkipped + statuses[index].NotRun = true + } + + setCheckTimes(&statuses[index], node) + } + + return statuses +} + +func terminalCheckStatuses( + workflow *unstructured.Unstructured, + run *signalv1.CertificationRun, +) ([]signalv1.CheckStatus, string, string) { + statuses := initialCheckStatuses(run.Spec.Checks) + + nodes, found, err := unstructured.NestedMap(workflow.Object, "status", "nodes") + if err != nil || !found { + for index := range statuses { + statuses[index].Phase = signalv1.CheckPhaseInconclusive + statuses[index].Verdict = "Inconclusive" + statuses[index].Reason = "WorkflowNodesUnavailable" + statuses[index].NotRun = true + } + + return statuses, "Inconclusive", "workflow task nodes are missing or malformed" + } + + aggregateVerdict := "Pass" + aggregateReason := "AllChecksPassed" + stoppedByPolicy := false + + for index := range run.Spec.Checks { + check := &run.Spec.Checks[index] + status := &statuses[index] + node, count := exactCheckNode(nodes, workflow.GetName(), check) + + if count != 1 { + status.Phase = signalv1.CheckPhaseInconclusive + status.Verdict = "Inconclusive" + status.Reason = "WorkflowTaskIdentityInvalid" + status.NotRun = true + aggregateVerdict = "Inconclusive" + aggregateReason = "workflow did not contain exactly one task node for every check" + + continue + } + + status.WorkflowNodeName = nodeString(node, "name") + setCheckTimes(status, node) + + phase := nodeString(node, "phase") + switch phase { + case "Omitted", "Skipped": + status.Phase = signalv1.CheckPhaseSkipped + status.Verdict = "Inconclusive" + status.Reason = "NotRun" + status.NotRun = true + + if run.Spec.ExecutionPolicy.OnFailure != signalv1.SuiteFailurePolicyStop || !stoppedByPolicy { + aggregateVerdict = "Inconclusive" + aggregateReason = "one or more checks were not run" + } + + continue + case "Failed", "Error": + status.Phase = signalv1.CheckPhaseInconclusive + status.Verdict = "Inconclusive" + status.Reason = "WorkflowTask" + phase + aggregateVerdict = "Inconclusive" + aggregateReason = "one or more check tasks failed at the workflow boundary" + stoppedByPolicy = true + + continue + case "Succeeded": + default: + status.Phase = signalv1.CheckPhaseInconclusive + status.Verdict = "Inconclusive" + status.Reason = "WorkflowTaskIncomplete" + status.NotRun = true + aggregateVerdict = "Inconclusive" + aggregateReason = "one or more check tasks were incomplete" + stoppedByPolicy = true + + continue + } + + outputs, outputErr := nodeWorkflowParameters(node, check.ResultBounds) + if outputErr != nil { + status.Phase = signalv1.CheckPhaseInconclusive + status.Verdict = "Inconclusive" + status.Reason = "OutputsRejected: " + outputErr.Error() + aggregateVerdict = "Inconclusive" + aggregateReason = "one or more check outputs were malformed or incomplete" + stoppedByPolicy = true + + continue + } + + status.Verdict = outputs["verdict"] + status.Reason = outputs["reason"] + status.Measurements = outputs["measurements"] + + _, globalIndexes, participantErr := participantsForCheck( + run.Spec.Participants, + check.ParticipantPolicy, + ) + if participantErr != nil { + status.Phase = signalv1.CheckPhaseInconclusive + status.Verdict = "Inconclusive" + status.Reason = "ParticipantPolicyInvalid: " + participantErr.Error() + aggregateVerdict = "Inconclusive" + aggregateReason = "one or more checks had invalid participant policy" + stoppedByPolicy = true + + continue + } + + normalizedEvidence, complete, evidenceErr := groupedBackendEvidenceForParticipants( + outputs["backend-evidence"], + globalIndexes, + ) + if evidenceErr != nil { + status.Phase = signalv1.CheckPhaseInconclusive + status.Verdict = "Inconclusive" + status.Reason = "BackendEvidenceRejected: " + evidenceErr.Error() + aggregateVerdict = "Inconclusive" + aggregateReason = "one or more checks had invalid backend evidence" + stoppedByPolicy = true + + continue + } + + status.BackendEvidence = signalv1.BackendEvidenceStatus{ + Version: "v1", + Complete: complete, + Digest: contentDigest(normalizedEvidence), + Content: normalizedEvidence, + } + + resources, resourceErr := backendResourceReferencesForParticipants( + outputs["backend-resources"], + globalIndexes, + ) + if resourceErr != nil { + status.Phase = signalv1.CheckPhaseInconclusive + status.Verdict = "Inconclusive" + status.Reason = "BackendResourcesRejected: " + resourceErr.Error() + aggregateVerdict = "Inconclusive" + aggregateReason = "one or more checks had invalid backend identities" + stoppedByPolicy = true + + continue + } + + status.BackendResources = resources + status.Cleanup.Complete = outputs["cleanup-failure"] == "" + status.Cleanup.Reason = outputs["cleanup-failure"] + + if !complete || !status.Cleanup.Complete || len(resources) == 0 { + status.Phase = signalv1.CheckPhaseInconclusive + status.Verdict = "Inconclusive" + status.Reason = "EvidenceOrCleanupIncomplete" + aggregateVerdict = "Inconclusive" + aggregateReason = "one or more checks lacked complete evidence, backend coverage, or cleanup" + stoppedByPolicy = true + + continue + } + + switch status.Verdict { + case "Pass": + status.Phase = signalv1.CheckPhaseSucceeded + case "Fail": + status.Phase = signalv1.CheckPhaseFailed + stoppedByPolicy = true + + if aggregateVerdict == "Pass" { + aggregateVerdict = "Fail" + aggregateReason = "OneOrMoreChecksFailed" + } + case "Inconclusive": + status.Phase = signalv1.CheckPhaseInconclusive + aggregateVerdict = "Inconclusive" + aggregateReason = "one or more checks were inconclusive" + stoppedByPolicy = true + default: + status.Phase = signalv1.CheckPhaseInconclusive + status.Verdict = "Inconclusive" + status.Reason = "InvalidVerdict" + aggregateVerdict = "Inconclusive" + aggregateReason = "one or more checks returned an invalid verdict" + stoppedByPolicy = true + } + } + + return statuses, aggregateVerdict, aggregateReason +} + +func exactCheckNode( + nodes map[string]any, + workflowName string, + check *signalv1.CertificationCheckSnapshot, +) (map[string]any, int) { + expected := workflowName + "." + checkTaskName(check) + + var match map[string]any + + count := 0 + + for _, raw := range nodes { + node, ok := raw.(map[string]any) + if !ok { + continue + } + + name, ok := node["name"].(string) + if ok && name == expected { + match = node + count++ + } + } + + return match, count +} + +func nodeWorkflowParameters( + node map[string]any, + bounds signalv1.ResultBounds, +) (map[string]string, error) { + outputs, ok := node["outputs"].(map[string]any) + if !ok { + return nil, errors.New("task outputs are missing or malformed") + } + + parameters, ok := outputs["parameters"].([]any) + if !ok { + return nil, errors.New("task output parameters are missing or malformed") + } + + limits := map[string]int{ + "verdict": workflowOutputLimits["verdict"], + "reason": int(bounds.ReasonBytes), + "measurements": int(bounds.MeasurementsBytes), + "backend-resources": int(bounds.BackendResourcesBytes), + "backend-evidence": int(bounds.BackendEvidenceBytes), + "cleanup-failure": workflowOutputLimits["cleanup-failure"], + } + + decoded, err := decodeWorkflowParametersWithLimits(parameters, limits) + if err != nil { + return nil, err + } + + for _, required := range []string{ + "verdict", + "reason", + "measurements", + "backend-resources", + "backend-evidence", + "cleanup-failure", + } { + if _, exists := decoded[required]; !exists { + return nil, fmt.Errorf("required parameter %q is missing", required) + } + } + + return decoded, nil +} + +func setCheckTimes(status *signalv1.CheckStatus, node map[string]any) { + started := nodeString(node, "startedAt") + finished := nodeString(node, "finishedAt") + + if parsed, err := time.Parse(time.RFC3339, started); err == nil { + value := metav1.NewTime(parsed) + status.StartedAt = &value + } + + if parsed, err := time.Parse(time.RFC3339, finished); err == nil { + value := metav1.NewTime(parsed) + status.FinishedAt = &value + } + + if status.StartedAt != nil && status.FinishedAt != nil { + status.Duration = metav1.Duration{Duration: status.FinishedAt.Sub(status.StartedAt.Time)} + } +} + +func nodeString(node map[string]any, key string) string { + value, ok := node[key].(string) + if !ok { + return "" + } + + return value +} + func (r *RunReconciler) finish( ctx context.Context, run *signalv1.CertificationRun, @@ -1100,34 +2292,94 @@ func (r *RunReconciler) evidenceDocument( verdict string, reason string, ) ([]byte, error) { + if !isGroupedRun(run) { + checks := runChecks(run) + if len(checks) != 1 { + return nil, errors.New("legacy run must contain one check") + } + + check := &checks[0] + + var snapshot corev1.ConfigMap + if err := r.Get( + ctx, + client.ObjectKey{Namespace: run.Namespace, Name: check.Script.ConfigMapName}, + &snapshot, + ); err != nil { + return nil, fmt.Errorf("read script snapshot for evidence: %w", err) + } + + doc := map[string]any{ + "apiVersion": "signal.unbounded-cloud.io/evidence/v1alpha1", + "runID": string(run.UID), + "runName": run.Name, + "profile": run.Spec.Profile, + "participants": run.Spec.Participants, + "workflow": run.Status.Workflow, + "workflowOutputs": run.Status.Workflow.Outputs, + "backend": check.Backend, + "runnerImage": check.RunnerImage, + "workloadPolicy": check.WorkloadPolicy, + "purpose": run.Spec.Purpose, + "stagePlan": run.Spec.StagePlan, + "runtimeStage": check.RuntimeStage, + "backendActiveDeadline": check.BackendActiveDeadline, + "parameters": check.Parameters, + "scriptSnapshot": check.Script, + "scriptContent": snapshot.Data[check.Script.Key], + "executionState": executionState, + "verdict": verdict, + "reason": reason, + "startedAt": run.Status.StartedAt, + "finishedAt": run.Status.FinishedAt.UTC().Format(time.RFC3339Nano), + } + + return json.MarshalIndent(doc, "", " ") + } + + checks := runChecks(run) + if len(checks) == 0 { + return nil, errors.New("run has no certification checks") + } + var snapshot corev1.ConfigMap - if err := r.Get(ctx, client.ObjectKey{Namespace: run.Namespace, Name: run.Spec.Script.ConfigMapName}, &snapshot); err != nil { + if err := r.Get( + ctx, + client.ObjectKey{Namespace: run.Namespace, Name: checks[0].Script.ConfigMapName}, + &snapshot, + ); err != nil { return nil, fmt.Errorf("read script snapshot for evidence: %w", err) } + scriptContents := make(map[string]string, len(checks)) + for index := range checks { + content, exists := snapshot.Data[checks[index].Script.Key] + if !exists { + return nil, fmt.Errorf("script snapshot is missing check %s key %s", checks[index].ID, checks[index].Script.Key) + } + + scriptContents[string(checks[index].ID)] = content + } + doc := map[string]any{ - "apiVersion": "signal.unbounded-cloud.io/evidence/v1alpha1", - "runID": string(run.UID), - "runName": run.Name, - "profile": run.Spec.Profile, - "participants": run.Spec.Participants, - "workflow": run.Status.Workflow, - "workflowOutputs": run.Status.Workflow.Outputs, - "backend": run.Spec.Backend, - "runnerImage": run.Spec.RunnerImage, - "workloadPolicy": run.Spec.WorkloadPolicy, - "purpose": run.Spec.Purpose, - "stagePlan": run.Spec.StagePlan, - "runtimeStage": run.Spec.RuntimeStage, - "backendActiveDeadline": run.Spec.BackendActiveDeadline, - "parameters": run.Spec.Parameters, - "scriptSnapshot": run.Spec.Script, - "scriptContent": snapshot.Data[run.Spec.Script.Key], - "executionState": executionState, - "verdict": verdict, - "reason": reason, - "startedAt": run.Status.StartedAt, - "finishedAt": run.Status.FinishedAt.UTC().Format(time.RFC3339Nano), + "apiVersion": "signal.unbounded-cloud.io/evidence/v1alpha1", + "runID": string(run.UID), + "runName": run.Name, + "profile": run.Spec.Profile, + "participants": run.Spec.Participants, + "participantRoles": run.Spec.ParticipantRoles, + "topologyConstraints": run.Spec.TopologyConstraints, + "workflow": run.Status.Workflow, + "checks": checks, + "checkStatus": run.Status.Checks, + "scriptContents": scriptContents, + "executionPolicy": run.Spec.ExecutionPolicy, + "purpose": run.Spec.Purpose, + "executionState": executionState, + "verdict": verdict, + "reason": reason, + "startedAt": run.Status.StartedAt, + "finishedAt": run.Status.FinishedAt.UTC().Format(time.RFC3339Nano), } return json.MarshalIndent(doc, "", " ") @@ -1211,11 +2463,30 @@ func backendResourceReferences( } } - if participantCount > 1 { - for index, isCovered := range covered { - if !isCovered { - return nil, fmt.Errorf("backend-resources does not cover participant index %d", index) - } + if participantCount > 1 { + for index, isCovered := range covered { + if !isCovered { + return nil, fmt.Errorf("backend-resources does not cover participant index %d", index) + } + } + } + + return references, nil +} + +func backendResourceReferencesForParticipants( + value string, + globalIndexes []int32, +) ([]signalv1.BackendResourceReference, error) { + references, err := backendResourceReferences(value, len(globalIndexes)) + if err != nil { + return nil, err + } + + for referenceIndex := range references { + for participantIndex := range references[referenceIndex].ParticipantIndexes { + localIndex := references[referenceIndex].ParticipantIndexes[participantIndex] + references[referenceIndex].ParticipantIndexes[participantIndex] = globalIndexes[localIndex] } } @@ -1236,6 +2507,10 @@ func workflowParameters(workflow *unstructured.Unstructured) (map[string]string, } func decodeWorkflowParameters(parameters []any) (map[string]string, error) { + return decodeWorkflowParametersWithLimits(parameters, workflowOutputLimits) +} + +func decodeWorkflowParametersWithLimits(parameters []any, limits map[string]int) (map[string]string, error) { outputs := make(map[string]string, len(parameters)) for _, raw := range parameters { parameter, ok := raw.(map[string]any) @@ -1250,7 +2525,7 @@ func decodeWorkflowParameters(parameters []any) (map[string]string, error) { return nil, fmt.Errorf("parameter name and value must be strings") } - limit, allowed := workflowOutputLimits[name] + limit, allowed := limits[name] if !allowed { return nil, fmt.Errorf("unexpected parameter %q", name) } @@ -1393,6 +2668,134 @@ func backendEvidenceComplete(content string, participantCount int) (bool, error) return evidence.Complete, nil } +func groupedBackendEvidenceComplete(content string, participantCount int) (bool, error) { + if content == "" { + return false, errors.New("backend-evidence is required") + } + + var evidence struct { + Version string `json:"version"` + Complete bool `json:"complete"` + CaptureError string `json:"captureError,omitempty"` + Participants []struct { + ParticipantIndex int `json:"participantIndex"` + Resource struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Namespace string `json:"namespace,omitempty"` + Name string `json:"name"` + UID types.UID `json:"uid"` + } `json:"resource"` + LauncherJob any `json:"launcherJob,omitempty"` + Pods []any `json:"pods"` + CaptureError string `json:"captureError,omitempty"` + } `json:"participants"` + } + + decoder := json.NewDecoder(strings.NewReader(content)) + if err := decoder.Decode(&evidence); err != nil { + return false, fmt.Errorf("decode backend-evidence: %w", err) + } + + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + return false, errors.New("backend-evidence contains trailing JSON values") + } + + if evidence.Version != "v1" { + return false, fmt.Errorf("backend-evidence version %q is invalid", evidence.Version) + } + + if len(evidence.Participants) != participantCount { + return false, fmt.Errorf( + "backend-evidence contains %d participants, want %d", + len(evidence.Participants), + participantCount, + ) + } + + seen := make([]bool, participantCount) + + for index := range evidence.Participants { + participant := &evidence.Participants[index] + if participant.ParticipantIndex < 0 || participant.ParticipantIndex >= participantCount { + return false, fmt.Errorf( + "backend-evidence participant index %d is out of range", + participant.ParticipantIndex, + ) + } + + if seen[participant.ParticipantIndex] { + return false, fmt.Errorf( + "backend-evidence repeats participant index %d", + participant.ParticipantIndex, + ) + } + + seen[participant.ParticipantIndex] = true + + if participant.Resource.APIVersion == "" || + participant.Resource.Kind == "" || + participant.Resource.Name == "" || + participant.Resource.UID == "" { + return false, fmt.Errorf( + "backend-evidence participant %d has incomplete resource identity", + participant.ParticipantIndex, + ) + } + } + + return evidence.Complete, nil +} + +func groupedBackendEvidenceForParticipants( + content string, + globalIndexes []int32, +) (string, bool, error) { + complete, err := groupedBackendEvidenceComplete(content, len(globalIndexes)) + if err != nil { + return "", false, err + } + + var evidence map[string]json.RawMessage + if err := json.Unmarshal([]byte(content), &evidence); err != nil { + return "", false, fmt.Errorf("decode backend-evidence for participant mapping: %w", err) + } + + var participants []map[string]json.RawMessage + if err := json.Unmarshal(evidence["participants"], &participants); err != nil { + return "", false, fmt.Errorf("decode backend-evidence participants for mapping: %w", err) + } + + for index := range participants { + var localIndex int + if err := json.Unmarshal(participants[index]["participantIndex"], &localIndex); err != nil { + return "", false, fmt.Errorf("decode backend-evidence participant index: %w", err) + } + + mapped, err := json.Marshal(globalIndexes[localIndex]) + if err != nil { + return "", false, fmt.Errorf("encode mapped backend-evidence participant index: %w", err) + } + + participants[index]["participantIndex"] = mapped + } + + mappedParticipants, err := json.Marshal(participants) + if err != nil { + return "", false, fmt.Errorf("encode mapped backend-evidence participants: %w", err) + } + + evidence["participants"] = mappedParticipants + + normalized, err := json.Marshal(evidence) + if err != nil { + return "", false, fmt.Errorf("encode mapped backend-evidence: %w", err) + } + + return string(normalized), complete, nil +} + func (r *RunReconciler) cancel(ctx context.Context, run *signalv1.CertificationRun) (ctrl.Result, error) { stopped, err := r.ensureWorkflowStopped(ctx, run) if err != nil { @@ -1407,7 +2810,30 @@ func (r *RunReconciler) cancel(ctx context.Context, run *signalv1.CertificationR run.Status.Phase = signalv1.RunPhaseCancelled run.Status.ExecutionState = "Interrupted" run.Status.Verdict = "Inconclusive" + run.Status.Reason = "canceled by operator" + if isGroupedRun(run) { + statuses := run.Status.Checks + if len(statuses) != len(run.Spec.Checks) { + statuses = initialCheckStatuses(run.Spec.Checks) + } + + for index := range statuses { + if statuses[index].Phase == signalv1.CheckPhaseSucceeded || + statuses[index].Phase == signalv1.CheckPhaseFailed || + statuses[index].Phase == signalv1.CheckPhaseInconclusive { + continue + } + + statuses[index].Phase = signalv1.CheckPhaseSkipped + statuses[index].Verdict = "Inconclusive" + statuses[index].Reason = "Canceled" + statuses[index].NotRun = true + } + + run.Status.Checks = statuses + } + now := metav1.NewTime(r.now()) run.Status.FinishedAt = &now @@ -1430,7 +2856,14 @@ func (r *RunReconciler) cleanup(ctx context.Context, run *signalv1.Certification deleted := false - snapshot := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: run.Namespace, Name: run.Spec.Script.ConfigMapName}} + checks := runChecks(run) + if len(checks) == 0 { + return ctrl.Result{}, errors.New("run has no certification checks") + } + + snapshot := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: run.Namespace, Name: checks[0].Script.ConfigMapName}, + } if err := r.Delete(ctx, snapshot); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err } @@ -1713,6 +3146,397 @@ func validateStagePlan(plan []signalv1.CertificationStage) error { return nil } +func profileChecks(profile *signalv1.CertificationProfile) []signalv1.CertificationCheck { + if len(profile.Spec.Checks) > 0 { + checks := make([]signalv1.CertificationCheck, len(profile.Spec.Checks)) + for index := range profile.Spec.Checks { + checks[index] = profile.Spec.Checks[index] + checks[index].WorkloadPolicy = *profile.Spec.Checks[index].WorkloadPolicy.DeepCopy() + checks[index].RuntimeStage = copyRuntimeStage(profile.Spec.Checks[index].RuntimeStage) + checks[index].Parameters = maps.Clone(profile.Spec.Checks[index].Parameters) + checks[index].ParticipantPolicy.RequiredRoles = slices.Clone( + profile.Spec.Checks[index].ParticipantPolicy.RequiredRoles, + ) + checks[index].ParticipantPolicy.ExcludedRoles = slices.Clone( + profile.Spec.Checks[index].ParticipantPolicy.ExcludedRoles, + ) + } + + return checks + } + + maxAttempts := profile.Spec.MaxAttempts + if maxAttempts < 1 { + maxAttempts = DefaultMaxAttempts + } + + return []signalv1.CertificationCheck{{ + ID: "legacy", + Order: 0, + Script: profile.Spec.Script, + RunnerImage: profile.Spec.RunnerImage, + Backend: profile.Spec.Backend, + WorkloadPolicy: *profile.Spec.WorkloadPolicy.DeepCopy(), + RuntimeStage: copyRuntimeStage(profile.Spec.RuntimeStage), + Parameters: maps.Clone(profile.Spec.Parameters), + BackendActiveDeadline: profile.Spec.BackendActiveDeadline, + ResultBounds: signalv1.ResultBounds{ + ReasonBytes: int32(workflowOutputLimits["reason"]), + MeasurementsBytes: int32(workflowOutputLimits["measurements"]), + BackendResourcesBytes: int32(workflowOutputLimits["backend-resources"]), + BackendEvidenceBytes: 64 * 1024, + }, + MaxAttempts: maxAttempts, + }} +} + +func runChecks(run *signalv1.CertificationRun) []signalv1.CertificationCheckSnapshot { + if len(run.Spec.Checks) > 0 { + return run.Spec.Checks + } + + return []signalv1.CertificationCheckSnapshot{{ + ID: "legacy", + Order: 0, + Script: signalv1.ScriptSnapshot{ + ConfigMapName: run.Spec.Script.ConfigMapName, + Key: run.Spec.Script.Key, + RequiresRuntimeStage: run.Spec.Script.RequiresRuntimeStage, + Digest: run.Spec.Script.Digest, + }, + RunnerImage: run.Spec.RunnerImage, + Backend: run.Spec.Backend, + WorkloadPolicy: run.Spec.WorkloadPolicy, + RuntimeStage: run.Spec.RuntimeStage, + Parameters: run.Spec.Parameters, + BackendActiveDeadline: run.Spec.BackendActiveDeadline, + ResultBounds: signalv1.ResultBounds{ + ReasonBytes: int32(workflowOutputLimits["reason"]), + MeasurementsBytes: int32(workflowOutputLimits["measurements"]), + BackendResourcesBytes: int32(workflowOutputLimits["backend-resources"]), + BackendEvidenceBytes: 64 * 1024, + }, + MaxAttempts: run.Spec.MaxAttempts, + }} +} + +func isGroupedRun(run *signalv1.CertificationRun) bool { + return len(run.Spec.Checks) == groupedCheckCount +} + +func validateCheckPlan(checks []signalv1.CertificationCheck, grouped bool) error { + if grouped && len(checks) != groupedCheckCount { + return fmt.Errorf("must contain exactly %d checks", groupedCheckCount) + } + + seen := make(map[signalv1.CheckID]struct{}, len(checks)) + for index := range checks { + check := &checks[index] + if check.ID == "" { + return fmt.Errorf("check %d has an empty ID", index) + } + + if check.Order != int32(index) { + return fmt.Errorf("check %s order is %d, want %d", check.ID, check.Order, index) + } + + if grouped && string(check.ID) != fmt.Sprintf("R%02d", index+1) { + return fmt.Errorf("check order %d must use ID R%02d", index, index+1) + } + + if _, duplicate := seen[check.ID]; duplicate { + return fmt.Errorf("duplicate check ID %q", check.ID) + } + + seen[check.ID] = struct{}{} + + if grouped && check.MaxAttempts != 1 { + return fmt.Errorf("check %s maxAttempts must be exactly 1", check.ID) + } + + if err := validateResultBounds(check.ID, check.ResultBounds); err != nil { + return err + } + + if err := validateCheckParticipantPolicy(check.ID, check.ParticipantPolicy); err != nil { + return err + } + } + + return nil +} + +func validateRunCheckPlan(checks []signalv1.CertificationCheckSnapshot, grouped bool) error { + if grouped && len(checks) != groupedCheckCount { + return fmt.Errorf("run must contain exactly %d checks", groupedCheckCount) + } + + seen := make(map[signalv1.CheckID]struct{}, len(checks)) + for index := range checks { + check := &checks[index] + if check.ID == "" || check.Order != int32(index) { + return fmt.Errorf("run check %d has invalid identity or order", index) + } + + if grouped && string(check.ID) != fmt.Sprintf("R%02d", index+1) { + return fmt.Errorf("run check order %d must use ID R%02d", index, index+1) + } + + if _, duplicate := seen[check.ID]; duplicate { + return fmt.Errorf("run contains duplicate check ID %q", check.ID) + } + + seen[check.ID] = struct{}{} + + if grouped && check.MaxAttempts != 1 { + return fmt.Errorf("run check %s maxAttempts must be exactly 1", check.ID) + } + + if err := validateResultBounds(check.ID, check.ResultBounds); err != nil { + return err + } + } + + return nil +} + +func validateResultBounds(id signalv1.CheckID, bounds signalv1.ResultBounds) error { + switch { + case bounds.ReasonBytes < 1 || bounds.ReasonBytes > int32(workflowOutputLimits["reason"]): + return fmt.Errorf("check %s reasonBytes must be between 1 and %d", id, workflowOutputLimits["reason"]) + case bounds.MeasurementsBytes < 1 || bounds.MeasurementsBytes > int32(workflowOutputLimits["measurements"]): + return fmt.Errorf( + "check %s measurementsBytes must be between 1 and %d", + id, + workflowOutputLimits["measurements"], + ) + case bounds.BackendResourcesBytes < 1 || + bounds.BackendResourcesBytes > int32(workflowOutputLimits["backend-resources"]): + return fmt.Errorf( + "check %s backendResourcesBytes must be between 1 and %d", + id, + workflowOutputLimits["backend-resources"], + ) + case bounds.BackendEvidenceBytes < 1 || bounds.BackendEvidenceBytes > 64*1024: + return fmt.Errorf("check %s backendEvidenceBytes must be between 1 and 65536", id) + } + + return nil +} + +func validateCheckParticipantPolicy(id signalv1.CheckID, policy signalv1.CheckParticipantPolicy) error { + required := make(map[string]struct{}, len(policy.RequiredRoles)) + for _, role := range policy.RequiredRoles { + if role == "" { + return fmt.Errorf("check %s contains an empty required role", id) + } + + if _, duplicate := required[role]; duplicate { + return fmt.Errorf("check %s repeats required role %q", id, role) + } + + required[role] = struct{}{} + } + + excluded := make(map[string]struct{}, len(policy.ExcludedRoles)) + for _, role := range policy.ExcludedRoles { + if role == "" { + return fmt.Errorf("check %s contains an empty excluded role", id) + } + + if _, duplicate := excluded[role]; duplicate { + return fmt.Errorf("check %s repeats excluded role %q", id, role) + } + + if _, conflict := required[role]; conflict { + return fmt.Errorf("check %s both requires and excludes role %q", id, role) + } + + excluded[role] = struct{}{} + } + + return nil +} + +func validateParticipantConstraints( + profile *signalv1.CertificationProfile, + checks []signalv1.CertificationCheck, +) error { + roles := make(map[string]struct{}, len(profile.Spec.ParticipantRoles)) + + var assigned int32 + + for index := range profile.Spec.ParticipantRoles { + role := &profile.Spec.ParticipantRoles[index] + if role.Name == "" || role.Count < 1 { + return fmt.Errorf("participant role %d has invalid name or count", index) + } + + if _, duplicate := roles[role.Name]; duplicate { + return fmt.Errorf("duplicate participant role %q", role.Name) + } + + if _, err := metav1.LabelSelectorAsSelector(&role.Selector); err != nil { + return fmt.Errorf("participant role %s selector: %w", role.Name, err) + } + + roles[role.Name] = struct{}{} + assigned += role.Count + } + + if assigned > profile.Spec.Participants.Count { + return fmt.Errorf( + "participant roles require %d assignments but participant count is %d", + assigned, + profile.Spec.Participants.Count, + ) + } + + for index := range profile.Spec.TopologyConstraints { + constraint := &profile.Spec.TopologyConstraints[index] + if _, exists := roles[constraint.Role]; !exists { + return fmt.Errorf("topology constraint references unknown role %q", constraint.Role) + } + + if constraint.LabelKey == "" || constraint.MinDistinctValues < 1 { + return fmt.Errorf("topology constraint %d has invalid label key or minimum", index) + } + } + + for index := range checks { + for _, role := range append( + slices.Clone(checks[index].ParticipantPolicy.RequiredRoles), + checks[index].ParticipantPolicy.ExcludedRoles..., + ) { + if _, exists := roles[role]; !exists { + return fmt.Errorf("check %s references unknown participant role %q", checks[index].ID, role) + } + } + } + + return nil +} + +func participantsForCheck( + participants []signalv1.Participant, + policy signalv1.CheckParticipantPolicy, +) ([]signalv1.Participant, []int32, error) { + required := make(map[string]bool, len(policy.RequiredRoles)) + for _, role := range policy.RequiredRoles { + required[role] = false + } + + excluded := make(map[string]struct{}, len(policy.ExcludedRoles)) + for _, role := range policy.ExcludedRoles { + excluded[role] = struct{}{} + } + + selected := make([]signalv1.Participant, 0, len(participants)) + + globalIndexes := make([]int32, 0, len(participants)) + for index := range participants { + participant := participants[index] + include := len(required) == 0 + + for _, role := range participant.Roles { + if _, blocked := excluded[role]; blocked { + include = false + + break + } + + if _, wanted := required[role]; wanted { + required[role] = true + include = true + } + } + + if !include { + continue + } + + globalIndexes = append(globalIndexes, participant.Index) + participant.Index = int32(len(selected)) + selected = append(selected, participant) + } + + if len(selected) == 0 { + return nil, nil, errors.New("participant policy selected no participants") + } + + for role, present := range required { + if !present { + return nil, nil, fmt.Errorf("required role %q has no selected participant", role) + } + } + + return selected, globalIndexes, nil +} + +func setLegacyRunAliases(spec *signalv1.CertificationRunSpec) { + if len(spec.Checks) == 0 { + return + } + + first := &spec.Checks[0] + spec.Script = signalv1.LegacyScriptSnapshot{ + ConfigMapName: first.Script.ConfigMapName, + Key: first.Script.Key, + RequiresRuntimeStage: first.Script.RequiresRuntimeStage, + Digest: first.Script.Digest, + } + spec.Backend = first.Backend + spec.RunnerImage = first.RunnerImage + spec.WorkloadPolicy = first.WorkloadPolicy + spec.RuntimeStage = first.RuntimeStage + spec.Parameters = first.Parameters + spec.BackendActiveDeadline = first.BackendActiveDeadline + spec.MaxAttempts = first.MaxAttempts +} + +func sanitizeTaskName(value string) string { + value = strings.ToLower(value) + value = strings.Map(func(character rune) rune { + if character >= 'a' && character <= 'z' || + character >= '0' && character <= '9' || + character == '-' { + return character + } + + return '-' + }, value) + + value = strings.Trim(value, "-") + if value == "" { + value = "check" + } + + if len(value) > 48 { + value = strings.TrimRight(value[:48], "-") + } + + return value +} + +func checkTaskName(check *signalv1.CertificationCheckSnapshot) string { + return fmt.Sprintf("check-%02d-%s", check.Order, sanitizeTaskName(string(check.ID))) +} + +func checkRunName(runName string, checkID signalv1.CheckID) string { + suffix := sanitizeTaskName(string(checkID)) + + candidate := runName + "-" + suffix + if len(candidate) <= 63 { + return candidate + } + + sum := sha256.Sum256([]byte(candidate)) + hash := hex.EncodeToString(sum[:4]) + prefixLength := 63 - len(hash) - 1 + + return strings.TrimRight(candidate[:prefixLength], "-") + "-" + hash +} + func copyRuntimeStage(stage *signalv1.RuntimeStage) *signalv1.RuntimeStage { if stage == nil { return nil diff --git a/internal/signal/controller_test.go b/internal/signal/controller_test.go index 96f42c299e..151297704e 100644 --- a/internal/signal/controller_test.go +++ b/internal/signal/controller_test.go @@ -35,10 +35,10 @@ func TestAdmissionCreatesImmutableSnapshotAndPreventsDuplicate(t *testing.T) { node := readyNode("gpu-0", "example-gpu") node.Labels[RequestLabel] = "gpu-smoke" profile := testProfile("gpu-smoke", "example-gpu") - profile.Spec.Script.RequiresRuntimeStage = true - profile.Spec.RuntimeStage = &signalv1.RuntimeStage{Name: "collective-bandwidth"} - profile.Spec.Backend.Queue = "certification" - profile.Spec.WorkloadPolicy = signalv1.WorkloadPolicy{ + profile.Spec.Checks[0].Script.RequiresRuntimeStage = true + profile.Spec.Checks[0].RuntimeStage = &signalv1.RuntimeStage{Name: "collective-bandwidth"} + profile.Spec.Checks[0].Backend.Queue = "certification" + profile.Spec.Checks[0].WorkloadPolicy = signalv1.WorkloadPolicy{ Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("4")}, Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("8")}, @@ -82,38 +82,36 @@ func TestAdmissionCreatesImmutableSnapshotAndPreventsDuplicate(t *testing.T) { } run := runs.Items[0] - if run.Spec.Script.Digest != digestString(payload) { - t.Fatalf("script digest = %q", run.Spec.Script.Digest) - } - if len(run.Spec.StagePlan) != 1 || run.Spec.StagePlan[0].ID != "workload" { - t.Fatalf("stage plan = %#v, want immutable profile plan", run.Spec.StagePlan) + check := run.Spec.Checks[0] + if check.Script.Digest != digestString(payload) { + t.Fatalf("script digest = %q", check.Script.Digest) } - if !run.Spec.Script.RequiresRuntimeStage || - run.Spec.RuntimeStage == nil || - run.Spec.RuntimeStage.Name != "collective-bandwidth" || - run.Spec.BackendActiveDeadline.Duration != 9*time.Minute { + if !check.Script.RequiresRuntimeStage || + check.RuntimeStage == nil || + check.RuntimeStage.Name != "collective-bandwidth" || + check.BackendActiveDeadline.Duration != 9*time.Minute { t.Fatalf("runtime contract = script %#v stage %#v backend deadline %s", - run.Spec.Script, run.Spec.RuntimeStage, run.Spec.BackendActiveDeadline.Duration) + check.Script, check.RuntimeStage, check.BackendActiveDeadline.Duration) } if run.Spec.Purpose != signalv1.PurposeQuickCertification { t.Fatalf("purpose = %q, want %q", run.Spec.Purpose, signalv1.PurposeQuickCertification) } - if !run.Spec.WorkloadPolicy.HostNetwork || - run.Spec.WorkloadPolicy.KueueTopology == nil || - run.Spec.WorkloadPolicy.KueueTopology.RequiredLevel != corev1.LabelHostname || - run.Spec.WorkloadPolicy.Resources.Limits.Cpu().String() != "8" || - len(run.Spec.WorkloadPolicy.Volumes) != 1 || - run.Spec.WorkloadPolicy.Volumes[0].EmptyDir.SizeLimit.String() != "8Gi" { - t.Fatalf("workload policy = %#v, want immutable profile policy", run.Spec.WorkloadPolicy) + if !check.WorkloadPolicy.HostNetwork || + check.WorkloadPolicy.KueueTopology == nil || + check.WorkloadPolicy.KueueTopology.RequiredLevel != corev1.LabelHostname || + check.WorkloadPolicy.Resources.Limits.Cpu().String() != "8" || + len(check.WorkloadPolicy.Volumes) != 1 || + check.WorkloadPolicy.Volumes[0].EmptyDir.SizeLimit.String() != "8Gi" { + t.Fatalf("workload policy = %#v, want immutable profile policy", check.WorkloadPolicy) } - if run.Spec.Parameters["observed-gpu-capacity"] != "8" || - run.Spec.Parameters["observed-gpu-allocatable"] != "8" { - t.Fatalf("observed GPU inventory = %#v", run.Spec.Parameters) + if check.Parameters["observed-gpu-capacity"] != "8" || + check.Parameters["observed-gpu-allocatable"] != "8" { + t.Fatalf("observed GPU inventory = %#v", check.Parameters) } if run.Annotations[RunTriggerAnnotation] != "manual" { @@ -121,7 +119,7 @@ func TestAdmissionCreatesImmutableSnapshotAndPreventsDuplicate(t *testing.T) { } var snapshot corev1.ConfigMap - if err := kube.Get(context.Background(), client.ObjectKey{Namespace: "signal", Name: run.Spec.Script.ConfigMapName}, &snapshot); err != nil { + if err := kube.Get(context.Background(), client.ObjectKey{Namespace: "signal", Name: check.Script.ConfigMapName}, &snapshot); err != nil { t.Fatalf("get snapshot: %v", err) } @@ -150,7 +148,7 @@ func TestAdmissionRejectsMissingRequiredRuntimeStageBeforeReservation(t *testing node := readyNode("gpu-0", "example-gpu") node.Labels[RequestLabel] = "gpu-smoke" profile := testProfile("gpu-smoke", "example-gpu") - profile.Spec.Script.RequiresRuntimeStage = true + profile.Spec.Checks[0].Script.RequiresRuntimeStage = true source := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, Data: map[string]string{"payload.txt": "test payload\n"}, @@ -183,7 +181,7 @@ func TestAdmissionRejectsInsufficientBackendDeadlineReserveBeforeReservation(t * node := readyNode("gpu-0", "example-gpu") node.Labels[RequestLabel] = "gpu-smoke" profile := testProfile("gpu-smoke", "example-gpu") - profile.Spec.BackendActiveDeadline = metav1.Duration{ + profile.Spec.Checks[0].BackendActiveDeadline = metav1.Duration{ Duration: profile.Spec.Deadline.Duration - 59*time.Second, } source := &corev1.ConfigMap{ @@ -218,7 +216,7 @@ func TestAdmissionRejectsMutableRunnerImageBeforeReservation(t *testing.T) { node := readyNode("gpu-0", "example-gpu") node.Labels[RequestLabel] = "gpu-smoke" profile := testProfile("gpu-smoke", "example-gpu") - profile.Spec.RunnerImage = "registry.example/runner:latest" + profile.Spec.Checks[0].RunnerImage = "registry.example/runner:latest" source := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, Data: map[string]string{"payload.txt": "test payload\n"}, @@ -252,7 +250,7 @@ func TestAdmissionRejectsInvalidWorkloadPolicyBeforeReservation(t *testing.T) { node := readyNode("gpu-0", "example-gpu") node.Labels[RequestLabel] = "gpu-smoke" profile := testProfile("gpu-smoke", "example-gpu") - profile.Spec.WorkloadPolicy.Volumes = []signalv1.WorkloadVolume{{ + profile.Spec.Checks[0].WorkloadPolicy.Volumes = []signalv1.WorkloadVolume{{ Name: "replacement", MountPath: "/results", EmptyDir: &corev1.EmptyDirVolumeSource{}, @@ -289,8 +287,8 @@ func TestAdmissionRejectsKueueTopologyWithoutQueueBeforeReservation(t *testing.T node := readyNode("gpu-0", "example-gpu") node.Labels[RequestLabel] = "gpu-smoke" profile := testProfile("gpu-smoke", "example-gpu") - profile.Spec.Backend = signalv1.ExecutionBackend{APIVersion: "v1", Kind: "Pod"} - profile.Spec.WorkloadPolicy.KueueTopology = &signalv1.KueueTopologyPolicy{ + profile.Spec.Checks[0].Backend = signalv1.ExecutionBackend{APIVersion: "v1", Kind: "Pod"} + profile.Spec.Checks[0].WorkloadPolicy.KueueTopology = &signalv1.KueueTopologyPolicy{ RequiredLevel: corev1.LabelHostname, } source := &corev1.ConfigMap{ @@ -720,19 +718,19 @@ func TestWorkflowPinsAttemptInputs(t *testing.T) { run := testRun("run-1") run.UID = types.UID("run-uid") - run.Spec.Parameters = map[string]string{ + run.Spec.Checks[0].Parameters = map[string]string{ "expected-gpu-count": "8", "health-mode": "snapshot", "mpi-payload-mode": "script", } - run.Spec.Script.RequiresRuntimeStage = true - run.Spec.RuntimeStage = &signalv1.RuntimeStage{Name: "collective-bandwidth"} + run.Spec.Checks[0].Script.RequiresRuntimeStage = true + run.Spec.Checks[0].RuntimeStage = &signalv1.RuntimeStage{Name: "collective-bandwidth"} run.Spec.Tolerations = []corev1.Toleration{{ Key: "nvidia.com/gpu", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule, }} - run.Spec.WorkloadPolicy = signalv1.WorkloadPolicy{ + run.Spec.Checks[0].WorkloadPolicy = signalv1.WorkloadPolicy{ Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("4")}, Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("8")}, @@ -785,7 +783,9 @@ func TestWorkflowPinsAttemptInputs(t *testing.T) { values[parameter["name"].(string)] = parameter["value"].(string) } - if values["script-digest"] != run.Spec.Script.Digest || values["backend-kind"] != "MPIJob" || values["attempt"] != "2" { + if values["script-digest"] != run.Spec.Checks[0].Script.Digest || + values["backend-kind"] != "MPIJob" || + values["attempt"] != "2" { t.Fatalf("workflow values = %#v", values) } @@ -810,7 +810,8 @@ func TestWorkflowPinsAttemptInputs(t *testing.T) { t.Fatalf("workflow participants = %#v", participants) } - if values["script-configmap"] != run.Spec.Script.ConfigMapName || values["script-configmap"] == "scripts" { + if values["script-configmap"] != run.Spec.Checks[0].Script.ConfigMapName || + values["script-configmap"] == "scripts" { t.Fatalf("workflow must mount the immutable snapshot, values = %#v", values) } @@ -858,7 +859,7 @@ func TestBuildWorkflowRejectsParameterThatCannotBecomeEnvironmentVariable(t *tes t.Parallel() run := testRun("run-invalid-parameter") - run.Spec.Parameters = map[string]string{"invalid.parameter": "value"} + run.Spec.Checks[0].Parameters = map[string]string{"invalid.parameter": "value"} if _, err := buildWorkflow(run, 1); err == nil || !strings.Contains(err.Error(), "cannot be mapped to a script environment variable") { @@ -1030,8 +1031,8 @@ func TestMultiNodePassWithoutBackendIdentityBecomesInconclusive(t *testing.T) { StartedAt: ptrTime(metav1.NewTime(time.Unix(100, 0))), } snapshot := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Script.ConfigMapName, Namespace: run.Namespace}, - Data: map[string]string{run.Spec.Script.Key: "test payload\n"}, + ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Checks[0].Script.ConfigMapName, Namespace: run.Namespace}, + Data: map[string]string{run.Spec.Checks[0].Script.Key: "test payload\n"}, } workflow := workflowObject(run, "Succeeded", "Pass", "Healthy") store := &memoryEvidenceStore{} @@ -1073,8 +1074,8 @@ func TestMultiNodePassRetainsCoveredBackendIdentities(t *testing.T) { StartedAt: ptrTime(metav1.NewTime(time.Unix(100, 0))), } snapshot := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Script.ConfigMapName, Namespace: run.Namespace}, - Data: map[string]string{run.Spec.Script.Key: "test payload\n"}, + ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Checks[0].Script.ConfigMapName, Namespace: run.Namespace}, + Data: map[string]string{run.Spec.Checks[0].Script.Key: "test payload\n"}, } workflow := workflowObject(run, "Succeeded", "Pass", "Healthy") @@ -1282,8 +1283,8 @@ func TestTerminalVerdictUsesEntrypointOutputsWhenTopLevelAbsent(t *testing.T) { StartedAt: ptrTime(metav1.NewTime(time.Unix(100, 0))), } snapshot := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Script.ConfigMapName, Namespace: run.Namespace}, - Data: map[string]string{run.Spec.Script.Key: "test payload\n"}, + ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Checks[0].Script.ConfigMapName, Namespace: run.Namespace}, + Data: map[string]string{run.Spec.Checks[0].Script.Key: "test payload\n"}, } workflow := workflowObject(run, "Succeeded", "Pass", "Healthy") @@ -1354,9 +1355,9 @@ func TestTerminalVerdictPersistsEvidenceAndCleanupReleasesNode(t *testing.T) { scheme := testScheme(t) run := testRun("run-pass") - run.Spec.RuntimeStage = &signalv1.RuntimeStage{Name: "collective-bandwidth"} - run.Spec.Backend.Queue = "certification" - run.Spec.WorkloadPolicy.KueueTopology = &signalv1.KueueTopologyPolicy{ + run.Spec.Checks[0].RuntimeStage = &signalv1.RuntimeStage{Name: "collective-bandwidth"} + run.Spec.Checks[0].Backend.Queue = "certification" + run.Spec.Checks[0].WorkloadPolicy.KueueTopology = &signalv1.KueueTopologyPolicy{ RequiredLevel: corev1.LabelHostname, } run.Status = signalv1.CertificationRunStatus{ @@ -1368,8 +1369,8 @@ func TestTerminalVerdictPersistsEvidenceAndCleanupReleasesNode(t *testing.T) { node.Labels[RequestLabel] = run.Spec.Profile.Name node.Annotations = map[string]string{ReservationLabel: run.Name} snapshot := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Script.ConfigMapName, Namespace: run.Namespace}, - Data: map[string]string{run.Spec.Script.Key: "test payload\n"}, + ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Checks[0].Script.ConfigMapName, Namespace: run.Namespace}, + Data: map[string]string{run.Spec.Checks[0].Script.Key: "test payload\n"}, } workflow := workflowObject(run, "Succeeded", "Pass", "Healthy") store := &memoryEvidenceStore{} @@ -1427,12 +1428,16 @@ func TestTerminalVerdictPersistsEvidenceAndCleanupReleasesNode(t *testing.T) { t.Fatalf("evidence workflow = %#v", evidence.Workflow) } - if !reflect.DeepEqual(evidence.WorkloadPolicy, run.Spec.WorkloadPolicy) { - t.Fatalf("evidence workload policy = %#v, want %#v", evidence.WorkloadPolicy, run.Spec.WorkloadPolicy) + if !reflect.DeepEqual(evidence.WorkloadPolicy, run.Spec.Checks[0].WorkloadPolicy) { + t.Fatalf( + "evidence workload policy = %#v, want %#v", + evidence.WorkloadPolicy, + run.Spec.Checks[0].WorkloadPolicy, + ) } - if !reflect.DeepEqual(evidence.RuntimeStage, run.Spec.RuntimeStage) || - evidence.BackendActiveDeadline != run.Spec.BackendActiveDeadline { + if !reflect.DeepEqual(evidence.RuntimeStage, run.Spec.Checks[0].RuntimeStage) || + evidence.BackendActiveDeadline != run.Spec.Checks[0].BackendActiveDeadline { t.Fatalf("evidence runtime contract = stage %#v deadline %s", evidence.RuntimeStage, evidence.BackendActiveDeadline.Duration) } @@ -1474,8 +1479,8 @@ func TestTerminalFinalizationRetriesWithIdenticalEvidence(t *testing.T) { node := readyNode("gpu-0", "A100") node.Annotations = map[string]string{ReservationLabel: run.Name} snapshot := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Script.ConfigMapName, Namespace: run.Namespace}, - Data: map[string]string{run.Spec.Script.Key: "test payload\n"}, + ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Checks[0].Script.ConfigMapName, Namespace: run.Namespace}, + Data: map[string]string{run.Spec.Checks[0].Script.Key: "test payload\n"}, } workflow := workflowObject(run, "Succeeded", "Pass", "Healthy") statusPatches := 0 @@ -1639,7 +1644,7 @@ func TestWorkflowFailureRetriesWithoutSwitchingPinnedInputs(t *testing.T) { scheme := testScheme(t) run := testRun("run-retry") - run.Spec.MaxAttempts = 2 + run.Spec.Checks[0].MaxAttempts = 2 run.Status = signalv1.CertificationRunStatus{ Phase: signalv1.RunPhaseRunning, Workflow: signalv1.WorkflowStatus{Name: "run-retry-a1", Attempt: 1}, @@ -1672,11 +1677,11 @@ func TestWorkflowFailureRetriesWithoutSwitchingPinnedInputs(t *testing.T) { t.Fatalf("get retry workflow: %v", err) } - if value := workflowArgument(&retry, "script-digest"); value != run.Spec.Script.Digest { + if value := workflowArgument(&retry, "script-digest"); value != run.Spec.Checks[0].Script.Digest { t.Fatalf("retry script digest = %q", value) } - if value := workflowArgument(&retry, "backend-kind"); value != run.Spec.Backend.Kind { + if value := workflowArgument(&retry, "backend-kind"); value != run.Spec.Checks[0].Backend.Kind { t.Fatalf("retry backend = %q", value) } } @@ -1692,8 +1697,8 @@ func TestCanceledRunCleansUpDespiteCancelFlag(t *testing.T) { node.Labels[RequestLabel] = run.Spec.Profile.Name node.Annotations = map[string]string{ReservationLabel: run.Name} snapshot := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Script.ConfigMapName, Namespace: run.Namespace}, - Data: map[string]string{run.Spec.Script.Key: "test payload\n"}, + ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Checks[0].Script.ConfigMapName, Namespace: run.Namespace}, + Data: map[string]string{run.Spec.Checks[0].Script.Key: "test payload\n"}, } kube := fake.NewClientBuilder(). WithScheme(scheme). @@ -1726,6 +1731,14 @@ func (s *memoryEvidenceStore) Put(_ context.Context, path string, data []byte) ( return "memory://" + path, digestString(string(data)), nil } +func (s *memoryEvidenceStore) Get(_ context.Context, path string) ([]byte, string, string, bool, error) { + if s.data == nil { + return nil, "", "", false, nil + } + + return append([]byte(nil), s.data...), "memory://" + path, digestString(string(s.data)), true, nil +} + type immutableMemoryEvidenceStore struct { data []byte puts int @@ -1742,6 +1755,14 @@ func (s *immutableMemoryEvidenceStore) Put(_ context.Context, path string, data return "memory://" + path, digestString(string(data)), nil } +func (s *immutableMemoryEvidenceStore) Get(_ context.Context, path string) ([]byte, string, string, bool, error) { + if s.data == nil { + return nil, "", "", false, nil + } + + return append([]byte(nil), s.data...), "memory://" + path, digestString(string(s.data)), true, nil +} + func testScheme(t *testing.T) *runtime.Scheme { t.Helper() @@ -1778,7 +1799,7 @@ func readyNode(name, sku string) *corev1.Node { } func testProfile(name, sku string) *signalv1.CertificationProfile { - return &signalv1.CertificationProfile{ + profile := &signalv1.CertificationProfile{ ObjectMeta: metav1.ObjectMeta{Name: name, UID: types.UID(name + "-uid"), Generation: 3}, Spec: signalv1.CertificationProfileSpec{ Participants: signalv1.ParticipantPolicy{ @@ -1788,7 +1809,11 @@ func testProfile(name, sku string) *signalv1.CertificationProfile { }}, RequiredResources: corev1.ResourceList{corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("1")}, }, - Script: signalv1.ScriptReference{Name: "scripts", Key: "payload.txt"}, + Script: signalv1.ScriptReference{ + Name: "scripts", + Key: "payload.txt", + Digest: digestString("test payload\n"), + }, Workflow: signalv1.WorkflowReference{Name: "gpu-cert"}, Backend: signalv1.ExecutionBackend{APIVersion: "kubeflow.org/v2beta1", Kind: "MPIJob"}, RunnerImage: "runner@sha256:" + strings.Repeat("a", 64), @@ -1800,10 +1825,31 @@ func testProfile(name, sku string) *signalv1.CertificationProfile { MaxAttempts: 2, }, } + + profile.Spec.Checks = []signalv1.CertificationCheck{{ + ID: "legacy", + Order: 0, + Script: profile.Spec.Script, + RunnerImage: profile.Spec.RunnerImage, + Backend: profile.Spec.Backend, + WorkloadPolicy: profile.Spec.WorkloadPolicy, + RuntimeStage: profile.Spec.RuntimeStage, + Parameters: profile.Spec.Parameters, + BackendActiveDeadline: profile.Spec.BackendActiveDeadline, + ResultBounds: signalv1.ResultBounds{ + ReasonBytes: 4096, + MeasurementsBytes: 16384, + BackendResourcesBytes: 16384, + BackendEvidenceBytes: 65536, + }, + MaxAttempts: profile.Spec.MaxAttempts, + }} + + return profile } func testRun(name string) *signalv1.CertificationRun { - return &signalv1.CertificationRun{ + run := &signalv1.CertificationRun{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: "signal", @@ -1813,7 +1859,7 @@ func testRun(name string) *signalv1.CertificationRun { Spec: signalv1.CertificationRunSpec{ Profile: signalv1.PinnedProfile{Name: "gpu-profile", UID: "profile-uid", Generation: 3}, Participants: []signalv1.Participant{{NodeName: "gpu-0", NodeUID: "gpu-0-uid"}}, - Script: signalv1.ScriptSnapshot{ConfigMapName: name + "-scripts", Key: "payload.txt", Digest: digestString("test payload\n")}, + Script: signalv1.LegacyScriptSnapshot{ConfigMapName: name + "-scripts", Key: "payload.txt", Digest: digestString("test payload\n")}, Workflow: signalv1.WorkflowReference{Name: "gpu-cert"}, Backend: signalv1.ExecutionBackend{APIVersion: "kubeflow.org/v2beta1", Kind: "MPIJob"}, RunnerImage: "runner@sha256:" + strings.Repeat("a", 64), @@ -1824,6 +1870,27 @@ func testRun(name string) *signalv1.CertificationRun { RequestNodeName: "gpu-0", }, } + + run.Spec.Checks = []signalv1.CertificationCheckSnapshot{{ + ID: "legacy", + Order: 0, + Script: signalv1.ScriptSnapshot(run.Spec.Script), + RunnerImage: run.Spec.RunnerImage, + Backend: run.Spec.Backend, + WorkloadPolicy: run.Spec.WorkloadPolicy, + RuntimeStage: run.Spec.RuntimeStage, + Parameters: run.Spec.Parameters, + BackendActiveDeadline: run.Spec.BackendActiveDeadline, + ResultBounds: signalv1.ResultBounds{ + ReasonBytes: 4096, + MeasurementsBytes: 16384, + BackendResourcesBytes: 16384, + BackendEvidenceBytes: 65536, + }, + MaxAttempts: run.Spec.MaxAttempts, + }} + + return run } func workflowObject(run *signalv1.CertificationRun, phase, verdict, reason string) *unstructured.Unstructured { diff --git a/internal/signal/evidence.go b/internal/signal/evidence.go index 6a6524002b..c70c75798c 100644 --- a/internal/signal/evidence.go +++ b/internal/signal/evidence.go @@ -7,8 +7,12 @@ import ( "bytes" "compress/gzip" "context" + "encoding/json" "fmt" + "io" + "maps" "net/url" + "strconv" "strings" corev1 "k8s.io/api/core/v1" @@ -19,6 +23,7 @@ import ( type EvidenceStore interface { Put(context.Context, string, []byte) (string, string, error) + Get(context.Context, string) ([]byte, string, string, bool, error) } type KubernetesEvidenceStore struct { @@ -26,6 +31,14 @@ type KubernetesEvidenceStore struct { namespace string } +const maximumEvidenceChunkBytes = 700 * 1024 + +type evidenceChunkManifest struct { + Version string `json:"version"` + Chunks int `json:"chunks"` + CompressedDigest string `json:"compressedDigest"` +} + func NewKubernetesEvidenceStore(kube client.Client, namespace string) (*KubernetesEvidenceStore, error) { if kube == nil { return nil, fmt.Errorf("kubernetes client is required") @@ -39,7 +52,7 @@ func NewKubernetesEvidenceStore(kube client.Client, namespace string) (*Kubernet } func (s *KubernetesEvidenceStore) Put(ctx context.Context, path string, data []byte) (string, string, error) { - name := strings.TrimSuffix(url.PathEscape(strings.TrimSuffix(path[strings.LastIndex(path, "/")+1:], ".json")), ".json") + name := evidenceName(path) if name == "" { return "", "", fmt.Errorf("evidence path is required") } @@ -64,28 +77,187 @@ func (s *KubernetesEvidenceStore) Put(ctx context.Context, path string, data []b "signal.unbounded-cloud.io/evidence-digest": digest, }, }, - Immutable: &immutable, - BinaryData: map[string][]byte{ - "evidence.json.gz": compressed, - }, + Immutable: &immutable, + BinaryData: map[string][]byte{}, + } + uriKey := "evidence.json.gz" + + if len(compressed) <= maximumEvidenceChunkBytes { + evidence.BinaryData["evidence.json.gz"] = compressed + } else { + uriKey = "manifest.json" + + chunkCount := (len(compressed) + maximumEvidenceChunkBytes - 1) / maximumEvidenceChunkBytes + for index := range chunkCount { + start := index * maximumEvidenceChunkBytes + end := min(start+maximumEvidenceChunkBytes, len(compressed)) + + chunk := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-evidence-%03d", name, index), + Namespace: s.namespace, + Labels: map[string]string{ + SnapshotRunLabel: name, + }, + Annotations: map[string]string{ + "signal.unbounded-cloud.io/evidence-digest": digest, + "signal.unbounded-cloud.io/chunk-index": strconv.Itoa(index), + }, + }, + Immutable: &immutable, + BinaryData: map[string][]byte{ + "chunk": compressed[start:end], + }, + } + if err := s.createOrVerifyEvidenceConfigMap(ctx, chunk); err != nil { + return "", "", err + } + } + + manifest, err := json.Marshal(evidenceChunkManifest{ + Version: "v1", + Chunks: chunkCount, + CompressedDigest: digestBytes(compressed), + }) + if err != nil { + return "", "", fmt.Errorf("encode evidence chunk manifest: %w", err) + } + + evidence.BinaryData["manifest.json"] = manifest + } + + if err := s.createOrVerifyEvidenceConfigMap(ctx, evidence); err != nil { + return "", "", err + } + + return evidenceURI(s.namespace, evidence.Name, uriKey), digest, nil +} + +func (s *KubernetesEvidenceStore) Get( + ctx context.Context, + path string, +) ([]byte, string, string, bool, error) { + name := evidenceName(path) + if name == "" { + return nil, "", "", false, fmt.Errorf("evidence path is required") + } + + var evidence corev1.ConfigMap + if err := s.kube.Get( + ctx, + client.ObjectKey{Namespace: s.namespace, Name: name + "-evidence"}, + &evidence, + ); err != nil { + if apierrors.IsNotFound(err) { + return nil, "", "", false, nil + } + + return nil, "", "", false, fmt.Errorf("get evidence ConfigMap: %w", err) + } + + if evidence.Immutable == nil || !*evidence.Immutable { + return nil, "", "", false, fmt.Errorf("evidence ConfigMap %s is mutable", evidence.Name) + } + + compressed, exists := evidence.BinaryData["evidence.json.gz"] + + uriKey := "evidence.json.gz" + if !exists { + uriKey = "manifest.json" + + var manifest evidenceChunkManifest + if err := json.Unmarshal(evidence.BinaryData["manifest.json"], &manifest); err != nil { + return nil, "", "", false, fmt.Errorf("decode evidence ConfigMap %s chunk manifest: %w", evidence.Name, err) + } + + if manifest.Version != "v1" || manifest.Chunks < 1 { + return nil, "", "", false, fmt.Errorf("evidence ConfigMap %s has an invalid chunk manifest", evidence.Name) + } + + compressed = make([]byte, 0, manifest.Chunks*maximumEvidenceChunkBytes) + for index := range manifest.Chunks { + var chunk corev1.ConfigMap + if err := s.kube.Get( + ctx, + client.ObjectKey{ + Namespace: s.namespace, + Name: fmt.Sprintf("%s-%03d", evidence.Name, index), + }, + &chunk, + ); err != nil { + return nil, "", "", false, fmt.Errorf("get evidence chunk %d: %w", index, err) + } + + if chunk.Immutable == nil || !*chunk.Immutable { + return nil, "", "", false, fmt.Errorf("evidence chunk %s is mutable", chunk.Name) + } + + if chunk.Annotations["signal.unbounded-cloud.io/evidence-digest"] != + evidence.Annotations["signal.unbounded-cloud.io/evidence-digest"] || + chunk.Annotations["signal.unbounded-cloud.io/chunk-index"] != strconv.Itoa(index) { + return nil, "", "", false, fmt.Errorf("evidence chunk %s identity does not match its manifest", chunk.Name) + } + + compressed = append(compressed, chunk.BinaryData["chunk"]...) + } + + if digestBytes(compressed) != manifest.CompressedDigest { + return nil, "", "", false, fmt.Errorf("evidence ConfigMap %s chunks do not match their manifest", evidence.Name) + } } + + data, err := decompressEvidence(compressed) + if err != nil { + return nil, "", "", false, fmt.Errorf("decompress evidence ConfigMap %s: %w", evidence.Name, err) + } + + digest := digestBytes(data) + if evidence.Annotations["signal.unbounded-cloud.io/evidence-digest"] != digest { + return nil, "", "", false, fmt.Errorf("evidence ConfigMap %s digest does not match its content", evidence.Name) + } + + uri := evidenceURI(s.namespace, evidence.Name, uriKey) + + return data, uri, digest, true, nil +} + +func evidenceURI(namespace, name, key string) string { + return fmt.Sprintf("kubernetes://%s/configmaps/%s/%s", namespace, name, key) +} + +func (s *KubernetesEvidenceStore) createOrVerifyEvidenceConfigMap( + ctx context.Context, + evidence *corev1.ConfigMap, +) error { if err := s.kube.Create(ctx, evidence); err != nil { if !apierrors.IsAlreadyExists(err) { - return "", "", fmt.Errorf("create evidence ConfigMap: %w", err) + return fmt.Errorf("create evidence ConfigMap %s: %w", evidence.Name, err) } var existing corev1.ConfigMap if getErr := s.kube.Get(ctx, client.ObjectKeyFromObject(evidence), &existing); getErr != nil { - return "", "", fmt.Errorf("get existing evidence ConfigMap: %w", getErr) + return fmt.Errorf("get existing evidence ConfigMap %s: %w", evidence.Name, getErr) + } + + if existing.Immutable == nil || !*existing.Immutable { + return fmt.Errorf("evidence ConfigMap %s already exists but is mutable", evidence.Name) } - if existing.Annotations["signal.unbounded-cloud.io/evidence-digest"] != digest || - !bytes.Equal(existing.BinaryData["evidence.json.gz"], compressed) { - return "", "", fmt.Errorf("evidence ConfigMap %s already exists with different content", evidence.Name) + if !maps.Equal(existing.Labels, evidence.Labels) || + !maps.Equal(existing.Annotations, evidence.Annotations) || + !maps.EqualFunc(existing.BinaryData, evidence.BinaryData, bytes.Equal) { + return fmt.Errorf("evidence ConfigMap %s already exists with different content", evidence.Name) } } - return fmt.Sprintf("kubernetes://%s/configmaps/%s/evidence.json.gz", s.namespace, evidence.Name), digest, nil + return nil +} + +func evidenceName(path string) string { + return strings.TrimSuffix( + url.PathEscape(strings.TrimSuffix(path[strings.LastIndex(path, "/")+1:], ".json")), + ".json", + ) } func compressEvidence(data []byte) ([]byte, error) { @@ -102,3 +274,23 @@ func compressEvidence(data []byte) ([]byte, error) { return compressed.Bytes(), nil } + +func decompressEvidence(compressed []byte) ([]byte, error) { + reader, err := gzip.NewReader(bytes.NewReader(compressed)) + if err != nil { + return nil, err + } + + data, readErr := io.ReadAll(reader) + closeErr := reader.Close() + + if readErr != nil { + return nil, readErr + } + + if closeErr != nil { + return nil, closeErr + } + + return data, nil +} diff --git a/internal/signal/evidence_test.go b/internal/signal/evidence_test.go index 8c7915f986..9f79009eca 100644 --- a/internal/signal/evidence_test.go +++ b/internal/signal/evidence_test.go @@ -74,4 +74,90 @@ func TestKubernetesEvidenceStoreReturnsQueryableRunURIAndDigest(t *testing.T) { if string(got) != string(data) { t.Fatalf("persisted evidence = %q", got) } + + recovered, recoveredURI, recoveredDigest, found, err := store.Get( + context.Background(), + "project-signal/example/run-a.json", + ) + if err != nil { + t.Fatal(err) + } + + if !found || + recoveredURI != uri || + recoveredDigest != digest || + !bytes.Equal(recovered, data) { + t.Fatalf( + "recovered evidence = found:%t uri:%q digest:%q data:%q", + found, + recoveredURI, + recoveredDigest, + recovered, + ) + } +} + +func TestKubernetesEvidenceStoreChunksLargeEvidence(t *testing.T) { + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + kube := fake.NewClientBuilder().WithScheme(scheme).Build() + + store, err := NewKubernetesEvidenceStore(kube, "project-signal") + if err != nil { + t.Fatal(err) + } + + data := make([]byte, 2*maximumEvidenceChunkBytes) + + state := uint32(1) + for index := range data { + state = state*1664525 + 1013904223 + data[index] = byte(state >> 24) + } + + uri, digest, err := store.Put(context.Background(), "project-signal/example/run-large.json", data) + if err != nil { + t.Fatal(err) + } + + if uri != "kubernetes://project-signal/configmaps/run-large-evidence/manifest.json" { + t.Fatalf("unexpected chunked URI %q", uri) + } + + var manifest corev1.ConfigMap + if err := kube.Get( + context.Background(), + client.ObjectKey{Namespace: "project-signal", Name: "run-large-evidence"}, + &manifest, + ); err != nil { + t.Fatal(err) + } + + if len(manifest.BinaryData["manifest.json"]) == 0 { + t.Fatal("large evidence did not use a chunk manifest") + } + + recovered, recoveredURI, recoveredDigest, found, err := store.Get( + context.Background(), + "project-signal/example/run-large.json", + ) + if err != nil { + t.Fatal(err) + } + + if !found || + recoveredURI != uri || + recoveredDigest != digest || + !bytes.Equal(recovered, data) { + t.Fatalf( + "recovered large evidence = found:%t uri:%q digest:%q bytes:%d", + found, + recoveredURI, + recoveredDigest, + len(recovered), + ) + } } diff --git a/internal/signal/grouped_controller_test.go b/internal/signal/grouped_controller_test.go new file mode 100644 index 0000000000..4b93e802e1 --- /dev/null +++ b/internal/signal/grouped_controller_test.go @@ -0,0 +1,854 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signal + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" +) + +func TestGroupedAdmissionSnapshotsCompletePlanOnce(t *testing.T) { + t.Parallel() + + node := readyNode("gpu-0", "example-gpu") + node.Labels[RequestLabel] = "grouped-profile" + profile := testProfile("grouped-profile", "example-gpu") + profile.Spec.Checks = groupedProfileChecks() + profile.Spec.ExecutionPolicy = signalv1.SuiteExecutionPolicy{OnFailure: signalv1.SuiteFailurePolicyStop} + profile.Spec.Script = signalv1.ScriptReference{} + profile.Spec.Backend = signalv1.ExecutionBackend{} + profile.Spec.RunnerImage = "" + profile.Spec.WorkloadPolicy = signalv1.WorkloadPolicy{} + profile.Spec.StagePlan = nil + profile.Spec.Parameters = nil + profile.Spec.BackendActiveDeadline = metav1.Duration{} + profile.Spec.MaxAttempts = 0 + + source := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, + Data: map[string]string{}, + } + for index := range profile.Spec.Checks { + source.Data[profile.Spec.Checks[index].Script.Key] = fmt.Sprintf("check-%d\n", index) + } + + scheme := testScheme(t) + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node, profile, source).Build() + reconciler := &AdmissionReconciler{ + Client: kube, + Namespace: "signal", + ScriptNamespace: "signal", + Now: func() time.Time { return time.Unix(100, 0) }, + } + + if _, err := reconciler.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}, + ); err != nil { + t.Fatal(err) + } + + var runs signalv1.CertificationRunList + if err := kube.List(context.Background(), &runs); err != nil { + t.Fatal(err) + } + + if len(runs.Items) != 1 || len(runs.Items[0].Spec.Checks) != groupedCheckCount { + t.Fatalf("runs = %#v", runs.Items) + } + + run := &runs.Items[0] + + var snapshot corev1.ConfigMap + if err := kube.Get( + context.Background(), + client.ObjectKey{Namespace: run.Namespace, Name: run.Spec.Checks[0].Script.ConfigMapName}, + &snapshot, + ); err != nil { + t.Fatal(err) + } + + if snapshot.Immutable == nil || !*snapshot.Immutable || len(snapshot.Data) != groupedCheckCount { + t.Fatalf("snapshot = immutable:%v keys:%d", snapshot.Immutable, len(snapshot.Data)) + } + + for index := range run.Spec.Checks { + check := &run.Spec.Checks[index] + if check.Order != int32(index) || + check.Script.Digest != digestString(fmt.Sprintf("check-%d\n", index)) || + snapshot.Data[check.Script.Key] != fmt.Sprintf("check-%d\n", index) { + t.Fatalf("check %d = %#v", index, check) + } + } +} + +func TestGroupedAdmissionRejectsScriptDigestMismatch(t *testing.T) { + t.Parallel() + + node := readyNode("gpu-0", "example-gpu") + node.Labels[RequestLabel] = "grouped-profile" + profile := testProfile("grouped-profile", "example-gpu") + profile.Spec.Checks = groupedProfileChecks() + profile.Spec.ExecutionPolicy = signalv1.SuiteExecutionPolicy{OnFailure: signalv1.SuiteFailurePolicyStop} + + source := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, + Data: map[string]string{}, + } + for index := range profile.Spec.Checks { + source.Data[profile.Spec.Checks[index].Script.Key] = fmt.Sprintf("check-%d\n", index) + } + + source.Data[profile.Spec.Checks[8].Script.Key] = "changed\n" + + kube := fake.NewClientBuilder().WithScheme(testScheme(t)).WithObjects(node, profile, source).Build() + reconciler := &AdmissionReconciler{ + Client: kube, + Namespace: "signal", + ScriptNamespace: "signal", + } + + _, err := reconciler.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}, + ) + if err == nil || !strings.Contains(err.Error(), "source script digest for check R09") { + t.Fatalf("Reconcile error = %v", err) + } + + var got corev1.Node + if getErr := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &got); getErr != nil { + t.Fatal(getErr) + } + + if got.Annotations[ReservationLabel] != "" { + t.Fatalf("reservation remained after digest rejection: %#v", got.Annotations) + } +} + +func TestParticipantRoleAndTopologyConstraints(t *testing.T) { + t.Parallel() + + service := readyNode("service", "example-gpu") + service.Labels["signal.example/role"] = "service" + service.Labels["topology.example/zone"] = "zone-a" + witnessA := readyNode("witness-a", "example-gpu") + witnessA.Labels["signal.example/role"] = "witness" + witnessA.Labels["topology.example/zone"] = "zone-a" + witnessB := readyNode("witness-b", "example-gpu") + witnessB.Labels["signal.example/role"] = "witness" + witnessB.Labels["topology.example/zone"] = "zone-b" + witnessC := readyNode("witness-c", "example-gpu") + witnessC.Labels["signal.example/role"] = "witness" + witnessC.Labels["topology.example/zone"] = "zone-b" + + participants := []signalv1.Participant{ + participantSnapshot(service, 0), + participantSnapshot(witnessA, 1), + participantSnapshot(witnessB, 2), + participantSnapshot(witnessC, 3), + } + kube := fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithObjects(service, witnessA, witnessB, witnessC). + Build() + reconciler := &AdmissionReconciler{Client: kube} + + err := reconciler.assignParticipantRoles( + context.Background(), + participants, + []signalv1.ParticipantRoleConstraint{ + { + Name: "service", + Count: 1, + Selector: metav1.LabelSelector{MatchLabels: map[string]string{"signal.example/role": "service"}}, + }, + { + Name: "witness", + Count: 3, + Selector: metav1.LabelSelector{MatchLabels: map[string]string{"signal.example/role": "witness"}}, + }, + }, + []signalv1.ParticipantTopologyConstraint{{ + Role: "witness", + LabelKey: "topology.example/zone", + MinDistinctValues: 2, + }}, + ) + if err != nil { + t.Fatal(err) + } + + if !slices.Equal(participants[0].Roles, []string{"service"}) { + t.Fatalf("service roles = %#v", participants[0].Roles) + } + + for index := 1; index < len(participants); index++ { + if !slices.Equal(participants[index].Roles, []string{"witness"}) { + t.Fatalf("witness %d roles = %#v", index, participants[index].Roles) + } + } +} + +func TestGroupedWorkflowBuildsVisibleSerialDAG(t *testing.T) { + t.Parallel() + + run := groupedTestRun("grouped") + run.Spec.ExecutionPolicy.OnFailure = signalv1.SuiteFailurePolicyContinue + run.Spec.Participants = []signalv1.Participant{ + {Index: 0, NodeName: "service", NodeUID: "service-uid", Roles: []string{"service"}}, + {Index: 1, NodeName: "witness-a", NodeUID: "witness-a-uid", Roles: []string{"witness"}}, + {Index: 2, NodeName: "witness-b", NodeUID: "witness-b-uid", Roles: []string{"witness"}}, + {Index: 3, NodeName: "witness-c", NodeUID: "witness-c-uid", Roles: []string{"witness"}}, + } + run.Spec.Checks[14].ParticipantPolicy.ExcludedRoles = []string{"service"} + + workflow, err := buildWorkflow(run, 1) + if err != nil { + t.Fatal(err) + } + + tasks, found, err := unstructured.NestedSlice( + workflow.Object, + "spec", + "templates", + ) + if err != nil || !found || len(tasks) != 1 { + t.Fatalf("templates: found=%t len=%d err=%v", found, len(tasks), err) + } + + template := tasks[0].(map[string]any) + dag := template["dag"].(map[string]any) + + dagTasks := dag["tasks"].([]any) + if len(dagTasks) != groupedCheckCount { + t.Fatalf("DAG tasks = %d, want %d", len(dagTasks), groupedCheckCount) + } + + for index := range dagTasks { + task := dagTasks[index].(map[string]any) + if index > 0 { + want := dagTasks[index-1].(map[string]any)["name"].(string) + ".Completed" + if task["depends"] != want { + t.Fatalf("task %d dependency = %q, want %q", index, task["depends"], want) + } + } + + templateRef := task["templateRef"].(map[string]any) + if templateRef["name"] != run.Spec.Workflow.Name || templateRef["template"] != "certification" { + t.Fatalf("task %d templateRef = %#v", index, templateRef) + } + } + + last := dagTasks[14].(map[string]any) + parameters := last["arguments"].(map[string]any)["parameters"].([]any) + + values := parameterValues(parameters) + if values["participant-count"] != "3" { + t.Fatalf("row 15 participant count = %q, want 3 witnesses", values["participant-count"]) + } +} + +func TestGroupedWorkflowStopPolicyRequiresDiagnosticPass(t *testing.T) { + t.Parallel() + + run := groupedTestRun("grouped-stop") + run.Spec.ExecutionPolicy.OnFailure = signalv1.SuiteFailurePolicyStop + + workflow, err := buildWorkflow(run, 1) + if err != nil { + t.Fatal(err) + } + + templates, _, _ := unstructured.NestedSlice(workflow.Object, "spec", "templates") + tasks := templates[0].(map[string]any)["dag"].(map[string]any)["tasks"].([]any) + + for index := 1; index < len(tasks); index++ { + previous := tasks[index-1].(map[string]any)["name"].(string) + + task := tasks[index].(map[string]any) + if task["depends"] != previous+".Succeeded" { + t.Fatalf("task %d dependency = %q", index, task["depends"]) + } + + wantWhen := "{{tasks." + previous + ".outputs.parameters.verdict}} == Pass" + if task["when"] != wantWhen { + t.Fatalf("task %d when = %q, want %q", index, task["when"], wantWhen) + } + } +} + +func TestGroupedTerminalAggregationRequiresExactCompleteCoverage(t *testing.T) { + t.Parallel() + + run := groupedTestRun("aggregate") + workflow := groupedWorkflow(run, "Succeeded") + + statuses, verdict, reason := terminalCheckStatuses(workflow, run) + if verdict != "Pass" || reason != "AllChecksPassed" { + t.Fatalf("aggregate = %s %s", verdict, reason) + } + + if len(statuses) != groupedCheckCount { + t.Fatalf("statuses = %d", len(statuses)) + } + + for index := range statuses { + if statuses[index].Phase != signalv1.CheckPhaseSucceeded || + !statuses[index].BackendEvidence.Complete || + !statuses[index].Cleanup.Complete || + len(statuses[index].BackendResources) != 1 { + t.Fatalf("status %d = %#v", index, statuses[index]) + } + } +} + +func TestGroupedTerminalAggregationMapsFilteredParticipantIndexes(t *testing.T) { + t.Parallel() + + run := groupedTestRun("participant-indexes") + run.Spec.Participants = []signalv1.Participant{ + {Index: 0, NodeName: "service", NodeUID: "service-uid", Roles: []string{"service"}}, + {Index: 1, NodeName: "witness-a", NodeUID: "witness-a-uid", Roles: []string{"witness"}}, + {Index: 2, NodeName: "witness-b", NodeUID: "witness-b-uid", Roles: []string{"witness"}}, + {Index: 3, NodeName: "witness-c", NodeUID: "witness-c-uid", Roles: []string{"witness"}}, + } + run.Spec.Checks[14].ParticipantPolicy.ExcludedRoles = []string{"service"} + + workflow := groupedWorkflow(run, "Succeeded") + node := groupedNode(workflow, run, 14) + evidence, _ := json.Marshal(map[string]any{ + "version": "v1", + "complete": true, + "participants": []any{ + groupedEvidenceParticipant(run, 14, 0), + groupedEvidenceParticipant(run, 14, 1), + groupedEvidenceParticipant(run, 14, 2), + }, + }) + resources, _ := json.Marshal([]signalv1.BackendResourceReference{{ + APIVersion: "v1", + Kind: "Pod", + Namespace: run.Namespace, + Name: "row-15", + UID: "row-15-uid", + ParticipantIndexes: []int32{0, 1, 2}, + }}) + + setNodeParameter(node, "backend-evidence", string(evidence)) + setNodeParameter(node, "backend-resources", string(resources)) + + statuses, _, _ := terminalCheckStatuses(workflow, run) + status := statuses[14] + + if !slices.Equal(status.BackendResources[0].ParticipantIndexes, []int32{1, 2, 3}) { + t.Fatalf("backend participant indexes = %#v", status.BackendResources[0].ParticipantIndexes) + } + + var normalized struct { + Participants []struct { + ParticipantIndex int32 `json:"participantIndex"` + } `json:"participants"` + } + if err := json.Unmarshal([]byte(status.BackendEvidence.Content), &normalized); err != nil { + t.Fatal(err) + } + + got := make([]int32, len(normalized.Participants)) + for index := range normalized.Participants { + got[index] = normalized.Participants[index].ParticipantIndex + } + + if !slices.Equal(got, []int32{1, 2, 3}) { + t.Fatalf("evidence participant indexes = %#v", got) + } +} + +func TestGroupedTerminalAggregationRetainsFailureAfterPolicyStop(t *testing.T) { + t.Parallel() + + run := groupedTestRun("stop-after-fail") + run.Spec.ExecutionPolicy.OnFailure = signalv1.SuiteFailurePolicyStop + workflow := groupedWorkflow(run, "Succeeded") + setNodeParameter(groupedNode(workflow, run, 2), "verdict", "Fail") + + for index := 3; index < groupedCheckCount; index++ { + groupedNode(workflow, run, index)["phase"] = "Omitted" + } + + statuses, verdict, reason := terminalCheckStatuses(workflow, run) + if verdict != "Fail" || reason != "OneOrMoreChecksFailed" { + t.Fatalf("aggregate = %q %q", verdict, reason) + } + + if statuses[2].Phase != signalv1.CheckPhaseFailed { + t.Fatalf("failed check status = %#v", statuses[2]) + } + + for index := 3; index < groupedCheckCount; index++ { + if statuses[index].Phase != signalv1.CheckPhaseSkipped || !statuses[index].NotRun { + t.Fatalf("stopped check %d = %#v", index, statuses[index]) + } + } +} + +func TestCompactCheckStatusesRetainsEvidenceDigest(t *testing.T) { + t.Parallel() + + statuses := []signalv1.CheckStatus{{ + BackendEvidence: signalv1.BackendEvidenceStatus{ + Version: "v1", + Complete: true, + Digest: contentDigest(`{"version":"v1"}`), + Content: `{"version":"v1"}`, + }, + }} + + compacted := compactCheckStatuses(statuses) + if compacted[0].BackendEvidence.Content != "" { + t.Fatalf("compacted backend evidence retained content") + } + + if compacted[0].BackendEvidence.Digest != statuses[0].BackendEvidence.Digest { + t.Fatalf("compacted digest = %q, want %q", compacted[0].BackendEvidence.Digest, statuses[0].BackendEvidence.Digest) + } + + if statuses[0].BackendEvidence.Content == "" { + t.Fatalf("compaction mutated source status") + } +} + +func TestGroupedReconcileRecoversPublishedEvidenceBeforeCancellation(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + run := groupedTestRun("recover-evidence") + run.UID = "recover-evidence-uid" + run.Spec.Cancel = true + run.Spec.Participants = []signalv1.Participant{{ + Index: 0, + NodeName: "gpu-0", + NodeUID: "gpu-0-uid", + }} + run.Status.Workflow = signalv1.WorkflowStatus{Name: run.Name + "-a1", UID: "workflow-uid", Attempt: 1, Phase: "Succeeded"} + + finishedAt := time.Date(2026, 10, 22, 10, 0, 0, 0, time.UTC) + + statuses := initialCheckStatuses(run.Spec.Checks) + for index := range statuses { + statuses[index].Phase = signalv1.CheckPhaseSucceeded + statuses[index].Verdict = "Pass" + statuses[index].BackendEvidence = signalv1.BackendEvidenceStatus{ + Version: "v1", + Complete: true, + Digest: contentDigest(`{"version":"v1"}`), + Content: `{"version":"v1"}`, + } + } + + document, err := json.Marshal(map[string]any{ + "runID": string(run.UID), + "runName": run.Name, + "workflow": run.Status.Workflow, + "checkStatus": statuses, + "executionState": "Completed", + "verdict": "Pass", + "reason": "AllChecksPassed", + "finishedAt": finishedAt.Format(time.RFC3339Nano), + }) + if err != nil { + t.Fatal(err) + } + + store := &immutableMemoryEvidenceStore{} + if _, _, err := store.Put(context.Background(), run.Spec.EvidencePath, document); err != nil { + t.Fatal(err) + } + + node := readyNode("gpu-0", "example-gpu") + node.UID = "gpu-0-uid" + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationRun{}). + WithObjects(run, node). + Build() + reconciler := &RunReconciler{Client: kube, Evidence: store} + + if _, err := reconciler.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}, + ); err != nil { + t.Fatal(err) + } + + var got signalv1.CertificationRun + if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { + t.Fatal(err) + } + + if got.Status.Phase != signalv1.RunPhaseSucceeded || + got.Status.Verdict != "Pass" || + got.Status.ExecutionState != "Completed" { + t.Fatalf("recovered status = %#v", got.Status) + } + + if got.Status.Checks[0].BackendEvidence.Content != "" || + got.Status.Checks[0].BackendEvidence.Digest == "" { + t.Fatalf("recovered check evidence was not compacted: %#v", got.Status.Checks[0].BackendEvidence) + } +} + +func TestGroupedTerminalAggregationFailsClosed(t *testing.T) { + t.Parallel() + + tests := map[string]func(*unstructured.Unstructured, *signalv1.CertificationRun){ + "missing task": func(workflow *unstructured.Unstructured, run *signalv1.CertificationRun) { + nodes := workflow.Object["status"].(map[string]any)["nodes"].(map[string]any) + delete(nodes, string(run.Spec.Checks[3].ID)) + }, + "duplicate identity": func(workflow *unstructured.Unstructured, run *signalv1.CertificationRun) { + nodes := workflow.Object["status"].(map[string]any)["nodes"].(map[string]any) + nodes["duplicate"] = nodes[string(run.Spec.Checks[4].ID)] + }, + "cleanup failure": func(workflow *unstructured.Unstructured, run *signalv1.CertificationRun) { + node := groupedNode(workflow, run, 5) + setNodeParameter(node, "cleanup-failure", "BackendCleanupFailed") + }, + "malformed evidence": func(workflow *unstructured.Unstructured, run *signalv1.CertificationRun) { + node := groupedNode(workflow, run, 6) + setNodeParameter(node, "backend-evidence", "{") + }, + "skipped": func(workflow *unstructured.Unstructured, run *signalv1.CertificationRun) { + groupedNode(workflow, run, 7)["phase"] = "Omitted" + }, + } + + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + run := groupedTestRun("fail-closed") + workflow := groupedWorkflow(run, "Succeeded") + mutate(workflow, run) + + statuses, verdict, _ := terminalCheckStatuses(workflow, run) + if verdict != "Inconclusive" { + t.Fatalf("verdict = %q", verdict) + } + + found := false + + for index := range statuses { + if statuses[index].Phase == signalv1.CheckPhaseInconclusive || + statuses[index].Phase == signalv1.CheckPhaseSkipped { + found = true + } + } + + if !found { + t.Fatalf("statuses did not retain fail-closed check state: %#v", statuses) + } + }) + } +} + +func TestSelectParticipantPlanFindsSatisfiableTopology(t *testing.T) { + t.Parallel() + + candidates := []corev1.Node{ + participantNode("service", "service", "zone-a"), + participantNode("witness-a", "witness", "zone-a"), + participantNode("witness-b", "witness", "zone-a"), + participantNode("witness-c", "witness", "zone-a"), + participantNode("witness-d", "witness", "zone-b"), + } + + participants, err := selectParticipantPlan( + candidates, + "service", + 4, + []signalv1.ParticipantRoleConstraint{ + { + Name: "service", + Count: 1, + Selector: metav1.LabelSelector{MatchLabels: map[string]string{"signal.example/role": "service"}}, + }, + { + Name: "witness", + Count: 3, + Selector: metav1.LabelSelector{MatchLabels: map[string]string{"signal.example/role": "witness"}}, + }, + }, + []signalv1.ParticipantTopologyConstraint{{ + Role: "witness", + LabelKey: "topology.example/zone", + MinDistinctValues: 2, + }}, + ) + if err != nil { + t.Fatal(err) + } + + names := make([]string, len(participants)) + for index := range participants { + names[index] = participants[index].NodeName + } + + if !slices.Contains(names, "witness-d") { + t.Fatalf("selected participants = %#v, missing satisfiable zone-b witness", names) + } +} + +func TestSelectParticipantPlanPrioritizesDistinctTopologyDomains(t *testing.T) { + t.Parallel() + + candidates := []corev1.Node{participantNode("service", "service", "zone-a")} + for index := 1; index < 63; index++ { + candidates = append(candidates, participantNode( + fmt.Sprintf("witness-%02d", index), + "witness", + "zone-a", + )) + } + + candidates = append(candidates, participantNode("witness-63", "witness", "zone-b")) + + participants, err := selectParticipantPlan( + candidates, + "service", + 7, + []signalv1.ParticipantRoleConstraint{ + { + Name: "service", + Count: 1, + Selector: metav1.LabelSelector{MatchLabels: map[string]string{"signal.example/role": "service"}}, + }, + { + Name: "witness", + Count: 6, + Selector: metav1.LabelSelector{MatchLabels: map[string]string{"signal.example/role": "witness"}}, + }, + }, + []signalv1.ParticipantTopologyConstraint{{ + Role: "witness", + LabelKey: "topology.example/zone", + MinDistinctValues: 2, + }}, + ) + if err != nil { + t.Fatal(err) + } + + names := make([]string, len(participants)) + for index := range participants { + names[index] = participants[index].NodeName + } + + if !slices.Contains(names, "witness-63") { + t.Fatalf("selected participants = %#v, missing late distinct domain", names) + } +} + +func groupedTestRun(name string) *signalv1.CertificationRun { + run := testRun(name) + run.Spec.Script = signalv1.LegacyScriptSnapshot{} + run.Spec.Backend = signalv1.ExecutionBackend{} + run.Spec.RunnerImage = "" + run.Spec.WorkloadPolicy = signalv1.WorkloadPolicy{} + run.Spec.Parameters = nil + run.Spec.BackendActiveDeadline = metav1.Duration{} + run.Spec.MaxAttempts = 0 + run.Spec.ExecutionPolicy = signalv1.SuiteExecutionPolicy{OnFailure: signalv1.SuiteFailurePolicyStop} + run.Spec.Checks = make([]signalv1.CertificationCheckSnapshot, groupedCheckCount) + + for index := range run.Spec.Checks { + run.Spec.Checks[index] = signalv1.CertificationCheckSnapshot{ + ID: signalv1.CheckID(fmt.Sprintf("R%02d", index+1)), + Order: int32(index), + Script: signalv1.ScriptSnapshot{ + ConfigMapName: name + "-scripts", + Key: fmt.Sprintf("%02d-check", index), + Digest: digestString(fmt.Sprintf("check-%d\n", index)), + }, + RunnerImage: "runner@sha256:" + stringsRepeat("a", 64), + Backend: signalv1.ExecutionBackend{APIVersion: "v1", Kind: "Pod"}, + BackendActiveDeadline: metav1.Duration{Duration: 9 * time.Minute}, + ResultBounds: signalv1.ResultBounds{ + ReasonBytes: 4096, + MeasurementsBytes: 16384, + BackendResourcesBytes: 16384, + BackendEvidenceBytes: 65536, + }, + MaxAttempts: 1, + } + } + + return run +} + +func groupedProfileChecks() []signalv1.CertificationCheck { + checks := make([]signalv1.CertificationCheck, groupedCheckCount) + for index := range checks { + checks[index] = signalv1.CertificationCheck{ + ID: signalv1.CheckID(fmt.Sprintf("R%02d", index+1)), + Order: int32(index), + Script: signalv1.ScriptReference{ + Name: "scripts", + Key: fmt.Sprintf("check-%02d.sh", index+1), + Digest: digestString(fmt.Sprintf("check-%d\n", index)), + }, + RunnerImage: "runner@sha256:" + stringsRepeat("a", 64), + Backend: signalv1.ExecutionBackend{APIVersion: "v1", Kind: "Pod"}, + BackendActiveDeadline: metav1.Duration{Duration: 9 * time.Minute}, + ResultBounds: signalv1.ResultBounds{ + ReasonBytes: 4096, + MeasurementsBytes: 16384, + BackendResourcesBytes: 16384, + BackendEvidenceBytes: 65536, + }, + MaxAttempts: 1, + } + } + + return checks +} + +func groupedWorkflow(run *signalv1.CertificationRun, phase string) *unstructured.Unstructured { + nodes := make(map[string]any, len(run.Spec.Checks)) + + for index := range run.Spec.Checks { + check := &run.Spec.Checks[index] + taskName := checkTaskName(check) + evidence, _ := json.Marshal(map[string]any{ + "version": "v1", + "complete": true, + "participants": []any{map[string]any{ + "participantIndex": 0, + "resource": map[string]any{ + "apiVersion": "v1", + "kind": "Pod", + "namespace": run.Namespace, + "name": taskName, + "uid": "uid-" + taskName, + }, + "pods": []any{}, + }}, + }) + resources, _ := json.Marshal([]signalv1.BackendResourceReference{{ + APIVersion: "v1", + Kind: "Pod", + Namespace: run.Namespace, + Name: taskName, + UID: types.UID("uid-" + taskName), + ParticipantIndexes: []int32{0}, + }}) + nodes[string(check.ID)] = map[string]any{ + "name": run.Name + "-a1." + taskName, + "phase": "Succeeded", + "startedAt": "2026-10-22T10:00:00Z", + "finishedAt": "2026-10-22T10:00:01Z", + "outputs": map[string]any{"parameters": []any{ + map[string]any{"name": "verdict", "value": "Pass"}, + map[string]any{"name": "reason", "value": "Healthy"}, + map[string]any{"name": "measurements", "value": "{}"}, + map[string]any{"name": "backend-resources", "value": string(resources)}, + map[string]any{"name": "backend-evidence", "value": string(evidence)}, + map[string]any{"name": "cleanup-failure", "value": ""}, + }}, + } + } + + return &unstructured.Unstructured{Object: map[string]any{ + "metadata": map[string]any{"name": run.Name + "-a1"}, + "status": map[string]any{ + "phase": phase, + "nodes": nodes, + }, + }} +} + +func groupedNode( + workflow *unstructured.Unstructured, + run *signalv1.CertificationRun, + index int, +) map[string]any { + return workflow.Object["status"].(map[string]any)["nodes"].(map[string]any)[string(run.Spec.Checks[index].ID)].(map[string]any) +} + +func groupedEvidenceParticipant( + run *signalv1.CertificationRun, + checkIndex int, + participantIndex int, +) map[string]any { + taskName := checkTaskName(&run.Spec.Checks[checkIndex]) + + return map[string]any{ + "participantIndex": participantIndex, + "resource": map[string]any{ + "apiVersion": "v1", + "kind": "Pod", + "namespace": run.Namespace, + "name": fmt.Sprintf("%s-p%d", taskName, participantIndex), + "uid": fmt.Sprintf("uid-%s-p%d", taskName, participantIndex), + }, + "pods": []any{}, + } +} + +func participantNode(name, role, zone string) corev1.Node { + return corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(name + "-uid"), + Labels: map[string]string{ + "signal.example/role": role, + "topology.example/zone": zone, + }, + }} +} + +func setNodeParameter(node map[string]any, name, value string) { + parameters := node["outputs"].(map[string]any)["parameters"].([]any) + for _, raw := range parameters { + parameter := raw.(map[string]any) + if parameter["name"] == name { + parameter["value"] = value + + return + } + } +} + +func parameterValues(parameters []any) map[string]string { + values := make(map[string]string, len(parameters)) + for _, raw := range parameters { + parameter := raw.(map[string]any) + values[parameter["name"].(string)] = parameter["value"].(string) + } + + return values +} + +func stringsRepeat(value string, count int) string { + result := "" + for range count { + result += value + } + + return result +} diff --git a/internal/signal/projection.go b/internal/signal/projection.go index 4ed10d4a30..bfb395bb10 100644 --- a/internal/signal/projection.go +++ b/internal/signal/projection.go @@ -81,7 +81,7 @@ func projectTerminal(ctx context.Context, kube client.Client, run *signalv1.Cert node.Annotations[LastRunAnnotation] = boundedAnnotationValue(run.Name) node.Annotations[LastRunAtAnnotation] = finishedAt node.Annotations[EvidenceDigestAnnotation] = boundedAnnotationValue(run.Status.Evidence.Digest) - node.Annotations[ScriptDigestAnnotation] = boundedAnnotationValue(run.Spec.Script.Digest) + node.Annotations[ScriptDigestAnnotation] = boundedAnnotationValue(runScriptDigest(run)) switch run.Status.Phase { case signalv1.RunPhaseSucceeded: @@ -104,6 +104,22 @@ func projectTerminal(ctx context.Context, kube client.Client, run *signalv1.Cert }) } +func runScriptDigest(run *signalv1.CertificationRun) string { + if len(run.Spec.Checks) == 0 { + return run.Spec.Script.Digest + } + + hash := sha256.New() + for index := range run.Spec.Checks { + _, _ = hash.Write([]byte(run.Spec.Checks[index].ID)) + _, _ = hash.Write([]byte{0}) + _, _ = hash.Write([]byte(run.Spec.Checks[index].Script.Digest)) + _, _ = hash.Write([]byte{0}) + } + + return "sha256:" + hex.EncodeToString(hash.Sum(nil)) +} + func projectRecurring( ctx context.Context, kube client.Client, diff --git a/internal/signal/projection_test.go b/internal/signal/projection_test.go index 928b5de346..e04828f9c1 100644 --- a/internal/signal/projection_test.go +++ b/internal/signal/projection_test.go @@ -60,7 +60,7 @@ func TestNodeProjectionTransitionsAndRetainsTerminalSummary(t *testing.T) { if got.Annotations[LastRunAnnotation] != run.Name || got.Annotations[LastSuccessfulRunAnnotation] != run.Name || got.Annotations[EvidenceDigestAnnotation] != run.Status.Evidence.Digest || - got.Annotations[ScriptDigestAnnotation] != run.Spec.Script.Digest { + got.Annotations[ScriptDigestAnnotation] != runScriptDigest(run) { t.Fatalf("terminal annotations = %v", got.Annotations) } } From fee5169062f6c347bf80b86a2c92895f491177d8 Mon Sep 17 00:00:00 2001 From: chokevin Date: Thu, 22 Oct 2026 14:52:30 -0700 Subject: [PATCH 22/36] fix(signal): resolve grouped workflow template inputs --- .../04-adapter-workflowtemplate.yaml.tmpl | 155 +++++++++++++++--- deploy/signal/manifest_test.go | 45 +++++ 2 files changed, 177 insertions(+), 23 deletions(-) diff --git a/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl b/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl index 3ab4f44a3f..6e203efcd2 100644 --- a/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl +++ b/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl @@ -5,7 +5,7 @@ metadata: namespace: {{ default "project-signal" .Namespace }} spec: serviceAccountName: project-signal-adapter - entrypoint: certification + entrypoint: certification-legacy arguments: parameters: - name: run-name @@ -48,7 +48,116 @@ spec: value: "false" - name: attempt templates: + - name: certification-legacy + steps: + - - name: certification + template: certification + arguments: + parameters: + - name: run-name + value: "{{`{{workflow.parameters.run-name}}`}}" + - name: run-uid + value: "{{`{{workflow.parameters.run-uid}}`}}" + - name: participants + value: "{{`{{workflow.parameters.participants}}`}}" + - name: participants-json + value: "{{`{{workflow.parameters.participants-json}}`}}" + - name: participant-count + value: "{{`{{workflow.parameters.participant-count}}`}}" + - name: request-node-name + value: "{{`{{workflow.parameters.request-node-name}}`}}" + - name: script-configmap + value: "{{`{{workflow.parameters.script-configmap}}`}}" + - name: script-key + value: "{{`{{workflow.parameters.script-key}}`}}" + - name: script-digest + value: "{{`{{workflow.parameters.script-digest}}`}}" + - name: script-requires-runtime-stage + value: "{{`{{workflow.parameters.script-requires-runtime-stage}}`}}" + - name: runtime-stage-name + value: "{{`{{workflow.parameters.runtime-stage-name}}`}}" + - name: backend-api-version + value: "{{`{{workflow.parameters.backend-api-version}}`}}" + - name: backend-kind + value: "{{`{{workflow.parameters.backend-kind}}`}}" + - name: backend-active-deadline-seconds + value: "{{`{{workflow.parameters.backend-active-deadline-seconds}}`}}" + - name: queue + value: "{{`{{workflow.parameters.queue}}`}}" + - name: runner-image + value: "{{`{{workflow.parameters.runner-image}}`}}" + - name: tolerations + value: "{{`{{workflow.parameters.tolerations}}`}}" + - name: script-environment + value: "{{`{{workflow.parameters.script-environment}}`}}" + - name: workload-policy + value: "{{`{{workflow.parameters.workload-policy}}`}}" + - name: mpi-payload-mode + value: "{{`{{workflow.parameters.mpi-payload-mode}}`}}" + - name: expected-gpu-product-regex + value: "{{`{{workflow.parameters.expected-gpu-product-regex}}`}}" + - name: expected-gpu-count + value: "{{`{{workflow.parameters.expected-gpu-count}}`}}" + - name: observed-gpu-capacity + value: "{{`{{workflow.parameters.observed-gpu-capacity}}`}}" + - name: observed-gpu-allocatable + value: "{{`{{workflow.parameters.observed-gpu-allocatable}}`}}" + - name: gpu-resource-count + value: "{{`{{workflow.parameters.gpu-resource-count}}`}}" + - name: privileged + value: "{{`{{workflow.parameters.privileged}}`}}" + - name: attempt + value: "{{`{{workflow.parameters.attempt}}`}}" + outputs: + parameters: + - name: verdict + valueFrom: + parameter: "{{`{{steps.certification.outputs.parameters.verdict}}`}}" + - name: reason + valueFrom: + parameter: "{{`{{steps.certification.outputs.parameters.reason}}`}}" + - name: cleanup-failure + valueFrom: + parameter: "{{`{{steps.certification.outputs.parameters.cleanup-failure}}`}}" + - name: measurements + valueFrom: + parameter: "{{`{{steps.certification.outputs.parameters.measurements}}`}}" + - name: backend-resources + valueFrom: + parameter: "{{`{{steps.certification.outputs.parameters.backend-resources}}`}}" + - name: backend-evidence + valueFrom: + parameter: "{{`{{steps.certification.outputs.parameters.backend-evidence}}`}}" - name: certification + inputs: + parameters: + - name: run-name + - name: run-uid + - name: participants + - name: participants-json + - name: participant-count + - name: request-node-name + - name: script-configmap + - name: script-key + - name: script-digest + - name: script-requires-runtime-stage + - name: runtime-stage-name + - name: backend-api-version + - name: backend-kind + - name: backend-active-deadline-seconds + - name: queue + - name: runner-image + - name: tolerations + - name: script-environment + - name: workload-policy + - name: mpi-payload-mode + - name: expected-gpu-product-regex + - name: expected-gpu-count + - name: observed-gpu-capacity + - name: observed-gpu-allocatable + - name: gpu-resource-count + - name: privileged + - name: attempt container: image: {{ default "example.invalid/project-signal-adapter@sha256:0000000000000000000000000000000000000000000000000000000000000000" .AdapterImage }} imagePullPolicy: IfNotPresent @@ -58,47 +167,47 @@ spec: - --workflow-name - "$(WORKFLOW_NAME)" - --run-name - - "{{`{{workflow.parameters.run-name}}`}}" + - "{{`{{inputs.parameters.run-name}}`}}" - --run-uid - - "{{`{{workflow.parameters.run-uid}}`}}" + - "{{`{{inputs.parameters.run-uid}}`}}" - --participants - - "{{`{{workflow.parameters.participants}}`}}" + - "{{`{{inputs.parameters.participants}}`}}" - --participants-json - - "{{`{{workflow.parameters.participants-json}}`}}" + - "{{`{{inputs.parameters.participants-json}}`}}" - --participant-count - - "{{`{{workflow.parameters.participant-count}}`}}" + - "{{`{{inputs.parameters.participant-count}}`}}" - --request-node-name - - "{{`{{workflow.parameters.request-node-name}}`}}" + - "{{`{{inputs.parameters.request-node-name}}`}}" - --script-configmap - - "{{`{{workflow.parameters.script-configmap}}`}}" + - "{{`{{inputs.parameters.script-configmap}}`}}" - --script-key - - "{{`{{workflow.parameters.script-key}}`}}" + - "{{`{{inputs.parameters.script-key}}`}}" - --script-digest - - "{{`{{workflow.parameters.script-digest}}`}}" - - "--script-requires-runtime-stage={{`{{workflow.parameters.script-requires-runtime-stage}}`}}" + - "{{`{{inputs.parameters.script-digest}}`}}" + - "--script-requires-runtime-stage={{`{{inputs.parameters.script-requires-runtime-stage}}`}}" - --runtime-stage-name - - "{{`{{workflow.parameters.runtime-stage-name}}`}}" + - "{{`{{inputs.parameters.runtime-stage-name}}`}}" - --runner-image - - "{{`{{workflow.parameters.runner-image}}`}}" + - "{{`{{inputs.parameters.runner-image}}`}}" - --backend-api-version - - "{{`{{workflow.parameters.backend-api-version}}`}}" + - "{{`{{inputs.parameters.backend-api-version}}`}}" - --backend-kind - - "{{`{{workflow.parameters.backend-kind}}`}}" + - "{{`{{inputs.parameters.backend-kind}}`}}" - --backend-active-deadline-seconds - - "{{`{{workflow.parameters.backend-active-deadline-seconds}}`}}" + - "{{`{{inputs.parameters.backend-active-deadline-seconds}}`}}" - --queue - - "{{`{{workflow.parameters.queue}}`}}" + - "{{`{{inputs.parameters.queue}}`}}" - --tolerations - - "{{`{{workflow.parameters.tolerations}}`}}" + - "{{`{{inputs.parameters.tolerations}}`}}" - --script-environment - - "{{`{{workflow.parameters.script-environment}}`}}" + - "{{`{{inputs.parameters.script-environment}}`}}" - --workload-policy - - "{{`{{workflow.parameters.workload-policy}}`}}" + - "{{`{{inputs.parameters.workload-policy}}`}}" - --gpu-resource-count - - "{{`{{workflow.parameters.gpu-resource-count}}`}}" - - "--privileged={{`{{workflow.parameters.privileged}}`}}" + - "{{`{{inputs.parameters.gpu-resource-count}}`}}" + - "--privileged={{`{{inputs.parameters.privileged}}`}}" - --attempt - - "{{`{{workflow.parameters.attempt}}`}}" + - "{{`{{inputs.parameters.attempt}}`}}" env: - name: POD_NAMESPACE valueFrom: diff --git a/deploy/signal/manifest_test.go b/deploy/signal/manifest_test.go index 35fe8bcd2c..210dd57080 100644 --- a/deploy/signal/manifest_test.go +++ b/deploy/signal/manifest_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "reflect" + "regexp" "sort" "strings" "testing" @@ -90,6 +91,50 @@ func TestAdapterManifestIsGenericAndParticipantAware(t *testing.T) { } } +func TestAdapterTemplateResolvesGroupedTaskArgumentsThroughInputs(t *testing.T) { + t.Parallel() + + content, err := os.ReadFile("04-adapter-workflowtemplate.yaml.tmpl") + if err != nil { + t.Fatal(err) + } + + manifest := string(content) + if !strings.Contains(manifest, "entrypoint: certification-legacy") { + t.Fatal("adapter WorkflowTemplate does not preserve the legacy workflow entrypoint") + } + + const certificationMarker = " - name: certification\n inputs:\n" + certificationIndex := strings.LastIndex(manifest, certificationMarker) + if certificationIndex < 0 { + t.Fatal("adapter WorkflowTemplate does not declare certification template inputs") + } + + certification := manifest[certificationIndex:] + if strings.Contains(certification, "workflow.parameters.") { + t.Fatal("grouped certification template references undeclared workflow parameters") + } + + inputReference := regexp.MustCompile(`inputs\.parameters\.([a-z0-9-]+)`) + for _, match := range inputReference.FindAllStringSubmatch(certification, -1) { + declaration := " - name: " + match[1] + if !strings.Contains(certification, declaration) { + t.Fatalf("certification template references undeclared input %q", match[1]) + } + } + + legacy := manifest[:certificationIndex] + for _, required := range []string{ + "value: \"{{`{{workflow.parameters.run-name}}`}}\"", + "parameter: \"{{`{{steps.certification.outputs.parameters.verdict}}`}}\"", + "parameter: \"{{`{{steps.certification.outputs.parameters.backend-evidence}}`}}\"", + } { + if !strings.Contains(legacy, required) { + t.Fatalf("legacy adapter wrapper does not contain %q", required) + } + } +} + func TestAdapterRBACAllowsBackendEvidenceCapture(t *testing.T) { t.Parallel() From 1d7cfe249c7fe7e0eb476df911b8e2d415cbac80 Mon Sep 17 00:00:00 2001 From: chokevin Date: Thu, 22 Oct 2026 14:55:04 -0700 Subject: [PATCH 23/36] fix(signal): use valid terminal DAG dependencies --- internal/signal/controller.go | 5 ++++- internal/signal/grouped_controller_test.go | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/internal/signal/controller.go b/internal/signal/controller.go index da317d0707..149f344354 100644 --- a/internal/signal/controller.go +++ b/internal/signal/controller.go @@ -1345,7 +1345,10 @@ func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured if index > 0 { previous := checkTaskName(&checks[index-1]) if run.Spec.ExecutionPolicy.OnFailure == signalv1.SuiteFailurePolicyContinue { - task["depends"] = previous + ".Completed" + task["depends"] = fmt.Sprintf( + "%[1]s.Succeeded || %[1]s.Failed || %[1]s.Errored || %[1]s.Skipped || %[1]s.Omitted", + previous, + ) } else { task["depends"] = previous + ".Succeeded" task["when"] = fmt.Sprintf( diff --git a/internal/signal/grouped_controller_test.go b/internal/signal/grouped_controller_test.go index 4b93e802e1..28d8428c48 100644 --- a/internal/signal/grouped_controller_test.go +++ b/internal/signal/grouped_controller_test.go @@ -244,7 +244,11 @@ func TestGroupedWorkflowBuildsVisibleSerialDAG(t *testing.T) { for index := range dagTasks { task := dagTasks[index].(map[string]any) if index > 0 { - want := dagTasks[index-1].(map[string]any)["name"].(string) + ".Completed" + previous := dagTasks[index-1].(map[string]any)["name"].(string) + want := fmt.Sprintf( + "%[1]s.Succeeded || %[1]s.Failed || %[1]s.Errored || %[1]s.Skipped || %[1]s.Omitted", + previous, + ) if task["depends"] != want { t.Fatalf("task %d dependency = %q, want %q", index, task["depends"], want) } From 21e2d042eaed5179d8631fe558eaf4a32bcdb590 Mon Sep 17 00:00:00 2001 From: chokevin Date: Thu, 22 Oct 2026 15:25:57 -0700 Subject: [PATCH 24/36] test(signal): satisfy grouped workflow lint --- deploy/signal/manifest_test.go | 1 + internal/signal/grouped_controller_test.go | 1 + 2 files changed, 2 insertions(+) diff --git a/deploy/signal/manifest_test.go b/deploy/signal/manifest_test.go index 210dd57080..2de78cef39 100644 --- a/deploy/signal/manifest_test.go +++ b/deploy/signal/manifest_test.go @@ -105,6 +105,7 @@ func TestAdapterTemplateResolvesGroupedTaskArgumentsThroughInputs(t *testing.T) } const certificationMarker = " - name: certification\n inputs:\n" + certificationIndex := strings.LastIndex(manifest, certificationMarker) if certificationIndex < 0 { t.Fatal("adapter WorkflowTemplate does not declare certification template inputs") diff --git a/internal/signal/grouped_controller_test.go b/internal/signal/grouped_controller_test.go index 28d8428c48..14795a19de 100644 --- a/internal/signal/grouped_controller_test.go +++ b/internal/signal/grouped_controller_test.go @@ -245,6 +245,7 @@ func TestGroupedWorkflowBuildsVisibleSerialDAG(t *testing.T) { task := dagTasks[index].(map[string]any) if index > 0 { previous := dagTasks[index-1].(map[string]any)["name"].(string) + want := fmt.Sprintf( "%[1]s.Succeeded || %[1]s.Failed || %[1]s.Errored || %[1]s.Skipped || %[1]s.Omitted", previous, From 7724a4e0f83e54c7c8303acc22e1e8eeefefaf78 Mon Sep 17 00:00:00 2001 From: chokevin Date: Thu, 22 Oct 2026 16:06:29 -0700 Subject: [PATCH 25/36] fix(signal): run grouped tasks with adapter service account --- internal/signal/controller.go | 6 +++++- internal/signal/grouped_controller_test.go | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/internal/signal/controller.go b/internal/signal/controller.go index 149f344354..bc8f91f7ac 100644 --- a/internal/signal/controller.go +++ b/internal/signal/controller.go @@ -83,7 +83,10 @@ var workflowOutputLimits = map[string]int{ "verdict": 32, } -const groupedCheckCount = 15 +const ( + groupedCheckCount = 15 + adapterServiceAccountName = "project-signal-adapter" +) var workflowGVK = schema.GroupVersionKind{ Group: "argoproj.io", @@ -1373,6 +1376,7 @@ func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured }, "spec": map[string]any{ "activeDeadlineSeconds": int64(run.Spec.Deadline.Seconds()), + "serviceAccountName": adapterServiceAccountName, "ttlStrategy": map[string]any{ "secondsAfterCompletion": int64(6 * time.Hour / time.Second), "secondsAfterSuccess": int64(6 * time.Hour / time.Second), diff --git a/internal/signal/grouped_controller_test.go b/internal/signal/grouped_controller_test.go index 14795a19de..e87e843b77 100644 --- a/internal/signal/grouped_controller_test.go +++ b/internal/signal/grouped_controller_test.go @@ -224,6 +224,20 @@ func TestGroupedWorkflowBuildsVisibleSerialDAG(t *testing.T) { t.Fatal(err) } + serviceAccountName, found, err := unstructured.NestedString( + workflow.Object, + "spec", + "serviceAccountName", + ) + if err != nil || !found || serviceAccountName != adapterServiceAccountName { + t.Fatalf( + "service account: found=%t value=%q err=%v", + found, + serviceAccountName, + err, + ) + } + tasks, found, err := unstructured.NestedSlice( workflow.Object, "spec", From 44bdf83cfd6757181a46dfbdbafb1d2fe939d1fb Mon Sep 17 00:00:00 2001 From: chokevin Date: Thu, 22 Oct 2026 19:45:51 -0700 Subject: [PATCH 26/36] refactor(signal): trim certification to production path --- .github/workflows/ci.yaml | 17 - Makefile | 22 +- .../v1alpha1/certificationprofile_types.go | 63 +- .../v1alpha1/certificationrun_envtest_test.go | 185 -- api/signal/v1alpha1/certificationrun_types.go | 19 - api/signal/v1alpha1/digest_test.go | 36 - api/signal/v1alpha1/runtime_contract_test.go | 98 - api/signal/v1alpha1/zz_generated.deepcopy.go | 173 -- cmd/project-signal-adapter/main.go | 2 - cmd/project-signal-adapter/main_test.go | 66 - cmd/project-signal-controller/main.go | 10 - .../04-adapter-workflowtemplate.yaml.tmpl | 128 -- deploy/signal/README.md | 4 +- ...ounded-cloud.io_certificationprofiles.yaml | 147 +- deploy/signal/manifest_test.go | 294 --- internal/signal/controller.go | 714 +----- internal/signal/controller_test.go | 1979 ----------------- internal/signal/evidence_test.go | 163 -- internal/signal/grouped_controller_test.go | 873 -------- internal/signal/policy.go | 718 ------ internal/signal/policy_test.go | 323 --- internal/signal/projection.go | 85 +- internal/signal/projection_test.go | 227 -- internal/signaladapter/adapter.go | 46 +- internal/signaladapter/adapter_test.go | 1470 ------------ .../signaladapter/backend_evidence_test.go | 481 ---- internal/signaladapter/runner.go | 60 - internal/signaladapter/runner_test.go | 404 ---- internal/signalworkload/policy_test.go | 198 -- 29 files changed, 80 insertions(+), 8925 deletions(-) delete mode 100644 api/signal/v1alpha1/certificationrun_envtest_test.go delete mode 100644 api/signal/v1alpha1/digest_test.go delete mode 100644 api/signal/v1alpha1/runtime_contract_test.go delete mode 100644 cmd/project-signal-adapter/main_test.go delete mode 100644 deploy/signal/manifest_test.go delete mode 100644 internal/signal/controller_test.go delete mode 100644 internal/signal/evidence_test.go delete mode 100644 internal/signal/grouped_controller_test.go delete mode 100644 internal/signal/policy.go delete mode 100644 internal/signal/policy_test.go delete mode 100644 internal/signal/projection_test.go delete mode 100644 internal/signaladapter/adapter_test.go delete mode 100644 internal/signaladapter/backend_evidence_test.go delete mode 100644 internal/signaladapter/runner_test.go delete mode 100644 internal/signalworkload/policy_test.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 38959183b7..5e0288eb77 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -391,23 +391,6 @@ jobs: - name: Provision envtest and run controller API-server race tests run: timeout --signal=TERM --kill-after=10s 300s make racer-envtest-ci - signal-envtest: - name: Project Signal API Server Integration Tests - runs-on: ubuntu-24.04 - timeout-minutes: 15 - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: go.mod - cache-dependency-path: go.sum - - - name: Provision envtest and run Project Signal API-server race tests - run: timeout --signal=TERM --kill-after=10s 300s make signal-envtest-ci - # ---------- Orca Integration Tests ---------- # Spins up Garage and Azurite via testcontainers-go and runs the # orca in-process integration suite (internal/orca/inttest). Docker diff --git a/Makefile b/Makefile index cfd224d4d6..48ff025ebd 100644 --- a/Makefile +++ b/Makefile @@ -244,7 +244,7 @@ NET_FRONTEND_CACHE_FILE := $(NET_FRONTEND_DIST_DIR)/.frontend-build-key # Frontend build toggle (dev builds produce unminified output with sourcemaps). REACT_DEV ?= false -.PHONY: all help fmt lint lint-actions test build vulncheck check-deps kubectl-unbounded kubectl-unbounded-build install-tools install-protoc install-helm generate kubectl-unbounded forge relctl relctl-build agent-artifacts-builder agent-artifacts-builder-build orcadev unbounded-agent machina machina-build machina-oci machina-oci-push machina-manifests machine-ops-controller machine-ops-controller-build machine-ops-controller-oci machine-ops-controller-oci-push machine-ops-manifests project-signal-controller project-signal-controller-build project-signal-adapter project-signal-adapter-build signal-envtest signal-envtest-ci signal-manifests image-project-signal-controller-local image-project-signal-adapter-local metalman metalman-build metalman-oci metalman-oci-push unbounded-operator unbounded-operator-build unbounded-operator-manifests playpen-manifests e2e-gantry e2e-playpen e2e-racer gomod docs-serve unbounded-net-controller unbounded-net-controller-build unbounded-net-node unbounded-net-node-build unbounded-net-routeplan-debug unping unping-build unroute unroute-build license-check notice notice-check gantry gantry-build gantry-manifests inventory-manifests +.PHONY: all help fmt lint lint-actions test build vulncheck check-deps kubectl-unbounded kubectl-unbounded-build install-tools install-protoc install-helm generate kubectl-unbounded forge relctl relctl-build agent-artifacts-builder agent-artifacts-builder-build orcadev unbounded-agent machina machina-build machina-oci machina-oci-push machina-manifests machine-ops-controller machine-ops-controller-build machine-ops-controller-oci machine-ops-controller-oci-push machine-ops-manifests project-signal-controller project-signal-controller-build project-signal-adapter project-signal-adapter-build signal-manifests image-project-signal-controller-local image-project-signal-adapter-local metalman metalman-build metalman-oci metalman-oci-push unbounded-operator unbounded-operator-build unbounded-operator-manifests playpen-manifests e2e-gantry e2e-playpen e2e-racer gomod docs-serve unbounded-net-controller unbounded-net-controller-build unbounded-net-node unbounded-net-node-build unbounded-net-routeplan-debug unping unping-build unroute unroute-build license-check notice notice-check gantry gantry-build gantry-manifests inventory-manifests .PHONY: net-frontend net-frontend-clean net-ebpf-build net-ebpf-generate net-ebpf-verify net-manifests gantry-chart-lint gantry-chart-package release-bom release-manifests unbounded-operator-release-manifest .PHONY: image-machina-local image-token-refresher-local image-machine-ops-controller-local image-metalman-local image-unbounded-operator-local image-unbounded-operator-push image-playpen-local image-net-controller-local image-net-node-local image-gantry-local image-gantry-push images-local .PHONY: image-net-controller-push image-net-node-push images-net-all images-net-all-push @@ -305,7 +305,6 @@ help: ## Show this help @echo " machine-ops-controller Build machine-ops-controller" @echo " project-signal-controller Build Project Signal controller" @echo " project-signal-adapter Build Project Signal participant adapter" - @echo " signal-envtest-ci Provision and run Project Signal API-server tests" @echo " signal-manifests Render Project Signal manifests" @echo " metalman | metalman-build Build metalman controller (with/without lint/test)" @echo " unbounded-operator | unbounded-operator-build Build the top-level Site operator" @@ -330,9 +329,6 @@ help: ## Show this help @echo " image-machine-ops-controller-local Build machine-ops-controller image" @echo " image-project-signal-controller-local Build Project Signal controller image" @echo " image-project-signal-adapter-local Build Project Signal adapter image" - @echo " Build Project Signal recovery verifier image" - @echo " Build Project Signal GPU runner image" - @echo " Build and push Project Signal GPU runner image" @echo " image-metalman-local Build metalman image" @echo " image-unbounded-operator-local Build unbounded-operator image" @echo " image-unbounded-operator-push Build and push unbounded-operator image" @@ -616,16 +612,6 @@ racer-envtest: ## Run real API-server, manager election, TLS and crash-recovery @mkdir -p tmp/racer-envtest TMPDIR="$(CURDIR)/tmp/racer-envtest" KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" timeout --signal=TERM --kill-after=10s 300s $(GOTEST) -race ./api/racer/v1alpha1 ./internal/racer ./internal/racer/authority ./deploy/racer ./internal/operator/components/racer -run '^TestEnvtest' -count=1 -v -timeout=5m -signal-envtest-ci: $(SETUP_ENVTEST) ## Provision pinned local API-server assets and require Project Signal envtest - @mkdir -p tmp/signal-envtest - @assets=$$(TMPDIR="$(CURDIR)/tmp/signal-envtest" timeout --signal=TERM --kill-after=10s 300s "$(SETUP_ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(CURDIR)/bin/envtest" -p path) && \ - $(MAKE) signal-envtest KUBEBUILDER_ASSETS="$$assets" - -signal-envtest: ## Run Project Signal API-server validation tests - @test -n "$(KUBEBUILDER_ASSETS)" || { echo "Set KUBEBUILDER_ASSETS to repository-local envtest binaries"; exit 1; } - @mkdir -p tmp/signal-envtest - TMPDIR="$(CURDIR)/tmp/signal-envtest" KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" timeout --signal=TERM --kill-after=10s 300s $(GOTEST) -race ./api/signal/v1alpha1 -run '^TestEnvtest' -count=1 -v -timeout=5m - racer-generate: ## Generate Racer deepcopy and CRD artifacts timeout --signal=TERM --kill-after=10s 300s $(GOCMD) generate ./api/racer/v1alpha1 @@ -688,11 +674,11 @@ image-racer-object-local: ## Build the racer-object container image locally (sin --build-arg GIT_COMMIT=$(GIT_COMMIT) \ --build-arg BUILD_TIME=$(BUILD_TIME) \ -t racer-object:$(VERSION_TAG) -t $(RACER_OBJECT_IMAGE) \ - -f ./images/racer-object/Containerfile . - $(call trivy-maybe,$(RACER_OBJECT_IMAGE)) + -f ./images/racer-object/Containerfile . + $(call trivy-maybe,$(RACER_OBJECT_IMAGE)) build: machina-manifests token-refresher-manifests machine-ops-manifests signal-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Build all Go packages - $(GOBUILD) ./... + $(GOBUILD) ./... generate: install-protoc ## Run go generate for API types (deepcopy, CRDs) and protobuf PATH="$(PROTOC_DIR)/bin:$$PATH" $(GOCMD) generate $(GO_PACKAGES) diff --git a/api/signal/v1alpha1/certificationprofile_types.go b/api/signal/v1alpha1/certificationprofile_types.go index d7a38d9dd9..58a3515e61 100644 --- a/api/signal/v1alpha1/certificationprofile_types.go +++ b/api/signal/v1alpha1/certificationprofile_types.go @@ -31,16 +31,6 @@ const ( PurposeDiagnostic CertificationPurpose = "Diagnostic" ) -type CertificationStage struct { - ID string `json:"id"` - DisplayName string `json:"displayName"` - Category string `json:"category"` - Description string `json:"description,omitempty"` - ExpectedDuration metav1.Duration `json:"expectedDuration,omitempty"` - Required bool `json:"required"` - Conditional bool `json:"conditional,omitempty"` -} - type CheckID string type ResultBounds struct { @@ -179,7 +169,6 @@ type WorkloadPolicy struct { // +kubebuilder:validation:XValidation:rule="self.checks.map(c, c.order) == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]",message="checks must use immutable contiguous order 0 through 14" // +kubebuilder:validation:XValidation:rule="self.checks.map(c, c.id) == ['R01', 'R02', 'R03', 'R04', 'R05', 'R06', 'R07', 'R08', 'R09', 'R10', 'R11', 'R12', 'R13', 'R14', 'R15']",message="checks must use ordered IDs R01 through R15" // +kubebuilder:validation:XValidation:rule="self.checks.all(c, duration(self.deadline) - duration(c.backendActiveDeadline) >= duration('60s'))",message="every backendActiveDeadline must leave at least 60 seconds before deadline" -// +kubebuilder:validation:XValidation:rule="!has(self.recurring) || !self.recurring.enabled",message="recurring certification is not supported for grouped suites" type CertificationProfileSpec struct { Participants ParticipantPolicy `json:"participants"` // +kubebuilder:validation:MaxItems=16 @@ -196,67 +185,17 @@ type CertificationProfileSpec struct { EvidencePath string `json:"evidencePath"` Deadline metav1.Duration `json:"deadline"` Tolerations []corev1.Toleration `json:"tolerations,omitempty"` - Recurring *RecurringPolicy `json:"recurring,omitempty"` - - // Deprecated Go-only aliases retained for unit fixtures. They are not part - // of the v1alpha1 wire contract. - Script ScriptReference `json:"-"` - Backend ExecutionBackend `json:"-"` - RunnerImage string `json:"-"` - WorkloadPolicy WorkloadPolicy `json:"-"` - StagePlan []CertificationStage `json:"-"` - RuntimeStage *RuntimeStage `json:"-"` - Parameters map[string]string `json:"-"` - BackendActiveDeadline metav1.Duration `json:"-"` - MaxAttempts int32 `json:"-"` -} - -type RecurringPolicy struct { - Enabled bool `json:"enabled"` - NodeNames []string `json:"nodeNames"` - MinIdleDuration metav1.Duration `json:"minIdleDuration"` - ValidFor metav1.Duration `json:"validFor"` - NotBefore *metav1.Time `json:"notBefore,omitempty"` - ExpiresAt metav1.Time `json:"expiresAt"` - EvaluationInterval metav1.Duration `json:"evaluationInterval,omitempty"` - FailureBackoff metav1.Duration `json:"failureBackoff"` - MaxConsecutiveFailures int32 `json:"maxConsecutiveFailures"` - MaxConcurrent int32 `json:"maxConcurrent"` - FailureDomainLabel string `json:"failureDomainLabel,omitempty"` - MaxConcurrentPerFailureDomain int32 `json:"maxConcurrentPerFailureDomain,omitempty"` - ExclusiveProfiles []string `json:"exclusiveProfiles,omitempty"` -} - -type RecurringNodeStatus struct { - NodeName string `json:"nodeName"` - IdleSince *metav1.Time `json:"idleSince,omitempty"` - LastEvaluatedAt *metav1.Time `json:"lastEvaluatedAt,omitempty"` - LastSuccessfulRun string `json:"lastSuccessfulRun,omitempty"` - LastSuccessfulAt *metav1.Time `json:"lastSuccessfulAt,omitempty"` - NextDueAt *metav1.Time `json:"nextDueAt,omitempty"` - ConsecutiveFailures int32 `json:"consecutiveFailures,omitempty"` - Suspended bool `json:"suspended,omitempty"` - Reason string `json:"reason,omitempty"` -} - -type CertificationProfileStatus struct { - ObservedGeneration int64 `json:"observedGeneration,omitempty"` - Nodes []RecurringNodeStatus `json:"nodes,omitempty"` - Conditions []metav1.Condition `json:"conditions,omitempty"` } // +kubebuilder:object:root=true // +kubebuilder:resource:scope=Cluster,shortName=certprofile -// +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="Checks",type=integer,JSONPath=`.spec.checks.size()` // +kubebuilder:printcolumn:name="Participants",type=integer,JSONPath=`.spec.participants.count` -// +kubebuilder:printcolumn:name="Recurring",type=boolean,JSONPath=`.spec.recurring.enabled` type CertificationProfile struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec CertificationProfileSpec `json:"spec"` - Status CertificationProfileStatus `json:"status,omitempty"` + Spec CertificationProfileSpec `json:"spec"` } // +kubebuilder:object:root=true diff --git a/api/signal/v1alpha1/certificationrun_envtest_test.go b/api/signal/v1alpha1/certificationrun_envtest_test.go deleted file mode 100644 index 3268185d71..0000000000 --- a/api/signal/v1alpha1/certificationrun_envtest_test.go +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -package v1alpha1 - -import ( - "fmt" - "os" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest" -) - -func TestEnvtestCertificationRunOptionalImmutability(t *testing.T) { - assets := os.Getenv("KUBEBUILDER_ASSETS") - if assets == "" { - t.Skip("set KUBEBUILDER_ASSETS for real API validation tests") - } - - environment := &envtest.Environment{ - BinaryAssetsDirectory: assets, - CRDDirectoryPaths: []string{"../../../deploy/signal/crd"}, - ErrorIfCRDPathMissing: true, - AttachControlPlaneOutput: false, - } - restConfig, err := environment.Start() - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, environment.Stop()) }) - - scheme := runtime.NewScheme() - require.NoError(t, corev1.AddToScheme(scheme)) - require.NoError(t, AddToScheme(scheme)) - - apiClient, err := client.New(restConfig, client.Options{Scheme: scheme}) - require.NoError(t, err) - require.NoError(t, apiClient.Create(t.Context(), &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{Name: "signal"}, - })) - - createRun := func(name string, stage *RuntimeStage) *CertificationRun { - t.Helper() - - run := validEnvtestCertificationRun(name) - run.Spec.Checks[0].RuntimeStage = stage - run.Spec.Checks[0].Script.RequiresRuntimeStage = stage != nil - require.NoError(t, apiClient.Create(t.Context(), run)) - - return run - } - - t.Run("omitted runtime stage allows terminal status update", func(t *testing.T) { - run := createRun("runtime-stage-absent", nil) - run.Status.Phase = RunPhaseInconclusive - run.Status.ExecutionState = "Completed" - run.Status.Verdict = "Inconclusive" - run.Status.Reason = "DiagnosticUnavailable" - require.NoError(t, apiClient.Status().Update(t.Context(), run)) - }) - - t.Run("present equal remains valid", func(t *testing.T) { - run := createRun("runtime-stage-equal", &RuntimeStage{Name: "collective-bandwidth"}) - run.Spec.Cancel = true - require.NoError(t, apiClient.Update(t.Context(), run)) - }) - - for name, test := range map[string]struct { - initial *RuntimeStage - updated *RuntimeStage - }{ - "absent to present": { - updated: &RuntimeStage{Name: "collective-bandwidth"}, - }, - "present to absent": { - initial: &RuntimeStage{Name: "collective-bandwidth"}, - }, - "present unequal": { - initial: &RuntimeStage{Name: "collective-bandwidth"}, - updated: &RuntimeStage{Name: "memory-bandwidth"}, - }, - } { - t.Run(name+" is rejected", func(t *testing.T) { - run := createRun("runtime-stage-"+strings.ReplaceAll(name, " ", "-"), test.initial) - run.Spec.Checks[0].RuntimeStage = test.updated - run.Spec.Checks[0].Script.RequiresRuntimeStage = test.updated != nil - - err := apiClient.Update(t.Context(), run) - require.True(t, apierrors.IsInvalid(err), "%v", err) - require.ErrorContains(t, err, "checks are immutable") - }) - } - - for name, mutate := range map[string]func(*CertificationRun){ - "fourteen checks": func(run *CertificationRun) { - run.Spec.Checks = run.Spec.Checks[:14] - }, - "duplicate check id": func(run *CertificationRun) { - run.Spec.Checks[1].ID = run.Spec.Checks[0].ID - }, - "wrong ordered check id": func(run *CertificationRun) { - run.Spec.Checks[4].ID = "R15" - run.Spec.Checks[14].ID = "R05" - }, - "noncontiguous order": func(run *CertificationRun) { - run.Spec.Checks[7].Order = 8 - }, - "retry enabled": func(run *CertificationRun) { - run.Spec.Checks[2].MaxAttempts = 2 - }, - "insufficient cleanup reserve": func(run *CertificationRun) { - run.Spec.Checks[3].BackendActiveDeadline = metav1.Duration{Duration: 61 * time.Second} - }, - } { - t.Run(name+" is rejected", func(t *testing.T) { - run := validEnvtestCertificationRun("invalid-" + strings.ReplaceAll(name, " ", "-")) - mutate(run) - - err := apiClient.Create(t.Context(), run) - require.True(t, apierrors.IsInvalid(err), "%v", err) - }) - } -} - -func validEnvtestCertificationRun(name string) *CertificationRun { - return &CertificationRun{ - TypeMeta: metav1.TypeMeta{APIVersion: GroupVersion.String(), Kind: "CertificationRun"}, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: "signal", - }, - Spec: CertificationRunSpec{ - Profile: PinnedProfile{ - Name: "profile", - UID: types.UID("profile-uid"), - Generation: 1, - }, - Participants: []Participant{{ - Index: 0, - NodeName: "node-a", - NodeUID: types.UID("node-a-uid"), - }}, - Checks: validEnvtestChecks(), - ExecutionPolicy: SuiteExecutionPolicy{OnFailure: SuiteFailurePolicyStop}, - Workflow: WorkflowReference{Name: "workflow"}, - EvidencePath: "signal/" + name + ".json", - Deadline: metav1.Duration{Duration: 2 * time.Minute}, - RequestNodeName: "node-a", - }, - } -} - -func validEnvtestChecks() []CertificationCheckSnapshot { - checks := make([]CertificationCheckSnapshot, 15) - for index := range checks { - checks[index] = CertificationCheckSnapshot{ - ID: CheckID(fmt.Sprintf("R%02d", index+1)), - Order: int32(index), - Script: ScriptSnapshot{ - ConfigMapName: "scripts", - Key: "check.sh", - Digest: "sha256:" + strings.Repeat("a", 64), - }, - RunnerImage: "example.invalid/runner@sha256:" + strings.Repeat("b", 64), - Backend: ExecutionBackend{APIVersion: "v1", Kind: "Pod"}, - BackendActiveDeadline: metav1.Duration{Duration: time.Minute}, - ResultBounds: ResultBounds{ - ReasonBytes: 4096, - MeasurementsBytes: 16384, - BackendResourcesBytes: 16384, - BackendEvidenceBytes: 65536, - }, - MaxAttempts: 1, - } - } - - return checks -} diff --git a/api/signal/v1alpha1/certificationrun_types.go b/api/signal/v1alpha1/certificationrun_types.go index 196127329c..3193856bf4 100644 --- a/api/signal/v1alpha1/certificationrun_types.go +++ b/api/signal/v1alpha1/certificationrun_types.go @@ -66,13 +66,6 @@ type ScriptSnapshot struct { Digest string `json:"digest"` } -type LegacyScriptSnapshot struct { - ConfigMapName string `json:"configMapName,omitempty"` - Key string `json:"key,omitempty"` - RequiresRuntimeStage bool `json:"requiresRuntimeStage,omitempty"` - Digest string `json:"digest,omitempty"` -} - // +kubebuilder:validation:XValidation:rule="self.maxAttempts == 1",message="maxAttempts must be exactly 1" // +kubebuilder:validation:XValidation:rule="duration(self.backendActiveDeadline) > duration('0s')",message="backendActiveDeadline must be positive" // +kubebuilder:validation:XValidation:rule="!has(self.script.requiresRuntimeStage) || !self.script.requiresRuntimeStage || has(self.runtimeStage)",message="runtimeStage is required when the script requires a runtime stage" @@ -133,18 +126,6 @@ type CertificationRunSpec struct { Tolerations []corev1.Toleration `json:"tolerations,omitempty"` RequestNodeName string `json:"requestNodeName"` Cancel bool `json:"cancel,omitempty"` - - // Deprecated Go-only aliases retained for unit fixtures. They are not part - // of the v1alpha1 wire contract. - Script LegacyScriptSnapshot `json:"-"` - Backend ExecutionBackend `json:"-"` - RunnerImage string `json:"-"` - WorkloadPolicy WorkloadPolicy `json:"-"` - StagePlan []CertificationStage `json:"-"` - RuntimeStage *RuntimeStage `json:"-"` - Parameters map[string]string `json:"-"` - BackendActiveDeadline metav1.Duration `json:"-"` - MaxAttempts int32 `json:"-"` } type WorkflowStatus struct { diff --git a/api/signal/v1alpha1/digest_test.go b/api/signal/v1alpha1/digest_test.go deleted file mode 100644 index e0e1559325..0000000000 --- a/api/signal/v1alpha1/digest_test.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -package v1alpha1 - -import ( - "strings" - "testing" -) - -func TestIsSHA256Digest(t *testing.T) { - t.Parallel() - - valid := "sha256:" + strings.Repeat("3c", 32) - if !IsSHA256Digest(valid) { - t.Fatalf("IsSHA256Digest(%q) = false", valid) - } - - for name, value := range map[string]string{ - "bare hex": strings.Repeat("3c", 32), - "uppercase hex": "sha256:" + strings.Repeat("3C", 32), - "wrong algorithm": "sha512:" + strings.Repeat("3c", 32), - "short": "sha256:" + strings.Repeat("3c", 31), - "long": "sha256:" + strings.Repeat("3c", 33), - "leading space": " " + valid, - "trailing newline": valid + "\n", - } { - t.Run(name, func(t *testing.T) { - t.Parallel() - - if IsSHA256Digest(value) { - t.Fatalf("IsSHA256Digest(%q) = true", value) - } - }) - } -} diff --git a/api/signal/v1alpha1/runtime_contract_test.go b/api/signal/v1alpha1/runtime_contract_test.go deleted file mode 100644 index ff7c49cf63..0000000000 --- a/api/signal/v1alpha1/runtime_contract_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -package v1alpha1 - -import ( - "strings" - "testing" - "time" -) - -func TestValidateRuntimeContractAcceptsExactDeadlineReserve(t *testing.T) { - t.Parallel() - - err := ValidateRuntimeContract( - true, - &RuntimeStage{Name: "collective-bandwidth"}, - 9*time.Minute, - 10*time.Minute, - ) - if err != nil { - t.Fatal(err) - } -} - -func TestValidateRuntimeContractAllowsNonStageScript(t *testing.T) { - t.Parallel() - - if err := ValidateRuntimeContract(false, nil, time.Minute, 2*time.Minute); err != nil { - t.Fatal(err) - } -} - -func TestValidateRuntimeContractRejectsInvalidStageAndDeadlines(t *testing.T) { - t.Parallel() - - tests := map[string]struct { - requiresStage bool - stage *RuntimeStage - backendDeadline time.Duration - outerDeadline time.Duration - want string - }{ - "missing required stage": { - requiresStage: true, - backendDeadline: time.Minute, - outerDeadline: 2 * time.Minute, - want: "runtime stage is required", - }, - "empty stage": { - stage: &RuntimeStage{}, - backendDeadline: time.Minute, - outerDeadline: 2 * time.Minute, - want: "runtime stage name is invalid", - }, - "malformed stage": { - stage: &RuntimeStage{Name: "Collective/Bandwidth"}, - backendDeadline: time.Minute, - outerDeadline: 2 * time.Minute, - want: "runtime stage name is invalid", - }, - "zero backend deadline": { - backendDeadline: 0, - outerDeadline: 2 * time.Minute, - want: "must be positive", - }, - "fractional seconds": { - backendDeadline: time.Minute + time.Millisecond, - outerDeadline: 2 * time.Minute, - want: "whole seconds", - }, - "reserve one second too short": { - backendDeadline: 61 * time.Second, - outerDeadline: 2 * time.Minute, - want: "at least 1m0s", - }, - "backend equals outer": { - backendDeadline: 2 * time.Minute, - outerDeadline: 2 * time.Minute, - want: "at least 1m0s", - }, - } - for name, test := range tests { - t.Run(name, func(t *testing.T) { - t.Parallel() - - err := ValidateRuntimeContract( - test.requiresStage, - test.stage, - test.backendDeadline, - test.outerDeadline, - ) - if err == nil || !strings.Contains(err.Error(), test.want) { - t.Fatalf("ValidateRuntimeContract() error = %v, want %q", err, test.want) - } - }) - } -} diff --git a/api/signal/v1alpha1/zz_generated.deepcopy.go b/api/signal/v1alpha1/zz_generated.deepcopy.go index 907981bee4..dc2279b510 100644 --- a/api/signal/v1alpha1/zz_generated.deepcopy.go +++ b/api/signal/v1alpha1/zz_generated.deepcopy.go @@ -120,7 +120,6 @@ func (in *CertificationProfile) DeepCopyInto(out *CertificationProfile) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationProfile. @@ -206,32 +205,6 @@ func (in *CertificationProfileSpec) DeepCopyInto(out *CertificationProfileSpec) (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.Recurring != nil { - in, out := &in.Recurring, &out.Recurring - *out = new(RecurringPolicy) - (*in).DeepCopyInto(*out) - } - out.Script = in.Script - out.Backend = in.Backend - in.WorkloadPolicy.DeepCopyInto(&out.WorkloadPolicy) - if in.StagePlan != nil { - in, out := &in.StagePlan, &out.StagePlan - *out = make([]CertificationStage, len(*in)) - copy(*out, *in) - } - if in.RuntimeStage != nil { - in, out := &in.RuntimeStage, &out.RuntimeStage - *out = new(RuntimeStage) - **out = **in - } - if in.Parameters != nil { - in, out := &in.Parameters, &out.Parameters - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - out.BackendActiveDeadline = in.BackendActiveDeadline } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationProfileSpec. @@ -244,35 +217,6 @@ func (in *CertificationProfileSpec) DeepCopy() *CertificationProfileSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CertificationProfileStatus) DeepCopyInto(out *CertificationProfileStatus) { - *out = *in - if in.Nodes != nil { - in, out := &in.Nodes, &out.Nodes - *out = make([]RecurringNodeStatus, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.Conditions != nil { - in, out := &in.Conditions, &out.Conditions - *out = make([]metav1.Condition, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationProfileStatus. -func (in *CertificationProfileStatus) DeepCopy() *CertificationProfileStatus { - if in == nil { - return nil - } - out := new(CertificationProfileStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CertificationRun) DeepCopyInto(out *CertificationRun) { *out = *in @@ -372,27 +316,6 @@ func (in *CertificationRunSpec) DeepCopyInto(out *CertificationRunSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - out.Script = in.Script - out.Backend = in.Backend - in.WorkloadPolicy.DeepCopyInto(&out.WorkloadPolicy) - if in.StagePlan != nil { - in, out := &in.StagePlan, &out.StagePlan - *out = make([]CertificationStage, len(*in)) - copy(*out, *in) - } - if in.RuntimeStage != nil { - in, out := &in.RuntimeStage, &out.RuntimeStage - *out = new(RuntimeStage) - **out = **in - } - if in.Parameters != nil { - in, out := &in.Parameters, &out.Parameters - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - out.BackendActiveDeadline = in.BackendActiveDeadline } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationRunSpec. @@ -445,22 +368,6 @@ func (in *CertificationRunStatus) DeepCopy() *CertificationRunStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CertificationStage) DeepCopyInto(out *CertificationStage) { - *out = *in - out.ExpectedDuration = in.ExpectedDuration -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationStage. -func (in *CertificationStage) DeepCopy() *CertificationStage { - if in == nil { - return nil - } - out := new(CertificationStage) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CheckCleanupStatus) DeepCopyInto(out *CheckCleanupStatus) { *out = *in @@ -594,21 +501,6 @@ func (in *KueueTopologyPolicy) DeepCopy() *KueueTopologyPolicy { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LegacyScriptSnapshot) DeepCopyInto(out *LegacyScriptSnapshot) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LegacyScriptSnapshot. -func (in *LegacyScriptSnapshot) DeepCopy() *LegacyScriptSnapshot { - if in == nil { - return nil - } - out := new(LegacyScriptSnapshot) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Participant) DeepCopyInto(out *Participant) { *out = *in @@ -712,71 +604,6 @@ func (in *PinnedProfile) DeepCopy() *PinnedProfile { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RecurringNodeStatus) DeepCopyInto(out *RecurringNodeStatus) { - *out = *in - if in.IdleSince != nil { - in, out := &in.IdleSince, &out.IdleSince - *out = (*in).DeepCopy() - } - if in.LastEvaluatedAt != nil { - in, out := &in.LastEvaluatedAt, &out.LastEvaluatedAt - *out = (*in).DeepCopy() - } - if in.LastSuccessfulAt != nil { - in, out := &in.LastSuccessfulAt, &out.LastSuccessfulAt - *out = (*in).DeepCopy() - } - if in.NextDueAt != nil { - in, out := &in.NextDueAt, &out.NextDueAt - *out = (*in).DeepCopy() - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RecurringNodeStatus. -func (in *RecurringNodeStatus) DeepCopy() *RecurringNodeStatus { - if in == nil { - return nil - } - out := new(RecurringNodeStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RecurringPolicy) DeepCopyInto(out *RecurringPolicy) { - *out = *in - if in.NodeNames != nil { - in, out := &in.NodeNames, &out.NodeNames - *out = make([]string, len(*in)) - copy(*out, *in) - } - out.MinIdleDuration = in.MinIdleDuration - out.ValidFor = in.ValidFor - if in.NotBefore != nil { - in, out := &in.NotBefore, &out.NotBefore - *out = (*in).DeepCopy() - } - in.ExpiresAt.DeepCopyInto(&out.ExpiresAt) - out.EvaluationInterval = in.EvaluationInterval - out.FailureBackoff = in.FailureBackoff - if in.ExclusiveProfiles != nil { - in, out := &in.ExclusiveProfiles, &out.ExclusiveProfiles - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RecurringPolicy. -func (in *RecurringPolicy) DeepCopy() *RecurringPolicy { - if in == nil { - return nil - } - out := new(RecurringPolicy) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResultBounds) DeepCopyInto(out *ResultBounds) { *out = *in diff --git a/cmd/project-signal-adapter/main.go b/cmd/project-signal-adapter/main.go index 4e68dfcf73..216370f704 100644 --- a/cmd/project-signal-adapter/main.go +++ b/cmd/project-signal-adapter/main.go @@ -52,8 +52,6 @@ func main() { flags.Var(uidValue{target: &cfg.RunUID}, "run-uid", "CertificationRun UID") flags.StringVar(&cfg.ParticipantsJSON, "participants-json", "", "Ordered participant snapshot JSON") flags.IntVar(&cfg.ParticipantCount, "participant-count", 0, "Declared participant count") - flags.StringVar(&cfg.RequestNodeName, "request-node-name", "", "Compatibility request Node name") - flags.StringVar(&cfg.LegacyParticipants, "participants", "", "Compatibility comma-separated Node names") flags.StringVar(&cfg.ScriptConfigMap, "script-configmap", "", "Immutable script ConfigMap") flags.StringVar(&cfg.ScriptKey, "script-key", "", "Script ConfigMap key") flags.StringVar( diff --git a/cmd/project-signal-adapter/main_test.go b/cmd/project-signal-adapter/main_test.go deleted file mode 100644 index 6d7e2dd1a9..0000000000 --- a/cmd/project-signal-adapter/main_test.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "os" - "path/filepath" - "testing" - - "github.com/Azure/unbounded/internal/signaladapter" -) - -func TestWriteOutputsProjectsCleanupFailure(t *testing.T) { - t.Parallel() - - directory := t.TempDir() - - output := signaladapter.AggregateOutput{ - ExecutionState: "Error", - Verdict: "Inconclusive", - Reason: "BackendExecutionFailed", - CleanupFailure: "BackendCleanupFailed", - BackendEvidence: `{"version":"v1","complete":true,"participants":[{}]}`, - } - if err := writeOutputs(directory, output); err != nil { - t.Fatal(err) - } - - for name, want := range map[string]string{ - "reason": "BackendExecutionFailed", - "cleanup-failure": "BackendCleanupFailed", - "backend-evidence": `{"version":"v1","complete":true,"participants":[{}]}`, - } { - got, err := os.ReadFile(filepath.Join(directory, name)) - if err != nil { - t.Fatal(err) - } - - if string(got) != want { - t.Fatalf("%s = %q, want %q", name, got, want) - } - } -} - -func TestWriteOutputsRejectsOversizedCleanupFailure(t *testing.T) { - t.Parallel() - - err := writeOutputs(t.TempDir(), signaladapter.AggregateOutput{ - CleanupFailure: string(make([]byte, signaladapter.MaxCleanupFailureBytes+1)), - }) - if err == nil { - t.Fatal("writeOutputs accepted oversized cleanup failure") - } -} - -func TestWriteOutputsRejectsMalformedBackendEvidence(t *testing.T) { - t.Parallel() - - err := writeOutputs(t.TempDir(), signaladapter.AggregateOutput{ - BackendEvidence: `{"version":"v1","complete":true}`, - }) - if err == nil { - t.Fatal("writeOutputs accepted malformed backend evidence") - } -} diff --git a/cmd/project-signal-controller/main.go b/cmd/project-signal-controller/main.go index 9002e90ddb..4656b1284d 100644 --- a/cmd/project-signal-controller/main.go +++ b/cmd/project-signal-controller/main.go @@ -136,16 +136,6 @@ func run(ctx context.Context, cfg config) error { return fmt.Errorf("register run reconciler: %w", err) } - recurring := &projectsignal.RecurringReconciler{ - Client: mgr.GetClient(), - Reader: mgr.GetAPIReader(), - Namespace: cfg.namespace, - Admission: admission, - } - if err := recurring.SetupWithManager(mgr); err != nil { - return fmt.Errorf("register recurring reconciler: %w", err) - } - if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { return err } diff --git a/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl b/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl index 6e203efcd2..5f3bdb8c6a 100644 --- a/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl +++ b/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl @@ -5,138 +5,14 @@ metadata: namespace: {{ default "project-signal" .Namespace }} spec: serviceAccountName: project-signal-adapter - entrypoint: certification-legacy - arguments: - parameters: - - name: run-name - - name: run-uid - - name: participants - - name: participants-json - - name: participant-count - - name: request-node-name - - name: script-configmap - - name: script-key - - name: script-digest - - name: script-requires-runtime-stage - value: "false" - - name: runtime-stage-name - value: "" - - name: backend-api-version - - name: backend-kind - - name: backend-active-deadline-seconds - - name: queue - - name: runner-image - - name: tolerations - value: "[]" - - name: script-environment - value: "{}" - - name: workload-policy - value: "{}" - - name: mpi-payload-mode - value: "" - - name: expected-gpu-product-regex - value: "" - - name: expected-gpu-count - value: "0" - - name: observed-gpu-capacity - value: "0" - - name: observed-gpu-allocatable - value: "0" - - name: gpu-resource-count - value: "0" - - name: privileged - value: "false" - - name: attempt templates: - - name: certification-legacy - steps: - - - name: certification - template: certification - arguments: - parameters: - - name: run-name - value: "{{`{{workflow.parameters.run-name}}`}}" - - name: run-uid - value: "{{`{{workflow.parameters.run-uid}}`}}" - - name: participants - value: "{{`{{workflow.parameters.participants}}`}}" - - name: participants-json - value: "{{`{{workflow.parameters.participants-json}}`}}" - - name: participant-count - value: "{{`{{workflow.parameters.participant-count}}`}}" - - name: request-node-name - value: "{{`{{workflow.parameters.request-node-name}}`}}" - - name: script-configmap - value: "{{`{{workflow.parameters.script-configmap}}`}}" - - name: script-key - value: "{{`{{workflow.parameters.script-key}}`}}" - - name: script-digest - value: "{{`{{workflow.parameters.script-digest}}`}}" - - name: script-requires-runtime-stage - value: "{{`{{workflow.parameters.script-requires-runtime-stage}}`}}" - - name: runtime-stage-name - value: "{{`{{workflow.parameters.runtime-stage-name}}`}}" - - name: backend-api-version - value: "{{`{{workflow.parameters.backend-api-version}}`}}" - - name: backend-kind - value: "{{`{{workflow.parameters.backend-kind}}`}}" - - name: backend-active-deadline-seconds - value: "{{`{{workflow.parameters.backend-active-deadline-seconds}}`}}" - - name: queue - value: "{{`{{workflow.parameters.queue}}`}}" - - name: runner-image - value: "{{`{{workflow.parameters.runner-image}}`}}" - - name: tolerations - value: "{{`{{workflow.parameters.tolerations}}`}}" - - name: script-environment - value: "{{`{{workflow.parameters.script-environment}}`}}" - - name: workload-policy - value: "{{`{{workflow.parameters.workload-policy}}`}}" - - name: mpi-payload-mode - value: "{{`{{workflow.parameters.mpi-payload-mode}}`}}" - - name: expected-gpu-product-regex - value: "{{`{{workflow.parameters.expected-gpu-product-regex}}`}}" - - name: expected-gpu-count - value: "{{`{{workflow.parameters.expected-gpu-count}}`}}" - - name: observed-gpu-capacity - value: "{{`{{workflow.parameters.observed-gpu-capacity}}`}}" - - name: observed-gpu-allocatable - value: "{{`{{workflow.parameters.observed-gpu-allocatable}}`}}" - - name: gpu-resource-count - value: "{{`{{workflow.parameters.gpu-resource-count}}`}}" - - name: privileged - value: "{{`{{workflow.parameters.privileged}}`}}" - - name: attempt - value: "{{`{{workflow.parameters.attempt}}`}}" - outputs: - parameters: - - name: verdict - valueFrom: - parameter: "{{`{{steps.certification.outputs.parameters.verdict}}`}}" - - name: reason - valueFrom: - parameter: "{{`{{steps.certification.outputs.parameters.reason}}`}}" - - name: cleanup-failure - valueFrom: - parameter: "{{`{{steps.certification.outputs.parameters.cleanup-failure}}`}}" - - name: measurements - valueFrom: - parameter: "{{`{{steps.certification.outputs.parameters.measurements}}`}}" - - name: backend-resources - valueFrom: - parameter: "{{`{{steps.certification.outputs.parameters.backend-resources}}`}}" - - name: backend-evidence - valueFrom: - parameter: "{{`{{steps.certification.outputs.parameters.backend-evidence}}`}}" - name: certification inputs: parameters: - name: run-name - name: run-uid - - name: participants - name: participants-json - name: participant-count - - name: request-node-name - name: script-configmap - name: script-key - name: script-digest @@ -170,14 +46,10 @@ spec: - "{{`{{inputs.parameters.run-name}}`}}" - --run-uid - "{{`{{inputs.parameters.run-uid}}`}}" - - --participants - - "{{`{{inputs.parameters.participants}}`}}" - --participants-json - "{{`{{inputs.parameters.participants-json}}`}}" - --participant-count - "{{`{{inputs.parameters.participant-count}}`}}" - - --request-node-name - - "{{`{{inputs.parameters.request-node-name}}`}}" - --script-configmap - "{{`{{inputs.parameters.script-configmap}}`}}" - --script-key diff --git a/deploy/signal/README.md b/deploy/signal/README.md index 24af8e79c8..2965681bba 100644 --- a/deploy/signal/README.md +++ b/deploy/signal/README.md @@ -62,10 +62,8 @@ The referenced WorkflowTemplate must accept these string parameters: | --- | --- | | `run-name` | CertificationRun name | | `run-uid` | CertificationRun UID | -| `participants` | Comma-separated admitted Node names retained for compatibility | | `participants-json` | Ordered JSON array of participant indexes, Node names, immutable UIDs, capacity, and allocatable resources | | `participant-count` | Number of atomically admitted participants | -| `request-node-name` | Node that triggered the grouped run | | `script-configmap` | Immutable run-owned ConfigMap | | `script-key` | Selected ConfigMap key | | `script-digest` | Algorithm-qualified SHA-256 of the selected script bytes (`sha256:<64 lowercase hex>`) | @@ -379,7 +377,7 @@ task invokes the configured adapter `certification` template through `templateRef`; dependencies make execution serial and visible in `status.nodes`. `executionPolicy.onFailure` is immutable and is either `Stop` or `Continue`. A stopped task is recorded as skipped/not-run, never inferred as -a diagnostic result. Grouped suites cannot enable recurring admission. +a diagnostic result. Participant roles and topology constraints are generic typed policy. Profiles can, for example, bind one role to one participant, another role to three diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml index d8c5d91450..6f5f5ecea7 100644 --- a/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml @@ -25,9 +25,6 @@ spec: - jsonPath: .spec.participants.count name: Participants type: integer - - jsonPath: .spec.recurring.enabled - name: Recurring - type: boolean name: v1alpha1 schema: openAPIV3Schema: @@ -742,53 +739,6 @@ spec: - TargetedVerification - Diagnostic type: string - recurring: - properties: - enabled: - type: boolean - evaluationInterval: - type: string - exclusiveProfiles: - items: - type: string - type: array - expiresAt: - format: date-time - type: string - failureBackoff: - type: string - failureDomainLabel: - type: string - maxConcurrent: - format: int32 - type: integer - maxConcurrentPerFailureDomain: - format: int32 - type: integer - maxConsecutiveFailures: - format: int32 - type: integer - minIdleDuration: - type: string - nodeNames: - items: - type: string - type: array - notBefore: - format: date-time - type: string - validFor: - type: string - required: - - enabled - - expiresAt - - failureBackoff - - maxConcurrent - - maxConsecutiveFailures - - minIdleDuration - - nodeNames - - validFor - type: object tolerations: items: description: |- @@ -879,104 +829,9 @@ spec: before deadline rule: self.checks.all(c, duration(self.deadline) - duration(c.backendActiveDeadline) >= duration('60s')) - - message: recurring certification is not supported for grouped suites - rule: '!has(self.recurring) || !self.recurring.enabled' - status: - properties: - conditions: - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - nodes: - items: - properties: - consecutiveFailures: - format: int32 - type: integer - idleSince: - format: date-time - type: string - lastEvaluatedAt: - format: date-time - type: string - lastSuccessfulAt: - format: date-time - type: string - lastSuccessfulRun: - type: string - nextDueAt: - format: date-time - type: string - nodeName: - type: string - reason: - type: string - suspended: - type: boolean - required: - - nodeName - type: object - type: array - observedGeneration: - format: int64 - type: integer - type: object required: - spec type: object served: true storage: true - subresources: - status: {} + subresources: {} diff --git a/deploy/signal/manifest_test.go b/deploy/signal/manifest_test.go deleted file mode 100644 index 2de78cef39..0000000000 --- a/deploy/signal/manifest_test.go +++ /dev/null @@ -1,294 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -package signalmanifest_test - -import ( - "os" - "path/filepath" - "reflect" - "regexp" - "sort" - "strings" - "testing" -) - -func TestUpstreamManifestsContainControllerAndGenericAdapter(t *testing.T) { - t.Parallel() - - templates, err := filepath.Glob("*.yaml.tmpl") - if err != nil { - t.Fatal(err) - } - - sort.Strings(templates) - - want := []string{ - "00-namespace.yaml.tmpl", - "01-serviceaccounts.yaml.tmpl", - "02-rbac.yaml.tmpl", - "03-adapter-serviceaccount-rbac.yaml.tmpl", - "04-adapter-workflowtemplate.yaml.tmpl", - "06-deployment.yaml.tmpl", - } - if !reflect.DeepEqual(templates, want) { - t.Fatalf("manifest templates = %v, want %v", templates, want) - } - - for _, path := range []string{"01-serviceaccounts.yaml.tmpl", "02-rbac.yaml.tmpl"} { - content, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - - manifest := string(content) - if strings.Contains(manifest, "project-signal-runner") || - strings.Contains(manifest, "kubeflow.org") { - t.Fatalf("%s contains bundled runner surface", path) - } - } -} - -func TestAdapterManifestIsGenericAndParticipantAware(t *testing.T) { - t.Parallel() - - content, err := os.ReadFile("04-adapter-workflowtemplate.yaml.tmpl") - if err != nil { - t.Fatal(err) - } - - manifest := string(content) - for _, required := range []string{ - "participants-json", - "participant-count", - "request-node-name", - "backend-resources", - "backend-evidence", - "/outputs/backend-evidence", - "cleanup-failure", - "/outputs/cleanup-failure", - "backend-api-version", - "backend-kind", - "backend-active-deadline-seconds", - "queue", - "runtime-stage-name", - "script-requires-runtime-stage", - "workload-policy", - } { - if !strings.Contains(manifest, required) { - t.Fatalf("adapter WorkflowTemplate does not contain %q", required) - } - } - - for _, forbidden := range []string{ - "project-signal-certification/", - "REPLACE_WITH_", - "internal-node", - } { - if strings.Contains(manifest, forbidden) { - t.Fatalf("adapter WorkflowTemplate contains downstream-specific value %q", forbidden) - } - } -} - -func TestAdapterTemplateResolvesGroupedTaskArgumentsThroughInputs(t *testing.T) { - t.Parallel() - - content, err := os.ReadFile("04-adapter-workflowtemplate.yaml.tmpl") - if err != nil { - t.Fatal(err) - } - - manifest := string(content) - if !strings.Contains(manifest, "entrypoint: certification-legacy") { - t.Fatal("adapter WorkflowTemplate does not preserve the legacy workflow entrypoint") - } - - const certificationMarker = " - name: certification\n inputs:\n" - - certificationIndex := strings.LastIndex(manifest, certificationMarker) - if certificationIndex < 0 { - t.Fatal("adapter WorkflowTemplate does not declare certification template inputs") - } - - certification := manifest[certificationIndex:] - if strings.Contains(certification, "workflow.parameters.") { - t.Fatal("grouped certification template references undeclared workflow parameters") - } - - inputReference := regexp.MustCompile(`inputs\.parameters\.([a-z0-9-]+)`) - for _, match := range inputReference.FindAllStringSubmatch(certification, -1) { - declaration := " - name: " + match[1] - if !strings.Contains(certification, declaration) { - t.Fatalf("certification template references undeclared input %q", match[1]) - } - } - - legacy := manifest[:certificationIndex] - for _, required := range []string{ - "value: \"{{`{{workflow.parameters.run-name}}`}}\"", - "parameter: \"{{`{{steps.certification.outputs.parameters.verdict}}`}}\"", - "parameter: \"{{`{{steps.certification.outputs.parameters.backend-evidence}}`}}\"", - } { - if !strings.Contains(legacy, required) { - t.Fatalf("legacy adapter wrapper does not contain %q", required) - } - } -} - -func TestAdapterRBACAllowsBackendEvidenceCapture(t *testing.T) { - t.Parallel() - - content, err := os.ReadFile("03-adapter-serviceaccount-rbac.yaml.tmpl") - if err != nil { - t.Fatal(err) - } - - manifest := string(content) - for _, required := range []string{ - `resources: ["pods"]`, - `resources: ["pods/log"]`, - `apiGroups: ["batch"]`, - `resources: ["jobs"]`, - `verbs: ["get", "list"]`, - } { - if !strings.Contains(manifest, required) { - t.Fatalf("adapter RBAC does not contain %q", required) - } - } -} - -func TestSignalCRDsExposeGroupedRuntimeAndDeadlineContract(t *testing.T) { - t.Parallel() - - for _, path := range []string{ - "crd/signal.unbounded-cloud.io_certificationprofiles.yaml", - "crd/signal.unbounded-cloud.io_certificationruns.yaml", - } { - content, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - - manifest := string(content) - for _, required := range []string{ - "checks:", - "maxItems: 15", - "minItems: 15", - "executionPolicy:", - "participantRoles:", - "topologyConstraints:", - "resultBounds:", - "backendActiveDeadline:", - "runtimeStage:", - "requiresRuntimeStage:", - "every backendActiveDeadline must leave at least 60 seconds", - "runtimeStage is required when the script requires a runtime", - "maxAttempts must be exactly 1", - "checks must contain unique IDs", - "maxLength: 63", - } { - if !strings.Contains(manifest, required) { - t.Fatalf("%s does not contain %q", path, required) - } - - profileContent, err := os.ReadFile("crd/signal.unbounded-cloud.io_certificationprofiles.yaml") - if err != nil { - t.Fatal(err) - } - - if !strings.Contains(string(profileContent), "recurring certification is not supported for grouped suites") { - t.Fatal("CertificationProfile CRD does not reject grouped recurrence") - } - - runContent, err := os.ReadFile("crd/signal.unbounded-cloud.io_certificationruns.yaml") - if err != nil { - t.Fatal(err) - } - - for _, required := range []string{"backendEvidence:", "workflowNodeName:", "retryCount:", "notRun:"} { - if !strings.Contains(string(runContent), required) { - t.Fatalf("CertificationRun CRD does not contain per-check status field %q", required) - } - } - } - } -} - -func TestCertificationRunCRDMakesGroupedSnapshotImmutable(t *testing.T) { - t.Parallel() - - content, err := os.ReadFile("crd/signal.unbounded-cloud.io_certificationruns.yaml") - if err != nil { - t.Fatal(err) - } - - manifest := string(content) - requiredRules := []string{ - "checks are immutable", - "participants are immutable", - "participantRoles are immutable", - "topologyConstraints are immutable", - "executionPolicy is immutable", - "workflow is immutable", - } - - for _, rule := range requiredRules { - if !strings.Contains(manifest, rule) { - t.Fatalf("CertificationRun CRD does not contain rule %q", rule) - } - } -} - -func TestCertificationRunCRDRequiresCanonicalDigests(t *testing.T) { - t.Parallel() - - content, err := os.ReadFile("crd/signal.unbounded-cloud.io_certificationruns.yaml") - if err != nil { - t.Fatal(err) - } - - const digestPattern = "pattern: ^sha256:[0-9a-f]{64}$" - if count := strings.Count(string(content), digestPattern); count != 3 { - t.Fatalf("canonical digest patterns = %d, want 3", count) - } -} - -func TestSignalCRDsExposeBoundedKueueTopologyPolicy(t *testing.T) { - t.Parallel() - - for _, path := range []string{ - "crd/signal.unbounded-cloud.io_certificationprofiles.yaml", - "crd/signal.unbounded-cloud.io_certificationruns.yaml", - } { - content, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - - manifest := string(content) - for _, required := range []string{ - "kueueTopology:", - "requiredLevel:", - "maxLength: 316", - "minLength: 1", - } { - if !strings.Contains(manifest, required) { - t.Fatalf("%s does not contain %q", path, required) - } - } - } -} - -func TestUpstreamDoesNotPublishCertificationPackage(t *testing.T) { - t.Parallel() - - for _, path := range []string{ - "../../project-signal-certification", - "../../project-signal-certification-runner", - } { - if _, err := os.Stat(path); !os.IsNotExist(err) { - t.Fatalf("%s must not be published upstream: %v", path, err) - } - } -} diff --git a/internal/signal/controller.go b/internal/signal/controller.go index bc8f91f7ac..ecda2070f2 100644 --- a/internal/signal/controller.go +++ b/internal/signal/controller.go @@ -50,7 +50,6 @@ const ( RunTriggerAnnotation = "signal.unbounded-cloud.io/trigger" RunFinalizer = "signal.unbounded-cloud.io/cleanup" SnapshotRunLabel = "signal.unbounded-cloud.io/run" - DefaultMaxAttempts = int32(1) ) var runnerImageDigestPattern = regexp.MustCompile(`^[^@\s]+@sha256:[0-9a-f]{64}$`) @@ -231,7 +230,7 @@ func (r *AdmissionReconciler) admitNodeWithTriggerLocked( } checks := profileChecks(&profile) - if err := validateCheckPlan(checks, len(profile.Spec.Checks) == groupedCheckCount); err != nil { + if err := validateCheckPlan(checks); err != nil { return fmt.Errorf("profile %s checks: %w", profile.Name, err) } @@ -350,11 +349,6 @@ func (r *AdmissionReconciler) admitNodeWithTriggerLocked( RequestNodeName: node.Name, }, } - if len(profile.Spec.Checks) != groupedCheckCount { - setLegacyRunAliases(&run.Spec) - run.Spec.StagePlan = append([]signalv1.CertificationStage(nil), profile.Spec.StagePlan...) - } - if err := r.Create(ctx, run); err != nil { deleteErr := r.Delete(context.WithoutCancel(ctx), snapshot) @@ -514,40 +508,6 @@ func participantSnapshot(node *corev1.Node, index int32) signalv1.Participant { } } -func (r *AdmissionReconciler) assignParticipantRoles( - ctx context.Context, - participants []signalv1.Participant, - roles []signalv1.ParticipantRoleConstraint, - topology []signalv1.ParticipantTopologyConstraint, -) error { - if len(roles) == 0 { - return nil - } - - reader := r.APIReader - if reader == nil { - reader = r.Client - } - - nodes := make([]corev1.Node, len(participants)) - for index := range participants { - if err := reader.Get(ctx, client.ObjectKey{Name: participants[index].NodeName}, &nodes[index]); err != nil { - return fmt.Errorf("read participant %s for role assignment: %w", participants[index].NodeName, err) - } - } - - assignments, err := solveParticipantRoles(nodes, "", int32(len(nodes)), roles, topology) - if err != nil { - return err - } - - for index := range participants { - participants[index].Roles = assignments[index] - } - - return nil -} - type participantRoleSlot struct { name string } @@ -1040,9 +1000,6 @@ func (r *AdmissionReconciler) createSnapshot( } key := fmt.Sprintf("%02d-%s", check.Order, sanitizeTaskName(string(check.ID))) - if len(checks) == 1 && check.ID == "legacy" { - key = check.Script.Key - } snapshot.Data[key] = content checkSnapshots[index] = signalv1.CertificationCheckSnapshot{ @@ -1087,24 +1044,13 @@ func (r *RunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R return r.cleanup(ctx, &run) } - if isGroupedRun(&run) && run.Status.Evidence.URI == "" { + if run.Status.Evidence.URI == "" { result, recovered, err := r.resumeGroupedEvidence(ctx, &run) if err != nil || recovered { return result, err } } - if run.Status.FinishedAt != nil && !isGroupedRun(&run) { - return r.finish( - ctx, - &run, - nil, - run.Status.ExecutionState, - run.Status.Verdict, - run.Status.Reason, - ) - } - if run.Spec.Cancel { return r.cancel(ctx, &run) } @@ -1123,17 +1069,13 @@ func (r *RunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R if err := reader.Get(ctx, client.ObjectKey{Namespace: run.Namespace, Name: run.Status.Workflow.Name}, workflow); err != nil { if apierrors.IsNotFound(err) { - if isGroupedRun(&run) { - workflow.SetName(run.Status.Workflow.Name) - workflow.Object["status"] = map[string]any{ - "phase": "Error", - "message": "workflow disappeared before terminal status", - } - - return r.finishGrouped(ctx, &run, workflow, "Error") + workflow.SetName(run.Status.Workflow.Name) + workflow.Object["status"] = map[string]any{ + "phase": "Error", + "message": "workflow disappeared before terminal status", } - return r.retryOrFinish(ctx, &run, "Error", "WorkflowMissing", "workflow disappeared before terminal status") + return r.finishGrouped(ctx, &run, workflow, "Error") } return ctrl.Result{}, err @@ -1159,10 +1101,7 @@ func (r *RunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R return ctrl.Result{}, err } - checkStatuses := run.Status.Checks - if isGroupedRun(&run) { - checkStatuses = runningCheckStatuses(workflow, &run) - } + checkStatuses := runningCheckStatuses(workflow, &run) if run.Status.Phase != signalv1.RunPhaseRunning || run.Status.Workflow.Phase != phase || @@ -1183,25 +1122,9 @@ func (r *RunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R return ctrl.Result{RequeueAfter: 15 * time.Second}, nil case "Succeeded": - if isGroupedRun(&run) { - return r.finishGrouped(ctx, &run, workflow, phase) - } - - verdict := workflowParameter(workflow, "verdict") - reason := workflowParameter(workflow, "reason") - - if verdict == "" { - verdict = "Inconclusive" - reason = "workflow completed without a verdict" - } - - return r.finish(ctx, &run, workflow, "Completed", verdict, reason) + return r.finishGrouped(ctx, &run, workflow, phase) case "Failed", "Error": - if isGroupedRun(&run) { - return r.finishGrouped(ctx, &run, workflow, phase) - } - - return r.retryOrFinish(ctx, &run, phase, "Workflow"+phase, workflowMessage(workflow)) + return r.finishGrouped(ctx, &run, workflow, phase) default: return ctrl.Result{}, fmt.Errorf("unsupported Workflow phase %q", phase) } @@ -1242,9 +1165,7 @@ func (r *RunReconciler) startAttempt(ctx context.Context, run *signalv1.Certific Attempt: attempt, Phase: "Pending", } - if isGroupedRun(run) { - run.Status.Checks = initialCheckStatuses(run.Spec.Checks) - } + run.Status.Checks = initialCheckStatuses(run.Spec.Checks) if run.Status.StartedAt == nil { now := metav1.NewTime(r.now()) @@ -1264,12 +1185,8 @@ func (r *RunReconciler) startAttempt(ctx context.Context, run *signalv1.Certific } func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured.Unstructured, error) { - if !isGroupedRun(run) { - return buildLegacyWorkflow(run, attempt) - } - - checks := runChecks(run) - if err := validateRunCheckPlan(checks, isGroupedRun(run)); err != nil { + checks := run.Spec.Checks + if err := validateRunCheckPlan(checks); err != nil { return nil, err } @@ -1298,11 +1215,6 @@ func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured return nil, fmt.Errorf("check %s participants: %w", check.ID, participantErr) } - participantNames := make([]string, 0, len(checkParticipants)) - for participantIndex := range checkParticipants { - participantNames = append(participantNames, checkParticipants[participantIndex].NodeName) - } - participantsJSON, marshalErr := json.Marshal(checkParticipants) if marshalErr != nil { return nil, fmt.Errorf("encode check %s participants: %w", check.ID, marshalErr) @@ -1336,7 +1248,6 @@ func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured check, attempt, checkParticipants, - participantNames, string(participantsJSON), string(tolerationsJSON), string(environmentJSON), @@ -1405,103 +1316,11 @@ func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured return workflow, nil } -func buildLegacyWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured.Unstructured, error) { - checks := runChecks(run) - if len(checks) != 1 { - return nil, errors.New("legacy workflow requires exactly one check") - } - - check := &checks[0] - - participantNames := make([]string, 0, len(run.Spec.Participants)) - for _, participant := range run.Spec.Participants { - participantNames = append(participantNames, participant.NodeName) - } - - tolerations := uniqueTolerations(append([]corev1.Toleration{{ - Key: "nvidia.com/gpu", - Operator: corev1.TolerationOpExists, - Effect: corev1.TaintEffectNoSchedule, - }}, run.Spec.Tolerations...)) - - tolerationsJSON, err := json.Marshal(tolerations) - if err != nil { - return nil, fmt.Errorf("encode Workflow tolerations: %w", err) - } - - environment, err := scriptEnvironment(check.Parameters) - if err != nil { - return nil, err - } - - environmentJSON, err := json.Marshal(environment) - if err != nil { - return nil, fmt.Errorf("encode script environment: %w", err) - } - - workloadPolicyJSON, err := signalworkload.Encode(check.WorkloadPolicy) - if err != nil { - return nil, fmt.Errorf("encode workload policy: %w", err) - } - - participantsJSON, err := json.Marshal(run.Spec.Participants) - if err != nil { - return nil, fmt.Errorf("encode Workflow participants: %w", err) - } - - name := fmt.Sprintf("%s-a%d", run.Name, attempt) - workflow := &unstructured.Unstructured{Object: map[string]any{ - "apiVersion": workflowGVK.GroupVersion().String(), - "kind": workflowGVK.Kind, - "metadata": map[string]any{ - "name": name, - "namespace": run.Namespace, - "labels": map[string]any{ - SnapshotRunLabel: run.Name, - }, - }, - "spec": map[string]any{ - "activeDeadlineSeconds": int64(run.Spec.Deadline.Seconds()), - "ttlStrategy": map[string]any{ - "secondsAfterCompletion": int64(6 * time.Hour / time.Second), - "secondsAfterSuccess": int64(6 * time.Hour / time.Second), - "secondsAfterFailure": int64(6 * time.Hour / time.Second), - }, - "podGC": map[string]any{ - "strategy": "OnWorkflowCompletion", - "deleteDelayDuration": "2h", - }, - "workflowTemplateRef": map[string]any{ - "name": run.Spec.Workflow.Name, - }, - "arguments": map[string]any{ - "parameters": checkWorkflowParameters( - run, - check, - attempt, - run.Spec.Participants, - participantNames, - string(participantsJSON), - string(tolerationsJSON), - string(environmentJSON), - workloadPolicyJSON, - ), - }, - }, - }} - workflow.SetOwnerReferences([]metav1.OwnerReference{ - *metav1.NewControllerRef(run, signalv1.GroupVersion.WithKind("CertificationRun")), - }) - - return workflow, nil -} - func checkWorkflowParameters( run *signalv1.CertificationRun, check *signalv1.CertificationCheckSnapshot, attempt int32, participants []signalv1.Participant, - participantNames []string, participantsJSON string, tolerationsJSON string, scriptEnvironmentJSON string, @@ -1510,10 +1329,8 @@ func checkWorkflowParameters( return []any{ map[string]any{"name": "run-name", "value": checkRunName(run.Name, check.ID)}, map[string]any{"name": "run-uid", "value": string(run.UID)}, - map[string]any{"name": "participants", "value": strings.Join(participantNames, ",")}, map[string]any{"name": "participants-json", "value": participantsJSON}, map[string]any{"name": "participant-count", "value": fmt.Sprintf("%d", len(participants))}, - map[string]any{"name": "request-node-name", "value": run.Spec.RequestNodeName}, map[string]any{"name": "script-configmap", "value": check.Script.ConfigMapName}, map[string]any{"name": "script-key", "value": check.Script.Key}, map[string]any{"name": "script-digest", "value": check.Script.Digest}, @@ -1597,25 +1414,6 @@ func scriptEnvironment(parameters map[string]string) (map[string]string, error) return environment, nil } -func (r *RunReconciler) retryOrFinish( - ctx context.Context, - run *signalv1.CertificationRun, - executionState string, - reason string, - message string, -) (ctrl.Result, error) { - maxAttempts := run.Spec.MaxAttempts - if checks := runChecks(run); len(checks) == 1 { - maxAttempts = checks[0].MaxAttempts - } - - if run.Status.Workflow.Attempt < maxAttempts { - return r.startAttempt(ctx, run, run.Status.Workflow.Attempt+1) - } - - return r.finish(ctx, run, nil, executionState, "Inconclusive", reason+": "+message) -} - func (r *RunReconciler) finishGrouped( ctx context.Context, run *signalv1.CertificationRun, @@ -2161,116 +1959,6 @@ func nodeString(node map[string]any, key string) string { return value } -func (r *RunReconciler) finish( - ctx context.Context, - run *signalv1.CertificationRun, - workflow *unstructured.Unstructured, - executionState string, - verdict string, - reason string, -) (ctrl.Result, error) { - if run.Status.FinishedAt == nil { - before := run.DeepCopy() - run.Status.ExecutionState = executionState - run.Status.Verdict = verdict - finalReason := reason - - if workflow != nil { - outputs, err := workflowParameters(workflow) - if err != nil { - run.Status.Verdict = "Inconclusive" - finalReason = "workflow outputs rejected: " + err.Error() - } else { - run.Status.Workflow.Outputs = outputs - - evidenceComplete, evidenceErr := backendEvidenceComplete( - outputs["backend-evidence"], - len(run.Spec.Participants), - ) - if evidenceErr != nil { - run.Status.Verdict = "Inconclusive" - finalReason = "workflow backend evidence rejected: " + evidenceErr.Error() - } else if run.Status.Verdict == "Pass" && !evidenceComplete { - run.Status.Verdict = "Inconclusive" - finalReason = "workflow backend evidence capture was incomplete" - } - - backendResources, backendErr := backendResourceReferences( - outputs["backend-resources"], - len(run.Spec.Participants), - ) - if backendErr != nil { - run.Status.Verdict = "Inconclusive" - finalReason = "workflow backend resources rejected: " + backendErr.Error() - } else { - run.Status.Workflow.BackendResources = backendResources - if run.Status.Verdict == "Pass" && - len(run.Spec.Participants) > 1 && - len(backendResources) == 0 { - run.Status.Verdict = "Inconclusive" - finalReason = "workflow did not identify backend resources for all participants" - } - } - } - } - - run.Status.Reason = finalReason - if executionState == "Completed" { - run.Status.Workflow.Phase = "Succeeded" - } else { - run.Status.Workflow.Phase = executionState - } - - now := metav1.NewTime(r.now()) - run.Status.FinishedAt = &now - - if err := r.Status().Patch(ctx, run, client.MergeFrom(before)); err != nil { - return ctrl.Result{}, err - } - } - - executionState = run.Status.ExecutionState - verdict = run.Status.Verdict - reason = run.Status.Reason - - evidence, err := r.evidenceDocument(ctx, run, executionState, verdict, reason) - if err != nil { - return ctrl.Result{}, err - } - - uri, digest, err := r.Evidence.Put(ctx, run.Spec.EvidencePath, evidence) - if err != nil { - return ctrl.Result{}, err - } - - before := run.DeepCopy() - run.Status.Evidence = signalv1.EvidenceStatus{URI: uri, Digest: digest} - - switch verdict { - case "Pass": - run.Status.Phase = signalv1.RunPhaseSucceeded - case "Fail", "Suspect": - run.Status.Phase = signalv1.RunPhaseFailed - default: - run.Status.Phase = signalv1.RunPhaseInconclusive - } - - if err := r.Status().Patch(ctx, run, client.MergeFrom(before)); err != nil { - return ctrl.Result{}, err - } - - if err := projectTerminal(ctx, r.Client, run); err != nil { - return ctrl.Result{}, err - } - - if r.Recorder != nil { - r.Recorder.Eventf(run, nil, corev1.EventTypeNormal, "CertificationFinished", "FinishCertification", - "ExecutionState=%s Verdict=%s Reason=%s", executionState, verdict, reason) - } - - return ctrl.Result{RequeueAfter: time.Millisecond}, nil -} - func uniqueTolerations(tolerations []corev1.Toleration) []corev1.Toleration { unique := make([]corev1.Toleration, 0, len(tolerations)) for _, toleration := range tolerations { @@ -2299,52 +1987,7 @@ func (r *RunReconciler) evidenceDocument( verdict string, reason string, ) ([]byte, error) { - if !isGroupedRun(run) { - checks := runChecks(run) - if len(checks) != 1 { - return nil, errors.New("legacy run must contain one check") - } - - check := &checks[0] - - var snapshot corev1.ConfigMap - if err := r.Get( - ctx, - client.ObjectKey{Namespace: run.Namespace, Name: check.Script.ConfigMapName}, - &snapshot, - ); err != nil { - return nil, fmt.Errorf("read script snapshot for evidence: %w", err) - } - - doc := map[string]any{ - "apiVersion": "signal.unbounded-cloud.io/evidence/v1alpha1", - "runID": string(run.UID), - "runName": run.Name, - "profile": run.Spec.Profile, - "participants": run.Spec.Participants, - "workflow": run.Status.Workflow, - "workflowOutputs": run.Status.Workflow.Outputs, - "backend": check.Backend, - "runnerImage": check.RunnerImage, - "workloadPolicy": check.WorkloadPolicy, - "purpose": run.Spec.Purpose, - "stagePlan": run.Spec.StagePlan, - "runtimeStage": check.RuntimeStage, - "backendActiveDeadline": check.BackendActiveDeadline, - "parameters": check.Parameters, - "scriptSnapshot": check.Script, - "scriptContent": snapshot.Data[check.Script.Key], - "executionState": executionState, - "verdict": verdict, - "reason": reason, - "startedAt": run.Status.StartedAt, - "finishedAt": run.Status.FinishedAt.UTC().Format(time.RFC3339Nano), - } - - return json.MarshalIndent(doc, "", " ") - } - - checks := runChecks(run) + checks := run.Spec.Checks if len(checks) == 0 { return nil, errors.New("run has no certification checks") } @@ -2500,23 +2143,6 @@ func backendResourceReferencesForParticipants( return references, nil } -func workflowParameters(workflow *unstructured.Unstructured) (map[string]string, error) { - parameters, found, err := unstructured.NestedSlice(workflow.Object, "status", "outputs", "parameters") - if err != nil { - return nil, fmt.Errorf("read top-level parameters: %w", err) - } - - if found { - return decodeWorkflowParameters(parameters) - } - - return entrypointWorkflowParameters(workflow) -} - -func decodeWorkflowParameters(parameters []any) (map[string]string, error) { - return decodeWorkflowParametersWithLimits(parameters, workflowOutputLimits) -} - func decodeWorkflowParametersWithLimits(parameters []any, limits map[string]int) (map[string]string, error) { outputs := make(map[string]string, len(parameters)) for _, raw := range parameters { @@ -2551,130 +2177,6 @@ func decodeWorkflowParametersWithLimits(parameters []any, limits map[string]int) return outputs, nil } -func entrypointWorkflowParameters(workflow *unstructured.Unstructured) (map[string]string, error) { - workflowName := workflow.GetName() - if workflowName == "" { - return nil, errors.New("workflow name is required to identify entrypoint outputs") - } - - nodes, found, err := unstructured.NestedMap(workflow.Object, "status", "nodes") - if err != nil { - return nil, fmt.Errorf("read workflow nodes: %w", err) - } - - if !found { - return nil, errors.New("top-level outputs are absent and workflow nodes are missing") - } - - var entrypoint map[string]any - - entrypointCount := 0 - rootKeyMismatch := "" - - for key, raw := range nodes { - node, ok := raw.(map[string]any) - if !ok { - if key == workflowName { - return nil, fmt.Errorf("entrypoint node %q is not an object", key) - } - - continue - } - - nodeName, nameOK := node["name"].(string) - if key == workflowName && !nameOK { - return nil, fmt.Errorf("entrypoint node %q requires a string name", key) - } - - if key == workflowName && nodeName != workflowName { - rootKeyMismatch = nodeName - } - - if nameOK && nodeName == workflowName { - entrypoint = node - entrypointCount++ - } - } - - if rootKeyMismatch != "" && entrypointCount > 0 { - return nil, fmt.Errorf( - "entrypoint identity is ambiguous: node key %q names %q while another node names the workflow", - workflowName, - rootKeyMismatch, - ) - } - - if rootKeyMismatch != "" { - return nil, fmt.Errorf( - "entrypoint node %q has mismatched name %q", - workflowName, - rootKeyMismatch, - ) - } - - if entrypointCount == 0 { - return nil, fmt.Errorf("entrypoint node named %q was not found", workflowName) - } - - if entrypointCount > 1 { - return nil, fmt.Errorf("entrypoint node named %q is duplicated", workflowName) - } - - outputs, ok := entrypoint["outputs"].(map[string]any) - if !ok { - return nil, fmt.Errorf("entrypoint node %q outputs are missing or malformed", workflowName) - } - - parameters, ok := outputs["parameters"].([]any) - if !ok { - return nil, fmt.Errorf("entrypoint node %q parameters are missing or malformed", workflowName) - } - - return decodeWorkflowParameters(parameters) -} - -func backendEvidenceComplete(content string, participantCount int) (bool, error) { - if content == "" { - return false, errors.New("backend-evidence is required") - } - - var evidence struct { - Version string `json:"version"` - Complete bool `json:"complete"` - Participants []any `json:"participants"` - } - - decoder := json.NewDecoder(strings.NewReader(content)) - decoder.DisallowUnknownFields() - - if err := decoder.Decode(&evidence); err != nil { - return false, fmt.Errorf("decode backend-evidence: %w", err) - } - - var trailing any - if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { - if err == nil { - return false, errors.New("backend-evidence contains trailing JSON values") - } - - return false, fmt.Errorf("decode trailing backend-evidence value: %w", err) - } - - if evidence.Version != "v1" { - return false, fmt.Errorf("backend-evidence version %q is invalid", evidence.Version) - } - - if len(evidence.Participants) != participantCount { - return false, fmt.Errorf( - "backend-evidence contains %d participants, want %d", - len(evidence.Participants), - participantCount, - ) - } - - return evidence.Complete, nil -} - func groupedBackendEvidenceComplete(content string, participantCount int) (bool, error) { if content == "" { return false, errors.New("backend-evidence is required") @@ -2819,28 +2321,27 @@ func (r *RunReconciler) cancel(ctx context.Context, run *signalv1.CertificationR run.Status.Verdict = "Inconclusive" run.Status.Reason = "canceled by operator" - if isGroupedRun(run) { - statuses := run.Status.Checks - if len(statuses) != len(run.Spec.Checks) { - statuses = initialCheckStatuses(run.Spec.Checks) - } - for index := range statuses { - if statuses[index].Phase == signalv1.CheckPhaseSucceeded || - statuses[index].Phase == signalv1.CheckPhaseFailed || - statuses[index].Phase == signalv1.CheckPhaseInconclusive { - continue - } + statuses := run.Status.Checks + if len(statuses) != len(run.Spec.Checks) { + statuses = initialCheckStatuses(run.Spec.Checks) + } - statuses[index].Phase = signalv1.CheckPhaseSkipped - statuses[index].Verdict = "Inconclusive" - statuses[index].Reason = "Canceled" - statuses[index].NotRun = true + for index := range statuses { + if statuses[index].Phase == signalv1.CheckPhaseSucceeded || + statuses[index].Phase == signalv1.CheckPhaseFailed || + statuses[index].Phase == signalv1.CheckPhaseInconclusive { + continue } - run.Status.Checks = statuses + statuses[index].Phase = signalv1.CheckPhaseSkipped + statuses[index].Verdict = "Inconclusive" + statuses[index].Reason = "Canceled" + statuses[index].NotRun = true } + run.Status.Checks = statuses + now := metav1.NewTime(r.now()) run.Status.FinishedAt = &now @@ -2863,7 +2364,7 @@ func (r *RunReconciler) cleanup(ctx context.Context, run *signalv1.Certification deleted := false - checks := runChecks(run) + checks := run.Spec.Checks if len(checks) == 0 { return ctrl.Result{}, errors.New("run has no certification checks") } @@ -3043,15 +2544,6 @@ func participantNodeUID(run *signalv1.CertificationRun, nodeName string) types.U return "" } -func workflowParameter(workflow *unstructured.Unstructured, name string) string { - outputs, err := workflowParameters(workflow) - if err != nil { - return "" - } - - return outputs[name] -} - func workflowMessage(workflow *unstructured.Unstructured) string { message, _, err := unstructured.NestedString(workflow.Object, "status", "message") if err != nil { @@ -3130,109 +2622,26 @@ func removeString(values []string, remove string) []string { return result } -func validateStagePlan(plan []signalv1.CertificationStage) error { - seen := map[string]struct{}{} - - for i := range plan { - stage := &plan[i] - if stage.ID == "" { - return fmt.Errorf("stage %d has an empty ID", i) - } - - if stage.DisplayName == "" { - return fmt.Errorf("stage %q has an empty display name", stage.ID) - } - - if _, ok := seen[stage.ID]; ok { - return fmt.Errorf("duplicate stage ID %q", stage.ID) - } - - seen[stage.ID] = struct{}{} - } - - return nil -} - func profileChecks(profile *signalv1.CertificationProfile) []signalv1.CertificationCheck { - if len(profile.Spec.Checks) > 0 { - checks := make([]signalv1.CertificationCheck, len(profile.Spec.Checks)) - for index := range profile.Spec.Checks { - checks[index] = profile.Spec.Checks[index] - checks[index].WorkloadPolicy = *profile.Spec.Checks[index].WorkloadPolicy.DeepCopy() - checks[index].RuntimeStage = copyRuntimeStage(profile.Spec.Checks[index].RuntimeStage) - checks[index].Parameters = maps.Clone(profile.Spec.Checks[index].Parameters) - checks[index].ParticipantPolicy.RequiredRoles = slices.Clone( - profile.Spec.Checks[index].ParticipantPolicy.RequiredRoles, - ) - checks[index].ParticipantPolicy.ExcludedRoles = slices.Clone( - profile.Spec.Checks[index].ParticipantPolicy.ExcludedRoles, - ) - } - - return checks - } - - maxAttempts := profile.Spec.MaxAttempts - if maxAttempts < 1 { - maxAttempts = DefaultMaxAttempts + checks := make([]signalv1.CertificationCheck, len(profile.Spec.Checks)) + for index := range profile.Spec.Checks { + checks[index] = profile.Spec.Checks[index] + checks[index].WorkloadPolicy = *profile.Spec.Checks[index].WorkloadPolicy.DeepCopy() + checks[index].RuntimeStage = copyRuntimeStage(profile.Spec.Checks[index].RuntimeStage) + checks[index].Parameters = maps.Clone(profile.Spec.Checks[index].Parameters) + checks[index].ParticipantPolicy.RequiredRoles = slices.Clone( + profile.Spec.Checks[index].ParticipantPolicy.RequiredRoles, + ) + checks[index].ParticipantPolicy.ExcludedRoles = slices.Clone( + profile.Spec.Checks[index].ParticipantPolicy.ExcludedRoles, + ) } - return []signalv1.CertificationCheck{{ - ID: "legacy", - Order: 0, - Script: profile.Spec.Script, - RunnerImage: profile.Spec.RunnerImage, - Backend: profile.Spec.Backend, - WorkloadPolicy: *profile.Spec.WorkloadPolicy.DeepCopy(), - RuntimeStage: copyRuntimeStage(profile.Spec.RuntimeStage), - Parameters: maps.Clone(profile.Spec.Parameters), - BackendActiveDeadline: profile.Spec.BackendActiveDeadline, - ResultBounds: signalv1.ResultBounds{ - ReasonBytes: int32(workflowOutputLimits["reason"]), - MeasurementsBytes: int32(workflowOutputLimits["measurements"]), - BackendResourcesBytes: int32(workflowOutputLimits["backend-resources"]), - BackendEvidenceBytes: 64 * 1024, - }, - MaxAttempts: maxAttempts, - }} + return checks } -func runChecks(run *signalv1.CertificationRun) []signalv1.CertificationCheckSnapshot { - if len(run.Spec.Checks) > 0 { - return run.Spec.Checks - } - - return []signalv1.CertificationCheckSnapshot{{ - ID: "legacy", - Order: 0, - Script: signalv1.ScriptSnapshot{ - ConfigMapName: run.Spec.Script.ConfigMapName, - Key: run.Spec.Script.Key, - RequiresRuntimeStage: run.Spec.Script.RequiresRuntimeStage, - Digest: run.Spec.Script.Digest, - }, - RunnerImage: run.Spec.RunnerImage, - Backend: run.Spec.Backend, - WorkloadPolicy: run.Spec.WorkloadPolicy, - RuntimeStage: run.Spec.RuntimeStage, - Parameters: run.Spec.Parameters, - BackendActiveDeadline: run.Spec.BackendActiveDeadline, - ResultBounds: signalv1.ResultBounds{ - ReasonBytes: int32(workflowOutputLimits["reason"]), - MeasurementsBytes: int32(workflowOutputLimits["measurements"]), - BackendResourcesBytes: int32(workflowOutputLimits["backend-resources"]), - BackendEvidenceBytes: 64 * 1024, - }, - MaxAttempts: run.Spec.MaxAttempts, - }} -} - -func isGroupedRun(run *signalv1.CertificationRun) bool { - return len(run.Spec.Checks) == groupedCheckCount -} - -func validateCheckPlan(checks []signalv1.CertificationCheck, grouped bool) error { - if grouped && len(checks) != groupedCheckCount { +func validateCheckPlan(checks []signalv1.CertificationCheck) error { + if len(checks) != groupedCheckCount { return fmt.Errorf("must contain exactly %d checks", groupedCheckCount) } @@ -3247,7 +2656,7 @@ func validateCheckPlan(checks []signalv1.CertificationCheck, grouped bool) error return fmt.Errorf("check %s order is %d, want %d", check.ID, check.Order, index) } - if grouped && string(check.ID) != fmt.Sprintf("R%02d", index+1) { + if string(check.ID) != fmt.Sprintf("R%02d", index+1) { return fmt.Errorf("check order %d must use ID R%02d", index, index+1) } @@ -3257,7 +2666,7 @@ func validateCheckPlan(checks []signalv1.CertificationCheck, grouped bool) error seen[check.ID] = struct{}{} - if grouped && check.MaxAttempts != 1 { + if check.MaxAttempts != 1 { return fmt.Errorf("check %s maxAttempts must be exactly 1", check.ID) } @@ -3273,8 +2682,8 @@ func validateCheckPlan(checks []signalv1.CertificationCheck, grouped bool) error return nil } -func validateRunCheckPlan(checks []signalv1.CertificationCheckSnapshot, grouped bool) error { - if grouped && len(checks) != groupedCheckCount { +func validateRunCheckPlan(checks []signalv1.CertificationCheckSnapshot) error { + if len(checks) != groupedCheckCount { return fmt.Errorf("run must contain exactly %d checks", groupedCheckCount) } @@ -3285,7 +2694,7 @@ func validateRunCheckPlan(checks []signalv1.CertificationCheckSnapshot, grouped return fmt.Errorf("run check %d has invalid identity or order", index) } - if grouped && string(check.ID) != fmt.Sprintf("R%02d", index+1) { + if string(check.ID) != fmt.Sprintf("R%02d", index+1) { return fmt.Errorf("run check order %d must use ID R%02d", index, index+1) } @@ -3295,7 +2704,7 @@ func validateRunCheckPlan(checks []signalv1.CertificationCheckSnapshot, grouped seen[check.ID] = struct{}{} - if grouped && check.MaxAttempts != 1 { + if check.MaxAttempts != 1 { return fmt.Errorf("run check %s maxAttempts must be exactly 1", check.ID) } @@ -3480,27 +2889,6 @@ func participantsForCheck( return selected, globalIndexes, nil } -func setLegacyRunAliases(spec *signalv1.CertificationRunSpec) { - if len(spec.Checks) == 0 { - return - } - - first := &spec.Checks[0] - spec.Script = signalv1.LegacyScriptSnapshot{ - ConfigMapName: first.Script.ConfigMapName, - Key: first.Script.Key, - RequiresRuntimeStage: first.Script.RequiresRuntimeStage, - Digest: first.Script.Digest, - } - spec.Backend = first.Backend - spec.RunnerImage = first.RunnerImage - spec.WorkloadPolicy = first.WorkloadPolicy - spec.RuntimeStage = first.RuntimeStage - spec.Parameters = first.Parameters - spec.BackendActiveDeadline = first.BackendActiveDeadline - spec.MaxAttempts = first.MaxAttempts -} - func sanitizeTaskName(value string) string { value = strings.ToLower(value) value = strings.Map(func(character rune) rune { diff --git a/internal/signal/controller_test.go b/internal/signal/controller_test.go deleted file mode 100644 index 151297704e..0000000000 --- a/internal/signal/controller_test.go +++ /dev/null @@ -1,1979 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -package signal - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "reflect" - "strings" - "testing" - "time" - - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" -) - -func TestAdmissionCreatesImmutableSnapshotAndPreventsDuplicate(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - node := readyNode("gpu-0", "example-gpu") - node.Labels[RequestLabel] = "gpu-smoke" - profile := testProfile("gpu-smoke", "example-gpu") - profile.Spec.Checks[0].Script.RequiresRuntimeStage = true - profile.Spec.Checks[0].RuntimeStage = &signalv1.RuntimeStage{Name: "collective-bandwidth"} - profile.Spec.Checks[0].Backend.Queue = "certification" - profile.Spec.Checks[0].WorkloadPolicy = signalv1.WorkloadPolicy{ - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("4")}, - Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("8")}, - }, - HostNetwork: true, - KueueTopology: &signalv1.KueueTopologyPolicy{ - RequiredLevel: corev1.LabelHostname, - }, - Volumes: []signalv1.WorkloadVolume{{ - Name: "work", - MountPath: "/work", - EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: quantity("8Gi")}, - }}, - } - - const payload = "test payload\n" - - source := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, - Data: map[string]string{"payload.txt": payload}, - } - kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node, profile, source).Build() - now := time.Unix(100, 0) - r := &AdmissionReconciler{Client: kube, Namespace: "signal", ScriptNamespace: "signal", Now: func() time.Time { return now }} - - if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}); err != nil { - t.Fatalf("Reconcile: %v", err) - } - - if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}); err != nil { - t.Fatalf("duplicate Reconcile: %v", err) - } - - var runs signalv1.CertificationRunList - if err := kube.List(context.Background(), &runs); err != nil { - t.Fatalf("list runs: %v", err) - } - - if len(runs.Items) != 1 { - t.Fatalf("runs = %d, want 1", len(runs.Items)) - } - - run := runs.Items[0] - - check := run.Spec.Checks[0] - if check.Script.Digest != digestString(payload) { - t.Fatalf("script digest = %q", check.Script.Digest) - } - - if !check.Script.RequiresRuntimeStage || - check.RuntimeStage == nil || - check.RuntimeStage.Name != "collective-bandwidth" || - check.BackendActiveDeadline.Duration != 9*time.Minute { - t.Fatalf("runtime contract = script %#v stage %#v backend deadline %s", - check.Script, check.RuntimeStage, check.BackendActiveDeadline.Duration) - } - - if run.Spec.Purpose != signalv1.PurposeQuickCertification { - t.Fatalf("purpose = %q, want %q", run.Spec.Purpose, signalv1.PurposeQuickCertification) - } - - if !check.WorkloadPolicy.HostNetwork || - check.WorkloadPolicy.KueueTopology == nil || - check.WorkloadPolicy.KueueTopology.RequiredLevel != corev1.LabelHostname || - check.WorkloadPolicy.Resources.Limits.Cpu().String() != "8" || - len(check.WorkloadPolicy.Volumes) != 1 || - check.WorkloadPolicy.Volumes[0].EmptyDir.SizeLimit.String() != "8Gi" { - t.Fatalf("workload policy = %#v, want immutable profile policy", check.WorkloadPolicy) - } - - if check.Parameters["observed-gpu-capacity"] != "8" || - check.Parameters["observed-gpu-allocatable"] != "8" { - t.Fatalf("observed GPU inventory = %#v", check.Parameters) - } - - if run.Annotations[RunTriggerAnnotation] != "manual" { - t.Fatalf("trigger = %q, want manual", run.Annotations[RunTriggerAnnotation]) - } - - var snapshot corev1.ConfigMap - if err := kube.Get(context.Background(), client.ObjectKey{Namespace: "signal", Name: check.Script.ConfigMapName}, &snapshot); err != nil { - t.Fatalf("get snapshot: %v", err) - } - - if snapshot.Immutable == nil || !*snapshot.Immutable { - t.Fatal("snapshot is mutable") - } - - if snapshot.Data["payload.txt"] != payload { - t.Fatalf("snapshot payload = %q", snapshot.Data["payload.txt"]) - } - - var locked corev1.Node - if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &locked); err != nil { - t.Fatalf("get locked node: %v", err) - } - - if locked.Annotations[ReservationLabel] != run.Name { - t.Fatalf("reservation = %q, want %q", locked.Annotations[ReservationLabel], run.Name) - } -} - -func TestAdmissionRejectsMissingRequiredRuntimeStageBeforeReservation(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - node := readyNode("gpu-0", "example-gpu") - node.Labels[RequestLabel] = "gpu-smoke" - profile := testProfile("gpu-smoke", "example-gpu") - profile.Spec.Checks[0].Script.RequiresRuntimeStage = true - source := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, - Data: map[string]string{"payload.txt": "test payload\n"}, - } - kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node, profile, source).Build() - reconciler := &AdmissionReconciler{Client: kube, Namespace: "signal", ScriptNamespace: "signal"} - - _, err := reconciler.Reconcile( - context.Background(), - reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}, - ) - if err == nil || !strings.Contains(err.Error(), "runtime stage is required") { - t.Fatalf("Reconcile error = %v", err) - } - - var got corev1.Node - if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &got); err != nil { - t.Fatal(err) - } - - if got.Annotations[ReservationLabel] != "" { - t.Fatalf("node was reserved after rejected runtime contract: %#v", got.Annotations) - } -} - -func TestAdmissionRejectsInsufficientBackendDeadlineReserveBeforeReservation(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - node := readyNode("gpu-0", "example-gpu") - node.Labels[RequestLabel] = "gpu-smoke" - profile := testProfile("gpu-smoke", "example-gpu") - profile.Spec.Checks[0].BackendActiveDeadline = metav1.Duration{ - Duration: profile.Spec.Deadline.Duration - 59*time.Second, - } - source := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, - Data: map[string]string{"payload.txt": "test payload\n"}, - } - kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node, profile, source).Build() - reconciler := &AdmissionReconciler{Client: kube, Namespace: "signal", ScriptNamespace: "signal"} - - _, err := reconciler.Reconcile( - context.Background(), - reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}, - ) - if err == nil || !strings.Contains(err.Error(), "leave at least 1m0s") { - t.Fatalf("Reconcile error = %v", err) - } - - var got corev1.Node - if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &got); err != nil { - t.Fatal(err) - } - - if got.Annotations[ReservationLabel] != "" { - t.Fatalf("node was reserved after rejected deadlines: %#v", got.Annotations) - } -} - -func TestAdmissionRejectsMutableRunnerImageBeforeReservation(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - node := readyNode("gpu-0", "example-gpu") - node.Labels[RequestLabel] = "gpu-smoke" - profile := testProfile("gpu-smoke", "example-gpu") - profile.Spec.Checks[0].RunnerImage = "registry.example/runner:latest" - source := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, - Data: map[string]string{"payload.txt": "test payload\n"}, - } - - kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node, profile, source).Build() - reconciler := &AdmissionReconciler{Client: kube, Namespace: "signal", ScriptNamespace: "signal"} - - _, err := reconciler.Reconcile( - context.Background(), - reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}, - ) - if err == nil || !strings.Contains(err.Error(), "immutable sha256 digest") { - t.Fatalf("Reconcile error = %v", err) - } - - var got corev1.Node - if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &got); err != nil { - t.Fatal(err) - } - - if got.Annotations[ReservationLabel] != "" { - t.Fatalf("node was reserved after rejected profile: %#v", got.Annotations) - } -} - -func TestAdmissionRejectsInvalidWorkloadPolicyBeforeReservation(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - node := readyNode("gpu-0", "example-gpu") - node.Labels[RequestLabel] = "gpu-smoke" - profile := testProfile("gpu-smoke", "example-gpu") - profile.Spec.Checks[0].WorkloadPolicy.Volumes = []signalv1.WorkloadVolume{{ - Name: "replacement", - MountPath: "/results", - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }} - source := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, - Data: map[string]string{"payload.txt": "test payload\n"}, - } - kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node, profile, source).Build() - reconciler := &AdmissionReconciler{Client: kube, Namespace: "signal", ScriptNamespace: "signal"} - - _, err := reconciler.Reconcile( - context.Background(), - reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}, - ) - if err == nil || !strings.Contains(err.Error(), "reserved or existing mountPath") { - t.Fatalf("Reconcile error = %v", err) - } - - var got corev1.Node - if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &got); err != nil { - t.Fatal(err) - } - - if got.Annotations[ReservationLabel] != "" { - t.Fatalf("node was reserved after rejected workload policy: %#v", got.Annotations) - } -} - -func TestAdmissionRejectsKueueTopologyWithoutQueueBeforeReservation(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - node := readyNode("gpu-0", "example-gpu") - node.Labels[RequestLabel] = "gpu-smoke" - profile := testProfile("gpu-smoke", "example-gpu") - profile.Spec.Checks[0].Backend = signalv1.ExecutionBackend{APIVersion: "v1", Kind: "Pod"} - profile.Spec.Checks[0].WorkloadPolicy.KueueTopology = &signalv1.KueueTopologyPolicy{ - RequiredLevel: corev1.LabelHostname, - } - source := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, - Data: map[string]string{"payload.txt": "test payload\n"}, - } - kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node, profile, source).Build() - reconciler := &AdmissionReconciler{Client: kube, Namespace: "signal", ScriptNamespace: "signal"} - - _, err := reconciler.Reconcile( - context.Background(), - reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}, - ) - if err == nil || !strings.Contains(err.Error(), "kueueTopology requires a backend queue") { - t.Fatalf("Reconcile error = %v", err) - } - - var got corev1.Node - if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &got); err != nil { - t.Fatal(err) - } - - if got.Annotations[ReservationLabel] != "" { - t.Fatalf("node was reserved after rejected topology policy: %#v", got.Annotations) - } -} - -func TestParticipantSelectionIsDeterministicAndSkipsLockedNodes(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - request := readyNode("gpu-0", "H200") - peer := readyNode("gpu-1", "H200") - locked := readyNode("gpu-2", "H200") - locked.Annotations = map[string]string{ReservationLabel: "other-run"} - wrong := readyNode("gpu-3", "A100") - profile := testProfile("h200-pair", "H200") - profile.Spec.Participants.Count = 2 - kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(request, peer, locked, wrong).Build() - r := &AdmissionReconciler{Client: kube} - - participants, err := r.selectParticipants(context.Background(), request, profile) - if err != nil { - t.Fatalf("selectParticipants: %v", err) - } - - if len(participants) != 2 || - participants[0].Index != 0 || - participants[0].NodeName != request.Name || - participants[1].Index != 1 || - participants[1].NodeName != peer.Name { - t.Fatalf("participants = %#v", participants) - } - - gpu := corev1.ResourceName("nvidia.com/gpu") - capacity := participants[1].Capacity[gpu] - - allocatable := participants[1].Allocatable[gpu] - if capacity.Cmp(resource.MustParse("8")) != 0 || - allocatable.Cmp(resource.MustParse("8")) != 0 { - t.Fatalf("participant inventory = %#v", participants[1]) - } -} - -func TestParticipantSelectionRejectsInsufficientEligibleNodes(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - request := readyNode("gpu-0", "H200") - profile := testProfile("h200-pair", "H200") - profile.Spec.Participants.Count = 2 - kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(request).Build() - r := &AdmissionReconciler{Client: kube} - - if _, err := r.selectParticipants(context.Background(), request, profile); err == nil || - !strings.Contains(err.Error(), "requires 2 eligible unlocked participants, found 1") { - t.Fatalf("selectParticipants error = %v", err) - } -} - -func TestMultiNodeAdmissionReservesEveryParticipant(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - request := readyNode("gpu-0", "H200") - request.Labels[RequestLabel] = "h200-pair" - peer := readyNode("gpu-1", "H200") - profile := testProfile("h200-pair", "H200") - profile.Spec.Participants.Count = 2 - source := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, - Data: map[string]string{"payload.txt": "test payload\n"}, - } - kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(request, peer, profile, source).Build() - r := &AdmissionReconciler{ - Client: kube, - Namespace: "signal", - ScriptNamespace: "signal", - Now: func() time.Time { return time.Unix(100, 0) }, - } - - if _, err := r.Reconcile( - context.Background(), - reconcile.Request{NamespacedName: client.ObjectKey{Name: request.Name}}, - ); err != nil { - t.Fatalf("Reconcile: %v", err) - } - - var runs signalv1.CertificationRunList - if err := kube.List(context.Background(), &runs); err != nil { - t.Fatal(err) - } - - if len(runs.Items) != 1 || len(runs.Items[0].Spec.Participants) != 2 { - t.Fatalf("runs = %#v", runs.Items) - } - - for index, name := range []string{"gpu-0", "gpu-1"} { - participant := runs.Items[0].Spec.Participants[index] - if participant.Index != int32(index) || participant.NodeName != name { - t.Fatalf("participant %d = %#v", index, participant) - } - - var node corev1.Node - if err := kube.Get(context.Background(), client.ObjectKey{Name: name}, &node); err != nil { - t.Fatal(err) - } - - if node.Annotations[ReservationLabel] != runs.Items[0].Name { - t.Fatalf("Node %s reservation = %q", name, node.Annotations[ReservationLabel]) - } - } -} - -func TestMultiNodeReservationRollsBackPartialFailure(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - first := readyNode("gpu-0", "H200") - second := readyNode("gpu-1", "H200") - kube := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(first, second). - WithInterceptorFuncs(interceptor.Funcs{ - Patch: func( - ctx context.Context, - c client.WithWatch, - obj client.Object, - patch client.Patch, - opts ...client.PatchOption, - ) error { - node, ok := obj.(*corev1.Node) - if ok && node.Name == second.Name && node.Annotations[ReservationLabel] == "run-pair" { - return errors.New("reservation conflict") - } - - return c.Patch(ctx, obj, patch, opts...) - }, - }). - Build() - r := &AdmissionReconciler{Client: kube} - participants := []signalv1.Participant{ - participantSnapshot(first, 0), - participantSnapshot(second, 1), - } - - if err := r.reserveParticipants(context.Background(), participants, "run-pair"); err == nil { - t.Fatal("reserveParticipants error = nil") - } - - var recovered corev1.Node - if err := kube.Get(context.Background(), client.ObjectKey{Name: first.Name}, &recovered); err != nil { - t.Fatal(err) - } - - if recovered.Annotations[ReservationLabel] != "" { - t.Fatalf("partial reservation was not rolled back: %#v", recovered.Annotations) - } -} - -func TestAdmissionFailureMakesCreatedRunInertAndReleasesReservation(t *testing.T) { - t.Parallel() - - for _, tc := range []struct { - name string - failSnapshot bool - failProjection bool - }{ - {name: "snapshot binding", failSnapshot: true}, - {name: "pending projection", failProjection: true}, - } { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - node := readyNode("gpu-0", "example-gpu") - node.Labels[RequestLabel] = "gpu-smoke" - profile := testProfile("gpu-smoke", "example-gpu") - source := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, - Data: map[string]string{"payload.txt": "test payload\n"}, - } - kube := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(node, profile, source). - WithInterceptorFuncs(interceptor.Funcs{ - Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { - if snapshot, ok := obj.(*corev1.ConfigMap); ok && - tc.failSnapshot && snapshot.Name != source.Name { - return errors.New("snapshot update failed") - } - - return c.Update(ctx, obj, opts...) - }, - Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { - if projected, ok := obj.(*corev1.Node); ok && - tc.failProjection && projected.Labels[StateLabel] == "pending" { - return errors.New("projection patch failed") - } - - return c.Patch(ctx, obj, patch, opts...) - }, - }). - Build() - r := &AdmissionReconciler{ - Client: kube, - Namespace: "signal", - ScriptNamespace: "signal", - Now: func() time.Time { return time.Unix(100, 0) }, - } - - if _, err := r.Reconcile( - context.Background(), - reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}, - ); err == nil { - t.Fatal("Reconcile error = nil, want failed admission") - } - - var runs signalv1.CertificationRunList - if err := kube.List(context.Background(), &runs); err != nil { - t.Fatalf("list runs: %v", err) - } - - for i := range runs.Items { - if runs.Items[i].DeletionTimestamp.IsZero() { - t.Fatalf("run %s remained active after failed admission", runs.Items[i].Name) - } - } - - var cleaned corev1.Node - if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &cleaned); err != nil { - t.Fatalf("get cleaned node: %v", err) - } - - if cleaned.Annotations[ReservationLabel] != "" { - t.Fatalf("reservation remained after inerting failed run: %q", cleaned.Annotations[ReservationLabel]) - } - }) - } -} - -func TestAdmissionRecoversOrphanedReservationAndSnapshot(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - node := readyNode("gpu-0", "example-gpu") - node.Labels[RequestLabel] = "gpu-smoke" - node.Annotations = map[string]string{ReservationLabel: "orphaned-run"} - peer := readyNode("gpu-1", "example-gpu") - peer.Annotations = map[string]string{ReservationLabel: "orphaned-run"} - profile := testProfile("gpu-smoke", "example-gpu") - profile.Spec.Participants.Count = 2 - source := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, - Data: map[string]string{"payload.txt": "test payload\n"}, - } - immutable := true - orphanedSnapshot := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "orphaned-run-scripts", - Namespace: "signal", - Labels: map[string]string{SnapshotRunLabel: "orphaned-run"}, - }, - Immutable: &immutable, - Data: map[string]string{"payload.txt": "orphaned\n"}, - } - kube := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(node, peer, profile, source, orphanedSnapshot). - Build() - r := &AdmissionReconciler{ - Client: kube, - APIReader: kube, - Namespace: "signal", - ScriptNamespace: "signal", - Now: func() time.Time { return time.Unix(100, 0) }, - } - - if _, err := r.Reconcile( - context.Background(), - reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}, - ); err != nil { - t.Fatalf("Reconcile: %v", err) - } - - if err := kube.Get( - context.Background(), - client.ObjectKeyFromObject(orphanedSnapshot), - &corev1.ConfigMap{}, - ); !apierrors.IsNotFound(err) { - t.Fatalf("orphaned snapshot get error = %v, want NotFound", err) - } - - var runs signalv1.CertificationRunList - if err := kube.List(context.Background(), &runs); err != nil { - t.Fatalf("list runs: %v", err) - } - - if len(runs.Items) != 1 { - t.Fatalf("runs = %d, want recovered admission", len(runs.Items)) - } - - for _, nodeName := range []string{node.Name, peer.Name} { - var recovered corev1.Node - if err := kube.Get(context.Background(), client.ObjectKey{Name: nodeName}, &recovered); err != nil { - t.Fatalf("get recovered node %s: %v", nodeName, err) - } - - if recovered.Annotations[ReservationLabel] != runs.Items[0].Name { - t.Fatalf( - "Node %s reservation = %q, want %q", - nodeName, - recovered.Annotations[ReservationLabel], - runs.Items[0].Name, - ) - } - } -} - -func TestDeletingRunClearsReservationAndRequest(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - run := testRun("deleting-run") - run.Finalizers = []string{RunFinalizer} - node := readyNode(run.Spec.RequestNodeName, "A100") - node.Labels[RequestLabel] = run.Spec.Profile.Name - node.Annotations = map[string]string{ReservationLabel: run.Name} - kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(run, node).Build() - reconciler := &RunReconciler{Client: kube} - - if _, err := reconciler.reconcileDeleting(context.Background(), run); err != nil { - t.Fatalf("reconcileDeleting: %v", err) - } - - var cleaned corev1.Node - if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &cleaned); err != nil { - t.Fatalf("get cleaned node: %v", err) - } - - if cleaned.Annotations[ReservationLabel] != "" || cleaned.Labels[RequestLabel] != "" { - t.Fatalf("deletion cleanup left trigger metadata: labels=%v annotations=%v", cleaned.Labels, cleaned.Annotations) - } -} - -func TestCancellationWaitsForWorkflowDeletionBeforeReleasingNode(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - run := testRun("canceling-run") - run.Spec.Cancel = true - run.Status.Phase = signalv1.RunPhaseRunning - run.Status.Workflow = signalv1.WorkflowStatus{Name: "canceling-run-a1", Attempt: 1} - node := readyNode(run.Spec.RequestNodeName, "A100") - node.Annotations = map[string]string{ReservationLabel: run.Name} - workflow := workflowObject(run, "Running", "", "") - kube := fake.NewClientBuilder(). - WithScheme(scheme). - WithStatusSubresource(&signalv1.CertificationRun{}). - WithObjects(run, node, workflow). - WithInterceptorFuncs(interceptor.Funcs{ - Delete: func( - _ context.Context, - _ client.WithWatch, - _ client.Object, - _ ...client.DeleteOption, - ) error { - return nil - }, - }). - Build() - reconciler := &RunReconciler{Client: kube} - - result, err := reconciler.Reconcile( - context.Background(), - reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}, - ) - if err != nil { - t.Fatalf("Reconcile: %v", err) - } - - if result.RequeueAfter == 0 { - t.Fatal("cancellation did not wait for Workflow deletion") - } - - var gotRun signalv1.CertificationRun - if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &gotRun); err != nil { - t.Fatal(err) - } - - if gotRun.Status.Phase.Terminal() { - t.Fatalf("run became terminal before Workflow deletion: %#v", gotRun.Status) - } - - var gotNode corev1.Node - if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &gotNode); err != nil { - t.Fatal(err) - } - - if gotNode.Annotations[ReservationLabel] != run.Name { - t.Fatalf("reservation was released before Workflow deletion: %v", gotNode.Annotations) - } -} - -func TestWorkflowPinsAttemptInputs(t *testing.T) { - t.Parallel() - - run := testRun("run-1") - run.UID = types.UID("run-uid") - run.Spec.Checks[0].Parameters = map[string]string{ - "expected-gpu-count": "8", - "health-mode": "snapshot", - "mpi-payload-mode": "script", - } - run.Spec.Checks[0].Script.RequiresRuntimeStage = true - run.Spec.Checks[0].RuntimeStage = &signalv1.RuntimeStage{Name: "collective-bandwidth"} - run.Spec.Tolerations = []corev1.Toleration{{ - Key: "nvidia.com/gpu", - Operator: corev1.TolerationOpExists, - Effect: corev1.TaintEffectNoSchedule, - }} - run.Spec.Checks[0].WorkloadPolicy = signalv1.WorkloadPolicy{ - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("4")}, - Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("8")}, - }, - HostNetwork: true, - KueueTopology: &signalv1.KueueTopologyPolicy{ - RequiredLevel: corev1.LabelHostname, - }, - Volumes: []signalv1.WorkloadVolume{{ - Name: "work", - MountPath: "/work", - EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: quantity("8Gi")}, - }}, - } - - workflow, err := buildWorkflow(run, 2) - if err != nil { - t.Fatal(err) - } - - if workflow.GetName() != "run-1-a2" { - t.Fatalf("workflow name = %q", workflow.GetName()) - } - - if got, _, _ := unstructured.NestedInt64(workflow.Object, "spec", "ttlStrategy", "secondsAfterCompletion"); got != 21600 { - t.Fatalf("workflow completion TTL = %d, want 21600", got) - } - - if got, _, _ := unstructured.NestedInt64(workflow.Object, "spec", "activeDeadlineSeconds"); got != 600 { - t.Fatalf("workflow active deadline = %d, want outer deadline 600", got) - } - - if got, _, _ := unstructured.NestedString(workflow.Object, "spec", "podGC", "deleteDelayDuration"); got != "2h" { - t.Fatalf("workflow pod GC delay = %q, want 2h", got) - } - - if len(workflow.GetOwnerReferences()) != 1 || workflow.GetOwnerReferences()[0].UID != run.UID { - t.Fatalf("owner references = %#v", workflow.GetOwnerReferences()) - } - - parameters, found, err := unstructured.NestedSlice(workflow.Object, "spec", "arguments", "parameters") - if err != nil || !found { - t.Fatalf("workflow parameters: found=%t err=%v", found, err) - } - - values := map[string]string{} - - for _, raw := range parameters { - parameter := raw.(map[string]any) - values[parameter["name"].(string)] = parameter["value"].(string) - } - - if values["script-digest"] != run.Spec.Checks[0].Script.Digest || - values["backend-kind"] != "MPIJob" || - values["attempt"] != "2" { - t.Fatalf("workflow values = %#v", values) - } - - if values["script-requires-runtime-stage"] != "true" || - values["runtime-stage-name"] != "collective-bandwidth" || - values["backend-active-deadline-seconds"] != "540" { - t.Fatalf("workflow runtime contract = %#v", values) - } - - if values["participant-count"] != "1" || values["request-node-name"] != run.Spec.RequestNodeName { - t.Fatalf("workflow participant metadata = %#v", values) - } - - var participants []signalv1.Participant - if err := json.Unmarshal([]byte(values["participants-json"]), &participants); err != nil { - t.Fatalf("decode participants: %v", err) - } - - if len(participants) != 1 || - participants[0].NodeName != run.Spec.Participants[0].NodeName || - participants[0].NodeUID != run.Spec.Participants[0].NodeUID { - t.Fatalf("workflow participants = %#v", participants) - } - - if values["script-configmap"] != run.Spec.Checks[0].Script.ConfigMapName || - values["script-configmap"] == "scripts" { - t.Fatalf("workflow must mount the immutable snapshot, values = %#v", values) - } - - if !strings.Contains(values["tolerations"], `"key":"nvidia.com/gpu"`) { - t.Fatalf("workflow tolerations = %q", values["tolerations"]) - } - - var environment map[string]string - if err := json.Unmarshal([]byte(values["script-environment"]), &environment); err != nil { - t.Fatalf("decode script environment: %v", err) - } - - if environment["EXPECTED_GPU_COUNT"] != "8" || - environment["HEALTH_MODE"] != "snapshot" || - values["mpi-payload-mode"] != "script" { - t.Fatalf("workflow script policy = %#v, parameters = %#v", environment, values) - } - - var tolerations []corev1.Toleration - if err := json.Unmarshal([]byte(values["tolerations"]), &tolerations); err != nil { - t.Fatalf("decode workflow tolerations: %v", err) - } - - if len(tolerations) != 1 { - t.Fatalf("workflow tolerations = %#v, want one deduplicated GPU toleration", tolerations) - } - - var policy signalv1.WorkloadPolicy - if err := json.Unmarshal([]byte(values["workload-policy"]), &policy); err != nil { - t.Fatalf("decode workload policy: %v", err) - } - - if !policy.HostNetwork || - policy.KueueTopology == nil || - policy.KueueTopology.RequiredLevel != corev1.LabelHostname || - policy.Resources.Requests.Cpu().String() != "4" || - policy.Resources.Limits.Cpu().String() != "8" || - len(policy.Volumes) != 1 || - policy.Volumes[0].EmptyDir.SizeLimit.String() != "8Gi" { - t.Fatalf("workflow workload policy = %#v", policy) - } -} - -func TestBuildWorkflowRejectsParameterThatCannotBecomeEnvironmentVariable(t *testing.T) { - t.Parallel() - - run := testRun("run-invalid-parameter") - run.Spec.Checks[0].Parameters = map[string]string{"invalid.parameter": "value"} - - if _, err := buildWorkflow(run, 1); err == nil || - !strings.Contains(err.Error(), "cannot be mapped to a script environment variable") { - t.Fatalf("buildWorkflow error = %v", err) - } -} - -func TestScriptEnvironmentRejectsCollidingParameterNames(t *testing.T) { - t.Parallel() - - _, err := scriptEnvironment(map[string]string{ - "gpu-devices": "0,1", - "gpu_devices": "2,3", - }) - if err == nil || !strings.Contains(err.Error(), "both map to GPU_DEVICES") { - t.Fatalf("scriptEnvironment error = %v", err) - } -} - -func TestScriptEnvironmentRejectsReservedNames(t *testing.T) { - t.Parallel() - - for _, parameter := range []string{"path", "bash-env", "project-signal-run-uid"} { - _, err := scriptEnvironment(map[string]string{parameter: "value"}) - if err == nil || !strings.Contains(err.Error(), "reserved environment variable") { - t.Fatalf("scriptEnvironment(%q) error = %v", parameter, err) - } - } -} - -func TestWorkflowParametersRejectMalformedEvidence(t *testing.T) { - t.Parallel() - - tests := map[string][]any{ - "duplicate": { - map[string]any{"name": "verdict", "value": "Pass"}, - map[string]any{"name": "verdict", "value": "Fail"}, - }, - "unexpected": { - map[string]any{"name": "unbounded", "value": "value"}, - }, - "oversized": { - map[string]any{"name": "measurements", "value": strings.Repeat("x", 16*1024+1)}, - }, - "oversized cleanup failure": { - map[string]any{"name": "cleanup-failure", "value": strings.Repeat("x", 65)}, - }, - "oversized backend evidence": { - map[string]any{"name": "backend-evidence", "value": strings.Repeat("x", 256*1024+1)}, - }, - "wrong type": { - map[string]any{"name": "verdict", "value": true}, - }, - } - - for name, parameters := range tests { - t.Run(name, func(t *testing.T) { - t.Parallel() - - workflow := &unstructured.Unstructured{Object: map[string]any{ - "status": map[string]any{ - "outputs": map[string]any{"parameters": parameters}, - }, - }} - - if _, err := workflowParameters(workflow); err == nil { - t.Fatal("workflowParameters succeeded") - } - }) - } -} - -func TestBackendEvidenceCompleteValidation(t *testing.T) { - t.Parallel() - - for name, test := range map[string]struct { - content string - count int - want bool - wantErr bool - }{ - "complete": { - content: `{"version":"v1","complete":true,"participants":[{}]}`, - count: 1, - want: true, - }, - "incomplete": { - content: `{"version":"v1","complete":false,"participants":[{}]}`, - count: 1, - }, - "missing": { - count: 1, - wantErr: true, - }, - "wrong count": { - content: `{"version":"v1","complete":true,"participants":[{},{}]}`, - count: 1, - wantErr: true, - }, - "malformed": { - content: `{"version":"v1","complete":true,"participants":`, - count: 1, - wantErr: true, - }, - "unknown version": { - content: `{"version":"v2","complete":true,"participants":[{}]}`, - count: 1, - wantErr: true, - }, - } { - t.Run(name, func(t *testing.T) { - t.Parallel() - - got, err := backendEvidenceComplete(test.content, test.count) - if (err != nil) != test.wantErr || got != test.want { - t.Fatalf("backendEvidenceComplete() = %t, %v", got, err) - } - }) - } -} - -func TestBackendResourceReferencesRequireCompleteMultiNodeCoverage(t *testing.T) { - t.Parallel() - - value := `[ - {"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","namespace":"signal","name":"node-0","uid":"uid-0","participantIndexes":[0]}, - {"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","namespace":"signal","name":"node-1","uid":"uid-1","participantIndexes":[1]} - ]` - - references, err := backendResourceReferences(value, 2) - if err != nil { - t.Fatalf("backendResourceReferences: %v", err) - } - - if len(references) != 2 || references[1].ParticipantIndexes[0] != 1 { - t.Fatalf("references = %#v", references) - } - - for name, malformed := range map[string]string{ - "missing participant": `[{"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","name":"node-0","uid":"uid-0","participantIndexes":[0]}]`, - "out of range": `[{"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","name":"node-0","uid":"uid-0","participantIndexes":[0,2]}]`, - "empty coverage": `[{"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","name":"node-0","uid":"uid-0"}]`, - "missing uid": `[{"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","name":"node-0","participantIndexes":[0,1]}]`, - "unknown field": `[{"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","name":"node-0","uid":"uid-0","participantIndexes":[0,1],"extra":true}]`, - "trailing value": `[{"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","name":"node-0","uid":"uid-0","participantIndexes":[0,1]}] {}`, - } { - t.Run(name, func(t *testing.T) { - t.Parallel() - - if _, err := backendResourceReferences(malformed, 2); err == nil { - t.Fatalf("backendResourceReferences(%s) error = nil", malformed) - } - }) - } -} - -func TestMultiNodePassWithoutBackendIdentityBecomesInconclusive(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - run := testRun("run-multi") - run.Spec.Participants = []signalv1.Participant{ - participantSnapshot(readyNode("gpu-0", "H200"), 0), - participantSnapshot(readyNode("gpu-1", "H200"), 1), - } - run.Status = signalv1.CertificationRunStatus{ - Phase: signalv1.RunPhaseRunning, - Workflow: signalv1.WorkflowStatus{Name: "run-multi-a1", Attempt: 1}, - StartedAt: ptrTime(metav1.NewTime(time.Unix(100, 0))), - } - snapshot := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Checks[0].Script.ConfigMapName, Namespace: run.Namespace}, - Data: map[string]string{run.Spec.Checks[0].Script.Key: "test payload\n"}, - } - workflow := workflowObject(run, "Succeeded", "Pass", "Healthy") - store := &memoryEvidenceStore{} - kube := fake.NewClientBuilder(). - WithScheme(scheme). - WithStatusSubresource(&signalv1.CertificationRun{}). - WithObjects(run, snapshot, workflow). - Build() - r := &RunReconciler{Client: kube, Evidence: store, Now: func() time.Time { return time.Unix(200, 0) }} - - if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}); err != nil { - t.Fatalf("Reconcile: %v", err) - } - - var got signalv1.CertificationRun - if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { - t.Fatal(err) - } - - if got.Status.Phase != signalv1.RunPhaseInconclusive || - got.Status.Verdict != "Inconclusive" || - !strings.Contains(got.Status.Reason, "did not identify backend resources") { - t.Fatalf("status = %#v", got.Status) - } -} - -func TestMultiNodePassRetainsCoveredBackendIdentities(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - run := testRun("run-multi-pass") - run.Spec.Participants = []signalv1.Participant{ - participantSnapshot(readyNode("gpu-0", "H200"), 0), - participantSnapshot(readyNode("gpu-1", "H200"), 1), - } - run.Status = signalv1.CertificationRunStatus{ - Phase: signalv1.RunPhaseRunning, - Workflow: signalv1.WorkflowStatus{Name: "run-multi-pass-a1", Attempt: 1}, - StartedAt: ptrTime(metav1.NewTime(time.Unix(100, 0))), - } - snapshot := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Checks[0].Script.ConfigMapName, Namespace: run.Namespace}, - Data: map[string]string{run.Spec.Checks[0].Script.Key: "test payload\n"}, - } - workflow := workflowObject(run, "Succeeded", "Pass", "Healthy") - - outputs, _, err := unstructured.NestedSlice(workflow.Object, "status", "outputs", "parameters") - if err != nil { - t.Fatal(err) - } - - outputs = append(outputs, map[string]any{ - "name": "backend-resources", - "value": `[ - {"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","namespace":"signal","name":"node-0","uid":"uid-0","participantIndexes":[0]}, - {"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","namespace":"signal","name":"node-1","uid":"uid-1","participantIndexes":[1]} - ]`, - }) - if err := unstructured.SetNestedSlice(workflow.Object, outputs, "status", "outputs", "parameters"); err != nil { - t.Fatal(err) - } - - store := &memoryEvidenceStore{} - kube := fake.NewClientBuilder(). - WithScheme(scheme). - WithStatusSubresource(&signalv1.CertificationRun{}). - WithObjects(run, snapshot, workflow). - Build() - r := &RunReconciler{Client: kube, Evidence: store, Now: func() time.Time { return time.Unix(200, 0) }} - - if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}); err != nil { - t.Fatalf("Reconcile: %v", err) - } - - var got signalv1.CertificationRun - if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { - t.Fatal(err) - } - - if got.Status.Phase != signalv1.RunPhaseSucceeded || - len(got.Status.Workflow.BackendResources) != 2 || - got.Status.Workflow.BackendResources[1].ParticipantIndexes[0] != 1 { - t.Fatalf("status = %#v", got.Status) - } -} - -func TestStartAttemptRecoversExistingOwnedWorkflow(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - run := testRun("run-recovered") - - workflow, err := buildWorkflow(run, 1) - if err != nil { - t.Fatal(err) - } - - workflow.SetUID("workflow-uid") - kube := fake.NewClientBuilder(). - WithScheme(scheme). - WithStatusSubresource(&signalv1.CertificationRun{}). - WithObjects(run, workflow). - Build() - r := &RunReconciler{Client: kube, Evidence: &memoryEvidenceStore{}} - - if _, err := r.startAttempt(context.Background(), run, 1); err != nil { - t.Fatalf("startAttempt: %v", err) - } - - var got signalv1.CertificationRun - if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { - t.Fatalf("get run: %v", err) - } - - if got.Status.Workflow.Name != workflow.GetName() || - got.Status.Workflow.UID != string(workflow.GetUID()) || - got.Status.Workflow.Attempt != 1 { - t.Fatalf("workflow status = %#v", got.Status.Workflow) - } -} - -func TestWorkflowParameterFallsBackToEntrypointNodeOutputs(t *testing.T) { - t.Parallel() - - workflow := &unstructured.Unstructured{Object: map[string]any{ - "metadata": map[string]any{"name": "run-a1"}, - "status": map[string]any{ - "nodes": map[string]any{ - "run-a1": map[string]any{ - "name": "run-a1", - "outputs": map[string]any{ - "parameters": []any{map[string]any{"name": "verdict", "value": "Pass"}}, - }, - }, - }, - }, - }} - - if got := workflowParameter(workflow, "verdict"); got != "Pass" { - t.Fatalf("verdict = %q", got) - } -} - -func TestWorkflowParametersPreferPresentTopLevelOutputs(t *testing.T) { - t.Parallel() - - workflow := workflowWithEntrypointOutputs( - []any{map[string]any{"name": "verdict", "value": "NodePass"}}, - ) - if err := unstructured.SetNestedSlice( - workflow.Object, - []any{map[string]any{"name": "verdict", "value": "TopLevelPass"}}, - "status", - "outputs", - "parameters", - ); err != nil { - t.Fatal(err) - } - - outputs, err := workflowParameters(workflow) - if err != nil { - t.Fatal(err) - } - - if outputs["verdict"] != "TopLevelPass" { - t.Fatalf("verdict = %q", outputs["verdict"]) - } -} - -func TestWorkflowParametersFallBackToAuthoritativeEntrypoint(t *testing.T) { - t.Parallel() - - workflow := workflowWithEntrypointOutputs( - []any{ - map[string]any{"name": "verdict", "value": "Pass"}, - map[string]any{"name": "reason", "value": "Healthy"}, - }, - ) - - outputs, err := workflowParameters(workflow) - if err != nil { - t.Fatal(err) - } - - if outputs["verdict"] != "Pass" || outputs["reason"] != "Healthy" { - t.Fatalf("outputs = %#v", outputs) - } -} - -func TestWorkflowParametersRejectInvalidEntrypointFallback(t *testing.T) { - t.Parallel() - - for name, mutate := range map[string]func(*unstructured.Unstructured){ - "missing entrypoint": func(workflow *unstructured.Unstructured) { - workflow.Object["status"].(map[string]any)["nodes"] = map[string]any{ - "child": map[string]any{"name": "run-a1.child"}, - } - }, - "malformed entrypoint identity": func(workflow *unstructured.Unstructured) { - workflow.Object["status"].(map[string]any)["nodes"] = map[string]any{ - "run-a1": map[string]any{"name": true}, - } - }, - "duplicate entrypoint": func(workflow *unstructured.Unstructured) { - nodes := workflow.Object["status"].(map[string]any)["nodes"].(map[string]any) - nodes["duplicate"] = map[string]any{ - "name": "run-a1", - "outputs": map[string]any{"parameters": []any{}}, - } - }, - "ambiguous entrypoint": func(workflow *unstructured.Unstructured) { - workflow.Object["status"].(map[string]any)["nodes"] = map[string]any{ - "run-a1": map[string]any{"name": "run-a1.child"}, - "other": map[string]any{ - "name": "run-a1", - "outputs": map[string]any{"parameters": []any{}}, - }, - } - }, - "malformed outputs": func(workflow *unstructured.Unstructured) { - nodes := workflow.Object["status"].(map[string]any)["nodes"].(map[string]any) - nodes["run-a1"].(map[string]any)["outputs"] = map[string]any{ - "parameters": "not-a-list", - } - }, - } { - t.Run(name, func(t *testing.T) { - t.Parallel() - - workflow := workflowWithEntrypointOutputs([]any{}) - mutate(workflow) - - if _, err := workflowParameters(workflow); err == nil { - t.Fatal("workflowParameters accepted invalid entrypoint fallback") - } - }) - } -} - -func TestTerminalVerdictUsesEntrypointOutputsWhenTopLevelAbsent(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - run := testRun("run-entrypoint-output") - run.Status = signalv1.CertificationRunStatus{ - Phase: signalv1.RunPhaseRunning, - Workflow: signalv1.WorkflowStatus{Name: "run-entrypoint-output-a1", Attempt: 1}, - StartedAt: ptrTime(metav1.NewTime(time.Unix(100, 0))), - } - snapshot := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Checks[0].Script.ConfigMapName, Namespace: run.Namespace}, - Data: map[string]string{run.Spec.Checks[0].Script.Key: "test payload\n"}, - } - workflow := workflowObject(run, "Succeeded", "Pass", "Healthy") - - parameters, found, err := unstructured.NestedSlice( - workflow.Object, - "status", - "outputs", - "parameters", - ) - if err != nil || !found { - t.Fatalf("read top-level outputs: found=%t, err=%v", found, err) - } - - delete(workflow.Object["status"].(map[string]any), "outputs") - workflow.Object["status"].(map[string]any)["nodes"] = map[string]any{ - workflow.GetName(): map[string]any{ - "name": workflow.GetName(), - "outputs": map[string]any{ - "parameters": parameters, - }, - }, - } - - store := &memoryEvidenceStore{} - kube := fake.NewClientBuilder(). - WithScheme(scheme). - WithStatusSubresource(&signalv1.CertificationRun{}). - WithObjects(run, snapshot, workflow). - Build() - r := &RunReconciler{Client: kube, Evidence: store, Now: func() time.Time { return time.Unix(200, 0) }} - - if _, err := r.Reconcile( - context.Background(), - reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}, - ); err != nil { - t.Fatalf("Reconcile: %v", err) - } - - var got signalv1.CertificationRun - if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { - t.Fatal(err) - } - - if got.Status.Phase != signalv1.RunPhaseSucceeded || - got.Status.Verdict != "Pass" || - got.Status.Workflow.Outputs["reason"] != "Healthy" { - t.Fatalf("status = %#v", got.Status) - } -} - -func TestValidateStagePlanRejectsInvalidAndDuplicateStages(t *testing.T) { - t.Parallel() - - if err := validateStagePlan([]signalv1.CertificationStage{{DisplayName: "Missing ID"}}); err == nil { - t.Fatal("expected empty stage ID to fail validation") - } - - if err := validateStagePlan([]signalv1.CertificationStage{ - {ID: "workload", DisplayName: "First"}, - {ID: "workload", DisplayName: "Duplicate"}, - }); err == nil { - t.Fatal("expected duplicate stage IDs to fail validation") - } -} - -func TestTerminalVerdictPersistsEvidenceAndCleanupReleasesNode(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - run := testRun("run-pass") - run.Spec.Checks[0].RuntimeStage = &signalv1.RuntimeStage{Name: "collective-bandwidth"} - run.Spec.Checks[0].Backend.Queue = "certification" - run.Spec.Checks[0].WorkloadPolicy.KueueTopology = &signalv1.KueueTopologyPolicy{ - RequiredLevel: corev1.LabelHostname, - } - run.Status = signalv1.CertificationRunStatus{ - Phase: signalv1.RunPhaseRunning, - Workflow: signalv1.WorkflowStatus{Name: "run-pass-a1", Attempt: 1}, - StartedAt: ptrTime(metav1.NewTime(time.Unix(100, 0))), - } - node := readyNode("gpu-0", "A100") - node.Labels[RequestLabel] = run.Spec.Profile.Name - node.Annotations = map[string]string{ReservationLabel: run.Name} - snapshot := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Checks[0].Script.ConfigMapName, Namespace: run.Namespace}, - Data: map[string]string{run.Spec.Checks[0].Script.Key: "test payload\n"}, - } - workflow := workflowObject(run, "Succeeded", "Pass", "Healthy") - store := &memoryEvidenceStore{} - kube := fake.NewClientBuilder(). - WithScheme(scheme). - WithStatusSubresource(&signalv1.CertificationRun{}). - WithObjects(run, node, snapshot, workflow). - Build() - r := &RunReconciler{Client: kube, Evidence: store, Now: func() time.Time { return time.Unix(200, 0) }} - - if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}); err != nil { - t.Fatalf("terminal Reconcile: %v", err) - } - - if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}); err != nil { - t.Fatalf("cleanup Reconcile: %v", err) - } - - var got signalv1.CertificationRun - if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { - t.Fatalf("get run: %v", err) - } - - if got.Status.Phase != signalv1.RunPhaseSucceeded || - got.Status.Verdict != "Pass" || - got.Status.Workflow.Phase != "Succeeded" { - t.Fatalf("status = %#v", got.Status) - } - - if got.Status.Workflow.Outputs["measurements"] != "checked=14" { - t.Fatalf("retained workflow outputs = %#v", got.Status.Workflow.Outputs) - } - - if got.Status.Workflow.Outputs["backend-evidence"] == "" { - t.Fatalf("backend evidence was not retained: %#v", got.Status.Workflow.Outputs) - } - - if got.Status.Evidence.URI == "" || len(store.data) == 0 { - t.Fatal("evidence was not persisted") - } - - var evidence struct { - Workflow signalv1.WorkflowStatus `json:"workflow"` - WorkloadPolicy signalv1.WorkloadPolicy `json:"workloadPolicy"` - RuntimeStage *signalv1.RuntimeStage `json:"runtimeStage"` - BackendActiveDeadline metav1.Duration `json:"backendActiveDeadline"` - } - if err := json.Unmarshal(store.data, &evidence); err != nil { - t.Fatalf("decode evidence: %v", err) - } - - if evidence.Workflow.Phase != "Succeeded" || - evidence.Workflow.Outputs["measurements"] != "checked=14" || - evidence.Workflow.Outputs["backend-evidence"] != got.Status.Workflow.Outputs["backend-evidence"] { - t.Fatalf("evidence workflow = %#v", evidence.Workflow) - } - - if !reflect.DeepEqual(evidence.WorkloadPolicy, run.Spec.Checks[0].WorkloadPolicy) { - t.Fatalf( - "evidence workload policy = %#v, want %#v", - evidence.WorkloadPolicy, - run.Spec.Checks[0].WorkloadPolicy, - ) - } - - if !reflect.DeepEqual(evidence.RuntimeStage, run.Spec.Checks[0].RuntimeStage) || - evidence.BackendActiveDeadline != run.Spec.Checks[0].BackendActiveDeadline { - t.Fatalf("evidence runtime contract = stage %#v deadline %s", - evidence.RuntimeStage, evidence.BackendActiveDeadline.Duration) - } - - if !got.Status.Cleanup.ReservationsReleased || !got.Status.Cleanup.SnapshotDeleted { - t.Fatalf("cleanup status = %#v", got.Status.Cleanup) - } - - if containsString(got.Finalizers, RunFinalizer) { - t.Fatal("cleanup finalizer remains") - } - - var released corev1.Node - if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &released); err != nil { - t.Fatalf("get released node: %v", err) - } - - if released.Annotations[ReservationLabel] != "" || released.Labels[RequestLabel] != "" { - t.Fatalf("node was not released: labels=%v annotations=%v", released.Labels, released.Annotations) - } - - if released.Labels[StateLabel] != "passed" || - released.Labels[EligibilityLabel] != "production" || - released.Annotations[LastSuccessfulRunAnnotation] != run.Name { - t.Fatalf("terminal projection was not retained: labels=%v annotations=%v", released.Labels, released.Annotations) - } -} - -func TestTerminalFinalizationRetriesWithIdenticalEvidence(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - run := testRun("run-retry-finalization") - run.Status = signalv1.CertificationRunStatus{ - Phase: signalv1.RunPhaseRunning, - Workflow: signalv1.WorkflowStatus{Name: "run-retry-finalization-a1", Attempt: 1}, - StartedAt: ptrTime(metav1.NewTime(time.Unix(100, 0))), - } - node := readyNode("gpu-0", "A100") - node.Annotations = map[string]string{ReservationLabel: run.Name} - snapshot := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Checks[0].Script.ConfigMapName, Namespace: run.Namespace}, - Data: map[string]string{run.Spec.Checks[0].Script.Key: "test payload\n"}, - } - workflow := workflowObject(run, "Succeeded", "Pass", "Healthy") - statusPatches := 0 - base := fake.NewClientBuilder(). - WithScheme(scheme). - WithStatusSubresource(&signalv1.CertificationRun{}). - WithObjects(run, node, snapshot, workflow). - Build() - kube := interceptor.NewClient(base, interceptor.Funcs{ - SubResourcePatch: func( - ctx context.Context, - c client.Client, - subResource string, - obj client.Object, - patch client.Patch, - opts ...client.SubResourcePatchOption, - ) error { - if subResource == "status" { - statusPatches++ - if statusPatches == 2 { - return errors.New("status patch interrupted after evidence persistence") - } - } - - return c.SubResource(subResource).Patch(ctx, obj, patch, opts...) - }, - }) - store := &immutableMemoryEvidenceStore{} - r := &RunReconciler{ - Client: kube, - Evidence: store, - Now: func() time.Time { return time.Unix(200+int64(statusPatches), 0) }, - } - key := reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)} - - if _, err := r.Reconcile(context.Background(), key); err == nil { - t.Fatal("first Reconcile error = nil, want interrupted status patch") - } - - var interrupted signalv1.CertificationRun - if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &interrupted); err != nil { - t.Fatalf("get interrupted run: %v", err) - } - - beforeCancel := interrupted.DeepCopy() - interrupted.Spec.Cancel = true - - if err := kube.Patch(context.Background(), &interrupted, client.MergeFrom(beforeCancel)); err != nil { - t.Fatalf("request cancellation: %v", err) - } - - if _, err := r.Reconcile(context.Background(), key); err != nil { - t.Fatalf("retry Reconcile: %v", err) - } - - var got signalv1.CertificationRun - if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { - t.Fatalf("get run: %v", err) - } - - if got.Status.Phase != signalv1.RunPhaseSucceeded { - t.Fatalf("phase = %q, want Succeeded", got.Status.Phase) - } - - if store.puts != 2 { - t.Fatalf("evidence puts = %d, want 2 identical attempts", store.puts) - } -} - -func TestWorkflowReplacementIsRejected(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - run := testRun("run-workflow-replaced") - run.Status = signalv1.CertificationRunStatus{ - Phase: signalv1.RunPhaseRunning, - Workflow: signalv1.WorkflowStatus{ - Name: "run-workflow-replaced-a1", - UID: "original-workflow-uid", - Attempt: 1, - }, - } - replacement := workflowObject(run, "Succeeded", "Pass", "forged") - replacement.SetUID("replacement-workflow-uid") - kube := fake.NewClientBuilder(). - WithScheme(scheme). - WithStatusSubresource(&signalv1.CertificationRun{}). - WithObjects(run, replacement). - Build() - r := &RunReconciler{Client: kube, Evidence: &memoryEvidenceStore{}} - - if _, err := r.Reconcile( - context.Background(), - reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}, - ); err == nil || !strings.Contains(err.Error(), "refusing concurrent retry") { - t.Fatalf("Reconcile error = %v, want identity rejection", err) - } - - var got signalv1.CertificationRun - if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { - t.Fatalf("get run: %v", err) - } - - if got.Status.Workflow.Attempt != 1 || - got.Status.Workflow.Name != "run-workflow-replaced-a1" { - t.Fatalf("replacement workflow triggered a retry: %#v", got.Status.Workflow) - } - - var workflows unstructured.UnstructuredList - workflows.SetGroupVersionKind(workflowGVK.GroupVersion().WithKind("WorkflowList")) - - if err := kube.List(context.Background(), &workflows, client.InNamespace(run.Namespace)); err != nil { - t.Fatalf("list Workflows: %v", err) - } - - if len(workflows.Items) != 1 { - t.Fatalf("workflows = %d, want only mismatched original name", len(workflows.Items)) - } -} - -func TestCancellationRefusesToDeleteReplacementWorkflow(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - run := testRun("run-cancel-replacement") - run.Spec.Cancel = true - run.Status = signalv1.CertificationRunStatus{ - Phase: signalv1.RunPhaseRunning, - Workflow: signalv1.WorkflowStatus{ - Name: "run-cancel-replacement-a1", - UID: "original-workflow-uid", - Attempt: 1, - }, - } - replacement := workflowObject(run, "Running", "", "") - replacement.SetUID("replacement-workflow-uid") - kube := fake.NewClientBuilder(). - WithScheme(scheme). - WithStatusSubresource(&signalv1.CertificationRun{}). - WithObjects(run, replacement). - Build() - r := &RunReconciler{Client: kube} - - if _, err := r.Reconcile( - context.Background(), - reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}, - ); err == nil || !strings.Contains(err.Error(), "refusing to delete Workflow") { - t.Fatalf("Reconcile error = %v, want identity rejection", err) - } - - var remaining unstructured.Unstructured - remaining.SetGroupVersionKind(workflowGVK) - - if err := kube.Get(context.Background(), client.ObjectKeyFromObject(replacement), &remaining); err != nil { - t.Fatalf("replacement Workflow was deleted: %v", err) - } -} - -func TestWorkflowFailureRetriesWithoutSwitchingPinnedInputs(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - run := testRun("run-retry") - run.Spec.Checks[0].MaxAttempts = 2 - run.Status = signalv1.CertificationRunStatus{ - Phase: signalv1.RunPhaseRunning, - Workflow: signalv1.WorkflowStatus{Name: "run-retry-a1", Attempt: 1}, - } - workflow := workflowObject(run, "Failed", "", "") - kube := fake.NewClientBuilder(). - WithScheme(scheme). - WithStatusSubresource(&signalv1.CertificationRun{}). - WithObjects(run, workflow). - Build() - r := &RunReconciler{Client: kube, Evidence: &memoryEvidenceStore{}} - - if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}); err != nil { - t.Fatalf("Reconcile: %v", err) - } - - var got signalv1.CertificationRun - if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { - t.Fatalf("get run: %v", err) - } - - if got.Status.Workflow.Attempt != 2 || got.Status.Workflow.Name != "run-retry-a2" { - t.Fatalf("workflow status = %#v", got.Status.Workflow) - } - - var retry unstructured.Unstructured - retry.SetGroupVersionKind(workflowGVK) - - if err := kube.Get(context.Background(), client.ObjectKey{Namespace: run.Namespace, Name: "run-retry-a2"}, &retry); err != nil { - t.Fatalf("get retry workflow: %v", err) - } - - if value := workflowArgument(&retry, "script-digest"); value != run.Spec.Checks[0].Script.Digest { - t.Fatalf("retry script digest = %q", value) - } - - if value := workflowArgument(&retry, "backend-kind"); value != run.Spec.Checks[0].Backend.Kind { - t.Fatalf("retry backend = %q", value) - } -} - -func TestCanceledRunCleansUpDespiteCancelFlag(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - run := testRun("run-canceled") - run.Spec.Cancel = true - run.Status.Phase = signalv1.RunPhaseCancelled - node := readyNode("gpu-0", "A100") - node.Labels[RequestLabel] = run.Spec.Profile.Name - node.Annotations = map[string]string{ReservationLabel: run.Name} - snapshot := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Checks[0].Script.ConfigMapName, Namespace: run.Namespace}, - Data: map[string]string{run.Spec.Checks[0].Script.Key: "test payload\n"}, - } - kube := fake.NewClientBuilder(). - WithScheme(scheme). - WithStatusSubresource(&signalv1.CertificationRun{}). - WithObjects(run, node, snapshot). - Build() - r := &RunReconciler{Client: kube, Evidence: &memoryEvidenceStore{}} - - if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}); err != nil { - t.Fatalf("Reconcile: %v", err) - } - - var got signalv1.CertificationRun - if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { - t.Fatalf("get run: %v", err) - } - - if containsString(got.Finalizers, RunFinalizer) || !got.Status.Cleanup.ReservationsReleased { - t.Fatalf("run was not cleaned up: %#v", got) - } -} - -type memoryEvidenceStore struct { - data []byte -} - -func (s *memoryEvidenceStore) Put(_ context.Context, path string, data []byte) (string, string, error) { - s.data = append([]byte(nil), data...) - - return "memory://" + path, digestString(string(data)), nil -} - -func (s *memoryEvidenceStore) Get(_ context.Context, path string) ([]byte, string, string, bool, error) { - if s.data == nil { - return nil, "", "", false, nil - } - - return append([]byte(nil), s.data...), "memory://" + path, digestString(string(s.data)), true, nil -} - -type immutableMemoryEvidenceStore struct { - data []byte - puts int -} - -func (s *immutableMemoryEvidenceStore) Put(_ context.Context, path string, data []byte) (string, string, error) { - s.puts++ - if s.data != nil && string(s.data) != string(data) { - return "", "", errors.New("evidence changed across retry") - } - - s.data = append([]byte(nil), data...) - - return "memory://" + path, digestString(string(data)), nil -} - -func (s *immutableMemoryEvidenceStore) Get(_ context.Context, path string) ([]byte, string, string, bool, error) { - if s.data == nil { - return nil, "", "", false, nil - } - - return append([]byte(nil), s.data...), "memory://" + path, digestString(string(s.data)), true, nil -} - -func testScheme(t *testing.T) *runtime.Scheme { - t.Helper() - - scheme := runtime.NewScheme() - if err := corev1.AddToScheme(scheme); err != nil { - t.Fatal(err) - } - - if err := signalv1.AddToScheme(scheme); err != nil { - t.Fatal(err) - } - - scheme.AddKnownTypeWithName(workflowGVK, &unstructured.Unstructured{}) - scheme.AddKnownTypeWithName(workflowGVK.GroupVersion().WithKind("WorkflowList"), &unstructured.UnstructuredList{}) - - return scheme -} - -func readyNode(name, sku string) *corev1.Node { - return &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - UID: types.UID(name + "-uid"), - Labels: map[string]string{ - "node.kubernetes.io/instance-type": sku, - }, - }, - Status: corev1.NodeStatus{ - Capacity: corev1.ResourceList{corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("8")}, - Allocatable: corev1.ResourceList{corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("8")}, - Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}, - }, - } -} - -func testProfile(name, sku string) *signalv1.CertificationProfile { - profile := &signalv1.CertificationProfile{ - ObjectMeta: metav1.ObjectMeta{Name: name, UID: types.UID(name + "-uid"), Generation: 3}, - Spec: signalv1.CertificationProfileSpec{ - Participants: signalv1.ParticipantPolicy{ - Count: 1, - Selector: metav1.LabelSelector{MatchLabels: map[string]string{ - "node.kubernetes.io/instance-type": sku, - }}, - RequiredResources: corev1.ResourceList{corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("1")}, - }, - Script: signalv1.ScriptReference{ - Name: "scripts", - Key: "payload.txt", - Digest: digestString("test payload\n"), - }, - Workflow: signalv1.WorkflowReference{Name: "gpu-cert"}, - Backend: signalv1.ExecutionBackend{APIVersion: "kubeflow.org/v2beta1", Kind: "MPIJob"}, - RunnerImage: "runner@sha256:" + strings.Repeat("a", 64), - Purpose: signalv1.PurposeQuickCertification, - StagePlan: []signalv1.CertificationStage{{ID: "workload", DisplayName: "Workload", Category: "compute", Required: true}}, - EvidencePath: "signal/" + sku, - Deadline: metav1.Duration{Duration: 10 * time.Minute}, - BackendActiveDeadline: metav1.Duration{Duration: 9 * time.Minute}, - MaxAttempts: 2, - }, - } - - profile.Spec.Checks = []signalv1.CertificationCheck{{ - ID: "legacy", - Order: 0, - Script: profile.Spec.Script, - RunnerImage: profile.Spec.RunnerImage, - Backend: profile.Spec.Backend, - WorkloadPolicy: profile.Spec.WorkloadPolicy, - RuntimeStage: profile.Spec.RuntimeStage, - Parameters: profile.Spec.Parameters, - BackendActiveDeadline: profile.Spec.BackendActiveDeadline, - ResultBounds: signalv1.ResultBounds{ - ReasonBytes: 4096, - MeasurementsBytes: 16384, - BackendResourcesBytes: 16384, - BackendEvidenceBytes: 65536, - }, - MaxAttempts: profile.Spec.MaxAttempts, - }} - - return profile -} - -func testRun(name string) *signalv1.CertificationRun { - run := &signalv1.CertificationRun{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: "signal", - UID: types.UID(name + "-uid"), - Finalizers: []string{RunFinalizer}, - }, - Spec: signalv1.CertificationRunSpec{ - Profile: signalv1.PinnedProfile{Name: "gpu-profile", UID: "profile-uid", Generation: 3}, - Participants: []signalv1.Participant{{NodeName: "gpu-0", NodeUID: "gpu-0-uid"}}, - Script: signalv1.LegacyScriptSnapshot{ConfigMapName: name + "-scripts", Key: "payload.txt", Digest: digestString("test payload\n")}, - Workflow: signalv1.WorkflowReference{Name: "gpu-cert"}, - Backend: signalv1.ExecutionBackend{APIVersion: "kubeflow.org/v2beta1", Kind: "MPIJob"}, - RunnerImage: "runner@sha256:" + strings.Repeat("a", 64), - EvidencePath: "signal/" + name + ".json", - Deadline: metav1.Duration{Duration: 10 * time.Minute}, - BackendActiveDeadline: metav1.Duration{Duration: 9 * time.Minute}, - MaxAttempts: 2, - RequestNodeName: "gpu-0", - }, - } - - run.Spec.Checks = []signalv1.CertificationCheckSnapshot{{ - ID: "legacy", - Order: 0, - Script: signalv1.ScriptSnapshot(run.Spec.Script), - RunnerImage: run.Spec.RunnerImage, - Backend: run.Spec.Backend, - WorkloadPolicy: run.Spec.WorkloadPolicy, - RuntimeStage: run.Spec.RuntimeStage, - Parameters: run.Spec.Parameters, - BackendActiveDeadline: run.Spec.BackendActiveDeadline, - ResultBounds: signalv1.ResultBounds{ - ReasonBytes: 4096, - MeasurementsBytes: 16384, - BackendResourcesBytes: 16384, - BackendEvidenceBytes: 65536, - }, - MaxAttempts: run.Spec.MaxAttempts, - }} - - return run -} - -func workflowObject(run *signalv1.CertificationRun, phase, verdict, reason string) *unstructured.Unstructured { - const workflowUID = "workflow-uid" - - run.Status.Workflow.UID = workflowUID - - backendEvidence, err := json.Marshal(map[string]any{ - "version": "v1", - "complete": true, - "participants": make([]map[string]any, len(run.Spec.Participants)), - }) - if err != nil { - panic(err) - } - - workflow := &unstructured.Unstructured{Object: map[string]any{ - "apiVersion": workflowGVK.GroupVersion().String(), - "kind": workflowGVK.Kind, - "metadata": map[string]any{ - "name": run.Status.Workflow.Name, - "namespace": run.Namespace, - "uid": workflowUID, - "ownerReferences": []any{ - map[string]any{ - "apiVersion": signalv1.GroupVersion.String(), - "kind": "CertificationRun", - "name": run.Name, - "uid": string(run.UID), - "controller": true, - "blockOwnerDeletion": true, - }, - }, - }, - "status": map[string]any{ - "phase": phase, - "message": "test workflow message", - "outputs": map[string]any{"parameters": []any{ - map[string]any{"name": "verdict", "value": verdict}, - map[string]any{"name": "reason", "value": reason}, - map[string]any{"name": "measurements", "value": "checked=14"}, - map[string]any{"name": "backend-evidence", "value": string(backendEvidence)}, - }}, - }, - }} - - return workflow -} - -func workflowWithEntrypointOutputs(parameters []any) *unstructured.Unstructured { - return &unstructured.Unstructured{Object: map[string]any{ - "metadata": map[string]any{"name": "run-a1"}, - "status": map[string]any{ - "nodes": map[string]any{ - "run-a1": map[string]any{ - "name": "run-a1", - "outputs": map[string]any{ - "parameters": parameters, - }, - }, - }, - }, - }} -} - -func workflowArgument(workflow *unstructured.Unstructured, name string) string { - parameters, _, _ := unstructured.NestedSlice(workflow.Object, "spec", "arguments", "parameters") - for _, raw := range parameters { - parameter := raw.(map[string]any) - if parameter["name"] == name { - return fmt.Sprint(parameter["value"]) - } - } - - return "" -} - -func ptrTime(value metav1.Time) *metav1.Time { - return &value -} - -func quantity(value string) *resource.Quantity { - result := resource.MustParse(value) - - return &result -} diff --git a/internal/signal/evidence_test.go b/internal/signal/evidence_test.go deleted file mode 100644 index 9f79009eca..0000000000 --- a/internal/signal/evidence_test.go +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -package signal - -import ( - "bytes" - "compress/gzip" - "context" - "io" - "testing" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -func TestKubernetesEvidenceStoreReturnsQueryableRunURIAndDigest(t *testing.T) { - scheme := runtime.NewScheme() - if err := corev1.AddToScheme(scheme); err != nil { - t.Fatal(err) - } - - kube := fake.NewClientBuilder().WithScheme(scheme).Build() - - store, err := NewKubernetesEvidenceStore(kube, "project-signal") - if err != nil { - t.Fatal(err) - } - - data := []byte(`{"verdict":"Pass"}`) - - uri, digest, err := store.Put(context.Background(), "project-signal/example/run-a.json", data) - if err != nil { - t.Fatal(err) - } - - if uri != "kubernetes://project-signal/configmaps/run-a-evidence/evidence.json.gz" { - t.Fatalf("unexpected URI %q", uri) - } - - if digest != digestBytes(data) { - t.Fatalf("digest = %q, want %q", digest, digestBytes(data)) - } - - var evidence corev1.ConfigMap - if err := kube.Get( - context.Background(), - client.ObjectKey{Namespace: "project-signal", Name: "run-a-evidence"}, - &evidence, - ); err != nil { - t.Fatal(err) - } - - if evidence.Immutable == nil || !*evidence.Immutable { - t.Fatal("evidence ConfigMap is mutable") - } - - reader, err := gzip.NewReader(bytes.NewReader(evidence.BinaryData["evidence.json.gz"])) - if err != nil { - t.Fatal(err) - } - - got, err := io.ReadAll(reader) - if err != nil { - t.Fatal(err) - } - - if err := reader.Close(); err != nil { - t.Fatal(err) - } - - if string(got) != string(data) { - t.Fatalf("persisted evidence = %q", got) - } - - recovered, recoveredURI, recoveredDigest, found, err := store.Get( - context.Background(), - "project-signal/example/run-a.json", - ) - if err != nil { - t.Fatal(err) - } - - if !found || - recoveredURI != uri || - recoveredDigest != digest || - !bytes.Equal(recovered, data) { - t.Fatalf( - "recovered evidence = found:%t uri:%q digest:%q data:%q", - found, - recoveredURI, - recoveredDigest, - recovered, - ) - } -} - -func TestKubernetesEvidenceStoreChunksLargeEvidence(t *testing.T) { - scheme := runtime.NewScheme() - if err := corev1.AddToScheme(scheme); err != nil { - t.Fatal(err) - } - - kube := fake.NewClientBuilder().WithScheme(scheme).Build() - - store, err := NewKubernetesEvidenceStore(kube, "project-signal") - if err != nil { - t.Fatal(err) - } - - data := make([]byte, 2*maximumEvidenceChunkBytes) - - state := uint32(1) - for index := range data { - state = state*1664525 + 1013904223 - data[index] = byte(state >> 24) - } - - uri, digest, err := store.Put(context.Background(), "project-signal/example/run-large.json", data) - if err != nil { - t.Fatal(err) - } - - if uri != "kubernetes://project-signal/configmaps/run-large-evidence/manifest.json" { - t.Fatalf("unexpected chunked URI %q", uri) - } - - var manifest corev1.ConfigMap - if err := kube.Get( - context.Background(), - client.ObjectKey{Namespace: "project-signal", Name: "run-large-evidence"}, - &manifest, - ); err != nil { - t.Fatal(err) - } - - if len(manifest.BinaryData["manifest.json"]) == 0 { - t.Fatal("large evidence did not use a chunk manifest") - } - - recovered, recoveredURI, recoveredDigest, found, err := store.Get( - context.Background(), - "project-signal/example/run-large.json", - ) - if err != nil { - t.Fatal(err) - } - - if !found || - recoveredURI != uri || - recoveredDigest != digest || - !bytes.Equal(recovered, data) { - t.Fatalf( - "recovered large evidence = found:%t uri:%q digest:%q bytes:%d", - found, - recoveredURI, - recoveredDigest, - len(recovered), - ) - } -} diff --git a/internal/signal/grouped_controller_test.go b/internal/signal/grouped_controller_test.go deleted file mode 100644 index e87e843b77..0000000000 --- a/internal/signal/grouped_controller_test.go +++ /dev/null @@ -1,873 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -package signal - -import ( - "context" - "encoding/json" - "fmt" - "slices" - "strings" - "testing" - "time" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" -) - -func TestGroupedAdmissionSnapshotsCompletePlanOnce(t *testing.T) { - t.Parallel() - - node := readyNode("gpu-0", "example-gpu") - node.Labels[RequestLabel] = "grouped-profile" - profile := testProfile("grouped-profile", "example-gpu") - profile.Spec.Checks = groupedProfileChecks() - profile.Spec.ExecutionPolicy = signalv1.SuiteExecutionPolicy{OnFailure: signalv1.SuiteFailurePolicyStop} - profile.Spec.Script = signalv1.ScriptReference{} - profile.Spec.Backend = signalv1.ExecutionBackend{} - profile.Spec.RunnerImage = "" - profile.Spec.WorkloadPolicy = signalv1.WorkloadPolicy{} - profile.Spec.StagePlan = nil - profile.Spec.Parameters = nil - profile.Spec.BackendActiveDeadline = metav1.Duration{} - profile.Spec.MaxAttempts = 0 - - source := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, - Data: map[string]string{}, - } - for index := range profile.Spec.Checks { - source.Data[profile.Spec.Checks[index].Script.Key] = fmt.Sprintf("check-%d\n", index) - } - - scheme := testScheme(t) - kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node, profile, source).Build() - reconciler := &AdmissionReconciler{ - Client: kube, - Namespace: "signal", - ScriptNamespace: "signal", - Now: func() time.Time { return time.Unix(100, 0) }, - } - - if _, err := reconciler.Reconcile( - context.Background(), - reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}, - ); err != nil { - t.Fatal(err) - } - - var runs signalv1.CertificationRunList - if err := kube.List(context.Background(), &runs); err != nil { - t.Fatal(err) - } - - if len(runs.Items) != 1 || len(runs.Items[0].Spec.Checks) != groupedCheckCount { - t.Fatalf("runs = %#v", runs.Items) - } - - run := &runs.Items[0] - - var snapshot corev1.ConfigMap - if err := kube.Get( - context.Background(), - client.ObjectKey{Namespace: run.Namespace, Name: run.Spec.Checks[0].Script.ConfigMapName}, - &snapshot, - ); err != nil { - t.Fatal(err) - } - - if snapshot.Immutable == nil || !*snapshot.Immutable || len(snapshot.Data) != groupedCheckCount { - t.Fatalf("snapshot = immutable:%v keys:%d", snapshot.Immutable, len(snapshot.Data)) - } - - for index := range run.Spec.Checks { - check := &run.Spec.Checks[index] - if check.Order != int32(index) || - check.Script.Digest != digestString(fmt.Sprintf("check-%d\n", index)) || - snapshot.Data[check.Script.Key] != fmt.Sprintf("check-%d\n", index) { - t.Fatalf("check %d = %#v", index, check) - } - } -} - -func TestGroupedAdmissionRejectsScriptDigestMismatch(t *testing.T) { - t.Parallel() - - node := readyNode("gpu-0", "example-gpu") - node.Labels[RequestLabel] = "grouped-profile" - profile := testProfile("grouped-profile", "example-gpu") - profile.Spec.Checks = groupedProfileChecks() - profile.Spec.ExecutionPolicy = signalv1.SuiteExecutionPolicy{OnFailure: signalv1.SuiteFailurePolicyStop} - - source := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, - Data: map[string]string{}, - } - for index := range profile.Spec.Checks { - source.Data[profile.Spec.Checks[index].Script.Key] = fmt.Sprintf("check-%d\n", index) - } - - source.Data[profile.Spec.Checks[8].Script.Key] = "changed\n" - - kube := fake.NewClientBuilder().WithScheme(testScheme(t)).WithObjects(node, profile, source).Build() - reconciler := &AdmissionReconciler{ - Client: kube, - Namespace: "signal", - ScriptNamespace: "signal", - } - - _, err := reconciler.Reconcile( - context.Background(), - reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}, - ) - if err == nil || !strings.Contains(err.Error(), "source script digest for check R09") { - t.Fatalf("Reconcile error = %v", err) - } - - var got corev1.Node - if getErr := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &got); getErr != nil { - t.Fatal(getErr) - } - - if got.Annotations[ReservationLabel] != "" { - t.Fatalf("reservation remained after digest rejection: %#v", got.Annotations) - } -} - -func TestParticipantRoleAndTopologyConstraints(t *testing.T) { - t.Parallel() - - service := readyNode("service", "example-gpu") - service.Labels["signal.example/role"] = "service" - service.Labels["topology.example/zone"] = "zone-a" - witnessA := readyNode("witness-a", "example-gpu") - witnessA.Labels["signal.example/role"] = "witness" - witnessA.Labels["topology.example/zone"] = "zone-a" - witnessB := readyNode("witness-b", "example-gpu") - witnessB.Labels["signal.example/role"] = "witness" - witnessB.Labels["topology.example/zone"] = "zone-b" - witnessC := readyNode("witness-c", "example-gpu") - witnessC.Labels["signal.example/role"] = "witness" - witnessC.Labels["topology.example/zone"] = "zone-b" - - participants := []signalv1.Participant{ - participantSnapshot(service, 0), - participantSnapshot(witnessA, 1), - participantSnapshot(witnessB, 2), - participantSnapshot(witnessC, 3), - } - kube := fake.NewClientBuilder(). - WithScheme(testScheme(t)). - WithObjects(service, witnessA, witnessB, witnessC). - Build() - reconciler := &AdmissionReconciler{Client: kube} - - err := reconciler.assignParticipantRoles( - context.Background(), - participants, - []signalv1.ParticipantRoleConstraint{ - { - Name: "service", - Count: 1, - Selector: metav1.LabelSelector{MatchLabels: map[string]string{"signal.example/role": "service"}}, - }, - { - Name: "witness", - Count: 3, - Selector: metav1.LabelSelector{MatchLabels: map[string]string{"signal.example/role": "witness"}}, - }, - }, - []signalv1.ParticipantTopologyConstraint{{ - Role: "witness", - LabelKey: "topology.example/zone", - MinDistinctValues: 2, - }}, - ) - if err != nil { - t.Fatal(err) - } - - if !slices.Equal(participants[0].Roles, []string{"service"}) { - t.Fatalf("service roles = %#v", participants[0].Roles) - } - - for index := 1; index < len(participants); index++ { - if !slices.Equal(participants[index].Roles, []string{"witness"}) { - t.Fatalf("witness %d roles = %#v", index, participants[index].Roles) - } - } -} - -func TestGroupedWorkflowBuildsVisibleSerialDAG(t *testing.T) { - t.Parallel() - - run := groupedTestRun("grouped") - run.Spec.ExecutionPolicy.OnFailure = signalv1.SuiteFailurePolicyContinue - run.Spec.Participants = []signalv1.Participant{ - {Index: 0, NodeName: "service", NodeUID: "service-uid", Roles: []string{"service"}}, - {Index: 1, NodeName: "witness-a", NodeUID: "witness-a-uid", Roles: []string{"witness"}}, - {Index: 2, NodeName: "witness-b", NodeUID: "witness-b-uid", Roles: []string{"witness"}}, - {Index: 3, NodeName: "witness-c", NodeUID: "witness-c-uid", Roles: []string{"witness"}}, - } - run.Spec.Checks[14].ParticipantPolicy.ExcludedRoles = []string{"service"} - - workflow, err := buildWorkflow(run, 1) - if err != nil { - t.Fatal(err) - } - - serviceAccountName, found, err := unstructured.NestedString( - workflow.Object, - "spec", - "serviceAccountName", - ) - if err != nil || !found || serviceAccountName != adapterServiceAccountName { - t.Fatalf( - "service account: found=%t value=%q err=%v", - found, - serviceAccountName, - err, - ) - } - - tasks, found, err := unstructured.NestedSlice( - workflow.Object, - "spec", - "templates", - ) - if err != nil || !found || len(tasks) != 1 { - t.Fatalf("templates: found=%t len=%d err=%v", found, len(tasks), err) - } - - template := tasks[0].(map[string]any) - dag := template["dag"].(map[string]any) - - dagTasks := dag["tasks"].([]any) - if len(dagTasks) != groupedCheckCount { - t.Fatalf("DAG tasks = %d, want %d", len(dagTasks), groupedCheckCount) - } - - for index := range dagTasks { - task := dagTasks[index].(map[string]any) - if index > 0 { - previous := dagTasks[index-1].(map[string]any)["name"].(string) - - want := fmt.Sprintf( - "%[1]s.Succeeded || %[1]s.Failed || %[1]s.Errored || %[1]s.Skipped || %[1]s.Omitted", - previous, - ) - if task["depends"] != want { - t.Fatalf("task %d dependency = %q, want %q", index, task["depends"], want) - } - } - - templateRef := task["templateRef"].(map[string]any) - if templateRef["name"] != run.Spec.Workflow.Name || templateRef["template"] != "certification" { - t.Fatalf("task %d templateRef = %#v", index, templateRef) - } - } - - last := dagTasks[14].(map[string]any) - parameters := last["arguments"].(map[string]any)["parameters"].([]any) - - values := parameterValues(parameters) - if values["participant-count"] != "3" { - t.Fatalf("row 15 participant count = %q, want 3 witnesses", values["participant-count"]) - } -} - -func TestGroupedWorkflowStopPolicyRequiresDiagnosticPass(t *testing.T) { - t.Parallel() - - run := groupedTestRun("grouped-stop") - run.Spec.ExecutionPolicy.OnFailure = signalv1.SuiteFailurePolicyStop - - workflow, err := buildWorkflow(run, 1) - if err != nil { - t.Fatal(err) - } - - templates, _, _ := unstructured.NestedSlice(workflow.Object, "spec", "templates") - tasks := templates[0].(map[string]any)["dag"].(map[string]any)["tasks"].([]any) - - for index := 1; index < len(tasks); index++ { - previous := tasks[index-1].(map[string]any)["name"].(string) - - task := tasks[index].(map[string]any) - if task["depends"] != previous+".Succeeded" { - t.Fatalf("task %d dependency = %q", index, task["depends"]) - } - - wantWhen := "{{tasks." + previous + ".outputs.parameters.verdict}} == Pass" - if task["when"] != wantWhen { - t.Fatalf("task %d when = %q, want %q", index, task["when"], wantWhen) - } - } -} - -func TestGroupedTerminalAggregationRequiresExactCompleteCoverage(t *testing.T) { - t.Parallel() - - run := groupedTestRun("aggregate") - workflow := groupedWorkflow(run, "Succeeded") - - statuses, verdict, reason := terminalCheckStatuses(workflow, run) - if verdict != "Pass" || reason != "AllChecksPassed" { - t.Fatalf("aggregate = %s %s", verdict, reason) - } - - if len(statuses) != groupedCheckCount { - t.Fatalf("statuses = %d", len(statuses)) - } - - for index := range statuses { - if statuses[index].Phase != signalv1.CheckPhaseSucceeded || - !statuses[index].BackendEvidence.Complete || - !statuses[index].Cleanup.Complete || - len(statuses[index].BackendResources) != 1 { - t.Fatalf("status %d = %#v", index, statuses[index]) - } - } -} - -func TestGroupedTerminalAggregationMapsFilteredParticipantIndexes(t *testing.T) { - t.Parallel() - - run := groupedTestRun("participant-indexes") - run.Spec.Participants = []signalv1.Participant{ - {Index: 0, NodeName: "service", NodeUID: "service-uid", Roles: []string{"service"}}, - {Index: 1, NodeName: "witness-a", NodeUID: "witness-a-uid", Roles: []string{"witness"}}, - {Index: 2, NodeName: "witness-b", NodeUID: "witness-b-uid", Roles: []string{"witness"}}, - {Index: 3, NodeName: "witness-c", NodeUID: "witness-c-uid", Roles: []string{"witness"}}, - } - run.Spec.Checks[14].ParticipantPolicy.ExcludedRoles = []string{"service"} - - workflow := groupedWorkflow(run, "Succeeded") - node := groupedNode(workflow, run, 14) - evidence, _ := json.Marshal(map[string]any{ - "version": "v1", - "complete": true, - "participants": []any{ - groupedEvidenceParticipant(run, 14, 0), - groupedEvidenceParticipant(run, 14, 1), - groupedEvidenceParticipant(run, 14, 2), - }, - }) - resources, _ := json.Marshal([]signalv1.BackendResourceReference{{ - APIVersion: "v1", - Kind: "Pod", - Namespace: run.Namespace, - Name: "row-15", - UID: "row-15-uid", - ParticipantIndexes: []int32{0, 1, 2}, - }}) - - setNodeParameter(node, "backend-evidence", string(evidence)) - setNodeParameter(node, "backend-resources", string(resources)) - - statuses, _, _ := terminalCheckStatuses(workflow, run) - status := statuses[14] - - if !slices.Equal(status.BackendResources[0].ParticipantIndexes, []int32{1, 2, 3}) { - t.Fatalf("backend participant indexes = %#v", status.BackendResources[0].ParticipantIndexes) - } - - var normalized struct { - Participants []struct { - ParticipantIndex int32 `json:"participantIndex"` - } `json:"participants"` - } - if err := json.Unmarshal([]byte(status.BackendEvidence.Content), &normalized); err != nil { - t.Fatal(err) - } - - got := make([]int32, len(normalized.Participants)) - for index := range normalized.Participants { - got[index] = normalized.Participants[index].ParticipantIndex - } - - if !slices.Equal(got, []int32{1, 2, 3}) { - t.Fatalf("evidence participant indexes = %#v", got) - } -} - -func TestGroupedTerminalAggregationRetainsFailureAfterPolicyStop(t *testing.T) { - t.Parallel() - - run := groupedTestRun("stop-after-fail") - run.Spec.ExecutionPolicy.OnFailure = signalv1.SuiteFailurePolicyStop - workflow := groupedWorkflow(run, "Succeeded") - setNodeParameter(groupedNode(workflow, run, 2), "verdict", "Fail") - - for index := 3; index < groupedCheckCount; index++ { - groupedNode(workflow, run, index)["phase"] = "Omitted" - } - - statuses, verdict, reason := terminalCheckStatuses(workflow, run) - if verdict != "Fail" || reason != "OneOrMoreChecksFailed" { - t.Fatalf("aggregate = %q %q", verdict, reason) - } - - if statuses[2].Phase != signalv1.CheckPhaseFailed { - t.Fatalf("failed check status = %#v", statuses[2]) - } - - for index := 3; index < groupedCheckCount; index++ { - if statuses[index].Phase != signalv1.CheckPhaseSkipped || !statuses[index].NotRun { - t.Fatalf("stopped check %d = %#v", index, statuses[index]) - } - } -} - -func TestCompactCheckStatusesRetainsEvidenceDigest(t *testing.T) { - t.Parallel() - - statuses := []signalv1.CheckStatus{{ - BackendEvidence: signalv1.BackendEvidenceStatus{ - Version: "v1", - Complete: true, - Digest: contentDigest(`{"version":"v1"}`), - Content: `{"version":"v1"}`, - }, - }} - - compacted := compactCheckStatuses(statuses) - if compacted[0].BackendEvidence.Content != "" { - t.Fatalf("compacted backend evidence retained content") - } - - if compacted[0].BackendEvidence.Digest != statuses[0].BackendEvidence.Digest { - t.Fatalf("compacted digest = %q, want %q", compacted[0].BackendEvidence.Digest, statuses[0].BackendEvidence.Digest) - } - - if statuses[0].BackendEvidence.Content == "" { - t.Fatalf("compaction mutated source status") - } -} - -func TestGroupedReconcileRecoversPublishedEvidenceBeforeCancellation(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - run := groupedTestRun("recover-evidence") - run.UID = "recover-evidence-uid" - run.Spec.Cancel = true - run.Spec.Participants = []signalv1.Participant{{ - Index: 0, - NodeName: "gpu-0", - NodeUID: "gpu-0-uid", - }} - run.Status.Workflow = signalv1.WorkflowStatus{Name: run.Name + "-a1", UID: "workflow-uid", Attempt: 1, Phase: "Succeeded"} - - finishedAt := time.Date(2026, 10, 22, 10, 0, 0, 0, time.UTC) - - statuses := initialCheckStatuses(run.Spec.Checks) - for index := range statuses { - statuses[index].Phase = signalv1.CheckPhaseSucceeded - statuses[index].Verdict = "Pass" - statuses[index].BackendEvidence = signalv1.BackendEvidenceStatus{ - Version: "v1", - Complete: true, - Digest: contentDigest(`{"version":"v1"}`), - Content: `{"version":"v1"}`, - } - } - - document, err := json.Marshal(map[string]any{ - "runID": string(run.UID), - "runName": run.Name, - "workflow": run.Status.Workflow, - "checkStatus": statuses, - "executionState": "Completed", - "verdict": "Pass", - "reason": "AllChecksPassed", - "finishedAt": finishedAt.Format(time.RFC3339Nano), - }) - if err != nil { - t.Fatal(err) - } - - store := &immutableMemoryEvidenceStore{} - if _, _, err := store.Put(context.Background(), run.Spec.EvidencePath, document); err != nil { - t.Fatal(err) - } - - node := readyNode("gpu-0", "example-gpu") - node.UID = "gpu-0-uid" - kube := fake.NewClientBuilder(). - WithScheme(scheme). - WithStatusSubresource(&signalv1.CertificationRun{}). - WithObjects(run, node). - Build() - reconciler := &RunReconciler{Client: kube, Evidence: store} - - if _, err := reconciler.Reconcile( - context.Background(), - reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}, - ); err != nil { - t.Fatal(err) - } - - var got signalv1.CertificationRun - if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { - t.Fatal(err) - } - - if got.Status.Phase != signalv1.RunPhaseSucceeded || - got.Status.Verdict != "Pass" || - got.Status.ExecutionState != "Completed" { - t.Fatalf("recovered status = %#v", got.Status) - } - - if got.Status.Checks[0].BackendEvidence.Content != "" || - got.Status.Checks[0].BackendEvidence.Digest == "" { - t.Fatalf("recovered check evidence was not compacted: %#v", got.Status.Checks[0].BackendEvidence) - } -} - -func TestGroupedTerminalAggregationFailsClosed(t *testing.T) { - t.Parallel() - - tests := map[string]func(*unstructured.Unstructured, *signalv1.CertificationRun){ - "missing task": func(workflow *unstructured.Unstructured, run *signalv1.CertificationRun) { - nodes := workflow.Object["status"].(map[string]any)["nodes"].(map[string]any) - delete(nodes, string(run.Spec.Checks[3].ID)) - }, - "duplicate identity": func(workflow *unstructured.Unstructured, run *signalv1.CertificationRun) { - nodes := workflow.Object["status"].(map[string]any)["nodes"].(map[string]any) - nodes["duplicate"] = nodes[string(run.Spec.Checks[4].ID)] - }, - "cleanup failure": func(workflow *unstructured.Unstructured, run *signalv1.CertificationRun) { - node := groupedNode(workflow, run, 5) - setNodeParameter(node, "cleanup-failure", "BackendCleanupFailed") - }, - "malformed evidence": func(workflow *unstructured.Unstructured, run *signalv1.CertificationRun) { - node := groupedNode(workflow, run, 6) - setNodeParameter(node, "backend-evidence", "{") - }, - "skipped": func(workflow *unstructured.Unstructured, run *signalv1.CertificationRun) { - groupedNode(workflow, run, 7)["phase"] = "Omitted" - }, - } - - for name, mutate := range tests { - t.Run(name, func(t *testing.T) { - t.Parallel() - - run := groupedTestRun("fail-closed") - workflow := groupedWorkflow(run, "Succeeded") - mutate(workflow, run) - - statuses, verdict, _ := terminalCheckStatuses(workflow, run) - if verdict != "Inconclusive" { - t.Fatalf("verdict = %q", verdict) - } - - found := false - - for index := range statuses { - if statuses[index].Phase == signalv1.CheckPhaseInconclusive || - statuses[index].Phase == signalv1.CheckPhaseSkipped { - found = true - } - } - - if !found { - t.Fatalf("statuses did not retain fail-closed check state: %#v", statuses) - } - }) - } -} - -func TestSelectParticipantPlanFindsSatisfiableTopology(t *testing.T) { - t.Parallel() - - candidates := []corev1.Node{ - participantNode("service", "service", "zone-a"), - participantNode("witness-a", "witness", "zone-a"), - participantNode("witness-b", "witness", "zone-a"), - participantNode("witness-c", "witness", "zone-a"), - participantNode("witness-d", "witness", "zone-b"), - } - - participants, err := selectParticipantPlan( - candidates, - "service", - 4, - []signalv1.ParticipantRoleConstraint{ - { - Name: "service", - Count: 1, - Selector: metav1.LabelSelector{MatchLabels: map[string]string{"signal.example/role": "service"}}, - }, - { - Name: "witness", - Count: 3, - Selector: metav1.LabelSelector{MatchLabels: map[string]string{"signal.example/role": "witness"}}, - }, - }, - []signalv1.ParticipantTopologyConstraint{{ - Role: "witness", - LabelKey: "topology.example/zone", - MinDistinctValues: 2, - }}, - ) - if err != nil { - t.Fatal(err) - } - - names := make([]string, len(participants)) - for index := range participants { - names[index] = participants[index].NodeName - } - - if !slices.Contains(names, "witness-d") { - t.Fatalf("selected participants = %#v, missing satisfiable zone-b witness", names) - } -} - -func TestSelectParticipantPlanPrioritizesDistinctTopologyDomains(t *testing.T) { - t.Parallel() - - candidates := []corev1.Node{participantNode("service", "service", "zone-a")} - for index := 1; index < 63; index++ { - candidates = append(candidates, participantNode( - fmt.Sprintf("witness-%02d", index), - "witness", - "zone-a", - )) - } - - candidates = append(candidates, participantNode("witness-63", "witness", "zone-b")) - - participants, err := selectParticipantPlan( - candidates, - "service", - 7, - []signalv1.ParticipantRoleConstraint{ - { - Name: "service", - Count: 1, - Selector: metav1.LabelSelector{MatchLabels: map[string]string{"signal.example/role": "service"}}, - }, - { - Name: "witness", - Count: 6, - Selector: metav1.LabelSelector{MatchLabels: map[string]string{"signal.example/role": "witness"}}, - }, - }, - []signalv1.ParticipantTopologyConstraint{{ - Role: "witness", - LabelKey: "topology.example/zone", - MinDistinctValues: 2, - }}, - ) - if err != nil { - t.Fatal(err) - } - - names := make([]string, len(participants)) - for index := range participants { - names[index] = participants[index].NodeName - } - - if !slices.Contains(names, "witness-63") { - t.Fatalf("selected participants = %#v, missing late distinct domain", names) - } -} - -func groupedTestRun(name string) *signalv1.CertificationRun { - run := testRun(name) - run.Spec.Script = signalv1.LegacyScriptSnapshot{} - run.Spec.Backend = signalv1.ExecutionBackend{} - run.Spec.RunnerImage = "" - run.Spec.WorkloadPolicy = signalv1.WorkloadPolicy{} - run.Spec.Parameters = nil - run.Spec.BackendActiveDeadline = metav1.Duration{} - run.Spec.MaxAttempts = 0 - run.Spec.ExecutionPolicy = signalv1.SuiteExecutionPolicy{OnFailure: signalv1.SuiteFailurePolicyStop} - run.Spec.Checks = make([]signalv1.CertificationCheckSnapshot, groupedCheckCount) - - for index := range run.Spec.Checks { - run.Spec.Checks[index] = signalv1.CertificationCheckSnapshot{ - ID: signalv1.CheckID(fmt.Sprintf("R%02d", index+1)), - Order: int32(index), - Script: signalv1.ScriptSnapshot{ - ConfigMapName: name + "-scripts", - Key: fmt.Sprintf("%02d-check", index), - Digest: digestString(fmt.Sprintf("check-%d\n", index)), - }, - RunnerImage: "runner@sha256:" + stringsRepeat("a", 64), - Backend: signalv1.ExecutionBackend{APIVersion: "v1", Kind: "Pod"}, - BackendActiveDeadline: metav1.Duration{Duration: 9 * time.Minute}, - ResultBounds: signalv1.ResultBounds{ - ReasonBytes: 4096, - MeasurementsBytes: 16384, - BackendResourcesBytes: 16384, - BackendEvidenceBytes: 65536, - }, - MaxAttempts: 1, - } - } - - return run -} - -func groupedProfileChecks() []signalv1.CertificationCheck { - checks := make([]signalv1.CertificationCheck, groupedCheckCount) - for index := range checks { - checks[index] = signalv1.CertificationCheck{ - ID: signalv1.CheckID(fmt.Sprintf("R%02d", index+1)), - Order: int32(index), - Script: signalv1.ScriptReference{ - Name: "scripts", - Key: fmt.Sprintf("check-%02d.sh", index+1), - Digest: digestString(fmt.Sprintf("check-%d\n", index)), - }, - RunnerImage: "runner@sha256:" + stringsRepeat("a", 64), - Backend: signalv1.ExecutionBackend{APIVersion: "v1", Kind: "Pod"}, - BackendActiveDeadline: metav1.Duration{Duration: 9 * time.Minute}, - ResultBounds: signalv1.ResultBounds{ - ReasonBytes: 4096, - MeasurementsBytes: 16384, - BackendResourcesBytes: 16384, - BackendEvidenceBytes: 65536, - }, - MaxAttempts: 1, - } - } - - return checks -} - -func groupedWorkflow(run *signalv1.CertificationRun, phase string) *unstructured.Unstructured { - nodes := make(map[string]any, len(run.Spec.Checks)) - - for index := range run.Spec.Checks { - check := &run.Spec.Checks[index] - taskName := checkTaskName(check) - evidence, _ := json.Marshal(map[string]any{ - "version": "v1", - "complete": true, - "participants": []any{map[string]any{ - "participantIndex": 0, - "resource": map[string]any{ - "apiVersion": "v1", - "kind": "Pod", - "namespace": run.Namespace, - "name": taskName, - "uid": "uid-" + taskName, - }, - "pods": []any{}, - }}, - }) - resources, _ := json.Marshal([]signalv1.BackendResourceReference{{ - APIVersion: "v1", - Kind: "Pod", - Namespace: run.Namespace, - Name: taskName, - UID: types.UID("uid-" + taskName), - ParticipantIndexes: []int32{0}, - }}) - nodes[string(check.ID)] = map[string]any{ - "name": run.Name + "-a1." + taskName, - "phase": "Succeeded", - "startedAt": "2026-10-22T10:00:00Z", - "finishedAt": "2026-10-22T10:00:01Z", - "outputs": map[string]any{"parameters": []any{ - map[string]any{"name": "verdict", "value": "Pass"}, - map[string]any{"name": "reason", "value": "Healthy"}, - map[string]any{"name": "measurements", "value": "{}"}, - map[string]any{"name": "backend-resources", "value": string(resources)}, - map[string]any{"name": "backend-evidence", "value": string(evidence)}, - map[string]any{"name": "cleanup-failure", "value": ""}, - }}, - } - } - - return &unstructured.Unstructured{Object: map[string]any{ - "metadata": map[string]any{"name": run.Name + "-a1"}, - "status": map[string]any{ - "phase": phase, - "nodes": nodes, - }, - }} -} - -func groupedNode( - workflow *unstructured.Unstructured, - run *signalv1.CertificationRun, - index int, -) map[string]any { - return workflow.Object["status"].(map[string]any)["nodes"].(map[string]any)[string(run.Spec.Checks[index].ID)].(map[string]any) -} - -func groupedEvidenceParticipant( - run *signalv1.CertificationRun, - checkIndex int, - participantIndex int, -) map[string]any { - taskName := checkTaskName(&run.Spec.Checks[checkIndex]) - - return map[string]any{ - "participantIndex": participantIndex, - "resource": map[string]any{ - "apiVersion": "v1", - "kind": "Pod", - "namespace": run.Namespace, - "name": fmt.Sprintf("%s-p%d", taskName, participantIndex), - "uid": fmt.Sprintf("uid-%s-p%d", taskName, participantIndex), - }, - "pods": []any{}, - } -} - -func participantNode(name, role, zone string) corev1.Node { - return corev1.Node{ObjectMeta: metav1.ObjectMeta{ - Name: name, - UID: types.UID(name + "-uid"), - Labels: map[string]string{ - "signal.example/role": role, - "topology.example/zone": zone, - }, - }} -} - -func setNodeParameter(node map[string]any, name, value string) { - parameters := node["outputs"].(map[string]any)["parameters"].([]any) - for _, raw := range parameters { - parameter := raw.(map[string]any) - if parameter["name"] == name { - parameter["value"] = value - - return - } - } -} - -func parameterValues(parameters []any) map[string]string { - values := make(map[string]string, len(parameters)) - for _, raw := range parameters { - parameter := raw.(map[string]any) - values[parameter["name"].(string)] = parameter["value"].(string) - } - - return values -} - -func stringsRepeat(value string, count int) string { - result := "" - for range count { - result += value - } - - return result -} diff --git a/internal/signal/policy.go b/internal/signal/policy.go deleted file mode 100644 index bdcd9d39a4..0000000000 --- a/internal/signal/policy.go +++ /dev/null @@ -1,718 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -package signal - -import ( - "context" - "fmt" - "slices" - "sort" - "time" - - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - apimeta "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - - machinav1 "github.com/Azure/unbounded/api/machina/v1alpha3" - signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" -) - -const defaultPolicyEvaluationInterval = time.Minute - -type RecurringReconciler struct { - client.Client - Reader client.Reader - Namespace string - Admission *AdmissionReconciler - Now func() time.Time -} - -func (r *RecurringReconciler) SetupWithManager(mgr ctrl.Manager) error { - if r.Reader == nil { - r.Reader = mgr.GetAPIReader() - } - - return ctrl.NewControllerManagedBy(mgr). - For(&signalv1.CertificationProfile{}). - WithOptions(controller.Options{MaxConcurrentReconciles: 1}). - Complete(r) -} - -func (r *RecurringReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - var profile signalv1.CertificationProfile - if err := r.Get(ctx, req.NamespacedName, &profile); err != nil { - return ctrl.Result{}, client.IgnoreNotFound(err) - } - - policy := profile.Spec.Recurring - if policy == nil || !policy.Enabled { - return ctrl.Result{}, nil - } - - now := r.now() - - interval := policy.EvaluationInterval.Duration - if interval <= 0 { - interval = defaultPolicyEvaluationInterval - } - - before := profile.DeepCopy() - profile.Status.ObservedGeneration = profile.Generation - - if err := validateRecurringPolicy(policy); err != nil { - setProfileCondition(&profile, now, metav1.ConditionFalse, "InvalidPolicy", err.Error()) - - return ctrl.Result{RequeueAfter: interval}, r.Status().Patch(ctx, &profile, client.MergeFrom(before)) - } - - if policyExpired(policy, now) { - for index := range profile.Status.Nodes { - profile.Status.Nodes[index].Suspended = true - profile.Status.Nodes[index].Reason = "PolicyExpired" - profile.Status.Nodes[index].NextDueAt = nil - } - - setProfileCondition(&profile, now, metav1.ConditionFalse, "Expired", - fmt.Sprintf("recurring admission expired at %s", policy.ExpiresAt.UTC().Format(time.RFC3339))) - - if err := r.Status().Patch(ctx, &profile, client.MergeFrom(before)); err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{}, r.projectStatuses(ctx, &profile) - } - - if r.Admission == nil { - return ctrl.Result{}, fmt.Errorf("admission reconciler is required") - } - - r.Admission.admissionMu.Lock() - defer r.Admission.admissionMu.Unlock() - - var runs signalv1.CertificationRunList - if err := r.Reader.List(ctx, &runs, client.InNamespace(r.Namespace)); err != nil { - return ctrl.Result{}, fmt.Errorf("list CertificationRuns: %w", err) - } - - activeByProfile, activeByDomain := activeBudgets(runs.Items, &profile) - if policy.FailureDomainLabel != "" { - for index := range runs.Items { - run := &runs.Items[index] - if run.Spec.Profile.Name != profile.Name || run.Status.Phase.Terminal() { - continue - } - - for _, participant := range run.Spec.Participants { - var node corev1.Node - if err := r.Reader.Get(ctx, client.ObjectKey{Name: participant.NodeName}, &node); err != nil { - return ctrl.Result{}, fmt.Errorf( - "get active participant %s for failure-domain accounting: %w", - participant.NodeName, - err, - ) - } - - activeByDomain[node.Labels[policy.FailureDomainLabel]]++ - } - } - } - - statuses := make([]signalv1.RecurringNodeStatus, 0, len(policy.NodeNames)) - - for _, nodeName := range policy.NodeNames { - old := recurringNodeStatus(profile.Status.Nodes, nodeName) - - status, admitted, err := r.evaluateNode(ctx, &profile, nodeName, old, runs.Items, activeByProfile, activeByDomain, now) - if err != nil { - status.Reason = "EvaluationError: " + err.Error() - statuses = append(statuses, status) - - continue - } - - statuses = append(statuses, status) - - if admitted { - activeByProfile++ - - if policy.FailureDomainLabel != "" { - if err := r.accountAdmittedFailureDomains(ctx, nodeName, policy.FailureDomainLabel, activeByDomain); err != nil { - return ctrl.Result{}, err - } - } - } - } - - profile.Status.Nodes = statuses - setProfileCondition(&profile, now, metav1.ConditionTrue, "Evaluated", - fmt.Sprintf("evaluated %d explicitly opted-in nodes", len(statuses))) - - if err := r.Status().Patch(ctx, &profile, client.MergeFrom(before)); err != nil { - return ctrl.Result{}, err - } - - if err := r.projectStatuses(ctx, &profile); err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{RequeueAfter: interval}, nil -} - -func (r *RecurringReconciler) accountAdmittedFailureDomains( - ctx context.Context, - requestNodeName string, - failureDomainLabel string, - activeByDomain map[string]int32, -) error { - var requestNode corev1.Node - if err := r.Reader.Get(ctx, client.ObjectKey{Name: requestNodeName}, &requestNode); err != nil { - return fmt.Errorf("get admitted request Node %s for failure-domain accounting: %w", requestNodeName, err) - } - - runName := requestNode.Annotations[ReservationLabel] - if runName == "" { - return fmt.Errorf("admitted request Node %s has no reservation", requestNodeName) - } - - var run signalv1.CertificationRun - if err := r.Reader.Get(ctx, client.ObjectKey{Namespace: r.Namespace, Name: runName}, &run); err != nil { - return fmt.Errorf("get admitted run %s for failure-domain accounting: %w", runName, err) - } - - for _, participant := range run.Spec.Participants { - var node corev1.Node - if err := r.Reader.Get(ctx, client.ObjectKey{Name: participant.NodeName}, &node); err != nil { - return fmt.Errorf( - "get admitted participant %s for failure-domain accounting: %w", - participant.NodeName, - err, - ) - } - - if participant.NodeUID != "" && node.UID != participant.NodeUID { - return fmt.Errorf("admitted participant %s identity changed", participant.NodeName) - } - - activeByDomain[node.Labels[failureDomainLabel]]++ - } - - return nil -} - -func (r *RecurringReconciler) projectStatuses(ctx context.Context, profile *signalv1.CertificationProfile) error { - for _, status := range profile.Status.Nodes { - if err := projectRecurring(ctx, r.Client, profile, status); err != nil { - return err - } - } - - return nil -} - -func (r *RecurringReconciler) evaluateNode( - ctx context.Context, - profile *signalv1.CertificationProfile, - nodeName string, - old signalv1.RecurringNodeStatus, - runs []signalv1.CertificationRun, - activeByProfile int32, - activeByDomain map[string]int32, - now time.Time, -) (signalv1.RecurringNodeStatus, bool, error) { - status := old - status.NodeName = nodeName - evaluated := metav1.NewTime(now) - status.LastEvaluatedAt = &evaluated - status.Suspended = false - - var node corev1.Node - if err := r.Reader.Get(ctx, client.ObjectKey{Name: nodeName}, &node); err != nil { - status.IdleSince = nil - - return status, false, fmt.Errorf("get Node: %w", err) - } - - selector, err := metav1.LabelSelectorAsSelector(&profile.Spec.Participants.Selector) - if err != nil { - return status, false, fmt.Errorf("participant selector: %w", err) - } - - if !selector.Matches(labels.Set(node.Labels)) || !nodeEligible(&node, profile.Spec.Participants.RequiredResources) { - status.IdleSince = nil - status.Reason = "NodeIneligible" - - return status, false, nil - } - - busy, reason, err := r.nodeBusy(ctx, &node) - if err != nil { - status.IdleSince = nil - - return status, false, err - } - - if busy { - status.IdleSince = nil - status.Reason = reason - - return status, false, nil - } - - if status.IdleSince == nil { - idleSince := metav1.NewTime(now) - status.IdleSince = &idleSince - } - - nodeRuns := runsForNodeProfile(runs, nodeName, profile.Name) - lastSuccess, consecutiveFailures, latestFailure := runHistory(nodeRuns) - status.ConsecutiveFailures = consecutiveFailures - - if lastSuccess != nil { - status.LastSuccessfulRun = lastSuccess.Name - status.LastSuccessfulAt = lastSuccess.Status.FinishedAt - } - - due := dueAt(profile.Spec.Recurring, status.IdleSince, lastSuccess) - status.NextDueAt = ptrMetaTime(due) - - if suspended, reason := failureGate(profile.Spec.Recurring, consecutiveFailures, latestFailure, now); reason != "" { - status.Suspended = suspended - status.Reason = reason - - return status, false, nil - } - - if now.Before(due) { - status.Reason = "NotDue" - - return status, false, nil - } - - if activeRunForNode(runs, nodeName) || node.Annotations[ReservationLabel] != "" { - status.Reason = "RunActive" - - return status, false, nil - } - - if excludedRunActive(runs, nodeName, profile.Spec.Recurring.ExclusiveProfiles) { - status.Reason = "ExclusiveProfileActive" - - return status, false, nil - } - - if activeByProfile >= profile.Spec.Recurring.MaxConcurrent { - status.Reason = "ConcurrencyBudget" - - return status, false, nil - } - - domain := node.Labels[profile.Spec.Recurring.FailureDomainLabel] - if profile.Spec.Recurring.FailureDomainLabel != "" && - profile.Spec.Recurring.MaxConcurrentPerFailureDomain > 0 && - activeByDomain[domain] >= profile.Spec.Recurring.MaxConcurrentPerFailureDomain { - status.Reason = "FailureDomainBudget" - - return status, false, nil - } - - if profile.Spec.Recurring.FailureDomainLabel != "" && - profile.Spec.Recurring.MaxConcurrentPerFailureDomain > 0 { - participants, err := r.Admission.selectParticipants(ctx, &node, profile) - if err != nil { - return status, false, fmt.Errorf("select recurring participants: %w", err) - } - - fits, err := r.participantsFitFailureDomainBudget( - ctx, - participants, - profile.Spec.Recurring.FailureDomainLabel, - profile.Spec.Recurring.MaxConcurrentPerFailureDomain, - activeByDomain, - ) - if err != nil { - return status, false, err - } - - if !fits { - status.Reason = "FailureDomainBudget" - - return status, false, nil - } - } - - if err := r.Admission.admitNodeWithTriggerLocked(ctx, &node, profile.Name, "recurring"); err != nil { - if apierrors.IsAlreadyExists(err) || apierrors.IsConflict(err) { - status.Reason = "AdmissionRaced" - - return status, false, nil - } - - return status, false, fmt.Errorf("automatic admission: %w", err) - } - - status.Reason = "Admitted" - status.NextDueAt = nil - - return status, true, nil -} - -func (r *RecurringReconciler) participantsFitFailureDomainBudget( - ctx context.Context, - participants []signalv1.Participant, - failureDomainLabel string, - maxConcurrent int32, - activeByDomain map[string]int32, -) (bool, error) { - prospective := make(map[string]int32) - - for _, participant := range participants { - var node corev1.Node - if err := r.Reader.Get(ctx, client.ObjectKey{Name: participant.NodeName}, &node); err != nil { - return false, fmt.Errorf( - "get selected participant %s for failure-domain accounting: %w", - participant.NodeName, - err, - ) - } - - if participant.NodeUID != "" && node.UID != participant.NodeUID { - return false, fmt.Errorf("selected participant %s identity changed", participant.NodeName) - } - - domain := node.Labels[failureDomainLabel] - prospective[domain]++ - - if activeByDomain[domain]+prospective[domain] > maxConcurrent { - return false, nil - } - } - - return true, nil -} - -func (r *RecurringReconciler) nodeBusy(ctx context.Context, node *corev1.Node) (bool, string, error) { - var pods corev1.PodList - if err := r.Reader.List(ctx, &pods, client.MatchingFields{"spec.nodeName": node.Name}); err != nil { - return true, "WorkloadStateUnavailable", fmt.Errorf("list Node pods: %w", err) - } - - for index := range pods.Items { - pod := &pods.Items[index] - if pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { - continue - } - - if controlledByDaemonSet(pod) { - continue - } - - return true, "WorkloadActive", nil - } - - active, err := r.activeMachineOperation(ctx, node.Name) - if err != nil { - return true, "MaintenanceStateUnavailable", err - } - - if active { - return true, "MaintenanceOperationActive", nil - } - - return false, "", nil -} - -func (r *RecurringReconciler) activeMachineOperation(ctx context.Context, nodeName string) (bool, error) { - var machines machinav1.MachineList - if err := r.Reader.List(ctx, &machines); err != nil { - if apierrors.IsNotFound(err) { - return false, nil - } - - return false, fmt.Errorf("list Machines: %w", err) - } - - machineNames := map[string]struct{}{} - - for index := range machines.Items { - machine := &machines.Items[index] - if machine.Spec.Kubernetes != nil && - machine.Spec.Kubernetes.NodeRef != nil && - machine.Spec.Kubernetes.NodeRef.Name == nodeName { - machineNames[machine.Name] = struct{}{} - } - } - - if len(machineNames) == 0 { - return false, nil - } - - var operations machinav1.MachineOperationList - if err := r.Reader.List(ctx, &operations); err != nil { - return false, fmt.Errorf("list MachineOperations: %w", err) - } - - for index := range operations.Items { - operation := &operations.Items[index] - if operation.Status.Phase != machinav1.OperationPhasePending && - operation.Status.Phase != machinav1.OperationPhaseInProgress { - continue - } - - if _, ok := machineNames[operation.Spec.MachineRef]; ok { - return true, nil - } - - if operation.Spec.MachineSelector != nil { - selector, err := metav1.LabelSelectorAsSelector(operation.Spec.MachineSelector) - if err != nil { - return false, fmt.Errorf("MachineOperation %s selector: %w", operation.Name, err) - } - - for machineIndex := range machines.Items { - machine := &machines.Items[machineIndex] - if _, ok := machineNames[machine.Name]; ok && selector.Matches(labels.Set(machine.Labels)) { - return true, nil - } - } - } - } - - return false, nil -} - -func controlledByDaemonSet(pod *corev1.Pod) bool { - for _, owner := range pod.OwnerReferences { - if owner.Controller != nil && *owner.Controller && owner.Kind == "DaemonSet" { - return true - } - } - - return false -} - -func validateRecurringPolicy(policy *signalv1.RecurringPolicy) error { - switch { - case len(policy.NodeNames) == 0: - return fmt.Errorf("nodeNames must explicitly opt in at least one Node") - case policy.MinIdleDuration.Duration < 15*time.Minute: - return fmt.Errorf("minIdleDuration must be at least 15 minutes") - case policy.ValidFor.Duration <= 0: - return fmt.Errorf("validFor must be positive") - case policy.ExpiresAt.IsZero(): - return fmt.Errorf("expiresAt is required") - case policy.MaxConcurrent < 1: - return fmt.Errorf("maxConcurrent must be positive") - case policy.MaxConsecutiveFailures < 1: - return fmt.Errorf("maxConsecutiveFailures must be positive") - } - - return nil -} - -func policyExpired(policy *signalv1.RecurringPolicy, now time.Time) bool { - return !now.Before(policy.ExpiresAt.Time) -} - -func failureGate( - policy *signalv1.RecurringPolicy, - consecutiveFailures int32, - latestFailure *time.Time, - now time.Time, -) (bool, string) { - if policy.MaxConsecutiveFailures > 0 && consecutiveFailures >= policy.MaxConsecutiveFailures { - return true, "CircuitOpen" - } - - if latestFailure != nil && now.Before(latestFailure.Add(policy.FailureBackoff.Duration)) { - return false, "FailureBackoff" - } - - return false, "" -} - -func dueAt( - policy *signalv1.RecurringPolicy, - idleSince *metav1.Time, - lastSuccess *signalv1.CertificationRun, -) time.Time { - due := idleSince.Add(policy.MinIdleDuration.Duration) - if policy.NotBefore != nil && due.Before(policy.NotBefore.Time) { - due = policy.NotBefore.Time - } - - if lastSuccess != nil && lastSuccess.Status.FinishedAt != nil { - freshUntil := lastSuccess.Status.FinishedAt.Add(policy.ValidFor.Duration) - if due.Before(freshUntil) { - due = freshUntil - } - } - - return due -} - -func activeBudgets(runs []signalv1.CertificationRun, profile *signalv1.CertificationProfile) (int32, map[string]int32) { - var active int32 - - domains := map[string]int32{} - - for index := range runs { - run := &runs[index] - if run.Spec.Profile.Name != profile.Name || run.Status.Phase.Terminal() { - continue - } - - active++ - } - - return active, domains -} - -func activeRunForNode(runs []signalv1.CertificationRun, nodeName string) bool { - for index := range runs { - run := &runs[index] - if run.Status.Phase.Terminal() { - continue - } - - for _, participant := range run.Spec.Participants { - if participant.NodeName == nodeName { - return true - } - } - } - - return false -} - -func excludedRunActive(runs []signalv1.CertificationRun, nodeName string, profiles []string) bool { - for index := range runs { - run := &runs[index] - if run.Status.Phase.Terminal() || !slices.Contains(profiles, run.Spec.Profile.Name) { - continue - } - - for _, participant := range run.Spec.Participants { - if participant.NodeName == nodeName { - return true - } - } - } - - return false -} - -func runsForNodeProfile( - runs []signalv1.CertificationRun, - nodeName string, - profileName string, -) []signalv1.CertificationRun { - result := make([]signalv1.CertificationRun, 0) - - for index := range runs { - run := runs[index] - if run.Spec.Profile.Name != profileName || !run.Status.Phase.Terminal() { - continue - } - - for _, participant := range run.Spec.Participants { - if participant.NodeName == nodeName { - result = append(result, run) - - break - } - } - } - - sort.Slice(result, func(i, j int) bool { - return completionTime(&result[i]).After(completionTime(&result[j])) - }) - - return result -} - -func runHistory( - runs []signalv1.CertificationRun, -) (*signalv1.CertificationRun, int32, *time.Time) { - var ( - lastSuccess *signalv1.CertificationRun - latestFailure *time.Time - consecutiveFailure int32 - ) - - for index := range runs { - run := &runs[index] - if run.Status.Phase == signalv1.RunPhaseSucceeded { - lastSuccess = run - - break - } - - if run.Status.Phase != signalv1.RunPhaseFailed && run.Status.Phase != signalv1.RunPhaseInconclusive { - continue - } - - consecutiveFailure++ - - if latestFailure == nil { - value := completionTime(run) - latestFailure = &value - } - } - - return lastSuccess, consecutiveFailure, latestFailure -} - -func completionTime(run *signalv1.CertificationRun) time.Time { - if run.Status.FinishedAt != nil { - return run.Status.FinishedAt.Time - } - - return run.CreationTimestamp.Time -} - -func recurringNodeStatus(statuses []signalv1.RecurringNodeStatus, nodeName string) signalv1.RecurringNodeStatus { - for _, status := range statuses { - if status.NodeName == nodeName { - return status - } - } - - return signalv1.RecurringNodeStatus{NodeName: nodeName} -} - -func ptrMetaTime(value time.Time) *metav1.Time { - result := metav1.NewTime(value) - - return &result -} - -func setProfileCondition( - profile *signalv1.CertificationProfile, - now time.Time, - status metav1.ConditionStatus, - reason string, - message string, -) { - apimeta.SetStatusCondition(&profile.Status.Conditions, metav1.Condition{ - Type: "RecurringAdmissionReady", - Status: status, - ObservedGeneration: profile.Generation, - LastTransitionTime: metav1.NewTime(now), - Reason: reason, - Message: message, - }) -} - -func (r *RecurringReconciler) now() time.Time { - if r.Now != nil { - return r.Now() - } - - return time.Now() -} diff --git a/internal/signal/policy_test.go b/internal/signal/policy_test.go deleted file mode 100644 index f856e4dc76..0000000000 --- a/internal/signal/policy_test.go +++ /dev/null @@ -1,323 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -package signal - -import ( - "context" - "errors" - "strings" - "testing" - "time" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" -) - -func TestRecurringRunBudgetsUseAuthoritativeReader(t *testing.T) { - t.Parallel() - - scheme := runtime.NewScheme() - if err := corev1.AddToScheme(scheme); err != nil { - t.Fatal(err) - } - - if err := signalv1.AddToScheme(scheme); err != nil { - t.Fatal(err) - } - - now := time.Date(2026, 10, 10, 20, 0, 0, 0, time.UTC) - node := readyNode("gpu-0", "wrong-gpu") - profile := testProfile("gpu-smoke", "expected-gpu") - profile.Spec.Recurring = &signalv1.RecurringPolicy{ - Enabled: true, - NodeNames: []string{node.Name}, - MinIdleDuration: metav1.Duration{Duration: 15 * time.Minute}, - ValidFor: metav1.Duration{Duration: time.Hour}, - ExpiresAt: metav1.NewTime(now.Add(time.Hour)), - FailureBackoff: metav1.Duration{Duration: time.Minute}, - MaxConsecutiveFailures: 3, - MaxConcurrent: 1, - } - base := fake.NewClientBuilder(). - WithScheme(scheme). - WithStatusSubresource(&signalv1.CertificationProfile{}). - WithObjects(node, profile). - Build() - cached := interceptor.NewClient(base, interceptor.Funcs{ - List: func( - context.Context, - client.WithWatch, - client.ObjectList, - ...client.ListOption, - ) error { - return errors.New("cached list must not define admission budgets") - }, - }) - r := &RecurringReconciler{ - Client: cached, - Reader: base, - Namespace: "signal", - Admission: &AdmissionReconciler{Client: cached}, - Now: func() time.Time { return now }, - } - - if _, err := r.Reconcile( - context.Background(), - reconcile.Request{NamespacedName: client.ObjectKey{Name: profile.Name}}, - ); err != nil { - t.Fatalf("Reconcile: %v", err) - } -} - -func TestRecurringFailureDomainAccountingFailsClosed(t *testing.T) { - t.Parallel() - - scheme := runtime.NewScheme() - if err := corev1.AddToScheme(scheme); err != nil { - t.Fatal(err) - } - - if err := signalv1.AddToScheme(scheme); err != nil { - t.Fatal(err) - } - - now := time.Date(2026, 10, 10, 20, 0, 0, 0, time.UTC) - profile := testProfile("gpu-smoke", "expected-gpu") - profile.Spec.Recurring = &signalv1.RecurringPolicy{ - Enabled: true, - NodeNames: []string{"gpu-0"}, - MinIdleDuration: metav1.Duration{Duration: 15 * time.Minute}, - ValidFor: metav1.Duration{Duration: time.Hour}, - ExpiresAt: metav1.NewTime(now.Add(time.Hour)), - FailureBackoff: metav1.Duration{Duration: time.Minute}, - MaxConsecutiveFailures: 3, - MaxConcurrent: 1, - FailureDomainLabel: "topology.kubernetes.io/zone", - MaxConcurrentPerFailureDomain: 1, - } - active := testRun("active") - active.Spec.Profile.Name = profile.Name - active.Spec.Participants = []signalv1.Participant{{NodeName: "missing-node", NodeUID: "missing-uid"}} - active.Status.Phase = signalv1.RunPhaseRunning - base := fake.NewClientBuilder(). - WithScheme(scheme). - WithStatusSubresource(&signalv1.CertificationProfile{}). - WithObjects(profile, active). - Build() - r := &RecurringReconciler{ - Client: base, - Reader: base, - Namespace: "signal", - Admission: &AdmissionReconciler{Client: base}, - Now: func() time.Time { return now }, - } - - if _, err := r.Reconcile( - context.Background(), - reconcile.Request{NamespacedName: client.ObjectKey{Name: profile.Name}}, - ); err == nil || !strings.Contains(err.Error(), "failure-domain accounting") { - t.Fatalf("Reconcile error = %v, want fail-closed accounting", err) - } -} - -func TestRecurringMultiNodeFailureDomainBudgetAccountsWholeGroup(t *testing.T) { - t.Parallel() - - scheme := runtime.NewScheme() - if err := corev1.AddToScheme(scheme); err != nil { - t.Fatal(err) - } - - if err := signalv1.AddToScheme(scheme); err != nil { - t.Fatal(err) - } - - first := readyNode("gpu-0", "H200") - first.Labels["topology.kubernetes.io/zone"] = "zone-a" - second := readyNode("gpu-1", "H200") - second.Labels["topology.kubernetes.io/zone"] = "zone-a" - kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(first, second).Build() - r := &RecurringReconciler{Reader: kube} - participants := []signalv1.Participant{ - participantSnapshot(first, 0), - participantSnapshot(second, 1), - } - - fits, err := r.participantsFitFailureDomainBudget( - context.Background(), - participants, - "topology.kubernetes.io/zone", - 1, - map[string]int32{}, - ) - if err != nil { - t.Fatal(err) - } - - if fits { - t.Fatal("two participants in one domain exceeded the per-domain budget") - } - - second.Labels["topology.kubernetes.io/zone"] = "zone-b" - if err := kube.Update(context.Background(), second); err != nil { - t.Fatal(err) - } - - fits, err = r.participantsFitFailureDomainBudget( - context.Background(), - participants, - "topology.kubernetes.io/zone", - 1, - map[string]int32{}, - ) - if err != nil { - t.Fatal(err) - } - - if !fits { - t.Fatal("participants in distinct domains should fit the budget") - } -} - -func TestRecurringDueUsesContinuousIdleAndFreshness(t *testing.T) { - t.Parallel() - - start := time.Date(2026, 10, 10, 20, 0, 0, 0, time.UTC) - idleSince := metav1.NewTime(start) - policy := &signalv1.RecurringPolicy{ - MinIdleDuration: metav1.Duration{Duration: 15 * time.Minute}, - ValidFor: metav1.Duration{Duration: time.Hour}, - } - - if got := dueAt(policy, &idleSince, nil); !got.Equal(start.Add(15 * time.Minute)) { - t.Fatalf("initial due = %s", got) - } - - success := testRun("success") - success.Status.Phase = signalv1.RunPhaseSucceeded - success.Status.FinishedAt = ptrTime(metav1.NewTime(start.Add(10 * time.Minute))) - - if got := dueAt(policy, &idleSince, success); !got.Equal(start.Add(70 * time.Minute)) { - t.Fatalf("freshness due = %s", got) - } -} - -func TestRecurringIdleStateSurvivesRestart(t *testing.T) { - t.Parallel() - - idleSince := metav1.NewTime(time.Date(2026, 10, 10, 20, 0, 0, 0, time.UTC)) - statuses := []signalv1.RecurringNodeStatus{{NodeName: "gpu-0", IdleSince: &idleSince, Reason: "NotDue"}} - - got := recurringNodeStatus(statuses, "gpu-0") - if got.IdleSince == nil || !got.IdleSince.Equal(&idleSince) { - t.Fatalf("idleSince was not recovered: %#v", got) - } -} - -func TestRecurringExpiryFailsClosed(t *testing.T) { - t.Parallel() - - expiry := metav1.NewTime(time.Date(2026, 10, 12, 14, 0, 0, 0, time.UTC)) - policy := &signalv1.RecurringPolicy{ExpiresAt: expiry} - - if policyExpired(policy, expiry.Add(-time.Nanosecond)) { - t.Fatal("policy expired early") - } - - if !policyExpired(policy, expiry.Time) { - t.Fatal("policy remained active at its expiry") - } -} - -func TestRecurringConcurrencyAndDuplicatePrevention(t *testing.T) { - t.Parallel() - - active := *testRun("active") - active.Status.Phase = signalv1.RunPhaseRunning - active.Spec.Profile.Name = "short" - profile := testProfile("short", "A100") - - count, _ := activeBudgets([]signalv1.CertificationRun{active}, profile) - if count != 1 || !activeRunForNode([]signalv1.CertificationRun{active}, "gpu-0") { - t.Fatalf("active budget=%d duplicate=%t", count, activeRunForNode([]signalv1.CertificationRun{active}, "gpu-0")) - } -} - -func TestRecurringShortLongMutualExclusion(t *testing.T) { - t.Parallel() - - longRun := *testRun("long") - longRun.Spec.Profile.Name = "gpu-long" - longRun.Status.Phase = signalv1.RunPhaseRunning - - if !excludedRunActive( - []signalv1.CertificationRun{longRun}, - "gpu-0", - []string{"gpu-long"}, - ) { - t.Fatal("active long run did not exclude the short profile") - } -} - -func TestRecurringFailureBackoffAndCircuitBreaker(t *testing.T) { - t.Parallel() - - now := time.Date(2026, 10, 10, 20, 0, 0, 0, time.UTC) - failed := *testRun("failed") - failed.Status.Phase = signalv1.RunPhaseFailed - failed.Status.FinishedAt = ptrTime(metav1.NewTime(now.Add(-5 * time.Minute))) - inconclusive := *testRun("inconclusive") - inconclusive.Status.Phase = signalv1.RunPhaseInconclusive - inconclusive.Status.FinishedAt = ptrTime(metav1.NewTime(now.Add(-10 * time.Minute))) - runs := []signalv1.CertificationRun{failed, inconclusive} - - _, count, latest := runHistory(runs) - policy := &signalv1.RecurringPolicy{ - FailureBackoff: metav1.Duration{Duration: 30 * time.Minute}, - MaxConsecutiveFailures: 3, - } - - suspended, reason := failureGate(policy, count, latest, now) - if suspended || reason != "FailureBackoff" { - t.Fatalf("backoff gate = (%t, %q)", suspended, reason) - } - - policy.MaxConsecutiveFailures = 2 - - suspended, reason = failureGate(policy, count, latest, now) - if !suspended || reason != "CircuitOpen" { - t.Fatalf("circuit gate = (%t, %q)", suspended, reason) - } -} - -func TestIdleDetectionIgnoresOnlyDaemonSetsAndTerminalPods(t *testing.T) { - t.Parallel() - - controller := true - daemon := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{OwnerReferences: []metav1.OwnerReference{{ - APIVersion: appsv1.SchemeGroupVersion.String(), - Kind: "DaemonSet", - Controller: &controller, - }}}, - } - deployment := daemon.DeepCopy() - deployment.OwnerReferences[0].Kind = "ReplicaSet" - - if !controlledByDaemonSet(daemon) { - t.Fatal("DaemonSet pod was treated as workload activity") - } - - if controlledByDaemonSet(deployment) { - t.Fatal("non-DaemonSet pod was treated as disruptable") - } -} diff --git a/internal/signal/projection.go b/internal/signal/projection.go index bfb395bb10..37ef80316c 100644 --- a/internal/signal/projection.go +++ b/internal/signal/projection.go @@ -12,7 +12,6 @@ import ( "time" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" @@ -25,17 +24,14 @@ const ( EligibilityLabel = "signal.unbounded-cloud.io/eligibility" ProfileLabel = "signal.unbounded-cloud.io/profile" - ProfileNameAnnotation = "signal.unbounded-cloud.io/profile-name" - ProfileGenerationAnnotation = "signal.unbounded-cloud.io/profile-generation" - LastRunAnnotation = "signal.unbounded-cloud.io/last-run" - LastRunAtAnnotation = "signal.unbounded-cloud.io/last-run-at" - LastSuccessfulRunAnnotation = "signal.unbounded-cloud.io/last-successful-run" - LastSuccessfulAtAnnotation = "signal.unbounded-cloud.io/last-successful-at" - EvidenceDigestAnnotation = "signal.unbounded-cloud.io/evidence-digest" - ScriptDigestAnnotation = "signal.unbounded-cloud.io/script-digest" - NextDueAtAnnotation = "signal.unbounded-cloud.io/next-due-at" - RecurringReasonAnnotation = "signal.unbounded-cloud.io/recurring-reason" - RecurringSuspendedAnnotation = "signal.unbounded-cloud.io/recurring-suspended" + ProfileNameAnnotation = "signal.unbounded-cloud.io/profile-name" + ProfileGenerationAnnotation = "signal.unbounded-cloud.io/profile-generation" + LastRunAnnotation = "signal.unbounded-cloud.io/last-run" + LastRunAtAnnotation = "signal.unbounded-cloud.io/last-run-at" + LastSuccessfulRunAnnotation = "signal.unbounded-cloud.io/last-successful-run" + LastSuccessfulAtAnnotation = "signal.unbounded-cloud.io/last-successful-at" + EvidenceDigestAnnotation = "signal.unbounded-cloud.io/evidence-digest" + ScriptDigestAnnotation = "signal.unbounded-cloud.io/script-digest" maxProjectionAnnotationValue = 1024 ) @@ -105,10 +101,6 @@ func projectTerminal(ctx context.Context, kube client.Client, run *signalv1.Cert } func runScriptDigest(run *signalv1.CertificationRun) string { - if len(run.Spec.Checks) == 0 { - return run.Spec.Script.Digest - } - hash := sha256.New() for index := range run.Spec.Checks { _, _ = hash.Write([]byte(run.Spec.Checks[index].ID)) @@ -120,49 +112,6 @@ func runScriptDigest(run *signalv1.CertificationRun) string { return "sha256:" + hex.EncodeToString(hash.Sum(nil)) } -func projectRecurring( - ctx context.Context, - kube client.Client, - profile *signalv1.CertificationProfile, - status signalv1.RecurringNodeStatus, -) error { - participant := signalv1.Participant{NodeName: status.NodeName} - pinned := signalv1.PinnedProfile{Name: profile.Name, UID: profile.UID, Generation: profile.Generation} - - return projectParticipants(ctx, kube, []signalv1.Participant{participant}, func(node *corev1.Node) error { - selectedProfile := node.Annotations[ProfileNameAnnotation] - - active := node.Labels[StateLabel] == "pending" || node.Labels[StateLabel] == "testing" - if active && selectedProfile != profile.Name { - return nil - } - - if status.LastSuccessfulAt != nil && - !projectionTimeAfter(node.Annotations[LastSuccessfulAtAnnotation], status.LastSuccessfulAt.Time) { - setProfileProjection(node, pinned) - } else { - return nil - } - - setOrDeleteAnnotation(node, NextDueAtAnnotation, formatMetaTime(status.NextDueAt)) - setOrDeleteAnnotation(node, RecurringReasonAnnotation, status.Reason) - node.Annotations[RecurringSuspendedAnnotation] = strconv.FormatBool(status.Suspended) - - if status.LastSuccessfulAt != nil && - !projectionTimeAfter(node.Annotations[LastSuccessfulAtAnnotation], status.LastSuccessfulAt.Time) { - node.Annotations[LastSuccessfulRunAnnotation] = boundedAnnotationValue(status.LastSuccessfulRun) - node.Annotations[LastSuccessfulAtAnnotation] = status.LastSuccessfulAt.UTC().Format(time.RFC3339) - - if node.Labels[StateLabel] == "" { - node.Labels[StateLabel] = "passed" - node.Labels[EligibilityLabel] = "production" - } - } - - return nil - }) -} - func projectParticipants( ctx context.Context, kube client.Client, @@ -245,21 +194,3 @@ func projectionTimeAfter(value string, candidate time.Time) bool { return existing.After(candidate) } - -func setOrDeleteAnnotation(node *corev1.Node, key, value string) { - if value == "" { - delete(node.Annotations, key) - - return - } - - node.Annotations[key] = boundedAnnotationValue(value) -} - -func formatMetaTime(value *metav1.Time) string { - if value == nil { - return "" - } - - return value.UTC().Format(time.RFC3339) -} diff --git a/internal/signal/projection_test.go b/internal/signal/projection_test.go deleted file mode 100644 index e04828f9c1..0000000000 --- a/internal/signal/projection_test.go +++ /dev/null @@ -1,227 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -package signal - -import ( - "context" - "strings" - "testing" - "time" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - - signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" -) - -func TestNodeProjectionTransitionsAndRetainsTerminalSummary(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - node := readyNode("gpu-0", "A100") - kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node).Build() - run := testRun("run-pass") - - if err := projectPending(context.Background(), kube, run.Spec.Participants, run.Spec.Profile); err != nil { - t.Fatal(err) - } - - assertNodeProjection(t, kube, node.Name, "pending", "blocked") - - if err := projectTesting(context.Background(), kube, run); err != nil { - t.Fatal(err) - } - - assertNodeProjection(t, kube, node.Name, "testing", "blocked") - - run.Status.Phase = signalv1.RunPhaseSucceeded - run.Status.ExecutionState = "Completed" - run.Status.Verdict = "Pass" - run.Status.Reason = "Healthy" - run.Status.Evidence.Digest = "sha256:" + strings.Repeat("e", 64) - run.Status.FinishedAt = ptrTime(metav1.NewTime(time.Date(2026, 10, 11, 5, 0, 0, 0, time.UTC))) - - if err := projectTerminal(context.Background(), kube, run); err != nil { - t.Fatal(err) - } - - var got corev1.Node - if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &got); err != nil { - t.Fatal(err) - } - - if got.Labels[StateLabel] != "passed" || got.Labels[EligibilityLabel] != "production" { - t.Fatalf("terminal labels = %v", got.Labels) - } - - if got.Annotations[LastRunAnnotation] != run.Name || - got.Annotations[LastSuccessfulRunAnnotation] != run.Name || - got.Annotations[EvidenceDigestAnnotation] != run.Status.Evidence.Digest || - got.Annotations[ScriptDigestAnnotation] != runScriptDigest(run) { - t.Fatalf("terminal annotations = %v", got.Annotations) - } -} - -func TestNodeProjectionFailureAndInconclusiveBlockEligibility(t *testing.T) { - t.Parallel() - - for _, tc := range []struct { - name string - phase signalv1.RunPhase - state string - }{ - {name: "failed", phase: signalv1.RunPhaseFailed, state: "failed"}, - {name: "inconclusive", phase: signalv1.RunPhaseInconclusive, state: "inconclusive"}, - } { - t.Run(tc.name, func(t *testing.T) { - scheme := testScheme(t) - node := readyNode("gpu-0", "A100") - kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node).Build() - run := testRun("run-" + tc.name) - run.Status.Phase = tc.phase - run.Status.FinishedAt = ptrTime(metav1.Now()) - - if err := projectTerminal(context.Background(), kube, run); err != nil { - t.Fatal(err) - } - - assertNodeProjection(t, kube, node.Name, tc.state, "blocked") - }) - } -} - -func TestNodeProjectionDoesNotRegressAfterRestartOrOlderRetry(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - node := readyNode("gpu-0", "A100") - kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node).Build() - newer := testRun("newer") - newer.Status.Phase = signalv1.RunPhaseSucceeded - newer.Status.FinishedAt = ptrTime(metav1.NewTime(time.Date(2026, 10, 11, 6, 0, 0, 0, time.UTC))) - newer.Status.Evidence.Digest = "sha256:" + strings.Repeat("a", 64) - - if err := projectTerminal(context.Background(), kube, newer); err != nil { - t.Fatal(err) - } - - older := testRun("older") - older.Status.Phase = signalv1.RunPhaseFailed - older.Status.FinishedAt = ptrTime(metav1.NewTime(time.Date(2026, 10, 11, 5, 0, 0, 0, time.UTC))) - older.Status.Evidence.Digest = "sha256:" + strings.Repeat("b", 64) - - if err := projectTerminal(context.Background(), kube, older); err != nil { - t.Fatal(err) - } - - var got corev1.Node - if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &got); err != nil { - t.Fatal(err) - } - - if got.Labels[StateLabel] != "passed" || - got.Annotations[LastRunAnnotation] != newer.Name || - got.Annotations[EvidenceDigestAnnotation] != "sha256:"+strings.Repeat("a", 64) { - t.Fatalf("projection regressed: labels=%v annotations=%v", got.Labels, got.Annotations) - } -} - -func TestNodeProjectionDoesNotCertifyReplacementNode(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - replacement := readyNode("gpu-0", "A100") - replacement.UID = "replacement-uid" - kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(replacement).Build() - run := testRun("run-pass") - run.Status.Phase = signalv1.RunPhaseSucceeded - run.Status.Verdict = "Pass" - run.Status.FinishedAt = ptrTime(metav1.Now()) - - if err := projectTerminal(context.Background(), kube, run); err != nil { - t.Fatal(err) - } - - var got corev1.Node - if err := kube.Get(context.Background(), client.ObjectKey{Name: replacement.Name}, &got); err != nil { - t.Fatal(err) - } - - if got.Labels[StateLabel] != "" || got.Labels[EligibilityLabel] != "" { - t.Fatalf("replacement Node was certified: labels=%v", got.Labels) - } -} - -func TestProjectionValuesAreBoundedAndValid(t *testing.T) { - t.Parallel() - - longValue := strings.Repeat("Invalid/Profile:", 200) - - label := projectionLabelValue(longValue) - if len(label) > 63 { - t.Fatalf("label value is too long: %d", len(label)) - } - - if bounded := boundedAnnotationValue(longValue); len(bounded) > len("sha256:")+64 { - t.Fatalf("annotation value is unbounded: %d", len(bounded)) - } - - if got := projectionLabelValue("gpu-smoke-example"); got != "gpu-smoke-example" { - t.Fatalf("valid profile label changed to %q", got) - } -} - -func TestRecurringProjectionPublishesNextDueAndSuspension(t *testing.T) { - t.Parallel() - - scheme := testScheme(t) - node := readyNode("gpu-0", "A100") - profile := testProfile("gpu-smoke-example", "example-gpu") - kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node).Build() - nextDue := metav1.NewTime(time.Date(2026, 10, 11, 7, 0, 0, 0, time.UTC)) - status := signalv1.RecurringNodeStatus{ - NodeName: node.Name, - NextDueAt: &nextDue, - LastSuccessfulRun: "run-pass", - LastSuccessfulAt: ptrTime(metav1.NewTime(time.Date(2026, 10, 11, 6, 0, 0, 0, time.UTC))), - Suspended: true, - Reason: "CircuitOpen", - } - - if err := projectRecurring(context.Background(), kube, profile, status); err != nil { - t.Fatal(err) - } - - var got corev1.Node - if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &got); err != nil { - t.Fatal(err) - } - - if got.Annotations[NextDueAtAnnotation] != "2026-10-11T07:00:00Z" || - got.Annotations[RecurringReasonAnnotation] != "CircuitOpen" || - got.Annotations[RecurringSuspendedAnnotation] != "true" { - t.Fatalf("recurring annotations = %v", got.Annotations) - } -} - -func assertNodeProjection( - t *testing.T, - kube client.Client, - nodeName string, - state string, - eligibility string, -) { - t.Helper() - - var node corev1.Node - if err := kube.Get(context.Background(), client.ObjectKey{Name: nodeName}, &node); err != nil { - t.Fatal(err) - } - - if node.Labels[StateLabel] != state || node.Labels[EligibilityLabel] != eligibility { - t.Fatalf("projection = state %q eligibility %q", node.Labels[StateLabel], node.Labels[EligibilityLabel]) - } -} diff --git a/internal/signaladapter/adapter.go b/internal/signaladapter/adapter.go index 61df4a5bff..a83f4dd466 100644 --- a/internal/signaladapter/adapter.go +++ b/internal/signaladapter/adapter.go @@ -67,8 +67,6 @@ type Config struct { RunUID types.UID ParticipantsJSON string ParticipantCount int - RequestNodeName string - LegacyParticipants string ScriptConfigMap string ScriptKey string ScriptDigest string @@ -129,45 +127,31 @@ type AggregateOutput struct { func ParseParticipants( participantsJSON string, participantCount int, - requestNodeName string, - legacyParticipants string, ) ([]signalv1.Participant, error) { if participantCount < 1 || participantCount > MaxParticipants { return nil, fmt.Errorf("participant-count must be between 1 and %d", MaxParticipants) } - var participants []signalv1.Participant - - if strings.TrimSpace(participantsJSON) != "" { - decoder := json.NewDecoder(strings.NewReader(participantsJSON)) - decoder.DisallowUnknownFields() + if strings.TrimSpace(participantsJSON) == "" { + return nil, errors.New("participants-json is required") + } - if err := decoder.Decode(&participants); err != nil { - return nil, fmt.Errorf("decode participants-json: %w", err) - } + var participants []signalv1.Participant - var trailing any - if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { - if err == nil { - return nil, errors.New("participants-json contains trailing values") - } + decoder := json.NewDecoder(strings.NewReader(participantsJSON)) + decoder.DisallowUnknownFields() - return nil, fmt.Errorf("decode trailing participants-json value: %w", err) - } - } else { - nodeName := requestNodeName - if nodeName == "" { - names := strings.Split(legacyParticipants, ",") - if len(names) > 0 { - nodeName = strings.TrimSpace(names[0]) - } - } + if err := decoder.Decode(&participants); err != nil { + return nil, fmt.Errorf("decode participants-json: %w", err) + } - if nodeName == "" { - return nil, errors.New("participants-json or a compatibility Node name is required") + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + if err == nil { + return nil, errors.New("participants-json contains trailing values") } - participants = []signalv1.Participant{{Index: 0, NodeName: nodeName}} + return nil, fmt.Errorf("decode trailing participants-json value: %w", err) } if len(participants) != participantCount { @@ -320,8 +304,6 @@ func ValidateConfig(config Config) ([]signalv1.Participant, []corev1.Toleration, participants, err := ParseParticipants( config.ParticipantsJSON, config.ParticipantCount, - config.RequestNodeName, - config.LegacyParticipants, ) if err != nil { return nil, nil, nil, err diff --git a/internal/signaladapter/adapter_test.go b/internal/signaladapter/adapter_test.go deleted file mode 100644 index fc296c8f25..0000000000 --- a/internal/signaladapter/adapter_test.go +++ /dev/null @@ -1,1470 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -package signaladapter - -import ( - "bytes" - "context" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - "testing" - "time" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - - signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" -) - -func TestRenderOrdinaryPodsPerParticipant(t *testing.T) { - t.Parallel() - - config := testConfig() - config.BackendAPIVersion = "v1" - config.BackendKind = "Pod" - config.Queue = "" - - participants, tolerations, environment, err := ValidateConfig(config) - if err != nil { - t.Fatal(err) - } - - resources, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner()) - if err != nil { - t.Fatal(err) - } - - if len(resources) != 2 { - t.Fatalf("resources = %d, want 2", len(resources)) - } - - for index, resource := range resources { - if resource.GetKind() != "Pod" { - t.Fatalf("resource %d kind = %q", index, resource.GetKind()) - } - - nodeName, _, err := unstructured.NestedString(resource.Object, "spec", "nodeSelector", corev1.LabelHostname) - if err != nil { - t.Fatal(err) - } - - if nodeName != participants[index].NodeName { - t.Fatalf("resource %d node selector = %q", index, nodeName) - } - - restartPolicy, _, err := unstructured.NestedString(resource.Object, "spec", "restartPolicy") - if err != nil { - t.Fatal(err) - } - - if restartPolicy != string(corev1.RestartPolicyNever) { - t.Fatalf("resource %d restart policy = %q", index, restartPolicy) - } - - automount, _, err := unstructured.NestedBool(resource.Object, "spec", "automountServiceAccountToken") - if err != nil { - t.Fatal(err) - } - - if automount { - t.Fatalf("resource %d automounts a service account token", index) - } - - hostNetwork, _, err := unstructured.NestedBool(resource.Object, "spec", "hostNetwork") - if err != nil { - t.Fatal(err) - } - - if hostNetwork { - t.Fatalf("resource %d unexpectedly uses the host network", index) - } - - if resource.GetAnnotations()[requiredTopologyKey] != "" { - t.Fatalf("resource %d unexpectedly requests Kueue topology", index) - } - - activeDeadline, _, err := unstructured.NestedInt64(resource.Object, "spec", "activeDeadlineSeconds") - if err != nil || activeDeadline != config.BackendDeadlineSeconds { - t.Fatalf("resource %d active deadline = %d, err = %v", index, activeDeadline, err) - } - } -} - -func TestRenderOrdinaryPodWithKueueQueue(t *testing.T) { - t.Parallel() - - config := testConfig() - config.BackendAPIVersion = "v1" - config.BackendKind = "Pod" - config.Queue = "certification" - - participants, tolerations, environment, err := ValidateConfig(config) - if err != nil { - t.Fatal(err) - } - - resources, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner()) - if err != nil { - t.Fatal(err) - } - - t.Run("runtime stage is participant only", func(t *testing.T) { - t.Parallel() - - config := testConfig() - config.ScriptRequiresStage = true - config.RuntimeStageName = "collective-bandwidth" - - participants, tolerations, environment, err := ValidateConfig(config) - if err != nil { - t.Fatal(err) - } - - config.BackendAPIVersion = "v1" - config.BackendKind = "Pod" - config.Queue = "" - - pods, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner()) - if err != nil { - t.Fatal(err) - } - - podJSON, err := json.Marshal(pods[0].Object) - if err != nil { - t.Fatal(err) - } - - var pod corev1.Pod - if err := json.Unmarshal(podJSON, &pod); err != nil { - t.Fatal(err) - } - - if value := environmentValue(pod.Spec.Containers[0].Env, "PROJECT_SIGNAL_STAGE_NAME"); value != config.RuntimeStageName { - t.Fatalf("Pod runtime stage = %q", value) - } - - config.BackendAPIVersion = "kubeflow.org/v2beta1" - config.BackendKind = "MPIJob" - config.Queue = "certification" - - jobs, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner()) - if err != nil { - t.Fatal(err) - } - - launcher, _, err := unstructured.NestedSlice( - jobs[0].Object, - "spec", - "mpiReplicaSpecs", - "Launcher", - "template", - "spec", - "containers", - ) - if err != nil { - t.Fatal(err) - } - - worker, _, err := unstructured.NestedSlice( - jobs[0].Object, - "spec", - "mpiReplicaSpecs", - "Worker", - "template", - "spec", - "containers", - ) - if err != nil { - t.Fatal(err) - } - - launcherJSON, _ := json.Marshal(launcher) - workerJSON, _ := json.Marshal(worker) - - if strings.Contains(string(launcherJSON), "PROJECT_SIGNAL_STAGE_NAME") { - t.Fatalf("MPI launcher received runtime stage: %s", launcherJSON) - } - - if !strings.Contains(string(workerJSON), `"name":"PROJECT_SIGNAL_STAGE_NAME","value":"collective-bandwidth"`) { - t.Fatalf("MPI worker runtime stage missing: %s", workerJSON) - } - }) - - if queue := resources[0].GetLabels()["kueue.x-k8s.io/queue-name"]; queue != "certification" { - t.Fatalf("Pod queue label = %q", queue) - } -} - -func TestRenderMPIJobsPerParticipant(t *testing.T) { - t.Parallel() - - config := testConfig() - - participants, tolerations, environment, err := ValidateConfig(config) - if err != nil { - t.Fatal(err) - } - - resources, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner()) - if err != nil { - t.Fatal(err) - } - - for index, resource := range resources { - if resource.GetAPIVersion() != "kubeflow.org/v2beta1" || resource.GetKind() != "MPIJob" { - t.Fatalf("resource %d = %s %s", index, resource.GetAPIVersion(), resource.GetKind()) - } - - queue, _, err := unstructured.NestedString( - resource.Object, - "metadata", - "labels", - "kueue.x-k8s.io/queue-name", - ) - if err != nil { - t.Fatal(err) - } - - if queue != "certification" { - t.Fatalf("resource %d queue = %q", index, queue) - } - - activeDeadline, _, err := unstructured.NestedInt64(resource.Object, "spec", "runPolicy", "activeDeadlineSeconds") - if err != nil { - t.Fatal(err) - } - - if activeDeadline != config.BackendDeadlineSeconds { - t.Fatalf("resource %d active deadline = %d", index, activeDeadline) - } - - cleanPolicy, _, err := unstructured.NestedString(resource.Object, "spec", "runPolicy", "cleanPodPolicy") - if err != nil { - t.Fatal(err) - } - - if cleanPolicy != "All" { - t.Fatalf("resource %d cleanPodPolicy = %q", index, cleanPolicy) - } - - for _, replica := range []string{"Launcher", "Worker"} { - replicaRestartPolicy, _, err := unstructured.NestedString( - resource.Object, - "spec", - "mpiReplicaSpecs", - replica, - "restartPolicy", - ) - if err != nil { - t.Fatal(err) - } - - if replicaRestartPolicy != string(corev1.RestartPolicyNever) { - t.Fatalf("resource %d %s replica restart policy = %q", index, replica, replicaRestartPolicy) - } - - restartPolicy, _, err := unstructured.NestedString( - resource.Object, - "spec", - "mpiReplicaSpecs", - replica, - "template", - "spec", - "restartPolicy", - ) - if err != nil { - t.Fatal(err) - } - - if restartPolicy != string(corev1.RestartPolicyNever) { - t.Fatalf("resource %d %s restart policy = %q", index, replica, restartPolicy) - } - } - - if _, found, err := unstructured.NestedString(resource.Object, "spec", "restartPolicy"); err != nil || found { - t.Fatalf("resource %d has top-level restartPolicy: found=%t err=%v", index, found, err) - } - - workerNode, _, err := unstructured.NestedString( - resource.Object, - "spec", - "mpiReplicaSpecs", - "Worker", - "template", - "spec", - "nodeSelector", - corev1.LabelHostname, - ) - if err != nil { - t.Fatal(err) - } - - if workerNode != participants[index].NodeName { - t.Fatalf("resource %d worker node = %q", index, workerNode) - } - - launcherContainers, found, err := unstructured.NestedSlice( - resource.Object, - "spec", - "mpiReplicaSpecs", - "Launcher", - "template", - "spec", - "containers", - ) - if err != nil || !found { - t.Fatalf("resource %d launcher containers: found=%t err=%v", index, found, err) - } - - launcherJSON, err := json.Marshal(launcherContainers) - if err != nil { - t.Fatal(err) - } - - if strings.Contains(string(launcherJSON), "nvidia.com/gpu") { - t.Fatalf("resource %d launcher requests GPUs: %s", index, launcherJSON) - } - - if strings.Contains(string(launcherJSON), "/etc/mpi/hostfile") || - !strings.Contains(string(launcherJSON), mpiHostfilePath) || - !strings.Contains(string(launcherJSON), "OMPI_MCA_orte_default_hostfile") || - !strings.Contains(string(launcherJSON), "BatchMode=yes") || - !strings.Contains(string(launcherJSON), "ConnectionAttempts=1") || - !strings.Contains(string(launcherJSON), "StrictHostKeyChecking=accept-new") || - !strings.Contains(string(launcherJSON), "UserKnownHostsFile=") || - !strings.Contains(string(launcherJSON), mpiKnownHostsPath) || - !strings.Contains(string(launcherJSON), "/bin/true") || - !strings.Contains(string(launcherJSON), "Project Signal MPI readiness timed out") { - t.Fatalf("resource %d launcher readiness fence missing: %s", index, launcherJSON) - } - - jobName := resource.GetName() - - expectedHost := fmt.Sprintf( - "%s-worker-0.%s.%s.svc", - jobName, - jobName, - config.Namespace, - ) - if !strings.Contains(string(launcherJSON), expectedHost+" slots="+config.GPUResourceCount) { - t.Fatalf("resource %d launcher hostfile entry missing: %s", index, launcherJSON) - } - } -} - -func TestMPIWorkerReadinessTimeoutIsBoundedByBackendDeadline(t *testing.T) { - t.Parallel() - - for _, backendDeadline := range []int64{2, 60, 121, 540} { - timeout := mpiWorkerReadinessTimeout(backendDeadline) - if timeout < 1 || timeout >= backendDeadline || timeout > maxMPIWorkerReadyWait { - t.Fatalf("mpiWorkerReadinessTimeout(%d) = %d", backendDeadline, timeout) - } - } -} - -func TestMPIWorkerReadinessFenceWaitsForWorkers(t *testing.T) { - t.Parallel() - - directory := t.TempDir() - hostfile := filepath.Join(directory, "hostfile") - readyFile := filepath.Join(directory, "ready") - argumentsFile := filepath.Join(directory, "ssh-arguments") - binDirectory := fakeSSH(t, directory) - ready := make(chan error, 1) - hosts := []mpiWorkerHost{ - {Name: "worker-0.job.namespace.svc", Slots: 2}, - {Name: "worker-1.job.namespace.svc", Slots: 2}, - } - - go func() { - time.Sleep(100 * time.Millisecond) - - ready <- os.WriteFile(readyFile, nil, 0o600) - }() - - ctx, cancel := context.WithTimeout(t.Context(), 8*time.Second) - defer cancel() - - command := exec.CommandContext(ctx, "/bin/bash", "-c", mpiLauncherPrelude(hostfile, hosts, 5)) - - command.Env = append( - os.Environ(), - "PATH="+binDirectory+":"+os.Getenv("PATH"), - "PROJECT_SIGNAL_TEST_SSH_READY="+readyFile, - "PROJECT_SIGNAL_TEST_SSH_ARGUMENTS="+argumentsFile, - ) - - var stderr bytes.Buffer - - command.Stderr = &stderr - - if err := command.Run(); err != nil { - t.Fatalf("readiness fence failed: %v stderr=%q", err, stderr.String()) - } - - if err := <-ready; err != nil { - t.Fatal(err) - } - - hostfileContent, err := os.ReadFile(hostfile) - if err != nil { - t.Fatal(err) - } - - if string(hostfileContent) != "worker-0.job.namespace.svc slots=2\nworker-1.job.namespace.svc slots=2\n" { - t.Fatalf("generated hostfile = %q", hostfileContent) - } - - arguments, err := os.ReadFile(argumentsFile) - if err != nil { - t.Fatal(err) - } - - for _, expected := range []string{ - "-o BatchMode=yes", - "-o ConnectTimeout=2", - "-o ConnectionAttempts=1", - "-o StrictHostKeyChecking=accept-new", - "-o UserKnownHostsFile=" + mpiKnownHostsPath, - "worker-1.job.namespace.svc /bin/true", - } { - if !strings.Contains(string(arguments), expected) { - t.Fatalf("SSH arguments %q do not contain %q", arguments, expected) - } - } -} - -func TestMPIWorkerReadinessFenceTimesOut(t *testing.T) { - t.Parallel() - - directory := t.TempDir() - - hostfile := filepath.Join(directory, "hostfile") - command := exec.Command( - "/bin/bash", - "-c", - mpiLauncherPrelude( - hostfile, - []mpiWorkerHost{{Name: "worker-0.job.namespace.svc", Slots: 1}}, - 2, - ), - ) - - command.Env = append(os.Environ(), "PATH="+fakeSSH(t, directory)+":"+os.Getenv("PATH")) - - var stderr bytes.Buffer - - command.Stderr = &stderr - - err := command.Run() - if err == nil { - t.Fatal("readiness fence succeeded without a listening worker") - } - - var exitError *exec.ExitError - if !errors.As(err, &exitError) || exitError.ExitCode() != 124 { - t.Fatalf("readiness fence error = %v, want exit 124", err) - } - - if !strings.Contains(stderr.String(), "timed out after 2s") || - !strings.Contains(stderr.String(), hostfile) || - !strings.Contains(stderr.String(), "last unready host=worker-0.job.namespace.svc") || - !strings.Contains(stderr.String(), "simulated SSH connection failure") { - t.Fatalf("readiness timeout stderr = %q", stderr.String()) - } -} - -func TestMPILauncherPreludeIgnoresOperatorHostfile(t *testing.T) { - t.Parallel() - - directory := t.TempDir() - - operatorHostfile := filepath.Join(directory, "operator-hostfile") - if err := os.WriteFile(operatorHostfile, []byte("stale-worker slots=99\n"), 0o600); err != nil { - t.Fatal(err) - } - - generatedHostfile := filepath.Join(directory, "generated-hostfile") - readyFile := filepath.Join(directory, "ready") - observedOpenMPIHostfile := filepath.Join(directory, "observed-openmpi-hostfile") - - if err := os.WriteFile(readyFile, nil, 0o600); err != nil { - t.Fatal(err) - } - - command := exec.Command( - "/bin/bash", - "-c", - mpiLauncherPrelude( - generatedHostfile, - []mpiWorkerHost{{Name: "worker-0.job.namespace.svc", Slots: 4}}, - 2, - ), - ) - - command.Env = append( - os.Environ(), - "PATH="+fakeSSH(t, directory)+":"+os.Getenv("PATH"), - "PROJECT_SIGNAL_TEST_SSH_READY="+readyFile, - "PROJECT_SIGNAL_TEST_OMPI_HOSTFILE_OUTPUT="+observedOpenMPIHostfile, - "OMPI_MCA_orte_default_hostfile="+operatorHostfile, - ) - - if output, err := command.CombinedOutput(); err != nil { - t.Fatalf("launcher prelude failed: %v output=%q", err, output) - } - - content, err := os.ReadFile(generatedHostfile) - if err != nil { - t.Fatal(err) - } - - if strings.Contains(string(content), "stale-worker") || - string(content) != "worker-0.job.namespace.svc slots=4\n" { - t.Fatalf("generated hostfile = %q", content) - } - - observed, err := os.ReadFile(observedOpenMPIHostfile) - if err != nil { - t.Fatal(err) - } - - if string(observed) != generatedHostfile { - t.Fatalf("OpenMPI hostfile = %q, want %q", observed, generatedHostfile) - } -} - -func TestMPIWorkerHostsRejectsInvalidInputs(t *testing.T) { - t.Parallel() - - for name, test := range map[string]struct { - namespace string - jobName string - workerCount int64 - slots int64 - }{ - "zero workers": {namespace: "namespace", jobName: "job", slots: 1}, - "zero slots": {namespace: "namespace", jobName: "job", workerCount: 1}, - "invalid host": {namespace: "namespace", jobName: "INVALID", workerCount: 1, slots: 1}, - "too many hosts": {namespace: "namespace", jobName: "job", workerCount: MaxParticipants + 1, slots: 1}, - "too many slots": {namespace: "namespace", jobName: "job", workerCount: 1, slots: maxMPIWorkerSlots + 1}, - } { - t.Run(name, func(t *testing.T) { - t.Parallel() - - if _, err := mpiWorkerHosts( - test.namespace, - test.jobName, - test.workerCount, - test.slots, - ); err == nil { - t.Fatal("mpiWorkerHosts accepted invalid input") - } - }) - } -} - -func fakeSSH(t *testing.T, directory string) string { - t.Helper() - - binDirectory := filepath.Join(directory, "bin") - if err := os.Mkdir(binDirectory, 0o750); err != nil { - t.Fatal(err) - } - - content := []byte(`#!/bin/bash - if [[ -n "${PROJECT_SIGNAL_TEST_SSH_ARGUMENTS:-}" ]]; then - printf '%s\n' "$*" >> "${PROJECT_SIGNAL_TEST_SSH_ARGUMENTS}" -fi - if [[ -n "${PROJECT_SIGNAL_TEST_OMPI_HOSTFILE_OUTPUT:-}" ]]; then - printf '%s' "${OMPI_MCA_orte_default_hostfile:-}" > "${PROJECT_SIGNAL_TEST_OMPI_HOSTFILE_OUTPUT}" - fi - required=( - "-o BatchMode=yes" - "-o ConnectTimeout=2" - "-o ConnectionAttempts=1" - "-o StrictHostKeyChecking=accept-new" - "-o UserKnownHostsFile=/tmp/project-signal-known-hosts" - ) - arguments=" $* " - for option in "${required[@]}"; do - if [[ "${arguments}" != *" ${option} "* ]]; then - printf 'missing SSH option: %s\n' "${option}" >&2 - exit 255 - fi - done - if [[ -n "${PROJECT_SIGNAL_TEST_SSH_READY:-}" && -f "${PROJECT_SIGNAL_TEST_SSH_READY}" ]]; then - exit 0 - fi - printf 'simulated SSH connection failure for %s\n' "$*" >&2 - exit 255 - `) - if err := os.WriteFile(filepath.Join(binDirectory, "ssh"), content, 0o750); err != nil { - t.Fatal(err) - } - - return binDirectory -} - -func TestRenderMPIJobUsesExplicitLifecycleOverrides(t *testing.T) { - t.Parallel() - - config := testConfig() - config.ScriptEnvironmentJSON = `{ - "BACKEND_BACKOFF_LIMIT":"2", - "BACKEND_TTL_SECONDS_AFTER_FINISHED":"7200" - }` - - participants, tolerations, environment, err := ValidateConfig(config) - if err != nil { - t.Fatal(err) - } - - resources, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner()) - if err != nil { - t.Fatal(err) - } - - backoff, _, err := unstructured.NestedInt64(resources[0].Object, "spec", "runPolicy", "backoffLimit") - if err != nil { - t.Fatal(err) - } - - ttl, _, err := unstructured.NestedInt64(resources[0].Object, "spec", "runPolicy", "ttlSecondsAfterFinished") - if err != nil { - t.Fatal(err) - } - - if backoff != 2 || ttl != 7200 { - t.Fatalf("run policy backoff=%d ttl=%d", backoff, ttl) - } -} - -func TestRenderWorkloadPolicyOnParticipantContainers(t *testing.T) { - t.Parallel() - - policy := signalv1.WorkloadPolicy{ - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("8"), - corev1.ResourceMemory: resource.MustParse("32Gi"), - corev1.ResourceEphemeralStorage: resource.MustParse("10Gi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("8"), - corev1.ResourceMemory: resource.MustParse("32Gi"), - corev1.ResourceEphemeralStorage: resource.MustParse("20Gi"), - }, - }, - SecurityContext: &corev1.SecurityContext{ - RunAsUser: ptr(int64(0)), - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{"IPC_LOCK"}, - }, - }, - HostNetwork: true, - KueueTopology: &signalv1.KueueTopologyPolicy{ - RequiredLevel: corev1.LabelHostname, - }, - WorkingDir: "/opt/diagnostic", - Results: &corev1.EmptyDirVolumeSource{SizeLimit: quantity("2Gi")}, - Volumes: []signalv1.WorkloadVolume{ - { - Name: "host-proc", - MountPath: "/host/proc", - ReadOnly: true, - HostPath: &corev1.HostPathVolumeSource{Path: "/proc"}, - }, - { - Name: "work", - MountPath: "/work", - EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: quantity("8Gi")}, - }, - }, - } - - policyJSON, err := json.Marshal(policy) - if err != nil { - t.Fatal(err) - } - - config := testConfig() - config.BackendAPIVersion = "v1" - config.BackendKind = "Pod" - config.Queue = "certification" - config.GPUResourceCount = "8" - config.WorkloadPolicyJSON = string(policyJSON) - - participants, tolerations, environment, err := ValidateConfig(config) - if err != nil { - t.Fatal(err) - } - - resources, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner()) - if err != nil { - t.Fatal(err) - } - - var pod corev1.Pod - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(resources[0].Object, &pod); err != nil { - t.Fatal(err) - } - - container := pod.Spec.Containers[0] - - gpuLimit := container.Resources.Limits[corev1.ResourceName("nvidia.com/gpu")] - if !pod.Spec.HostNetwork || - pod.Spec.DNSPolicy != corev1.DNSClusterFirstWithHostNet || - pod.Annotations[requiredTopologyKey] != corev1.LabelHostname || - container.WorkingDir != "/opt/diagnostic" || - container.Resources.Requests.Cpu().String() != "8" || - container.Resources.Limits.Memory().String() != "32Gi" || - gpuLimit.String() != "8" || - container.SecurityContext == nil || - container.SecurityContext.RunAsUser == nil || - *container.SecurityContext.RunAsUser != 0 || - len(container.SecurityContext.Capabilities.Add) != 1 { - t.Fatalf("participant container = %#v", container) - } - - if len(pod.Spec.Volumes) != 4 || - pod.Spec.Volumes[1].EmptyDir.SizeLimit.String() != "2Gi" || - pod.Spec.Volumes[2].HostPath.Path != "/proc" || - pod.Spec.Volumes[3].EmptyDir.SizeLimit.String() != "8Gi" { - t.Fatalf("participant volumes = %#v", pod.Spec.Volumes) - } -} - -func TestMPIWorkloadPolicyDoesNotLeakOntoLauncher(t *testing.T) { - t.Parallel() - - policyJSON := `{ - "hostNetwork":true, - "kueueTopology":{"requiredLevel":"kubernetes.io/hostname"}, - "resources":{"requests":{"cpu":"4"},"limits":{"cpu":"4"}}, - "volumes":[{"name":"host-device","mountPath":"/dev/infiniband","hostPath":{"path":"/dev/infiniband"}}] - }` - config := testConfig() - config.WorkloadPolicyJSON = policyJSON - - participants, tolerations, environment, err := ValidateConfig(config) - if err != nil { - t.Fatal(err) - } - - resources, err := RenderResources(config, participants, tolerations, environment, testWorkflowOwner()) - if err != nil { - t.Fatal(err) - } - - content, err := json.Marshal(resources[0].Object) - if err != nil { - t.Fatal(err) - } - - var job struct { - Spec struct { - MPIReplicaSpecs map[string]struct { - Template corev1.PodTemplateSpec `json:"template"` - } `json:"mpiReplicaSpecs"` - } `json:"spec"` - } - if err := json.Unmarshal(content, &job); err != nil { - t.Fatal(err) - } - - launcherTemplate := job.Spec.MPIReplicaSpecs["Launcher"].Template - workerTemplate := job.Spec.MPIReplicaSpecs["Worker"].Template - launcher := launcherTemplate.Spec - worker := workerTemplate.Spec - - if launcher.HostNetwork || - launcher.DNSPolicy == corev1.DNSClusterFirstWithHostNet || - launcherTemplate.Annotations[requiredTopologyKey] != "" || - len(launcher.Volumes) != 2 || - len(launcher.Containers[0].VolumeMounts) != 2 { - t.Fatalf("launcher received participant policy: %#v", launcher) - } - - if !worker.HostNetwork || - worker.DNSPolicy != corev1.DNSClusterFirstWithHostNet || - workerTemplate.Annotations[requiredTopologyKey] != corev1.LabelHostname || - len(worker.Volumes) != 3 || - worker.Volumes[2].HostPath.Path != "/dev/infiniband" || - worker.Containers[0].Resources.Requests.Cpu().String() != "4" { - t.Fatalf("worker policy = %#v", worker) - } -} - -func TestValidateConfigRejectsKueueTopologyWithoutQueue(t *testing.T) { - t.Parallel() - - config := testConfig() - config.BackendAPIVersion = "v1" - config.BackendKind = "Pod" - config.Queue = "" - config.WorkloadPolicyJSON = `{"kueueTopology":{"requiredLevel":"kubernetes.io/hostname"}}` - - if _, _, _, err := ValidateConfig(config); err == nil || - !strings.Contains(err.Error(), "kueueTopology requires a queue") { - t.Fatalf("ValidateConfig topology error = %v", err) - } -} - -func TestValidateConfigRejectsMPIWorkerReadinessWithoutTimeBudget(t *testing.T) { - t.Parallel() - - config := testConfig() - config.BackendDeadlineSeconds = 1 - - if _, _, _, err := ValidateConfig(config); err == nil || - !strings.Contains(err.Error(), "at least 2 for MPIJob worker readiness") { - t.Fatalf("ValidateConfig deadline error = %v", err) - } -} - -func TestRenderRejectsGPUResourceInWorkloadPolicy(t *testing.T) { - t.Parallel() - - config := testConfig() - config.WorkloadPolicyJSON = `{"resources":{"limits":{"nvidia.com/gpu":"2"}}}` - - participants, tolerations, environment, err := ValidateConfig(config) - if err != nil { - t.Fatal(err) - } - - if _, err := RenderResources( - config, - participants, - tolerations, - environment, - testWorkflowOwner(), - ); err == nil || !strings.Contains(err.Error(), "gpu-resource-count") { - t.Fatalf("RenderResources error = %v", err) - } -} - -func TestParseParticipantsRejectsMalformedIdentity(t *testing.T) { - t.Parallel() - - tests := map[string]string{ - "noncontiguous index": `[{"index":1,"nodeName":"node-a","nodeUID":"uid-a"}]`, - "missing UID": `[{"index":0,"nodeName":"node-a","nodeUID":""}]`, - "duplicate node": `[{"index":0,"nodeName":"node-a","nodeUID":"uid-a"},{"index":1,"nodeName":"node-a","nodeUID":"uid-b"}]`, - "duplicate UID": `[{"index":0,"nodeName":"node-a","nodeUID":"uid-a"},{"index":1,"nodeName":"node-b","nodeUID":"uid-a"}]`, - } - - for name, participantsJSON := range tests { - t.Run(name, func(t *testing.T) { - t.Parallel() - - count := 1 - if strings.Contains(participantsJSON, `"index":1`) && strings.Count(participantsJSON, `"index"`) == 2 { - count = 2 - } - - if _, err := ParseParticipants(participantsJSON, count, "", ""); err == nil { - t.Fatal("ParseParticipants succeeded") - } - }) - } -} - -func TestParseParticipantsRejectsTrailingJSON(t *testing.T) { - t.Parallel() - - participantsJSON := `[{"index":0,"nodeName":"node-a","nodeUID":"uid-a"}] []` - if _, err := ParseParticipants(participantsJSON, 1, "", ""); err == nil { - t.Fatal("ParseParticipants accepted trailing JSON") - } -} - -func TestParseParticipantsCompatibilityFallbacks(t *testing.T) { - t.Parallel() - - for name, compatibility := range map[string][2]string{ - "request Node": {"node-request", ""}, - "legacy list": {"", "node-legacy,node-ignored"}, - } { - t.Run(name, func(t *testing.T) { - t.Parallel() - - participants, err := ParseParticipants("", 1, compatibility[0], compatibility[1]) - if err != nil { - t.Fatal(err) - } - - if len(participants) != 1 || participants[0].NodeName == "" { - t.Fatalf("participants = %#v", participants) - } - }) - } -} - -func TestValidateConfigRequiresDigestAndMPIQueue(t *testing.T) { - t.Parallel() - - config := testConfig() - - config.RunnerImage = "example.invalid/runner:latest" - if _, _, _, err := ValidateConfig(config); err == nil || !strings.Contains(err.Error(), "immutable sha256") { - t.Fatalf("digest validation error = %v", err) - } - - config = testConfig() - - config.Queue = "" - if _, _, _, err := ValidateConfig(config); err == nil || !strings.Contains(err.Error(), "queue is required") { - t.Fatalf("queue validation error = %v", err) - } -} - -func TestValidateConfigRequiresDeclaredRuntimeStage(t *testing.T) { - t.Parallel() - - config := testConfig() - config.ScriptRequiresStage = true - - if _, _, _, err := ValidateConfig(config); err == nil || - !strings.Contains(err.Error(), "runtime stage is required") { - t.Fatalf("ValidateConfig runtime stage error = %v", err) - } - - config.RuntimeStageName = "Collective/Bandwidth" - if _, _, _, err := ValidateConfig(config); err == nil || - !strings.Contains(err.Error(), "runtime stage name is invalid") { - t.Fatalf("ValidateConfig malformed stage error = %v", err) - } -} - -func TestValidateConfigAcceptsControllerScriptDigest(t *testing.T) { - t.Parallel() - - config := testConfig() - config.ScriptDigest = "sha256:3c11c8ff7b1182a1215a3ee4a0512901116c45646669b08a46710795120601a8" - - if _, _, _, err := ValidateConfig(config); err != nil { - t.Fatalf("ValidateConfig rejected controller script digest: %v", err) - } -} - -func TestValidateConfigRejectsMalformedScriptDigest(t *testing.T) { - t.Parallel() - - valid := "sha256:" + strings.Repeat("b", 64) - for name, digest := range map[string]string{ - "bare hex": strings.Repeat("b", 64), - "uppercase hex": "sha256:" + strings.Repeat("B", 64), - "wrong algorithm": "sha512:" + strings.Repeat("b", 64), - "short": "sha256:" + strings.Repeat("b", 63), - "long": "sha256:" + strings.Repeat("b", 65), - "leading space": " " + valid, - "trailing newline": valid + "\n", - } { - t.Run(name, func(t *testing.T) { - t.Parallel() - - config := testConfig() - config.ScriptDigest = digest - - if _, _, _, err := ValidateConfig(config); err == nil || - !strings.Contains(err.Error(), "algorithm-qualified SHA-256 digest") { - t.Fatalf("ValidateConfig script digest error = %v", err) - } - }) - } -} - -func TestValidateConfigRejectsReservedEnvironment(t *testing.T) { - t.Parallel() - - for _, environment := range []string{ - `{"PATH":"/tmp"}`, - `{"PROJECT_SIGNAL_RUN_UID":"replacement"}`, - } { - config := testConfig() - config.ScriptEnvironmentJSON = environment - - if _, _, _, err := ValidateConfig(config); err == nil || - !strings.Contains(err.Error(), "reserved name") { - t.Fatalf("environment %s error = %v", environment, err) - } - } -} - -func TestAggregatePassRequiresCompleteResultsAndIdentities(t *testing.T) { - t.Parallel() - - resources := testBackendResources() - results := []Result{ - {ParticipantIndex: 0, ExecutionState: "Succeeded", Verdict: "Pass", Reason: "Healthy"}, - {ParticipantIndex: 1, ExecutionState: "Succeeded", Verdict: "Pass", Reason: "Healthy"}, - } - - output, err := Aggregate(2, resources, results) - if err != nil { - t.Fatal(err) - } - - if output.Verdict != "Pass" || output.Reason != "AllParticipantsPassed" { - t.Fatalf("output = %#v", output) - } - - if _, err := Aggregate(2, resources[:1], results); err == nil { - t.Fatal("Aggregate accepted incomplete backend coverage") - } - - if _, err := Aggregate(2, resources, results[:1]); err == nil { - t.Fatal("Aggregate accepted incomplete results") - } -} - -func TestAggregateBackendFailurePrecedesDiagnosticFailure(t *testing.T) { - t.Parallel() - - output, err := Aggregate(2, testBackendResources(), []Result{ - {ParticipantIndex: 0, ExecutionState: "Succeeded", Verdict: "Fail", Reason: "DiagnosticFailed"}, - {ParticipantIndex: 1, ExecutionState: "Error", Verdict: "Inconclusive", Reason: "BackendFailed"}, - }) - if err != nil { - t.Fatal(err) - } - - if output.ExecutionState != "Error" || - output.Verdict != "Inconclusive" || - output.Reason != "BackendFailed" { - t.Fatalf("output = %#v", output) - } -} - -func TestAggregateRetainsDiagnosticFailureWhenBackendsSucceed(t *testing.T) { - t.Parallel() - - output, err := Aggregate(2, testBackendResources(), []Result{ - {ParticipantIndex: 0, ExecutionState: "Succeeded", Verdict: "Fail", Reason: "DiagnosticFailed"}, - {ParticipantIndex: 1, ExecutionState: "Succeeded", Verdict: "Pass", Reason: "Healthy"}, - }) - if err != nil { - t.Fatal(err) - } - - if output.Verdict != "Fail" || output.Reason != "ParticipantCertificationFailed" { - t.Fatalf("output = %#v", output) - } -} - -func TestAggregateBoundsResultLogMetadataAtMaximumParticipants(t *testing.T) { - t.Parallel() - - resources := make([]signalv1.BackendResourceReference, MaxParticipants) - - results := make([]Result, MaxParticipants) - for index := range MaxParticipants { - resources[index] = signalv1.BackendResourceReference{ - APIVersion: "kubeflow.org/v2beta1", - Kind: "MPIJob", - Namespace: "project-signal", - Name: fmt.Sprintf("run-p%d", index), - UID: types.UID(fmt.Sprintf("uid-%d", index)), - ParticipantIndexes: []int32{int32(index)}, - } - results[index] = Result{ - ParticipantIndex: index, - ExecutionState: "Succeeded", - Verdict: "Pass", - Reason: "Healthy", - ResultLog: &ResultLogMetadata{ - Current: ResultLogSourceMetadata{Status: "valid", Bytes: 128}, - Previous: ResultLogSourceMetadata{Status: "unavailable"}, - SelectedSource: "current", - }, - } - } - - output, err := Aggregate(MaxParticipants, resources, results) - if err != nil { - t.Fatal(err) - } - - if len(output.Measurements) > MaxMeasurementsBytes { - t.Fatalf("measurements size = %d", len(output.Measurements)) - } -} - -func TestAggregateRejectsDuplicateIdentityAndOversizedMeasurements(t *testing.T) { - t.Parallel() - - resources := testBackendResources() - resources[1] = resources[0] - resources[1].ParticipantIndexes = []int32{1} - - results := []Result{ - {ParticipantIndex: 0, ExecutionState: "Succeeded", Verdict: "Pass", Reason: "Healthy"}, - {ParticipantIndex: 1, ExecutionState: "Succeeded", Verdict: "Pass", Reason: "Healthy"}, - } - if _, err := Aggregate(2, resources, results); err == nil { - t.Fatal("Aggregate accepted duplicate backend identity") - } - - resources = testBackendResources() - - results[1].Measurements = strings.Repeat("x", MaxMeasurementsBytes+1) - if _, err := Aggregate(2, resources, results); err == nil { - t.Fatal("Aggregate accepted oversized measurements") - } -} - -func TestParticipantBounds(t *testing.T) { - t.Parallel() - - if _, err := ParseParticipants("[]", 0, "", ""); err == nil { - t.Fatal("ParseParticipants accepted zero participants") - } - - if _, err := ParseParticipants("[]", MaxParticipants+1, "", ""); err == nil { - t.Fatal("ParseParticipants accepted too many participants") - } -} - -func TestBackendResourceRequiresMatchingParticipantLabel(t *testing.T) { - t.Parallel() - - resource := ordinaryPod( - testConfig(), - testParticipants()[0], - nil, - nil, - signalv1.WorkloadPolicy{}, - testWorkflowOwner(), - 900, - 0, - ) - resource.UID = types.UID("pod-uid") - - content, err := json.Marshal(resource) - if err != nil { - t.Fatal(err) - } - - var object map[string]any - if err := json.Unmarshal(content, &object); err != nil { - t.Fatal(err) - } - - unstructuredResource := &unstructured.Unstructured{Object: object} - if _, err := BackendResource(unstructuredResource, 1); err == nil { - t.Fatal("BackendResource accepted mismatched participant label") - } -} - -func TestParseResultLogUsesLastCompleteRecord(t *testing.T) { - t.Parallel() - - encode := func(value string) string { - return base64.StdEncoding.EncodeToString([]byte(value)) - } - logContent := "noise\n" + resultPrefix + - encode("Pass") + " " + - encode("Healthy") + " " + - encode("temperature=42") + " 0\n" - - result, err := ParseResultLog(logContent, 1) - if err != nil { - t.Fatal(err) - } - - if result.ParticipantIndex != 1 || - result.Verdict != "Pass" || - result.Measurements != "temperature=42" { - t.Fatalf("result = %#v", result) - } -} - -func TestParseResultLogAllowsEmptyMeasurements(t *testing.T) { - t.Parallel() - - encode := func(value string) string { - return base64.StdEncoding.EncodeToString([]byte(value)) - } - logContent := resultPrefix + encode("Pass") + " " + encode("Healthy") + " - 0\n" - - result, err := ParseResultLog(logContent, 0) - if err != nil { - t.Fatal(err) - } - - if result.Measurements != "" { - t.Fatalf("measurements = %q", result.Measurements) - } -} - -func TestParseResultLogAcceptsInconclusiveExitTwo(t *testing.T) { - t.Parallel() - - encode := func(value string) string { - return base64.StdEncoding.EncodeToString([]byte(value)) - } - logContent := resultPrefix + - encode("Inconclusive") + " " + - encode("DiagnosticUnavailable") + " " + - encode("device_state=unknown") + " 2\n" - - result, err := ParseResultLog(logContent, 0) - if err != nil { - t.Fatal(err) - } - - if result.ExecutionState != "Succeeded" || - result.Verdict != "Inconclusive" || - result.ExitCode != 2 { - t.Fatalf("result = %#v", result) - } -} - -func TestParseResultLogRejectsContradictoryVerdictExitCode(t *testing.T) { - t.Parallel() - - encode := func(value string) string { - return base64.StdEncoding.EncodeToString([]byte(value)) - } - - for name, test := range map[string]struct { - verdict string - exitCode string - }{ - "Pass with diagnostic failure": {verdict: "Pass", exitCode: "1"}, - "Fail with successful process": {verdict: "Fail", exitCode: "0"}, - "Inconclusive with successful process": {verdict: "Inconclusive", exitCode: "0"}, - "unexpected exit code": {verdict: "Inconclusive", exitCode: "3"}, - } { - t.Run(name, func(t *testing.T) { - t.Parallel() - - logContent := resultPrefix + - encode(test.verdict) + " " + - encode("SyntheticReason") + " - " + - test.exitCode + "\n" - if _, err := ParseResultLog(logContent, 0); err == nil { - t.Fatal("ParseResultLog accepted contradictory result") - } - }) - } -} - -func TestParseResultLogRejectsMalformedAndMissingResults(t *testing.T) { - t.Parallel() - - encode := func(value string) string { - return base64.StdEncoding.EncodeToString([]byte(value)) - } - - for name, logContent := range map[string]string{ - "malformed verdict": resultPrefix + "%%% " + encode("Reason") + " - 2\n", - "missing verdict": resultPrefix + "- " + encode("Reason") + " - 2\n", - "missing reason": resultPrefix + encode("Inconclusive") + " - - 2\n", - "missing record": "diagnostic output only\n", - } { - t.Run(name, func(t *testing.T) { - t.Parallel() - - if _, err := ParseResultLog(logContent, 0); err == nil { - t.Fatal("ParseResultLog accepted invalid result") - } - }) - } -} - -func TestResultCommandClearsReservedFilesAndReportsExitCode(t *testing.T) { - t.Parallel() - - command := resultCommand() - if len(command) != 3 { - t.Fatalf("resultCommand() = %#v", command) - } - - for _, required := range []string{ - "rm -f /results/verdict /results/reason /results/measurements.txt", - `printf 'PROJECT_SIGNAL_RESULTS_V1 prepared\n'`, - "canonicalize_result_file /results/verdict", - "canonicalize_result_file /results/reason", - "canonicalize_result_file /results/measurements.txt", - `printf 'PROJECT_SIGNAL_RESULTS_V1 verdict=%s reason=%s measurements=%s\n'`, - `printf 'PROJECT_SIGNAL_RESULT_V2 %s %s %s %d\n'`, - `exit "$status"`, - } { - if !strings.Contains(command[2], required) { - t.Fatalf("result command does not contain %q:\n%s", required, command[2]) - } - } - - for _, forbidden := range []string{ - "ScriptDidNotProduceReason", - "script_exit_code=", - } { - if strings.Contains(command[2], forbidden) { - t.Fatalf("result command still contains fallback %q:\n%s", forbidden, command[2]) - } - } -} - -func TestResultCommandCanonicalizesTrailingNewlines(t *testing.T) { - t.Parallel() - - directory := t.TempDir() - resultsDirectory := filepath.Join(directory, "results") - - scriptsDirectory := filepath.Join(directory, "scripts") - if err := os.MkdirAll(scriptsDirectory, 0o750); err != nil { - t.Fatal(err) - } - - script := filepath.Join(scriptsDirectory, "diagnostic.sh") - if err := os.WriteFile(script, []byte(`#!/bin/bash -printf 'Pass\n' > "${PROJECT_SIGNAL_RESULTS_DIR}/verdict" -printf 'Healthy\n' > "${PROJECT_SIGNAL_RESULTS_DIR}/reason" -printf 'temperature=42\n' > "${PROJECT_SIGNAL_RESULTS_DIR}/measurements.txt" -`), 0o750); err != nil { - t.Fatal(err) - } - - command := resultCommand() - commandText := strings.ReplaceAll(command[2], "/results", resultsDirectory) - commandText = strings.ReplaceAll(commandText, "/scripts", scriptsDirectory) - process := exec.Command(command[0], command[1], commandText) - - process.Env = append( - os.Environ(), - "PROJECT_SIGNAL_SCRIPT_KEY=diagnostic.sh", - "PROJECT_SIGNAL_RESULTS_DIR="+resultsDirectory, - ) - - output, err := process.CombinedOutput() - if err != nil { - t.Fatalf("result command: %v\n%s", err, output) - } - - result, err := ParseResultLog(string(output), 0) - if err != nil { - t.Fatalf("ParseResultLog: %v\n%s", err, output) - } - - if result.Verdict != "Pass" || result.Reason != "Healthy" || result.Measurements != "temperature=42" { - t.Fatalf("result = %#v", result) - } - - for name, want := range map[string]string{ - "verdict": "Pass", - "reason": "Healthy", - "measurements.txt": "temperature=42", - } { - content, err := os.ReadFile(filepath.Join(resultsDirectory, name)) - if err != nil { - t.Fatal(err) - } - - if string(content) != want { - t.Fatalf("%s = %q, want %q", name, content, want) - } - } -} - -func testConfig() Config { - participantsJSON, err := json.Marshal(testParticipants()) - if err != nil { - panic(err) - } - - return Config{ - Namespace: "project-signal", - WorkflowName: "run-example-a1", - RunName: "run-example", - RunUID: types.UID("run-uid"), - ParticipantsJSON: string(participantsJSON), - ParticipantCount: 2, - RequestNodeName: "node-a", - LegacyParticipants: "node-a,node-b", - ScriptConfigMap: "run-example-scripts", - ScriptKey: "check.sh", - ScriptDigest: "sha256:" + strings.Repeat("b", 64), - RunnerImage: "example.invalid/runner@sha256:" + strings.Repeat("a", 64), - BackendAPIVersion: "kubeflow.org/v2beta1", - BackendKind: "MPIJob", - BackendDeadlineSeconds: 540, - Queue: "certification", - TolerationsJSON: `[{"key":"nvidia.com/gpu","operator":"Exists","effect":"NoSchedule"}]`, - ScriptEnvironmentJSON: `{"CHECK_MODE":"quick"}`, - GPUResourceCount: "1", - Attempt: 1, - BackoffLimit: 0, - TTLSeconds: 3600, - } -} - -func testParticipants() []signalv1.Participant { - return []signalv1.Participant{ - {Index: 0, NodeName: "node-a", NodeUID: types.UID("uid-a")}, - {Index: 1, NodeName: "node-b", NodeUID: types.UID("uid-b")}, - } -} - -func environmentValue(environment []corev1.EnvVar, name string) string { - for _, variable := range environment { - if variable.Name == name { - return variable.Value - } - } - - return "" -} - -func testBackendResources() []signalv1.BackendResourceReference { - return []signalv1.BackendResourceReference{ - { - APIVersion: "v1", - Kind: "Pod", - Namespace: "project-signal", - Name: "run-example-p0", - UID: types.UID("uid-0"), - ParticipantIndexes: []int32{0}, - }, - { - APIVersion: "v1", - Kind: "Pod", - Namespace: "project-signal", - Name: "run-example-p1", - UID: types.UID("uid-1"), - ParticipantIndexes: []int32{1}, - }, - } -} - -func testWorkflowOwner() metav1.OwnerReference { - controller := true - - return metav1.OwnerReference{ - APIVersion: "argoproj.io/v1alpha1", - Kind: "Workflow", - Name: "run-example-a1", - UID: types.UID("workflow-uid"), - Controller: &controller, - } -} - -func quantity(value string) *resource.Quantity { - result := resource.MustParse(value) - - return &result -} diff --git a/internal/signaladapter/backend_evidence_test.go b/internal/signaladapter/backend_evidence_test.go deleted file mode 100644 index 67f279f548..0000000000 --- a/internal/signaladapter/backend_evidence_test.go +++ /dev/null @@ -1,481 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -package signaladapter - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "strings" - "testing" - - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - dynamicfake "k8s.io/client-go/dynamic/fake" - kubernetesfake "k8s.io/client-go/kubernetes/fake" -) - -func TestCaptureBackendEvidencePreservesBackoffFailureAndPreviousInvalidResult(t *testing.T) { - t.Parallel() - - const ( - namespace = "project-signal" - mpiJobUID = types.UID("mpi-job-uid") - launcherJobUID = types.UID("launcher-job-uid") - ) - - mpiJob := &unstructured.Unstructured{Object: map[string]any{ - "apiVersion": "kubeflow.org/v2beta1", - "kind": "MPIJob", - "metadata": map[string]any{ - "name": "run-p0", - "namespace": namespace, - "uid": string(mpiJobUID), - }, - "status": map[string]any{ - "conditions": []any{map[string]any{ - "type": "Failed", - "status": "True", - "reason": "LauncherJobFailed", - "message": "launcher failed token=secret-value", - }}, - "replicaStatuses": map[string]any{ - "Launcher": map[string]any{"failed": int64(1)}, - "Worker": map[string]any{"active": int64(1)}, - }, - }, - }} - launcherJob := &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{ - Name: "run-p0-launcher", - Namespace: namespace, - UID: launcherJobUID, - OwnerReferences: []metav1.OwnerReference{{UID: mpiJobUID}}, - }, - Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{ - Type: batchv1.JobFailed, - Status: corev1.ConditionTrue, - Reason: batchv1.JobReasonBackoffLimitExceeded, - Message: "Job has reached the specified backoff limit", - }}}, - } - launcher := evidenceTestPod( - "run-p0-launcher-pod", - "launcher-pod-uid", - "launcher", - []metav1.OwnerReference{{UID: launcherJobUID}}, - 1, - ) - worker := evidenceTestPod( - "run-p0-worker-0", - "worker-pod-uid", - "worker", - []metav1.OwnerReference{{UID: mpiJobUID}}, - 0, - ) - - dynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds( - runtime.NewScheme(), - map[schema.GroupVersionResource]string{mpiJobGVR: "MPIJobList"}, - mpiJob, - ) - runner := &Runner{ - Dynamic: dynamicClient, - Kubernetes: kubernetesfake.NewSimpleClientset(launcherJob, launcher, worker), - ReadPodLog: func( - _ context.Context, - _, podName, containerName string, - previous bool, - ) (string, error) { - switch { - case podName == launcher.Name && containerName == "launcher" && !previous: - return "current launcher log password=hunter2\n", nil - case podName == launcher.Name && containerName == "launcher" && previous: - return resultsStatusPrefix + - "verdict=present reason=present measurements=missing\n" + - resultPrefix + "%%% bm90LXVzZWQ= - 2\n", nil - case podName == worker.Name && containerName == "worker" && !previous: - return resultsStatusPrefix + "prepared\nworker ready\n", nil - default: - return "", errors.New("unexpected log request") - } - }, - } - - content, complete, err := runner.captureBackendEvidence( - t.Context(), - Config{Namespace: namespace, RunName: "run"}, - []createdResource{{ - gvr: mpiJobGVR, - resource: mpiJob, - participantIndex: 0, - }}, - ) - if err != nil { - t.Fatal(err) - } - - if !complete { - t.Fatalf("captureBackendEvidence complete = false, content=%s", content) - } - - var evidence BackendEvidence - if err := json.Unmarshal([]byte(content), &evidence); err != nil { - t.Fatal(err) - } - - participant := evidence.Participants[0] - if participant.LauncherJob == nil || - len(participant.LauncherJob.Conditions) != 1 || - participant.LauncherJob.Conditions[0].Reason != batchv1.JobReasonBackoffLimitExceeded { - t.Fatalf("launcher Job evidence = %#v", participant.LauncherJob) - } - - launcherEvidence := podEvidenceByRole(t, participant.Pods, "launcher") - - container := launcherEvidence.Containers[0] - if container.RestartCount != 1 || - container.State.ExitCode != 1 || - container.PreviousLog.ResultMarkerStatus != "invalid" || - container.PreviousLog.ResultDirectoryStatus != - "verdict=present reason=present measurements=missing" { - t.Fatalf("launcher container evidence = %#v", container) - } - - if strings.Contains(container.CurrentLog.Content, "hunter2") || - !strings.Contains(container.CurrentLog.Content, "password=[REDACTED]") { - t.Fatalf("current log was not redacted: %q", container.CurrentLog.Content) - } - - result, hardFailure := backendCompletionFromLogs( - resultLogSnapshot{ - current: resultLogSourceWithContent("current launcher log\n", nil), - previous: resultLogSourceWithContent( - resultPrefix+"%%% bm90LXVzZWQ= - 2\n", - nil, - ), - }, - 0, - false, - ) - if !hardFailure { - t.Fatal("invalid previous result did not fail closed") - } - - output, err := Aggregate(1, testBackendResources()[:1], []Result{result}) - if err != nil { - t.Fatal(err) - } - - output.BackendEvidence = content - recordCleanupFailure(&output) - - if output.Reason != "BackendExecutionFailed" || - output.CleanupFailure != "BackendCleanupFailed" || - output.Verdict != "Inconclusive" { - t.Fatalf("output = %#v", output) - } -} - -func TestCaptureBackendEvidenceProjectsParticipantPod(t *testing.T) { - t.Parallel() - - backend := &unstructured.Unstructured{Object: map[string]any{ - "apiVersion": "v1", - "kind": "Pod", - "metadata": map[string]any{ - "name": "run-p0", - "namespace": "project-signal", - "uid": "pod-uid", - }, - "status": map[string]any{"phase": "Succeeded"}, - }} - pod := evidenceTestPod("run-p0", "pod-uid", "main", nil, 0) - pod.Status.Phase = corev1.PodSucceeded - pod.Status.ContainerStatuses[0].State.Terminated.ExitCode = 0 - pod.Status.ContainerStatuses[0].State.Terminated.Reason = "Completed" - - runner := &Runner{ - Dynamic: dynamicfake.NewSimpleDynamicClientWithCustomListKinds( - runtime.NewScheme(), - map[schema.GroupVersionResource]string{podGVR: "PodList"}, - backend, - ), - Kubernetes: kubernetesfake.NewSimpleClientset(pod), - ReadPodLog: func( - _ context.Context, - _, _, _ string, - previous bool, - ) (string, error) { - if previous { - t.Fatal("read previous log without a restart") - } - - return resultsStatusPrefix + "prepared\n", nil - }, - } - - content, complete, err := runner.captureBackendEvidence( - context.Background(), - Config{Namespace: "project-signal", RunName: "run"}, - []createdResource{{ - gvr: podGVR, - resource: backend, - participantIndex: 0, - }}, - ) - if err != nil { - t.Fatal(err) - } - - if !complete { - t.Fatalf("captureBackendEvidence complete = false, content=%s", content) - } - - var evidence BackendEvidence - if err := json.Unmarshal([]byte(content), &evidence); err != nil { - t.Fatal(err) - } - - participant := evidence.Participants[0] - if participant.Resource.Phase != "Succeeded" || - len(participant.Pods) != 1 || - participant.Pods[0].Role != "participant" || - participant.Pods[0].Containers[0].PreviousLog.Status != "not-applicable" { - t.Fatalf("participant evidence = %#v", participant) - } -} - -func TestCaptureBackendEvidenceFailurePreventsSuccess(t *testing.T) { - t.Parallel() - - output, err := Aggregate(1, testBackendResources()[:1], []Result{{ - ParticipantIndex: 0, - ExecutionState: "Succeeded", - Verdict: "Pass", - Reason: "Healthy", - }}) - if err != nil { - t.Fatal(err) - } - - recordEvidenceCaptureFailure(&output) - - if output.ExecutionState != "Error" || - output.Verdict != "Inconclusive" || - output.Reason != "BackendEvidenceCaptureFailed" { - t.Fatalf("output = %#v", output) - } -} - -func TestPersistBackendEvidenceFailurePreventsSuccess(t *testing.T) { - t.Parallel() - - output, err := Aggregate(1, testBackendResources()[:1], []Result{{ - ParticipantIndex: 0, - ExecutionState: "Succeeded", - Verdict: "Pass", - Reason: "Healthy", - }}) - if err != nil { - t.Fatal(err) - } - - output.BackendEvidence = `{"version":"v1","complete":true,"participants":[{}]}` - runner := &Runner{ - PersistBackendEvidence: func(string) error { - return errors.New("write failed") - }, - } - - if err := runner.persistEvidenceBeforeCleanup(&output); err != nil { - t.Fatal(err) - } - - var evidence BackendEvidence - if err := json.Unmarshal([]byte(output.BackendEvidence), &evidence); err != nil { - t.Fatal(err) - } - - if output.ExecutionState != "Error" || - output.Verdict != "Inconclusive" || - output.Reason != "BackendEvidenceCaptureFailed" || - evidence.Complete || - !strings.Contains(evidence.CaptureError, "persist backend evidence before cleanup") { - t.Fatalf("output = %#v, evidence = %#v", output, evidence) - } -} - -func TestCapturedLogIsBoundedAndRedacted(t *testing.T) { - t.Parallel() - - content := strings.Repeat("x", maxEvidenceLogBytes+64) + - "\nAuthorization: Bearer secret-token\n" + - "https://user:password@example.test/path\n" - captured := capturedLog(content, nil) - - if captured.CapturedBytes > maxEvidenceLogBytes || - !captured.Truncated || - strings.Contains(captured.Content, "secret-token") || - strings.Contains(captured.Content, "user:password") { - t.Fatalf("captured log = %#v", captured) - } -} - -func TestCapturedLogRedactsSensitiveValuesInRetainedTail(t *testing.T) { - t.Parallel() - - content := strings.Repeat("x", maxEvidenceLogBytes+64) + - "\nAuthorization: " + "Bearer " + "opaque-value\n" + - "https://" + "alice:credential-value@" + "example.test/path\n" - captured := capturedLog(content, nil) - - if strings.Contains(captured.Content, "opaque-value") || - strings.Contains(captured.Content, "alice:credential-value") { - t.Fatalf("captured log = %#v", captured) - } -} - -func TestResultDirectoryStatusRejectsUnstructuredMarker(t *testing.T) { - t.Parallel() - - const value = "do-not-retain" - - status := resultDirectoryStatus(resultsStatusPrefix + value) - if status != "invalid" || strings.Contains(status, value) { - t.Fatalf("resultDirectoryStatus() = %q", status) - } -} - -func TestMarshalBackendEvidenceBoundsOversizedProjection(t *testing.T) { - t.Parallel() - - evidence := BackendEvidence{ - Version: backendEvidenceVersion, - Complete: true, - Participants: make([]ParticipantBackendEvidence, 64), - } - for participantIndex := range evidence.Participants { - participant := &evidence.Participants[participantIndex] - participant.ParticipantIndex = participantIndex - participant.Resource = BackendObjectEvidence{ - APIVersion: "kubeflow.org/v2beta1", - Kind: "MPIJob", - Namespace: "project-signal", - Name: fmt.Sprintf("run-p%d", participantIndex), - UID: types.UID(fmt.Sprintf("mpi-%d", participantIndex)), - Message: strings.Repeat("m", maxEvidenceTextBytes), - } - - participant.Pods = make([]PodEvidence, 4) - for podIndex := range participant.Pods { - pod := &participant.Pods[podIndex] - pod.Name = fmt.Sprintf("run-p%d-%d", participantIndex, podIndex) - pod.UID = types.UID(fmt.Sprintf("pod-%d-%d", participantIndex, podIndex)) - pod.Containers = []ContainerEvidence{{ - Name: "participant", - CurrentLog: CapturedLogEvidence{ - Status: "captured", - Bytes: maxEvidenceLogBytes, - CapturedBytes: maxEvidenceLogBytes, - Content: strings.Repeat("x", maxEvidenceLogBytes), - }, - PreviousLog: CapturedLogEvidence{ - Status: "captured", - Bytes: maxEvidenceLogBytes, - CapturedBytes: maxEvidenceLogBytes, - Content: strings.Repeat("y", maxEvidenceLogBytes), - }, - }} - } - } - - content, complete, err := marshalBackendEvidence(evidence) - if err != nil { - t.Fatal(err) - } - - if complete { - t.Fatal("oversized evidence remained complete") - } - - if len(content) > MaxBackendEvidenceBytes { - t.Fatalf("backend evidence size = %d, want <= %d", len(content), MaxBackendEvidenceBytes) - } - - if err := ValidateBackendEvidence(string(content)); err != nil { - t.Fatalf("ValidateBackendEvidence() error = %v", err) - } - - var bounded BackendEvidence - if err := json.Unmarshal(content, &bounded); err != nil { - t.Fatal(err) - } - - if bounded.Complete || bounded.Participants[0].CaptureError == "" { - t.Fatalf("bounded evidence = %#v", bounded) - } -} - -func evidenceTestPod( - name string, - uid types.UID, - containerName string, - owners []metav1.OwnerReference, - restarts int32, -) *corev1.Pod { - lastState := corev1.ContainerState{} - if restarts > 0 { - lastState.Terminated = &corev1.ContainerStateTerminated{ - ExitCode: 1, - Reason: "Error", - } - } - - return &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: "project-signal", - UID: uid, - OwnerReferences: owners, - Labels: map[string]string{ - "signal.unbounded-cloud.io/run": "run", - "signal.unbounded-cloud.io/participant-index": "0", - }, - }, - Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: containerName}}}, - Status: corev1.PodStatus{ - Phase: corev1.PodFailed, - ContainerStatuses: []corev1.ContainerStatus{{ - Name: containerName, - RestartCount: restarts, - State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ - ExitCode: 1, - Reason: "Error", - }}, - LastTerminationState: lastState, - }}, - }, - } -} - -func podEvidenceByRole(t *testing.T, pods []PodEvidence, role string) PodEvidence { - t.Helper() - - for _, pod := range pods { - if pod.Role == role { - return pod - } - } - - t.Fatalf("pod role %q was not captured: %#v", role, pods) - - return PodEvidence{} -} diff --git a/internal/signaladapter/runner.go b/internal/signaladapter/runner.go index f6a5609798..a2eff9962e 100644 --- a/internal/signaladapter/runner.go +++ b/internal/signaladapter/runner.go @@ -335,66 +335,6 @@ func (r *Runner) waitForResult(ctx context.Context, config Config, backend creat } } -func backendCompletionResult( - logContent string, - logErr error, - participantIndex int, - backendSucceeded bool, -) Result { - return backendCompletionFromSource( - resultLogSource{ - content: logContent, - err: logErr, - metadata: resultLogSourceMetadata( - logContent, - logErr, - ), - }, - participantIndex, - backendSucceeded, - ) -} - -func backendCompletionFromSource( - source resultLogSource, - participantIndex int, - backendSucceeded bool, -) Result { - result := Result{ - ParticipantIndex: participantIndex, - ExecutionState: "Error", - Verdict: "Inconclusive", - Reason: "BackendExecutionFailed", - ResultLog: &ResultLogMetadata{ - Current: source.metadata, - Previous: ResultLogSourceMetadata{Status: "not-read"}, - SelectedSource: "", - }, - } - - if source.err != nil { - if backendSucceeded { - result.Reason = "BackendResultUnavailable: " + source.err.Error() - } - - return result - } - - parsed, err := ParseResultLog(source.content, participantIndex) - if err != nil { - if backendSucceeded { - result.Reason = "BackendResultInvalid: " + err.Error() - } - - return result - } - - parsed.ResultLog = result.ResultLog - parsed.ResultLog.SelectedSource = "current" - - return completedResult(parsed, backendSucceeded) -} - func backendCompletionFromLogs( logs resultLogSnapshot, participantIndex int, diff --git a/internal/signaladapter/runner_test.go b/internal/signaladapter/runner_test.go deleted file mode 100644 index 6ec2d68f5a..0000000000 --- a/internal/signaladapter/runner_test.go +++ /dev/null @@ -1,404 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -package signaladapter - -import ( - "context" - "encoding/base64" - "fmt" - "strings" - "testing" - "time" - - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - dynamicfake "k8s.io/client-go/dynamic/fake" - kubernetesfake "k8s.io/client-go/kubernetes/fake" -) - -func TestCompletedResultAcceptsVerdictAssociatedTerminalState(t *testing.T) { - t.Parallel() - - for name, test := range map[string]struct { - result Result - backendSucceeded bool - }{ - "Pass": { - result: Result{ParticipantIndex: 0, ExecutionState: "Succeeded", Verdict: "Pass", Reason: "Healthy"}, - backendSucceeded: true, - }, - "Fail": { - result: Result{ - ParticipantIndex: 0, - ExecutionState: "Succeeded", - Verdict: "Fail", - Reason: "DiagnosticFailed", - ExitCode: 1, - }, - }, - "Inconclusive": { - result: Result{ - ParticipantIndex: 0, - ExecutionState: "Succeeded", - Verdict: "Inconclusive", - Reason: "DiagnosticUnavailable", - ExitCode: 2, - }, - }, - } { - t.Run(name, func(t *testing.T) { - t.Parallel() - - result := completedResult(test.result, test.backendSucceeded) - if result.ExecutionState != "Succeeded" || result.Verdict != test.result.Verdict { - t.Fatalf("completedResult() = %#v", result) - } - }) - } -} - -func TestBackendCompletionResultKeepsValidInconclusiveFromFailedBackend(t *testing.T) { - t.Parallel() - - logContent := encodedResultRecord("Inconclusive", "DiagnosticUnavailable", "device_state=unknown", 2) - - result := backendCompletionResult(logContent, nil, 0, false) - if result.ExecutionState != "Succeeded" || - result.Verdict != "Inconclusive" || - result.Reason != "DiagnosticUnavailable" { - t.Fatalf("backendCompletionResult() = %#v", result) - } -} - -func TestBackendCompletionResultFailsClosedWithoutValidResult(t *testing.T) { - t.Parallel() - - for name, logContent := range map[string]string{ - "malformed result": resultPrefix + "%%% bm90LXVzZWQ= - 2\n", - "missing result": "diagnostic output only\n", - } { - t.Run(name, func(t *testing.T) { - t.Parallel() - - result := backendCompletionResult(logContent, nil, 0, false) - if result.ExecutionState != "Error" || - result.Verdict != "Inconclusive" || - result.Reason != "BackendExecutionFailed" { - t.Fatalf("backendCompletionResult() = %#v", result) - } - }) - } -} - -func TestBackendCompletionFromLogsRecoversPreviousResult(t *testing.T) { - t.Parallel() - - logs := resultLogSnapshot{ - current: resultLogSourceWithContent("launcher restarted\n", nil), - previous: resultLogSourceWithContent( - encodedResultRecord("Inconclusive", "DiagnosticUnavailable", "state=unknown", 2), - nil, - ), - } - - result, hardFailure := backendCompletionFromLogs(logs, 0, false) - if hardFailure || - result.ExecutionState != "Succeeded" || - result.Verdict != "Inconclusive" || - result.Reason != "DiagnosticUnavailable" || - result.ResultLog == nil || - result.ResultLog.SelectedSource != "previous" || - result.ResultLog.Current.Status != "missing" || - result.ResultLog.Previous.Status != "valid" { - t.Fatalf("backendCompletionFromLogs() = %#v, hardFailure=%t", result, hardFailure) - } -} - -func TestBackendCompletionFromLogsPrefersCurrentResult(t *testing.T) { - t.Parallel() - - record := encodedResultRecord("Pass", "Healthy", "state=ready", 0) - logs := resultLogSnapshot{ - current: resultLogSourceWithContent(record, nil), - previous: resultLogSourceWithContent(record, nil), - } - - result, hardFailure := backendCompletionFromLogs(logs, 0, true) - if hardFailure || - result.ExecutionState != "Succeeded" || - result.Verdict != "Pass" || - result.ResultLog == nil || - result.ResultLog.SelectedSource != "current" { - t.Fatalf("backendCompletionFromLogs() = %#v, hardFailure=%t", result, hardFailure) - } -} - -func TestBackendCompletionFromLogsRejectsConflicts(t *testing.T) { - t.Parallel() - - logs := resultLogSnapshot{ - current: resultLogSourceWithContent( - encodedResultRecord("Pass", "Healthy", "state=ready", 0), - nil, - ), - previous: resultLogSourceWithContent( - encodedResultRecord("Inconclusive", "DiagnosticUnavailable", "state=unknown", 2), - nil, - ), - } - - result, hardFailure := backendCompletionFromLogs(logs, 0, false) - if !hardFailure || - result.ExecutionState != "Error" || - result.Verdict != "Inconclusive" || - !strings.Contains(result.Reason, "conflicting results") || - result.ResultLog == nil || - result.ResultLog.Current.Status != "valid" || - result.ResultLog.Previous.Status != "valid" { - t.Fatalf("backendCompletionFromLogs() = %#v, hardFailure=%t", result, hardFailure) - } -} - -func TestBackendCompletionFromLogsRejectsMalformedCurrentBeforePreviousFallback(t *testing.T) { - t.Parallel() - - logs := resultLogSnapshot{ - current: resultLogSourceWithContent(resultPrefix+"%%% bm90LXVzZWQ= - 2\n", nil), - previous: resultLogSourceWithContent( - encodedResultRecord("Inconclusive", "DiagnosticUnavailable", "state=unknown", 2), - nil, - ), - } - - result, hardFailure := backendCompletionFromLogs(logs, 0, false) - if !hardFailure || - result.ExecutionState != "Error" || - result.Reason != "BackendExecutionFailed" || - result.ResultLog == nil || - result.ResultLog.Current.Status != "invalid" { - t.Fatalf("backendCompletionFromLogs() = %#v, hardFailure=%t", result, hardFailure) - } -} - -func TestResultLogsReadsCurrentAndPreviousLauncherLogs(t *testing.T) { - t.Parallel() - - job := &unstructured.Unstructured{} - job.SetAPIVersion("kubeflow.org/v2beta1") - job.SetKind("MPIJob") - job.SetName("run-p0") - job.SetNamespace("project-signal") - job.SetUID("job-uid") - - launcher := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "run-p0-launcher", - Namespace: "project-signal", - Labels: map[string]string{ - "signal.unbounded-cloud.io/run": "run", - "signal.unbounded-cloud.io/participant-index": "0", - }, - OwnerReferences: []metav1.OwnerReference{{UID: "launcher-job-uid"}}, - }, - Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "launcher"}}}, - } - launcherJob := &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{ - Name: "run-p0-launcher", - Namespace: "project-signal", - UID: "launcher-job-uid", - OwnerReferences: []metav1.OwnerReference{{UID: "job-uid"}}, - }, - } - - var previousValues []bool - - runner := &Runner{ - Kubernetes: kubernetesfake.NewSimpleClientset(launcher, launcherJob), - ReadPodLog: func( - _ context.Context, - namespace, podName, containerName string, - previous bool, - ) (string, error) { - if namespace != launcher.Namespace || - podName != launcher.Name || - containerName != "launcher" { - t.Fatalf("read log identity = %s/%s %s", namespace, podName, containerName) - } - - previousValues = append(previousValues, previous) - - return encodedResultRecord("Pass", "Healthy", "", 0), nil - }, - } - - logs := runner.resultLogs( - t.Context(), - Config{Namespace: "project-signal", RunName: "run"}, - createdResource{resource: job, participantIndex: 0}, - true, - ) - if len(previousValues) != 2 || previousValues[0] || !previousValues[1] { - t.Fatalf("previous log reads = %#v", previousValues) - } - - if logs.current.metadata.Status != "valid" || logs.previous.metadata.Status != "valid" { - t.Fatalf("result log metadata = %#v", logs) - } -} - -func TestCleanupFailurePreservesExecutionFailure(t *testing.T) { - t.Parallel() - - output, err := Aggregate(1, testBackendResources()[:1], []Result{{ - ParticipantIndex: 0, - ExecutionState: "Error", - Verdict: "Inconclusive", - Reason: "BackendExecutionFailed", - }}) - if err != nil { - t.Fatal(err) - } - - recordCleanupFailure(&output) - - if output.ExecutionState != "Error" || - output.Verdict != "Inconclusive" || - output.Reason != "BackendExecutionFailed" || - output.CleanupFailure != "BackendCleanupFailed" { - t.Fatalf("output = %#v", output) - } -} - -func TestCleanupFailurePreventsPass(t *testing.T) { - t.Parallel() - - output, err := Aggregate(1, testBackendResources()[:1], []Result{{ - ParticipantIndex: 0, - ExecutionState: "Succeeded", - Verdict: "Pass", - Reason: "Healthy", - }}) - if err != nil { - t.Fatal(err) - } - - recordCleanupFailure(&output) - - if output.ExecutionState != "Error" || - output.Verdict != "Inconclusive" || - output.Reason != "BackendCleanupFailed" || - output.CleanupFailure != "BackendCleanupFailed" { - t.Fatalf("output = %#v", output) - } -} - -func TestCompletedResultRejectsTerminalStateMismatch(t *testing.T) { - t.Parallel() - - for name, test := range map[string]struct { - exitCode int - backendSucceeded bool - }{ - "zero exit with failed backend": {backendSucceeded: false}, - "nonzero exit with succeeded backend": {exitCode: 2, backendSucceeded: true}, - } { - t.Run(name, func(t *testing.T) { - t.Parallel() - - result := completedResult(Result{ - ParticipantIndex: 1, - ExecutionState: "Succeeded", - Verdict: "Inconclusive", - Reason: "SyntheticReason", - ExitCode: test.exitCode, - }, test.backendSucceeded) - if result.ExecutionState != "Error" || - result.Verdict != "Inconclusive" || - !strings.Contains(result.Reason, "contradicts backend terminal state") { - t.Fatalf("completedResult() = %#v", result) - } - }) - } -} - -func encodedResultRecord(verdict, reason, measurements string, exitCode int) string { - encode := func(value string) string { - if value == "" { - return "-" - } - - return base64.StdEncoding.EncodeToString([]byte(value)) - } - - return fmt.Sprintf( - "%s%s %s %s %d\n", - resultPrefix, - encode(verdict), - encode(reason), - encode(measurements), - exitCode, - ) -} - -func TestCleanupDeletesCreatedBackends(t *testing.T) { - t.Parallel() - - scheme := runtime.NewScheme() - pod := &unstructured.Unstructured{Object: map[string]any{ - "apiVersion": "v1", - "kind": "Pod", - "metadata": map[string]any{ - "name": "run-p0", - "namespace": "project-signal", - "uid": "pod-uid", - }, - }} - job := &unstructured.Unstructured{Object: map[string]any{ - "apiVersion": "kubeflow.org/v2beta1", - "kind": "MPIJob", - "metadata": map[string]any{ - "name": "run-p1", - "namespace": "project-signal", - "uid": "job-uid", - }, - }} - - client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds( - scheme, - map[schema.GroupVersionResource]string{ - podGVR: "PodList", - mpiJobGVR: "MPIJobList", - }, - pod, - job, - ) - runner := &Runner{Dynamic: client, PollInterval: time.Millisecond} - - err := runner.cleanup(context.Background(), "project-signal", []createdResource{ - {gvr: podGVR, resource: pod, participantIndex: 0}, - {gvr: mpiJobGVR, resource: job, participantIndex: 1}, - }) - if err != nil { - t.Fatal(err) - } - - for _, item := range []struct { - gvr schema.GroupVersionResource - name string - }{ - {gvr: podGVR, name: pod.GetName()}, - {gvr: mpiJobGVR, name: job.GetName()}, - } { - _, err := client.Resource(item.gvr).Namespace("project-signal"). - Get(context.Background(), item.name, metav1.GetOptions{}) - if err == nil { - t.Fatalf("%s still exists", item.name) - } - } -} diff --git a/internal/signalworkload/policy_test.go b/internal/signalworkload/policy_test.go deleted file mode 100644 index 1ca48a0346..0000000000 --- a/internal/signalworkload/policy_test.go +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -package signalworkload - -import ( - "strings" - "testing" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - - signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" -) - -func TestPolicyRoundTripPreservesBoundedExecutionSettings(t *testing.T) { - t.Parallel() - - policy := signalv1.WorkloadPolicy{ - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("8"), - corev1.ResourceMemory: resource.MustParse("32Gi"), - corev1.ResourceEphemeralStorage: resource.MustParse("20Gi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("8"), - corev1.ResourceMemory: resource.MustParse("32Gi"), - corev1.ResourceEphemeralStorage: resource.MustParse("40Gi"), - }, - }, - SecurityContext: &corev1.SecurityContext{ - Privileged: ptr(false), - RunAsUser: ptr(int64(0)), - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{"IPC_LOCK"}, - Drop: []corev1.Capability{"NET_RAW"}, - }, - }, - WorkingDir: "/opt/diagnostic", - HostNetwork: true, - KueueTopology: &signalv1.KueueTopologyPolicy{ - RequiredLevel: corev1.LabelHostname, - }, - Results: &corev1.EmptyDirVolumeSource{SizeLimit: quantity("2Gi")}, - Volumes: []signalv1.WorkloadVolume{ - { - Name: "host-proc", - MountPath: "/host/proc", - ReadOnly: true, - HostPath: &corev1.HostPathVolumeSource{Path: "/proc"}, - }, - { - Name: "work", - MountPath: "/work", - EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: quantity("8Gi")}, - }, - { - Name: "scratch", - MountPath: "/scratch", - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "diagnostic-scratch"}, - }, - }, - } - - content, err := Encode(policy) - if err != nil { - t.Fatal(err) - } - - got, err := Decode(content) - if err != nil { - t.Fatal(err) - } - - if got.WorkingDir != policy.WorkingDir || - !got.HostNetwork || - got.KueueTopology == nil || - got.KueueTopology.RequiredLevel != corev1.LabelHostname || - got.Resources.Limits.Cpu().Cmp(*policy.Resources.Limits.Cpu()) != 0 || - len(got.Volumes) != 3 || - got.Volumes[0].HostPath.Path != "/proc" || - got.Volumes[1].EmptyDir.SizeLimit.Cmp(*quantity("8Gi")) != 0 || - got.Volumes[2].PersistentVolumeClaim.ClaimName != "diagnostic-scratch" { - t.Fatalf("decoded policy = %#v", got) - } -} - -func TestValidateRejectsUnsafeOrAmbiguousPolicy(t *testing.T) { - t.Parallel() - - tests := map[string]signalv1.WorkloadPolicy{ - "reserved mount": { - Volumes: []signalv1.WorkloadVolume{{ - Name: "replacement", - MountPath: "/results", - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }}, - }, - "multiple sources": { - Volumes: []signalv1.WorkloadVolume{{ - Name: "scratch", - MountPath: "/scratch", - HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/scratch"}, - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }}, - }, - "relative host path": { - Volumes: []signalv1.WorkloadVolume{{ - Name: "scratch", - MountPath: "/scratch", - HostPath: &corev1.HostPathVolumeSource{Path: "scratch"}, - }}, - }, - "limit below request": { - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("8")}, - Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("4")}, - }, - }, - "capability overlap": { - SecurityContext: &corev1.SecurityContext{Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{"SYS_ADMIN"}, - Drop: []corev1.Capability{"SYS_ADMIN"}, - }}, - }, - "empty topology level": { - KueueTopology: &signalv1.KueueTopologyPolicy{}, - }, - "invalid topology level": { - KueueTopology: &signalv1.KueueTopologyPolicy{RequiredLevel: "not a label key"}, - }, - } - - for name, policy := range tests { - t.Run(name, func(t *testing.T) { - t.Parallel() - - if err := Validate(policy); err == nil { - t.Fatal("Validate accepted invalid policy") - } - }) - } -} - -func TestDecodeRejectsOversizedPolicy(t *testing.T) { - t.Parallel() - - policy := signalv1.WorkloadPolicy{WorkingDir: "/" + strings.Repeat("a", MaxPolicyBytes)} - if _, err := Encode(policy); err == nil { - t.Fatal("Encode accepted oversized policy") - } -} - -func TestDecodeRejectsMalformedHostNetworkPolicy(t *testing.T) { - t.Parallel() - - for name, content := range map[string]string{ - "wrong type": `{"hostNetwork":"true"}`, - "unknown field": `{"hostNetworking":true}`, - } { - t.Run(name, func(t *testing.T) { - t.Parallel() - - if _, err := Decode(content); err == nil { - t.Fatal("Decode accepted malformed host network policy") - } - }) - } -} - -func TestDecodeRejectsMalformedKueueTopologyPolicy(t *testing.T) { - t.Parallel() - - for name, content := range map[string]string{ - "wrong type": `{"kueueTopology":"kubernetes.io/hostname"}`, - "unknown field": `{"kueueTopology":{"requiredTopology":"kubernetes.io/hostname"}}`, - "empty object": `{"kueueTopology":{}}`, - } { - t.Run(name, func(t *testing.T) { - t.Parallel() - - if _, err := Decode(content); err == nil { - t.Fatal("Decode accepted malformed Kueue topology policy") - } - }) - } -} - -func quantity(value string) *resource.Quantity { - result := resource.MustParse(value) - - return &result -} - -func ptr[T any](value T) *T { - return &value -} From db222a3c788faa67305d5febf298444d26d3f6ef Mon Sep 17 00:00:00 2001 From: chokevin Date: Fri, 23 Oct 2026 00:46:47 -0700 Subject: [PATCH 27/36] fix(signal): fence certification cleanup and validate execution evidence --- cmd/project-signal-adapter/main.go | 1 + cmd/project-signal-controller/main.go | 6 - deploy/signal/02-rbac.yaml.tmpl | 29 +- .../04-adapter-workflowtemplate.yaml.tmpl | 4 + deploy/signal/06-deployment.yaml.tmpl | 2 +- deploy/signal/README.md | 48 +- internal/signal/controller.go | 424 ++++++++++++++---- internal/signal/projection.go | 21 +- internal/signaladapter/adapter.go | 7 +- internal/signaladapter/backend_evidence.go | 37 +- internal/signaladapter/runner.go | 36 +- 11 files changed, 469 insertions(+), 146 deletions(-) diff --git a/cmd/project-signal-adapter/main.go b/cmd/project-signal-adapter/main.go index 216370f704..4c249da2d2 100644 --- a/cmd/project-signal-adapter/main.go +++ b/cmd/project-signal-adapter/main.go @@ -80,6 +80,7 @@ func main() { flags.StringVar(&cfg.TolerationsJSON, "tolerations", "[]", "JSON-encoded Pod tolerations") flags.StringVar(&cfg.ScriptEnvironmentJSON, "script-environment", "{}", "JSON-encoded script environment") flags.StringVar(&cfg.WorkloadPolicyJSON, "workload-policy", "{}", "JSON-encoded participant workload policy") + flags.IntVar(&cfg.BackendEvidenceBytes, "backend-evidence-bytes", signaladapter.MaxBackendEvidenceBytes, "Maximum backend evidence output bytes") flags.StringVar(&cfg.GPUResourceCount, "gpu-resource-count", "0", "GPU resource limit per participant") flags.BoolVar(&cfg.Privileged, "privileged", false, "Run the participant workload with a privileged security context") flags.IntVar(&cfg.Attempt, "attempt", 0, "One-based Workflow attempt number") diff --git a/cmd/project-signal-controller/main.go b/cmd/project-signal-controller/main.go index 4656b1284d..b8afa3bf80 100644 --- a/cmd/project-signal-controller/main.go +++ b/cmd/project-signal-controller/main.go @@ -21,7 +21,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - machinav1 "github.com/Azure/unbounded/api/machina/v1alpha3" signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" projectsignal "github.com/Azure/unbounded/internal/signal" "github.com/Azure/unbounded/internal/version" @@ -83,10 +82,6 @@ func run(ctx context.Context, cfg config) error { return err } - if err := machinav1.AddToScheme(scheme); err != nil { - return err - } - watchedNamespaces := map[string]cache.Config{cfg.namespace: {}} watchedNamespaces[cfg.scriptNamespace] = cache.Config{} @@ -127,7 +122,6 @@ func run(ctx context.Context, cfg config) error { runs := &projectsignal.RunReconciler{ Client: mgr.GetClient(), APIReader: mgr.GetAPIReader(), - Namespace: cfg.namespace, Evidence: evidence, MaxConcurrentRuns: cfg.maxConcurrentRuns, Recorder: mgr.GetEventRecorder(controllerName), diff --git a/deploy/signal/02-rbac.yaml.tmpl b/deploy/signal/02-rbac.yaml.tmpl index 1b2bc29d89..88136f8293 100644 --- a/deploy/signal/02-rbac.yaml.tmpl +++ b/deploy/signal/02-rbac.yaml.tmpl @@ -6,27 +6,18 @@ rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch", "patch"] - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list"] - apiGroups: ["signal.unbounded-cloud.io"] resources: ["certificationprofiles"] verbs: ["get", "list", "watch"] - - apiGroups: ["signal.unbounded-cloud.io"] - resources: ["certificationprofiles/status"] - verbs: ["get", "update", "patch"] - apiGroups: ["signal.unbounded-cloud.io"] resources: ["certificationruns"] - verbs: ["get", "list", "watch", "create", "update", "patch"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - apiGroups: ["signal.unbounded-cloud.io"] resources: ["certificationruns/status"] verbs: ["get", "update", "patch"] - apiGroups: ["signal.unbounded-cloud.io"] resources: ["certificationruns/finalizers"] verbs: ["update"] - - apiGroups: ["unbounded-cloud.io"] - resources: ["machines", "machineoperations"] - verbs: ["get", "list"] --- apiVersion: rbac.authorization.k8s.io/v1 @@ -49,6 +40,24 @@ metadata: name: project-signal-controller namespace: {{ default "project-signal" .Namespace }} rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "patch", "delete"] + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "list", "patch", "delete"] + - apiGroups: ["kubeflow.org"] + resources: ["mpijobs"] + verbs: ["get", "list", "patch", "delete"] + - apiGroups: ["batch"] + resources: ["jobs/finalizers"] + verbs: ["update"] + - apiGroups: ["kubeflow.org"] + resources: ["mpijobs/finalizers"] + verbs: ["update"] + - apiGroups: ["argoproj.io"] + resources: ["workflows/finalizers"] + verbs: ["update"] - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] diff --git a/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl b/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl index 5f3bdb8c6a..7d5dd4f35b 100644 --- a/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl +++ b/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl @@ -26,6 +26,8 @@ spec: - name: tolerations - name: script-environment - name: workload-policy + - name: backend-evidence-bytes + default: "65536" - name: mpi-payload-mode - name: expected-gpu-product-regex - name: expected-gpu-count @@ -75,6 +77,8 @@ spec: - "{{`{{inputs.parameters.script-environment}}`}}" - --workload-policy - "{{`{{inputs.parameters.workload-policy}}`}}" + - --backend-evidence-bytes + - "{{`{{inputs.parameters.backend-evidence-bytes}}`}}" - --gpu-resource-count - "{{`{{inputs.parameters.gpu-resource-count}}`}}" - "--privileged={{`{{inputs.parameters.privileged}}`}}" diff --git a/deploy/signal/06-deployment.yaml.tmpl b/deploy/signal/06-deployment.yaml.tmpl index d58ea202ba..5763f36013 100644 --- a/deploy/signal/06-deployment.yaml.tmpl +++ b/deploy/signal/06-deployment.yaml.tmpl @@ -20,7 +20,7 @@ spec: imagePullPolicy: IfNotPresent args: - --namespace={{ default "project-signal" .Namespace }} - - --script-namespace={{ default "project-signal" .Namespace }} + - --script-namespace={{ default (default "project-signal" .Namespace) .ScriptNamespace }} - --leader-elect=true ports: - name: metrics diff --git a/deploy/signal/README.md b/deploy/signal/README.md index 2965681bba..e9bcfcd222 100644 --- a/deploy/signal/README.md +++ b/deploy/signal/README.md @@ -77,13 +77,14 @@ The referenced WorkflowTemplate must accept these string parameters: | `tolerations` | JSON-encoded toleration list | | `script-environment` | JSON-encoded, validated environment map | | `workload-policy` | JSON-encoded immutable participant-container resources, security context, and bounded volume policy | -| `mpi-payload-mode` | Optional operator-defined execution mode | -| `expected-gpu-product-regex` | Optional operator policy | -| `expected-gpu-count` | Optional operator policy | -| `observed-gpu-capacity` | Capacity copied from the admitted Node | -| `observed-gpu-allocatable` | Allocatable count copied from the admitted Node | -| `gpu-resource-count` | Optional operator policy | -| `privileged` | Optional operator policy | +| `backend-evidence-bytes` | Per-check evidence output bound, at most 64 KiB | +| `mpi-payload-mode` | Compatibility input for operator-defined templates | +| `expected-gpu-product-regex` | Compatibility input for operator-defined templates | +| `expected-gpu-count` | Compatibility input for operator-defined templates | +| `observed-gpu-capacity` | Compatibility input copied from admitted Node inventory | +| `observed-gpu-allocatable` | Compatibility input copied from admitted Node inventory | +| `gpu-resource-count` | GPU count, defaults to `0` for Pod diagnostics; must be positive for MPIJobs | +| `privileged` | Explicit boolean, defaults to `false` | | `attempt` | One-based attempt number | The Workflow must publish only these output parameters: @@ -100,11 +101,12 @@ Unexpected, duplicate, malformed, or oversized outputs are rejected and the run remains fail-closed. Each `backend-resources` entry must contain `apiVersion`, `kind`, `name`, `uid`, and the zero-based `participantIndexes` that the workload executed. `namespace` is optional. A multi-Node successful -result must cover every admitted participant. The controller prefers present -top-level Workflow outputs. When Argo leaves those outputs absent, it accepts -outputs only from the unique Workflow entrypoint node whose name exactly -matches the Workflow name; missing, malformed, duplicate, or ambiguous -entrypoint identity and outputs fail closed. +result must cover every admitted participant. The controller reads outputs only +from each check's unique deterministic DAG task node. Missing, malformed, +duplicate, or ambiguous task identity and outputs fail closed. Operator script +parameters, including GPU expectations and observed inventory, are carried in +`script-environment`. Their existing individual compatibility inputs remain +available to external templates; the supplied adapter uses the environment. Before foreground deletion, the adapter captures and writes a typed v1 projection for every participant backend to the Argo output path. The @@ -120,7 +122,7 @@ secret-like assignments, URL user information, private-key blocks, and complete result-marker payloads. Object reason and message fields are also bounded. If a required object or current log cannot be captured, a restarted container's previous log is unavailable, or the projection must omit content to fit the -256 KiB output limit, `complete` is false and the run cannot pass. Capture and +configured per-check output limit (at most 64 KiB), `complete` is false and the run cannot pass. Capture and cleanup failures remain explicit, while an earlier backend execution failure continues to be the causal result reason. @@ -132,6 +134,23 @@ inferred. A termination after the write may leave the evidence file available to Argo, but collection still depends on the Workflow executor surviving long enough to publish output parameters. +The controller cross-checks typed backend evidence against the declared backend, +namespace, exact workload UID, participant coverage, and required Pod/container +and current/previous log records. A template's `complete: true` alone is not +sufficient. Log collection requests the latest 64 lines, retains at most 64 KiB +for result parsing, and projects redacted tails of at most 512 bytes into evidence. +Earlier log content is intentionally outside this bounded result contract. + +After durable terminal evidence is recorded, the controller foreground-deletes +the Workflow and waits for owned backend descendants to be removed before +releasing Node reservations. This also covers adapter crashes, cancellation, +and a controller crash between Workflow creation and status persistence. +The six-hour Workflow TTL is a fallback, not the normal evidence-retention +mechanism; immutable terminal evidence remains available after Workflow cleanup. +The controller's namespace-scoped backend patch/delete and owner-finalizer +permissions allow Kubernetes' owner-reference admission checks to authorize +blocking deletion fences. It does not receive backend workload creation rights. + ## Multi-Node execution Participant admission is deterministic and atomic. The requesting Node is @@ -425,6 +444,9 @@ Controller-owned labels: Pending, testing, failed, and inconclusive states use `eligibility=blocked`. A successful run projects `passed/production`. Older runs cannot overwrite a newer terminal projection. +Projection and request clearing also require the current reservation to name +the same Run, so historical Runs cannot overwrite a newer pending/testing run +or consume its request. Completed finalizer cleanup is not replayed. ## Operations diff --git a/internal/signal/controller.go b/internal/signal/controller.go index ecda2070f2..0f9442f8d6 100644 --- a/internal/signal/controller.go +++ b/internal/signal/controller.go @@ -16,6 +16,7 @@ import ( "regexp" "slices" "sort" + "strconv" "strings" "sync" "time" @@ -39,6 +40,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/recorder" signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" + "github.com/Azure/unbounded/internal/signaladapter" "github.com/Azure/unbounded/internal/signalworkload" ) @@ -75,7 +77,7 @@ var reservedScriptEnvironmentNames = map[string]struct{}{ var workflowOutputLimits = map[string]int{ "backend-resources": 16 * 1024, - "backend-evidence": 256 * 1024, + "backend-evidence": signaladapter.MaxBackendEvidenceBytes, "cleanup-failure": 64, "measurements": 16 * 1024, "reason": 4 * 1024, @@ -105,13 +107,11 @@ type AdmissionReconciler struct { type RunReconciler struct { client.Client - APIReader client.Reader - Namespace string - Evidence EvidenceStore - Now func() time.Time - MaxConcurrentRuns int - WorkflowAPIVersion string - Recorder recorder.EventRecorder + APIReader client.Reader + Evidence EvidenceStore + Now func() time.Time + MaxConcurrentRuns int + Recorder recorder.EventRecorder } func (r *AdmissionReconciler) SetupWithManager(mgr ctrl.Manager) error { @@ -155,27 +155,9 @@ func (r *AdmissionReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } func (r *AdmissionReconciler) admitNode(ctx context.Context, node *corev1.Node, profileName string) error { - return r.admitNodeWithTrigger(ctx, node, profileName, "manual") -} - -func (r *AdmissionReconciler) admitNodeWithTrigger( - ctx context.Context, - node *corev1.Node, - profileName string, - trigger string, -) error { r.admissionMu.Lock() defer r.admissionMu.Unlock() - return r.admitNodeWithTriggerLocked(ctx, node, profileName, trigger) -} - -func (r *AdmissionReconciler) admitNodeWithTriggerLocked( - ctx context.Context, - node *corev1.Node, - profileName string, - trigger string, -) error { if runName := node.Annotations[ReservationLabel]; runName != "" { var run signalv1.CertificationRun @@ -271,6 +253,20 @@ func (r *AdmissionReconciler) admitNodeWithTriggerLocked( return err } + for index := range checks { + if _, _, err := participantsForCheck(participants, checks[index].ParticipantPolicy); err != nil { + return fmt.Errorf("check %s participants: %w", checks[index].ID, err) + } + + if _, err := signalworkload.Encode(checks[index].WorkloadPolicy); err != nil { + return fmt.Errorf("check %s workload policy: %w", checks[index].ID, err) + } + + if _, _, err := adapterParameterValues(checks[index].Parameters, checks[index].Backend.Kind); err != nil { + return fmt.Errorf("check %s parameters: %w", checks[index].ID, err) + } + } + runName := runName(node.Name, profile.Name, r.now()) if err := r.reserveParticipants(ctx, participants, runName); err != nil { return err @@ -326,7 +322,7 @@ func (r *AdmissionReconciler) admitNodeWithTriggerLocked( RunProfileLabel: profile.Name, }, Annotations: map[string]string{ - RunTriggerAnnotation: trigger, + RunTriggerAnnotation: "manual", }, Finalizers: []string{RunFinalizer}, }, @@ -349,6 +345,12 @@ func (r *AdmissionReconciler) admitNodeWithTriggerLocked( RequestNodeName: node.Name, }, } + if _, err := buildWorkflow(run, 1); err != nil { + deleteErr := r.Delete(context.WithoutCancel(ctx), snapshot) + + return errors.Join(fmt.Errorf("validate executable plan: %w", err), client.IgnoreNotFound(deleteErr)) + } + if err := r.Create(ctx, run); err != nil { deleteErr := r.Delete(context.WithoutCancel(ctx), snapshot) @@ -356,16 +358,11 @@ func (r *AdmissionReconciler) admitNodeWithTriggerLocked( } cleanupFailedAdmission := func(cause error) error { + releaseOnFailure = false deleteRunErr := r.Delete(context.WithoutCancel(ctx), run) - if deleteRunErr != nil && !apierrors.IsNotFound(deleteRunErr) { - // Keep the reservation if the run could not be made inert. This - // prevents another admission from overlapping the surviving run. - releaseOnFailure = false - } - - deleteSnapshotErr := r.Delete(context.WithoutCancel(ctx), snapshot) - return errors.Join(cause, client.IgnoreNotFound(deleteRunErr), client.IgnoreNotFound(deleteSnapshotErr)) + // The Run finalizer owns rollback once the Run exists, including its snapshot. + return errors.Join(cause, client.IgnoreNotFound(deleteRunErr)) } snapshot.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(run, signalv1.GroupVersion.WithKind("CertificationRun"))} @@ -373,7 +370,7 @@ func (r *AdmissionReconciler) admitNodeWithTriggerLocked( return cleanupFailedAdmission(fmt.Errorf("bind script snapshot to run: %w", err)) } - if err := projectPending(ctx, r.Client, participants, run.Spec.Profile); err != nil { + if err := projectPending(ctx, r.Client, run); err != nil { return cleanupFailedAdmission(fmt.Errorf("project pending certification state: %w", err)) } @@ -941,7 +938,7 @@ func (r *AdmissionReconciler) releaseParticipants(ctx context.Context, participa before := node.DeepCopy() delete(node.Annotations, ReservationLabel) - if err := r.Patch(ctx, &node, client.MergeFrom(before)); err != nil { + if err := r.Patch(ctx, &node, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil { errs = append(errs, err) } } @@ -1041,6 +1038,10 @@ func (r *RunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R } if run.Status.Phase.Terminal() { + if !containsString(run.Finalizers, RunFinalizer) { + return ctrl.Result{}, nil + } + return r.cleanup(ctx, &run) } @@ -1133,7 +1134,15 @@ func (r *RunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R func (r *RunReconciler) startAttempt(ctx context.Context, run *signalv1.CertificationRun, attempt int32) (ctrl.Result, error) { workflow, err := buildWorkflow(run, attempt) if err != nil { - return ctrl.Result{}, err + before := run.DeepCopy() + now := metav1.NewTime(r.now()) + run.Status.Phase = signalv1.RunPhaseInconclusive + run.Status.ExecutionState = "Error" + run.Status.Verdict = "Inconclusive" + run.Status.Reason = "InvalidExecutionPlan: " + err.Error() + run.Status.FinishedAt = &now + + return ctrl.Result{RequeueAfter: time.Millisecond}, r.Status().Patch(ctx, run, client.MergeFrom(before)) } if err := r.Create(ctx, workflow); err != nil { @@ -1235,6 +1244,11 @@ func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured return nil, fmt.Errorf("encode check %s workload policy: %w", check.ID, encodeErr) } + gpuCount, privileged, err := adapterParameterValues(check.Parameters, check.Backend.Kind) + if err != nil { + return nil, fmt.Errorf("check %s parameters: %w", check.ID, err) + } + taskName := checkTaskName(check) task := map[string]any{ "name": taskName, @@ -1252,6 +1266,8 @@ func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured string(tolerationsJSON), string(environmentJSON), workloadPolicyJSON, + gpuCount, + privileged, ), }, } @@ -1325,6 +1341,8 @@ func checkWorkflowParameters( tolerationsJSON string, scriptEnvironmentJSON string, workloadPolicyJSON string, + gpuCount string, + privileged string, ) []any { return []any{ map[string]any{"name": "run-name", "value": checkRunName(run.Name, check.ID)}, @@ -1355,12 +1373,37 @@ func checkWorkflowParameters( map[string]any{"name": "expected-gpu-count", "value": check.Parameters["expected-gpu-count"]}, map[string]any{"name": "observed-gpu-capacity", "value": check.Parameters["observed-gpu-capacity"]}, map[string]any{"name": "observed-gpu-allocatable", "value": check.Parameters["observed-gpu-allocatable"]}, - map[string]any{"name": "gpu-resource-count", "value": check.Parameters["gpu-resource-count"]}, - map[string]any{"name": "privileged", "value": check.Parameters["privileged"]}, + map[string]any{"name": "backend-evidence-bytes", "value": strconv.Itoa(int(check.ResultBounds.BackendEvidenceBytes))}, + map[string]any{"name": "gpu-resource-count", "value": gpuCount}, + map[string]any{"name": "privileged", "value": privileged}, map[string]any{"name": "attempt", "value": fmt.Sprintf("%d", attempt)}, } } +func adapterParameterValues(parameters map[string]string, backendKind string) (string, string, error) { + gpuCount := parameters["gpu-resource-count"] + if gpuCount == "" { + gpuCount = "0" + } + + count, err := strconv.ParseInt(gpuCount, 10, 64) + if err != nil || count < 0 || (backendKind == "MPIJob" && count == 0) { + return "", "", errors.New("gpu-resource-count must be nonnegative and positive for MPIJob") + } + + privileged := parameters["privileged"] + if privileged == "" { + privileged = "false" + } + + value, err := strconv.ParseBool(privileged) + if err != nil { + return "", "", fmt.Errorf("privileged must be a boolean: %w", err) + } + + return strconv.FormatInt(count, 10), strconv.FormatBool(value), nil +} + func validEnvironmentName(value string) bool { if value == "" || (value[0] != '_' && (value[0] < 'A' || value[0] > 'Z')) { return false @@ -1498,11 +1541,16 @@ func (r *RunReconciler) resumeGroupedEvidence( return ctrl.Result{}, false, fmt.Errorf("decode persisted grouped evidence: %w", err) } - if document.RunName != run.Name || - (document.RunID != "" && run.UID != "" && document.RunID != string(run.UID)) { + if document.RunName != run.Name || document.RunID == "" || document.RunID != string(run.UID) { return ctrl.Result{}, false, fmt.Errorf("persisted grouped evidence identity does not match run %s", run.Name) } + if document.Workflow.Name != "" && + (document.Workflow.Name != fmt.Sprintf("%s-a1", run.Name) || document.Workflow.UID == "" || + (run.Status.Workflow.UID != "" && document.Workflow.UID != run.Status.Workflow.UID)) { + return ctrl.Result{}, false, errors.New("persisted evidence Workflow identity does not match run") + } + if len(document.CheckStatus) != len(run.Spec.Checks) { return ctrl.Result{}, false, fmt.Errorf( "persisted grouped evidence contains %d check statuses, want %d", @@ -1782,6 +1830,9 @@ func terminalCheckStatuses( normalizedEvidence, complete, evidenceErr := groupedBackendEvidenceForParticipants( outputs["backend-evidence"], globalIndexes, + outputs["backend-resources"], + check.Backend, + run.Namespace, ) if evidenceErr != nil { status.Phase = signalv1.CheckPhaseInconclusive @@ -2177,42 +2228,20 @@ func decodeWorkflowParametersWithLimits(parameters []any, limits map[string]int) return outputs, nil } -func groupedBackendEvidenceComplete(content string, participantCount int) (bool, error) { - if content == "" { - return false, errors.New("backend-evidence is required") - } - - var evidence struct { - Version string `json:"version"` - Complete bool `json:"complete"` - CaptureError string `json:"captureError,omitempty"` - Participants []struct { - ParticipantIndex int `json:"participantIndex"` - Resource struct { - APIVersion string `json:"apiVersion"` - Kind string `json:"kind"` - Namespace string `json:"namespace,omitempty"` - Name string `json:"name"` - UID types.UID `json:"uid"` - } `json:"resource"` - LauncherJob any `json:"launcherJob,omitempty"` - Pods []any `json:"pods"` - CaptureError string `json:"captureError,omitempty"` - } `json:"participants"` - } - - decoder := json.NewDecoder(strings.NewReader(content)) - if err := decoder.Decode(&evidence); err != nil { - return false, fmt.Errorf("decode backend-evidence: %w", err) - } - - var trailing any - if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { - return false, errors.New("backend-evidence contains trailing JSON values") +func groupedBackendEvidenceComplete( + content string, + participantCount int, + references []signalv1.BackendResourceReference, + backend signalv1.ExecutionBackend, + namespace string, +) (bool, error) { + if err := signaladapter.ValidateBackendEvidence(content); err != nil { + return false, err } - if evidence.Version != "v1" { - return false, fmt.Errorf("backend-evidence version %q is invalid", evidence.Version) + var evidence signaladapter.BackendEvidence + if err := json.Unmarshal([]byte(content), &evidence); err != nil { + return false, err } if len(evidence.Participants) != participantCount { @@ -2224,6 +2253,8 @@ func groupedBackendEvidenceComplete(content string, participantCount int) (bool, } seen := make([]bool, participantCount) + complete := evidence.Complete && evidence.CaptureError == "" + podUIDs := map[types.UID]struct{}{} for index := range evidence.Participants { participant := &evidence.Participants[index] @@ -2252,16 +2283,85 @@ func groupedBackendEvidenceComplete(content string, participantCount int) (bool, participant.ParticipantIndex, ) } + + matches := 0 + + for _, reference := range references { + if !slices.Contains(reference.ParticipantIndexes, int32(participant.ParticipantIndex)) { + continue + } + + if reference.APIVersion != participant.Resource.APIVersion || + reference.Kind != participant.Resource.Kind || + reference.Namespace != participant.Resource.Namespace || + reference.Name != participant.Resource.Name || + reference.UID != participant.Resource.UID { + return false, fmt.Errorf("backend-evidence participant %d resource differs from backend-resources", participant.ParticipantIndex) + } + + matches++ + } + + if matches != 1 || participant.Resource.APIVersion != backend.APIVersion || + participant.Resource.Kind != backend.Kind || participant.Resource.Namespace != namespace { + return false, fmt.Errorf("backend-evidence participant %d does not match the declared backend", participant.ParticipantIndex) + } + + complete = complete && participant.CaptureError == "" && len(participant.Pods) > 0 + launcher, worker := false, false + + for _, pod := range participant.Pods { + if pod.UID == "" || pod.Name == "" { + return false, errors.New("backend-evidence Pod identity is incomplete") + } + + if _, duplicate := podUIDs[pod.UID]; duplicate { + return false, errors.New("backend-evidence repeats a Pod UID") + } + + podUIDs[pod.UID] = struct{}{} + if backend.Kind == "Pod" && (pod.UID != participant.Resource.UID || + pod.Name != participant.Resource.Name || pod.Role != "participant" || len(participant.Pods) != 1) { + return false, errors.New("backend-evidence Pod differs from its backend resource") + } + + launcher = launcher || pod.Role == "launcher" + worker = worker || pod.Role == "worker" + + complete = complete && pod.Phase != "" && len(pod.Containers) > 0 + for _, container := range pod.Containers { + complete = complete && container.Name != "" && container.State.State != "" && + container.CurrentLog.Status == "captured" && container.CurrentLog.Error == "" && + (container.PreviousLog.Status == "captured" || container.PreviousLog.Status == "not-applicable") && + container.PreviousLog.Error == "" + if container.RestartCount > 0 || container.LastState.State == "terminated" { + complete = complete && container.PreviousLog.Status == "captured" + } + } + } + + if backend.Kind == "MPIJob" { + complete = complete && launcher && worker && participant.LauncherJob != nil && + participant.LauncherJob.Name != "" && participant.LauncherJob.UID != "" + } } - return evidence.Complete, nil + return complete, nil } func groupedBackendEvidenceForParticipants( content string, globalIndexes []int32, + resources string, + backend signalv1.ExecutionBackend, + namespace string, ) (string, bool, error) { - complete, err := groupedBackendEvidenceComplete(content, len(globalIndexes)) + references, err := backendResourceReferences(resources, len(globalIndexes)) + if err != nil { + return "", false, err + } + + complete, err := groupedBackendEvidenceComplete(content, len(globalIndexes), references, backend, namespace) if err != nil { return "", false, err } @@ -2353,10 +2453,23 @@ func (r *RunReconciler) cancel(ctx context.Context, run *signalv1.CertificationR } func (r *RunReconciler) cleanup(ctx context.Context, run *signalv1.CertificationRun) (ctrl.Result, error) { + stopped, err := r.ensureWorkflowStopped(ctx, run) + if err != nil { + return ctrl.Result{}, err + } + + if !stopped { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + if err := projectTerminal(ctx, r.Client, run); err != nil { return ctrl.Result{}, err } + if err := r.clearRequest(ctx, run); err != nil { + return ctrl.Result{}, err + } + released, err := r.releaseRunReservations(ctx, run) if err != nil { return ctrl.Result{}, err @@ -2378,10 +2491,6 @@ func (r *RunReconciler) cleanup(ctx context.Context, run *signalv1.Certification deleted = true - if err := r.clearRequest(ctx, run); err != nil { - return ctrl.Result{}, err - } - statusBefore := run.DeepCopy() run.Status.Cleanup = signalv1.CleanupStatus{ReservationsReleased: released, SnapshotDeleted: deleted} @@ -2418,14 +2527,37 @@ func (r *RunReconciler) reconcileDeleting(ctx context.Context, run *signalv1.Cer return ctrl.Result{RequeueAfter: time.Second}, nil } - if _, err := r.releaseRunReservations(ctx, run); err != nil { + if err := r.clearRequest(ctx, run); err != nil { return ctrl.Result{}, err } - if err := r.clearRequest(ctx, run); err != nil { + if _, err := r.releaseRunReservations(ctx, run); err != nil { return ctrl.Result{}, err } + if len(run.Spec.Checks) > 0 { + snapshot := &corev1.ConfigMap{} + + reader := r.APIReader + if reader == nil { + reader = r.Client + } + + err := reader.Get(ctx, client.ObjectKey{Namespace: run.Namespace, Name: run.Spec.Checks[0].Script.ConfigMapName}, snapshot) + if err == nil { + if snapshot.Labels[SnapshotRunLabel] != run.Name { + return ctrl.Result{}, errors.New("refusing to delete script snapshot with different run identity") + } + + uid := snapshot.UID + if err := r.Delete(ctx, snapshot, client.Preconditions{UID: &uid}); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + } else if !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + } + before := run.DeepCopy() run.Finalizers = removeString(run.Finalizers, RunFinalizer) @@ -2454,7 +2586,7 @@ func (r *RunReconciler) releaseRunReservations(ctx context.Context, run *signalv before := node.DeepCopy() delete(node.Annotations, ReservationLabel) - if err := r.Patch(ctx, &node, client.MergeFrom(before)); err != nil { + if err := r.Patch(ctx, &node, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil { return false, err } } @@ -2472,25 +2604,26 @@ func (r *RunReconciler) clearRequest(ctx context.Context, run *signalv1.Certific return nil } - if node.Labels[RequestLabel] != run.Spec.Profile.Name { + if node.Labels[RequestLabel] != run.Spec.Profile.Name || node.Annotations[ReservationLabel] != run.Name { return nil } before := node.DeepCopy() delete(node.Labels, RequestLabel) - return r.Patch(ctx, &node, client.MergeFrom(before)) + return r.Patch(ctx, &node, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})) } func (r *RunReconciler) ensureWorkflowStopped(ctx context.Context, run *signalv1.CertificationRun) (bool, error) { - if run.Status.Workflow.Name == "" { - return true, nil + name := run.Status.Workflow.Name + if name == "" { + name = fmt.Sprintf("%s-a1", run.Name) } workflow := &unstructured.Unstructured{} workflow.SetGroupVersionKind(workflowGVK) - key := client.ObjectKey{Namespace: run.Namespace, Name: run.Status.Workflow.Name} + key := client.ObjectKey{Namespace: run.Namespace, Name: name} reader := r.APIReader if reader == nil { @@ -2499,18 +2632,21 @@ func (r *RunReconciler) ensureWorkflowStopped(ctx context.Context, run *signalv1 if err := reader.Get(ctx, key, workflow); err != nil { if apierrors.IsNotFound(err) { - return true, nil + return r.runWorkloadsGone(ctx, reader, run) } return false, fmt.Errorf("get Workflow before deletion: %w", err) } - if run.Status.Workflow.UID == "" || - string(workflow.GetUID()) != run.Status.Workflow.UID || + if (run.Status.Workflow.UID != "" && string(workflow.GetUID()) != run.Status.Workflow.UID) || !metav1.IsControlledBy(workflow, run) { return false, fmt.Errorf("refusing to delete Workflow with changed identity or ownership") } + if err := r.fenceWorkflowDependents(ctx, reader, workflow, run); err != nil { + return false, err + } + propagation := metav1.DeletePropagationForeground uid := workflow.GetUID() @@ -2525,7 +2661,7 @@ func (r *RunReconciler) ensureWorkflowStopped(ctx context.Context, run *signalv1 if err := reader.Get(ctx, key, workflow); err != nil { if apierrors.IsNotFound(err) { - return true, nil + return r.runWorkloadsGone(ctx, reader, run) } return false, fmt.Errorf("confirm Workflow deletion: %w", err) @@ -2534,6 +2670,106 @@ func (r *RunReconciler) ensureWorkflowStopped(ctx context.Context, run *signalv1 return false, nil } +func (r *RunReconciler) runWorkloadsGone(ctx context.Context, reader client.Reader, run *signalv1.CertificationRun) (bool, error) { + for _, check := range run.Spec.Checks { + for _, kind := range []schema.GroupVersionKind{ + corev1.SchemeGroupVersion.WithKind("Pod"), + {Group: "batch", Version: "v1", Kind: "Job"}, + {Group: "kubeflow.org", Version: "v2beta1", Kind: "MPIJob"}, + } { + if kind.Kind != "Pod" && check.Backend.Kind != "MPIJob" { + continue + } + + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(kind.GroupVersion().WithKind(kind.Kind + "List")) + + if err := reader.List(ctx, list, client.InNamespace(run.Namespace), + client.MatchingLabels{SnapshotRunLabel: checkRunName(run.Name, check.ID)}); err != nil { + return false, fmt.Errorf("confirm %s cleanup: %w", kind.Kind, err) + } + + if len(list.Items) > 0 { + return false, nil + } + } + } + + return true, nil +} + +func (r *RunReconciler) fenceWorkflowDependents( + ctx context.Context, + reader client.Reader, + workflow *unstructured.Unstructured, + run *signalv1.CertificationRun, +) error { + owners := map[types.UID]struct{}{workflow.GetUID(): {}} + kinds := []schema.GroupVersionKind{} + + for _, check := range run.Spec.Checks { + if check.Backend.Kind == "MPIJob" { + kinds = append(kinds, schema.GroupVersionKind{Group: "kubeflow.org", Version: "v2beta1", Kind: "MPIJob"}) + + break + } + } + + kinds = append( + kinds, + schema.GroupVersionKind{Group: "batch", Version: "v1", Kind: "Job"}, + corev1.SchemeGroupVersion.WithKind("Pod"), + ) + + // Fence every level before deleting the Workflow so foreground GC also waits + // for existing backend descendants whose original owner reference did not block. + for _, kind := range kinds { + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(kind.GroupVersion().WithKind(kind.Kind + "List")) + + if err := reader.List(ctx, list, client.InNamespace(run.Namespace)); err != nil { + return fmt.Errorf("list %s before Workflow cleanup: %w", kind.Kind, err) + } + + for index := range list.Items { + object := &list.Items[index] + references := object.GetOwnerReferences() + owned := false + changed := false + + for ownerIndex := range references { + if _, found := owners[references[ownerIndex].UID]; !found { + continue + } + + owned = true + + if references[ownerIndex].BlockOwnerDeletion == nil || !*references[ownerIndex].BlockOwnerDeletion { + block := true + references[ownerIndex].BlockOwnerDeletion = &block + changed = true + } + } + + if !owned { + continue + } + + owners[object.GetUID()] = struct{}{} + if changed { + before := object.DeepCopy() + object.SetOwnerReferences(references) + + if err := r.Patch(ctx, object, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil { + return fmt.Errorf("fence %s %s before Workflow cleanup: %w", kind.Kind, object.GetName(), err) + } + } + } + } + + return nil +} + func participantNodeUID(run *signalv1.CertificationRun, nodeName string) types.UID { for _, participant := range run.Spec.Participants { if participant.NodeName == nodeName { diff --git a/internal/signal/projection.go b/internal/signal/projection.go index 37ef80316c..9e32376908 100644 --- a/internal/signal/projection.go +++ b/internal/signal/projection.go @@ -39,11 +39,14 @@ const ( func projectPending( ctx context.Context, kube client.Client, - participants []signalv1.Participant, - profile signalv1.PinnedProfile, + run *signalv1.CertificationRun, ) error { - return projectParticipants(ctx, kube, participants, func(node *corev1.Node) error { - setProfileProjection(node, profile) + return projectParticipants(ctx, kube, run.Spec.Participants, func(node *corev1.Node) error { + if node.Annotations[ReservationLabel] != run.Name { + return nil + } + + setProfileProjection(node, run.Spec.Profile) node.Labels[StateLabel] = "pending" node.Labels[EligibilityLabel] = "blocked" @@ -53,6 +56,10 @@ func projectPending( func projectTesting(ctx context.Context, kube client.Client, run *signalv1.CertificationRun) error { return projectParticipants(ctx, kube, run.Spec.Participants, func(node *corev1.Node) error { + if node.Annotations[ReservationLabel] != run.Name { + return nil + } + setProfileProjection(node, run.Spec.Profile) node.Labels[StateLabel] = "testing" node.Labels[EligibilityLabel] = "blocked" @@ -69,6 +76,10 @@ func projectTerminal(ctx context.Context, kube client.Client, run *signalv1.Cert finishedAt := run.Status.FinishedAt.UTC().Format(time.RFC3339) return projectParticipants(ctx, kube, run.Spec.Participants, func(node *corev1.Node) error { + if node.Annotations[ReservationLabel] != run.Name { + return nil + } + if projectionTimeAfter(node.Annotations[LastRunAtAnnotation], run.Status.FinishedAt.Time) { return nil } @@ -140,7 +151,7 @@ func projectParticipants( return err } - return kube.Patch(ctx, &node, client.MergeFrom(before)) + return kube.Patch(ctx, &node, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})) }); err != nil { return fmt.Errorf("project Node %s status: %w", participant.NodeName, err) } diff --git a/internal/signaladapter/adapter.go b/internal/signaladapter/adapter.go index a83f4dd466..42b22c2a91 100644 --- a/internal/signaladapter/adapter.go +++ b/internal/signaladapter/adapter.go @@ -31,7 +31,7 @@ const ( MaxReasonBytes = 4 * 1024 MaxMeasurementsBytes = 16 * 1024 MaxBackendOutputBytes = 16 * 1024 - MaxBackendEvidenceBytes = 256 * 1024 + MaxBackendEvidenceBytes = 64 * 1024 MaxCleanupFailureBytes = 64 maxMPIWorkerReadyWait = int64(30) maxMPIWorkerSlots = int64(1<<31 - 1) @@ -85,6 +85,7 @@ type Config struct { Attempt int BackoffLimit int64 TTLSeconds int64 + BackendEvidenceBytes int } type Result struct { @@ -292,6 +293,10 @@ func ValidateConfig(config Config) ([]signalv1.Participant, []corev1.Toleration, return nil, nil, nil, errors.New("attempt must be positive") } + if config.BackendEvidenceBytes < 0 || config.BackendEvidenceBytes > MaxBackendEvidenceBytes { + return nil, nil, nil, fmt.Errorf("backend-evidence-bytes must be between 1 and %d when set", MaxBackendEvidenceBytes) + } + policy, err := signalworkload.Decode(config.WorkloadPolicyJSON) if err != nil { return nil, nil, nil, err diff --git a/internal/signaladapter/backend_evidence.go b/internal/signaladapter/backend_evidence.go index adb3b7a0a5..3068289e18 100644 --- a/internal/signaladapter/backend_evidence.go +++ b/internal/signaladapter/backend_evidence.go @@ -143,7 +143,7 @@ func (r *Runner) captureBackendEvidence( evidence.Complete = evidence.Complete && complete } - content, complete, err := marshalBackendEvidence(evidence) + content, complete, err := marshalBackendEvidence(evidence, config.BackendEvidenceBytes) if err != nil { return "", false, err } @@ -151,13 +151,21 @@ func (r *Runner) captureBackendEvidence( return string(content), complete, nil } -func marshalBackendEvidence(evidence BackendEvidence) ([]byte, bool, error) { +func marshalBackendEvidence(evidence BackendEvidence, limit int) ([]byte, bool, error) { + if limit == 0 { + limit = MaxBackendEvidenceBytes + } + + if limit < 1 || limit > MaxBackendEvidenceBytes { + return nil, false, fmt.Errorf("backend evidence limit must be between 1 and %d bytes", MaxBackendEvidenceBytes) + } + content, err := json.Marshal(evidence) if err != nil { return nil, false, fmt.Errorf("encode backend evidence: %w", err) } - if len(content) <= MaxBackendEvidenceBytes { + if len(content) <= limit { return content, evidence.Complete, nil } @@ -181,7 +189,7 @@ func marshalBackendEvidence(evidence BackendEvidence) ([]byte, bool, error) { return nil, false, fmt.Errorf("encode bounded backend evidence: %w", err) } - if len(content) <= MaxBackendEvidenceBytes { + if len(content) <= limit { return content, false, nil } @@ -204,8 +212,8 @@ func marshalBackendEvidence(evidence BackendEvidence) ([]byte, bool, error) { return nil, false, fmt.Errorf("encode reduced backend evidence: %w", err) } - if len(content) > MaxBackendEvidenceBytes { - return nil, false, fmt.Errorf("reduced backend evidence exceeds %d bytes", MaxBackendEvidenceBytes) + if len(content) > limit { + return nil, false, fmt.Errorf("reduced backend evidence exceeds %d bytes", limit) } return content, false, nil @@ -222,7 +230,7 @@ func omitLogContent(logEvidence *CapturedLogEvidence) { logEvidence.Error = "content omitted because backend evidence exceeded the size limit" } -func markBackendEvidenceIncomplete(content, captureError string) (string, error) { +func markBackendEvidenceIncomplete(content, captureError string, limit int) (string, error) { var evidence BackendEvidence if err := json.Unmarshal([]byte(content), &evidence); err != nil { @@ -232,7 +240,7 @@ func markBackendEvidenceIncomplete(content, captureError string) (string, error) evidence.Complete = false evidence.CaptureError = boundedRedactedText(captureError, maxEvidenceTextBytes) - bounded, _, err := marshalBackendEvidence(evidence) + bounded, _, err := marshalBackendEvidence(evidence, limit) if err != nil { return "", err } @@ -262,6 +270,9 @@ func (r *Runner) captureParticipantEvidence( if err != nil { participant.CaptureError = boundedRedactedText("get backend object: "+err.Error(), maxEvidenceTextBytes) complete = false + } else if object.GetUID() != backend.resource.GetUID() { + participant.CaptureError = "backend object UID changed before evidence capture" + complete = false } else { projected, projectErr := projectBackendObject(object) if projectErr != nil { @@ -285,6 +296,12 @@ func (r *Runner) captureParticipantEvidence( return participant, false } + if pod.UID != backend.resource.GetUID() { + participant.CaptureError = joinCaptureError(participant.CaptureError, "participant Pod UID changed before evidence capture") + + return participant, false + } + podEvidence, podComplete := r.capturePodEvidence(ctx, config.Namespace, pod, "participant") participant.Pods = []PodEvidence{podEvidence} complete = complete && podComplete @@ -602,7 +619,9 @@ func capturedLog(content string, err error) CapturedLogEvidence { func redactLog(content string) string { lines := strings.Split(content, "\n") - inPrivateKey := false + begin := strings.Index(content, "BEGIN ") + end := strings.Index(content, "END ") + inPrivateKey := end >= 0 && (begin < 0 || end < begin) && strings.Contains(content[end:], "PRIVATE KEY") for index, line := range lines { if strings.Contains(line, "BEGIN ") && strings.Contains(line, "PRIVATE KEY") { diff --git a/internal/signaladapter/runner.go b/internal/signaladapter/runner.go index a2eff9962e..842928dab8 100644 --- a/internal/signaladapter/runner.go +++ b/internal/signaladapter/runner.go @@ -202,7 +202,7 @@ func (r *Runner) Execute(ctx context.Context, config Config) (output AggregateOu recordEvidenceCaptureFailure(&output) } - if err := r.persistEvidenceBeforeCleanup(&output); err != nil { + if err := r.persistEvidenceBeforeCleanup(&output, config.BackendEvidenceBytes); err != nil { return AggregateOutput{}, err } @@ -220,7 +220,7 @@ func (r *Runner) Execute(ctx context.Context, config Config) (output AggregateOu return output, nil } -func (r *Runner) persistEvidenceBeforeCleanup(output *AggregateOutput) error { +func (r *Runner) persistEvidenceBeforeCleanup(output *AggregateOutput, limit int) error { if r.PersistBackendEvidence == nil { return nil } @@ -231,6 +231,7 @@ func (r *Runner) persistEvidenceBeforeCleanup(output *AggregateOutput) error { output.BackendEvidence, markErr = markBackendEvidenceIncomplete( output.BackendEvidence, "persist backend evidence before cleanup: "+err.Error(), + limit, ) if markErr != nil { return markErr @@ -570,14 +571,18 @@ func (r *Runner) readPodLog( return r.ReadPodLog(ctx, namespace, podName, containerName, previous) } + tailLines := int64(64) + stream, err := r.Kubernetes.CoreV1().Pods(namespace). - GetLogs(podName, &corev1.PodLogOptions{Container: containerName, Previous: previous}). + GetLogs(podName, &corev1.PodLogOptions{Container: containerName, Previous: previous, TailLines: &tailLines}). Stream(ctx) if err != nil { return "", fmt.Errorf("open result log: %w", err) } - content, err := io.ReadAll(io.LimitReader(stream, maxResultLogBytes+1)) + tail := &resultLogTail{} + + _, err = io.Copy(tail, stream) if err != nil { closeErr := stream.Close() if closeErr != nil { @@ -594,11 +599,28 @@ func (r *Runner) readPodLog( return "", fmt.Errorf("close result log: %w", err) } - if len(content) > maxResultLogBytes { - return "", fmt.Errorf("result log exceeds %d bytes", maxResultLogBytes) + return string(tail.content), nil +} + +type resultLogTail struct { + content []byte +} + +func (tail *resultLogTail) Write(content []byte) (int, error) { + count := len(content) + if len(content) >= maxResultLogBytes { + tail.content = append(tail.content[:0], content[len(content)-maxResultLogBytes:]...) + + return count, nil + } + + if excess := len(tail.content) + len(content) - maxResultLogBytes; excess > 0 { + tail.content = tail.content[excess:] } - return string(content), nil + tail.content = append(tail.content, content...) + + return count, nil } func resultLogSourceWithContent(content string, err error) resultLogSource { From ed49b0b787c1badfeb539e2e569bacfc106cf30d Mon Sep 17 00:00:00 2001 From: chokevin Date: Fri, 23 Oct 2026 10:21:18 -0700 Subject: [PATCH 28/36] fix(signal): default backend references to the run namespace --- deploy/signal/README.md | 6 ++++-- internal/signal/controller.go | 15 +++++++++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/deploy/signal/README.md b/deploy/signal/README.md index e9bcfcd222..e2d6051abb 100644 --- a/deploy/signal/README.md +++ b/deploy/signal/README.md @@ -100,8 +100,10 @@ The Workflow must publish only these output parameters: Unexpected, duplicate, malformed, or oversized outputs are rejected and the run remains fail-closed. Each `backend-resources` entry must contain `apiVersion`, `kind`, `name`, `uid`, and the zero-based `participantIndexes` -that the workload executed. `namespace` is optional. A multi-Node successful -result must cover every admitted participant. The controller reads outputs only +that the workload executed. `namespace` is optional and defaults to the +CertificationRun namespace; an explicitly different namespace is rejected. +A multi-Node successful result must cover every admitted participant. +The controller reads outputs only from each check's unique deterministic DAG task node. Missing, malformed, duplicate, or ambiguous task identity and outputs fail closed. Operator script parameters, including GPU expectations and observed inventory, are carried in diff --git a/internal/signal/controller.go b/internal/signal/controller.go index 0f9442f8d6..e1aefbe052 100644 --- a/internal/signal/controller.go +++ b/internal/signal/controller.go @@ -1855,6 +1855,7 @@ func terminalCheckStatuses( resources, resourceErr := backendResourceReferencesForParticipants( outputs["backend-resources"], globalIndexes, + run.Namespace, ) if resourceErr != nil { status.Phase = signalv1.CheckPhaseInconclusive @@ -2089,6 +2090,7 @@ func (r *RunReconciler) evidenceDocument( func backendResourceReferences( value string, participantCount int, + namespace string, ) ([]signalv1.BackendResourceReference, error) { if participantCount < 1 || participantCount > 64 { return nil, fmt.Errorf("participant count must be between 1 and 64") @@ -2124,6 +2126,14 @@ func backendResourceReferences( return nil, fmt.Errorf("backend-resources[%d] has an incomplete identity", index) } + if reference.Namespace == "" { + reference.Namespace = namespace + } + + if reference.Namespace != namespace { + return nil, fmt.Errorf("backend-resources[%d] namespace does not match the Run namespace", index) + } + identity := strings.Join([]string{ reference.APIVersion, reference.Kind, @@ -2178,8 +2188,9 @@ func backendResourceReferences( func backendResourceReferencesForParticipants( value string, globalIndexes []int32, + namespace string, ) ([]signalv1.BackendResourceReference, error) { - references, err := backendResourceReferences(value, len(globalIndexes)) + references, err := backendResourceReferences(value, len(globalIndexes), namespace) if err != nil { return nil, err } @@ -2356,7 +2367,7 @@ func groupedBackendEvidenceForParticipants( backend signalv1.ExecutionBackend, namespace string, ) (string, bool, error) { - references, err := backendResourceReferences(resources, len(globalIndexes)) + references, err := backendResourceReferences(resources, len(globalIndexes), namespace) if err != nil { return "", false, err } From 7c166757dfb1cc552a58157d55cb18ff45b2397c Mon Sep 17 00:00:00 2001 From: chokevin Date: Fri, 23 Oct 2026 13:46:20 -0700 Subject: [PATCH 29/36] fix(signal): preserve standalone certification semantics --- deploy/signal/README.md | 25 ++++++++++++------- internal/signal/controller.go | 36 +++++++++++++++++++++++++--- internal/signaladapter/adapter.go | 40 +++++++++++++++++++++++-------- 3 files changed, 80 insertions(+), 21 deletions(-) diff --git a/deploy/signal/README.md b/deploy/signal/README.md index e2d6051abb..fa5be43c74 100644 --- a/deploy/signal/README.md +++ b/deploy/signal/README.md @@ -165,10 +165,12 @@ The participant group is a certification boundary, not one distributed process group. The included adapter creates one independently pinned backend workload per participant. Ordinary diagnostics use one Pod per participant. MPI diagnostics use one MPIJob per participant with one launcher and one GPU worker; -only the worker is pinned to the participant Node and receives the configured -GPU limit. The adapter propagates the Workflow active deadline, uses the -profile's queue, records API-assigned workload UIDs, reads the final typed result -from bounded workload logs, aggregates diagnostic state separately from backend +ordinary Pods and MPI workers use required Node affinity on the immutable +`metadata.name` field to bind to the exact admitted Node without bypassing +scheduler admission. Only the worker receives the configured GPU limit. The +adapter propagates the Workflow active deadline, applies the profile's optional +queue, records API-assigned workload UIDs, reads the final typed result from +bounded workload logs, aggregates diagnostic state separately from backend execution state, and foreground-deletes every created workload. The selected script is invoked with no arguments for a Pod backend, @@ -269,7 +271,7 @@ checks: maxAttempts: 1 ``` -or an explicit Kueue queue: +or an MPIJob backend, optionally submitted through a Kueue queue: ```yaml checks: @@ -281,9 +283,10 @@ checks: queue: ``` -When a Pod backend supplies a queue, the adapter writes the standard -`kueue.x-k8s.io/queue-name` label. MPIJob backends require a queue; Pod backends -may omit it only when the operator intentionally runs them outside Kueue. +When either backend supplies a queue, the adapter writes the standard +`kueue.x-k8s.io/queue-name` label. Pod and MPIJob backends may both omit it for +standalone execution without Kueue. A `kueueTopology` policy still requires an +explicit queue. Checks may set `spec.checks[].workloadPolicy` when a diagnostic needs explicit participant-container policy. The controller validates and snapshots this @@ -400,6 +403,12 @@ task invokes the configured adapter `certification` template through or `Continue`. A stopped task is recorded as skipped/not-run, never inferred as a diagnostic result. +Before creating the Workflow, the controller durably records its deterministic +name and attempt in CertificationRun status. Once that creation intent exists, +a missing Workflow terminates the Run as Inconclusive and is never recreated. +This fail-closed fence prevents a controller outage plus Workflow TTL cleanup +from executing the one-time certification suite twice. + Participant roles and topology constraints are generic typed policy. Profiles can, for example, bind one role to one participant, another role to three participants, require the latter to span at least two values of an operator diff --git a/internal/signal/controller.go b/internal/signal/controller.go index e1aefbe052..32ee4fbce1 100644 --- a/internal/signal/controller.go +++ b/internal/signal/controller.go @@ -1082,9 +1082,18 @@ func (r *RunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R return ctrl.Result{}, err } - if run.Status.Workflow.UID == "" || - string(workflow.GetUID()) != run.Status.Workflow.UID || - !metav1.IsControlledBy(workflow, &run) { + if !metav1.IsControlledBy(workflow, &run) { + return ctrl.Result{}, fmt.Errorf( + "workflow %s identity or ownership changed; refusing concurrent retry", + workflow.GetName(), + ) + } + + if run.Status.Workflow.UID == "" { + return r.recordWorkflowStarted(ctx, &run, workflow, run.Status.Workflow.Attempt) + } + + if string(workflow.GetUID()) != run.Status.Workflow.UID { return ctrl.Result{}, fmt.Errorf( "workflow %s identity or ownership changed; refusing concurrent retry", workflow.GetName(), @@ -1145,6 +1154,18 @@ func (r *RunReconciler) startAttempt(ctx context.Context, run *signalv1.Certific return ctrl.Result{RequeueAfter: time.Millisecond}, r.Status().Patch(ctx, run, client.MergeFrom(before)) } + before := run.DeepCopy() + run.Status.Workflow = signalv1.WorkflowStatus{ + Name: workflow.GetName(), + Attempt: attempt, + Phase: "Creating", + } + run.Status.Checks = initialCheckStatuses(run.Spec.Checks) + + if err := r.Status().Patch(ctx, run, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, fmt.Errorf("record Workflow creation intent: %w", err) + } + if err := r.Create(ctx, workflow); err != nil { if !apierrors.IsAlreadyExists(err) { return ctrl.Result{}, fmt.Errorf("create Workflow attempt %d: %w", attempt, err) @@ -1164,6 +1185,15 @@ func (r *RunReconciler) startAttempt(ctx context.Context, run *signalv1.Certific workflow = existing } + return r.recordWorkflowStarted(ctx, run, workflow, attempt) +} + +func (r *RunReconciler) recordWorkflowStarted( + ctx context.Context, + run *signalv1.CertificationRun, + workflow *unstructured.Unstructured, + attempt int32, +) (ctrl.Result, error) { before := run.DeepCopy() run.Status.Phase = signalv1.RunPhaseRunning run.Status.ExecutionState = "Running" diff --git a/internal/signaladapter/adapter.go b/internal/signaladapter/adapter.go index 42b22c2a91..6c1ce075c7 100644 --- a/internal/signaladapter/adapter.go +++ b/internal/signaladapter/adapter.go @@ -258,10 +258,6 @@ func ValidateConfig(config Config) ([]signalv1.Participant, []corev1.Toleration, switch { case config.BackendKind == "Pod" && config.BackendAPIVersion == "v1": case config.BackendKind == "MPIJob" && config.BackendAPIVersion == "kubeflow.org/v2beta1": - if config.Queue == "" { - return nil, nil, nil, errors.New("queue is required for MPIJob execution") - } - if config.BackendDeadlineSeconds < 2 { return nil, nil, nil, errors.New( "backend-active-deadline-seconds must be at least 2 for MPIJob worker readiness", @@ -912,7 +908,7 @@ func ordinaryPod( }, Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyNever, - NodeSelector: map[string]string{corev1.LabelHostname: participant.NodeName}, + Affinity: participantNodeAffinity(participant.NodeName), Tolerations: tolerations, ActiveDeadlineSeconds: &backendDeadlineSeconds, AutomountServiceAccountToken: ptr(false), @@ -936,6 +932,10 @@ func mpiJob( gpuCount int64, ) (*unstructured.Unstructured, error) { labels := resourceLabels(config, participant) + if config.Queue != "" { + labels = withLabel(labels, "kueue.x-k8s.io/queue-name", config.Queue) + } + jobName := resourceNameWithLimit(config.RunName, participant.Index, 63-len("-worker-0")) launcherEnv := workloadEnvironment(config, participant, environment, false) workerEnv := workloadEnvironment(config, participant, environment, true) @@ -964,7 +964,7 @@ func mpiJob( "metadata": map[string]any{ "name": jobName, "namespace": config.Namespace, - "labels": stringMapAny(withLabel(labels, "kueue.x-k8s.io/queue-name", config.Queue)), + "labels": stringMapAny(labels), "ownerReferences": []any{ownerReferenceMap(workflow)}, }, "spec": map[string]any{ @@ -984,7 +984,7 @@ func mpiJob( "restartPolicy": string(corev1.RestartPolicyNever), "template": podTemplate( labels, - nil, + "", nil, corev1.Container{ Name: "launcher", @@ -1005,7 +1005,7 @@ func mpiJob( "restartPolicy": string(corev1.RestartPolicyNever), "template": podTemplate( labels, - map[string]string{corev1.LabelHostname: participant.NodeName}, + participant.NodeName, tolerations, worker, config.ScriptConfigMap, @@ -1021,7 +1021,7 @@ func mpiJob( func podTemplate( labels map[string]string, - nodeSelector map[string]string, + nodeName string, tolerations []corev1.Toleration, container corev1.Container, scriptConfigMap string, @@ -1039,7 +1039,7 @@ func podTemplate( }, Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyNever, - NodeSelector: nodeSelector, + Affinity: participantNodeAffinity(nodeName), Tolerations: tolerations, AutomountServiceAccountToken: ptr(false), Containers: []corev1.Container{container}, @@ -1056,6 +1056,26 @@ func podTemplate( return content } +func participantNodeAffinity(nodeName string) *corev1.Affinity { + if nodeName == "" { + return nil + } + + return &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{ + MatchFields: []corev1.NodeSelectorRequirement{{ + Key: "metadata.name", + Operator: corev1.NodeSelectorOpIn, + Values: []string{nodeName}, + }}, + }}, + }, + }, + } +} + func workloadAnnotations(policy signalv1.WorkloadPolicy) map[string]string { if policy.KueueTopology == nil { return nil From 3ada75f5a3e870d9b45dea399b3a3c2b53bd53c7 Mon Sep 17 00:00:00 2001 From: chokevin Date: Fri, 23 Oct 2026 14:01:38 -0700 Subject: [PATCH 30/36] fix(signal): guard one-time certification invariants --- api/signal/v1alpha1/certificationrun_types.go | 13 +- api/signal/v1alpha1/zz_generated.deepcopy.go | 16 +- deploy/signal/02-rbac.yaml.tmpl | 6 +- deploy/signal/README.md | 26 ++-- ....unbounded-cloud.io_certificationruns.yaml | 36 ----- internal/signal/controller.go | 138 ++++++++++++++++-- internal/signaladapter/adapter.go | 14 +- 7 files changed, 161 insertions(+), 88 deletions(-) diff --git a/api/signal/v1alpha1/certificationrun_types.go b/api/signal/v1alpha1/certificationrun_types.go index 3193856bf4..732155e212 100644 --- a/api/signal/v1alpha1/certificationrun_types.go +++ b/api/signal/v1alpha1/certificationrun_types.go @@ -12,6 +12,8 @@ import ( type RunPhase string const ( + ReservationNodeUIDLabel = "signal.unbounded-cloud.io/reserved-node-uid" + RunPhasePending RunPhase = "Pending" RunPhaseRunning RunPhase = "Running" RunPhaseSucceeded RunPhase = "Succeeded" @@ -129,13 +131,10 @@ type CertificationRunSpec struct { } type WorkflowStatus struct { - Name string `json:"name,omitempty"` - UID string `json:"uid,omitempty"` - Attempt int32 `json:"attempt,omitempty"` - Phase string `json:"phase,omitempty"` - Outputs map[string]string `json:"outputs,omitempty"` - // +kubebuilder:validation:MaxItems=64 - BackendResources []BackendResourceReference `json:"backendResources,omitempty"` + Name string `json:"name,omitempty"` + UID string `json:"uid,omitempty"` + Attempt int32 `json:"attempt,omitempty"` + Phase string `json:"phase,omitempty"` } type EvidenceStatus struct { diff --git a/api/signal/v1alpha1/zz_generated.deepcopy.go b/api/signal/v1alpha1/zz_generated.deepcopy.go index dc2279b510..0a84665b30 100644 --- a/api/signal/v1alpha1/zz_generated.deepcopy.go +++ b/api/signal/v1alpha1/zz_generated.deepcopy.go @@ -331,7 +331,7 @@ func (in *CertificationRunSpec) DeepCopy() *CertificationRunSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CertificationRunStatus) DeepCopyInto(out *CertificationRunStatus) { *out = *in - in.Workflow.DeepCopyInto(&out.Workflow) + out.Workflow = in.Workflow out.Evidence = in.Evidence out.Cleanup = in.Cleanup if in.Checks != nil { @@ -697,20 +697,6 @@ func (in *WorkflowReference) DeepCopy() *WorkflowReference { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WorkflowStatus) DeepCopyInto(out *WorkflowStatus) { *out = *in - if in.Outputs != nil { - in, out := &in.Outputs, &out.Outputs - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.BackendResources != nil { - in, out := &in.BackendResources, &out.BackendResources - *out = make([]BackendResourceReference, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkflowStatus. diff --git a/deploy/signal/02-rbac.yaml.tmpl b/deploy/signal/02-rbac.yaml.tmpl index 88136f8293..e834a22f7c 100644 --- a/deploy/signal/02-rbac.yaml.tmpl +++ b/deploy/signal/02-rbac.yaml.tmpl @@ -42,13 +42,13 @@ metadata: rules: - apiGroups: [""] resources: ["pods"] - verbs: ["get", "list", "patch", "delete"] + verbs: ["get", "list", "patch"] - apiGroups: ["batch"] resources: ["jobs"] - verbs: ["get", "list", "patch", "delete"] + verbs: ["get", "list", "patch"] - apiGroups: ["kubeflow.org"] resources: ["mpijobs"] - verbs: ["get", "list", "patch", "delete"] + verbs: ["get", "list", "patch"] - apiGroups: ["batch"] resources: ["jobs/finalizers"] verbs: ["update"] diff --git a/deploy/signal/README.md b/deploy/signal/README.md index fa5be43c74..699b05733f 100644 --- a/deploy/signal/README.md +++ b/deploy/signal/README.md @@ -149,9 +149,10 @@ releasing Node reservations. This also covers adapter crashes, cancellation, and a controller crash between Workflow creation and status persistence. The six-hour Workflow TTL is a fallback, not the normal evidence-retention mechanism; immutable terminal evidence remains available after Workflow cleanup. -The controller's namespace-scoped backend patch/delete and owner-finalizer -permissions allow Kubernetes' owner-reference admission checks to authorize -blocking deletion fences. It does not receive backend workload creation rights. +The controller's namespace-scoped backend patch and owner-finalizer permissions +allow Kubernetes' owner-reference admission checks to authorize blocking +deletion fences. It does not receive backend workload creation or deletion +rights. ## Multi-Node execution @@ -166,12 +167,14 @@ group. The included adapter creates one independently pinned backend workload per participant. Ordinary diagnostics use one Pod per participant. MPI diagnostics use one MPIJob per participant with one launcher and one GPU worker; ordinary Pods and MPI workers use required Node affinity on the immutable -`metadata.name` field to bind to the exact admitted Node without bypassing -scheduler admission. Only the worker receives the configured GPU limit. The -adapter propagates the Workflow active deadline, applies the profile's optional -queue, records API-assigned workload UIDs, reads the final typed result from -bounded workload logs, aggregates diagnostic state separately from backend -execution state, and foreground-deletes every created workload. +`metadata.name` field and a controller-owned reservation label containing the +admitted Node UID. This binds execution to the exact admitted Node without +bypassing scheduler admission or accepting a replacement Node that reuses its +name. Only the worker receives the configured GPU limit. The adapter propagates +the Workflow active deadline, applies the profile's optional queue, records +API-assigned workload UIDs, reads the final typed result from bounded workload +logs, aggregates diagnostic state separately from backend execution state, and +foreground-deletes every created workload. The selected script is invoked with no arguments for a Pod backend, `mpi-launcher ` in an MPIJob launcher, and `mpi-worker` in its worker. @@ -362,6 +365,11 @@ rendered to its participant Pod or MPIJob `activeDeadlineSeconds`. It must be positive and leave at least 60 seconds before the outer deadline; the controller rejects an invalid ordering before reserving Nodes. +The sum of all 15 checks' declared verdict, reason, measurements, backend +resource, backend evidence, and cleanup output bounds cannot exceed 768 KiB. +The controller rejects larger plans before reserving Nodes so valid per-check +bounds cannot collectively exceed a safe Argo Workflow status budget. + `kueueTopology.requiredLevel` is an explicit Kueue Topology Aware Scheduling request. It must be a valid Kubernetes label key and requires a nonempty backend queue. The adapter writes diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml index b097c308d6..ddaa70d26d 100644 --- a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml @@ -1062,44 +1062,8 @@ spec: attempt: format: int32 type: integer - backendResources: - items: - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - namespace: - type: string - participantIndexes: - items: - format: int32 - type: integer - maxItems: 64 - minItems: 1 - type: array - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - type: string - required: - - apiVersion - - kind - - name - - uid - type: object - maxItems: 64 - type: array name: type: string - outputs: - additionalProperties: - type: string - type: object phase: type: string uid: diff --git a/internal/signal/controller.go b/internal/signal/controller.go index 32ee4fbce1..867c8508e7 100644 --- a/internal/signal/controller.go +++ b/internal/signal/controller.go @@ -85,8 +85,9 @@ var workflowOutputLimits = map[string]int{ } const ( - groupedCheckCount = 15 - adapterServiceAccountName = "project-signal-adapter" + groupedCheckCount = 15 + adapterServiceAccountName = "project-signal-adapter" + maximumWorkflowOutputBytes = 768 * 1024 ) var workflowGVK = schema.GroupVersionKind{ @@ -439,7 +440,8 @@ func (r *AdmissionReconciler) selectParticipants( if candidate.Name == requestNode.Name || !selector.Matches(labels.Set(candidate.Labels)) || !nodeEligible(candidate, profile.Spec.Participants.RequiredResources) || - candidate.Annotations[ReservationLabel] != "" { + candidate.Annotations[ReservationLabel] != "" || + candidate.Labels[signalv1.ReservationNodeUIDLabel] != "" { continue } @@ -891,12 +893,25 @@ func (r *AdmissionReconciler) reserveParticipants(ctx context.Context, participa ) } + if value := node.Labels[signalv1.ReservationNodeUIDLabel]; value != "" { + return errors.Join( + fmt.Errorf("participant %s has stale reservation identity %s", participant.NodeName, value), + r.releaseParticipants(context.WithoutCancel(ctx), reserved, runName), + ) + } + before := node.DeepCopy() if node.Annotations == nil { node.Annotations = map[string]string{} } + if node.Labels == nil { + node.Labels = map[string]string{} + } + node.Annotations[ReservationLabel] = runName + node.Labels[signalv1.ReservationNodeUIDLabel] = string(participant.NodeUID) + if err := r.Patch( ctx, &node, @@ -935,8 +950,20 @@ func (r *AdmissionReconciler) releaseParticipants(ctx context.Context, participa continue } + if value := node.Labels[signalv1.ReservationNodeUIDLabel]; value != "" && value != string(participant.NodeUID) { + errs = append(errs, fmt.Errorf( + "participant %s reservation identity changed from %s to %s", + participant.NodeName, + participant.NodeUID, + value, + )) + + continue + } + before := node.DeepCopy() delete(node.Annotations, ReservationLabel) + delete(node.Labels, signalv1.ReservationNodeUIDLabel) if err := r.Patch(ctx, &node, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil { errs = append(errs, err) @@ -1510,6 +1537,7 @@ func (r *RunReconciler) finishGrouped( evidenceRun.Status.ExecutionState = executionState evidenceRun.Status.Verdict = verdict evidenceRun.Status.Reason = reason + evidenceRun.Status.Phase = runPhaseForVerdict(verdict) evidenceRun.Status.Workflow.Phase = workflowPhase finishedAt := groupedFinishedAt(workflow, run) @@ -1565,6 +1593,7 @@ func (r *RunReconciler) resumeGroupedEvidence( ExecutionState string `json:"executionState"` Verdict string `json:"verdict"` Reason string `json:"reason"` + Phase signalv1.RunPhase `json:"phase"` FinishedAt string `json:"finishedAt"` } if err := json.Unmarshal(data, &document); err != nil { @@ -1611,7 +1640,16 @@ func (r *RunReconciler) resumeGroupedEvidence( run.Status.Reason = document.Reason run.Status.Workflow = document.Workflow run.Status.Evidence = signalv1.EvidenceStatus{URI: uri, Digest: digest} - run.Status.Phase = runPhaseForVerdict(document.Verdict) + + if document.Phase == "" { + run.Status.Phase = runPhaseForVerdict(document.Verdict) + } else { + if !document.Phase.Terminal() { + return ctrl.Result{}, false, fmt.Errorf("persisted evidence contains nonterminal phase %q", document.Phase) + } + + run.Status.Phase = document.Phase + } value := metav1.NewTime(finishedAt) run.Status.FinishedAt = &value @@ -2110,6 +2148,7 @@ func (r *RunReconciler) evidenceDocument( "executionState": executionState, "verdict": verdict, "reason": reason, + "phase": run.Status.Phase, "startedAt": run.Status.StartedAt, "finishedAt": run.Status.FinishedAt.UTC().Format(time.RFC3339Nano), } @@ -2456,14 +2495,14 @@ func (r *RunReconciler) cancel(ctx context.Context, run *signalv1.CertificationR return ctrl.Result{RequeueAfter: time.Second}, nil } - before := run.DeepCopy() - run.Status.Phase = signalv1.RunPhaseCancelled - run.Status.ExecutionState = "Interrupted" - run.Status.Verdict = "Inconclusive" - - run.Status.Reason = "canceled by operator" + evidenceRun := run.DeepCopy() + evidenceRun.Status.Phase = signalv1.RunPhaseCancelled + evidenceRun.Status.ExecutionState = "Interrupted" + evidenceRun.Status.Verdict = "Inconclusive" + evidenceRun.Status.Reason = "canceled by operator" + evidenceRun.Status.Workflow.Phase = "Canceled" - statuses := run.Status.Checks + statuses := evidenceRun.Status.Checks if len(statuses) != len(run.Spec.Checks) { statuses = initialCheckStatuses(run.Spec.Checks) } @@ -2481,11 +2520,31 @@ func (r *RunReconciler) cancel(ctx context.Context, run *signalv1.CertificationR statuses[index].NotRun = true } - run.Status.Checks = statuses + evidenceRun.Status.Checks = statuses now := metav1.NewTime(r.now()) + evidenceRun.Status.FinishedAt = &now + + evidence, err := r.evidenceDocument( + ctx, + evidenceRun, + evidenceRun.Status.ExecutionState, + evidenceRun.Status.Verdict, + evidenceRun.Status.Reason, + ) + if err != nil { + return ctrl.Result{}, err + } + + uri, digest, err := r.Evidence.Put(ctx, run.Spec.EvidencePath, evidence) + if err != nil { + return ctrl.Result{}, err + } + + before := run.DeepCopy() + run.Status = evidenceRun.Status + run.Status.Evidence = signalv1.EvidenceStatus{URI: uri, Digest: digest} - run.Status.FinishedAt = &now if err := r.Status().Patch(ctx, run, client.MergeFrom(before)); err != nil { return ctrl.Result{}, err } @@ -2624,8 +2683,18 @@ func (r *RunReconciler) releaseRunReservations(ctx context.Context, run *signalv continue } + if value := node.Labels[signalv1.ReservationNodeUIDLabel]; value != "" && value != string(participant.NodeUID) { + return false, fmt.Errorf( + "participant %s reservation identity changed from %s to %s", + participant.NodeName, + participant.NodeUID, + value, + ) + } + before := node.DeepCopy() delete(node.Annotations, ReservationLabel) + delete(node.Labels, signalv1.ReservationNodeUIDLabel) if err := r.Patch(ctx, &node, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil { return false, err @@ -2922,7 +2991,11 @@ func validateCheckPlan(checks []signalv1.CertificationCheck) error { return fmt.Errorf("must contain exactly %d checks", groupedCheckCount) } - seen := make(map[signalv1.CheckID]struct{}, len(checks)) + var ( + seen = make(map[signalv1.CheckID]struct{}, len(checks)) + outputBytes int64 + ) + for index := range checks { check := &checks[index] if check.ID == "" { @@ -2951,11 +3024,21 @@ func validateCheckPlan(checks []signalv1.CertificationCheck) error { return err } + outputBytes += workflowOutputBytes(check.ResultBounds) + if err := validateCheckParticipantPolicy(check.ID, check.ParticipantPolicy); err != nil { return err } } + if outputBytes > maximumWorkflowOutputBytes { + return fmt.Errorf( + "aggregate check output budget is %d bytes, maximum is %d", + outputBytes, + maximumWorkflowOutputBytes, + ) + } + return nil } @@ -2964,7 +3047,11 @@ func validateRunCheckPlan(checks []signalv1.CertificationCheckSnapshot) error { return fmt.Errorf("run must contain exactly %d checks", groupedCheckCount) } - seen := make(map[signalv1.CheckID]struct{}, len(checks)) + var ( + seen = make(map[signalv1.CheckID]struct{}, len(checks)) + outputBytes int64 + ) + for index := range checks { check := &checks[index] if check.ID == "" || check.Order != int32(index) { @@ -2988,11 +3075,32 @@ func validateRunCheckPlan(checks []signalv1.CertificationCheckSnapshot) error { if err := validateResultBounds(check.ID, check.ResultBounds); err != nil { return err } + + outputBytes += workflowOutputBytes(check.ResultBounds) + } + + if outputBytes > maximumWorkflowOutputBytes { + return fmt.Errorf( + "run aggregate check output budget is %d bytes, maximum is %d", + outputBytes, + maximumWorkflowOutputBytes, + ) } return nil } +func workflowOutputBytes(bounds signalv1.ResultBounds) int64 { + return int64( + workflowOutputLimits["verdict"]+ + workflowOutputLimits["cleanup-failure"], + ) + + int64(bounds.ReasonBytes) + + int64(bounds.MeasurementsBytes) + + int64(bounds.BackendResourcesBytes) + + int64(bounds.BackendEvidenceBytes) +} + func validateResultBounds(id signalv1.CheckID, bounds signalv1.ResultBounds) error { switch { case bounds.ReasonBytes < 1 || bounds.ReasonBytes > int32(workflowOutputLimits["reason"]): diff --git a/internal/signaladapter/adapter.go b/internal/signaladapter/adapter.go index 6c1ce075c7..0c3e6ec502 100644 --- a/internal/signaladapter/adapter.go +++ b/internal/signaladapter/adapter.go @@ -908,7 +908,7 @@ func ordinaryPod( }, Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyNever, - Affinity: participantNodeAffinity(participant.NodeName), + Affinity: participantNodeAffinity(participant.NodeName, participant.NodeUID), Tolerations: tolerations, ActiveDeadlineSeconds: &backendDeadlineSeconds, AutomountServiceAccountToken: ptr(false), @@ -985,6 +985,7 @@ func mpiJob( "template": podTemplate( labels, "", + "", nil, corev1.Container{ Name: "launcher", @@ -1006,6 +1007,7 @@ func mpiJob( "template": podTemplate( labels, participant.NodeName, + participant.NodeUID, tolerations, worker, config.ScriptConfigMap, @@ -1022,6 +1024,7 @@ func mpiJob( func podTemplate( labels map[string]string, nodeName string, + nodeUID types.UID, tolerations []corev1.Toleration, container corev1.Container, scriptConfigMap string, @@ -1039,7 +1042,7 @@ func podTemplate( }, Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyNever, - Affinity: participantNodeAffinity(nodeName), + Affinity: participantNodeAffinity(nodeName, nodeUID), Tolerations: tolerations, AutomountServiceAccountToken: ptr(false), Containers: []corev1.Container{container}, @@ -1056,7 +1059,7 @@ func podTemplate( return content } -func participantNodeAffinity(nodeName string) *corev1.Affinity { +func participantNodeAffinity(nodeName string, nodeUID types.UID) *corev1.Affinity { if nodeName == "" { return nil } @@ -1070,6 +1073,11 @@ func participantNodeAffinity(nodeName string) *corev1.Affinity { Operator: corev1.NodeSelectorOpIn, Values: []string{nodeName}, }}, + MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: signalv1.ReservationNodeUIDLabel, + Operator: corev1.NodeSelectorOpIn, + Values: []string{string(nodeUID)}, + }}, }}, }, }, From f20d45fdcf0873ddb52b2318c285ff95a940079b Mon Sep 17 00:00:00 2001 From: chokevin Date: Fri, 23 Oct 2026 16:19:59 -0700 Subject: [PATCH 31/36] fix(signal): preserve certification audit state --- .../v1alpha1/certificationprofile_types.go | 8 +- api/signal/v1alpha1/certificationrun_types.go | 2 +- deploy/signal/README.md | 22 ++- ...ounded-cloud.io_certificationprofiles.yaml | 4 - ....unbounded-cloud.io_certificationruns.yaml | 4 - internal/signal/controller.go | 150 ++++++++++++++---- internal/signal/projection.go | 23 ++- internal/signaladapter/adapter.go | 65 ++++++++ internal/signaladapter/backend_evidence.go | 29 ++++ internal/signaladapter/runner.go | 112 +++++++++++-- 10 files changed, 353 insertions(+), 66 deletions(-) diff --git a/api/signal/v1alpha1/certificationprofile_types.go b/api/signal/v1alpha1/certificationprofile_types.go index 58a3515e61..b5bb178450 100644 --- a/api/signal/v1alpha1/certificationprofile_types.go +++ b/api/signal/v1alpha1/certificationprofile_types.go @@ -24,11 +24,7 @@ type WorkflowReference struct { type CertificationPurpose string const ( - PurposeFullCertification CertificationPurpose = "FullCertification" - PurposeQuickCertification CertificationPurpose = "QuickCertification" - PurposePreflight CertificationPurpose = "Preflight" - PurposeTargetedVerification CertificationPurpose = "TargetedVerification" - PurposeDiagnostic CertificationPurpose = "Diagnostic" + PurposeFullCertification CertificationPurpose = "FullCertification" ) type CheckID string @@ -180,7 +176,7 @@ type CertificationProfileSpec struct { // +kubebuilder:validation:MaxItems=15 Checks []CertificationCheck `json:"checks"` ExecutionPolicy SuiteExecutionPolicy `json:"executionPolicy"` - // +kubebuilder:validation:Enum=FullCertification;QuickCertification;Preflight;TargetedVerification;Diagnostic + // +kubebuilder:validation:Enum=FullCertification Purpose CertificationPurpose `json:"purpose,omitempty"` EvidencePath string `json:"evidencePath"` Deadline metav1.Duration `json:"deadline"` diff --git a/api/signal/v1alpha1/certificationrun_types.go b/api/signal/v1alpha1/certificationrun_types.go index 732155e212..cd39e9c7c9 100644 --- a/api/signal/v1alpha1/certificationrun_types.go +++ b/api/signal/v1alpha1/certificationrun_types.go @@ -121,7 +121,7 @@ type CertificationRunSpec struct { Checks []CertificationCheckSnapshot `json:"checks"` ExecutionPolicy SuiteExecutionPolicy `json:"executionPolicy"` Workflow WorkflowReference `json:"workflow"` - // +kubebuilder:validation:Enum=FullCertification;QuickCertification;Preflight;TargetedVerification;Diagnostic + // +kubebuilder:validation:Enum=FullCertification Purpose CertificationPurpose `json:"purpose,omitempty"` EvidencePath string `json:"evidencePath"` Deadline metav1.Duration `json:"deadline"` diff --git a/deploy/signal/README.md b/deploy/signal/README.md index 699b05733f..2819a6452f 100644 --- a/deploy/signal/README.md +++ b/deploy/signal/README.md @@ -143,6 +143,13 @@ sufficient. Log collection requests the latest 64 lines, retains at most 64 KiB for result parsing, and projects redacted tails of at most 512 bytes into evidence. Earlier log content is intentionally outside this bounded result contract. +After the adapter creates its first backend, every later setup or aggregation +failure is reported as an Inconclusive output with bounded partial backend +identity and typed evidence. The adapter persists that evidence before +foreground cleanup; missing participant coverage can never produce a passing +result. If the Argo output write fails, cleanup is blocked and the backends +remain available rather than destroying the only remaining audit state. + After durable terminal evidence is recorded, the controller foreground-deletes the Workflow and waits for owned backend descendants to be removed before releasing Node reservations. This also covers adapter crashes, cancellation, @@ -174,7 +181,8 @@ name. Only the worker receives the configured GPU limit. The adapter propagates the Workflow active deadline, applies the profile's optional queue, records API-assigned workload UIDs, reads the final typed result from bounded workload logs, aggregates diagnostic state separately from backend execution state, and -foreground-deletes every created workload. +foreground-deletes every created workload only after its evidence output is +durable. The selected script is invoked with no arguments for a Pod backend, `mpi-launcher ` in an MPIJob launcher, and `mpi-worker` in its worker. @@ -369,6 +377,10 @@ The sum of all 15 checks' declared verdict, reason, measurements, backend resource, backend evidence, and cleanup output bounds cannot exceed 768 KiB. The controller rejects larger plans before reserving Nodes so valid per-check bounds cannot collectively exceed a safe Argo Workflow status budget. +Each reason, measurements, backend-resource, and backend-evidence bound must +also fit the mandatory partial-failure envelope for the check's selected +participant count. Every accepted plan can therefore report a bounded partial +setup failure. `kueueTopology.requiredLevel` is an explicit Kueue Topology Aware Scheduling request. It must be a valid Kubernetes label key and requires a nonempty backend @@ -403,6 +415,9 @@ the complete plan and selected participants once into one immutable `CertificationRun`. Each check binds its script identity and digest, runner, backend, workload and topology policy, runtime inputs, deadline, result bounds, participant-role inclusion or exclusion, and `maxAttempts: 1`. +The only supported purpose is `FullCertification`; diagnostic, preflight, and +targeted modes are intentionally outside this one-time Node join contract and +cannot project production eligibility. The controller creates one Argo Workflow with an inline DAG entrypoint. Every task invokes the configured adapter `certification` template through @@ -417,6 +432,11 @@ a missing Workflow terminates the Run as Inconclusive and is never recreated. This fail-closed fence prevents a controller outage plus Workflow TTL cleanup from executing the one-time certification suite twice. +Admission also remains fail-closed until every selected Node has received the +pending projection. The Run is retained as the durable owner while a failed +Node patch is retried; its Workflow cannot start until admission is marked +ready. + Participant roles and topology constraints are generic typed policy. Profiles can, for example, bind one role to one participant, another role to three participants, require the latter to span at least two values of an operator diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml index 6f5f5ecea7..67672f4950 100644 --- a/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml @@ -734,10 +734,6 @@ spec: purpose: enum: - FullCertification - - QuickCertification - - Preflight - - TargetedVerification - - Diagnostic type: string tolerations: items: diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml index ddaa70d26d..4287392cab 100644 --- a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml @@ -741,10 +741,6 @@ spec: purpose: enum: - FullCertification - - QuickCertification - - Preflight - - TargetedVerification - - Diagnostic type: string requestNodeName: type: string diff --git a/internal/signal/controller.go b/internal/signal/controller.go index 867c8508e7..b0d2fc5487 100644 --- a/internal/signal/controller.go +++ b/internal/signal/controller.go @@ -45,13 +45,16 @@ import ( ) const ( - RequestLabel = "signal.unbounded-cloud.io/certification-request" - ReservationLabel = "signal.unbounded-cloud.io/reserved-by" - RunNodeLabel = "signal.unbounded-cloud.io/request-node" - RunProfileLabel = "signal.unbounded-cloud.io/profile" - RunTriggerAnnotation = "signal.unbounded-cloud.io/trigger" - RunFinalizer = "signal.unbounded-cloud.io/cleanup" - SnapshotRunLabel = "signal.unbounded-cloud.io/run" + RequestLabel = "signal.unbounded-cloud.io/certification-request" + ReservationLabel = "signal.unbounded-cloud.io/reserved-by" + RunNodeLabel = "signal.unbounded-cloud.io/request-node" + RunProfileLabel = "signal.unbounded-cloud.io/profile" + RunTriggerAnnotation = "signal.unbounded-cloud.io/trigger" + RunAdmissionAnnotation = "signal.unbounded-cloud.io/admission" + RunFinalizer = "signal.unbounded-cloud.io/cleanup" + SnapshotRunLabel = "signal.unbounded-cloud.io/run" + runAdmissionReady = "ready" + runAdmissionPending = "pending" ) var runnerImageDigestPattern = regexp.MustCompile(`^[^@\s]+@sha256:[0-9a-f]{64}$`) @@ -169,7 +172,7 @@ func (r *AdmissionReconciler) admitNode(ctx context.Context, node *corev1.Node, err := reader.Get(ctx, client.ObjectKey{Namespace: r.Namespace, Name: runName}, &run) if err == nil { - return nil + return r.completeRunAdmission(ctx, &run) } if !apierrors.IsNotFound(err) { @@ -221,6 +224,10 @@ func (r *AdmissionReconciler) admitNode(ctx context.Context, node *corev1.Node, return fmt.Errorf("profile %s participant constraints: %w", profile.Name, err) } + if profile.Spec.Purpose != "" && profile.Spec.Purpose != signalv1.PurposeFullCertification { + return fmt.Errorf("profile %s purpose must be %s", profile.Name, signalv1.PurposeFullCertification) + } + for index := range checks { check := &checks[index] if err := signalv1.ValidateRuntimeContract( @@ -255,10 +262,50 @@ func (r *AdmissionReconciler) admitNode(ctx context.Context, node *corev1.Node, } for index := range checks { - if _, _, err := participantsForCheck(participants, checks[index].ParticipantPolicy); err != nil { + checkParticipants, _, err := participantsForCheck(participants, checks[index].ParticipantPolicy) + if err != nil { return fmt.Errorf("check %s participants: %w", checks[index].ID, err) } + minimumEvidenceBytes := signaladapter.MinimumBackendEvidenceBytes(len(checkParticipants)) + if int(checks[index].ResultBounds.BackendEvidenceBytes) < minimumEvidenceBytes { + return fmt.Errorf( + "check %s backendEvidenceBytes must be at least %d for %d participants", + checks[index].ID, + minimumEvidenceBytes, + len(checkParticipants), + ) + } + + minimumReasonBytes := signaladapter.MinimumPartialReasonBytes() + if int(checks[index].ResultBounds.ReasonBytes) < minimumReasonBytes { + return fmt.Errorf( + "check %s reasonBytes must be at least %d for mandatory failure output", + checks[index].ID, + minimumReasonBytes, + ) + } + + minimumMeasurementsBytes := signaladapter.MinimumPartialMeasurementsBytes(len(checkParticipants)) + if int(checks[index].ResultBounds.MeasurementsBytes) < minimumMeasurementsBytes { + return fmt.Errorf( + "check %s measurementsBytes must be at least %d for %d participants", + checks[index].ID, + minimumMeasurementsBytes, + len(checkParticipants), + ) + } + + minimumBackendResourcesBytes := signaladapter.MinimumPartialBackendResourcesBytes(len(checkParticipants)) + if int(checks[index].ResultBounds.BackendResourcesBytes) < minimumBackendResourcesBytes { + return fmt.Errorf( + "check %s backendResourcesBytes must be at least %d for %d participants", + checks[index].ID, + minimumBackendResourcesBytes, + len(checkParticipants), + ) + } + if _, err := signalworkload.Encode(checks[index].WorkloadPolicy); err != nil { return fmt.Errorf("check %s workload policy: %w", checks[index].ID, err) } @@ -323,7 +370,8 @@ func (r *AdmissionReconciler) admitNode(ctx context.Context, node *corev1.Node, RunProfileLabel: profile.Name, }, Annotations: map[string]string{ - RunTriggerAnnotation: "manual", + RunTriggerAnnotation: "manual", + RunAdmissionAnnotation: runAdmissionPending, }, Finalizers: []string{RunFinalizer}, }, @@ -339,7 +387,7 @@ func (r *AdmissionReconciler) admitNode(ctx context.Context, node *corev1.Node, Checks: checkSnapshots, ExecutionPolicy: profile.Spec.ExecutionPolicy, Workflow: profile.Spec.Workflow, - Purpose: profile.Spec.Purpose, + Purpose: signalv1.PurposeFullCertification, EvidencePath: strings.Trim(profile.Spec.EvidencePath, "/") + "/" + runName + ".json", Deadline: profile.Spec.Deadline, Tolerations: profile.Spec.Tolerations, @@ -371,12 +419,12 @@ func (r *AdmissionReconciler) admitNode(ctx context.Context, node *corev1.Node, return cleanupFailedAdmission(fmt.Errorf("bind script snapshot to run: %w", err)) } - if err := projectPending(ctx, r.Client, run); err != nil { - return cleanupFailedAdmission(fmt.Errorf("project pending certification state: %w", err)) - } - releaseOnFailure = false + if err := r.completeRunAdmission(ctx, run); err != nil { + return err + } + if r.Recorder != nil { r.Recorder.Eventf(run, nil, corev1.EventTypeNormal, "CertificationAdmitted", "AdmitCertification", "Created CertificationRun %s with %d immutable checks", run.Name, len(checkSnapshots)) @@ -385,6 +433,36 @@ func (r *AdmissionReconciler) admitNode(ctx context.Context, node *corev1.Node, return nil } +func (r *AdmissionReconciler) completeRunAdmission( + ctx context.Context, + run *signalv1.CertificationRun, +) error { + if run.Annotations[RunAdmissionAnnotation] == runAdmissionReady { + return nil + } + + if err := projectPending(ctx, r.Client, run); err != nil { + return fmt.Errorf("project pending certification state: %w", err) + } + + before := run.DeepCopy() + if run.Annotations == nil { + run.Annotations = map[string]string{} + } + + run.Annotations[RunAdmissionAnnotation] = runAdmissionReady + + if err := r.Patch( + ctx, + run, + client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{}), + ); err != nil { + return fmt.Errorf("mark certification admission ready: %w", err) + } + + return nil +} + func (r *AdmissionReconciler) selectParticipants( ctx context.Context, requestNode *corev1.Node, @@ -1083,6 +1161,10 @@ func (r *RunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R return r.cancel(ctx, &run) } + if run.Annotations[RunAdmissionAnnotation] != runAdmissionReady { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + if run.Status.Workflow.Name == "" { return r.startAttempt(ctx, &run, 1) } @@ -1879,6 +1961,8 @@ func terminalCheckStatuses( status.Verdict = outputs["verdict"] status.Reason = outputs["reason"] status.Measurements = outputs["measurements"] + allowPartialBackendEvidence := status.Verdict == "Inconclusive" && + status.Reason == "BackendSetupFailed" _, globalIndexes, participantErr := participantsForCheck( run.Spec.Participants, @@ -1901,6 +1985,7 @@ func terminalCheckStatuses( outputs["backend-resources"], check.Backend, run.Namespace, + allowPartialBackendEvidence, ) if evidenceErr != nil { status.Phase = signalv1.CheckPhaseInconclusive @@ -1924,6 +2009,7 @@ func terminalCheckStatuses( outputs["backend-resources"], globalIndexes, run.Namespace, + allowPartialBackendEvidence, ) if resourceErr != nil { status.Phase = signalv1.CheckPhaseInconclusive @@ -2160,6 +2246,7 @@ func backendResourceReferences( value string, participantCount int, namespace string, + allowPartial bool, ) ([]signalv1.BackendResourceReference, error) { if participantCount < 1 || participantCount > 64 { return nil, fmt.Errorf("participant count must be between 1 and 64") @@ -2243,7 +2330,7 @@ func backendResourceReferences( } } - if participantCount > 1 { + if !allowPartial && participantCount > 1 { for index, isCovered := range covered { if !isCovered { return nil, fmt.Errorf("backend-resources does not cover participant index %d", index) @@ -2258,8 +2345,9 @@ func backendResourceReferencesForParticipants( value string, globalIndexes []int32, namespace string, + allowPartial bool, ) ([]signalv1.BackendResourceReference, error) { - references, err := backendResourceReferences(value, len(globalIndexes), namespace) + references, err := backendResourceReferences(value, len(globalIndexes), namespace, allowPartial) if err != nil { return nil, err } @@ -2314,6 +2402,7 @@ func groupedBackendEvidenceComplete( references []signalv1.BackendResourceReference, backend signalv1.ExecutionBackend, namespace string, + allowPartial bool, ) (bool, error) { if err := signaladapter.ValidateBackendEvidence(content); err != nil { return false, err @@ -2324,16 +2413,16 @@ func groupedBackendEvidenceComplete( return false, err } - if len(evidence.Participants) != participantCount { + if len(evidence.Participants) > participantCount || (!allowPartial && len(evidence.Participants) != participantCount) { return false, fmt.Errorf( - "backend-evidence contains %d participants, want %d", + "backend-evidence contains %d participants, want at most %d", len(evidence.Participants), participantCount, ) } seen := make([]bool, participantCount) - complete := evidence.Complete && evidence.CaptureError == "" + complete := !allowPartial && evidence.Complete && evidence.CaptureError == "" podUIDs := map[types.UID]struct{}{} for index := range evidence.Participants { @@ -2435,13 +2524,21 @@ func groupedBackendEvidenceForParticipants( resources string, backend signalv1.ExecutionBackend, namespace string, + allowPartial bool, ) (string, bool, error) { - references, err := backendResourceReferences(resources, len(globalIndexes), namespace) + references, err := backendResourceReferences(resources, len(globalIndexes), namespace, allowPartial) if err != nil { return "", false, err } - complete, err := groupedBackendEvidenceComplete(content, len(globalIndexes), references, backend, namespace) + complete, err := groupedBackendEvidenceComplete( + content, + len(globalIndexes), + references, + backend, + namespace, + allowPartial, + ) if err != nil { return "", false, err } @@ -2486,15 +2583,6 @@ func groupedBackendEvidenceForParticipants( } func (r *RunReconciler) cancel(ctx context.Context, run *signalv1.CertificationRun) (ctrl.Result, error) { - stopped, err := r.ensureWorkflowStopped(ctx, run) - if err != nil { - return ctrl.Result{}, err - } - - if !stopped { - return ctrl.Result{RequeueAfter: time.Second}, nil - } - evidenceRun := run.DeepCopy() evidenceRun.Status.Phase = signalv1.RunPhaseCancelled evidenceRun.Status.ExecutionState = "Interrupted" diff --git a/internal/signal/projection.go b/internal/signal/projection.go index 9e32376908..022a7a9e1b 100644 --- a/internal/signal/projection.go +++ b/internal/signal/projection.go @@ -41,9 +41,9 @@ func projectPending( kube client.Client, run *signalv1.CertificationRun, ) error { - return projectParticipants(ctx, kube, run.Spec.Participants, func(node *corev1.Node) error { + return projectParticipants(ctx, kube, run.Spec.Participants, true, func(node *corev1.Node) error { if node.Annotations[ReservationLabel] != run.Name { - return nil + return fmt.Errorf("reservation is not owned by Run %s", run.Name) } setProfileProjection(node, run.Spec.Profile) @@ -55,7 +55,7 @@ func projectPending( } func projectTesting(ctx context.Context, kube client.Client, run *signalv1.CertificationRun) error { - return projectParticipants(ctx, kube, run.Spec.Participants, func(node *corev1.Node) error { + return projectParticipants(ctx, kube, run.Spec.Participants, false, func(node *corev1.Node) error { if node.Annotations[ReservationLabel] != run.Name { return nil } @@ -75,7 +75,7 @@ func projectTerminal(ctx context.Context, kube client.Client, run *signalv1.Cert finishedAt := run.Status.FinishedAt.UTC().Format(time.RFC3339) - return projectParticipants(ctx, kube, run.Spec.Participants, func(node *corev1.Node) error { + return projectParticipants(ctx, kube, run.Spec.Participants, false, func(node *corev1.Node) error { if node.Annotations[ReservationLabel] != run.Name { return nil } @@ -127,6 +127,7 @@ func projectParticipants( ctx context.Context, kube client.Client, participants []signalv1.Participant, + strict bool, mutate func(*corev1.Node) error, ) error { for _, participant := range participants { @@ -137,10 +138,22 @@ func projectParticipants( if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { var node corev1.Node if err := kube.Get(ctx, client.ObjectKey{Name: participant.NodeName}, &node); err != nil { - return client.IgnoreNotFound(err) + if !strict { + return client.IgnoreNotFound(err) + } + + return err } if participant.NodeUID != "" && node.UID != participant.NodeUID { + if strict { + return fmt.Errorf( + "node UID changed from %s to %s", + participant.NodeUID, + node.UID, + ) + } + return nil } diff --git a/internal/signaladapter/adapter.go b/internal/signaladapter/adapter.go index 0c3e6ec502..096712e854 100644 --- a/internal/signaladapter/adapter.go +++ b/internal/signaladapter/adapter.go @@ -310,6 +310,15 @@ func ValidateConfig(config Config) ([]signalv1.Participant, []corev1.Toleration, return nil, nil, nil, err } + minimumEvidenceBytes := MinimumBackendEvidenceBytes(len(participants)) + if config.BackendEvidenceBytes > 0 && config.BackendEvidenceBytes < minimumEvidenceBytes { + return nil, nil, nil, fmt.Errorf( + "backend-evidence-bytes must be at least %d for %d participants", + minimumEvidenceBytes, + len(participants), + ) + } + var tolerations []corev1.Toleration if err := decodeJSONObject(config.TolerationsJSON, &tolerations); err != nil { return nil, nil, nil, fmt.Errorf("decode tolerations: %w", err) @@ -470,6 +479,13 @@ func BackendResource(resource *unstructured.Unstructured, participantIndex int32 ) } + return backendResourceIdentity(resource, participantIndex) +} + +func backendResourceIdentity( + resource *unstructured.Unstructured, + participantIndex int32, +) (signalv1.BackendResourceReference, error) { if resource.GetAPIVersion() == "" || resource.GetKind() == "" || resource.GetName() == "" || @@ -487,6 +503,55 @@ func BackendResource(resource *unstructured.Unstructured, participantIndex int32 }, nil } +func MinimumPartialReasonBytes() int { + return len("BackendSetupFailed") +} + +func MinimumPartialMeasurementsBytes(participantCount int) int { + results := make([]Result, participantCount) + for index := range results { + results[index] = Result{ + ParticipantIndex: index, + ExecutionState: "Error", + Verdict: "Inconclusive", + Reason: "BackendSetupFailed", + } + } + + content, err := json.Marshal(map[string]any{ + "participantCount": participantCount, + "createdBackends": participantCount, + "error": strings.Repeat("\x00", maxEvidenceTextBytes), + "results": results, + }) + if err != nil { + return MaxMeasurementsBytes + 1 + } + + return len(content) + 64 +} + +func MinimumPartialBackendResourcesBytes(participantCount int) int { + references := make([]signalv1.BackendResourceReference, participantCount) + for index := range references { + references[index] = signalv1.BackendResourceReference{ + APIVersion: strings.Repeat("a", 32), + Kind: strings.Repeat("a", 16), + Namespace: strings.Repeat("a", 63), + Name: strings.Repeat("a", 63), + UID: types.UID(strings.Repeat("a", 128)), + ParticipantIndexes: []int32{int32(index)}, + } + } + + content, err := json.Marshal(references) + if err != nil { + return MaxBackendOutputBytes + 1 + } + + return len(content) + 64 +} + func Aggregate( participantCount int, backendResources []signalv1.BackendResourceReference, diff --git a/internal/signaladapter/backend_evidence.go b/internal/signaladapter/backend_evidence.go index 3068289e18..95cb534c2f 100644 --- a/internal/signaladapter/backend_evidence.go +++ b/internal/signaladapter/backend_evidence.go @@ -126,6 +126,35 @@ type CapturedLogEvidence struct { Error string `json:"error,omitempty"` } +func MinimumBackendEvidenceBytes(participantCount int) int { + evidence := BackendEvidence{ + Version: backendEvidenceVersion, + Complete: false, + Participants: make([]ParticipantBackendEvidence, participantCount), + } + + for index := range evidence.Participants { + evidence.Participants[index] = ParticipantBackendEvidence{ + ParticipantIndex: index, + Resource: BackendObjectEvidence{ + APIVersion: strings.Repeat("a", 32), + Kind: strings.Repeat("a", 16), + Namespace: strings.Repeat("a", 63), + Name: strings.Repeat("a", 63), + UID: types.UID(strings.Repeat("a", 128)), + }, + CaptureError: strings.Repeat("a", maxEvidenceTextBytes), + } + } + + content, err := json.Marshal(evidence) + if err != nil { + return MaxBackendEvidenceBytes + 1 + } + + return len(content) + 64 +} + func (r *Runner) captureBackendEvidence( ctx context.Context, config Config, diff --git a/internal/signaladapter/runner.go b/internal/signaladapter/runner.go index 842928dab8..8315e8cddb 100644 --- a/internal/signaladapter/runner.go +++ b/internal/signaladapter/runner.go @@ -5,6 +5,7 @@ package signaladapter import ( "context" + "encoding/json" "errors" "fmt" "io" @@ -58,7 +59,7 @@ type resultLogSnapshot struct { previous resultLogSource } -func (r *Runner) Execute(ctx context.Context, config Config) (output AggregateOutput, resultErr error) { +func (r *Runner) Execute(ctx context.Context, config Config) (AggregateOutput, error) { if r.Dynamic == nil || r.Kubernetes == nil { return AggregateOutput{}, errors.New("kubernetes clients are required") } @@ -123,20 +124,23 @@ func (r *Runner) Execute(ctx context.Context, config Config) (output AggregateOu } created := make([]createdResource, 0, len(resources)) - defer func() { - if len(created) == 0 { - return + references := make([]signalv1.BackendResourceReference, 0, len(resources)) + finishBackendFailure := func(cause error) (AggregateOutput, error) { + output, err := r.backendFailureOutput(config, len(participants), created, references, cause) + if err != nil { + return AggregateOutput{}, err } cleanupContext, cancel := context.WithTimeout(context.Background(), r.cleanupWait()) defer cancel() if err := r.cleanup(cleanupContext, config.Namespace, created); err != nil { - resultErr = errors.Join(resultErr, fmt.Errorf("cleanup after adapter exit: %w", err)) + recordCleanupFailure(&output) } - }() - references := make([]signalv1.BackendResourceReference, 0, len(resources)) + return output, nil + } + for index, resource := range resources { gvr := podGVR if resource.GetKind() == "MPIJob" { @@ -146,6 +150,10 @@ func (r *Runner) Execute(ctx context.Context, config Config) (output AggregateOu createdObject, createErr := r.Dynamic.Resource(gvr).Namespace(config.Namespace). Create(executionContext, resource, metav1.CreateOptions{}) if createErr != nil { + if len(created) > 0 { + return finishBackendFailure(fmt.Errorf("create participant %d %s: %w", index, resource.GetKind(), createErr)) + } + return AggregateOutput{}, fmt.Errorf("create participant %d %s: %w", index, resource.GetKind(), createErr) } @@ -157,7 +165,17 @@ func (r *Runner) Execute(ctx context.Context, config Config) (output AggregateOu reference, referenceErr := BackendResource(createdObject, int32(index)) if referenceErr != nil { - return AggregateOutput{}, fmt.Errorf("record participant %d backend identity: %w", index, referenceErr) + fallbackReference, identityErr := backendResourceIdentity(createdObject, int32(index)) + if identityErr != nil { + return AggregateOutput{}, errors.Join( + fmt.Errorf("record participant %d backend identity: %w", index, referenceErr), + identityErr, + ) + } + + references = append(references, fallbackReference) + + return finishBackendFailure(fmt.Errorf("record participant %d backend identity: %w", index, referenceErr)) } references = append(references, reference) @@ -179,9 +197,9 @@ func (r *Runner) Execute(ctx context.Context, config Config) (output AggregateOu waitGroup.Wait() - output, err = Aggregate(len(participants), references, results) + output, err := Aggregate(len(participants), references, results) if err != nil { - return AggregateOutput{}, err + return finishBackendFailure(fmt.Errorf("aggregate participant results: %w", err)) } captureContext, cancelCapture := context.WithTimeout(context.Background(), r.cleanupWait()) @@ -195,7 +213,7 @@ func (r *Runner) Execute(ctx context.Context, config Config) (output AggregateOu created, ) if err != nil { - return AggregateOutput{}, err + return finishBackendFailure(fmt.Errorf("capture backend evidence: %w", err)) } if !evidenceComplete { @@ -203,7 +221,7 @@ func (r *Runner) Execute(ctx context.Context, config Config) (output AggregateOu } if err := r.persistEvidenceBeforeCleanup(&output, config.BackendEvidenceBytes); err != nil { - return AggregateOutput{}, err + return finishBackendFailure(fmt.Errorf("persist backend evidence: %w", err)) } cleanupContext, cancel := context.WithTimeout(context.Background(), r.cleanupWait()) @@ -215,7 +233,68 @@ func (r *Runner) Execute(ctx context.Context, config Config) (output AggregateOu return output, nil } - created = nil + return output, nil +} + +func (r *Runner) backendFailureOutput( + config Config, + participantCount int, + created []createdResource, + references []signalv1.BackendResourceReference, + cause error, +) (AggregateOutput, error) { + captureContext, cancelCapture := context.WithTimeout(context.Background(), r.cleanupWait()) + defer cancelCapture() + + evidence, _, err := r.captureBackendEvidence( + captureContext, + config, + created, + ) + if err != nil { + return AggregateOutput{}, fmt.Errorf("capture partial backend evidence: %w", err) + } + + backendJSON, err := json.Marshal(references) + if err != nil { + return AggregateOutput{}, fmt.Errorf("encode partial backend resources: %w", err) + } + + reason := boundedRedactedText(cause.Error(), maxEvidenceTextBytes) + + results := make([]Result, participantCount) + for index := range results { + results[index] = Result{ + ParticipantIndex: index, + ExecutionState: "Error", + Verdict: "Inconclusive", + Reason: "BackendSetupFailed", + } + } + + measurements, err := json.Marshal(map[string]any{ + "participantCount": participantCount, + "createdBackends": len(created), + "error": reason, + "results": results, + }) + if err != nil { + return AggregateOutput{}, fmt.Errorf("encode partial backend measurements: %w", err) + } + + output := AggregateOutput{ + ExecutionState: "Error", + Verdict: "Inconclusive", + Reason: "BackendSetupFailed", + Measurements: string(measurements), + BackendResources: references, + BackendJSON: string(backendJSON), + BackendEvidence: evidence, + } + + if err := r.persistEvidenceBeforeCleanup(&output, config.BackendEvidenceBytes); err != nil { + return AggregateOutput{}, err + } return output, nil } @@ -234,10 +313,15 @@ func (r *Runner) persistEvidenceBeforeCleanup(output *AggregateOutput, limit int limit, ) if markErr != nil { - return markErr + return errors.Join( + fmt.Errorf("persist backend evidence before cleanup: %w", err), + markErr, + ) } recordEvidenceCaptureFailure(output) + + return fmt.Errorf("persist backend evidence before cleanup: %w", err) } return nil From 3d7b130068661d775f78e693d5b556a67c12b17b Mon Sep 17 00:00:00 2001 From: chokevin Date: Fri, 23 Oct 2026 17:12:51 -0700 Subject: [PATCH 32/36] refactor(signal): narrow certification contract --- .../v1alpha1/certificationprofile_types.go | 84 +--- api/signal/v1alpha1/certificationrun_types.go | 19 +- api/signal/v1alpha1/zz_generated.deepcopy.go | 103 +---- cmd/project-signal-adapter/main.go | 5 - .../04-adapter-workflowtemplate.yaml.tmpl | 14 - deploy/signal/README.md | 125 ++---- ...ounded-cloud.io_certificationprofiles.yaml | 416 ++--------------- ....unbounded-cloud.io_certificationruns.yaml | 418 ++---------------- internal/signal/controller.go | 274 ++++-------- internal/signaladapter/adapter.go | 159 ++----- internal/signaladapter/backend_evidence.go | 2 +- internal/signaladapter/runner.go | 4 +- internal/signalworkload/policy.go | 126 ++---- 13 files changed, 298 insertions(+), 1451 deletions(-) diff --git a/api/signal/v1alpha1/certificationprofile_types.go b/api/signal/v1alpha1/certificationprofile_types.go index b5bb178450..0bbfa309e3 100644 --- a/api/signal/v1alpha1/certificationprofile_types.go +++ b/api/signal/v1alpha1/certificationprofile_types.go @@ -21,29 +21,8 @@ type WorkflowReference struct { Name string `json:"name"` } -type CertificationPurpose string - -const ( - PurposeFullCertification CertificationPurpose = "FullCertification" -) - type CheckID string -type ResultBounds struct { - // +kubebuilder:validation:Minimum=1 - // +kubebuilder:validation:Maximum=4096 - ReasonBytes int32 `json:"reasonBytes"` - // +kubebuilder:validation:Minimum=1 - // +kubebuilder:validation:Maximum=16384 - MeasurementsBytes int32 `json:"measurementsBytes"` - // +kubebuilder:validation:Minimum=1 - // +kubebuilder:validation:Maximum=16384 - BackendResourcesBytes int32 `json:"backendResourcesBytes"` - // +kubebuilder:validation:Minimum=1 - // +kubebuilder:validation:Maximum=65536 - BackendEvidenceBytes int32 `json:"backendEvidenceBytes"` -} - type ParticipantRoleConstraint struct { // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:MaxLength=63 @@ -73,7 +52,6 @@ type CheckParticipantPolicy struct { ExcludedRoles []string `json:"excludedRoles,omitempty"` } -// +kubebuilder:validation:XValidation:rule="self.maxAttempts == 1",message="maxAttempts must be exactly 1" // +kubebuilder:validation:XValidation:rule="duration(self.backendActiveDeadline) > duration('0s')",message="backendActiveDeadline must be positive" // +kubebuilder:validation:XValidation:rule="!has(self.script.requiresRuntimeStage) || !self.script.requiresRuntimeStage || has(self.runtimeStage)",message="runtimeStage is required when the script requires a runtime stage" type CertificationCheck struct { @@ -83,17 +61,17 @@ type CertificationCheck struct { ID CheckID `json:"id"` // +kubebuilder:validation:Minimum=0 // +kubebuilder:validation:Maximum=14 - Order int32 `json:"order"` - Script ScriptReference `json:"script"` - RunnerImage string `json:"runnerImage"` - Backend ExecutionBackend `json:"backend"` - WorkloadPolicy WorkloadPolicy `json:"workloadPolicy,omitempty"` - RuntimeStage *RuntimeStage `json:"runtimeStage,omitempty"` - Parameters map[string]string `json:"parameters,omitempty"` + Order int32 `json:"order"` + Script ScriptReference `json:"script"` + RunnerImage string `json:"runnerImage"` + Backend ExecutionBackend `json:"backend"` + WorkloadPolicy WorkloadPolicy `json:"workloadPolicy,omitempty"` + RuntimeStage *RuntimeStage `json:"runtimeStage,omitempty"` + // +kubebuilder:validation:Minimum=0 + GPUCount int32 `json:"gpuCount,omitempty"` + ScriptEnvironment map[string]string `json:"scriptEnvironment,omitempty"` BackendActiveDeadline metav1.Duration `json:"backendActiveDeadline"` - ResultBounds ResultBounds `json:"resultBounds"` ParticipantPolicy CheckParticipantPolicy `json:"participantPolicy,omitempty"` - MaxAttempts int32 `json:"maxAttempts"` } type SuiteFailurePolicy string @@ -118,7 +96,6 @@ type RuntimeStage struct { type ExecutionBackend struct { APIVersion string `json:"apiVersion"` Kind string `json:"kind"` - Queue string `json:"queue,omitempty"` } type ParticipantPolicy struct { @@ -131,34 +108,21 @@ type ParticipantPolicy struct { RequiredResources corev1.ResourceList `json:"requiredResources,omitempty"` } -type WorkloadVolume struct { - Name string `json:"name"` - MountPath string `json:"mountPath"` - ReadOnly bool `json:"readOnly,omitempty"` - - HostPath *corev1.HostPathVolumeSource `json:"hostPath,omitempty"` - EmptyDir *corev1.EmptyDirVolumeSource `json:"emptyDir,omitempty"` - PersistentVolumeClaim *corev1.PersistentVolumeClaimVolumeSource `json:"persistentVolumeClaim,omitempty"` -} - -type KueueTopologyPolicy struct { - // RequiredLevel is the Node label key for the minimum topology domain in which - // Kueue must place the participant PodSet. - // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:MaxLength=316 - // +kubebuilder:validation:Pattern=`^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$` - RequiredLevel string `json:"requiredLevel"` +type WorkloadHostPath struct { + Name string `json:"name"` + MountPath string `json:"mountPath"` + Path string `json:"path"` + Type corev1.HostPathType `json:"type,omitempty"` } type WorkloadPolicy struct { - Resources corev1.ResourceRequirements `json:"resources,omitempty"` - SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"` - HostNetwork bool `json:"hostNetwork,omitempty"` - KueueTopology *KueueTopologyPolicy `json:"kueueTopology,omitempty"` - WorkingDir string `json:"workingDir,omitempty"` - Results *corev1.EmptyDirVolumeSource `json:"results,omitempty"` + Resources corev1.ResourceRequirements `json:"resources,omitempty"` + Privileged bool `json:"privileged,omitempty"` + Capabilities []corev1.Capability `json:"capabilities,omitempty"` + HostNetwork bool `json:"hostNetwork,omitempty"` + WorkingDir string `json:"workingDir,omitempty"` // +kubebuilder:validation:MaxItems=16 - Volumes []WorkloadVolume `json:"volumes,omitempty"` + HostPaths []WorkloadHostPath `json:"hostPaths,omitempty"` } // +kubebuilder:validation:XValidation:rule="self.checks.all(c, self.checks.exists_one(x, x.id == c.id))",message="checks must contain unique IDs" @@ -176,11 +140,9 @@ type CertificationProfileSpec struct { // +kubebuilder:validation:MaxItems=15 Checks []CertificationCheck `json:"checks"` ExecutionPolicy SuiteExecutionPolicy `json:"executionPolicy"` - // +kubebuilder:validation:Enum=FullCertification - Purpose CertificationPurpose `json:"purpose,omitempty"` - EvidencePath string `json:"evidencePath"` - Deadline metav1.Duration `json:"deadline"` - Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + EvidencePath string `json:"evidencePath"` + Deadline metav1.Duration `json:"deadline"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` } // +kubebuilder:object:root=true diff --git a/api/signal/v1alpha1/certificationrun_types.go b/api/signal/v1alpha1/certificationrun_types.go index cd39e9c7c9..c94b7a4dae 100644 --- a/api/signal/v1alpha1/certificationrun_types.go +++ b/api/signal/v1alpha1/certificationrun_types.go @@ -68,7 +68,6 @@ type ScriptSnapshot struct { Digest string `json:"digest"` } -// +kubebuilder:validation:XValidation:rule="self.maxAttempts == 1",message="maxAttempts must be exactly 1" // +kubebuilder:validation:XValidation:rule="duration(self.backendActiveDeadline) > duration('0s')",message="backendActiveDeadline must be positive" // +kubebuilder:validation:XValidation:rule="!has(self.script.requiresRuntimeStage) || !self.script.requiresRuntimeStage || has(self.runtimeStage)",message="runtimeStage is required when the script requires a runtime stage" type CertificationCheckSnapshot struct { @@ -84,11 +83,10 @@ type CertificationCheckSnapshot struct { Backend ExecutionBackend `json:"backend"` WorkloadPolicy WorkloadPolicy `json:"workloadPolicy,omitempty"` RuntimeStage *RuntimeStage `json:"runtimeStage,omitempty"` - Parameters map[string]string `json:"parameters,omitempty"` + GPUCount int32 `json:"gpuCount,omitempty"` + ScriptEnvironment map[string]string `json:"scriptEnvironment,omitempty"` BackendActiveDeadline metav1.Duration `json:"backendActiveDeadline"` - ResultBounds ResultBounds `json:"resultBounds"` ParticipantPolicy CheckParticipantPolicy `json:"participantPolicy,omitempty"` - MaxAttempts int32 `json:"maxAttempts"` } // +kubebuilder:validation:XValidation:rule="self.profile == oldSelf.profile",message="profile is immutable" @@ -98,7 +96,6 @@ type CertificationCheckSnapshot struct { // +kubebuilder:validation:XValidation:rule="self.checks == oldSelf.checks",message="checks are immutable" // +kubebuilder:validation:XValidation:rule="self.executionPolicy == oldSelf.executionPolicy",message="executionPolicy is immutable" // +kubebuilder:validation:XValidation:rule="self.workflow == oldSelf.workflow",message="workflow is immutable" -// +kubebuilder:validation:XValidation:rule="has(self.purpose) == has(oldSelf.purpose) && (!has(self.purpose) || self.purpose == oldSelf.purpose)",message="purpose is immutable" // +kubebuilder:validation:XValidation:rule="self.evidencePath == oldSelf.evidencePath",message="evidencePath is immutable" // +kubebuilder:validation:XValidation:rule="self.deadline == oldSelf.deadline",message="deadline is immutable" // +kubebuilder:validation:XValidation:rule="has(self.tolerations) == has(oldSelf.tolerations) && (!has(self.tolerations) || self.tolerations == oldSelf.tolerations)",message="tolerations are immutable" @@ -121,13 +118,11 @@ type CertificationRunSpec struct { Checks []CertificationCheckSnapshot `json:"checks"` ExecutionPolicy SuiteExecutionPolicy `json:"executionPolicy"` Workflow WorkflowReference `json:"workflow"` - // +kubebuilder:validation:Enum=FullCertification - Purpose CertificationPurpose `json:"purpose,omitempty"` - EvidencePath string `json:"evidencePath"` - Deadline metav1.Duration `json:"deadline"` - Tolerations []corev1.Toleration `json:"tolerations,omitempty"` - RequestNodeName string `json:"requestNodeName"` - Cancel bool `json:"cancel,omitempty"` + EvidencePath string `json:"evidencePath"` + Deadline metav1.Duration `json:"deadline"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + RequestNodeName string `json:"requestNodeName"` + Cancel bool `json:"cancel,omitempty"` } type WorkflowStatus struct { diff --git a/api/signal/v1alpha1/zz_generated.deepcopy.go b/api/signal/v1alpha1/zz_generated.deepcopy.go index 0a84665b30..33d607cb52 100644 --- a/api/signal/v1alpha1/zz_generated.deepcopy.go +++ b/api/signal/v1alpha1/zz_generated.deepcopy.go @@ -59,15 +59,14 @@ func (in *CertificationCheck) DeepCopyInto(out *CertificationCheck) { *out = new(RuntimeStage) **out = **in } - if in.Parameters != nil { - in, out := &in.Parameters, &out.Parameters + if in.ScriptEnvironment != nil { + in, out := &in.ScriptEnvironment, &out.ScriptEnvironment *out = make(map[string]string, len(*in)) for key, val := range *in { (*out)[key] = val } } out.BackendActiveDeadline = in.BackendActiveDeadline - out.ResultBounds = in.ResultBounds in.ParticipantPolicy.DeepCopyInto(&out.ParticipantPolicy) } @@ -92,15 +91,14 @@ func (in *CertificationCheckSnapshot) DeepCopyInto(out *CertificationCheckSnapsh *out = new(RuntimeStage) **out = **in } - if in.Parameters != nil { - in, out := &in.Parameters, &out.Parameters + if in.ScriptEnvironment != nil { + in, out := &in.ScriptEnvironment, &out.ScriptEnvironment *out = make(map[string]string, len(*in)) for key, val := range *in { (*out)[key] = val } } out.BackendActiveDeadline = in.BackendActiveDeadline - out.ResultBounds = in.ResultBounds in.ParticipantPolicy.DeepCopyInto(&out.ParticipantPolicy) } @@ -486,21 +484,6 @@ func (in *ExecutionBackend) DeepCopy() *ExecutionBackend { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KueueTopologyPolicy) DeepCopyInto(out *KueueTopologyPolicy) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KueueTopologyPolicy. -func (in *KueueTopologyPolicy) DeepCopy() *KueueTopologyPolicy { - if in == nil { - return nil - } - out := new(KueueTopologyPolicy) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Participant) DeepCopyInto(out *Participant) { *out = *in @@ -604,21 +587,6 @@ func (in *PinnedProfile) DeepCopy() *PinnedProfile { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ResultBounds) DeepCopyInto(out *ResultBounds) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResultBounds. -func (in *ResultBounds) DeepCopy() *ResultBounds { - if in == nil { - return nil - } - out := new(ResultBounds) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RuntimeStage) DeepCopyInto(out *RuntimeStage) { *out = *in @@ -710,69 +678,42 @@ func (in *WorkflowStatus) DeepCopy() *WorkflowStatus { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *WorkloadPolicy) DeepCopyInto(out *WorkloadPolicy) { +func (in *WorkloadHostPath) DeepCopyInto(out *WorkloadHostPath) { *out = *in - in.Resources.DeepCopyInto(&out.Resources) - if in.SecurityContext != nil { - in, out := &in.SecurityContext, &out.SecurityContext - *out = new(v1.SecurityContext) - (*in).DeepCopyInto(*out) - } - if in.KueueTopology != nil { - in, out := &in.KueueTopology, &out.KueueTopology - *out = new(KueueTopologyPolicy) - **out = **in - } - if in.Results != nil { - in, out := &in.Results, &out.Results - *out = new(v1.EmptyDirVolumeSource) - (*in).DeepCopyInto(*out) - } - if in.Volumes != nil { - in, out := &in.Volumes, &out.Volumes - *out = make([]WorkloadVolume, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadPolicy. -func (in *WorkloadPolicy) DeepCopy() *WorkloadPolicy { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadHostPath. +func (in *WorkloadHostPath) DeepCopy() *WorkloadHostPath { if in == nil { return nil } - out := new(WorkloadPolicy) + out := new(WorkloadHostPath) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *WorkloadVolume) DeepCopyInto(out *WorkloadVolume) { +func (in *WorkloadPolicy) DeepCopyInto(out *WorkloadPolicy) { *out = *in - if in.HostPath != nil { - in, out := &in.HostPath, &out.HostPath - *out = new(v1.HostPathVolumeSource) - (*in).DeepCopyInto(*out) - } - if in.EmptyDir != nil { - in, out := &in.EmptyDir, &out.EmptyDir - *out = new(v1.EmptyDirVolumeSource) - (*in).DeepCopyInto(*out) - } - if in.PersistentVolumeClaim != nil { - in, out := &in.PersistentVolumeClaim, &out.PersistentVolumeClaim - *out = new(v1.PersistentVolumeClaimVolumeSource) - **out = **in + in.Resources.DeepCopyInto(&out.Resources) + if in.Capabilities != nil { + in, out := &in.Capabilities, &out.Capabilities + *out = make([]v1.Capability, len(*in)) + copy(*out, *in) + } + if in.HostPaths != nil { + in, out := &in.HostPaths, &out.HostPaths + *out = make([]WorkloadHostPath, len(*in)) + copy(*out, *in) } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadVolume. -func (in *WorkloadVolume) DeepCopy() *WorkloadVolume { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadPolicy. +func (in *WorkloadPolicy) DeepCopy() *WorkloadPolicy { if in == nil { return nil } - out := new(WorkloadVolume) + out := new(WorkloadPolicy) in.DeepCopyInto(out) return out } diff --git a/cmd/project-signal-adapter/main.go b/cmd/project-signal-adapter/main.go index 4c249da2d2..00d4efdf05 100644 --- a/cmd/project-signal-adapter/main.go +++ b/cmd/project-signal-adapter/main.go @@ -76,16 +76,11 @@ func main() { 0, "Active deadline for participant backend workloads", ) - flags.StringVar(&cfg.Queue, "queue", "", "Kueue LocalQueue name") flags.StringVar(&cfg.TolerationsJSON, "tolerations", "[]", "JSON-encoded Pod tolerations") flags.StringVar(&cfg.ScriptEnvironmentJSON, "script-environment", "{}", "JSON-encoded script environment") flags.StringVar(&cfg.WorkloadPolicyJSON, "workload-policy", "{}", "JSON-encoded participant workload policy") - flags.IntVar(&cfg.BackendEvidenceBytes, "backend-evidence-bytes", signaladapter.MaxBackendEvidenceBytes, "Maximum backend evidence output bytes") flags.StringVar(&cfg.GPUResourceCount, "gpu-resource-count", "0", "GPU resource limit per participant") - flags.BoolVar(&cfg.Privileged, "privileged", false, "Run the participant workload with a privileged security context") flags.IntVar(&cfg.Attempt, "attempt", 0, "One-based Workflow attempt number") - flags.Int64Var(&cfg.BackoffLimit, "backoff-limit", 0, "MPIJob backoff limit") - flags.Int64Var(&cfg.TTLSeconds, "ttl-seconds-after-finished", 3600, "MPIJob TTL after completion") flags.StringVar(&cfg.outputDirectory, "output-directory", "/outputs", "Argo output directory") command.CompletionOptions.DisableDefaultCmd = true diff --git a/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl b/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl index 7d5dd4f35b..f7a04baefc 100644 --- a/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl +++ b/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl @@ -21,20 +21,11 @@ spec: - name: backend-api-version - name: backend-kind - name: backend-active-deadline-seconds - - name: queue - name: runner-image - name: tolerations - name: script-environment - name: workload-policy - - name: backend-evidence-bytes - default: "65536" - - name: mpi-payload-mode - - name: expected-gpu-product-regex - - name: expected-gpu-count - - name: observed-gpu-capacity - - name: observed-gpu-allocatable - name: gpu-resource-count - - name: privileged - name: attempt container: image: {{ default "example.invalid/project-signal-adapter@sha256:0000000000000000000000000000000000000000000000000000000000000000" .AdapterImage }} @@ -69,19 +60,14 @@ spec: - "{{`{{inputs.parameters.backend-kind}}`}}" - --backend-active-deadline-seconds - "{{`{{inputs.parameters.backend-active-deadline-seconds}}`}}" - - --queue - - "{{`{{inputs.parameters.queue}}`}}" - --tolerations - "{{`{{inputs.parameters.tolerations}}`}}" - --script-environment - "{{`{{inputs.parameters.script-environment}}`}}" - --workload-policy - "{{`{{inputs.parameters.workload-policy}}`}}" - - --backend-evidence-bytes - - "{{`{{inputs.parameters.backend-evidence-bytes}}`}}" - --gpu-resource-count - "{{`{{inputs.parameters.gpu-resource-count}}`}}" - - "--privileged={{`{{inputs.parameters.privileged}}`}}" - --attempt - "{{`{{inputs.parameters.attempt}}`}}" env: diff --git a/deploy/signal/README.md b/deploy/signal/README.md index 2819a6452f..a22516e96a 100644 --- a/deploy/signal/README.md +++ b/deploy/signal/README.md @@ -5,7 +5,7 @@ certification workflows. The upstream integration provides the API, controller, CRDs, controller Deployment, and an optional generic participant adapter for ordinary Pods and Kubeflow MPIJobs. It intentionally does not publish certification scripts, runner images, CertificationProfiles, hardware policy, -or environment-specific queue configuration. +or environment-specific hardware configuration. ## Architecture @@ -49,8 +49,9 @@ by the selected script. The adapter image and execution image are separate: the upstream adapter creates and observes workloads, while the operator-owned execution image runs the script. -The included adapter supports `v1` Pods and `kubeflow.org/v2beta1` MPIJobs. -Project Signal does not install MPI Operator or Kueue. Its adapter RBAC is +The included adapter supports standalone `v1` Pods and +`kubeflow.org/v2beta1` MPIJobs. Project Signal does not install MPI Operator. +Kueue integration is intentionally deferred. The adapter RBAC is namespace-scoped and intentionally grants only the Pod, Pod log, MPIJob, Workflow, and Argo task-result access needed by the included template. @@ -72,19 +73,11 @@ The referenced WorkflowTemplate must accept these string parameters: | `backend-api-version` | Backend API version from the profile | | `backend-kind` | Backend kind from the profile | | `backend-active-deadline-seconds` | Participant backend deadline, shorter than the outer Workflow deadline | -| `queue` | Optional backend queue | | `runner-image` | Digest-pinned execution image | | `tolerations` | JSON-encoded toleration list | | `script-environment` | JSON-encoded, validated environment map | -| `workload-policy` | JSON-encoded immutable participant-container resources, security context, and bounded volume policy | -| `backend-evidence-bytes` | Per-check evidence output bound, at most 64 KiB | -| `mpi-payload-mode` | Compatibility input for operator-defined templates | -| `expected-gpu-product-regex` | Compatibility input for operator-defined templates | -| `expected-gpu-count` | Compatibility input for operator-defined templates | -| `observed-gpu-capacity` | Compatibility input copied from admitted Node inventory | -| `observed-gpu-allocatable` | Compatibility input copied from admitted Node inventory | +| `workload-policy` | JSON-encoded immutable resources, privilege/capabilities, host networking, working directory, and read-only host paths | | `gpu-resource-count` | GPU count, defaults to `0` for Pod diagnostics; must be positive for MPIJobs | -| `privileged` | Explicit boolean, defaults to `false` | | `attempt` | One-based attempt number | The Workflow must publish only these output parameters: @@ -92,10 +85,10 @@ The Workflow must publish only these output parameters: | Output | Limit | Meaning | | --- | ---: | --- | | `verdict` | 32 bytes | `Pass`, `Fail`, or `Inconclusive` | -| `reason` | 4 KiB | Bounded diagnostic reason | +| `reason` | 512 bytes | Bounded diagnostic reason | | `measurements` | 16 KiB | Bounded measurement summary | -| `backend-resources` | 16 KiB | Optional JSON array of backend workload identities and covered participant indexes; required for a multi-Node `Pass` | -| `backend-evidence` | 64 KiB per check | Required redacted pre-cleanup backend object, container state, and bounded current/previous log projection | +| `backend-resources` | 8 KiB | Optional JSON array of backend workload identities and covered participant indexes; required for a multi-Node `Pass` | +| `backend-evidence` | 24 KiB per check | Required redacted pre-cleanup backend object, container state, and bounded current/previous log projection | Unexpected, duplicate, malformed, or oversized outputs are rejected and the run remains fail-closed. Each `backend-resources` entry must contain @@ -103,12 +96,13 @@ run remains fail-closed. Each `backend-resources` entry must contain that the workload executed. `namespace` is optional and defaults to the CertificationRun namespace; an explicitly different namespace is rejected. A multi-Node successful result must cover every admitted participant. +Each check may select at most eight participants so all 15 checks' fixed output +bounds remain within the 768 KiB Workflow status budget. The controller reads outputs only from each check's unique deterministic DAG task node. Missing, malformed, duplicate, or ambiguous task identity and outputs fail closed. Operator script -parameters, including GPU expectations and observed inventory, are carried in -`script-environment`. Their existing individual compatibility inputs remain -available to external templates; the supplied adapter uses the environment. +inputs are explicit entries in `spec.checks[].scriptEnvironment`; the +controller validates their environment names before reserving Nodes. Before foreground deletion, the adapter captures and writes a typed v1 projection for every participant backend to the Argo output path. The @@ -178,8 +172,8 @@ ordinary Pods and MPI workers use required Node affinity on the immutable admitted Node UID. This binds execution to the exact admitted Node without bypassing scheduler admission or accepting a replacement Node that reuses its name. Only the worker receives the configured GPU limit. The adapter propagates -the Workflow active deadline, applies the profile's optional queue, records -API-assigned workload UIDs, reads the final typed result from bounded workload +the Workflow active deadline, records API-assigned workload UIDs, reads the +final typed result from bounded workload logs, aggregates diagnostic state separately from backend execution state, and foreground-deletes every created workload only after its evidence output is durable. @@ -272,17 +266,10 @@ checks: backend: apiVersion: v1 kind: Pod - queue: backendActiveDeadline: 19m - resultBounds: - reasonBytes: 4096 - measurementsBytes: 16384 - backendResourcesBytes: 16384 - backendEvidenceBytes: 65536 - maxAttempts: 1 ``` -or an MPIJob backend, optionally submitted through a Kueue queue: +or an MPIJob backend: ```yaml checks: @@ -291,14 +278,8 @@ checks: backend: apiVersion: kubeflow.org/v2beta1 kind: MPIJob - queue: ``` -When either backend supplies a queue, the adapter writes the standard -`kueue.x-k8s.io/queue-name` label. Pod and MPIJob backends may both omit it for -standalone execution without Kueue. A `kueueTopology` policy still requires an -explicit queue. - Checks may set `spec.checks[].workloadPolicy` when a diagnostic needs explicit participant-container policy. The controller validates and snapshots this policy into the immutable CertificationRun before reserving Nodes. The included @@ -308,8 +289,6 @@ extra mounts. ```yaml workloadPolicy: - kueueTopology: - requiredLevel: kubernetes.io/hostname resources: requests: cpu: "8" @@ -319,37 +298,26 @@ workloadPolicy: cpu: "8" memory: 32Gi ephemeral-storage: 40Gi - securityContext: - privileged: false - capabilities: - add: [IPC_LOCK] + privileged: false + capabilities: [IPC_LOCK] hostNetwork: true workingDir: /opt/diagnostic - results: - sizeLimit: 2Gi - volumes: + hostPaths: - name: work mountPath: /work - emptyDir: - sizeLimit: 16Gi - - name: host-proc - mountPath: /host/proc - readOnly: true - hostPath: - path: /proc - type: Directory + path: /var/lib/project-signal + type: Directory ``` -Each extra volume must select exactly one `hostPath`, `emptyDir`, or -`persistentVolumeClaim` source. Names and mount paths must be unique; +Host paths are always mounted read-only. Names and mount paths must be unique; `scripts`, `results`, `/scripts`, and `/results` are reserved. Paths must be clean and absolute, resource quantities must be nonnegative, and limits cannot -be below requests. The policy is bounded to 16 volumes and 16 KiB of serialized -input. GPU requests and limits remain controlled by `gpu-resource-count`, which -sets equal requests and limits on the participant container. Kubernetes -admission and operator policy still decide whether a capability, host path, -device, PVC, or privileged context is allowed; technical rendering is not -authorization. +be below requests. The policy is bounded to 16 host paths and 16 KiB of +serialized input. GPU requests and limits remain controlled by +`gpu-resource-count`, which sets equal requests and limits on the participant +container. Kubernetes admission and operator policy still decide whether a +capability, host path, device, or privileged context is allowed; technical +rendering is not authorization. Stage-aware scripts declare `spec.checks[].script.requiresRuntimeStage: true` and require an explicit @@ -373,22 +341,9 @@ rendered to its participant Pod or MPIJob `activeDeadlineSeconds`. It must be positive and leave at least 60 seconds before the outer deadline; the controller rejects an invalid ordering before reserving Nodes. -The sum of all 15 checks' declared verdict, reason, measurements, backend -resource, backend evidence, and cleanup output bounds cannot exceed 768 KiB. -The controller rejects larger plans before reserving Nodes so valid per-check -bounds cannot collectively exceed a safe Argo Workflow status budget. -Each reason, measurements, backend-resource, and backend-evidence bound must -also fit the mandatory partial-failure envelope for the check's selected -participant count. Every accepted plan can therefore report a bounded partial -setup failure. - -`kueueTopology.requiredLevel` is an explicit Kueue Topology Aware Scheduling -request. It must be a valid Kubernetes label key and requires a nonempty backend -queue. The adapter writes -`kueue.x-k8s.io/podset-required-topology: ` on ordinary -participant Pods and MPIJob worker Pod templates only. It never annotates the -unpinned MPI launcher, does not accept arbitrary annotations, and does not -choose a topology level when the field is omitted. +All checks use fixed output limits. Together, the 15 checks stay below the +768 KiB Workflow status budget, and every accepted plan can represent its +mandatory partial-failure envelope. `hostNetwork: true` is an explicit participant policy for diagnostics that must observe the selected Node's network namespace, such as physical NIC inventory. @@ -397,27 +352,19 @@ applies only to the Node-pinned worker; the unpinned launcher remains outside the host network namespace. Omitting the field keeps Kubernetes' default Pod networking. -`gpu-resource-count` remains an operator-supplied profile parameter. It may be -zero for Pod diagnostics and must be positive for MPIJobs. The MPIJob policy is +`spec.checks[].gpuCount` may be zero for Pod diagnostics and must be positive +for MPIJobs. The controller renders it as `gpu-resource-count` for the adapter. +The MPIJob policy is fail-closed: `backoffLimit: 0`, `cleanPodPolicy: All`, a one-hour post-finish TTL, replica-level `restartPolicy: Never`, `WaitForWorkersReady`, and `runLauncherAsWorker: false`. -Operators can override the nonnegative MPI retry and TTL values through profile -parameters `backend-backoff-limit` and -`backend-ttl-seconds-after-finished`. These become -`BACKEND_BACKOFF_LIMIT` and `BACKEND_TTL_SECONDS_AFTER_FINISHED` in the validated -script environment and remain explicit per-profile policy rather than cluster -defaults. - A `CertificationProfile` defines exactly 15 ordered checks. Admission snapshots the complete plan and selected participants once into one immutable `CertificationRun`. Each check binds its script identity and digest, runner, -backend, workload and topology policy, runtime inputs, deadline, result bounds, -participant-role inclusion or exclusion, and `maxAttempts: 1`. -The only supported purpose is `FullCertification`; diagnostic, preflight, and -targeted modes are intentionally outside this one-time Node join contract and -cannot project production eligibility. +backend, workload and topology policy, runtime inputs, deadline, and +participant-role inclusion or exclusion. Every run is a one-time full +certification; alternate purposes and retries are not public configuration. The controller creates one Argo Workflow with an inline DAG entrypoint. Every task invokes the configured adapter `certification` template through diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml index 67672f4950..4f424c7ac2 100644 --- a/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml @@ -57,31 +57,26 @@ spec: type: string kind: type: string - queue: - type: string required: - apiVersion - kind type: object backendActiveDeadline: type: string + gpuCount: + format: int32 + minimum: 0 + type: integer id: maxLength: 63 minLength: 1 pattern: ^R(0[1-9]|1[0-5])$ type: string - maxAttempts: - format: int32 - type: integer order: format: int32 maximum: 14 minimum: 0 type: integer - parameters: - additionalProperties: - type: string - type: object participantPolicy: properties: excludedRoles: @@ -95,34 +90,6 @@ spec: maxItems: 16 type: array type: object - resultBounds: - properties: - backendEvidenceBytes: - format: int32 - maximum: 65536 - minimum: 1 - type: integer - backendResourcesBytes: - format: int32 - maximum: 16384 - minimum: 1 - type: integer - measurementsBytes: - format: int32 - maximum: 16384 - minimum: 1 - type: integer - reasonBytes: - format: int32 - maximum: 4096 - minimum: 1 - type: integer - required: - - backendEvidenceBytes - - backendResourcesBytes - - measurementsBytes - - reasonBytes - type: object runnerImage: type: string runtimeStage: @@ -153,23 +120,39 @@ spec: - key - name type: object + scriptEnvironment: + additionalProperties: + type: string + type: object workloadPolicy: properties: + capabilities: + items: + description: Capability represent POSIX capabilities type + type: string + type: array hostNetwork: type: boolean - kueueTopology: - properties: - requiredLevel: - description: |- - RequiredLevel is the Node label key for the minimum topology domain in which - Kueue must place the participant PodSet. - maxLength: 316 - minLength: 1 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - requiredLevel - type: object + hostPaths: + items: + properties: + mountPath: + type: string + name: + type: string + path: + type: string + type: + type: string + required: + - mountPath + - name + - path + type: object + maxItems: 16 + type: array + privileged: + type: boolean resources: description: ResourceRequirements describes the compute resource requirements. @@ -231,331 +214,6 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object - results: - description: |- - Represents an empty directory for a pod. - Empty directory volumes support ownership management and SELinux relabeling. - properties: - medium: - description: |- - medium represents what type of storage medium should back this directory. - The default is "" which means to use the node's default medium. - Must be an empty string (default) or Memory. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - type: string - mode: - description: |- - mode specifies the permission bits for the emptyDir directory, in numeric - notation (e.g., 0755, 01777). Must be a value between 0000 and 01777. - If not specified, defaults to 0777. - This might be in conflict with other options that affect the file - mode, like fsGroup. If fsGroup is specified, the fsGroup permissions - will override the mode specified here. - This field has no effect on Windows. - This field is alpha and requires EmptyDirVolumeMode featuregate to be enabled. - format: int32 - type: integer - sizeLimit: - anyOf: - - type: integer - - type: string - description: |- - sizeLimit is the total amount of local storage required for this EmptyDir volume. - The size limit is also applicable for memory medium. - The maximum usage on memory medium EmptyDir would be the minimum value between - the SizeLimit specified here and the sum of memory limits of all containers in a pod. - The default is nil which means that the limit is undefined. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - securityContext: - description: |- - SecurityContext holds security configuration that will be applied to a container. - Some fields are present in both SecurityContext and PodSecurityContext. When both - are set, the values in SecurityContext take precedence. - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities - type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities - type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies - to the container. - type: string - role: - description: Role is a SELinux role label that applies - to the container. - type: string - type: - description: Type is a SELinux type label that applies - to the container. - type: string - user: - description: User is a SELinux user label that applies - to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name - of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - volumes: - items: - properties: - emptyDir: - description: |- - Represents an empty directory for a pod. - Empty directory volumes support ownership management and SELinux relabeling. - properties: - medium: - description: |- - medium represents what type of storage medium should back this directory. - The default is "" which means to use the node's default medium. - Must be an empty string (default) or Memory. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - type: string - mode: - description: |- - mode specifies the permission bits for the emptyDir directory, in numeric - notation (e.g., 0755, 01777). Must be a value between 0000 and 01777. - If not specified, defaults to 0777. - This might be in conflict with other options that affect the file - mode, like fsGroup. If fsGroup is specified, the fsGroup permissions - will override the mode specified here. - This field has no effect on Windows. - This field is alpha and requires EmptyDirVolumeMode featuregate to be enabled. - format: int32 - type: integer - sizeLimit: - anyOf: - - type: integer - - type: string - description: |- - sizeLimit is the total amount of local storage required for this EmptyDir volume. - The size limit is also applicable for memory medium. - The maximum usage on memory medium EmptyDir would be the minimum value between - the SizeLimit specified here and the sum of memory limits of all containers in a pod. - The default is nil which means that the limit is undefined. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - hostPath: - description: |- - Represents a host path mapped into a pod. - Host path volumes do not support ownership management or SELinux relabeling. - properties: - path: - description: |- - path of the directory on the host. - If the path is a symlink, it will follow the link to the real path. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - type: - description: |- - type for HostPath Volume - Defaults to "" - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - required: - - path - type: object - mountPath: - type: string - name: - type: string - persistentVolumeClaim: - description: |- - PersistentVolumeClaimVolumeSource references the user's PVC in the same namespace. - This volume finds the bound PV and mounts that volume for the pod. A - PersistentVolumeClaimVolumeSource is, essentially, a wrapper around another - type of volume that is owned by someone else (the system). - properties: - claimName: - description: |- - claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - type: string - readOnly: - description: |- - readOnly Will force the ReadOnly setting in VolumeMounts. - Default false. - type: boolean - required: - - claimName - type: object - readOnly: - type: boolean - required: - - mountPath - - name - type: object - maxItems: 16 - type: array workingDir: type: string type: object @@ -563,15 +221,11 @@ spec: - backend - backendActiveDeadline - id - - maxAttempts - order - - resultBounds - runnerImage - script type: object x-kubernetes-validations: - - message: maxAttempts must be exactly 1 - rule: self.maxAttempts == 1 - message: backendActiveDeadline must be positive rule: duration(self.backendActiveDeadline) > duration('0s') - message: runtimeStage is required when the script requires a runtime @@ -731,10 +385,6 @@ spec: - count - selector type: object - purpose: - enum: - - FullCertification - type: string tolerations: items: description: |- diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml index 4287392cab..f4f4512d66 100644 --- a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml @@ -68,31 +68,25 @@ spec: type: string kind: type: string - queue: - type: string required: - apiVersion - kind type: object backendActiveDeadline: type: string + gpuCount: + format: int32 + type: integer id: maxLength: 63 minLength: 1 pattern: ^R(0[1-9]|1[0-5])$ type: string - maxAttempts: - format: int32 - type: integer order: format: int32 maximum: 14 minimum: 0 type: integer - parameters: - additionalProperties: - type: string - type: object participantPolicy: properties: excludedRoles: @@ -106,34 +100,6 @@ spec: maxItems: 16 type: array type: object - resultBounds: - properties: - backendEvidenceBytes: - format: int32 - maximum: 65536 - minimum: 1 - type: integer - backendResourcesBytes: - format: int32 - maximum: 16384 - minimum: 1 - type: integer - measurementsBytes: - format: int32 - maximum: 16384 - minimum: 1 - type: integer - reasonBytes: - format: int32 - maximum: 4096 - minimum: 1 - type: integer - required: - - backendEvidenceBytes - - backendResourcesBytes - - measurementsBytes - - reasonBytes - type: object runnerImage: type: string runtimeStage: @@ -164,23 +130,39 @@ spec: - digest - key type: object + scriptEnvironment: + additionalProperties: + type: string + type: object workloadPolicy: properties: + capabilities: + items: + description: Capability represent POSIX capabilities type + type: string + type: array hostNetwork: type: boolean - kueueTopology: - properties: - requiredLevel: - description: |- - RequiredLevel is the Node label key for the minimum topology domain in which - Kueue must place the participant PodSet. - maxLength: 316 - minLength: 1 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - requiredLevel - type: object + hostPaths: + items: + properties: + mountPath: + type: string + name: + type: string + path: + type: string + type: + type: string + required: + - mountPath + - name + - path + type: object + maxItems: 16 + type: array + privileged: + type: boolean resources: description: ResourceRequirements describes the compute resource requirements. @@ -242,331 +224,6 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object - results: - description: |- - Represents an empty directory for a pod. - Empty directory volumes support ownership management and SELinux relabeling. - properties: - medium: - description: |- - medium represents what type of storage medium should back this directory. - The default is "" which means to use the node's default medium. - Must be an empty string (default) or Memory. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - type: string - mode: - description: |- - mode specifies the permission bits for the emptyDir directory, in numeric - notation (e.g., 0755, 01777). Must be a value between 0000 and 01777. - If not specified, defaults to 0777. - This might be in conflict with other options that affect the file - mode, like fsGroup. If fsGroup is specified, the fsGroup permissions - will override the mode specified here. - This field has no effect on Windows. - This field is alpha and requires EmptyDirVolumeMode featuregate to be enabled. - format: int32 - type: integer - sizeLimit: - anyOf: - - type: integer - - type: string - description: |- - sizeLimit is the total amount of local storage required for this EmptyDir volume. - The size limit is also applicable for memory medium. - The maximum usage on memory medium EmptyDir would be the minimum value between - the SizeLimit specified here and the sum of memory limits of all containers in a pod. - The default is nil which means that the limit is undefined. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - securityContext: - description: |- - SecurityContext holds security configuration that will be applied to a container. - Some fields are present in both SecurityContext and PodSecurityContext. When both - are set, the values in SecurityContext take precedence. - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities - type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities - type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies - to the container. - type: string - role: - description: Role is a SELinux role label that applies - to the container. - type: string - type: - description: Type is a SELinux type label that applies - to the container. - type: string - user: - description: User is a SELinux user label that applies - to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name - of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - volumes: - items: - properties: - emptyDir: - description: |- - Represents an empty directory for a pod. - Empty directory volumes support ownership management and SELinux relabeling. - properties: - medium: - description: |- - medium represents what type of storage medium should back this directory. - The default is "" which means to use the node's default medium. - Must be an empty string (default) or Memory. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - type: string - mode: - description: |- - mode specifies the permission bits for the emptyDir directory, in numeric - notation (e.g., 0755, 01777). Must be a value between 0000 and 01777. - If not specified, defaults to 0777. - This might be in conflict with other options that affect the file - mode, like fsGroup. If fsGroup is specified, the fsGroup permissions - will override the mode specified here. - This field has no effect on Windows. - This field is alpha and requires EmptyDirVolumeMode featuregate to be enabled. - format: int32 - type: integer - sizeLimit: - anyOf: - - type: integer - - type: string - description: |- - sizeLimit is the total amount of local storage required for this EmptyDir volume. - The size limit is also applicable for memory medium. - The maximum usage on memory medium EmptyDir would be the minimum value between - the SizeLimit specified here and the sum of memory limits of all containers in a pod. - The default is nil which means that the limit is undefined. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - hostPath: - description: |- - Represents a host path mapped into a pod. - Host path volumes do not support ownership management or SELinux relabeling. - properties: - path: - description: |- - path of the directory on the host. - If the path is a symlink, it will follow the link to the real path. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - type: - description: |- - type for HostPath Volume - Defaults to "" - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - required: - - path - type: object - mountPath: - type: string - name: - type: string - persistentVolumeClaim: - description: |- - PersistentVolumeClaimVolumeSource references the user's PVC in the same namespace. - This volume finds the bound PV and mounts that volume for the pod. A - PersistentVolumeClaimVolumeSource is, essentially, a wrapper around another - type of volume that is owned by someone else (the system). - properties: - claimName: - description: |- - claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - type: string - readOnly: - description: |- - readOnly Will force the ReadOnly setting in VolumeMounts. - Default false. - type: boolean - required: - - claimName - type: object - readOnly: - type: boolean - required: - - mountPath - - name - type: object - maxItems: 16 - type: array workingDir: type: string type: object @@ -574,15 +231,11 @@ spec: - backend - backendActiveDeadline - id - - maxAttempts - order - - resultBounds - runnerImage - script type: object x-kubernetes-validations: - - message: maxAttempts must be exactly 1 - rule: self.maxAttempts == 1 - message: backendActiveDeadline must be positive rule: duration(self.backendActiveDeadline) > duration('0s') - message: runtimeStage is required when the script requires a runtime @@ -738,10 +391,6 @@ spec: - name - uid type: object - purpose: - enum: - - FullCertification - type: string requestNodeName: type: string tolerations: @@ -841,9 +490,6 @@ spec: rule: self.executionPolicy == oldSelf.executionPolicy - message: workflow is immutable rule: self.workflow == oldSelf.workflow - - message: purpose is immutable - rule: has(self.purpose) == has(oldSelf.purpose) && (!has(self.purpose) - || self.purpose == oldSelf.purpose) - message: evidencePath is immutable rule: self.evidencePath == oldSelf.evidencePath - message: deadline is immutable diff --git a/internal/signal/controller.go b/internal/signal/controller.go index b0d2fc5487..692c4481ea 100644 --- a/internal/signal/controller.go +++ b/internal/signal/controller.go @@ -79,11 +79,11 @@ var reservedScriptEnvironmentNames = map[string]struct{}{ } var workflowOutputLimits = map[string]int{ - "backend-resources": 16 * 1024, + "backend-resources": signaladapter.MaxBackendOutputBytes, "backend-evidence": signaladapter.MaxBackendEvidenceBytes, - "cleanup-failure": 64, - "measurements": 16 * 1024, - "reason": 4 * 1024, + "cleanup-failure": signaladapter.MaxCleanupFailureBytes, + "measurements": signaladapter.MaxMeasurementsBytes, + "reason": signaladapter.MaxReasonBytes, "verdict": 32, } @@ -224,10 +224,6 @@ func (r *AdmissionReconciler) admitNode(ctx context.Context, node *corev1.Node, return fmt.Errorf("profile %s participant constraints: %w", profile.Name, err) } - if profile.Spec.Purpose != "" && profile.Spec.Purpose != signalv1.PurposeFullCertification { - return fmt.Errorf("profile %s purpose must be %s", profile.Name, signalv1.PurposeFullCertification) - } - for index := range checks { check := &checks[index] if err := signalv1.ValidateRuntimeContract( @@ -243,17 +239,13 @@ func (r *AdmissionReconciler) admitNode(ctx context.Context, node *corev1.Node, return fmt.Errorf("profile %s check %s runner image must use an immutable sha256 digest", profile.Name, check.ID) } - if _, err := scriptEnvironment(check.Parameters); err != nil { - return fmt.Errorf("profile %s check %s parameters: %w", profile.Name, check.ID, err) + if err := validateScriptEnvironment(check.ScriptEnvironment); err != nil { + return fmt.Errorf("profile %s check %s script environment: %w", profile.Name, check.ID, err) } if err := signalworkload.Validate(check.WorkloadPolicy); err != nil { return fmt.Errorf("profile %s check %s workload policy: %w", profile.Name, check.ID, err) } - - if check.WorkloadPolicy.KueueTopology != nil && check.Backend.Queue == "" { - return fmt.Errorf("profile %s check %s workload policy kueueTopology requires a backend queue", profile.Name, check.ID) - } } participants, err := r.selectParticipants(ctx, node, &profile) @@ -267,42 +259,55 @@ func (r *AdmissionReconciler) admitNode(ctx context.Context, node *corev1.Node, return fmt.Errorf("check %s participants: %w", checks[index].ID, err) } + if len(checkParticipants) > signaladapter.MaxCheckParticipants { + return fmt.Errorf( + "check %s selects %d participants, maximum is %d", + checks[index].ID, + len(checkParticipants), + signaladapter.MaxCheckParticipants, + ) + } + minimumEvidenceBytes := signaladapter.MinimumBackendEvidenceBytes(len(checkParticipants)) - if int(checks[index].ResultBounds.BackendEvidenceBytes) < minimumEvidenceBytes { + if signaladapter.MaxBackendEvidenceBytes < minimumEvidenceBytes { return fmt.Errorf( - "check %s backendEvidenceBytes must be at least %d for %d participants", + "check %s fixed backend evidence limit %d cannot represent %d participants; requires at least %d bytes", checks[index].ID, - minimumEvidenceBytes, + signaladapter.MaxBackendEvidenceBytes, len(checkParticipants), + minimumEvidenceBytes, ) } minimumReasonBytes := signaladapter.MinimumPartialReasonBytes() - if int(checks[index].ResultBounds.ReasonBytes) < minimumReasonBytes { + if signaladapter.MaxReasonBytes < minimumReasonBytes { return fmt.Errorf( - "check %s reasonBytes must be at least %d for mandatory failure output", + "check %s fixed reason limit %d cannot represent mandatory failure output; requires at least %d bytes", checks[index].ID, + signaladapter.MaxReasonBytes, minimumReasonBytes, ) } minimumMeasurementsBytes := signaladapter.MinimumPartialMeasurementsBytes(len(checkParticipants)) - if int(checks[index].ResultBounds.MeasurementsBytes) < minimumMeasurementsBytes { + if signaladapter.MaxMeasurementsBytes < minimumMeasurementsBytes { return fmt.Errorf( - "check %s measurementsBytes must be at least %d for %d participants", + "check %s fixed measurements limit %d cannot represent %d participants; requires at least %d bytes", checks[index].ID, - minimumMeasurementsBytes, + signaladapter.MaxMeasurementsBytes, len(checkParticipants), + minimumMeasurementsBytes, ) } minimumBackendResourcesBytes := signaladapter.MinimumPartialBackendResourcesBytes(len(checkParticipants)) - if int(checks[index].ResultBounds.BackendResourcesBytes) < minimumBackendResourcesBytes { + if signaladapter.MaxBackendOutputBytes < minimumBackendResourcesBytes { return fmt.Errorf( - "check %s backendResourcesBytes must be at least %d for %d participants", + "check %s fixed backend resource limit %d cannot represent %d participants; requires at least %d bytes", checks[index].ID, - minimumBackendResourcesBytes, + signaladapter.MaxBackendOutputBytes, len(checkParticipants), + minimumBackendResourcesBytes, ) } @@ -310,8 +315,8 @@ func (r *AdmissionReconciler) admitNode(ctx context.Context, node *corev1.Node, return fmt.Errorf("check %s workload policy: %w", checks[index].ID, err) } - if _, _, err := adapterParameterValues(checks[index].Parameters, checks[index].Backend.Kind); err != nil { - return fmt.Errorf("check %s parameters: %w", checks[index].ID, err) + if err := validateGPUCount(checks[index].GPUCount, checks[index].Backend.Kind); err != nil { + return fmt.Errorf("check %s GPU count: %w", checks[index].ID, err) } } @@ -329,32 +334,6 @@ func (r *AdmissionReconciler) admitNode(ctx context.Context, node *corev1.Node, } }() - if len(participants) == 1 { - var participantNode corev1.Node - - reader := r.APIReader - if reader == nil { - reader = r.Client - } - - if err := reader.Get(ctx, client.ObjectKey{Name: participants[0].NodeName}, &participantNode); err != nil { - return fmt.Errorf("read admitted participant inventory: %w", err) - } - - gpuResource := corev1.ResourceName("nvidia.com/gpu") - gpuCapacity := participantNode.Status.Capacity[gpuResource] - gpuAllocatable := participantNode.Status.Allocatable[gpuResource] - - for index := range checks { - if checks[index].Parameters == nil { - checks[index].Parameters = map[string]string{} - } - - checks[index].Parameters["observed-gpu-capacity"] = gpuCapacity.String() - checks[index].Parameters["observed-gpu-allocatable"] = gpuAllocatable.String() - } - } - snapshot, checkSnapshots, err := r.createSnapshot(ctx, runName, checks) if err != nil { return err @@ -387,7 +366,6 @@ func (r *AdmissionReconciler) admitNode(ctx context.Context, node *corev1.Node, Checks: checkSnapshots, ExecutionPolicy: profile.Spec.ExecutionPolicy, Workflow: profile.Spec.Workflow, - Purpose: signalv1.PurposeFullCertification, EvidencePath: strings.Trim(profile.Spec.EvidencePath, "/") + "/" + runName + ".json", Deadline: profile.Spec.Deadline, Tolerations: profile.Spec.Tolerations, @@ -1117,11 +1095,10 @@ func (r *AdmissionReconciler) createSnapshot( Backend: check.Backend, WorkloadPolicy: *check.WorkloadPolicy.DeepCopy(), RuntimeStage: copyRuntimeStage(check.RuntimeStage), - Parameters: maps.Clone(check.Parameters), + GPUCount: check.GPUCount, + ScriptEnvironment: maps.Clone(check.ScriptEnvironment), BackendActiveDeadline: check.BackendActiveDeadline, - ResultBounds: check.ResultBounds, ParticipantPolicy: check.ParticipantPolicy, - MaxAttempts: check.MaxAttempts, } } @@ -1368,12 +1345,7 @@ func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured return nil, fmt.Errorf("encode check %s participants: %w", check.ID, marshalErr) } - environment, environmentErr := scriptEnvironment(check.Parameters) - if environmentErr != nil { - return nil, fmt.Errorf("check %s: %w", check.ID, environmentErr) - } - - environmentJSON, marshalErr := json.Marshal(environment) + environmentJSON, marshalErr := json.Marshal(check.ScriptEnvironment) if marshalErr != nil { return nil, fmt.Errorf("encode check %s script environment: %w", check.ID, marshalErr) } @@ -1383,11 +1355,6 @@ func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured return nil, fmt.Errorf("encode check %s workload policy: %w", check.ID, encodeErr) } - gpuCount, privileged, err := adapterParameterValues(check.Parameters, check.Backend.Kind) - if err != nil { - return nil, fmt.Errorf("check %s parameters: %w", check.ID, err) - } - taskName := checkTaskName(check) task := map[string]any{ "name": taskName, @@ -1405,8 +1372,7 @@ func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured string(tolerationsJSON), string(environmentJSON), workloadPolicyJSON, - gpuCount, - privileged, + strconv.FormatInt(int64(check.GPUCount), 10), ), }, } @@ -1481,7 +1447,6 @@ func checkWorkflowParameters( scriptEnvironmentJSON string, workloadPolicyJSON string, gpuCount string, - privileged string, ) []any { return []any{ map[string]any{"name": "run-name", "value": checkRunName(run.Name, check.ID)}, @@ -1502,45 +1467,21 @@ func checkWorkflowParameters( "name": "backend-active-deadline-seconds", "value": fmt.Sprintf("%d", int64(check.BackendActiveDeadline.Seconds())), }, - map[string]any{"name": "queue", "value": check.Backend.Queue}, map[string]any{"name": "runner-image", "value": check.RunnerImage}, map[string]any{"name": "tolerations", "value": tolerationsJSON}, map[string]any{"name": "script-environment", "value": scriptEnvironmentJSON}, map[string]any{"name": "workload-policy", "value": workloadPolicyJSON}, - map[string]any{"name": "mpi-payload-mode", "value": check.Parameters["mpi-payload-mode"]}, - map[string]any{"name": "expected-gpu-product-regex", "value": check.Parameters["expected-gpu-product-regex"]}, - map[string]any{"name": "expected-gpu-count", "value": check.Parameters["expected-gpu-count"]}, - map[string]any{"name": "observed-gpu-capacity", "value": check.Parameters["observed-gpu-capacity"]}, - map[string]any{"name": "observed-gpu-allocatable", "value": check.Parameters["observed-gpu-allocatable"]}, - map[string]any{"name": "backend-evidence-bytes", "value": strconv.Itoa(int(check.ResultBounds.BackendEvidenceBytes))}, map[string]any{"name": "gpu-resource-count", "value": gpuCount}, - map[string]any{"name": "privileged", "value": privileged}, map[string]any{"name": "attempt", "value": fmt.Sprintf("%d", attempt)}, } } -func adapterParameterValues(parameters map[string]string, backendKind string) (string, string, error) { - gpuCount := parameters["gpu-resource-count"] - if gpuCount == "" { - gpuCount = "0" +func validateGPUCount(gpuCount int32, backendKind string) error { + if gpuCount < 0 || (backendKind == "MPIJob" && gpuCount == 0) { + return errors.New("must be nonnegative and positive for MPIJob") } - count, err := strconv.ParseInt(gpuCount, 10, 64) - if err != nil || count < 0 || (backendKind == "MPIJob" && count == 0) { - return "", "", errors.New("gpu-resource-count must be nonnegative and positive for MPIJob") - } - - privileged := parameters["privileged"] - if privileged == "" { - privileged = "false" - } - - value, err := strconv.ParseBool(privileged) - if err != nil { - return "", "", fmt.Errorf("privileged must be a boolean: %w", err) - } - - return strconv.FormatInt(count, 10), strconv.FormatBool(value), nil + return nil } func validEnvironmentName(value string) bool { @@ -1560,40 +1501,29 @@ func validEnvironmentName(value string) bool { return true } -func scriptEnvironment(parameters map[string]string) (map[string]string, error) { - keys := make([]string, 0, len(parameters)) - for key := range parameters { +func validateScriptEnvironment(environment map[string]string) error { + keys := make([]string, 0, len(environment)) + for key := range environment { keys = append(keys, key) } sort.Strings(keys) - environment := make(map[string]string, len(parameters)) - parameterByEnvironment := make(map[string]string, len(parameters)) - - for _, key := range keys { - environmentName := strings.ToUpper(strings.ReplaceAll(key, "-", "_")) - if !validEnvironmentName(environmentName) { - return nil, fmt.Errorf("profile parameter %q cannot be mapped to a script environment variable", key) - } - - if strings.HasPrefix(environmentName, "PROJECT_SIGNAL_") { - return nil, fmt.Errorf("profile parameter %q maps to reserved environment variable %s", key, environmentName) + for _, name := range keys { + if !validEnvironmentName(name) { + return fmt.Errorf("name %q is invalid", name) } - if _, reserved := reservedScriptEnvironmentNames[environmentName]; reserved { - return nil, fmt.Errorf("profile parameter %q maps to reserved environment variable %s", key, environmentName) + if strings.HasPrefix(name, "PROJECT_SIGNAL_") { + return fmt.Errorf("name %q is reserved", name) } - if previous, exists := parameterByEnvironment[environmentName]; exists { - return nil, fmt.Errorf("profile parameters %q and %q both map to %s", previous, key, environmentName) + if _, reserved := reservedScriptEnvironmentNames[name]; reserved { + return fmt.Errorf("name %q is reserved", name) } - - parameterByEnvironment[environmentName] = key - environment[environmentName] = parameters[key] } - return environment, nil + return nil } func (r *RunReconciler) finishGrouped( @@ -1946,7 +1876,7 @@ func terminalCheckStatuses( continue } - outputs, outputErr := nodeWorkflowParameters(node, check.ResultBounds) + outputs, outputErr := nodeWorkflowParameters(node) if outputErr != nil { status.Phase = signalv1.CheckPhaseInconclusive status.Verdict = "Inconclusive" @@ -2093,10 +2023,7 @@ func exactCheckNode( return match, count } -func nodeWorkflowParameters( - node map[string]any, - bounds signalv1.ResultBounds, -) (map[string]string, error) { +func nodeWorkflowParameters(node map[string]any) (map[string]string, error) { outputs, ok := node["outputs"].(map[string]any) if !ok { return nil, errors.New("task outputs are missing or malformed") @@ -2109,10 +2036,10 @@ func nodeWorkflowParameters( limits := map[string]int{ "verdict": workflowOutputLimits["verdict"], - "reason": int(bounds.ReasonBytes), - "measurements": int(bounds.MeasurementsBytes), - "backend-resources": int(bounds.BackendResourcesBytes), - "backend-evidence": int(bounds.BackendEvidenceBytes), + "reason": workflowOutputLimits["reason"], + "measurements": workflowOutputLimits["measurements"], + "backend-resources": workflowOutputLimits["backend-resources"], + "backend-evidence": workflowOutputLimits["backend-evidence"], "cleanup-failure": workflowOutputLimits["cleanup-failure"], } @@ -2230,7 +2157,6 @@ func (r *RunReconciler) evidenceDocument( "checkStatus": run.Status.Checks, "scriptContents": scriptContents, "executionPolicy": run.Spec.ExecutionPolicy, - "purpose": run.Spec.Purpose, "executionState": executionState, "verdict": verdict, "reason": reason, @@ -3062,7 +2988,7 @@ func profileChecks(profile *signalv1.CertificationProfile) []signalv1.Certificat checks[index] = profile.Spec.Checks[index] checks[index].WorkloadPolicy = *profile.Spec.Checks[index].WorkloadPolicy.DeepCopy() checks[index].RuntimeStage = copyRuntimeStage(profile.Spec.Checks[index].RuntimeStage) - checks[index].Parameters = maps.Clone(profile.Spec.Checks[index].Parameters) + checks[index].ScriptEnvironment = maps.Clone(profile.Spec.Checks[index].ScriptEnvironment) checks[index].ParticipantPolicy.RequiredRoles = slices.Clone( profile.Spec.Checks[index].ParticipantPolicy.RequiredRoles, ) @@ -3079,10 +3005,7 @@ func validateCheckPlan(checks []signalv1.CertificationCheck) error { return fmt.Errorf("must contain exactly %d checks", groupedCheckCount) } - var ( - seen = make(map[signalv1.CheckID]struct{}, len(checks)) - outputBytes int64 - ) + seen := make(map[signalv1.CheckID]struct{}, len(checks)) for index := range checks { check := &checks[index] @@ -3104,25 +3027,15 @@ func validateCheckPlan(checks []signalv1.CertificationCheck) error { seen[check.ID] = struct{}{} - if check.MaxAttempts != 1 { - return fmt.Errorf("check %s maxAttempts must be exactly 1", check.ID) - } - - if err := validateResultBounds(check.ID, check.ResultBounds); err != nil { - return err - } - - outputBytes += workflowOutputBytes(check.ResultBounds) - if err := validateCheckParticipantPolicy(check.ID, check.ParticipantPolicy); err != nil { return err } } - if outputBytes > maximumWorkflowOutputBytes { + if fixedWorkflowOutputBytes() > maximumWorkflowOutputBytes { return fmt.Errorf( - "aggregate check output budget is %d bytes, maximum is %d", - outputBytes, + "fixed aggregate check output budget is %d bytes, maximum is %d", + fixedWorkflowOutputBytes(), maximumWorkflowOutputBytes, ) } @@ -3135,10 +3048,7 @@ func validateRunCheckPlan(checks []signalv1.CertificationCheckSnapshot) error { return fmt.Errorf("run must contain exactly %d checks", groupedCheckCount) } - var ( - seen = make(map[signalv1.CheckID]struct{}, len(checks)) - outputBytes int64 - ) + seen := make(map[signalv1.CheckID]struct{}, len(checks)) for index := range checks { check := &checks[index] @@ -3155,22 +3065,12 @@ func validateRunCheckPlan(checks []signalv1.CertificationCheckSnapshot) error { } seen[check.ID] = struct{}{} - - if check.MaxAttempts != 1 { - return fmt.Errorf("run check %s maxAttempts must be exactly 1", check.ID) - } - - if err := validateResultBounds(check.ID, check.ResultBounds); err != nil { - return err - } - - outputBytes += workflowOutputBytes(check.ResultBounds) } - if outputBytes > maximumWorkflowOutputBytes { + if fixedWorkflowOutputBytes() > maximumWorkflowOutputBytes { return fmt.Errorf( - "run aggregate check output budget is %d bytes, maximum is %d", - outputBytes, + "fixed run aggregate check output budget is %d bytes, maximum is %d", + fixedWorkflowOutputBytes(), maximumWorkflowOutputBytes, ) } @@ -3178,39 +3078,17 @@ func validateRunCheckPlan(checks []signalv1.CertificationCheckSnapshot) error { return nil } -func workflowOutputBytes(bounds signalv1.ResultBounds) int64 { - return int64( - workflowOutputLimits["verdict"]+ - workflowOutputLimits["cleanup-failure"], - ) + - int64(bounds.ReasonBytes) + - int64(bounds.MeasurementsBytes) + - int64(bounds.BackendResourcesBytes) + - int64(bounds.BackendEvidenceBytes) -} - -func validateResultBounds(id signalv1.CheckID, bounds signalv1.ResultBounds) error { - switch { - case bounds.ReasonBytes < 1 || bounds.ReasonBytes > int32(workflowOutputLimits["reason"]): - return fmt.Errorf("check %s reasonBytes must be between 1 and %d", id, workflowOutputLimits["reason"]) - case bounds.MeasurementsBytes < 1 || bounds.MeasurementsBytes > int32(workflowOutputLimits["measurements"]): - return fmt.Errorf( - "check %s measurementsBytes must be between 1 and %d", - id, - workflowOutputLimits["measurements"], - ) - case bounds.BackendResourcesBytes < 1 || - bounds.BackendResourcesBytes > int32(workflowOutputLimits["backend-resources"]): - return fmt.Errorf( - "check %s backendResourcesBytes must be between 1 and %d", - id, - workflowOutputLimits["backend-resources"], - ) - case bounds.BackendEvidenceBytes < 1 || bounds.BackendEvidenceBytes > 64*1024: - return fmt.Errorf("check %s backendEvidenceBytes must be between 1 and 65536", id) - } +func fixedWorkflowOutputBytes() int64 { + perCheck := int64( + workflowOutputLimits["verdict"] + + workflowOutputLimits["cleanup-failure"] + + workflowOutputLimits["reason"] + + workflowOutputLimits["measurements"] + + workflowOutputLimits["backend-resources"] + + workflowOutputLimits["backend-evidence"], + ) - return nil + return groupedCheckCount * perCheck } func validateCheckParticipantPolicy(id signalv1.CheckID, policy signalv1.CheckParticipantPolicy) error { diff --git a/internal/signaladapter/adapter.go b/internal/signaladapter/adapter.go index 096712e854..abfe4bbd1a 100644 --- a/internal/signaladapter/adapter.go +++ b/internal/signaladapter/adapter.go @@ -9,7 +9,6 @@ import ( "errors" "fmt" "io" - "maps" "regexp" "slices" "strconv" @@ -28,17 +27,19 @@ import ( const ( MaxParticipants = 64 - MaxReasonBytes = 4 * 1024 + MaxCheckParticipants = 8 + MaxReasonBytes = 512 MaxMeasurementsBytes = 16 * 1024 - MaxBackendOutputBytes = 16 * 1024 - MaxBackendEvidenceBytes = 64 * 1024 + MaxBackendOutputBytes = 8 * 1024 + MaxBackendEvidenceBytes = 24 * 1024 MaxCleanupFailureBytes = 64 maxMPIWorkerReadyWait = int64(30) maxMPIWorkerSlots = int64(1<<31 - 1) + mpiBackoffLimit = int64(0) + mpiTTLSeconds = int64(3600) mpiHostfilePath = "/tmp/project-signal-hostfile" mpiKnownHostsPath = "/tmp/project-signal-known-hosts" resultPrefix = "PROJECT_SIGNAL_RESULT_V2 " - requiredTopologyKey = "kueue.x-k8s.io/podset-required-topology" ) var runnerImagePattern = regexp.MustCompile(`^[^@\s]+@sha256:[0-9a-f]{64}$`) @@ -76,16 +77,11 @@ type Config struct { BackendAPIVersion string BackendKind string BackendDeadlineSeconds int64 - Queue string TolerationsJSON string ScriptEnvironmentJSON string WorkloadPolicyJSON string GPUResourceCount string - Privileged bool Attempt int - BackoffLimit int64 - TTLSeconds int64 - BackendEvidenceBytes int } type Result struct { @@ -129,8 +125,8 @@ func ParseParticipants( participantsJSON string, participantCount int, ) ([]signalv1.Participant, error) { - if participantCount < 1 || participantCount > MaxParticipants { - return nil, fmt.Errorf("participant-count must be between 1 and %d", MaxParticipants) + if participantCount < 1 || participantCount > MaxCheckParticipants { + return nil, fmt.Errorf("participant-count must be between 1 and %d", MaxCheckParticipants) } if strings.TrimSpace(participantsJSON) == "" { @@ -271,37 +267,14 @@ func ValidateConfig(config Config) ([]signalv1.Participant, []corev1.Toleration, ) } - if config.Queue != "" { - if errors := validation.IsDNS1123Subdomain(config.Queue); len(errors) > 0 { - return nil, nil, nil, fmt.Errorf("queue is invalid: %s", strings.Join(errors, ", ")) - } - } - - if config.BackoffLimit < 0 { - return nil, nil, nil, errors.New("backoff-limit must not be negative") - } - - if config.TTLSeconds < 0 { - return nil, nil, nil, errors.New("ttl-seconds-after-finished must not be negative") - } - if config.Attempt < 1 { return nil, nil, nil, errors.New("attempt must be positive") } - if config.BackendEvidenceBytes < 0 || config.BackendEvidenceBytes > MaxBackendEvidenceBytes { - return nil, nil, nil, fmt.Errorf("backend-evidence-bytes must be between 1 and %d when set", MaxBackendEvidenceBytes) - } - - policy, err := signalworkload.Decode(config.WorkloadPolicyJSON) - if err != nil { + if _, err := signalworkload.Decode(config.WorkloadPolicyJSON); err != nil { return nil, nil, nil, err } - if policy.KueueTopology != nil && config.Queue == "" { - return nil, nil, nil, errors.New("workload policy kueueTopology requires a queue") - } - participants, err := ParseParticipants( config.ParticipantsJSON, config.ParticipantCount, @@ -311,11 +284,12 @@ func ValidateConfig(config Config) ([]signalv1.Participant, []corev1.Toleration, } minimumEvidenceBytes := MinimumBackendEvidenceBytes(len(participants)) - if config.BackendEvidenceBytes > 0 && config.BackendEvidenceBytes < minimumEvidenceBytes { + if minimumEvidenceBytes > MaxBackendEvidenceBytes { return nil, nil, nil, fmt.Errorf( - "backend-evidence-bytes must be at least %d for %d participants", - minimumEvidenceBytes, + "fixed backend evidence limit %d cannot represent %d participants; requires at least %d bytes", + MaxBackendEvidenceBytes, len(participants), + minimumEvidenceBytes, ) } @@ -329,24 +303,6 @@ func ValidateConfig(config Config) ([]signalv1.Participant, []corev1.Toleration, return nil, nil, nil, fmt.Errorf("decode script-environment: %w", err) } - if value := environment["BACKEND_BACKOFF_LIMIT"]; value != "" { - parsed, err := strconv.ParseInt(value, 10, 64) - if err != nil || parsed < 0 { - return nil, nil, nil, errors.New("BACKEND_BACKOFF_LIMIT must be a nonnegative integer") - } - - config.BackoffLimit = parsed - } - - if value := environment["BACKEND_TTL_SECONDS_AFTER_FINISHED"]; value != "" { - parsed, err := strconv.ParseInt(value, 10, 64) - if err != nil || parsed < 0 { - return nil, nil, nil, errors.New("BACKEND_TTL_SECONDS_AFTER_FINISHED must be a nonnegative integer") - } - - config.TTLSeconds = parsed - } - environmentNames := make([]string, 0, len(environment)) for name := range environment { environmentNames = append(environmentNames, name) @@ -406,25 +362,6 @@ func RenderResources( return nil, errors.New("workload policy must not set nvidia.com/gpu limits; use gpu-resource-count") } - var policyEnvironment map[string]string - if err := decodeJSONObject(config.ScriptEnvironmentJSON, &policyEnvironment); err != nil { - return nil, fmt.Errorf("decode adapter policy: %w", err) - } - - if value := policyEnvironment["BACKEND_BACKOFF_LIMIT"]; value != "" { - config.BackoffLimit, err = strconv.ParseInt(value, 10, 64) - if err != nil { - return nil, fmt.Errorf("parse BACKEND_BACKOFF_LIMIT: %w", err) - } - } - - if value := policyEnvironment["BACKEND_TTL_SECONDS_AFTER_FINISHED"]; value != "" { - config.TTLSeconds, err = strconv.ParseInt(value, 10, 64) - if err != nil { - return nil, fmt.Errorf("parse BACKEND_TTL_SECONDS_AFTER_FINISHED: %w", err) - } - } - for _, participant := range participants { var object any @@ -945,9 +882,6 @@ func ordinaryPod( gpuCount int64, ) *corev1.Pod { labels := resourceLabels(config, participant) - if config.Queue != "" { - labels = withLabel(labels, "kueue.x-k8s.io/queue-name", config.Queue) - } env := workloadEnvironment(config, participant, environment, true) container := workloadContainer( @@ -968,7 +902,6 @@ func ordinaryPod( Name: resourceName(config.RunName, participant.Index), Namespace: config.Namespace, Labels: labels, - Annotations: workloadAnnotations(policy), OwnerReferences: []metav1.OwnerReference{workflow}, }, Spec: corev1.PodSpec{ @@ -997,9 +930,6 @@ func mpiJob( gpuCount int64, ) (*unstructured.Unstructured, error) { labels := resourceLabels(config, participant) - if config.Queue != "" { - labels = withLabel(labels, "kueue.x-k8s.io/queue-name", config.Queue) - } jobName := resourceNameWithLimit(config.RunName, participant.Index, 63-len("-worker-0")) launcherEnv := workloadEnvironment(config, participant, environment, false) @@ -1039,9 +969,9 @@ func mpiJob( "slotsPerWorker": gpuCount, "runPolicy": map[string]any{ "activeDeadlineSeconds": backendDeadlineSeconds, - "backoffLimit": config.BackoffLimit, + "backoffLimit": mpiBackoffLimit, "cleanPodPolicy": "All", - "ttlSecondsAfterFinished": config.TTLSeconds, + "ttlSecondsAfterFinished": mpiTTLSeconds, }, "mpiReplicaSpecs": map[string]any{ "Launcher": map[string]any{ @@ -1102,8 +1032,7 @@ func podTemplate( template := corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - Annotations: workloadAnnotations(policy), + Labels: labels, }, Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyNever, @@ -1149,14 +1078,6 @@ func participantNodeAffinity(nodeName string, nodeUID types.UID) *corev1.Affinit } } -func workloadAnnotations(policy signalv1.WorkloadPolicy) map[string]string { - if policy.KueueTopology == nil { - return nil - } - - return map[string]string{requiredTopologyKey: policy.KueueTopology.RequiredLevel} -} - func applyPodPolicy(spec *corev1.PodSpec, policy signalv1.WorkloadPolicy) { if !policy.HostNetwork { return @@ -1233,10 +1154,15 @@ func workloadContainer( container.Resources.Limits[corev1.ResourceName("nvidia.com/gpu")] = gpu } - if policy.SecurityContext != nil { - container.SecurityContext = policy.SecurityContext.DeepCopy() - } else if config.Privileged { - container.SecurityContext = &corev1.SecurityContext{Privileged: ptr(true)} + if policy.Privileged || len(policy.Capabilities) > 0 { + container.SecurityContext = &corev1.SecurityContext{ + Privileged: ptr(policy.Privileged), + } + if len(policy.Capabilities) > 0 { + container.SecurityContext.Capabilities = &corev1.Capabilities{ + Add: slices.Clone(policy.Capabilities), + } + } } container.WorkingDir = policy.WorkingDir @@ -1247,18 +1173,16 @@ func workloadContainer( func workloadVolumes(scriptConfigMap string, policy signalv1.WorkloadPolicy) []corev1.Volume { volumes := resultVolumes(scriptConfigMap) - if policy.Results != nil { - volumes[1].EmptyDir = policy.Results.DeepCopy() - } - for index := range policy.Volumes { - volume := &policy.Volumes[index] + for index := range policy.HostPaths { + hostPath := &policy.HostPaths[index] volumes = append(volumes, corev1.Volume{ - Name: volume.Name, + Name: hostPath.Name, VolumeSource: corev1.VolumeSource{ - HostPath: volume.HostPath.DeepCopy(), - EmptyDir: volume.EmptyDir.DeepCopy(), - PersistentVolumeClaim: volume.PersistentVolumeClaim.DeepCopy(), + HostPath: &corev1.HostPathVolumeSource{ + Path: hostPath.Path, + Type: ptr(hostPath.Type), + }, }, }) } @@ -1267,13 +1191,13 @@ func workloadVolumes(scriptConfigMap string, policy signalv1.WorkloadPolicy) []c } func workloadVolumeMounts(policy signalv1.WorkloadPolicy) []corev1.VolumeMount { - mounts := make([]corev1.VolumeMount, 0, len(policy.Volumes)) - for index := range policy.Volumes { - volume := &policy.Volumes[index] + mounts := make([]corev1.VolumeMount, 0, len(policy.HostPaths)) + for index := range policy.HostPaths { + hostPath := &policy.HostPaths[index] mounts = append(mounts, corev1.VolumeMount{ - Name: volume.Name, - MountPath: volume.MountPath, - ReadOnly: volume.ReadOnly, + Name: hostPath.Name, + MountPath: hostPath.MountPath, + ReadOnly: true, }) } @@ -1400,13 +1324,6 @@ func stringMapAny(values map[string]string) map[string]any { return result } -func withLabel(values map[string]string, key, value string) map[string]string { - result := maps.Clone(values) - result[key] = value - - return result -} - func ownerReferenceMap(reference metav1.OwnerReference) map[string]any { return map[string]any{ "apiVersion": reference.APIVersion, diff --git a/internal/signaladapter/backend_evidence.go b/internal/signaladapter/backend_evidence.go index 95cb534c2f..7d1312ffb1 100644 --- a/internal/signaladapter/backend_evidence.go +++ b/internal/signaladapter/backend_evidence.go @@ -172,7 +172,7 @@ func (r *Runner) captureBackendEvidence( evidence.Complete = evidence.Complete && complete } - content, complete, err := marshalBackendEvidence(evidence, config.BackendEvidenceBytes) + content, complete, err := marshalBackendEvidence(evidence, MaxBackendEvidenceBytes) if err != nil { return "", false, err } diff --git a/internal/signaladapter/runner.go b/internal/signaladapter/runner.go index 8315e8cddb..80e7e8d38b 100644 --- a/internal/signaladapter/runner.go +++ b/internal/signaladapter/runner.go @@ -220,7 +220,7 @@ func (r *Runner) Execute(ctx context.Context, config Config) (AggregateOutput, e recordEvidenceCaptureFailure(&output) } - if err := r.persistEvidenceBeforeCleanup(&output, config.BackendEvidenceBytes); err != nil { + if err := r.persistEvidenceBeforeCleanup(&output, MaxBackendEvidenceBytes); err != nil { return finishBackendFailure(fmt.Errorf("persist backend evidence: %w", err)) } @@ -292,7 +292,7 @@ func (r *Runner) backendFailureOutput( BackendEvidence: evidence, } - if err := r.persistEvidenceBeforeCleanup(&output, config.BackendEvidenceBytes); err != nil { + if err := r.persistEvidenceBeforeCleanup(&output, MaxBackendEvidenceBytes); err != nil { return AggregateOutput{}, err } diff --git a/internal/signalworkload/policy.go b/internal/signalworkload/policy.go index 41adfa9d9e..8d12ed5cbc 100644 --- a/internal/signalworkload/policy.go +++ b/internal/signalworkload/policy.go @@ -13,7 +13,6 @@ import ( "strings" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/validation" signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" @@ -75,93 +74,54 @@ func Validate(policy signalv1.WorkloadPolicy) error { return err } - if err := validateSecurityContext(policy.SecurityContext); err != nil { + if err := validateCapabilities(policy.Capabilities); err != nil { return err } - if policy.KueueTopology != nil { - if errors := validation.IsQualifiedName(policy.KueueTopology.RequiredLevel); len(errors) > 0 { - return fmt.Errorf( - "workload policy kueueTopology requiredLevel is invalid: %s", - strings.Join(errors, ", "), - ) - } - } - if policy.WorkingDir != "" { if err := validateAbsolutePath("workingDir", policy.WorkingDir); err != nil { return err } } - if len(policy.Volumes) > 16 { - return errors.New("workload policy must not contain more than 16 volumes") + if len(policy.HostPaths) > 16 { + return errors.New("workload policy must not contain more than 16 host paths") } names := map[string]struct{}{"scripts": {}, "results": {}} mountPaths := map[string]struct{}{"/scripts": {}, "/results": {}} - for index := range policy.Volumes { - volume := &policy.Volumes[index] - if errors := validation.IsDNS1123Label(volume.Name); len(errors) > 0 { - return fmt.Errorf("workload policy volume %d name is invalid: %s", index, strings.Join(errors, ", ")) + for index := range policy.HostPaths { + hostPath := &policy.HostPaths[index] + if errors := validation.IsDNS1123Label(hostPath.Name); len(errors) > 0 { + return fmt.Errorf("workload policy host path %d name is invalid: %s", index, strings.Join(errors, ", ")) } - if _, exists := names[volume.Name]; exists { - return fmt.Errorf("workload policy volume %d duplicates reserved or existing name %q", index, volume.Name) + if _, exists := names[hostPath.Name]; exists { + return fmt.Errorf("workload policy host path %d duplicates reserved or existing name %q", index, hostPath.Name) } - names[volume.Name] = struct{}{} + names[hostPath.Name] = struct{}{} - if err := validateAbsolutePath(fmt.Sprintf("workload policy volume %d mountPath", index), volume.MountPath); err != nil { + if err := validateAbsolutePath(fmt.Sprintf("workload policy host path %d mountPath", index), hostPath.MountPath); err != nil { return err } - if _, exists := mountPaths[volume.MountPath]; exists { + if _, exists := mountPaths[hostPath.MountPath]; exists { return fmt.Errorf( - "workload policy volume %d duplicates reserved or existing mountPath %q", + "workload policy host path %d duplicates reserved or existing mountPath %q", index, - volume.MountPath, + hostPath.MountPath, ) } - mountPaths[volume.MountPath] = struct{}{} - - sources := 0 - if volume.HostPath != nil { - sources++ - - if err := validateAbsolutePath(fmt.Sprintf("workload policy volume %d hostPath", index), volume.HostPath.Path); err != nil { - return err - } - } - - if volume.EmptyDir != nil { - sources++ - - if err := validateQuantity(fmt.Sprintf("workload policy volume %d emptyDir sizeLimit", index), volume.EmptyDir.SizeLimit); err != nil { - return err - } - } - - if volume.PersistentVolumeClaim != nil { - sources++ - - if errors := validation.IsDNS1123Subdomain(volume.PersistentVolumeClaim.ClaimName); len(errors) > 0 { - return fmt.Errorf( - "workload policy volume %d persistentVolumeClaim claimName is invalid: %s", - index, - strings.Join(errors, ", "), - ) - } - } - - if sources != 1 { - return fmt.Errorf("workload policy volume %d must define exactly one source", index) + mountPaths[hostPath.MountPath] = struct{}{} + if err := validateAbsolutePath(fmt.Sprintf("workload policy host path %d path", index), hostPath.Path); err != nil { + return err } } - return validateQuantity("workload policy results sizeLimit", sizeLimit(policy.Results)) + return nil } func validateResources(resources corev1.ResourceRequirements) error { @@ -185,33 +145,19 @@ func validateResources(resources corev1.ResourceRequirements) error { return nil } -func validateSecurityContext(context *corev1.SecurityContext) error { - if context == nil || context.Capabilities == nil { - return nil - } - - seen := map[corev1.Capability]string{} - - for listName, capabilities := range map[string][]corev1.Capability{ - "add": context.Capabilities.Add, - "drop": context.Capabilities.Drop, - } { - for _, capability := range capabilities { - if !capabilityPattern.MatchString(string(capability)) { - return fmt.Errorf("workload policy capability %q is invalid", capability) - } +func validateCapabilities(capabilities []corev1.Capability) error { + seen := map[corev1.Capability]struct{}{} - if previous, exists := seen[capability]; exists { - return fmt.Errorf( - "workload policy capability %q appears in both or repeatedly within %s and %s", - capability, - previous, - listName, - ) - } + for _, capability := range capabilities { + if !capabilityPattern.MatchString(string(capability)) { + return fmt.Errorf("workload policy capability %q is invalid", capability) + } - seen[capability] = listName + if _, exists := seen[capability]; exists { + return fmt.Errorf("workload policy capability %q is repeated", capability) } + + seen[capability] = struct{}{} } return nil @@ -224,19 +170,3 @@ func validateAbsolutePath(name, value string) error { return nil } - -func validateQuantity(name string, quantity *resource.Quantity) error { - if quantity != nil && quantity.Sign() < 0 { - return fmt.Errorf("%s must not be negative", name) - } - - return nil -} - -func sizeLimit(source *corev1.EmptyDirVolumeSource) *resource.Quantity { - if source == nil { - return nil - } - - return source.SizeLimit -} From 5fd802580e0607e62c17ba4c5d4bdcfe057a09d8 Mon Sep 17 00:00:00 2001 From: chokevin Date: Fri, 23 Oct 2026 20:33:40 -0700 Subject: [PATCH 33/36] fix(signal): harden one-time certification lifecycle --- .../v1alpha1/certificationprofile_types.go | 73 +- api/signal/v1alpha1/certificationrun_types.go | 161 +-- api/signal/v1alpha1/runtime_contract.go | 55 +- api/signal/v1alpha1/zz_generated.deepcopy.go | 143 +- cmd/project-signal-adapter/main.go | 18 - .../04-adapter-workflowtemplate.yaml.tmpl | 21 +- deploy/signal/README.md | 645 +++------ ...ounded-cloud.io_certificationprofiles.yaml | 133 +- ....unbounded-cloud.io_certificationruns.yaml | 370 +----- internal/signal/controller.go | 1184 +++++++++++------ internal/signal/evidence.go | 15 +- internal/signaladapter/adapter.go | 145 +- internal/signaladapter/backend_evidence.go | 4 +- internal/signaladapter/runner.go | 180 ++- internal/signalworkload/policy.go | 42 +- 15 files changed, 1380 insertions(+), 1809 deletions(-) diff --git a/api/signal/v1alpha1/certificationprofile_types.go b/api/signal/v1alpha1/certificationprofile_types.go index 0bbfa309e3..d3d5a06da1 100644 --- a/api/signal/v1alpha1/certificationprofile_types.go +++ b/api/signal/v1alpha1/certificationprofile_types.go @@ -9,18 +9,13 @@ import ( ) type ScriptReference struct { - Name string `json:"name"` - Key string `json:"key"` - RequiresRuntimeStage bool `json:"requiresRuntimeStage,omitempty"` + Name string `json:"name"` + Key string `json:"key"` // Digest is the approved algorithm-qualified SHA-256 digest of the script bytes. // +kubebuilder:validation:Pattern=`^sha256:[0-9a-f]{64}$` Digest string `json:"digest"` } -type WorkflowReference struct { - Name string `json:"name"` -} - type CheckID string type ParticipantRoleConstraint struct { @@ -47,57 +42,35 @@ type ParticipantTopologyConstraint struct { type CheckParticipantPolicy struct { // +kubebuilder:validation:MaxItems=16 - RequiredRoles []string `json:"requiredRoles,omitempty"` - // +kubebuilder:validation:MaxItems=16 - ExcludedRoles []string `json:"excludedRoles,omitempty"` + Roles []string `json:"roles,omitempty"` } // +kubebuilder:validation:XValidation:rule="duration(self.backendActiveDeadline) > duration('0s')",message="backendActiveDeadline must be positive" -// +kubebuilder:validation:XValidation:rule="!has(self.script.requiresRuntimeStage) || !self.script.requiresRuntimeStage || has(self.runtimeStage)",message="runtimeStage is required when the script requires a runtime stage" +// +kubebuilder:validation:XValidation:rule="!has(self.scriptEnvironment) || self.scriptEnvironment.all(k, size(k) <= 253 && size(self.scriptEnvironment[k]) <= 4096)",message="scriptEnvironment names must not exceed 253 characters and values must not exceed 4096 characters" type CertificationCheck struct { - // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:MaxLength=63 - // +kubebuilder:validation:Pattern=`^R(0[1-9]|1[0-5])$` - ID CheckID `json:"id"` - // +kubebuilder:validation:Minimum=0 - // +kubebuilder:validation:Maximum=14 - Order int32 `json:"order"` + // ID and Order are derived from the check's position and are never serialized. + ID CheckID `json:"-"` + Order int32 `json:"-"` Script ScriptReference `json:"script"` RunnerImage string `json:"runnerImage"` Backend ExecutionBackend `json:"backend"` WorkloadPolicy WorkloadPolicy `json:"workloadPolicy,omitempty"` - RuntimeStage *RuntimeStage `json:"runtimeStage,omitempty"` // +kubebuilder:validation:Minimum=0 - GPUCount int32 `json:"gpuCount,omitempty"` + GPUCount int32 `json:"gpuCount,omitempty"` + // +kubebuilder:validation:MaxProperties=32 ScriptEnvironment map[string]string `json:"scriptEnvironment,omitempty"` BackendActiveDeadline metav1.Duration `json:"backendActiveDeadline"` ParticipantPolicy CheckParticipantPolicy `json:"participantPolicy,omitempty"` } -type SuiteFailurePolicy string +// +kubebuilder:validation:Enum=Pod;MPIJob +type ExecutionBackend string const ( - SuiteFailurePolicyStop SuiteFailurePolicy = "Stop" - SuiteFailurePolicyContinue SuiteFailurePolicy = "Continue" + ExecutionBackendPod ExecutionBackend = "Pod" + ExecutionBackendMPIJob ExecutionBackend = "MPIJob" ) -type SuiteExecutionPolicy struct { - // +kubebuilder:validation:Enum=Stop;Continue - OnFailure SuiteFailurePolicy `json:"onFailure"` -} - -type RuntimeStage struct { - // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:MaxLength=63 - // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` - Name string `json:"name"` -} - -type ExecutionBackend struct { - APIVersion string `json:"apiVersion"` - Kind string `json:"kind"` -} - type ParticipantPolicy struct { // +kubebuilder:validation:Minimum=1 // +kubebuilder:validation:Maximum=64 @@ -105,14 +78,12 @@ type ParticipantPolicy struct { Selector metav1.LabelSelector `json:"selector"` - RequiredResources corev1.ResourceList `json:"requiredResources,omitempty"` + MinimumAllocatable corev1.ResourceList `json:"minimumAllocatable,omitempty"` } type WorkloadHostPath struct { - Name string `json:"name"` - MountPath string `json:"mountPath"` - Path string `json:"path"` - Type corev1.HostPathType `json:"type,omitempty"` + MountPath string `json:"mountPath"` + Path string `json:"path"` } type WorkloadPolicy struct { @@ -125,24 +96,18 @@ type WorkloadPolicy struct { HostPaths []WorkloadHostPath `json:"hostPaths,omitempty"` } -// +kubebuilder:validation:XValidation:rule="self.checks.all(c, self.checks.exists_one(x, x.id == c.id))",message="checks must contain unique IDs" -// +kubebuilder:validation:XValidation:rule="self.checks.map(c, c.order) == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]",message="checks must use immutable contiguous order 0 through 14" -// +kubebuilder:validation:XValidation:rule="self.checks.map(c, c.id) == ['R01', 'R02', 'R03', 'R04', 'R05', 'R06', 'R07', 'R08', 'R09', 'R10', 'R11', 'R12', 'R13', 'R14', 'R15']",message="checks must use ordered IDs R01 through R15" // +kubebuilder:validation:XValidation:rule="self.checks.all(c, duration(self.deadline) - duration(c.backendActiveDeadline) >= duration('60s'))",message="every backendActiveDeadline must leave at least 60 seconds before deadline" +// +kubebuilder:validation:XValidation:rule="self.checks.map(c, duration(c.backendActiveDeadline) + duration('60s')).sum() <= duration(self.deadline)",message="deadline must cover every sequential backendActiveDeadline plus 60 seconds of evidence time per check" type CertificationProfileSpec struct { Participants ParticipantPolicy `json:"participants"` // +kubebuilder:validation:MaxItems=16 ParticipantRoles []ParticipantRoleConstraint `json:"participantRoles,omitempty"` // +kubebuilder:validation:MaxItems=16 TopologyConstraints []ParticipantTopologyConstraint `json:"topologyConstraints,omitempty"` - Workflow WorkflowReference `json:"workflow"` // +kubebuilder:validation:MinItems=15 // +kubebuilder:validation:MaxItems=15 - Checks []CertificationCheck `json:"checks"` - ExecutionPolicy SuiteExecutionPolicy `json:"executionPolicy"` - EvidencePath string `json:"evidencePath"` - Deadline metav1.Duration `json:"deadline"` - Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + Checks []CertificationCheck `json:"checks"` + Deadline metav1.Duration `json:"deadline"` } // +kubebuilder:object:root=true diff --git a/api/signal/v1alpha1/certificationrun_types.go b/api/signal/v1alpha1/certificationrun_types.go index c94b7a4dae..c9dfc817e8 100644 --- a/api/signal/v1alpha1/certificationrun_types.go +++ b/api/signal/v1alpha1/certificationrun_types.go @@ -9,10 +9,14 @@ import ( "k8s.io/apimachinery/pkg/types" ) +// +kubebuilder:validation:Enum=Pending;Running;Succeeded;Failed;Inconclusive;Canceled type RunPhase string const ( ReservationNodeUIDLabel = "signal.unbounded-cloud.io/reserved-node-uid" + RunConditionCordoned = "Cordoned" + RunConditionCleanup = "WorkloadCleanupComplete" + RunConditionReleased = "Released" RunPhasePending RunPhase = "Pending" RunPhaseRunning RunPhase = "Running" @@ -32,25 +36,22 @@ func (p RunPhase) Terminal() bool { } type Participant struct { - // +kubebuilder:validation:Minimum=0 - Index int32 `json:"index"` + // Index is derived from the participant's position and is never serialized. + Index int32 `json:"-"` NodeName string `json:"nodeName"` NodeUID types.UID `json:"nodeUID"` - Capacity corev1.ResourceList `json:"capacity,omitempty"` Allocatable corev1.ResourceList `json:"allocatable,omitempty"` - // +kubebuilder:validation:MaxItems=16 - Roles []string `json:"roles,omitempty"` + Role string `json:"role,omitempty"` + // +kubebuilder:validation:MaxProperties=16 + TopologyValues map[string]string `json:"topologyValues,omitempty"` } type BackendResourceReference struct { - APIVersion string `json:"apiVersion"` - Kind string `json:"kind"` - Namespace string `json:"namespace,omitempty"` - Name string `json:"name"` - UID types.UID `json:"uid"` + Name string `json:"name"` + UID types.UID `json:"uid"` // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=64 - ParticipantIndexes []int32 `json:"participantIndexes,omitempty"` + ParticipantIndexes []int32 `json:"participantIndexes"` } type PinnedProfile struct { @@ -59,31 +60,30 @@ type PinnedProfile struct { Generation int64 `json:"generation"` } +type ScriptBundleReference struct { + Name string `json:"name"` + UID types.UID `json:"uid"` +} + type ScriptSnapshot struct { - ConfigMapName string `json:"configMapName"` - Key string `json:"key"` - RequiresRuntimeStage bool `json:"requiresRuntimeStage,omitempty"` + Key string `json:"key"` // Digest is the algorithm-qualified SHA-256 digest of the selected script bytes. // +kubebuilder:validation:Pattern=`^sha256:[0-9a-f]{64}$` Digest string `json:"digest"` } // +kubebuilder:validation:XValidation:rule="duration(self.backendActiveDeadline) > duration('0s')",message="backendActiveDeadline must be positive" -// +kubebuilder:validation:XValidation:rule="!has(self.script.requiresRuntimeStage) || !self.script.requiresRuntimeStage || has(self.runtimeStage)",message="runtimeStage is required when the script requires a runtime stage" +// +kubebuilder:validation:XValidation:rule="!has(self.scriptEnvironment) || self.scriptEnvironment.all(k, size(k) <= 253 && size(self.scriptEnvironment[k]) <= 4096)",message="scriptEnvironment names must not exceed 253 characters and values must not exceed 4096 characters" type CertificationCheckSnapshot struct { - // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:MaxLength=63 - // +kubebuilder:validation:Pattern=`^R(0[1-9]|1[0-5])$` - ID CheckID `json:"id"` - // +kubebuilder:validation:Minimum=0 - // +kubebuilder:validation:Maximum=14 - Order int32 `json:"order"` - Script ScriptSnapshot `json:"script"` - RunnerImage string `json:"runnerImage"` - Backend ExecutionBackend `json:"backend"` - WorkloadPolicy WorkloadPolicy `json:"workloadPolicy,omitempty"` - RuntimeStage *RuntimeStage `json:"runtimeStage,omitempty"` - GPUCount int32 `json:"gpuCount,omitempty"` + // ID and Order are derived from the check's position and are never serialized. + ID CheckID `json:"-"` + Order int32 `json:"-"` + Script ScriptSnapshot `json:"script"` + RunnerImage string `json:"runnerImage"` + Backend ExecutionBackend `json:"backend"` + WorkloadPolicy WorkloadPolicy `json:"workloadPolicy,omitempty"` + GPUCount int32 `json:"gpuCount,omitempty"` + // +kubebuilder:validation:MaxProperties=32 ScriptEnvironment map[string]string `json:"scriptEnvironment,omitempty"` BackendActiveDeadline metav1.Duration `json:"backendActiveDeadline"` ParticipantPolicy CheckParticipantPolicy `json:"participantPolicy,omitempty"` @@ -91,45 +91,31 @@ type CertificationCheckSnapshot struct { // +kubebuilder:validation:XValidation:rule="self.profile == oldSelf.profile",message="profile is immutable" // +kubebuilder:validation:XValidation:rule="self.participants == oldSelf.participants",message="participants are immutable" -// +kubebuilder:validation:XValidation:rule="has(self.participantRoles) == has(oldSelf.participantRoles) && (!has(self.participantRoles) || self.participantRoles == oldSelf.participantRoles)",message="participantRoles are immutable" -// +kubebuilder:validation:XValidation:rule="has(self.topologyConstraints) == has(oldSelf.topologyConstraints) && (!has(self.topologyConstraints) || self.topologyConstraints == oldSelf.topologyConstraints)",message="topologyConstraints are immutable" +// +kubebuilder:validation:XValidation:rule="self.scriptBundle == oldSelf.scriptBundle",message="scriptBundle is immutable" // +kubebuilder:validation:XValidation:rule="self.checks == oldSelf.checks",message="checks are immutable" -// +kubebuilder:validation:XValidation:rule="self.executionPolicy == oldSelf.executionPolicy",message="executionPolicy is immutable" -// +kubebuilder:validation:XValidation:rule="self.workflow == oldSelf.workflow",message="workflow is immutable" -// +kubebuilder:validation:XValidation:rule="self.evidencePath == oldSelf.evidencePath",message="evidencePath is immutable" // +kubebuilder:validation:XValidation:rule="self.deadline == oldSelf.deadline",message="deadline is immutable" -// +kubebuilder:validation:XValidation:rule="has(self.tolerations) == has(oldSelf.tolerations) && (!has(self.tolerations) || self.tolerations == oldSelf.tolerations)",message="tolerations are immutable" -// +kubebuilder:validation:XValidation:rule="self.requestNodeName == oldSelf.requestNodeName",message="requestNodeName is immutable" -// +kubebuilder:validation:XValidation:rule="self.checks.all(c, self.checks.exists_one(x, x.id == c.id))",message="checks must contain unique IDs" -// +kubebuilder:validation:XValidation:rule="self.checks.map(c, c.order) == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]",message="checks must use immutable contiguous order 0 through 14" -// +kubebuilder:validation:XValidation:rule="self.checks.map(c, c.id) == ['R01', 'R02', 'R03', 'R04', 'R05', 'R06', 'R07', 'R08', 'R09', 'R10', 'R11', 'R12', 'R13', 'R14', 'R15']",message="checks must use ordered IDs R01 through R15" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.cancel) || !oldSelf.cancel || (has(self.cancel) && self.cancel)",message="cancel cannot be cleared" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.release) || !oldSelf.release || (has(self.release) && self.release)",message="release cannot be cleared" // +kubebuilder:validation:XValidation:rule="self.checks.all(c, duration(self.deadline) - duration(c.backendActiveDeadline) >= duration('60s'))",message="every backendActiveDeadline must leave at least 60 seconds before deadline" +// +kubebuilder:validation:XValidation:rule="self.checks.map(c, duration(c.backendActiveDeadline) + duration('60s')).sum() <= duration(self.deadline)",message="deadline must cover every sequential backendActiveDeadline plus 60 seconds of evidence time per check" type CertificationRunSpec struct { Profile PinnedProfile `json:"profile"` // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=64 - Participants []Participant `json:"participants"` - // +kubebuilder:validation:MaxItems=16 - ParticipantRoles []ParticipantRoleConstraint `json:"participantRoles,omitempty"` - // +kubebuilder:validation:MaxItems=16 - TopologyConstraints []ParticipantTopologyConstraint `json:"topologyConstraints,omitempty"` + Participants []Participant `json:"participants"` + ScriptBundle ScriptBundleReference `json:"scriptBundle"` // +kubebuilder:validation:MinItems=15 // +kubebuilder:validation:MaxItems=15 - Checks []CertificationCheckSnapshot `json:"checks"` - ExecutionPolicy SuiteExecutionPolicy `json:"executionPolicy"` - Workflow WorkflowReference `json:"workflow"` - EvidencePath string `json:"evidencePath"` - Deadline metav1.Duration `json:"deadline"` - Tolerations []corev1.Toleration `json:"tolerations,omitempty"` - RequestNodeName string `json:"requestNodeName"` - Cancel bool `json:"cancel,omitempty"` + Checks []CertificationCheckSnapshot `json:"checks"` + Deadline metav1.Duration `json:"deadline"` + Cancel bool `json:"cancel,omitempty"` + Release bool `json:"release,omitempty"` } type WorkflowStatus struct { - Name string `json:"name,omitempty"` - UID string `json:"uid,omitempty"` - Attempt int32 `json:"attempt,omitempty"` - Phase string `json:"phase,omitempty"` + Name string `json:"name,omitempty"` + UID string `json:"uid,omitempty"` + Phase string `json:"phase,omitempty"` } type EvidenceStatus struct { @@ -139,11 +125,7 @@ type EvidenceStatus struct { Digest string `json:"digest,omitempty"` } -type CleanupStatus struct { - ReservationsReleased bool `json:"reservationsReleased,omitempty"` - SnapshotDeleted bool `json:"snapshotDeleted,omitempty"` -} - +// +kubebuilder:validation:Enum=Pending;Running;Succeeded;Failed;Inconclusive;Skipped type CheckPhase string const ( @@ -156,53 +138,53 @@ const ( ) type BackendEvidenceStatus struct { - Version string `json:"version,omitempty"` - Complete bool `json:"complete,omitempty"` + Complete bool `json:"complete,omitempty"` // Digest identifies the complete backend-evidence v1 document retained in // the durable evidence artifact. // +kubebuilder:validation:Pattern=`^sha256:[0-9a-f]{64}$` Digest string `json:"digest,omitempty"` - // Content is populated only while constructing the durable evidence - // artifact and is not persisted in CertificationRun status. - Content string `json:"content,omitempty"` + // Content is transient input for the durable evidence artifact. + Content string `json:"-"` } type CheckCleanupStatus struct { - Complete bool `json:"complete,omitempty"` - Reason string `json:"reason,omitempty"` + Complete bool `json:"-"` + Reason string `json:"-"` } type CheckStatus struct { - ID CheckID `json:"id"` - Order int32 `json:"order"` - WorkflowNodeName string `json:"workflowNodeName,omitempty"` - Phase CheckPhase `json:"phase"` - Verdict string `json:"verdict,omitempty"` - Reason string `json:"reason,omitempty"` - Measurements string `json:"measurements,omitempty"` + // Internal fields are derived or transient and are never persisted in CR status. + ID CheckID `json:"-"` + Order int32 `json:"-"` + WorkflowNodeName string `json:"-"` + Phase CheckPhase `json:"phase"` + Verdict string `json:"-"` + // +kubebuilder:validation:MaxLength=1024 + Reason string `json:"reason,omitempty"` + // +kubebuilder:validation:MaxLength=16384 + Measurements string `json:"measurements,omitempty"` + // +kubebuilder:validation:MaxItems=64 BackendResources []BackendResourceReference `json:"backendResources,omitempty"` BackendEvidence BackendEvidenceStatus `json:"backendEvidence,omitempty"` StartedAt *metav1.Time `json:"startedAt,omitempty"` FinishedAt *metav1.Time `json:"finishedAt,omitempty"` - Duration metav1.Duration `json:"duration,omitempty"` - Attempt int32 `json:"attempt"` - RetryCount int32 `json:"retryCount"` - Cleanup CheckCleanupStatus `json:"cleanup,omitempty"` - NotRun bool `json:"notRun,omitempty"` + Cleanup CheckCleanupStatus `json:"-"` } type CertificationRunStatus struct { - Phase RunPhase `json:"phase,omitempty"` - ExecutionState string `json:"executionState,omitempty"` - Verdict string `json:"verdict,omitempty"` - Reason string `json:"reason,omitempty"` - Workflow WorkflowStatus `json:"workflow,omitempty"` - Evidence EvidenceStatus `json:"evidence,omitempty"` - Cleanup CleanupStatus `json:"cleanup,omitempty"` + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + Phase RunPhase `json:"phase,omitempty"` + // +kubebuilder:validation:MaxLength=1024 + Reason string `json:"reason,omitempty"` + Workflow WorkflowStatus `json:"workflow,omitempty"` + Evidence EvidenceStatus `json:"evidence,omitempty"` // +kubebuilder:validation:MaxItems=15 - Checks []CheckStatus `json:"checks,omitempty"` - StartedAt *metav1.Time `json:"startedAt,omitempty"` - FinishedAt *metav1.Time `json:"finishedAt,omitempty"` + Checks []CheckStatus `json:"checks,omitempty"` + StartedAt *metav1.Time `json:"startedAt,omitempty"` + FinishedAt *metav1.Time `json:"finishedAt,omitempty"` + // +listType=map + // +listMapKey=type + // +kubebuilder:validation:MaxItems=8 Conditions []metav1.Condition `json:"conditions,omitempty"` } @@ -211,7 +193,6 @@ type CertificationRunStatus struct { // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="Profile",type=string,JSONPath=`.spec.profile.name` // +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` -// +kubebuilder:printcolumn:name="Verdict",type=string,JSONPath=`.status.verdict` // +kubebuilder:printcolumn:name="Workflow",type=string,JSONPath=`.status.workflow.name` // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` type CertificationRun struct { diff --git a/api/signal/v1alpha1/runtime_contract.go b/api/signal/v1alpha1/runtime_contract.go index 217eb1da9e..d7376c7c58 100644 --- a/api/signal/v1alpha1/runtime_contract.go +++ b/api/signal/v1alpha1/runtime_contract.go @@ -7,51 +7,52 @@ import ( "errors" "fmt" "time" - - "k8s.io/apimachinery/pkg/util/validation" ) const MinimumBackendDeadlineReserve = time.Minute -func ValidateRuntimeContract( - scriptRequiresStage bool, - stage *RuntimeStage, - backendDeadline, outerDeadline time.Duration, -) error { - if err := ValidateRuntimeStage(scriptRequiresStage, stage); err != nil { - return err +func ValidateExecutionDeadlines(backendDeadline, outerDeadline time.Duration) error { + if backendDeadline <= 0 { + return errors.New("backend active deadline must be positive") } - return ValidateExecutionDeadlines(backendDeadline, outerDeadline) -} - -func ValidateRuntimeStage(scriptRequiresStage bool, stage *RuntimeStage) error { - if scriptRequiresStage && stage == nil { - return errors.New("runtime stage is required by the script") + if backendDeadline%time.Second != 0 || outerDeadline%time.Second != 0 { + return errors.New("backend and outer deadlines must use whole seconds") } - if stage != nil { - if errors := validation.IsDNS1123Label(stage.Name); len(errors) > 0 { - return fmt.Errorf("runtime stage name is invalid: %v", errors) - } + if outerDeadline-backendDeadline < MinimumBackendDeadlineReserve { + return fmt.Errorf( + "backend active deadline must leave at least %s before the outer deadline", + MinimumBackendDeadlineReserve, + ) } return nil } -func ValidateExecutionDeadlines(backendDeadline, outerDeadline time.Duration) error { - if backendDeadline <= 0 { - return errors.New("backend active deadline must be positive") +func ValidateSuiteDeadlines(backendDeadlines []time.Duration, suiteDeadline time.Duration) error { + if suiteDeadline <= 0 || suiteDeadline%time.Second != 0 { + return errors.New("suite deadline must be a positive whole number of seconds") } - if backendDeadline%time.Second != 0 || outerDeadline%time.Second != 0 { - return errors.New("backend and outer deadlines must use whole seconds") + var required time.Duration + for _, backendDeadline := range backendDeadlines { + if backendDeadline <= 0 || backendDeadline%time.Second != 0 { + return errors.New("backend active deadlines must be positive whole numbers of seconds") + } + + if backendDeadline > time.Duration(1<<63-1)-required-MinimumBackendDeadlineReserve { + return errors.New("combined check deadlines overflow") + } + + required += backendDeadline + MinimumBackendDeadlineReserve } - if outerDeadline-backendDeadline < MinimumBackendDeadlineReserve { + if required > suiteDeadline { return fmt.Errorf( - "backend active deadline must leave at least %s before the outer deadline", - MinimumBackendDeadlineReserve, + "combined backend deadlines and evidence reserves require %s, suite deadline is %s", + required, + suiteDeadline, ) } diff --git a/api/signal/v1alpha1/zz_generated.deepcopy.go b/api/signal/v1alpha1/zz_generated.deepcopy.go index 33d607cb52..2dbb4f175e 100644 --- a/api/signal/v1alpha1/zz_generated.deepcopy.go +++ b/api/signal/v1alpha1/zz_generated.deepcopy.go @@ -52,13 +52,7 @@ func (in *BackendResourceReference) DeepCopy() *BackendResourceReference { func (in *CertificationCheck) DeepCopyInto(out *CertificationCheck) { *out = *in out.Script = in.Script - out.Backend = in.Backend in.WorkloadPolicy.DeepCopyInto(&out.WorkloadPolicy) - if in.RuntimeStage != nil { - in, out := &in.RuntimeStage, &out.RuntimeStage - *out = new(RuntimeStage) - **out = **in - } if in.ScriptEnvironment != nil { in, out := &in.ScriptEnvironment, &out.ScriptEnvironment *out = make(map[string]string, len(*in)) @@ -84,13 +78,7 @@ func (in *CertificationCheck) DeepCopy() *CertificationCheck { func (in *CertificationCheckSnapshot) DeepCopyInto(out *CertificationCheckSnapshot) { *out = *in out.Script = in.Script - out.Backend = in.Backend in.WorkloadPolicy.DeepCopyInto(&out.WorkloadPolicy) - if in.RuntimeStage != nil { - in, out := &in.RuntimeStage, &out.RuntimeStage - *out = new(RuntimeStage) - **out = **in - } if in.ScriptEnvironment != nil { in, out := &in.ScriptEnvironment, &out.ScriptEnvironment *out = make(map[string]string, len(*in)) @@ -186,7 +174,6 @@ func (in *CertificationProfileSpec) DeepCopyInto(out *CertificationProfileSpec) *out = make([]ParticipantTopologyConstraint, len(*in)) copy(*out, *in) } - out.Workflow = in.Workflow if in.Checks != nil { in, out := &in.Checks, &out.Checks *out = make([]CertificationCheck, len(*in)) @@ -194,15 +181,7 @@ func (in *CertificationProfileSpec) DeepCopyInto(out *CertificationProfileSpec) (*in)[i].DeepCopyInto(&(*out)[i]) } } - out.ExecutionPolicy = in.ExecutionPolicy out.Deadline = in.Deadline - if in.Tolerations != nil { - in, out := &in.Tolerations, &out.Tolerations - *out = make([]v1.Toleration, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationProfileSpec. @@ -285,18 +264,7 @@ func (in *CertificationRunSpec) DeepCopyInto(out *CertificationRunSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.ParticipantRoles != nil { - in, out := &in.ParticipantRoles, &out.ParticipantRoles - *out = make([]ParticipantRoleConstraint, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.TopologyConstraints != nil { - in, out := &in.TopologyConstraints, &out.TopologyConstraints - *out = make([]ParticipantTopologyConstraint, len(*in)) - copy(*out, *in) - } + out.ScriptBundle = in.ScriptBundle if in.Checks != nil { in, out := &in.Checks, &out.Checks *out = make([]CertificationCheckSnapshot, len(*in)) @@ -304,16 +272,7 @@ func (in *CertificationRunSpec) DeepCopyInto(out *CertificationRunSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - out.ExecutionPolicy = in.ExecutionPolicy - out.Workflow = in.Workflow out.Deadline = in.Deadline - if in.Tolerations != nil { - in, out := &in.Tolerations, &out.Tolerations - *out = make([]v1.Toleration, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationRunSpec. @@ -331,7 +290,6 @@ func (in *CertificationRunStatus) DeepCopyInto(out *CertificationRunStatus) { *out = *in out.Workflow = in.Workflow out.Evidence = in.Evidence - out.Cleanup = in.Cleanup if in.Checks != nil { in, out := &in.Checks, &out.Checks *out = make([]CheckStatus, len(*in)) @@ -384,13 +342,8 @@ func (in *CheckCleanupStatus) DeepCopy() *CheckCleanupStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CheckParticipantPolicy) DeepCopyInto(out *CheckParticipantPolicy) { *out = *in - if in.RequiredRoles != nil { - in, out := &in.RequiredRoles, &out.RequiredRoles - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.ExcludedRoles != nil { - in, out := &in.ExcludedRoles, &out.ExcludedRoles + if in.Roles != nil { + in, out := &in.Roles, &out.Roles *out = make([]string, len(*in)) copy(*out, *in) } @@ -425,7 +378,6 @@ func (in *CheckStatus) DeepCopyInto(out *CheckStatus) { in, out := &in.FinishedAt, &out.FinishedAt *out = (*in).DeepCopy() } - out.Duration = in.Duration out.Cleanup = in.Cleanup } @@ -439,21 +391,6 @@ func (in *CheckStatus) DeepCopy() *CheckStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CleanupStatus) DeepCopyInto(out *CleanupStatus) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CleanupStatus. -func (in *CleanupStatus) DeepCopy() *CleanupStatus { - if in == nil { - return nil - } - out := new(CleanupStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EvidenceStatus) DeepCopyInto(out *EvidenceStatus) { *out = *in @@ -469,31 +406,9 @@ func (in *EvidenceStatus) DeepCopy() *EvidenceStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ExecutionBackend) DeepCopyInto(out *ExecutionBackend) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExecutionBackend. -func (in *ExecutionBackend) DeepCopy() *ExecutionBackend { - if in == nil { - return nil - } - out := new(ExecutionBackend) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Participant) DeepCopyInto(out *Participant) { *out = *in - if in.Capacity != nil { - in, out := &in.Capacity, &out.Capacity - *out = make(v1.ResourceList, len(*in)) - for key, val := range *in { - (*out)[key] = val.DeepCopy() - } - } if in.Allocatable != nil { in, out := &in.Allocatable, &out.Allocatable *out = make(v1.ResourceList, len(*in)) @@ -501,10 +416,12 @@ func (in *Participant) DeepCopyInto(out *Participant) { (*out)[key] = val.DeepCopy() } } - if in.Roles != nil { - in, out := &in.Roles, &out.Roles - *out = make([]string, len(*in)) - copy(*out, *in) + if in.TopologyValues != nil { + in, out := &in.TopologyValues, &out.TopologyValues + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } } } @@ -522,8 +439,8 @@ func (in *Participant) DeepCopy() *Participant { func (in *ParticipantPolicy) DeepCopyInto(out *ParticipantPolicy) { *out = *in in.Selector.DeepCopyInto(&out.Selector) - if in.RequiredResources != nil { - in, out := &in.RequiredResources, &out.RequiredResources + if in.MinimumAllocatable != nil { + in, out := &in.MinimumAllocatable, &out.MinimumAllocatable *out = make(v1.ResourceList, len(*in)) for key, val := range *in { (*out)[key] = val.DeepCopy() @@ -588,16 +505,16 @@ func (in *PinnedProfile) DeepCopy() *PinnedProfile { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RuntimeStage) DeepCopyInto(out *RuntimeStage) { +func (in *ScriptBundleReference) DeepCopyInto(out *ScriptBundleReference) { *out = *in } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuntimeStage. -func (in *RuntimeStage) DeepCopy() *RuntimeStage { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScriptBundleReference. +func (in *ScriptBundleReference) DeepCopy() *ScriptBundleReference { if in == nil { return nil } - out := new(RuntimeStage) + out := new(ScriptBundleReference) in.DeepCopyInto(out) return out } @@ -632,36 +549,6 @@ func (in *ScriptSnapshot) DeepCopy() *ScriptSnapshot { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *SuiteExecutionPolicy) DeepCopyInto(out *SuiteExecutionPolicy) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SuiteExecutionPolicy. -func (in *SuiteExecutionPolicy) DeepCopy() *SuiteExecutionPolicy { - if in == nil { - return nil - } - out := new(SuiteExecutionPolicy) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *WorkflowReference) DeepCopyInto(out *WorkflowReference) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkflowReference. -func (in *WorkflowReference) DeepCopy() *WorkflowReference { - if in == nil { - return nil - } - out := new(WorkflowReference) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WorkflowStatus) DeepCopyInto(out *WorkflowStatus) { *out = *in diff --git a/cmd/project-signal-adapter/main.go b/cmd/project-signal-adapter/main.go index 00d4efdf05..4b4b66ccb5 100644 --- a/cmd/project-signal-adapter/main.go +++ b/cmd/project-signal-adapter/main.go @@ -60,15 +60,7 @@ func main() { "", "algorithm-qualified SHA-256 digest of the script bytes (sha256:<64 lowercase hex>)", ) - flags.BoolVar( - &cfg.ScriptRequiresStage, - "script-requires-runtime-stage", - false, - "Require a validated runtime stage name", - ) - flags.StringVar(&cfg.RuntimeStageName, "runtime-stage-name", "", "Runtime stage name for participant containers") flags.StringVar(&cfg.RunnerImage, "runner-image", "", "Digest-pinned runner image") - flags.StringVar(&cfg.BackendAPIVersion, "backend-api-version", "", "Backend API version") flags.StringVar(&cfg.BackendKind, "backend-kind", "", "Backend kind") flags.Int64Var( &cfg.BackendDeadlineSeconds, @@ -76,11 +68,9 @@ func main() { 0, "Active deadline for participant backend workloads", ) - flags.StringVar(&cfg.TolerationsJSON, "tolerations", "[]", "JSON-encoded Pod tolerations") flags.StringVar(&cfg.ScriptEnvironmentJSON, "script-environment", "{}", "JSON-encoded script environment") flags.StringVar(&cfg.WorkloadPolicyJSON, "workload-policy", "{}", "JSON-encoded participant workload policy") flags.StringVar(&cfg.GPUResourceCount, "gpu-resource-count", "0", "GPU resource limit per participant") - flags.IntVar(&cfg.Attempt, "attempt", 0, "One-based Workflow attempt number") flags.StringVar(&cfg.outputDirectory, "output-directory", "/outputs", "Argo output directory") command.CompletionOptions.DisableDefaultCmd = true @@ -136,13 +126,6 @@ func writeOutputs(directory string, output signaladapter.AggregateOutput) error return errors.New("output-directory is required") } - if len(output.CleanupFailure) > signaladapter.MaxCleanupFailureBytes { - return fmt.Errorf( - "cleanup failure exceeds %d bytes", - signaladapter.MaxCleanupFailureBytes, - ) - } - if err := signaladapter.ValidateBackendEvidence(output.BackendEvidence); err != nil { return err } @@ -154,7 +137,6 @@ func writeOutputs(directory string, output signaladapter.AggregateOutput) error outputs := map[string]string{ "verdict": output.Verdict, "reason": output.Reason, - "cleanup-failure": output.CleanupFailure, "measurements": output.Measurements, "backend-resources": output.BackendJSON, "backend-evidence": output.BackendEvidence, diff --git a/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl b/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl index f7a04baefc..23a536e86c 100644 --- a/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl +++ b/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl @@ -16,17 +16,16 @@ spec: - name: script-configmap - name: script-key - name: script-digest - - name: script-requires-runtime-stage - - name: runtime-stage-name - - name: backend-api-version - name: backend-kind - name: backend-active-deadline-seconds - name: runner-image - - name: tolerations - name: script-environment - name: workload-policy - name: gpu-resource-count - - name: attempt + tolerations: + - key: node.kubernetes.io/unschedulable + operator: Exists + effect: NoSchedule container: image: {{ default "example.invalid/project-signal-adapter@sha256:0000000000000000000000000000000000000000000000000000000000000000" .AdapterImage }} imagePullPolicy: IfNotPresent @@ -49,27 +48,18 @@ spec: - "{{`{{inputs.parameters.script-key}}`}}" - --script-digest - "{{`{{inputs.parameters.script-digest}}`}}" - - "--script-requires-runtime-stage={{`{{inputs.parameters.script-requires-runtime-stage}}`}}" - - --runtime-stage-name - - "{{`{{inputs.parameters.runtime-stage-name}}`}}" - --runner-image - "{{`{{inputs.parameters.runner-image}}`}}" - - --backend-api-version - - "{{`{{inputs.parameters.backend-api-version}}`}}" - --backend-kind - "{{`{{inputs.parameters.backend-kind}}`}}" - --backend-active-deadline-seconds - "{{`{{inputs.parameters.backend-active-deadline-seconds}}`}}" - - --tolerations - - "{{`{{inputs.parameters.tolerations}}`}}" - --script-environment - "{{`{{inputs.parameters.script-environment}}`}}" - --workload-policy - "{{`{{inputs.parameters.workload-policy}}`}}" - --gpu-resource-count - "{{`{{inputs.parameters.gpu-resource-count}}`}}" - - --attempt - - "{{`{{inputs.parameters.attempt}}`}}" env: - name: POD_NAMESPACE valueFrom: @@ -93,9 +83,6 @@ spec: - name: reason valueFrom: path: /outputs/reason - - name: cleanup-failure - valueFrom: - path: /outputs/cleanup-failure - name: measurements valueFrom: path: /outputs/measurements diff --git a/deploy/signal/README.md b/deploy/signal/README.md index a22516e96a..bc808d10c7 100644 --- a/deploy/signal/README.md +++ b/deploy/signal/README.md @@ -1,467 +1,252 @@ # Project Signal controller -Project Signal is a Kubernetes controller for admitting approved idle Nodes to -certification workflows. The upstream integration provides the API, controller, -CRDs, controller Deployment, and an optional generic participant adapter for -ordinary Pods and Kubeflow MPIJobs. It intentionally does not publish -certification scripts, runner images, CertificationProfiles, hardware policy, -or environment-specific hardware configuration. - -## Architecture - -```text -approved Node request - | - v - CertificationProfile - | - v - reserve Nodes and snapshot the ordered 15-check script plan - | - v - immutable CertificationRun input + SHA-256 digest - | - v - one Argo Workflow with a serial, visible 15-task DAG - | - v - per-check typed results -> aggregate evidence -> Node projection -> cleanup -``` +Project Signal performs one certification run for a newly joined Node or a +small, explicitly selected group of Nodes. The controller snapshots the +approved profile and scripts, reserves and cordons every participant, runs 15 +ordered checks through Argo Workflows, persists one immutable evidence +document, and cleans up all temporary workloads. + +Project Signal does not integrate with Kueue. Queue admission and repeatable +fleet scheduling are intentionally deferred. + +## Components + +- `CertificationProfile` is cluster-scoped operator policy. +- `CertificationRun` is a namespaced, immutable execution snapshot. +- `project-signal-controller` admits Nodes, owns reservations and cordons, + creates the Workflow, persists status and evidence, and performs teardown. +- `project-signal-participant-adapter` creates Pod or MPIJob backends and + captures typed evidence. Ordinary Pods and completed MPI resources remain + until controller teardown; the MPI Operator removes still-running workers + after launcher completion so later sequential checks can use the GPUs. + On setup, observation, or wall-clock deadline failures, the adapter first + persists the available evidence and then quiesces uncertain backends. +- `project-signal-participant-adapter` is a fixed WorkflowTemplate used by + every check. Profiles cannot replace it or pass arbitrary Workflow wiring. + +Argo Workflows is required. The Kubeflow MPI Operator is required only when a +profile contains an `MPIJob` check. + +## First deployment -The controller snapshots all 15 selected script keys into one run-owned -immutable ConfigMap. Updating a source affects only later runs. The controller -does not contain or publish certification script bodies. - -## Prerequisites - -- Kubernetes with the `Node`, `Pod`, and `ConfigMap` APIs. -- Argo Workflows `argoproj.io/v1alpha1` Workflow API. -- A controller image built from `cmd/project-signal-controller`. -- An operator-owned script ConfigMap and execution image. -- An operator-owned WorkflowTemplate implementing the interface below, or the - included `project-signal-participant-adapter` WorkflowTemplate. -- Explicitly reviewed Node selectors, resource requests, tolerations, policy - values, and cleanup behavior. - -The execution image used with the included adapter must contain Bash and the -standard `mkdir`, `chmod`, `base64`, and `tr` utilities, plus every tool required -by the selected script. The adapter image and execution image are separate: the -upstream adapter creates and observes workloads, while the operator-owned -execution image runs the script. - -The included adapter supports standalone `v1` Pods and -`kubeflow.org/v2beta1` MPIJobs. Project Signal does not install MPI Operator. -Kueue integration is intentionally deferred. The adapter RBAC is -namespace-scoped and intentionally grants only the Pod, Pod log, MPIJob, -Workflow, and Argo task-result access needed by the included template. - -## WorkflowTemplate interface - -The referenced WorkflowTemplate must accept these string parameters: - -| Parameter | Meaning | -| --- | --- | -| `run-name` | CertificationRun name | -| `run-uid` | CertificationRun UID | -| `participants-json` | Ordered JSON array of participant indexes, Node names, immutable UIDs, capacity, and allocatable resources | -| `participant-count` | Number of atomically admitted participants | -| `script-configmap` | Immutable run-owned ConfigMap | -| `script-key` | Selected ConfigMap key | -| `script-digest` | Algorithm-qualified SHA-256 of the selected script bytes (`sha256:<64 lowercase hex>`) | -| `script-requires-runtime-stage` | Whether the selected script requires a runtime stage | -| `runtime-stage-name` | Optional validated runtime stage name | -| `backend-api-version` | Backend API version from the profile | -| `backend-kind` | Backend kind from the profile | -| `backend-active-deadline-seconds` | Participant backend deadline, shorter than the outer Workflow deadline | -| `runner-image` | Digest-pinned execution image | -| `tolerations` | JSON-encoded toleration list | -| `script-environment` | JSON-encoded, validated environment map | -| `workload-policy` | JSON-encoded immutable resources, privilege/capabilities, host networking, working directory, and read-only host paths | -| `gpu-resource-count` | GPU count, defaults to `0` for Pod diagnostics; must be positive for MPIJobs | -| `attempt` | One-based attempt number | - -The Workflow must publish only these output parameters: - -| Output | Limit | Meaning | -| --- | ---: | --- | -| `verdict` | 32 bytes | `Pass`, `Fail`, or `Inconclusive` | -| `reason` | 512 bytes | Bounded diagnostic reason | -| `measurements` | 16 KiB | Bounded measurement summary | -| `backend-resources` | 8 KiB | Optional JSON array of backend workload identities and covered participant indexes; required for a multi-Node `Pass` | -| `backend-evidence` | 24 KiB per check | Required redacted pre-cleanup backend object, container state, and bounded current/previous log projection | - -Unexpected, duplicate, malformed, or oversized outputs are rejected and the -run remains fail-closed. Each `backend-resources` entry must contain -`apiVersion`, `kind`, `name`, `uid`, and the zero-based `participantIndexes` -that the workload executed. `namespace` is optional and defaults to the -CertificationRun namespace; an explicitly different namespace is rejected. -A multi-Node successful result must cover every admitted participant. -Each check may select at most eight participants so all 15 checks' fixed output -bounds remain within the 768 KiB Workflow status budget. -The controller reads outputs only -from each check's unique deterministic DAG task node. Missing, malformed, -duplicate, or ambiguous task identity and outputs fail closed. Operator script -inputs are explicit entries in `spec.checks[].scriptEnvironment`; the -controller validates their environment names before reserving Nodes. - -Before foreground deletion, the adapter captures and writes a typed v1 -projection for every participant backend to the Argo output path. The -projection includes Pod or MPIJob state and -conditions, launcher Job conditions, participant container exit state and -restart count, and bounded current and previous container log tails. Generated -participant commands emit lifecycle markers for result-directory preparation -and final verdict, reason, and measurements file presence. The projection -records those markers without retaining the result payload itself. - -Each retained log tail is limited to 512 bytes and redacts bearer credentials, -secret-like assignments, URL user information, private-key blocks, and complete -result-marker payloads. Object reason and message fields are also bounded. If a -required object or current log cannot be captured, a restarted container's -previous log is unavailable, or the projection must omit content to fit the -configured per-check output limit (at most 64 KiB), `complete` is false and the run cannot pass. Capture and -cleanup failures remain explicit, while an earlier backend execution failure -continues to be the causal result reason. - -The adapter cannot observe or serialize its own state if the adapter container -or Node terminates before this pre-cleanup write. In that boundary the -`backend-evidence` output is missing and the controller rejects the Workflow -outputs as Inconclusive; the backend cause remains unknown rather than being -inferred. A termination after the write may leave the evidence file available -to Argo, but collection still depends on the Workflow executor surviving long -enough to publish output parameters. - -The controller cross-checks typed backend evidence against the declared backend, -namespace, exact workload UID, participant coverage, and required Pod/container -and current/previous log records. A template's `complete: true` alone is not -sufficient. Log collection requests the latest 64 lines, retains at most 64 KiB -for result parsing, and projects redacted tails of at most 512 bytes into evidence. -Earlier log content is intentionally outside this bounded result contract. - -After the adapter creates its first backend, every later setup or aggregation -failure is reported as an Inconclusive output with bounded partial backend -identity and typed evidence. The adapter persists that evidence before -foreground cleanup; missing participant coverage can never produce a passing -result. If the Argo output write fails, cleanup is blocked and the backends -remain available rather than destroying the only remaining audit state. - -After durable terminal evidence is recorded, the controller foreground-deletes -the Workflow and waits for owned backend descendants to be removed before -releasing Node reservations. This also covers adapter crashes, cancellation, -and a controller crash between Workflow creation and status persistence. -The six-hour Workflow TTL is a fallback, not the normal evidence-retention -mechanism; immutable terminal evidence remains available after Workflow cleanup. -The controller's namespace-scoped backend patch and owner-finalizer permissions -allow Kubernetes' owner-reference admission checks to authorize blocking -deletion fences. It does not receive backend workload creation or deletion -rights. - -## Multi-Node execution - -Participant admission is deterministic and atomic. The requesting Node is -participant zero, additional eligible Nodes are selected by name, and a failed -reservation rolls back every reservation already acquired. Each participant is -bound by Node UID and an immutable capacity and allocatable snapshot in the -CertificationRun. - -The participant group is a certification boundary, not one distributed process -group. The included adapter creates one independently pinned backend workload -per participant. Ordinary diagnostics use one Pod per participant. MPI -diagnostics use one MPIJob per participant with one launcher and one GPU worker; -ordinary Pods and MPI workers use required Node affinity on the immutable -`metadata.name` field and a controller-owned reservation label containing the -admitted Node UID. This binds execution to the exact admitted Node without -bypassing scheduler admission or accepting a replacement Node that reuses its -name. Only the worker receives the configured GPU limit. The adapter propagates -the Workflow active deadline, records API-assigned workload UIDs, reads the -final typed result from bounded workload -logs, aggregates diagnostic state separately from backend execution state, and -foreground-deletes every created workload only after its evidence output is -durable. - -The selected script is invoked with no arguments for a Pod backend, -`mpi-launcher ` in an MPIJob launcher, and `mpi-worker` in its worker. -Scripts write `verdict`, `reason`, and optional `measurements.txt` under -`/results`. The script exit code must match the result verdict: `Pass` exits 0, -`Fail` exits 1, and `Inconclusive` exits 2. A valid bounded result remains -authoritative when its verdict-associated exit code makes the backend Pod or -MPIJob terminally unsuccessful. The adapter removes reserved result files -before each invocation and removes trailing newline bytes from repository-owned -result files before encoding them. It rejects missing files, malformed values, -stale or contradictory result/exit combinations, and any other exit code. -Partial backend execution, identity gaps, deadline interruption, and cleanup -failure also remain `Inconclusive`. - -The MPI launcher also applies a compatibility fence for -[`kubeflow/mpi-operator#615`](https://github.com/kubeflow/mpi-operator/issues/615), -whose suspend/resume race is fixed upstream by -[`kubeflow/mpi-operator#868`](https://github.com/kubeflow/mpi-operator/pull/868) -at commit `3a1e56d8accd0d924703d79af3149e3fa8ba1b01`. Before invoking the diagnostic -launcher command, the adapter writes `/tmp/project-signal-hostfile` from the -operator's deterministic worker DNS contract -`-worker-...svc` and the validated -`slotsPerWorker`. It exports that file through -`OMPI_MCA_orte_default_hostfile`, so an empty or stale operator-injected -`/etc/mpi/hostfile` cannot control launcher behavior after resume. - -The launcher waits for every generated host to complete SSH `/bin/true` with -`BatchMode=yes`, `ConnectTimeout=2`, `ConnectionAttempts=1`, -`StrictHostKeyChecking=accept-new`, and a launcher-local -`UserKnownHostsFile=/tmp/project-signal-known-hosts`. SSH stderr remains visible -for diagnosis. The wait is capped at 30 seconds total and is always at least -one second shorter than `backendActiveDeadline`; timeout exits 124 and names -the last unready host. Launcher and worker replica restart policies are both -`Never`, preserving the result prelude and marker. - -The adapter keeps the current launcher log primary and reads the previous -container log defensively. A single valid previous result can recover a -restarted launcher when the current log has no result, identical current and -previous results select the current log, and conflicting results or a malformed -current marker fail closed. Bounded current/previous availability, byte counts, -parse status, and selected-source metadata are included in aggregate -measurements before foreground backend cleanup. This preserves the first -execution failure while keeping any cleanup failure separately visible. - -Other backend kinds remain operator-supplied and are not installed or granted -permissions by the included adapter. - -## Render and install - -Render the controller manifests: +Render the manifests with immutable controller and adapter images: ```bash make signal-manifests \ - SIGNAL_CONTROLLER_IMAGE= \ - SIGNAL_ADAPTER_IMAGE= + SIGNAL_CONTROLLER_IMAGE= \ + SIGNAL_ADAPTER_IMAGE=@sha256: ``` -Local adapter builds use the separate mutable -`SIGNAL_ADAPTER_BUILD_IMAGE=` staging reference because -container engines cannot tag a local build with an `image@sha256:` reference. - -Generated output is written to `deploy/signal/rendered/`. Validate it with the -target cluster before applying: +Install the rendered CRDs and namespaced resources: ```bash -kubectl apply --dry-run=server -f deploy/signal/rendered/crd/ -kubectl apply --dry-run=server -f deploy/signal/rendered/ kubectl apply -f deploy/signal/rendered/crd/ kubectl apply -f deploy/signal/rendered/ ``` -The upstream manifests install no profiles. Operators must create a -CertificationProfile with exactly 15 checks that reference operator-owned -script ConfigMaps and digest-pinned runner images. To use the included adapter, -set `spec.workflow.name: project-signal-participant-adapter`. Each -`spec.checks[]` entry selects either: - -```yaml -checks: - - id: - order: 0 - script: - name: - key: - digest: sha256: - runnerImage: @sha256: - backend: - apiVersion: v1 - kind: Pod - backendActiveDeadline: 19m -``` +The install creates: -or an MPIJob backend: +- the `project-signal` namespace; +- controller and adapter service accounts and RBAC; +- the fixed participant adapter WorkflowTemplate; +- the controller Deployment. -```yaml -checks: - - id: - order: <0-14> - backend: - apiVersion: kubeflow.org/v2beta1 - kind: MPIJob -``` +The controller requires cluster permission to get, list, watch, and patch +Nodes because it owns participant reservation markers and cordons. + +## CertificationProfile contract + +A Profile contains only operator choices that legitimately vary between +certification suites: + +| Field | Meaning | +|---|---| +| `spec.participants.count` | Total Nodes selected for a Run | +| `spec.participants.selector` | Labels every selected Node must match | +| `spec.participants.minimumAllocatable` | Admission-time allocatable resource floor | +| `spec.participantRoles` | Optional named role counts and role-specific selectors | +| `spec.topologyConstraints` | Optional minimum topology diversity for a role | +| `spec.checks` | Exactly 15 ordered checks; position defines R01 through R15 | +| `spec.deadline` | Whole-suite deadline | -Checks may set `spec.checks[].workloadPolicy` when a diagnostic needs explicit -participant-container policy. The controller validates and snapshots this -policy into the immutable CertificationRun before reserving Nodes. The included -adapter applies it to the ordinary Pod container or the MPIJob worker only; the -unpinned MPI launcher does not receive participant resources, host devices, or -extra mounts. - -```yaml -workloadPolicy: - resources: - requests: - cpu: "8" - memory: 32Gi - ephemeral-storage: 20Gi - limits: - cpu: "8" - memory: 32Gi - ephemeral-storage: 40Gi - privileged: false - capabilities: [IPC_LOCK] - hostNetwork: true - workingDir: /opt/diagnostic - hostPaths: - - name: work - mountPath: /work - path: /var/lib/project-signal - type: Directory +If roles are configured, their counts must sum to the total participant count. +The requesting Node is participant zero. The remaining participants are +selected deterministically from currently ready, schedulable, unreserved +Nodes. + +Each check contains: + +| Field | Meaning | +|---|---| +| `script` | Source ConfigMap name, key, and approved SHA-256 digest | +| `runnerImage` | Digest-pinned check image | +| `backend` | `Pod` or `MPIJob` | +| `backendActiveDeadline` | Backend execution deadline | +| `gpuCount` | GPU request; must be positive for MPIJob | +| `scriptEnvironment` | At most 32 bounded, non-reserved environment values | +| `participantPolicy.roles` | Optional included roles; omission means all participants | +| `workloadPolicy` | Narrow container resources and host access policy | + +Workload policy is intentionally constrained: + +- the only added Linux capability is `IPC_LOCK`; +- host sources must be under `/proc`, `/sys`, or + `/var/lib/project-signal`; +- host mounts are read-only and their volume names and `Directory` type are + controller-owned; +- scheduling tolerations are controller-owned; +- arbitrary PodSpec, Workflow, retry, TTL, queue, and cleanup settings are not + accepted. + +Profile edits affect only future Runs. + +## Starting one certification + +Add the request label to a schedulable Node: + +```bash +kubectl label node \ + signal.unbounded-cloud.io/certification-request= ``` -Host paths are always mounted read-only. Names and mount paths must be unique; -`scripts`, `results`, `/scripts`, and `/results` are reserved. Paths must be -clean and absolute, resource quantities must be nonnegative, and limits cannot -be below requests. The policy is bounded to 16 host paths and 16 KiB of -serialized input. GPU requests and limits remain controlled by -`gpu-resource-count`, which sets equal requests and limits on the participant -container. Kubernetes admission and operator policy still decide whether a -capability, host path, device, or privileged context is allowed; technical -rendering is not authorization. - -Stage-aware scripts declare -`spec.checks[].script.requiresRuntimeStage: true` and require an explicit -`spec.checks[].runtimeStage.name`. The name is a lowercase DNS label of at most -63 characters and is snapshotted into the immutable CertificationRun and -evidence. The adapter exports it as `PROJECT_SIGNAL_STAGE_NAME` only to ordinary -participant Pods and MPIJob workers. MPI launchers never receive it. Scripts -that are not stage-aware leave both fields unset. - -Every profile sets two distinct deadlines: - -```yaml -deadline: 20m -checks: - - backendActiveDeadline: 19m +Admission then: + +1. validates the Profile and all 15 checks; +2. selects the requesting Node plus any additional participants; +3. atomically records the Run reservation name, participant Node UID, and + `spec.unschedulable=true` on each participant; +4. copies all verified scripts into one immutable Run-owned ConfigMap; +5. creates one immutable `CertificationRun`; +6. projects the pending certification state and marks admission ready. + +Ordinary workloads cannot schedule onto cordoned participants. Signal backend +Pods receive only the fixed GPU toleration and the fixed +`node.kubernetes.io/unschedulable` toleration needed to execute on the Nodes +reserved by this Run. Required Node affinity pins every backend to the admitted +Node name and UID. + +## CertificationRun contract + +The Run spec snapshots: + +- Profile name, UID, and generation; +- ordered participants with Node name, Node UID, admission-time allocatable + resources, optional role, and referenced topology values; +- the immutable script bundle name and UID; +- 15 ordered check snapshots; +- the suite deadline; +- monotonic `cancel` and `release` actions. + +Participant and check identity is positional. Participant zero is the +requesting Node. Check positions zero through fourteen are R01 through R15. +The controller derives these identities after every API read rather than +serializing duplicate index and ID fields. + +Run status contains: + +- `observedGeneration`; +- phase and bounded reason; +- Workflow name, UID, and observed phase; +- evidence URI and SHA-256 digest; +- positional check phases, bounded reasons and measurements, backend + identities, and backend-evidence digest/completeness; +- start and finish timestamps; +- `Cordoned`, `WorkloadCleanupComplete`, and `Released` conditions. + +Detailed Pod and MPIJob evidence is retained in the durable evidence document, +not duplicated into CR status. + +## Terminal behavior + +The controller never releases a participant before evidence is durable and +all Workflow and backend resources are gone. + +- `Succeeded`: cleanup completes and participants are released automatically. +- `Failed`, `Inconclusive`, or `Canceled`: cleanup completes, but participants + remain reserved and cordoned. +- Set `spec.release: true` to release a non-successful Run after inspection. +- Deleting a Run requests teardown. Its finalizer stops and removes owned + workloads, verifies participant identity and reservation ownership, removes + owned cordons and reservations, deletes the script bundle, and then permits + deletion. + +Release uncordons a Node only when all of these still match: + +- the snapshotted Node UID; +- `signal.unbounded-cloud.io/reserved-by=`; +- `signal.unbounded-cloud.io/reserved-node-uid=`. + +Identity replacement or ownership loss is skipped during release. Signal +never uncordons or removes markers from a Node it cannot prove it still owns, +but it continues releasing the remaining exact identity matches. + +Cancel a live Run: + +```bash +kubectl patch certificationrun -n project-signal \ + --type=merge -p '{"spec":{"cancel":true}}' ``` -`deadline` is the outer Workflow deadline for backend observation, result and -evidence projection, and cleanup. Each check's `backendActiveDeadline` is -rendered to its participant Pod or MPIJob `activeDeadlineSeconds`. It must be -positive and leave at least 60 seconds before the outer deadline; the -controller rejects an invalid ordering before reserving Nodes. - -All checks use fixed output limits. Together, the 15 checks stay below the -768 KiB Workflow status budget, and every accepted plan can represent its -mandatory partial-failure envelope. - -`hostNetwork: true` is an explicit participant policy for diagnostics that must -observe the selected Node's network namespace, such as physical NIC inventory. -The adapter sets `dnsPolicy: ClusterFirstWithHostNet` with it. For MPIJobs this -applies only to the Node-pinned worker; the unpinned launcher remains outside -the host network namespace. Omitting the field keeps Kubernetes' default Pod -networking. - -`spec.checks[].gpuCount` may be zero for Pod diagnostics and must be positive -for MPIJobs. The controller renders it as `gpu-resource-count` for the adapter. -The MPIJob policy is -fail-closed: `backoffLimit: 0`, `cleanPodPolicy: All`, a one-hour post-finish -TTL, replica-level `restartPolicy: Never`, `WaitForWorkersReady`, and -`runLauncherAsWorker: false`. - -A `CertificationProfile` defines exactly 15 ordered checks. Admission snapshots -the complete plan and selected participants once into one immutable -`CertificationRun`. Each check binds its script identity and digest, runner, -backend, workload and topology policy, runtime inputs, deadline, and -participant-role inclusion or exclusion. Every run is a one-time full -certification; alternate purposes and retries are not public configuration. - -The controller creates one Argo Workflow with an inline DAG entrypoint. Every -task invokes the configured adapter `certification` template through -`templateRef`; dependencies make execution serial and visible in -`status.nodes`. `executionPolicy.onFailure` is immutable and is either `Stop` -or `Continue`. A stopped task is recorded as skipped/not-run, never inferred as -a diagnostic result. - -Before creating the Workflow, the controller durably records its deterministic -name and attempt in CertificationRun status. Once that creation intent exists, -a missing Workflow terminates the Run as Inconclusive and is never recreated. -This fail-closed fence prevents a controller outage plus Workflow TTL cleanup -from executing the one-time certification suite twice. - -Admission also remains fail-closed until every selected Node has received the -pending projection. The Run is retained as the durable owner while a failed -Node patch is retried; its Workflow cannot start until admission is marked -ready. - -Participant roles and topology constraints are generic typed policy. Profiles -can, for example, bind one role to one participant, another role to three -participants, require the latter to span at least two values of an operator -chosen topology label, and exclude a role from a specific check without -publishing environment-specific names or label values upstream. - -## Results and evidence - -Workflow infrastructure state and certification verdicts are separate: - -```text -executionState=Completed, verdict=Pass|Fail, reason= -executionState=Error, verdict=Inconclusive, reason= +Release a retained terminal Run: + +```bash +kubectl patch certificationrun -n project-signal \ + --type=merge -p '{"spec":{"release":true}}' ``` -Ambiguous or incomplete workflow execution remains `Inconclusive`. The -controller identifies each check only by its deterministic DAG task-node name -and rejects missing, duplicate, malformed, partial, or unexpected outputs. An -aggregate `Pass` requires all 15 checks, exact participant and backend -coverage, complete v1 backend evidence, and successful backend cleanup. -If backend execution and cleanup both fail, `reason` retains the first causal -backend failure and the bounded `cleanup-failure` Workflow output records -`BackendCleanupFailed` separately. Cleanup failure always forces -`executionState=Error` and `verdict=Inconclusive`, including after an otherwise -successful diagnostic. - -CertificationRun status retains bounded measurements, backend identities, and -the SHA-256 digest and completeness of each check's backend-evidence v1 -document. Full backend-evidence content and script snapshots are retained only -in the immutable terminal evidence artifact. Small artifacts use one -ConfigMap; larger artifacts use an immutable manifest plus bounded immutable -chunks so valid grouped evidence cannot exceed one Kubernetes object limit. - -## Node metadata - -Controller-owned labels: - -| Key | Values | -| --- | --- | -| `signal.unbounded-cloud.io/state` | `pending`, `testing`, `passed`, `failed`, `inconclusive` | -| `signal.unbounded-cloud.io/eligibility` | `blocked`, `production` | -| `signal.unbounded-cloud.io/profile` | bounded profile identifier | - -Pending, testing, failed, and inconclusive states use -`eligibility=blocked`. A successful run projects `passed/production`. Older -runs cannot overwrite a newer terminal projection. -Projection and request clearing also require the current reservation to name -the same Run, so historical Runs cannot overwrite a newer pending/testing run -or consume its request. Completed finalizer cleanup is not replayed. +## Evidence + +The evidence document includes: + +- Run, Profile, participant, script-bundle, and Workflow identities; +- the immutable check snapshots and script contents; +- aggregate execution state, verdict, reason, phase, and timestamps; +- each positional check's verdict and timing; +- typed Pod or MPIJob evidence, including Pods, containers, states, bounded + logs, and capture errors; +- backend identities and participant positions; +- a per-check cleanup disposition showing that deletion is deferred to the + controller. + +The adapter persists typed backend evidence and leaves completed resources in +place. MPI worker evidence is captured when still present; a worker may already +be absent because `cleanPodPolicy: Running` releases its GPU after launcher +completion. If an uncertain backend cannot be quiesced, the adapter task fails +and later checks are skipped rather than overlapping execution. The controller +deletes the Workflow and all remaining owned backends only after the final +evidence document is durable. +`WorkloadCleanupComplete=True` is the authoritative record that this +controller-owned cleanup finished. ## Operations -Request a run by applying the stable trigger label: +Inspect Runs and lifecycle conditions: ```bash -kubectl label node \ - signal.unbounded-cloud.io/certification-request= +kubectl get certificationruns -n project-signal +kubectl describe certificationrun -n project-signal ``` -Cancel an active run: +Inspect the immutable script bundle before terminal cleanup: ```bash -kubectl -n project-signal patch certificationrun \ - --type=merge -p '{"spec":{"cancel":true}}' +kubectl get configmap -n project-signal -scripts -o yaml ``` -Before removing the controller, wait for active grouped runs to become -terminal. Argo Workflows and any workload backend may be shared -cluster services and must not be removed as part of Project Signal cleanup. +Controller restarts are safe: -## Non-goals +- Workflow creation intent is recorded before creation; +- a missing Workflow after durable intent becomes Inconclusive and is not + recreated; +- positional identities are re-derived after every Run read; +- persisted terminal evidence can reconstruct compact Run status; +- owned premature uncordons are repaired while a Run is active or retained. -This upstream integration does not provide: +## Non-goals -- certification scripts or proprietary thresholds; -- a runner image; -- bundled CertificationProfiles or WorkflowTemplates; -- workload-backend installation or runner RBAC; -- destructive stress, reboot, cordon, drain, or remediation; -- an external evidence service; -- authoritative recovery of ambiguous external mutations. +- Kueue or other queue integration; +- recurring certification or automatic retry; +- arbitrary WorkflowTemplate, PodSpec, toleration, capability, or host-path + pass-through; +- general-purpose workload scheduling; +- live Node draining or eviction of workloads that existed before the cordon. diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml index 4f424c7ac2..4c18a27065 100644 --- a/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml @@ -52,39 +52,19 @@ spec: items: properties: backend: - properties: - apiVersion: - type: string - kind: - type: string - required: - - apiVersion - - kind - type: object + enum: + - Pod + - MPIJob + type: string backendActiveDeadline: type: string gpuCount: format: int32 minimum: 0 type: integer - id: - maxLength: 63 - minLength: 1 - pattern: ^R(0[1-9]|1[0-5])$ - type: string - order: - format: int32 - maximum: 14 - minimum: 0 - type: integer participantPolicy: properties: - excludedRoles: - items: - type: string - maxItems: 16 - type: array - requiredRoles: + roles: items: type: string maxItems: 16 @@ -92,16 +72,6 @@ spec: type: object runnerImage: type: string - runtimeStage: - properties: - name: - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - required: - - name - type: object script: properties: digest: @@ -113,8 +83,6 @@ spec: type: string name: type: string - requiresRuntimeStage: - type: boolean required: - digest - key @@ -123,6 +91,7 @@ spec: scriptEnvironment: additionalProperties: type: string + maxProperties: 32 type: object workloadPolicy: properties: @@ -138,15 +107,10 @@ spec: properties: mountPath: type: string - name: - type: string path: type: string - type: - type: string required: - mountPath - - name - path type: object maxItems: 16 @@ -220,35 +184,21 @@ spec: required: - backend - backendActiveDeadline - - id - - order - runnerImage - script type: object x-kubernetes-validations: - message: backendActiveDeadline must be positive rule: duration(self.backendActiveDeadline) > duration('0s') - - message: runtimeStage is required when the script requires a runtime - stage - rule: '!has(self.script.requiresRuntimeStage) || !self.script.requiresRuntimeStage - || has(self.runtimeStage)' + - message: scriptEnvironment names must not exceed 253 characters + and values must not exceed 4096 characters + rule: '!has(self.scriptEnvironment) || self.scriptEnvironment.all(k, + size(k) <= 253 && size(self.scriptEnvironment[k]) <= 4096)' maxItems: 15 minItems: 15 type: array deadline: type: string - evidencePath: - type: string - executionPolicy: - properties: - onFailure: - enum: - - Stop - - Continue - type: string - required: - - onFailure - type: object participantRoles: items: properties: @@ -322,7 +272,7 @@ spec: maximum: 64 minimum: 1 type: integer - requiredResources: + minimumAllocatable: additionalProperties: anyOf: - type: integer @@ -385,45 +335,6 @@ spec: - count - selector type: object - tolerations: - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array topologyConstraints: items: properties: @@ -447,34 +358,20 @@ spec: type: object maxItems: 16 type: array - workflow: - properties: - name: - type: string - required: - - name - type: object required: - checks - deadline - - evidencePath - - executionPolicy - participants - - workflow type: object x-kubernetes-validations: - - message: checks must contain unique IDs - rule: self.checks.all(c, self.checks.exists_one(x, x.id == c.id)) - - message: checks must use immutable contiguous order 0 through 14 - rule: self.checks.map(c, c.order) == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 10, 11, 12, 13, 14] - - message: checks must use ordered IDs R01 through R15 - rule: self.checks.map(c, c.id) == ['R01', 'R02', 'R03', 'R04', 'R05', - 'R06', 'R07', 'R08', 'R09', 'R10', 'R11', 'R12', 'R13', 'R14', 'R15'] - message: every backendActiveDeadline must leave at least 60 seconds before deadline rule: self.checks.all(c, duration(self.deadline) - duration(c.backendActiveDeadline) >= duration('60s')) + - message: deadline must cover every sequential backendActiveDeadline + plus 60 seconds of evidence time per check + rule: self.checks.map(c, duration(c.backendActiveDeadline) + duration('60s')).sum() + <= duration(self.deadline) required: - spec type: object diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml index f4f4512d66..2d61c5a6e9 100644 --- a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml @@ -25,9 +25,6 @@ spec: - jsonPath: .status.phase name: Phase type: string - - jsonPath: .status.verdict - name: Verdict - type: string - jsonPath: .status.workflow.name name: Workflow type: string @@ -63,38 +60,18 @@ spec: items: properties: backend: - properties: - apiVersion: - type: string - kind: - type: string - required: - - apiVersion - - kind - type: object + enum: + - Pod + - MPIJob + type: string backendActiveDeadline: type: string gpuCount: format: int32 type: integer - id: - maxLength: 63 - minLength: 1 - pattern: ^R(0[1-9]|1[0-5])$ - type: string - order: - format: int32 - maximum: 14 - minimum: 0 - type: integer participantPolicy: properties: - excludedRoles: - items: - type: string - maxItems: 16 - type: array - requiredRoles: + roles: items: type: string maxItems: 16 @@ -102,20 +79,8 @@ spec: type: object runnerImage: type: string - runtimeStage: - properties: - name: - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - required: - - name - type: object script: properties: - configMapName: - type: string digest: description: Digest is the algorithm-qualified SHA-256 digest of the selected script bytes. @@ -123,16 +88,14 @@ spec: type: string key: type: string - requiresRuntimeStage: - type: boolean required: - - configMapName - digest - key type: object scriptEnvironment: additionalProperties: type: string + maxProperties: 32 type: object workloadPolicy: properties: @@ -148,15 +111,10 @@ spec: properties: mountPath: type: string - name: - type: string path: type: string - type: - type: string required: - mountPath - - name - path type: object maxItems: 16 @@ -230,101 +188,21 @@ spec: required: - backend - backendActiveDeadline - - id - - order - runnerImage - script type: object x-kubernetes-validations: - message: backendActiveDeadline must be positive rule: duration(self.backendActiveDeadline) > duration('0s') - - message: runtimeStage is required when the script requires a runtime - stage - rule: '!has(self.script.requiresRuntimeStage) || !self.script.requiresRuntimeStage - || has(self.runtimeStage)' + - message: scriptEnvironment names must not exceed 253 characters + and values must not exceed 4096 characters + rule: '!has(self.scriptEnvironment) || self.scriptEnvironment.all(k, + size(k) <= 253 && size(self.scriptEnvironment[k]) <= 4096)' maxItems: 15 minItems: 15 type: array deadline: type: string - evidencePath: - type: string - executionPolicy: - properties: - onFailure: - enum: - - Stop - - Continue - type: string - required: - - onFailure - type: object - participantRoles: - items: - properties: - count: - format: int32 - maximum: 64 - minimum: 1 - type: integer - name: - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - selector: - description: Selector further constrains participants assigned - to this role. - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - required: - - count - - name - type: object - maxItems: 16 - type: array participants: items: properties: @@ -338,20 +216,6 @@ spec: description: ResourceList is a set of (resource name, quantity) pairs. type: object - capacity: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: ResourceList is a set of (resource name, quantity) - pairs. - type: object - index: - format: int32 - minimum: 0 - type: integer nodeName: type: string nodeUID: @@ -360,13 +224,14 @@ spec: don't ONLY use UUIDs, this is an alias to string. Being a type captures intent and helps make sure that UIDs and names do not get conflated. type: string - roles: - items: + role: + type: string + topologyValues: + additionalProperties: type: string - maxItems: 16 - type: array + maxProperties: 16 + type: object required: - - index - nodeName - nodeUID type: object @@ -391,163 +256,75 @@ spec: - name - uid type: object - requestNodeName: - type: string - tolerations: - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - topologyConstraints: - items: - properties: - labelKey: - description: LabelKey is the Node label used to identify topology - domains. - type: string - minDistinctValues: - format: int32 - maximum: 64 - minimum: 1 - type: integer - role: - description: Role names the participant role whose selected - Nodes are evaluated. - type: string - required: - - labelKey - - minDistinctValues - - role - type: object - maxItems: 16 - type: array - workflow: + release: + type: boolean + scriptBundle: properties: name: type: string + uid: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string required: - name + - uid type: object required: - checks - deadline - - evidencePath - - executionPolicy - participants - profile - - requestNodeName - - workflow + - scriptBundle type: object x-kubernetes-validations: - message: profile is immutable rule: self.profile == oldSelf.profile - message: participants are immutable rule: self.participants == oldSelf.participants - - message: participantRoles are immutable - rule: has(self.participantRoles) == has(oldSelf.participantRoles) && - (!has(self.participantRoles) || self.participantRoles == oldSelf.participantRoles) - - message: topologyConstraints are immutable - rule: has(self.topologyConstraints) == has(oldSelf.topologyConstraints) - && (!has(self.topologyConstraints) || self.topologyConstraints == - oldSelf.topologyConstraints) + - message: scriptBundle is immutable + rule: self.scriptBundle == oldSelf.scriptBundle - message: checks are immutable rule: self.checks == oldSelf.checks - - message: executionPolicy is immutable - rule: self.executionPolicy == oldSelf.executionPolicy - - message: workflow is immutable - rule: self.workflow == oldSelf.workflow - - message: evidencePath is immutable - rule: self.evidencePath == oldSelf.evidencePath - message: deadline is immutable rule: self.deadline == oldSelf.deadline - - message: tolerations are immutable - rule: has(self.tolerations) == has(oldSelf.tolerations) && (!has(self.tolerations) - || self.tolerations == oldSelf.tolerations) - - message: requestNodeName is immutable - rule: self.requestNodeName == oldSelf.requestNodeName - - message: checks must contain unique IDs - rule: self.checks.all(c, self.checks.exists_one(x, x.id == c.id)) - - message: checks must use immutable contiguous order 0 through 14 - rule: self.checks.map(c, c.order) == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 10, 11, 12, 13, 14] - - message: checks must use ordered IDs R01 through R15 - rule: self.checks.map(c, c.id) == ['R01', 'R02', 'R03', 'R04', 'R05', - 'R06', 'R07', 'R08', 'R09', 'R10', 'R11', 'R12', 'R13', 'R14', 'R15'] + - message: cancel cannot be cleared + rule: '!has(oldSelf.cancel) || !oldSelf.cancel || (has(self.cancel) + && self.cancel)' + - message: release cannot be cleared + rule: '!has(oldSelf.release) || !oldSelf.release || (has(self.release) + && self.release)' - message: every backendActiveDeadline must leave at least 60 seconds before deadline rule: self.checks.all(c, duration(self.deadline) - duration(c.backendActiveDeadline) >= duration('60s')) + - message: deadline must cover every sequential backendActiveDeadline + plus 60 seconds of evidence time per check + rule: self.checks.map(c, duration(c.backendActiveDeadline) + duration('60s')).sum() + <= duration(self.deadline) status: properties: checks: items: properties: - attempt: - format: int32 - type: integer backendEvidence: properties: complete: type: boolean - content: - description: |- - Content is populated only while constructing the durable evidence - artifact and is not persisted in CertificationRun status. - type: string digest: description: |- Digest identifies the complete backend-evidence v1 document retained in the durable evidence artifact. pattern: ^sha256:[0-9a-f]{64}$ type: string - version: - type: string type: object backendResources: items: properties: - apiVersion: - type: string - kind: - type: string name: type: string - namespace: - type: string participantIndexes: items: format: int32 @@ -562,63 +339,38 @@ spec: intent and helps make sure that UIDs and names do not get conflated. type: string required: - - apiVersion - - kind - name + - participantIndexes - uid type: object + maxItems: 64 type: array - cleanup: - properties: - complete: - type: boolean - reason: - type: string - type: object - duration: - type: string finishedAt: format: date-time type: string - id: - type: string measurements: + maxLength: 16384 type: string - notRun: - type: boolean - order: - format: int32 - type: integer phase: + enum: + - Pending + - Running + - Succeeded + - Failed + - Inconclusive + - Skipped type: string reason: + maxLength: 1024 type: string - retryCount: - format: int32 - type: integer startedAt: format: date-time type: string - verdict: - type: string - workflowNodeName: - type: string required: - - attempt - - id - - order - phase - - retryCount type: object maxItems: 15 type: array - cleanup: - properties: - reservationsReleased: - type: boolean - snapshotDeleted: - type: boolean - type: object conditions: items: description: Condition contains details for one aspect of the current @@ -674,7 +426,11 @@ spec: - status - type type: object + maxItems: 8 type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map evidence: properties: digest: @@ -685,25 +441,29 @@ spec: uri: type: string type: object - executionState: - type: string finishedAt: format: date-time type: string + observedGeneration: + format: int64 + type: integer phase: + enum: + - Pending + - Running + - Succeeded + - Failed + - Inconclusive + - Canceled type: string reason: + maxLength: 1024 type: string startedAt: format: date-time type: string - verdict: - type: string workflow: properties: - attempt: - format: int32 - type: integer name: type: string phase: diff --git a/internal/signal/controller.go b/internal/signal/controller.go index 692c4481ea..568769d952 100644 --- a/internal/signal/controller.go +++ b/internal/signal/controller.go @@ -23,6 +23,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" @@ -81,16 +82,17 @@ var reservedScriptEnvironmentNames = map[string]struct{}{ var workflowOutputLimits = map[string]int{ "backend-resources": signaladapter.MaxBackendOutputBytes, "backend-evidence": signaladapter.MaxBackendEvidenceBytes, - "cleanup-failure": signaladapter.MaxCleanupFailureBytes, "measurements": signaladapter.MaxMeasurementsBytes, "reason": signaladapter.MaxReasonBytes, "verdict": 32, } const ( - groupedCheckCount = 15 - adapterServiceAccountName = "project-signal-adapter" - maximumWorkflowOutputBytes = 768 * 1024 + groupedCheckCount = 15 + adapterServiceAccountName = "project-signal-adapter" + adapterWorkflowTemplateName = "project-signal-participant-adapter" + maximumWorkflowOutputBytes = 768 * 1024 + maximumStatusTextRunes = 1024 ) var workflowGVK = schema.GroupVersionKind{ @@ -152,7 +154,17 @@ func (r *AdmissionReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( profileName := node.Labels[RequestLabel] if profileName == "" { - return ctrl.Result{}, nil + runName := node.Annotations[ReservationLabel] + if runName == "" || node.Labels[signalv1.ReservationNodeUIDLabel] != string(node.UID) { + return ctrl.Result{}, nil + } + + r.admissionMu.Lock() + defer r.admissionMu.Unlock() + + _, err := r.resumeOrReleaseReservation(ctx, runName) + + return ctrl.Result{}, err } return ctrl.Result{}, r.admitNode(ctx, &node, profileName) @@ -163,51 +175,10 @@ func (r *AdmissionReconciler) admitNode(ctx context.Context, node *corev1.Node, defer r.admissionMu.Unlock() if runName := node.Annotations[ReservationLabel]; runName != "" { - var run signalv1.CertificationRun - - reader := r.APIReader - if reader == nil { - reader = r.Client - } - - err := reader.Get(ctx, client.ObjectKey{Namespace: r.Namespace, Name: runName}, &run) - if err == nil { - return r.completeRunAdmission(ctx, &run) - } - - if !apierrors.IsNotFound(err) { + handled, err := r.resumeOrReleaseReservation(ctx, runName) + if err != nil || handled { return err } - - var snapshot corev1.ConfigMap - - err = reader.Get( - ctx, - client.ObjectKey{ - Namespace: r.Namespace, - Name: runName + "-scripts", - }, - &snapshot, - ) - if err == nil { - if snapshot.Labels[SnapshotRunLabel] != runName { - return fmt.Errorf( - "orphaned snapshot %s is not bound to reservation %q", - snapshot.Name, - runName, - ) - } - - if err := r.Delete(ctx, &snapshot); err != nil && !apierrors.IsNotFound(err) { - return fmt.Errorf("delete orphaned script snapshot: %w", err) - } - } else if !apierrors.IsNotFound(err) { - return fmt.Errorf("get orphaned script snapshot: %w", err) - } - - if err := r.releaseOrphanedReservations(ctx, runName); err != nil { - return fmt.Errorf("release orphaned Project Signal reservations: %w", err) - } } var profile signalv1.CertificationProfile @@ -216,7 +187,7 @@ func (r *AdmissionReconciler) admitNode(ctx context.Context, node *corev1.Node, } checks := profileChecks(&profile) - if err := validateCheckPlan(checks); err != nil { + if err := validateCheckPlan(checks, profile.Spec.Deadline.Duration); err != nil { return fmt.Errorf("profile %s checks: %w", profile.Name, err) } @@ -226,9 +197,7 @@ func (r *AdmissionReconciler) admitNode(ctx context.Context, node *corev1.Node, for index := range checks { check := &checks[index] - if err := signalv1.ValidateRuntimeContract( - check.Script.RequiresRuntimeStage, - check.RuntimeStage, + if err := signalv1.ValidateExecutionDeadlines( check.BackendActiveDeadline.Duration, profile.Spec.Deadline.Duration, ); err != nil { @@ -315,7 +284,7 @@ func (r *AdmissionReconciler) admitNode(ctx context.Context, node *corev1.Node, return fmt.Errorf("check %s workload policy: %w", checks[index].ID, err) } - if err := validateGPUCount(checks[index].GPUCount, checks[index].Backend.Kind); err != nil { + if err := validateGPUCount(checks[index].GPUCount, checks[index].Backend); err != nil { return fmt.Errorf("check %s GPU count: %w", checks[index].ID, err) } } @@ -360,19 +329,16 @@ func (r *AdmissionReconciler) admitNode(ctx context.Context, node *corev1.Node, UID: profile.UID, Generation: profile.Generation, }, - Participants: participants, - ParticipantRoles: append([]signalv1.ParticipantRoleConstraint(nil), profile.Spec.ParticipantRoles...), - TopologyConstraints: append([]signalv1.ParticipantTopologyConstraint(nil), profile.Spec.TopologyConstraints...), - Checks: checkSnapshots, - ExecutionPolicy: profile.Spec.ExecutionPolicy, - Workflow: profile.Spec.Workflow, - EvidencePath: strings.Trim(profile.Spec.EvidencePath, "/") + "/" + runName + ".json", - Deadline: profile.Spec.Deadline, - Tolerations: profile.Spec.Tolerations, - RequestNodeName: node.Name, + Participants: participants, + ScriptBundle: signalv1.ScriptBundleReference{ + Name: snapshot.Name, + UID: snapshot.UID, + }, + Checks: checkSnapshots, + Deadline: profile.Spec.Deadline, }, } - if _, err := buildWorkflow(run, 1); err != nil { + if _, err := buildWorkflow(run); err != nil { deleteErr := r.Delete(context.WithoutCancel(ctx), snapshot) return errors.Join(fmt.Errorf("validate executable plan: %w", err), client.IgnoreNotFound(deleteErr)) @@ -411,6 +377,56 @@ func (r *AdmissionReconciler) admitNode(ctx context.Context, node *corev1.Node, return nil } +func (r *AdmissionReconciler) resumeOrReleaseReservation(ctx context.Context, runName string) (bool, error) { + var run signalv1.CertificationRun + + reader := r.APIReader + if reader == nil { + reader = r.Client + } + + err := reader.Get(ctx, client.ObjectKey{Namespace: r.Namespace, Name: runName}, &run) + if err == nil { + return true, r.completeRunAdmission(ctx, &run) + } + + if !apierrors.IsNotFound(err) { + return false, err + } + + var snapshot corev1.ConfigMap + + err = reader.Get( + ctx, + client.ObjectKey{ + Namespace: r.Namespace, + Name: runName + "-scripts", + }, + &snapshot, + ) + if err == nil { + if snapshot.Labels[SnapshotRunLabel] != runName { + return false, fmt.Errorf( + "orphaned snapshot %s is not bound to reservation %q", + snapshot.Name, + runName, + ) + } + + if err := r.Delete(ctx, &snapshot); err != nil && !apierrors.IsNotFound(err) { + return false, fmt.Errorf("delete orphaned script snapshot: %w", err) + } + } else if !apierrors.IsNotFound(err) { + return false, fmt.Errorf("get orphaned script snapshot: %w", err) + } + + if err := r.releaseOrphanedReservations(ctx, runName); err != nil { + return false, fmt.Errorf("release orphaned Project Signal reservations: %w", err) + } + + return false, nil +} + func (r *AdmissionReconciler) completeRunAdmission( ctx context.Context, run *signalv1.CertificationRun, @@ -476,7 +492,8 @@ func (r *AdmissionReconciler) selectParticipants( return nil, fmt.Errorf("profile %s participant selector: %w", profile.Name, err) } - if !selector.Matches(labels.Set(requestNode.Labels)) || !nodeEligible(requestNode, profile.Spec.Participants.RequiredResources) { + if !selector.Matches(labels.Set(requestNode.Labels)) || + !nodeEligible(requestNode, profile.Spec.Participants.MinimumAllocatable) { return nil, fmt.Errorf("request node %s does not satisfy profile %s", requestNode.Name, profile.Name) } @@ -495,7 +512,7 @@ func (r *AdmissionReconciler) selectParticipants( if candidate.Name == requestNode.Name || !selector.Matches(labels.Set(candidate.Labels)) || - !nodeEligible(candidate, profile.Spec.Participants.RequiredResources) || + !nodeEligible(candidate, profile.Spec.Participants.MinimumAllocatable) || candidate.Annotations[ReservationLabel] != "" || candidate.Labels[signalv1.ReservationNodeUIDLabel] != "" { continue @@ -558,7 +575,6 @@ func participantSnapshot(node *corev1.Node, index int32) signalv1.Participant { Index: index, NodeName: node.Name, NodeUID: node.UID, - Capacity: node.Status.Capacity.DeepCopy(), Allocatable: node.Status.Allocatable.DeepCopy(), } } @@ -620,13 +636,36 @@ func selectParticipantPlan( participants := make([]signalv1.Participant, 0, count) for _, candidateIndex := range selectedIndexes { participant := participantSnapshot(&candidates[candidateIndex], int32(len(participants))) - participant.Roles = slices.Clone(assignments[candidateIndex]) + if len(assignments[candidateIndex]) > 0 { + participant.Role = assignments[candidateIndex][0] + } + + participant.TopologyValues = topologyValues(&candidates[candidateIndex], topology) participants = append(participants, participant) } return participants, nil } +func topologyValues( + node *corev1.Node, + constraints []signalv1.ParticipantTopologyConstraint, +) map[string]string { + values := make(map[string]string) + for index := range constraints { + key := constraints[index].LabelKey + if value, exists := node.Labels[key]; exists { + values[key] = value + } + } + + if len(values) == 0 { + return nil + } + + return values +} + func solveParticipantRoles( candidates []corev1.Node, requestNodeName string, @@ -942,6 +981,13 @@ func (r *AdmissionReconciler) reserveParticipants(ctx context.Context, participa ) } + if node.Spec.Unschedulable { + return errors.Join( + fmt.Errorf("participant %s became unschedulable before reservation", participant.NodeName), + r.releaseParticipants(context.WithoutCancel(ctx), reserved, runName), + ) + } + if value := node.Annotations[ReservationLabel]; value != "" { return errors.Join( fmt.Errorf("participant %s is already reserved by %s", participant.NodeName, value), @@ -967,6 +1013,7 @@ func (r *AdmissionReconciler) reserveParticipants(ctx context.Context, participa node.Annotations[ReservationLabel] = runName node.Labels[signalv1.ReservationNodeUIDLabel] = string(participant.NodeUID) + node.Spec.Unschedulable = true if err := r.Patch( ctx, @@ -999,6 +1046,13 @@ func (r *AdmissionReconciler) releaseParticipants(ctx context.Context, participa } if participant.NodeUID != "" && node.UID != participant.NodeUID { + errs = append(errs, fmt.Errorf( + "participant %s identity changed from %s to %s", + participant.NodeName, + participant.NodeUID, + node.UID, + )) + continue } @@ -1006,7 +1060,7 @@ func (r *AdmissionReconciler) releaseParticipants(ctx context.Context, participa continue } - if value := node.Labels[signalv1.ReservationNodeUIDLabel]; value != "" && value != string(participant.NodeUID) { + if value := node.Labels[signalv1.ReservationNodeUIDLabel]; value != string(participant.NodeUID) { errs = append(errs, fmt.Errorf( "participant %s reservation identity changed from %s to %s", participant.NodeName, @@ -1018,6 +1072,7 @@ func (r *AdmissionReconciler) releaseParticipants(ctx context.Context, participa } before := node.DeepCopy() + node.Spec.Unschedulable = false delete(node.Annotations, ReservationLabel) delete(node.Labels, signalv1.ReservationNodeUIDLabel) @@ -1086,15 +1141,12 @@ func (r *AdmissionReconciler) createSnapshot( ID: check.ID, Order: check.Order, Script: signalv1.ScriptSnapshot{ - ConfigMapName: snapshot.Name, - Key: key, - RequiresRuntimeStage: check.Script.RequiresRuntimeStage, - Digest: digest, + Key: key, + Digest: digest, }, RunnerImage: check.RunnerImage, Backend: check.Backend, WorkloadPolicy: *check.WorkloadPolicy.DeepCopy(), - RuntimeStage: copyRuntimeStage(check.RuntimeStage), GPUCount: check.GPUCount, ScriptEnvironment: maps.Clone(check.ScriptEnvironment), BackendActiveDeadline: check.BackendActiveDeadline, @@ -1115,6 +1167,8 @@ func (r *RunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R return ctrl.Result{}, client.IgnoreNotFound(err) } + hydrateRunDerivedFields(&run) + if !run.DeletionTimestamp.IsZero() { return r.reconcileDeleting(ctx, &run) } @@ -1127,6 +1181,10 @@ func (r *RunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R return r.cleanup(ctx, &run) } + if err := r.verifyScriptBundle(ctx, &run); err != nil { + return ctrl.Result{}, err + } + if run.Status.Evidence.URI == "" { result, recovered, err := r.resumeGroupedEvidence(ctx, &run) if err != nil || recovered { @@ -1138,12 +1196,21 @@ func (r *RunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R return r.cancel(ctx, &run) } + participantsOwned, err := r.ensureParticipantsCordoned(ctx, &run) + if err != nil { + return ctrl.Result{}, err + } + + if !participantsOwned { + return r.finishParticipantOwnershipLoss(ctx, &run) + } + if run.Annotations[RunAdmissionAnnotation] != runAdmissionReady { return ctrl.Result{RequeueAfter: time.Second}, nil } if run.Status.Workflow.Name == "" { - return r.startAttempt(ctx, &run, 1) + return r.startWorkflow(ctx, &run) } workflow := &unstructured.Unstructured{} @@ -1176,7 +1243,7 @@ func (r *RunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R } if run.Status.Workflow.UID == "" { - return r.recordWorkflowStarted(ctx, &run, workflow, run.Status.Workflow.Attempt) + return r.recordWorkflowStarted(ctx, &run, workflow) } if string(workflow.GetUID()) != run.Status.Workflow.UID { @@ -1226,15 +1293,197 @@ func (r *RunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R } } -func (r *RunReconciler) startAttempt(ctx context.Context, run *signalv1.CertificationRun, attempt int32) (ctrl.Result, error) { - workflow, err := buildWorkflow(run, attempt) +func (r *RunReconciler) verifyScriptBundle(ctx context.Context, run *signalv1.CertificationRun) error { + reader := r.APIReader + if reader == nil { + reader = r.Client + } + + var snapshot corev1.ConfigMap + if err := reader.Get( + ctx, + client.ObjectKey{Namespace: run.Namespace, Name: run.Spec.ScriptBundle.Name}, + &snapshot, + ); err != nil { + return fmt.Errorf("read immutable script bundle: %w", err) + } + + if snapshot.UID != run.Spec.ScriptBundle.UID || snapshot.Labels[SnapshotRunLabel] != run.Name { + return errors.New("script bundle identity does not match the run") + } + + if snapshot.Immutable == nil || !*snapshot.Immutable { + return errors.New("script bundle is not immutable") + } + + for index := range run.Spec.Checks { + check := &run.Spec.Checks[index] + content, found := snapshot.Data[check.Script.Key] + if !found { + return fmt.Errorf("script bundle is missing check %s key %s", check.ID, check.Script.Key) + } + + if digestString(content) != check.Script.Digest { + return fmt.Errorf("script bundle digest does not match check %s", check.ID) + } + } + + return nil +} + +func hydrateRunDerivedFields(run *signalv1.CertificationRun) { + for index := range run.Spec.Participants { + run.Spec.Participants[index].Index = int32(index) + } + + for index := range run.Spec.Checks { + run.Spec.Checks[index].ID = signalv1.CheckID(fmt.Sprintf("R%02d", index+1)) + run.Spec.Checks[index].Order = int32(index) + } + + if len(run.Status.Checks) == len(run.Spec.Checks) { + for index := range run.Status.Checks { + run.Status.Checks[index].ID = run.Spec.Checks[index].ID + run.Status.Checks[index].Order = run.Spec.Checks[index].Order + } + } +} + +func (r *RunReconciler) ensureParticipantsCordoned( + ctx context.Context, + run *signalv1.CertificationRun, +) (bool, error) { + allOwned := true + + for _, participant := range run.Spec.Participants { + var node corev1.Node + if err := r.Get(ctx, client.ObjectKey{Name: participant.NodeName}, &node); err != nil { + if apierrors.IsNotFound(err) { + allOwned = false + + continue + } + + return false, fmt.Errorf("get cordoned participant %s: %w", participant.NodeName, err) + } + + if node.UID != participant.NodeUID { + allOwned = false + + continue + } + + if node.Annotations[ReservationLabel] != run.Name || + node.Labels[signalv1.ReservationNodeUIDLabel] != string(participant.NodeUID) { + allOwned = false + + continue + } + + if node.Spec.Unschedulable { + continue + } + + before := node.DeepCopy() + node.Spec.Unschedulable = true + if err := r.Patch( + ctx, + &node, + client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{}), + ); err != nil { + return false, fmt.Errorf("restore participant %s cordon: %w", participant.NodeName, err) + } + } + + status := metav1.ConditionTrue + reason := "ParticipantsCordoned" + message := "All participants are reserved and cordoned by this Run" + if !allOwned { + status = metav1.ConditionFalse + reason = "ParticipantOwnershipLost" + message = "One or more snapshotted participants are missing, replaced, or no longer owned by this Run" + } + + if err := r.patchRunCondition( + ctx, + run, + signalv1.RunConditionCordoned, + status, + reason, + message, + ); err != nil { + return false, err + } + + return allOwned, nil +} + +func (r *RunReconciler) finishParticipantOwnershipLoss( + ctx context.Context, + run *signalv1.CertificationRun, +) (ctrl.Result, error) { + workflow, err := r.workflowForCancellation(ctx, run) + if err != nil { + return ctrl.Result{}, err + } + + if workflow == nil { + workflow = &unstructured.Unstructured{} + workflow.SetGroupVersionKind(workflowGVK) + workflow.SetNamespace(run.Namespace) + workflow.SetName(fmt.Sprintf("%s-a1", run.Name)) + } else { + workflow = workflow.DeepCopy() + } + + if err := unstructured.SetNestedField(workflow.Object, "Error", "status", "phase"); err != nil { + return ctrl.Result{}, err + } + + if err := unstructured.SetNestedField( + workflow.Object, + "participant identity or reservation ownership changed", + "status", + "message", + ); err != nil { + return ctrl.Result{}, err + } + + return r.finishGrouped(ctx, run, workflow, "Error") +} + +func (r *RunReconciler) patchRunCondition( + ctx context.Context, + run *signalv1.CertificationRun, + conditionType string, + status metav1.ConditionStatus, + reason string, + message string, +) error { + before := run.DeepCopy() + run.Status.ObservedGeneration = run.Generation + apimeta.SetStatusCondition(&run.Status.Conditions, metav1.Condition{ + Type: conditionType, + Status: status, + ObservedGeneration: run.Generation, + Reason: reason, + Message: message, + }) + + if reflect.DeepEqual(before.Status, run.Status) { + return nil + } + + return r.Status().Patch(ctx, run, client.MergeFrom(before)) +} + +func (r *RunReconciler) startWorkflow(ctx context.Context, run *signalv1.CertificationRun) (ctrl.Result, error) { + workflow, err := buildWorkflow(run) if err != nil { before := run.DeepCopy() now := metav1.NewTime(r.now()) run.Status.Phase = signalv1.RunPhaseInconclusive - run.Status.ExecutionState = "Error" - run.Status.Verdict = "Inconclusive" - run.Status.Reason = "InvalidExecutionPlan: " + err.Error() + run.Status.Reason = boundedStatusText("InvalidExecutionPlan: " + err.Error()) run.Status.FinishedAt = &now return ctrl.Result{RequeueAfter: time.Millisecond}, r.Status().Patch(ctx, run, client.MergeFrom(before)) @@ -1242,9 +1491,8 @@ func (r *RunReconciler) startAttempt(ctx context.Context, run *signalv1.Certific before := run.DeepCopy() run.Status.Workflow = signalv1.WorkflowStatus{ - Name: workflow.GetName(), - Attempt: attempt, - Phase: "Creating", + Name: workflow.GetName(), + Phase: "Creating", } run.Status.Checks = initialCheckStatuses(run.Spec.Checks) @@ -1254,41 +1502,37 @@ func (r *RunReconciler) startAttempt(ctx context.Context, run *signalv1.Certific if err := r.Create(ctx, workflow); err != nil { if !apierrors.IsAlreadyExists(err) { - return ctrl.Result{}, fmt.Errorf("create Workflow attempt %d: %w", attempt, err) + return ctrl.Result{}, fmt.Errorf("create Workflow: %w", err) } existing := &unstructured.Unstructured{} existing.SetGroupVersionKind(workflowGVK) - if getErr := r.Get(ctx, client.ObjectKeyFromObject(workflow), existing); getErr != nil { - return ctrl.Result{}, fmt.Errorf("get existing Workflow attempt %d: %w", attempt, getErr) + return ctrl.Result{}, fmt.Errorf("get existing Workflow: %w", getErr) } if !metav1.IsControlledBy(existing, run) { - return ctrl.Result{}, fmt.Errorf("workflow attempt %d already exists without matching CertificationRun ownership", attempt) + return ctrl.Result{}, errors.New("Workflow already exists without matching CertificationRun ownership") } workflow = existing } - return r.recordWorkflowStarted(ctx, run, workflow, attempt) + return r.recordWorkflowStarted(ctx, run, workflow) } func (r *RunReconciler) recordWorkflowStarted( ctx context.Context, run *signalv1.CertificationRun, workflow *unstructured.Unstructured, - attempt int32, ) (ctrl.Result, error) { before := run.DeepCopy() run.Status.Phase = signalv1.RunPhaseRunning - run.Status.ExecutionState = "Running" run.Status.Workflow = signalv1.WorkflowStatus{ - Name: workflow.GetName(), - UID: string(workflow.GetUID()), - Attempt: attempt, - Phase: "Pending", + Name: workflow.GetName(), + UID: string(workflow.GetUID()), + Phase: "Pending", } run.Status.Checks = initialCheckStatuses(run.Spec.Checks) @@ -1298,35 +1542,24 @@ func (r *RunReconciler) recordWorkflowStarted( } if err := r.Status().Patch(ctx, run, client.MergeFrom(before)); err != nil { - return ctrl.Result{}, fmt.Errorf("record Workflow attempt: %w", err) + return ctrl.Result{}, fmt.Errorf("record Workflow identity: %w", err) } if r.Recorder != nil { r.Recorder.Eventf(run, nil, corev1.EventTypeNormal, "WorkflowStarted", "StartWorkflow", - "Created Workflow %s for attempt %d", workflow.GetName(), attempt) + "Created Workflow %s", workflow.GetName()) } return ctrl.Result{}, nil } -func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured.Unstructured, error) { +func buildWorkflow(run *signalv1.CertificationRun) (*unstructured.Unstructured, error) { checks := run.Spec.Checks - if err := validateRunCheckPlan(checks); err != nil { + if err := validateRunCheckPlan(checks, run.Spec.Deadline.Duration); err != nil { return nil, err } - tolerations := uniqueTolerations(append([]corev1.Toleration{{ - Key: "nvidia.com/gpu", - Operator: corev1.TolerationOpExists, - Effect: corev1.TaintEffectNoSchedule, - }}, run.Spec.Tolerations...)) - - tolerationsJSON, err := json.Marshal(tolerations) - if err != nil { - return nil, fmt.Errorf("encode Workflow tolerations: %w", err) - } - - name := fmt.Sprintf("%s-a%d", run.Name, attempt) + name := run.Name + "-a1" tasks := make([]any, 0, len(checks)) for index := range checks { @@ -1359,17 +1592,15 @@ func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured task := map[string]any{ "name": taskName, "templateRef": map[string]any{ - "name": run.Spec.Workflow.Name, + "name": adapterWorkflowTemplateName, "template": "certification", }, "arguments": map[string]any{ "parameters": checkWorkflowParameters( run, check, - attempt, checkParticipants, string(participantsJSON), - string(tolerationsJSON), string(environmentJSON), workloadPolicyJSON, strconv.FormatInt(int64(check.GPUCount), 10), @@ -1379,18 +1610,7 @@ func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured if index > 0 { previous := checkTaskName(&checks[index-1]) - if run.Spec.ExecutionPolicy.OnFailure == signalv1.SuiteFailurePolicyContinue { - task["depends"] = fmt.Sprintf( - "%[1]s.Succeeded || %[1]s.Failed || %[1]s.Errored || %[1]s.Skipped || %[1]s.Omitted", - previous, - ) - } else { - task["depends"] = previous + ".Succeeded" - task["when"] = fmt.Sprintf( - "{{tasks.%s.outputs.parameters.verdict}} == Pass", - previous, - ) - } + task["depends"] = previous + ".Succeeded" } tasks = append(tasks, task) @@ -1409,16 +1629,7 @@ func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured "spec": map[string]any{ "activeDeadlineSeconds": int64(run.Spec.Deadline.Seconds()), "serviceAccountName": adapterServiceAccountName, - "ttlStrategy": map[string]any{ - "secondsAfterCompletion": int64(6 * time.Hour / time.Second), - "secondsAfterSuccess": int64(6 * time.Hour / time.Second), - "secondsAfterFailure": int64(6 * time.Hour / time.Second), - }, - "podGC": map[string]any{ - "strategy": "OnWorkflowCompletion", - "deleteDelayDuration": "2h", - }, - "entrypoint": "certification-suite", + "entrypoint": "certification-suite", "templates": []any{ map[string]any{ "name": "certification-suite", @@ -1440,10 +1651,8 @@ func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured func checkWorkflowParameters( run *signalv1.CertificationRun, check *signalv1.CertificationCheckSnapshot, - attempt int32, participants []signalv1.Participant, participantsJSON string, - tolerationsJSON string, scriptEnvironmentJSON string, workloadPolicyJSON string, gpuCount string, @@ -1453,31 +1662,27 @@ func checkWorkflowParameters( map[string]any{"name": "run-uid", "value": string(run.UID)}, map[string]any{"name": "participants-json", "value": participantsJSON}, map[string]any{"name": "participant-count", "value": fmt.Sprintf("%d", len(participants))}, - map[string]any{"name": "script-configmap", "value": check.Script.ConfigMapName}, + map[string]any{"name": "script-configmap", "value": run.Spec.ScriptBundle.Name}, map[string]any{"name": "script-key", "value": check.Script.Key}, map[string]any{"name": "script-digest", "value": check.Script.Digest}, - map[string]any{ - "name": "script-requires-runtime-stage", - "value": fmt.Sprintf("%t", check.Script.RequiresRuntimeStage), - }, - map[string]any{"name": "runtime-stage-name", "value": runtimeStageName(check.RuntimeStage)}, - map[string]any{"name": "backend-api-version", "value": check.Backend.APIVersion}, - map[string]any{"name": "backend-kind", "value": check.Backend.Kind}, + map[string]any{"name": "backend-kind", "value": string(check.Backend)}, map[string]any{ "name": "backend-active-deadline-seconds", "value": fmt.Sprintf("%d", int64(check.BackendActiveDeadline.Seconds())), }, map[string]any{"name": "runner-image", "value": check.RunnerImage}, - map[string]any{"name": "tolerations", "value": tolerationsJSON}, map[string]any{"name": "script-environment", "value": scriptEnvironmentJSON}, map[string]any{"name": "workload-policy", "value": workloadPolicyJSON}, map[string]any{"name": "gpu-resource-count", "value": gpuCount}, - map[string]any{"name": "attempt", "value": fmt.Sprintf("%d", attempt)}, } } -func validateGPUCount(gpuCount int32, backendKind string) error { - if gpuCount < 0 || (backendKind == "MPIJob" && gpuCount == 0) { +func validateGPUCount(gpuCount int32, backend signalv1.ExecutionBackend) error { + if backend != signalv1.ExecutionBackendPod && backend != signalv1.ExecutionBackendMPIJob { + return fmt.Errorf("unsupported backend %q", backend) + } + + if gpuCount < 0 || (backend == signalv1.ExecutionBackendMPIJob && gpuCount == 0) { return errors.New("must be nonnegative and positive for MPIJob") } @@ -1502,6 +1707,10 @@ func validEnvironmentName(value string) bool { } func validateScriptEnvironment(environment map[string]string) error { + if len(environment) > 32 { + return errors.New("must not contain more than 32 entries") + } + keys := make([]string, 0, len(environment)) for key := range environment { keys = append(keys, key) @@ -1510,6 +1719,10 @@ func validateScriptEnvironment(environment map[string]string) error { sort.Strings(keys) for _, name := range keys { + if len(name) > 253 { + return fmt.Errorf("name %q exceeds 253 bytes", name) + } + if !validEnvironmentName(name) { return fmt.Errorf("name %q is invalid", name) } @@ -1521,6 +1734,10 @@ func validateScriptEnvironment(environment map[string]string) error { if _, reserved := reservedScriptEnvironmentNames[name]; reserved { return fmt.Errorf("name %q is reserved", name) } + + if len(environment[name]) > 4096 { + return fmt.Errorf("value for %q exceeds 4096 bytes", name) + } } return nil @@ -1546,8 +1763,6 @@ func (r *RunReconciler) finishGrouped( evidenceRun := run.DeepCopy() evidenceRun.Status.Checks = checks - evidenceRun.Status.ExecutionState = executionState - evidenceRun.Status.Verdict = verdict evidenceRun.Status.Reason = reason evidenceRun.Status.Phase = runPhaseForVerdict(verdict) evidenceRun.Status.Workflow.Phase = workflowPhase @@ -1566,7 +1781,7 @@ func (r *RunReconciler) finishGrouped( return ctrl.Result{}, err } - uri, digest, err := r.Evidence.Put(ctx, run.Spec.EvidencePath, evidence) + uri, digest, err := r.Evidence.Put(ctx, evidencePath(run), evidence) if err != nil { return ctrl.Result{}, err } @@ -1576,6 +1791,7 @@ func (r *RunReconciler) finishGrouped( run.Status.Checks = compactCheckStatuses(checks) run.Status.Evidence = signalv1.EvidenceStatus{URI: uri, Digest: digest} run.Status.Phase = runPhaseForVerdict(verdict) + run.Status.Reason = boundedStatusText(reason) if err := r.Status().Patch(ctx, run, client.MergeFrom(before)); err != nil { return ctrl.Result{}, err @@ -1592,7 +1808,7 @@ func (r *RunReconciler) resumeGroupedEvidence( ctx context.Context, run *signalv1.CertificationRun, ) (ctrl.Result, bool, error) { - data, uri, digest, found, err := r.Evidence.Get(ctx, run.Spec.EvidencePath) + data, uri, digest, found, err := r.Evidence.Get(ctx, evidencePath(run)) if err != nil || !found { return ctrl.Result{}, false, err } @@ -1601,7 +1817,7 @@ func (r *RunReconciler) resumeGroupedEvidence( RunID string `json:"runID"` RunName string `json:"runName"` Workflow signalv1.WorkflowStatus `json:"workflow"` - CheckStatus []signalv1.CheckStatus `json:"checkStatus"` + CheckStatus []evidenceCheckStatus `json:"checkStatus"` ExecutionState string `json:"executionState"` Verdict string `json:"verdict"` Reason string `json:"reason"` @@ -1616,10 +1832,18 @@ func (r *RunReconciler) resumeGroupedEvidence( return ctrl.Result{}, false, fmt.Errorf("persisted grouped evidence identity does not match run %s", run.Name) } - if document.Workflow.Name != "" && - (document.Workflow.Name != fmt.Sprintf("%s-a1", run.Name) || document.Workflow.UID == "" || - (run.Status.Workflow.UID != "" && document.Workflow.UID != run.Status.Workflow.UID)) { - return ctrl.Result{}, false, errors.New("persisted evidence Workflow identity does not match run") + if document.Workflow.Name != "" { + if document.Workflow.Name != fmt.Sprintf("%s-a1", run.Name) || + (run.Status.Workflow.UID != "" && document.Workflow.UID != run.Status.Workflow.UID) { + return ctrl.Result{}, false, errors.New("persisted evidence Workflow identity does not match run") + } + + neverObserved := document.Workflow.UID == "" && + ((document.Phase == signalv1.RunPhaseInconclusive && document.Workflow.Phase == "Error") || + (document.Phase == signalv1.RunPhaseCancelled && document.Workflow.Phase == "Canceled")) + if document.Workflow.UID == "" && !neverObserved { + return ctrl.Result{}, false, errors.New("persisted evidence is missing the observed Workflow UID") + } } if len(document.CheckStatus) != len(run.Spec.Checks) { @@ -1646,10 +1870,8 @@ func (r *RunReconciler) resumeGroupedEvidence( } before := run.DeepCopy() - run.Status.Checks = compactCheckStatuses(document.CheckStatus) - run.Status.ExecutionState = document.ExecutionState - run.Status.Verdict = document.Verdict - run.Status.Reason = document.Reason + run.Status.Checks = compactEvidenceCheckStatuses(document.CheckStatus) + run.Status.Reason = boundedStatusText(document.Reason) run.Status.Workflow = document.Workflow run.Status.Evidence = signalv1.EvidenceStatus{URI: uri, Digest: digest} @@ -1734,11 +1956,9 @@ func initialCheckStatuses(checks []signalv1.CertificationCheckSnapshot) []signal statuses := make([]signalv1.CheckStatus, len(checks)) for index := range checks { statuses[index] = signalv1.CheckStatus{ - ID: checks[index].ID, - Order: checks[index].Order, - Phase: signalv1.CheckPhasePending, - Attempt: 1, - RetryCount: 0, + ID: checks[index].ID, + Order: checks[index].Order, + Phase: signalv1.CheckPhasePending, } } @@ -1750,17 +1970,31 @@ func compactCheckStatuses(statuses []signalv1.CheckStatus) []signalv1.CheckStatu for index := range statuses { compacted[index] = *statuses[index].DeepCopy() compacted[index].BackendEvidence.Content = "" + compacted[index].Reason = boundedStatusText(compacted[index].Reason) } return compacted } +func boundedStatusText(value string) string { + runes := []rune(value) + if len(runes) <= maximumStatusTextRunes { + return value + } + + return string(runes[:maximumStatusTextRunes]) +} + func contentDigest(content string) string { sum := sha256.Sum256([]byte(content)) return fmt.Sprintf("sha256:%x", sum) } +func evidencePath(run *signalv1.CertificationRun) string { + return run.Name + ".json" +} + func runningCheckStatuses( workflow *unstructured.Unstructured, run *signalv1.CertificationRun, @@ -1790,7 +2024,6 @@ func runningCheckStatuses( statuses[index].Phase = signalv1.CheckPhaseInconclusive case "Omitted", "Skipped": statuses[index].Phase = signalv1.CheckPhaseSkipped - statuses[index].NotRun = true } setCheckTimes(&statuses[index], node) @@ -1811,7 +2044,6 @@ func terminalCheckStatuses( statuses[index].Phase = signalv1.CheckPhaseInconclusive statuses[index].Verdict = "Inconclusive" statuses[index].Reason = "WorkflowNodesUnavailable" - statuses[index].NotRun = true } return statuses, "Inconclusive", "workflow task nodes are missing or malformed" @@ -1819,7 +2051,6 @@ func terminalCheckStatuses( aggregateVerdict := "Pass" aggregateReason := "AllChecksPassed" - stoppedByPolicy := false for index := range run.Spec.Checks { check := &run.Spec.Checks[index] @@ -1830,7 +2061,6 @@ func terminalCheckStatuses( status.Phase = signalv1.CheckPhaseInconclusive status.Verdict = "Inconclusive" status.Reason = "WorkflowTaskIdentityInvalid" - status.NotRun = true aggregateVerdict = "Inconclusive" aggregateReason = "workflow did not contain exactly one task node for every check" @@ -1846,12 +2076,9 @@ func terminalCheckStatuses( status.Phase = signalv1.CheckPhaseSkipped status.Verdict = "Inconclusive" status.Reason = "NotRun" - status.NotRun = true - if run.Spec.ExecutionPolicy.OnFailure != signalv1.SuiteFailurePolicyStop || !stoppedByPolicy { - aggregateVerdict = "Inconclusive" - aggregateReason = "one or more checks were not run" - } + aggregateVerdict = "Inconclusive" + aggregateReason = "one or more checks were not run" continue case "Failed", "Error": @@ -1860,7 +2087,6 @@ func terminalCheckStatuses( status.Reason = "WorkflowTask" + phase aggregateVerdict = "Inconclusive" aggregateReason = "one or more check tasks failed at the workflow boundary" - stoppedByPolicy = true continue case "Succeeded": @@ -1868,10 +2094,8 @@ func terminalCheckStatuses( status.Phase = signalv1.CheckPhaseInconclusive status.Verdict = "Inconclusive" status.Reason = "WorkflowTaskIncomplete" - status.NotRun = true aggregateVerdict = "Inconclusive" aggregateReason = "one or more check tasks were incomplete" - stoppedByPolicy = true continue } @@ -1883,7 +2107,6 @@ func terminalCheckStatuses( status.Reason = "OutputsRejected: " + outputErr.Error() aggregateVerdict = "Inconclusive" aggregateReason = "one or more check outputs were malformed or incomplete" - stoppedByPolicy = true continue } @@ -1904,7 +2127,6 @@ func terminalCheckStatuses( status.Reason = "ParticipantPolicyInvalid: " + participantErr.Error() aggregateVerdict = "Inconclusive" aggregateReason = "one or more checks had invalid participant policy" - stoppedByPolicy = true continue } @@ -1923,13 +2145,11 @@ func terminalCheckStatuses( status.Reason = "BackendEvidenceRejected: " + evidenceErr.Error() aggregateVerdict = "Inconclusive" aggregateReason = "one or more checks had invalid backend evidence" - stoppedByPolicy = true continue } status.BackendEvidence = signalv1.BackendEvidenceStatus{ - Version: "v1", Complete: complete, Digest: contentDigest(normalizedEvidence), Content: normalizedEvidence, @@ -1947,22 +2167,20 @@ func terminalCheckStatuses( status.Reason = "BackendResourcesRejected: " + resourceErr.Error() aggregateVerdict = "Inconclusive" aggregateReason = "one or more checks had invalid backend identities" - stoppedByPolicy = true continue } status.BackendResources = resources - status.Cleanup.Complete = outputs["cleanup-failure"] == "" - status.Cleanup.Reason = outputs["cleanup-failure"] + status.Cleanup.Complete = false + status.Cleanup.Reason = "DeferredToController" - if !complete || !status.Cleanup.Complete || len(resources) == 0 { + if !complete || len(resources) == 0 { status.Phase = signalv1.CheckPhaseInconclusive status.Verdict = "Inconclusive" status.Reason = "EvidenceOrCleanupIncomplete" aggregateVerdict = "Inconclusive" - aggregateReason = "one or more checks lacked complete evidence, backend coverage, or cleanup" - stoppedByPolicy = true + aggregateReason = "one or more checks lacked complete evidence or backend coverage" continue } @@ -1972,7 +2190,6 @@ func terminalCheckStatuses( status.Phase = signalv1.CheckPhaseSucceeded case "Fail": status.Phase = signalv1.CheckPhaseFailed - stoppedByPolicy = true if aggregateVerdict == "Pass" { aggregateVerdict = "Fail" @@ -1982,14 +2199,12 @@ func terminalCheckStatuses( status.Phase = signalv1.CheckPhaseInconclusive aggregateVerdict = "Inconclusive" aggregateReason = "one or more checks were inconclusive" - stoppedByPolicy = true default: status.Phase = signalv1.CheckPhaseInconclusive status.Verdict = "Inconclusive" status.Reason = "InvalidVerdict" aggregateVerdict = "Inconclusive" aggregateReason = "one or more checks returned an invalid verdict" - stoppedByPolicy = true } } @@ -2040,7 +2255,6 @@ func nodeWorkflowParameters(node map[string]any) (map[string]string, error) { "measurements": workflowOutputLimits["measurements"], "backend-resources": workflowOutputLimits["backend-resources"], "backend-evidence": workflowOutputLimits["backend-evidence"], - "cleanup-failure": workflowOutputLimits["cleanup-failure"], } decoded, err := decodeWorkflowParametersWithLimits(parameters, limits) @@ -2054,7 +2268,6 @@ func nodeWorkflowParameters(node map[string]any) (map[string]string, error) { "measurements", "backend-resources", "backend-evidence", - "cleanup-failure", } { if _, exists := decoded[required]; !exists { return nil, fmt.Errorf("required parameter %q is missing", required) @@ -2077,10 +2290,6 @@ func setCheckTimes(status *signalv1.CheckStatus, node map[string]any) { value := metav1.NewTime(parsed) status.FinishedAt = &value } - - if status.StartedAt != nil && status.FinishedAt != nil { - status.Duration = metav1.Duration{Duration: status.FinishedAt.Sub(status.StartedAt.Time)} - } } func nodeString(node map[string]any, key string) string { @@ -2092,25 +2301,98 @@ func nodeString(node map[string]any, key string) string { return value } -func uniqueTolerations(tolerations []corev1.Toleration) []corev1.Toleration { - unique := make([]corev1.Toleration, 0, len(tolerations)) - for _, toleration := range tolerations { - duplicate := false +type evidenceBackendStatus struct { + Complete bool `json:"complete"` + Digest string `json:"digest,omitempty"` + Document json.RawMessage `json:"document,omitempty"` +} - for _, existing := range unique { - if reflect.DeepEqual(existing, toleration) { - duplicate = true +type evidenceCleanupStatus struct { + Complete bool `json:"complete"` + Reason string `json:"reason,omitempty"` +} - break +type evidenceCheckStatus struct { + ID signalv1.CheckID `json:"id"` + Order int32 `json:"order"` + WorkflowNodeName string `json:"workflowNodeName,omitempty"` + Phase signalv1.CheckPhase `json:"phase"` + Verdict string `json:"verdict"` + Reason string `json:"reason,omitempty"` + Measurements string `json:"measurements,omitempty"` + BackendResources []signalv1.BackendResourceReference `json:"backendResources,omitempty"` + BackendEvidence evidenceBackendStatus `json:"backendEvidence"` + Cleanup evidenceCleanupStatus `json:"cleanup"` + StartedAt *metav1.Time `json:"startedAt,omitempty"` + FinishedAt *metav1.Time `json:"finishedAt,omitempty"` +} + +func evidenceCheckStatuses(statuses []signalv1.CheckStatus) ([]evidenceCheckStatus, error) { + result := make([]evidenceCheckStatus, len(statuses)) + for index := range statuses { + status := &statuses[index] + var document json.RawMessage + + if status.BackendEvidence.Content != "" { + document = json.RawMessage(status.BackendEvidence.Content) + if !json.Valid(document) { + return nil, fmt.Errorf("check %s backend evidence is not valid JSON", status.ID) } } - if !duplicate { - unique = append(unique, toleration) + result[index] = evidenceCheckStatus{ + ID: status.ID, + Order: status.Order, + WorkflowNodeName: status.WorkflowNodeName, + Phase: status.Phase, + Verdict: status.Verdict, + Reason: status.Reason, + Measurements: status.Measurements, + BackendResources: status.BackendResources, + BackendEvidence: evidenceBackendStatus{ + Complete: status.BackendEvidence.Complete, + Digest: status.BackendEvidence.Digest, + Document: document, + }, + Cleanup: evidenceCleanupStatus{ + Complete: status.Cleanup.Complete, + Reason: status.Cleanup.Reason, + }, + StartedAt: status.StartedAt, + FinishedAt: status.FinishedAt, + } + } + + return result, nil +} + +func compactEvidenceCheckStatuses(statuses []evidenceCheckStatus) []signalv1.CheckStatus { + result := make([]signalv1.CheckStatus, len(statuses)) + for index := range statuses { + status := &statuses[index] + result[index] = signalv1.CheckStatus{ + ID: status.ID, + Order: status.Order, + WorkflowNodeName: status.WorkflowNodeName, + Phase: status.Phase, + Verdict: status.Verdict, + Reason: boundedStatusText(status.Reason), + Measurements: status.Measurements, + BackendResources: status.BackendResources, + BackendEvidence: signalv1.BackendEvidenceStatus{ + Complete: status.BackendEvidence.Complete, + Digest: status.BackendEvidence.Digest, + }, + StartedAt: status.StartedAt, + FinishedAt: status.FinishedAt, + Cleanup: signalv1.CheckCleanupStatus{ + Complete: status.Cleanup.Complete, + Reason: status.Cleanup.Reason, + }, } } - return unique + return result } func (r *RunReconciler) evidenceDocument( @@ -2128,12 +2410,16 @@ func (r *RunReconciler) evidenceDocument( var snapshot corev1.ConfigMap if err := r.Get( ctx, - client.ObjectKey{Namespace: run.Namespace, Name: checks[0].Script.ConfigMapName}, + client.ObjectKey{Namespace: run.Namespace, Name: run.Spec.ScriptBundle.Name}, &snapshot, ); err != nil { return nil, fmt.Errorf("read script snapshot for evidence: %w", err) } + if snapshot.UID != run.Spec.ScriptBundle.UID || snapshot.Labels[SnapshotRunLabel] != run.Name { + return nil, errors.New("script bundle identity does not match the run") + } + scriptContents := make(map[string]string, len(checks)) for index := range checks { content, exists := snapshot.Data[checks[index].Script.Key] @@ -2144,25 +2430,28 @@ func (r *RunReconciler) evidenceDocument( scriptContents[string(checks[index].ID)] = content } + checkStatuses, err := evidenceCheckStatuses(run.Status.Checks) + if err != nil { + return nil, err + } + doc := map[string]any{ - "apiVersion": "signal.unbounded-cloud.io/evidence/v1alpha1", - "runID": string(run.UID), - "runName": run.Name, - "profile": run.Spec.Profile, - "participants": run.Spec.Participants, - "participantRoles": run.Spec.ParticipantRoles, - "topologyConstraints": run.Spec.TopologyConstraints, - "workflow": run.Status.Workflow, - "checks": checks, - "checkStatus": run.Status.Checks, - "scriptContents": scriptContents, - "executionPolicy": run.Spec.ExecutionPolicy, - "executionState": executionState, - "verdict": verdict, - "reason": reason, - "phase": run.Status.Phase, - "startedAt": run.Status.StartedAt, - "finishedAt": run.Status.FinishedAt.UTC().Format(time.RFC3339Nano), + "apiVersion": "signal.unbounded-cloud.io/evidence/v1alpha1", + "runID": string(run.UID), + "runName": run.Name, + "profile": run.Spec.Profile, + "participants": run.Spec.Participants, + "scriptBundle": run.Spec.ScriptBundle, + "workflow": run.Status.Workflow, + "checks": checks, + "checkStatus": checkStatuses, + "scriptContents": scriptContents, + "executionState": executionState, + "verdict": verdict, + "reason": reason, + "phase": run.Status.Phase, + "startedAt": run.Status.StartedAt, + "finishedAt": run.Status.FinishedAt.UTC().Format(time.RFC3339Nano), } return json.MarshalIndent(doc, "", " ") @@ -2171,7 +2460,7 @@ func (r *RunReconciler) evidenceDocument( func backendResourceReferences( value string, participantCount int, - namespace string, + _ string, allowPartial bool, ) ([]signalv1.BackendResourceReference, error) { if participantCount < 1 || participantCount > 64 { @@ -2204,22 +2493,11 @@ func backendResourceReferences( for index := range references { reference := &references[index] - if reference.APIVersion == "" || reference.Kind == "" || reference.Name == "" || reference.UID == "" { + if reference.Name == "" || reference.UID == "" { return nil, fmt.Errorf("backend-resources[%d] has an incomplete identity", index) } - if reference.Namespace == "" { - reference.Namespace = namespace - } - - if reference.Namespace != namespace { - return nil, fmt.Errorf("backend-resources[%d] namespace does not match the Run namespace", index) - } - identity := strings.Join([]string{ - reference.APIVersion, - reference.Kind, - reference.Namespace, reference.Name, string(reference.UID), }, "\x00") @@ -2350,6 +2628,10 @@ func groupedBackendEvidenceComplete( seen := make([]bool, participantCount) complete := !allowPartial && evidence.Complete && evidence.CaptureError == "" podUIDs := map[types.UID]struct{}{} + expectedAPIVersion, expectedKind, err := backendIdentity(backend) + if err != nil { + return false, err + } for index := range evidence.Participants { participant := &evidence.Participants[index] @@ -2386,10 +2668,7 @@ func groupedBackendEvidenceComplete( continue } - if reference.APIVersion != participant.Resource.APIVersion || - reference.Kind != participant.Resource.Kind || - reference.Namespace != participant.Resource.Namespace || - reference.Name != participant.Resource.Name || + if reference.Name != participant.Resource.Name || reference.UID != participant.Resource.UID { return false, fmt.Errorf("backend-evidence participant %d resource differs from backend-resources", participant.ParticipantIndex) } @@ -2397,13 +2676,13 @@ func groupedBackendEvidenceComplete( matches++ } - if matches != 1 || participant.Resource.APIVersion != backend.APIVersion || - participant.Resource.Kind != backend.Kind || participant.Resource.Namespace != namespace { + if matches != 1 || participant.Resource.APIVersion != expectedAPIVersion || + participant.Resource.Kind != expectedKind || participant.Resource.Namespace != namespace { return false, fmt.Errorf("backend-evidence participant %d does not match the declared backend", participant.ParticipantIndex) } complete = complete && participant.CaptureError == "" && len(participant.Pods) > 0 - launcher, worker := false, false + launcher := false for _, pod := range participant.Pods { if pod.UID == "" || pod.Name == "" { @@ -2415,13 +2694,12 @@ func groupedBackendEvidenceComplete( } podUIDs[pod.UID] = struct{}{} - if backend.Kind == "Pod" && (pod.UID != participant.Resource.UID || + if backend == signalv1.ExecutionBackendPod && (pod.UID != participant.Resource.UID || pod.Name != participant.Resource.Name || pod.Role != "participant" || len(participant.Pods) != 1) { return false, errors.New("backend-evidence Pod differs from its backend resource") } launcher = launcher || pod.Role == "launcher" - worker = worker || pod.Role == "worker" complete = complete && pod.Phase != "" && len(pod.Containers) > 0 for _, container := range pod.Containers { @@ -2435,15 +2713,27 @@ func groupedBackendEvidenceComplete( } } - if backend.Kind == "MPIJob" { - complete = complete && launcher && worker && participant.LauncherJob != nil && + if backend == signalv1.ExecutionBackendMPIJob { + complete = complete && launcher && participant.LauncherJob != nil && participant.LauncherJob.Name != "" && participant.LauncherJob.UID != "" } + } return complete, nil } +func backendIdentity(backend signalv1.ExecutionBackend) (string, string, error) { + switch backend { + case signalv1.ExecutionBackendPod: + return "v1", "Pod", nil + case signalv1.ExecutionBackendMPIJob: + return "kubeflow.org/v2beta1", "MPIJob", nil + default: + return "", "", fmt.Errorf("unsupported backend %q", backend) + } +} + func groupedBackendEvidenceForParticipants( content string, globalIndexes []int32, @@ -2511,14 +2801,41 @@ func groupedBackendEvidenceForParticipants( func (r *RunReconciler) cancel(ctx context.Context, run *signalv1.CertificationRun) (ctrl.Result, error) { evidenceRun := run.DeepCopy() evidenceRun.Status.Phase = signalv1.RunPhaseCancelled - evidenceRun.Status.ExecutionState = "Interrupted" - evidenceRun.Status.Verdict = "Inconclusive" evidenceRun.Status.Reason = "canceled by operator" + if !run.DeletionTimestamp.IsZero() { + evidenceRun.Status.Reason = "canceled by deletion" + } evidenceRun.Status.Workflow.Phase = "Canceled" - statuses := evidenceRun.Status.Checks - if len(statuses) != len(run.Spec.Checks) { - statuses = initialCheckStatuses(run.Spec.Checks) + statuses := initialCheckStatuses(run.Spec.Checks) + workflow, err := r.workflowForCancellation(ctx, run) + if err != nil { + return ctrl.Result{}, err + } + + if workflow != nil { + evidenceRun.Status.Workflow.Name = workflow.GetName() + evidenceRun.Status.Workflow.UID = string(workflow.GetUID()) + if phase, found, phaseErr := unstructured.NestedString(workflow.Object, "status", "phase"); phaseErr != nil { + return ctrl.Result{}, fmt.Errorf("read Workflow phase during cancellation: %w", phaseErr) + } else if found { + evidenceRun.Status.Workflow.Phase = phase + } + + completed, _, _ := terminalCheckStatuses(workflow, run) + nodes, found, nodesErr := unstructured.NestedMap(workflow.Object, "status", "nodes") + if nodesErr != nil { + return ctrl.Result{}, fmt.Errorf("read Workflow nodes during cancellation: %w", nodesErr) + } + + if found { + for index := range run.Spec.Checks { + node, count := exactCheckNode(nodes, workflow.GetName(), &run.Spec.Checks[index]) + if count == 1 && nodeString(node, "phase") == "Succeeded" { + statuses[index] = completed[index] + } + } + } } for index := range statuses { @@ -2531,7 +2848,6 @@ func (r *RunReconciler) cancel(ctx context.Context, run *signalv1.CertificationR statuses[index].Phase = signalv1.CheckPhaseSkipped statuses[index].Verdict = "Inconclusive" statuses[index].Reason = "Canceled" - statuses[index].NotRun = true } evidenceRun.Status.Checks = statuses @@ -2542,15 +2858,15 @@ func (r *RunReconciler) cancel(ctx context.Context, run *signalv1.CertificationR evidence, err := r.evidenceDocument( ctx, evidenceRun, - evidenceRun.Status.ExecutionState, - evidenceRun.Status.Verdict, + "Interrupted", + "Inconclusive", evidenceRun.Status.Reason, ) if err != nil { return ctrl.Result{}, err } - uri, digest, err := r.Evidence.Put(ctx, run.Spec.EvidencePath, evidence) + uri, digest, err := r.Evidence.Put(ctx, evidencePath(run), evidence) if err != nil { return ctrl.Result{}, err } @@ -2566,7 +2882,52 @@ func (r *RunReconciler) cancel(ctx context.Context, run *signalv1.CertificationR return ctrl.Result{RequeueAfter: time.Millisecond}, nil } +func (r *RunReconciler) workflowForCancellation( + ctx context.Context, + run *signalv1.CertificationRun, +) (*unstructured.Unstructured, error) { + name := run.Status.Workflow.Name + if name == "" { + name = fmt.Sprintf("%s-a1", run.Name) + } + + workflow := &unstructured.Unstructured{} + workflow.SetGroupVersionKind(workflowGVK) + + reader := r.APIReader + if reader == nil { + reader = r.Client + } + + err := reader.Get( + ctx, + client.ObjectKey{Namespace: run.Namespace, Name: name}, + workflow, + ) + if apierrors.IsNotFound(err) { + return nil, nil + } + + if err != nil { + return nil, err + } + + if !metav1.IsControlledBy(workflow, run) || + (run.Status.Workflow.UID != "" && string(workflow.GetUID()) != run.Status.Workflow.UID) { + return nil, errors.New("Workflow identity changed during cancellation") + } + + return workflow, nil +} + func (r *RunReconciler) cleanup(ctx context.Context, run *signalv1.CertificationRun) (ctrl.Result, error) { + holdParticipants := run.Status.Phase != signalv1.RunPhaseSucceeded && !run.Spec.Release + if holdParticipants { + if _, err := r.ensureParticipantsCordoned(ctx, run); err != nil { + return ctrl.Result{}, err + } + } + stopped, err := r.ensureWorkflowStopped(ctx, run) if err != nil { return ctrl.Result{}, err @@ -2580,35 +2941,63 @@ func (r *RunReconciler) cleanup(ctx context.Context, run *signalv1.Certification return ctrl.Result{}, err } - if err := r.clearRequest(ctx, run); err != nil { + if err := r.deleteScriptBundle(ctx, run); err != nil { return ctrl.Result{}, err } - released, err := r.releaseRunReservations(ctx, run) - if err != nil { + if err := r.patchRunCondition( + ctx, + run, + signalv1.RunConditionCleanup, + metav1.ConditionTrue, + "WorkloadsDeleted", + "Workflow, backend descendants, and script bundle are deleted", + ); err != nil { return ctrl.Result{}, err } - deleted := false + if holdParticipants { + if err := r.patchRunCondition( + ctx, + run, + signalv1.RunConditionReleased, + metav1.ConditionFalse, + "AwaitingExplicitRelease", + "Participants remain reserved and cordoned until spec.release is true", + ); err != nil { + return ctrl.Result{}, err + } - checks := run.Spec.Checks - if len(checks) == 0 { - return ctrl.Result{}, errors.New("run has no certification checks") + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } - snapshot := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Namespace: run.Namespace, Name: checks[0].Script.ConfigMapName}, - } - if err := r.Delete(ctx, snapshot); err != nil && !apierrors.IsNotFound(err) { + if err := r.clearRequest(ctx, run); err != nil { return ctrl.Result{}, err } - deleted = true + if _, err := r.releaseRunReservations(ctx, run); err != nil { + return ctrl.Result{}, err + } - statusBefore := run.DeepCopy() + if err := r.patchRunCondition( + ctx, + run, + signalv1.RunConditionReleased, + metav1.ConditionTrue, + "ParticipantsReleased", + "Participant reservations and Signal-owned cordons are released", + ); err != nil { + return ctrl.Result{}, err + } - run.Status.Cleanup = signalv1.CleanupStatus{ReservationsReleased: released, SnapshotDeleted: deleted} - if err := r.Status().Patch(ctx, run, client.MergeFrom(statusBefore)); err != nil { + if err := r.patchRunCondition( + ctx, + run, + signalv1.RunConditionCordoned, + metav1.ConditionFalse, + "ParticipantsReleased", + "Signal no longer owns participant cordons", + ); err != nil { return ctrl.Result{}, err } @@ -2621,17 +3010,58 @@ func (r *RunReconciler) cleanup(ctx context.Context, run *signalv1.Certification if r.Recorder != nil { r.Recorder.Eventf(run, nil, corev1.EventTypeNormal, "CertificationCleaned", "CleanCertification", - "Released participant reservations, cleared the request label, and deleted the script snapshot") + "Deleted certification workloads and released participant reservations and cordons") } return ctrl.Result{}, nil } +func (r *RunReconciler) deleteScriptBundle(ctx context.Context, run *signalv1.CertificationRun) error { + snapshot := &corev1.ConfigMap{} + + reader := r.APIReader + if reader == nil { + reader = r.Client + } + + err := reader.Get( + ctx, + client.ObjectKey{Namespace: run.Namespace, Name: run.Spec.ScriptBundle.Name}, + snapshot, + ) + if apierrors.IsNotFound(err) { + return nil + } + + if err != nil { + return err + } + + if snapshot.UID != run.Spec.ScriptBundle.UID || snapshot.Labels[SnapshotRunLabel] != run.Name { + return errors.New("refusing to delete script bundle with changed identity") + } + + uid := snapshot.UID + + return client.IgnoreNotFound(r.Delete(ctx, snapshot, client.Preconditions{UID: &uid})) +} + func (r *RunReconciler) reconcileDeleting(ctx context.Context, run *signalv1.CertificationRun) (ctrl.Result, error) { if !containsString(run.Finalizers, RunFinalizer) { return ctrl.Result{}, nil } + if !run.Status.Phase.Terminal() && run.Status.Evidence.URI == "" { + result, recovered, err := r.resumeGroupedEvidence(ctx, run) + if err != nil || recovered { + return result, err + } + } + + if !run.Status.Phase.Terminal() && run.Status.Workflow.Name != "" { + return r.cancel(ctx, run) + } + stopped, err := r.ensureWorkflowStopped(ctx, run) if err != nil { return ctrl.Result{}, err @@ -2649,27 +3079,8 @@ func (r *RunReconciler) reconcileDeleting(ctx context.Context, run *signalv1.Cer return ctrl.Result{}, err } - if len(run.Spec.Checks) > 0 { - snapshot := &corev1.ConfigMap{} - - reader := r.APIReader - if reader == nil { - reader = r.Client - } - - err := reader.Get(ctx, client.ObjectKey{Namespace: run.Namespace, Name: run.Spec.Checks[0].Script.ConfigMapName}, snapshot) - if err == nil { - if snapshot.Labels[SnapshotRunLabel] != run.Name { - return ctrl.Result{}, errors.New("refusing to delete script snapshot with different run identity") - } - - uid := snapshot.UID - if err := r.Delete(ctx, snapshot, client.Preconditions{UID: &uid}); err != nil && !apierrors.IsNotFound(err) { - return ctrl.Result{}, err - } - } else if !apierrors.IsNotFound(err) { - return ctrl.Result{}, err - } + if err := r.deleteScriptBundle(ctx, run); err != nil { + return ctrl.Result{}, err } before := run.DeepCopy() @@ -2697,16 +3108,12 @@ func (r *RunReconciler) releaseRunReservations(ctx context.Context, run *signalv continue } - if value := node.Labels[signalv1.ReservationNodeUIDLabel]; value != "" && value != string(participant.NodeUID) { - return false, fmt.Errorf( - "participant %s reservation identity changed from %s to %s", - participant.NodeName, - participant.NodeUID, - value, - ) + if node.Labels[signalv1.ReservationNodeUIDLabel] != string(participant.NodeUID) { + continue } before := node.DeepCopy() + node.Spec.Unschedulable = false delete(node.Annotations, ReservationLabel) delete(node.Labels, signalv1.ReservationNodeUIDLabel) @@ -2719,12 +3126,18 @@ func (r *RunReconciler) releaseRunReservations(ctx context.Context, run *signalv } func (r *RunReconciler) clearRequest(ctx context.Context, run *signalv1.CertificationRun) error { + if len(run.Spec.Participants) == 0 { + return errors.New("run has no requesting participant") + } + + requestNode := run.Spec.Participants[0] + var node corev1.Node - if err := r.Get(ctx, client.ObjectKey{Name: run.Spec.RequestNodeName}, &node); err != nil { + if err := r.Get(ctx, client.ObjectKey{Name: requestNode.NodeName}, &node); err != nil { return client.IgnoreNotFound(err) } - if requestNodeUID := participantNodeUID(run, run.Spec.RequestNodeName); requestNodeUID != "" && node.UID != requestNodeUID { + if node.UID != requestNode.NodeUID { return nil } @@ -2801,7 +3214,7 @@ func (r *RunReconciler) runWorkloadsGone(ctx context.Context, reader client.Read {Group: "batch", Version: "v1", Kind: "Job"}, {Group: "kubeflow.org", Version: "v2beta1", Kind: "MPIJob"}, } { - if kind.Kind != "Pod" && check.Backend.Kind != "MPIJob" { + if kind.Kind != "Pod" && check.Backend != signalv1.ExecutionBackendMPIJob { continue } @@ -2832,7 +3245,7 @@ func (r *RunReconciler) fenceWorkflowDependents( kinds := []schema.GroupVersionKind{} for _, check := range run.Spec.Checks { - if check.Backend.Kind == "MPIJob" { + if check.Backend == signalv1.ExecutionBackendMPIJob { kinds = append(kinds, schema.GroupVersionKind{Group: "kubeflow.org", Version: "v2beta1", Kind: "MPIJob"}) break @@ -2986,48 +3399,33 @@ func profileChecks(profile *signalv1.CertificationProfile) []signalv1.Certificat checks := make([]signalv1.CertificationCheck, len(profile.Spec.Checks)) for index := range profile.Spec.Checks { checks[index] = profile.Spec.Checks[index] + checks[index].ID = signalv1.CheckID(fmt.Sprintf("R%02d", index+1)) + checks[index].Order = int32(index) checks[index].WorkloadPolicy = *profile.Spec.Checks[index].WorkloadPolicy.DeepCopy() - checks[index].RuntimeStage = copyRuntimeStage(profile.Spec.Checks[index].RuntimeStage) checks[index].ScriptEnvironment = maps.Clone(profile.Spec.Checks[index].ScriptEnvironment) - checks[index].ParticipantPolicy.RequiredRoles = slices.Clone( - profile.Spec.Checks[index].ParticipantPolicy.RequiredRoles, - ) - checks[index].ParticipantPolicy.ExcludedRoles = slices.Clone( - profile.Spec.Checks[index].ParticipantPolicy.ExcludedRoles, - ) + checks[index].ParticipantPolicy.Roles = slices.Clone(profile.Spec.Checks[index].ParticipantPolicy.Roles) } return checks } -func validateCheckPlan(checks []signalv1.CertificationCheck) error { +func validateCheckPlan(checks []signalv1.CertificationCheck, deadline time.Duration) error { if len(checks) != groupedCheckCount { return fmt.Errorf("must contain exactly %d checks", groupedCheckCount) } - seen := make(map[signalv1.CheckID]struct{}, len(checks)) - for index := range checks { check := &checks[index] - if check.ID == "" { - return fmt.Errorf("check %d has an empty ID", index) - } - - if check.Order != int32(index) { - return fmt.Errorf("check %s order is %d, want %d", check.ID, check.Order, index) - } - - if string(check.ID) != fmt.Sprintf("R%02d", index+1) { - return fmt.Errorf("check order %d must use ID R%02d", index, index+1) + if err := validateCheckParticipantPolicy(check.ID, check.ParticipantPolicy); err != nil { + return err } - if _, duplicate := seen[check.ID]; duplicate { - return fmt.Errorf("duplicate check ID %q", check.ID) + deadlines := make([]time.Duration, len(checks)) + for index := range checks { + deadlines[index] = checks[index].BackendActiveDeadline.Duration } - seen[check.ID] = struct{}{} - - if err := validateCheckParticipantPolicy(check.ID, check.ParticipantPolicy); err != nil { + if err := signalv1.ValidateSuiteDeadlines(deadlines, deadline); err != nil { return err } } @@ -3043,28 +3441,24 @@ func validateCheckPlan(checks []signalv1.CertificationCheck) error { return nil } -func validateRunCheckPlan(checks []signalv1.CertificationCheckSnapshot) error { +func validateRunCheckPlan(checks []signalv1.CertificationCheckSnapshot, deadline time.Duration) error { if len(checks) != groupedCheckCount { return fmt.Errorf("run must contain exactly %d checks", groupedCheckCount) } - seen := make(map[signalv1.CheckID]struct{}, len(checks)) - for index := range checks { check := &checks[index] - if check.ID == "" || check.Order != int32(index) { - return fmt.Errorf("run check %d has invalid identity or order", index) - } - - if string(check.ID) != fmt.Sprintf("R%02d", index+1) { - return fmt.Errorf("run check order %d must use ID R%02d", index, index+1) - } + check.ID = signalv1.CheckID(fmt.Sprintf("R%02d", index+1)) + check.Order = int32(index) + } - if _, duplicate := seen[check.ID]; duplicate { - return fmt.Errorf("run contains duplicate check ID %q", check.ID) - } + deadlines := make([]time.Duration, len(checks)) + for index := range checks { + deadlines[index] = checks[index].BackendActiveDeadline.Duration + } - seen[check.ID] = struct{}{} + if err := signalv1.ValidateSuiteDeadlines(deadlines, deadline); err != nil { + return err } if fixedWorkflowOutputBytes() > maximumWorkflowOutputBytes { @@ -3081,7 +3475,6 @@ func validateRunCheckPlan(checks []signalv1.CertificationCheckSnapshot) error { func fixedWorkflowOutputBytes() int64 { perCheck := int64( workflowOutputLimits["verdict"] + - workflowOutputLimits["cleanup-failure"] + workflowOutputLimits["reason"] + workflowOutputLimits["measurements"] + workflowOutputLimits["backend-resources"] + @@ -3092,34 +3485,17 @@ func fixedWorkflowOutputBytes() int64 { } func validateCheckParticipantPolicy(id signalv1.CheckID, policy signalv1.CheckParticipantPolicy) error { - required := make(map[string]struct{}, len(policy.RequiredRoles)) - for _, role := range policy.RequiredRoles { + roles := make(map[string]struct{}, len(policy.Roles)) + for _, role := range policy.Roles { if role == "" { - return fmt.Errorf("check %s contains an empty required role", id) + return fmt.Errorf("check %s contains an empty participant role", id) } - if _, duplicate := required[role]; duplicate { - return fmt.Errorf("check %s repeats required role %q", id, role) + if _, duplicate := roles[role]; duplicate { + return fmt.Errorf("check %s repeats participant role %q", id, role) } - required[role] = struct{}{} - } - - excluded := make(map[string]struct{}, len(policy.ExcludedRoles)) - for _, role := range policy.ExcludedRoles { - if role == "" { - return fmt.Errorf("check %s contains an empty excluded role", id) - } - - if _, duplicate := excluded[role]; duplicate { - return fmt.Errorf("check %s repeats excluded role %q", id, role) - } - - if _, conflict := required[role]; conflict { - return fmt.Errorf("check %s both requires and excludes role %q", id, role) - } - - excluded[role] = struct{}{} + roles[role] = struct{}{} } return nil @@ -3151,11 +3527,11 @@ func validateParticipantConstraints( assigned += role.Count } - if assigned > profile.Spec.Participants.Count { + if len(profile.Spec.ParticipantRoles) > 0 && assigned != profile.Spec.Participants.Count { return fmt.Errorf( - "participant roles require %d assignments but participant count is %d", - assigned, + "participant roles require exactly %d assignments, got %d", profile.Spec.Participants.Count, + assigned, ) } @@ -3171,10 +3547,7 @@ func validateParticipantConstraints( } for index := range checks { - for _, role := range append( - slices.Clone(checks[index].ParticipantPolicy.RequiredRoles), - checks[index].ParticipantPolicy.ExcludedRoles..., - ) { + for _, role := range checks[index].ParticipantPolicy.Roles { if _, exists := roles[role]; !exists { return fmt.Errorf("check %s references unknown participant role %q", checks[index].ID, role) } @@ -3188,14 +3561,9 @@ func participantsForCheck( participants []signalv1.Participant, policy signalv1.CheckParticipantPolicy, ) ([]signalv1.Participant, []int32, error) { - required := make(map[string]bool, len(policy.RequiredRoles)) - for _, role := range policy.RequiredRoles { - required[role] = false - } - - excluded := make(map[string]struct{}, len(policy.ExcludedRoles)) - for _, role := range policy.ExcludedRoles { - excluded[role] = struct{}{} + included := make(map[string]struct{}, len(policy.Roles)) + for _, role := range policy.Roles { + included[role] = struct{}{} } selected := make([]signalv1.Participant, 0, len(participants)) @@ -3203,26 +3571,14 @@ func participantsForCheck( globalIndexes := make([]int32, 0, len(participants)) for index := range participants { participant := participants[index] - include := len(required) == 0 - - for _, role := range participant.Roles { - if _, blocked := excluded[role]; blocked { - include = false - - break - } - - if _, wanted := required[role]; wanted { - required[role] = true - include = true - } - } + _, include := included[participant.Role] + include = len(included) == 0 || include if !include { continue } - globalIndexes = append(globalIndexes, participant.Index) + globalIndexes = append(globalIndexes, int32(index)) participant.Index = int32(len(selected)) selected = append(selected, participant) } @@ -3231,12 +3587,6 @@ func participantsForCheck( return nil, nil, errors.New("participant policy selected no participants") } - for role, present := range required { - if !present { - return nil, nil, fmt.Errorf("required role %q has no selected participant", role) - } - } - return selected, globalIndexes, nil } @@ -3283,22 +3633,6 @@ func checkRunName(runName string, checkID signalv1.CheckID) string { return strings.TrimRight(candidate[:prefixLength], "-") + "-" + hash } -func copyRuntimeStage(stage *signalv1.RuntimeStage) *signalv1.RuntimeStage { - if stage == nil { - return nil - } - - return &signalv1.RuntimeStage{Name: stage.Name} -} - -func runtimeStageName(stage *signalv1.RuntimeStage) string { - if stage == nil { - return "" - } - - return stage.Name -} - func AddToScheme(scheme *runtime.Scheme) error { return signalv1.AddToScheme(scheme) } diff --git a/internal/signal/evidence.go b/internal/signal/evidence.go index c70c75798c..b7a17ee56f 100644 --- a/internal/signal/evidence.go +++ b/internal/signal/evidence.go @@ -35,6 +35,7 @@ const maximumEvidenceChunkBytes = 700 * 1024 type evidenceChunkManifest struct { Version string `json:"version"` + Generation string `json:"generation"` Chunks int `json:"chunks"` CompressedDigest string `json:"compressedDigest"` } @@ -58,6 +59,7 @@ func (s *KubernetesEvidenceStore) Put(ctx context.Context, path string, data []b } digest := digestBytes(data) + generation := strings.TrimPrefix(digest, "sha256:") compressed, err := compressEvidence(data) if err != nil { @@ -94,7 +96,7 @@ func (s *KubernetesEvidenceStore) Put(ctx context.Context, path string, data []b chunk := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-evidence-%03d", name, index), + Name: fmt.Sprintf("%s-evidence-%s-%03d", name, generation, index), Namespace: s.namespace, Labels: map[string]string{ SnapshotRunLabel: name, @@ -115,7 +117,8 @@ func (s *KubernetesEvidenceStore) Put(ctx context.Context, path string, data []b } manifest, err := json.Marshal(evidenceChunkManifest{ - Version: "v1", + Version: "v2", + Generation: generation, Chunks: chunkCount, CompressedDigest: digestBytes(compressed), }) @@ -170,10 +173,14 @@ func (s *KubernetesEvidenceStore) Get( return nil, "", "", false, fmt.Errorf("decode evidence ConfigMap %s chunk manifest: %w", evidence.Name, err) } - if manifest.Version != "v1" || manifest.Chunks < 1 { + if manifest.Version != "v2" || manifest.Generation == "" || manifest.Chunks < 1 { return nil, "", "", false, fmt.Errorf("evidence ConfigMap %s has an invalid chunk manifest", evidence.Name) } + if "sha256:"+manifest.Generation != evidence.Annotations["signal.unbounded-cloud.io/evidence-digest"] { + return nil, "", "", false, fmt.Errorf("evidence ConfigMap %s chunk generation does not match its digest", evidence.Name) + } + compressed = make([]byte, 0, manifest.Chunks*maximumEvidenceChunkBytes) for index := range manifest.Chunks { var chunk corev1.ConfigMap @@ -181,7 +188,7 @@ func (s *KubernetesEvidenceStore) Get( ctx, client.ObjectKey{ Namespace: s.namespace, - Name: fmt.Sprintf("%s-%03d", evidence.Name, index), + Name: fmt.Sprintf("%s-%s-%03d", evidence.Name, manifest.Generation, index), }, &chunk, ); err != nil { diff --git a/internal/signaladapter/adapter.go b/internal/signaladapter/adapter.go index abfe4bbd1a..715051a1d3 100644 --- a/internal/signaladapter/adapter.go +++ b/internal/signaladapter/adapter.go @@ -4,6 +4,7 @@ package signaladapter import ( + "crypto/sha256" "encoding/base64" "encoding/json" "errors" @@ -32,11 +33,9 @@ const ( MaxMeasurementsBytes = 16 * 1024 MaxBackendOutputBytes = 8 * 1024 MaxBackendEvidenceBytes = 24 * 1024 - MaxCleanupFailureBytes = 64 maxMPIWorkerReadyWait = int64(30) maxMPIWorkerSlots = int64(1<<31 - 1) mpiBackoffLimit = int64(0) - mpiTTLSeconds = int64(3600) mpiHostfilePath = "/tmp/project-signal-hostfile" mpiKnownHostsPath = "/tmp/project-signal-known-hosts" resultPrefix = "PROJECT_SIGNAL_RESULT_V2 " @@ -71,17 +70,12 @@ type Config struct { ScriptConfigMap string ScriptKey string ScriptDigest string - ScriptRequiresStage bool - RuntimeStageName string RunnerImage string - BackendAPIVersion string BackendKind string BackendDeadlineSeconds int64 - TolerationsJSON string ScriptEnvironmentJSON string WorkloadPolicyJSON string GPUResourceCount string - Attempt int } type Result struct { @@ -114,7 +108,6 @@ type AggregateOutput struct { ExecutionState string `json:"executionState"` Verdict string `json:"verdict"` Reason string `json:"reason"` - CleanupFailure string `json:"cleanupFailure,omitempty"` Measurements string `json:"measurements"` BackendResources []signalv1.BackendResourceReference `json:"-"` BackendJSON string `json:"backend-resources"` @@ -164,9 +157,7 @@ func ParseParticipants( for index := range participants { participant := &participants[index] - if participant.Index != int32(index) { - return nil, fmt.Errorf("participants-json[%d].index must equal %d", index, index) - } + participant.Index = int32(index) if errors := validation.IsDNS1123Subdomain(participant.NodeName); len(errors) > 0 { return nil, fmt.Errorf("participants-json[%d].nodeName is invalid: %s", index, strings.Join(errors, ", ")) @@ -238,37 +229,20 @@ func ValidateConfig(config Config) ([]signalv1.Participant, []corev1.Toleration, ) } - var runtimeStage *signalv1.RuntimeStage - if config.RuntimeStageName != "" { - runtimeStage = &signalv1.RuntimeStage{Name: config.RuntimeStageName} - } - - if err := signalv1.ValidateRuntimeStage(config.ScriptRequiresStage, runtimeStage); err != nil { - return nil, nil, nil, fmt.Errorf("runtime stage: %w", err) - } - if config.BackendDeadlineSeconds <= 0 { return nil, nil, nil, errors.New("backend-active-deadline-seconds must be positive") } - switch { - case config.BackendKind == "Pod" && config.BackendAPIVersion == "v1": - case config.BackendKind == "MPIJob" && config.BackendAPIVersion == "kubeflow.org/v2beta1": + switch signalv1.ExecutionBackend(config.BackendKind) { + case signalv1.ExecutionBackendPod: + case signalv1.ExecutionBackendMPIJob: if config.BackendDeadlineSeconds < 2 { return nil, nil, nil, errors.New( "backend-active-deadline-seconds must be at least 2 for MPIJob worker readiness", ) } default: - return nil, nil, nil, fmt.Errorf( - "unsupported backend %s %s", - config.BackendAPIVersion, - config.BackendKind, - ) - } - - if config.Attempt < 1 { - return nil, nil, nil, errors.New("attempt must be positive") + return nil, nil, nil, fmt.Errorf("unsupported backend %q", config.BackendKind) } if _, err := signalworkload.Decode(config.WorkloadPolicyJSON); err != nil { @@ -293,16 +267,25 @@ func ValidateConfig(config Config) ([]signalv1.Participant, []corev1.Toleration, ) } - var tolerations []corev1.Toleration - if err := decodeJSONObject(config.TolerationsJSON, &tolerations); err != nil { - return nil, nil, nil, fmt.Errorf("decode tolerations: %w", err) - } + tolerations := []corev1.Toleration{{ + Key: "nvidia.com/gpu", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, { + Key: corev1.TaintNodeUnschedulable, + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }} var environment map[string]string if err := decodeJSONObject(config.ScriptEnvironmentJSON, &environment); err != nil { return nil, nil, nil, fmt.Errorf("decode script-environment: %w", err) } + if len(environment) > 32 { + return nil, nil, nil, errors.New("script-environment must not contain more than 32 entries") + } + environmentNames := make([]string, 0, len(environment)) for name := range environment { environmentNames = append(environmentNames, name) @@ -312,6 +295,10 @@ func ValidateConfig(config Config) ([]signalv1.Participant, []corev1.Toleration, env := make([]corev1.EnvVar, 0, len(environmentNames)+4) for _, name := range environmentNames { + if len(name) > 253 { + return nil, nil, nil, fmt.Errorf("script-environment name %q exceeds 253 bytes", name) + } + if errors := validation.IsEnvVarName(name); len(errors) > 0 { return nil, nil, nil, fmt.Errorf("script-environment contains invalid name %q", name) } @@ -324,6 +311,10 @@ func ValidateConfig(config Config) ([]signalv1.Participant, []corev1.Toleration, return nil, nil, nil, fmt.Errorf("script-environment contains reserved name %q", name) } + if len(environment[name]) > 4096 { + return nil, nil, nil, fmt.Errorf("script-environment value for %q exceeds 4096 bytes", name) + } + env = append(env, corev1.EnvVar{Name: name, Value: environment[name]}) } @@ -431,9 +422,6 @@ func backendResourceIdentity( } return signalv1.BackendResourceReference{ - APIVersion: resource.GetAPIVersion(), - Kind: resource.GetKind(), - Namespace: resource.GetNamespace(), Name: resource.GetName(), UID: resource.GetUID(), ParticipantIndexes: []int32{participantIndex}, @@ -472,9 +460,6 @@ func MinimumPartialBackendResourcesBytes(participantCount int) int { references := make([]signalv1.BackendResourceReference, participantCount) for index := range references { references[index] = signalv1.BackendResourceReference{ - APIVersion: strings.Repeat("a", 32), - Kind: strings.Repeat("a", 16), - Namespace: strings.Repeat("a", 63), Name: strings.Repeat("a", 63), UID: types.UID(strings.Repeat("a", 128)), ParticipantIndexes: []int32{int32(index)}, @@ -703,6 +688,10 @@ func resultCommand(arguments ...string) []string { } func resultCommandWithPrelude(prelude string, arguments ...string) []string { + if strings.TrimSpace(prelude) == "" { + prelude = ":" + } + quotedArguments := make([]string, len(arguments)) for index, argument := range arguments { quotedArguments[index] = shellQuote(argument) @@ -718,13 +707,33 @@ chmod 0755 /results rm -f /results/verdict /results/reason /results/measurements.txt printf 'PROJECT_SIGNAL_RESULTS_V1 prepared\n' set +e -%s -prelude_status=$? -if [[ "${prelude_status}" -eq 0 ]]; then - /bin/bash "/scripts/${PROJECT_SIGNAL_SCRIPT_KEY}" %s - status=$? +script_path="/scripts/${PROJECT_SIGNAL_SCRIPT_KEY}" +expected_digest="${PROJECT_SIGNAL_SCRIPT_DIGEST#sha256:}" +actual_digest="" +verification_reason="" +if command -v sha256sum >/dev/null 2>&1; then + actual_digest="$(sha256sum "${script_path}" | cut -d ' ' -f 1)" +elif command -v shasum >/dev/null 2>&1; then + actual_digest="$(shasum -a 256 "${script_path}" | cut -d ' ' -f 1)" +else + verification_reason="ScriptDigestVerificationUnavailable" +fi +if [[ -z "${verification_reason}" && "${actual_digest}" != "${expected_digest}" ]]; then + verification_reason="ScriptDigestMismatch" +fi +if [[ -n "${verification_reason}" ]]; then + printf 'Inconclusive' > /results/verdict + printf '%%s' "${verification_reason}" > /results/reason + status=125 else - status="${prelude_status}" + %s + prelude_status=$? + if [[ "${prelude_status}" -eq 0 ]]; then + /bin/bash "${script_path}" %s + status=$? + else + status="${prelude_status}" + fi fi set -e canonicalize_result_file() { @@ -883,7 +892,7 @@ func ordinaryPod( ) *corev1.Pod { labels := resourceLabels(config, participant) - env := workloadEnvironment(config, participant, environment, true) + env := workloadEnvironment(config, participant, environment) container := workloadContainer( config, policy, @@ -932,8 +941,8 @@ func mpiJob( labels := resourceLabels(config, participant) jobName := resourceNameWithLimit(config.RunName, participant.Index, 63-len("-worker-0")) - launcherEnv := workloadEnvironment(config, participant, environment, false) - workerEnv := workloadEnvironment(config, participant, environment, true) + launcherEnv := workloadEnvironment(config, participant, environment) + workerEnv := workloadEnvironment(config, participant, environment) readinessTimeout := mpiWorkerReadinessTimeout(backendDeadlineSeconds) workerHosts, err := mpiWorkerHosts(config.Namespace, jobName, 1, gpuCount) @@ -968,10 +977,9 @@ func mpiJob( "runLauncherAsWorker": false, "slotsPerWorker": gpuCount, "runPolicy": map[string]any{ - "activeDeadlineSeconds": backendDeadlineSeconds, - "backoffLimit": mpiBackoffLimit, - "cleanPodPolicy": "All", - "ttlSecondsAfterFinished": mpiTTLSeconds, + "activeDeadlineSeconds": backendDeadlineSeconds, + "backoffLimit": mpiBackoffLimit, + "cleanPodPolicy": "Running", }, "mpiReplicaSpecs": map[string]any{ "Launcher": map[string]any{ @@ -981,7 +989,7 @@ func mpiJob( labels, "", "", - nil, + tolerations, corev1.Container{ Name: "launcher", Image: config.RunnerImage, @@ -1091,7 +1099,6 @@ func workloadEnvironment( config Config, participant signalv1.Participant, environment []corev1.EnvVar, - includeRuntimeStage bool, ) []corev1.EnvVar { result := slices.Clone(environment) @@ -1104,17 +1111,9 @@ func workloadEnvironment( corev1.EnvVar{Name: "PROJECT_SIGNAL_NODE_UID", Value: string(participant.NodeUID)}, corev1.EnvVar{Name: "PROJECT_SIGNAL_SCRIPT_KEY", Value: config.ScriptKey}, corev1.EnvVar{Name: "PROJECT_SIGNAL_SCRIPT_DIGEST", Value: config.ScriptDigest}, - corev1.EnvVar{Name: "PROJECT_SIGNAL_ATTEMPT", Value: strconv.Itoa(config.Attempt)}, corev1.EnvVar{Name: "PROJECT_SIGNAL_RESULTS_DIR", Value: "/results"}, ) - if includeRuntimeStage && config.RuntimeStageName != "" { - result = append(result, corev1.EnvVar{ - Name: "PROJECT_SIGNAL_STAGE_NAME", - Value: config.RuntimeStageName, - }) - } - return result } @@ -1176,12 +1175,13 @@ func workloadVolumes(scriptConfigMap string, policy signalv1.WorkloadPolicy) []c for index := range policy.HostPaths { hostPath := &policy.HostPaths[index] + hostPathType := corev1.HostPathDirectory volumes = append(volumes, corev1.Volume{ - Name: hostPath.Name, + Name: signalworkload.HostPathVolumeName(index), VolumeSource: corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: hostPath.Path, - Type: ptr(hostPath.Type), + Type: &hostPathType, }, }, }) @@ -1195,7 +1195,7 @@ func workloadVolumeMounts(policy signalv1.WorkloadPolicy) []corev1.VolumeMount { for index := range policy.HostPaths { hostPath := &policy.HostPaths[index] mounts = append(mounts, corev1.VolumeMount{ - Name: hostPath.Name, + Name: signalworkload.HostPathVolumeName(index), MountPath: hostPath.MountPath, ReadOnly: true, }) @@ -1223,14 +1223,11 @@ func validateBackendCoverage( covered := make([]bool, participantCount) for index, resource := range resources { - if resource.APIVersion == "" || resource.Kind == "" || resource.Name == "" || resource.UID == "" { + if resource.Name == "" || resource.UID == "" { return fmt.Errorf("backend-resources[%d] has an incomplete identity", index) } identity := strings.Join([]string{ - resource.APIVersion, - resource.Kind, - resource.Namespace, resource.Name, string(resource.UID), }, "\x00") @@ -1309,7 +1306,9 @@ func resourceNameWithLimit(runName string, participantIndex int32, maximumLength maximumRunLength := maximumLength - len(suffix) if len(runName) > maximumRunLength { - runName = strings.TrimRight(runName[:maximumRunLength], "-.") + digest := fmt.Sprintf("%x", sha256.Sum256([]byte(runName)))[:8] + prefixLength := maximumRunLength - len(digest) - 1 + runName = strings.TrimRight(runName[:prefixLength], "-.") + "-" + digest } return runName + suffix diff --git a/internal/signaladapter/backend_evidence.go b/internal/signaladapter/backend_evidence.go index 7d1312ffb1..e356ef5a69 100644 --- a/internal/signaladapter/backend_evidence.go +++ b/internal/signaladapter/backend_evidence.go @@ -387,10 +387,10 @@ func (r *Runner) captureParticipantEvidence( complete = false } - if !hasPodRole(participant.Pods, "launcher") || !hasPodRole(participant.Pods, "worker") { + if !hasPodRole(participant.Pods, "launcher") { participant.CaptureError = joinCaptureError( participant.CaptureError, - "MPI launcher and worker Pods were not both found", + "MPI launcher Pod was not found", ) complete = false } diff --git a/internal/signaladapter/runner.go b/internal/signaladapter/runner.go index 80e7e8d38b..c648923d42 100644 --- a/internal/signaladapter/runner.go +++ b/internal/signaladapter/runner.go @@ -35,7 +35,7 @@ type Runner struct { Dynamic dynamic.Interface Kubernetes kubernetes.Interface PollInterval time.Duration - CleanupWait time.Duration + CaptureWait time.Duration ReadPodLog func(context.Context, string, string, string, bool) (string, error) PersistBackendEvidence func(string) error } @@ -88,9 +88,7 @@ func (r *Runner) Execute(ctx context.Context, config Config) (AggregateOutput, e return AggregateOutput{}, errors.New("owning Workflow requires a positive activeDeadlineSeconds") } - if err := signalv1.ValidateRuntimeContract( - config.ScriptRequiresStage, - runtimeStage(config.RuntimeStageName), + if err := signalv1.ValidateExecutionDeadlines( time.Duration(config.BackendDeadlineSeconds)*time.Second, time.Duration(activeDeadlineSeconds)*time.Second, ); err != nil { @@ -99,7 +97,7 @@ func (r *Runner) Execute(ctx context.Context, config Config) (AggregateOutput, e executionContext, cancelExecution := context.WithTimeout( ctx, - time.Duration(activeDeadlineSeconds)*time.Second, + time.Duration(config.BackendDeadlineSeconds)*time.Second, ) defer cancelExecution() @@ -126,19 +124,7 @@ func (r *Runner) Execute(ctx context.Context, config Config) (AggregateOutput, e created := make([]createdResource, 0, len(resources)) references := make([]signalv1.BackendResourceReference, 0, len(resources)) finishBackendFailure := func(cause error) (AggregateOutput, error) { - output, err := r.backendFailureOutput(config, len(participants), created, references, cause) - if err != nil { - return AggregateOutput{}, err - } - - cleanupContext, cancel := context.WithTimeout(context.Background(), r.cleanupWait()) - defer cancel() - - if err := r.cleanup(cleanupContext, config.Namespace, created); err != nil { - recordCleanupFailure(&output) - } - - return output, nil + return r.backendFailureOutput(config, len(participants), created, references, cause) } for index, resource := range resources { @@ -202,7 +188,7 @@ func (r *Runner) Execute(ctx context.Context, config Config) (AggregateOutput, e return finishBackendFailure(fmt.Errorf("aggregate participant results: %w", err)) } - captureContext, cancelCapture := context.WithTimeout(context.Background(), r.cleanupWait()) + captureContext, cancelCapture := context.WithTimeout(context.Background(), r.captureWait()) defer cancelCapture() var evidenceComplete bool @@ -220,17 +206,14 @@ func (r *Runner) Execute(ctx context.Context, config Config) (AggregateOutput, e recordEvidenceCaptureFailure(&output) } - if err := r.persistEvidenceBeforeCleanup(&output, MaxBackendEvidenceBytes); err != nil { + if err := r.persistBackendEvidence(&output, MaxBackendEvidenceBytes); err != nil { return finishBackendFailure(fmt.Errorf("persist backend evidence: %w", err)) } - cleanupContext, cancel := context.WithTimeout(context.Background(), r.cleanupWait()) - defer cancel() - - if err := r.cleanup(cleanupContext, config.Namespace, created); err != nil { - recordCleanupFailure(&output) - - return output, nil + if resultsRequireQuiescence(results) { + if err := r.quiesceBackends(created); err != nil { + return AggregateOutput{}, fmt.Errorf("quiesce uncertain backends: %w", err) + } } return output, nil @@ -243,7 +226,7 @@ func (r *Runner) backendFailureOutput( references []signalv1.BackendResourceReference, cause error, ) (AggregateOutput, error) { - captureContext, cancelCapture := context.WithTimeout(context.Background(), r.cleanupWait()) + captureContext, cancelCapture := context.WithTimeout(context.Background(), r.captureWait()) defer cancelCapture() evidence, _, err := r.captureBackendEvidence( @@ -292,14 +275,18 @@ func (r *Runner) backendFailureOutput( BackendEvidence: evidence, } - if err := r.persistEvidenceBeforeCleanup(&output, MaxBackendEvidenceBytes); err != nil { + if err := r.persistBackendEvidence(&output, MaxBackendEvidenceBytes); err != nil { return AggregateOutput{}, err } + if err := r.quiesceBackends(created); err != nil { + return AggregateOutput{}, fmt.Errorf("quiesce partial backends: %w", err) + } + return output, nil } -func (r *Runner) persistEvidenceBeforeCleanup(output *AggregateOutput, limit int) error { +func (r *Runner) persistBackendEvidence(output *AggregateOutput, limit int) error { if r.PersistBackendEvidence == nil { return nil } @@ -309,40 +296,81 @@ func (r *Runner) persistEvidenceBeforeCleanup(output *AggregateOutput, limit int output.BackendEvidence, markErr = markBackendEvidenceIncomplete( output.BackendEvidence, - "persist backend evidence before cleanup: "+err.Error(), + "persist backend evidence: "+err.Error(), limit, ) if markErr != nil { return errors.Join( - fmt.Errorf("persist backend evidence before cleanup: %w", err), + fmt.Errorf("persist backend evidence: %w", err), markErr, ) } recordEvidenceCaptureFailure(output) - return fmt.Errorf("persist backend evidence before cleanup: %w", err) + return fmt.Errorf("persist backend evidence: %w", err) } return nil } -func recordCleanupFailure(output *AggregateOutput) { - output.ExecutionState = "Error" - output.Verdict = "Inconclusive" - output.CleanupFailure = "BackendCleanupFailed" - - if output.Reason == "" || output.Reason == "AllParticipantsPassed" { - output.Reason = output.CleanupFailure +func resultsRequireQuiescence(results []Result) bool { + for _, result := range results { + if result.Reason == "BackendDeadlineExceeded" || + strings.HasPrefix(result.Reason, "BackendObservationFailed:") || + strings.HasPrefix(result.Reason, "BackendStatusInvalid:") { + return true + } } + + return false } -func runtimeStage(name string) *signalv1.RuntimeStage { - if name == "" { - return nil +func (r *Runner) quiesceBackends(backends []createdResource) error { + ctx, cancel := context.WithTimeout(context.Background(), r.captureWait()) + defer cancel() + + propagation := metav1.DeletePropagationForeground + for _, backend := range backends { + uid := backend.resource.GetUID() + err := r.Dynamic.Resource(backend.gvr).Namespace(backend.resource.GetNamespace()). + Delete(ctx, backend.resource.GetName(), metav1.DeleteOptions{ + Preconditions: &metav1.Preconditions{UID: &uid}, + PropagationPolicy: &propagation, + }) + if err != nil && !apierrors.IsNotFound(err) { + return err + } } - return &signalv1.RuntimeStage{Name: name} + ticker := time.NewTicker(r.pollInterval()) + defer ticker.Stop() + + for _, backend := range backends { + for { + object, err := r.Dynamic.Resource(backend.gvr).Namespace(backend.resource.GetNamespace()). + Get(ctx, backend.resource.GetName(), metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + break + } + + if err != nil { + return err + } + + if object.GetUID() != backend.resource.GetUID() { + return fmt.Errorf("backend %s/%s identity changed during quiescence", object.GetKind(), object.GetName()) + } + + select { + case <-ctx.Done(): + return fmt.Errorf("wait for backend %s/%s quiescence: %w", object.GetKind(), object.GetName(), ctx.Err()) + case <-ticker.C: + } + } + } + + return nil } func (r *Runner) waitForResult(ctx context.Context, config Config, backend createdResource) Result { @@ -744,64 +772,6 @@ func resultLogSourceMetadata(content string, err error) ResultLogSourceMetadata } } -func (r *Runner) cleanup( - ctx context.Context, - namespace string, - resources []createdResource, -) error { - propagation := metav1.DeletePropagationForeground - - var errs []error - - for _, resource := range resources { - err := r.Dynamic.Resource(resource.gvr).Namespace(namespace).Delete( - ctx, - resource.resource.GetName(), - metav1.DeleteOptions{ - Preconditions: &metav1.Preconditions{UID: ptr(resource.resource.GetUID())}, - PropagationPolicy: &propagation, - }, - ) - if err != nil && !apierrors.IsNotFound(err) { - errs = append(errs, fmt.Errorf("delete %s %s: %w", resource.resource.GetKind(), resource.resource.GetName(), err)) - } - } - - ticker := time.NewTicker(r.pollInterval()) - defer ticker.Stop() - - for _, resource := range resources { - for { - _, err := r.Dynamic.Resource(resource.gvr).Namespace(namespace). - Get(ctx, resource.resource.GetName(), metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - break - } - - if err != nil { - errs = append(errs, fmt.Errorf("observe deletion of %s %s: %w", resource.resource.GetKind(), resource.resource.GetName(), err)) - - break - } - - select { - case <-ctx.Done(): - errs = append(errs, fmt.Errorf( - "wait for deletion of %s %s: %w", - resource.resource.GetKind(), - resource.resource.GetName(), - ctx.Err(), - )) - - return errors.Join(errs...) - case <-ticker.C: - } - } - } - - return errors.Join(errs...) -} - func (r *Runner) pollInterval() time.Duration { if r.PollInterval > 0 { return r.PollInterval @@ -810,9 +780,9 @@ func (r *Runner) pollInterval() time.Duration { return 2 * time.Second } -func (r *Runner) cleanupWait() time.Duration { - if r.CleanupWait > 0 { - return r.CleanupWait +func (r *Runner) captureWait() time.Duration { + if r.CaptureWait > 0 { + return r.CaptureWait } return 30 * time.Second diff --git a/internal/signalworkload/policy.go b/internal/signalworkload/policy.go index 8d12ed5cbc..c29d0b2076 100644 --- a/internal/signalworkload/policy.go +++ b/internal/signalworkload/policy.go @@ -13,14 +13,23 @@ import ( "strings" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/validation" signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" ) const MaxPolicyBytes = 16 * 1024 -var capabilityPattern = regexp.MustCompile(`^[A-Z][A-Z0-9_]*$`) +var ( + capabilityPattern = regexp.MustCompile(`^[A-Z][A-Z0-9_]*$`) + allowedCapabilities = map[corev1.Capability]struct{}{ + "IPC_LOCK": {}, + } + allowedHostPathRoots = []string{"/proc", "/sys", "/var/lib/project-signal"} +) + +func HostPathVolumeName(index int) string { + return fmt.Sprintf("host-%02d", index) +} func Encode(policy signalv1.WorkloadPolicy) (string, error) { if err := Validate(policy); err != nil { @@ -88,21 +97,10 @@ func Validate(policy signalv1.WorkloadPolicy) error { return errors.New("workload policy must not contain more than 16 host paths") } - names := map[string]struct{}{"scripts": {}, "results": {}} mountPaths := map[string]struct{}{"/scripts": {}, "/results": {}} for index := range policy.HostPaths { hostPath := &policy.HostPaths[index] - if errors := validation.IsDNS1123Label(hostPath.Name); len(errors) > 0 { - return fmt.Errorf("workload policy host path %d name is invalid: %s", index, strings.Join(errors, ", ")) - } - - if _, exists := names[hostPath.Name]; exists { - return fmt.Errorf("workload policy host path %d duplicates reserved or existing name %q", index, hostPath.Name) - } - - names[hostPath.Name] = struct{}{} - if err := validateAbsolutePath(fmt.Sprintf("workload policy host path %d mountPath", index), hostPath.MountPath); err != nil { return err } @@ -119,6 +117,10 @@ func Validate(policy signalv1.WorkloadPolicy) error { if err := validateAbsolutePath(fmt.Sprintf("workload policy host path %d path", index), hostPath.Path); err != nil { return err } + + if !allowedHostPath(hostPath.Path) { + return fmt.Errorf("workload policy host path %d source %q is not allowlisted", index, hostPath.Path) + } } return nil @@ -153,6 +155,10 @@ func validateCapabilities(capabilities []corev1.Capability) error { return fmt.Errorf("workload policy capability %q is invalid", capability) } + if _, allowed := allowedCapabilities[capability]; !allowed { + return fmt.Errorf("workload policy capability %q is not allowlisted", capability) + } + if _, exists := seen[capability]; exists { return fmt.Errorf("workload policy capability %q is repeated", capability) } @@ -163,6 +169,16 @@ func validateCapabilities(capabilities []corev1.Capability) error { return nil } +func allowedHostPath(value string) bool { + for _, root := range allowedHostPathRoots { + if value == root || strings.HasPrefix(value, root+"/") { + return true + } + } + + return false +} + func validateAbsolutePath(name, value string) error { if value == "" || !strings.HasPrefix(value, "/") || path.Clean(value) != value { return fmt.Errorf("%s must be a clean absolute path", name) From 16e17e26a9c0eef5237d70c8d0625870d9f6b3d7 Mon Sep 17 00:00:00 2001 From: chokevin Date: Fri, 23 Oct 2026 20:57:33 -0700 Subject: [PATCH 34/36] fix(signal): satisfy lifecycle lint checks --- api/signal/v1alpha1/runtime_contract.go | 1 + internal/signal/controller.go | 28 +++++++++++++------------ internal/signaladapter/runner.go | 2 ++ 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/api/signal/v1alpha1/runtime_contract.go b/api/signal/v1alpha1/runtime_contract.go index d7376c7c58..1cb10c4648 100644 --- a/api/signal/v1alpha1/runtime_contract.go +++ b/api/signal/v1alpha1/runtime_contract.go @@ -36,6 +36,7 @@ func ValidateSuiteDeadlines(backendDeadlines []time.Duration, suiteDeadline time } var required time.Duration + for _, backendDeadline := range backendDeadlines { if backendDeadline <= 0 || backendDeadline%time.Second != 0 { return errors.New("backend active deadlines must be positive whole numbers of seconds") diff --git a/internal/signal/controller.go b/internal/signal/controller.go index 568769d952..b18d06c635 100644 --- a/internal/signal/controller.go +++ b/internal/signal/controller.go @@ -652,6 +652,7 @@ func topologyValues( constraints []signalv1.ParticipantTopologyConstraint, ) map[string]string { values := make(map[string]string) + for index := range constraints { key := constraints[index].LabelKey if value, exists := node.Labels[key]; exists { @@ -1319,6 +1320,7 @@ func (r *RunReconciler) verifyScriptBundle(ctx context.Context, run *signalv1.Ce for index := range run.Spec.Checks { check := &run.Spec.Checks[index] content, found := snapshot.Data[check.Script.Key] + if !found { return fmt.Errorf("script bundle is missing check %s key %s", check.ID, check.Script.Key) } @@ -1386,6 +1388,7 @@ func (r *RunReconciler) ensureParticipantsCordoned( before := node.DeepCopy() node.Spec.Unschedulable = true + if err := r.Patch( ctx, &node, @@ -1398,6 +1401,7 @@ func (r *RunReconciler) ensureParticipantsCordoned( status := metav1.ConditionTrue reason := "ParticipantsCordoned" message := "All participants are reserved and cordoned by this Run" + if !allOwned { status = metav1.ConditionFalse reason = "ParticipantOwnershipLost" @@ -1507,12 +1511,13 @@ func (r *RunReconciler) startWorkflow(ctx context.Context, run *signalv1.Certifi existing := &unstructured.Unstructured{} existing.SetGroupVersionKind(workflowGVK) + if getErr := r.Get(ctx, client.ObjectKeyFromObject(workflow), existing); getErr != nil { return ctrl.Result{}, fmt.Errorf("get existing Workflow: %w", getErr) } if !metav1.IsControlledBy(existing, run) { - return ctrl.Result{}, errors.New("Workflow already exists without matching CertificationRun ownership") + return ctrl.Result{}, errors.New("workflow already exists without matching CertificationRun ownership") } workflow = existing @@ -2329,6 +2334,7 @@ type evidenceCheckStatus struct { func evidenceCheckStatuses(statuses []signalv1.CheckStatus) ([]evidenceCheckStatus, error) { result := make([]evidenceCheckStatus, len(statuses)) + for index := range statuses { status := &statuses[index] var document json.RawMessage @@ -2629,6 +2635,7 @@ func groupedBackendEvidenceComplete( complete := !allowPartial && evidence.Complete && evidence.CaptureError == "" podUIDs := map[types.UID]struct{}{} expectedAPIVersion, expectedKind, err := backendIdentity(backend) + if err != nil { return false, err } @@ -2717,7 +2724,6 @@ func groupedBackendEvidenceComplete( complete = complete && launcher && participant.LauncherJob != nil && participant.LauncherJob.Name != "" && participant.LauncherJob.UID != "" } - } return complete, nil @@ -2802,13 +2808,16 @@ func (r *RunReconciler) cancel(ctx context.Context, run *signalv1.CertificationR evidenceRun := run.DeepCopy() evidenceRun.Status.Phase = signalv1.RunPhaseCancelled evidenceRun.Status.Reason = "canceled by operator" + if !run.DeletionTimestamp.IsZero() { evidenceRun.Status.Reason = "canceled by deletion" } + evidenceRun.Status.Workflow.Phase = "Canceled" statuses := initialCheckStatuses(run.Spec.Checks) workflow, err := r.workflowForCancellation(ctx, run) + if err != nil { return ctrl.Result{}, err } @@ -2816,6 +2825,7 @@ func (r *RunReconciler) cancel(ctx context.Context, run *signalv1.CertificationR if workflow != nil { evidenceRun.Status.Workflow.Name = workflow.GetName() evidenceRun.Status.Workflow.UID = string(workflow.GetUID()) + if phase, found, phaseErr := unstructured.NestedString(workflow.Object, "status", "phase"); phaseErr != nil { return ctrl.Result{}, fmt.Errorf("read Workflow phase during cancellation: %w", phaseErr) } else if found { @@ -2824,6 +2834,7 @@ func (r *RunReconciler) cancel(ctx context.Context, run *signalv1.CertificationR completed, _, _ := terminalCheckStatuses(workflow, run) nodes, found, nodesErr := unstructured.NestedMap(workflow.Object, "status", "nodes") + if nodesErr != nil { return ctrl.Result{}, fmt.Errorf("read Workflow nodes during cancellation: %w", nodesErr) } @@ -2831,6 +2842,7 @@ func (r *RunReconciler) cancel(ctx context.Context, run *signalv1.CertificationR if found { for index := range run.Spec.Checks { node, count := exactCheckNode(nodes, workflow.GetName(), &run.Spec.Checks[index]) + if count == 1 && nodeString(node, "phase") == "Succeeded" { statuses[index] = completed[index] } @@ -2914,7 +2926,7 @@ func (r *RunReconciler) workflowForCancellation( if !metav1.IsControlledBy(workflow, run) || (run.Status.Workflow.UID != "" && string(workflow.GetUID()) != run.Status.Workflow.UID) { - return nil, errors.New("Workflow identity changed during cancellation") + return nil, errors.New("workflow identity changed during cancellation") } return workflow, nil @@ -3307,16 +3319,6 @@ func (r *RunReconciler) fenceWorkflowDependents( return nil } -func participantNodeUID(run *signalv1.CertificationRun, nodeName string) types.UID { - for _, participant := range run.Spec.Participants { - if participant.NodeName == nodeName { - return participant.NodeUID - } - } - - return "" -} - func workflowMessage(workflow *unstructured.Unstructured) string { message, _, err := unstructured.NestedString(workflow.Object, "status", "message") if err != nil { diff --git a/internal/signaladapter/runner.go b/internal/signaladapter/runner.go index c648923d42..e48b6a2b6d 100644 --- a/internal/signaladapter/runner.go +++ b/internal/signaladapter/runner.go @@ -331,6 +331,7 @@ func (r *Runner) quiesceBackends(backends []createdResource) error { defer cancel() propagation := metav1.DeletePropagationForeground + for _, backend := range backends { uid := backend.resource.GetUID() err := r.Dynamic.Resource(backend.gvr).Namespace(backend.resource.GetNamespace()). @@ -338,6 +339,7 @@ func (r *Runner) quiesceBackends(backends []createdResource) error { Preconditions: &metav1.Preconditions{UID: &uid}, PropagationPolicy: &propagation, }) + if err != nil && !apierrors.IsNotFound(err) { return err } From 02715a501b9de746fbec5da483382f15c5bc0e67 Mon Sep 17 00:00:00 2001 From: chokevin Date: Fri, 23 Oct 2026 21:24:23 -0700 Subject: [PATCH 35/36] fix(signal): align lint spacing --- internal/signal/controller.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/signal/controller.go b/internal/signal/controller.go index b18d06c635..7ec8c64cad 100644 --- a/internal/signal/controller.go +++ b/internal/signal/controller.go @@ -2337,6 +2337,7 @@ func evidenceCheckStatuses(statuses []signalv1.CheckStatus) ([]evidenceCheckStat for index := range statuses { status := &statuses[index] + var document json.RawMessage if status.BackendEvidence.Content != "" { @@ -2634,8 +2635,8 @@ func groupedBackendEvidenceComplete( seen := make([]bool, participantCount) complete := !allowPartial && evidence.Complete && evidence.CaptureError == "" podUIDs := map[types.UID]struct{}{} - expectedAPIVersion, expectedKind, err := backendIdentity(backend) + expectedAPIVersion, expectedKind, err := backendIdentity(backend) if err != nil { return false, err } @@ -2816,8 +2817,8 @@ func (r *RunReconciler) cancel(ctx context.Context, run *signalv1.CertificationR evidenceRun.Status.Workflow.Phase = "Canceled" statuses := initialCheckStatuses(run.Spec.Checks) - workflow, err := r.workflowForCancellation(ctx, run) + workflow, err := r.workflowForCancellation(ctx, run) if err != nil { return ctrl.Result{}, err } From cd641dc76b58a1839d93f83de6aa69b7175e8b24 Mon Sep 17 00:00:00 2001 From: chokevin Date: Fri, 23 Oct 2026 21:47:11 -0700 Subject: [PATCH 36/36] fix(signal): separate cancellation evidence steps --- internal/signal/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/signal/controller.go b/internal/signal/controller.go index 7ec8c64cad..0b673c0214 100644 --- a/internal/signal/controller.go +++ b/internal/signal/controller.go @@ -2834,8 +2834,8 @@ func (r *RunReconciler) cancel(ctx context.Context, run *signalv1.CertificationR } completed, _, _ := terminalCheckStatuses(workflow, run) - nodes, found, nodesErr := unstructured.NestedMap(workflow.Object, "status", "nodes") + nodes, found, nodesErr := unstructured.NestedMap(workflow.Object, "status", "nodes") if nodesErr != nil { return ctrl.Result{}, fmt.Errorf("read Workflow nodes during cancellation: %w", nodesErr) }