diff --git a/.dockerignore b/.dockerignore index 9bbe7a5f07..5d20bc8b6b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,6 +6,7 @@ __debug* tmp/ .tmp/ dist/ +deploy/signal/rendered/ # Docker .dockerignore diff --git a/Makefile b/Makefile index 10bfc1e0bb..f401adc6b9 100644 --- a/Makefile +++ b/Makefile @@ -92,6 +92,13 @@ MACHINE_OPS_CONTROLLER_NAME ?= machine-ops-controller MACHINE_OPS_PROVIDER ?= MACHINE_OPS_SITE ?= +PROJECT_SIGNAL_CONTROLLER_BIN=bin/project-signal-controller +PROJECT_SIGNAL_CONTROLLER_CMD=./cmd/project-signal-controller +SIGNAL_CONTROLLER_IMAGE ?= example.invalid/project-signal-controller:replace-me +SIGNAL_NAMESPACE ?= project-signal +SIGNAL_MANIFEST_TEMPLATES_DIR := deploy/signal +SIGNAL_MANIFEST_RENDERED_DIR := deploy/signal/rendered + METALMAN_BIN=bin/metalman METALMAN_CMD=./cmd/metalman NETBOOT_IMAGE ?= $(CONTAINER_REGISTRY)/netboot:$(VERSION_TAG) @@ -230,14 +237,14 @@ NET_FRONTEND_CACHE_FILE := $(NET_FRONTEND_DIST_DIR)/.frontend-build-key # Frontend build toggle (dev builds produce unminified output with sourcemaps). REACT_DEV ?= false -.PHONY: all help fmt lint lint-actions test build vulncheck check-deps kubectl-unbounded kubectl-unbounded-build install-tools install-protoc install-helm generate kubectl-unbounded forge relctl relctl-build agent-artifacts-builder agent-artifacts-builder-build orcadev unbounded-agent machina machina-build machina-oci machina-oci-push machina-manifests machine-ops-controller machine-ops-controller-build machine-ops-controller-oci machine-ops-controller-oci-push machine-ops-manifests metalman metalman-build metalman-oci metalman-oci-push unbounded-operator unbounded-operator-build unbounded-operator-manifests playpen-manifests e2e-gantry e2e-playpen e2e-racer gomod docs-serve unbounded-net-controller unbounded-net-controller-build unbounded-net-node unbounded-net-node-build unbounded-net-routeplan-debug unping unping-build unroute unroute-build license-check notice notice-check gantry gantry-build gantry-manifests inventory-manifests +.PHONY: all help fmt lint lint-actions test build vulncheck check-deps kubectl-unbounded kubectl-unbounded-build install-tools install-protoc install-helm generate kubectl-unbounded forge relctl relctl-build agent-artifacts-builder agent-artifacts-builder-build orcadev unbounded-agent machina machina-build machina-oci machina-oci-push machina-manifests machine-ops-controller machine-ops-controller-build machine-ops-controller-oci machine-ops-controller-oci-push machine-ops-manifests project-signal-controller project-signal-controller-build signal-manifests image-project-signal-controller-local metalman metalman-build metalman-oci metalman-oci-push unbounded-operator unbounded-operator-build unbounded-operator-manifests playpen-manifests e2e-gantry e2e-playpen e2e-racer gomod docs-serve unbounded-net-controller unbounded-net-controller-build unbounded-net-node unbounded-net-node-build unbounded-net-routeplan-debug unping unping-build unroute unroute-build license-check notice notice-check gantry gantry-build gantry-manifests inventory-manifests .PHONY: net-frontend net-frontend-clean net-ebpf-build net-ebpf-generate net-ebpf-verify net-manifests gantry-chart-lint gantry-chart-package release-bom release-manifests unbounded-operator-release-manifest .PHONY: image-machina-local image-token-refresher-local image-machine-ops-controller-local image-metalman-local image-unbounded-operator-local image-unbounded-operator-push image-playpen-local image-net-controller-local image-net-node-local image-gantry-local image-gantry-push images-local .PHONY: image-net-controller-push image-net-node-push images-net-all images-net-all-push ##@ General -all: kubectl-unbounded forge relctl machina machine-ops-controller token-refresher unbounded-operator unbounded-net-controller unbounded-net-node unbounded-net-routeplan-debug unping unroute gantry ## Build all binaries (default) +all: kubectl-unbounded forge relctl machina machine-ops-controller project-signal-controller token-refresher unbounded-operator unbounded-net-controller unbounded-net-node unbounded-net-routeplan-debug unping unroute gantry ## Build all binaries (default) help: ## Show this help @echo "" @@ -289,6 +296,8 @@ help: ## Show this help @echo " unbounded-agent Build unbounded-agent (linux)" @echo " machina | machina-build Build machina controller (with/without lint/test)" @echo " machine-ops-controller Build machine-ops-controller" + @echo " project-signal-controller Build Project Signal controller" + @echo " signal-manifests Render Project Signal manifests" @echo " metalman | metalman-build Build metalman controller (with/without lint/test)" @echo " unbounded-operator | unbounded-operator-build Build the top-level Site operator" @echo " unbounded-net-controller Build net controller" @@ -309,6 +318,10 @@ help: ## Show this help @echo " image-machina-local Build machina image with \$$(CONTAINER_ENGINE)" @echo " image-token-refresher-local Build token-refresher image" @echo " image-machine-ops-controller-local Build machine-ops-controller image" + @echo " image-project-signal-controller-local Build Project Signal controller image" + @echo " Build Project Signal recovery verifier image" + @echo " Build Project Signal GPU runner image" + @echo " Build and push Project Signal GPU runner image" @echo " image-metalman-local Build metalman image" @echo " image-unbounded-operator-local Build unbounded-operator image" @echo " image-unbounded-operator-push Build and push unbounded-operator image" @@ -543,13 +556,13 @@ lint-actions: ## Run actionlint over .github/workflows ifdef CI # In CI each job is independent; skip chained prerequisites. -test: machina-manifests token-refresher-manifests machine-ops-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Run all tests with race detector +test: machina-manifests token-refresher-manifests machine-ops-manifests signal-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Run all tests with race detector $(GOTEST) -race ./... else # Locally, chain test -> lint for convenience. -test: lint machina-manifests token-refresher-manifests machine-ops-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Run all tests (implies lint) +test: lint machina-manifests token-refresher-manifests machine-ops-manifests signal-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Run all tests (implies lint) $(GOTEST) ./... endif @@ -654,16 +667,16 @@ image-racer-object-local: ## Build the racer-object container image locally (sin --build-arg GIT_COMMIT=$(GIT_COMMIT) \ --build-arg BUILD_TIME=$(BUILD_TIME) \ -t racer-object:$(VERSION_TAG) -t $(RACER_OBJECT_IMAGE) \ - -f ./images/racer-object/Containerfile . - $(call trivy-maybe,$(RACER_OBJECT_IMAGE)) + -f ./images/racer-object/Containerfile . + $(call trivy-maybe,$(RACER_OBJECT_IMAGE)) -build: machina-manifests token-refresher-manifests machine-ops-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Build all Go packages - $(GOBUILD) ./... +build: machina-manifests token-refresher-manifests machine-ops-manifests signal-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Build all Go packages + $(GOBUILD) ./... generate: install-protoc ## Run go generate for API types (deepcopy, CRDs) and protobuf PATH="$(PROTOC_DIR)/bin:$$PATH" $(GOCMD) generate $(GO_PACKAGES) -vulncheck: machina-manifests token-refresher-manifests machine-ops-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Run govulncheck; fails only on reachable vulnerabilities that have a final-release fix +vulncheck: machina-manifests token-refresher-manifests machine-ops-manifests signal-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Run govulncheck; fails only on reachable vulnerabilities that have a final-release fix @# The JSON stream is the documented programmatic interface. The gate owns @# the verdict, so govulncheck is not asked for one: in JSON mode it exits 0 @# whether or not it found anything, and a non-zero exit here means the scan @@ -1075,6 +1088,21 @@ machine-ops-controller-oci: image-machine-ops-controller-local ## Alias for imag machine-ops-controller-oci-push: machine-ops-controller-oci ## Build and push the machine-ops-controller image $(CONTAINER_ENGINE) push $(MACHINE_OPS_CONTROLLER_IMAGE) +project-signal-controller-build: signal-manifests ## Build Project Signal controller without lint/test + @mkdir -p bin + $(GOBUILD) -trimpath -ldflags '$(STAMP_LDFLAGS)' -o $(PROJECT_SIGNAL_CONTROLLER_BIN) $(PROJECT_SIGNAL_CONTROLLER_CMD) + +project-signal-controller: test project-signal-controller-build ## Build Project Signal controller (implies test) + +image-project-signal-controller-local: ## Build Project Signal controller image locally + $(CONTAINER_ENGINE) build \ + --build-arg VERSION=$(VERSION) \ + --build-arg GIT_COMMIT=$(GIT_COMMIT) \ + --build-arg BUILD_TIME=$(BUILD_TIME) \ + -t project-signal-controller:$(VERSION_TAG) -t $(SIGNAL_CONTROLLER_IMAGE) \ + -f ./images/project-signal-controller/Containerfile . + $(call trivy-maybe,$(SIGNAL_CONTROLLER_IMAGE)) + MACHINA_NAMESPACE ?= $(UNBOUNDED_NAMESPACE) MACHINA_API_SERVER_ENDPOINT ?= MACHINA_MANIFEST_TEMPLATES_DIR := deploy/machina @@ -1147,6 +1175,18 @@ machine-ops-manifests: ## Render machine-ops-controller manifests into deploy/ma --set APIServerEndpoint=$(MACHINE_OPS_API_SERVER_ENDPOINT) @echo "Rendered machine-ops manifests into $(MACHINE_OPS_MANIFEST_RENDERED_DIR) (image: $(MACHINE_OPS_CONTROLLER_IMAGE))" +signal-manifests: ## Render Project Signal manifests into deploy/signal/rendered + @mkdir -p $(SIGNAL_MANIFEST_RENDERED_DIR)/crd + @find $(SIGNAL_MANIFEST_RENDERED_DIR) -mindepth 1 -not -name .gitignore -delete + $(GOCMD) run ./hack/cmd/render-manifests \ + --templates-dir $(SIGNAL_MANIFEST_TEMPLATES_DIR) \ + --output-dir $(SIGNAL_MANIFEST_RENDERED_DIR) \ + --set Namespace=$(SIGNAL_NAMESPACE) \ + --set ControllerImage=$(SIGNAL_CONTROLLER_IMAGE) + @mkdir -p $(SIGNAL_MANIFEST_RENDERED_DIR)/crd + @cp $(SIGNAL_MANIFEST_TEMPLATES_DIR)/crd/*.yaml $(SIGNAL_MANIFEST_RENDERED_DIR)/crd/ + @echo "Rendered Project Signal manifests into $(SIGNAL_MANIFEST_RENDERED_DIR) (image: $(SIGNAL_CONTROLLER_IMAGE))" + playpen-manifests: ## Render playpen operator and runner manifests into deploy/playpen/rendered @mkdir -p $(PLAYPEN_MANIFEST_RENDERED_DIR) @find $(PLAYPEN_MANIFEST_RENDERED_DIR) -mindepth 1 -not -name .gitignore -delete diff --git a/api/signal/v1alpha1/certificationprofile_types.go b/api/signal/v1alpha1/certificationprofile_types.go new file mode 100644 index 0000000000..4666975ccf --- /dev/null +++ b/api/signal/v1alpha1/certificationprofile_types.go @@ -0,0 +1,126 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type ScriptReference struct { + Name string `json:"name"` + Key string `json:"key"` +} + +type WorkflowReference struct { + Name string `json:"name"` +} + +type CertificationPurpose string + +const ( + PurposeFullCertification CertificationPurpose = "FullCertification" + PurposeQuickCertification CertificationPurpose = "QuickCertification" + PurposePreflight CertificationPurpose = "Preflight" + PurposeTargetedVerification CertificationPurpose = "TargetedVerification" + PurposeDiagnostic CertificationPurpose = "Diagnostic" +) + +type CertificationStage struct { + ID string `json:"id"` + DisplayName string `json:"displayName"` + Category string `json:"category"` + Description string `json:"description,omitempty"` + ExpectedDuration metav1.Duration `json:"expectedDuration,omitempty"` + Required bool `json:"required"` + Conditional bool `json:"conditional,omitempty"` +} + +type ExecutionBackend struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Queue string `json:"queue,omitempty"` +} + +type ParticipantPolicy struct { + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=64 + Count int32 `json:"count"` + + Selector metav1.LabelSelector `json:"selector"` + + RequiredResources corev1.ResourceList `json:"requiredResources,omitempty"` +} + +type CertificationProfileSpec struct { + Participants ParticipantPolicy `json:"participants"` + Script ScriptReference `json:"script"` + Workflow WorkflowReference `json:"workflow"` + Backend ExecutionBackend `json:"backend"` + RunnerImage string `json:"runnerImage"` + // +kubebuilder:validation:Enum=FullCertification;QuickCertification;Preflight;TargetedVerification;Diagnostic + Purpose CertificationPurpose `json:"purpose,omitempty"` + StagePlan []CertificationStage `json:"stagePlan,omitempty"` + Parameters map[string]string `json:"parameters,omitempty"` + EvidencePath string `json:"evidencePath"` + Deadline metav1.Duration `json:"deadline"` + MaxAttempts int32 `json:"maxAttempts,omitempty"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + Recurring *RecurringPolicy `json:"recurring,omitempty"` +} + +type RecurringPolicy struct { + Enabled bool `json:"enabled"` + NodeNames []string `json:"nodeNames"` + MinIdleDuration metav1.Duration `json:"minIdleDuration"` + ValidFor metav1.Duration `json:"validFor"` + NotBefore *metav1.Time `json:"notBefore,omitempty"` + ExpiresAt metav1.Time `json:"expiresAt"` + EvaluationInterval metav1.Duration `json:"evaluationInterval,omitempty"` + FailureBackoff metav1.Duration `json:"failureBackoff"` + MaxConsecutiveFailures int32 `json:"maxConsecutiveFailures"` + MaxConcurrent int32 `json:"maxConcurrent"` + FailureDomainLabel string `json:"failureDomainLabel,omitempty"` + MaxConcurrentPerFailureDomain int32 `json:"maxConcurrentPerFailureDomain,omitempty"` + ExclusiveProfiles []string `json:"exclusiveProfiles,omitempty"` +} + +type RecurringNodeStatus struct { + NodeName string `json:"nodeName"` + IdleSince *metav1.Time `json:"idleSince,omitempty"` + LastEvaluatedAt *metav1.Time `json:"lastEvaluatedAt,omitempty"` + LastSuccessfulRun string `json:"lastSuccessfulRun,omitempty"` + LastSuccessfulAt *metav1.Time `json:"lastSuccessfulAt,omitempty"` + NextDueAt *metav1.Time `json:"nextDueAt,omitempty"` + ConsecutiveFailures int32 `json:"consecutiveFailures,omitempty"` + Suspended bool `json:"suspended,omitempty"` + Reason string `json:"reason,omitempty"` +} + +type CertificationProfileStatus struct { + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + Nodes []RecurringNodeStatus `json:"nodes,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster,shortName=certprofile +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Backend",type=string,JSONPath=`.spec.backend.kind` +// +kubebuilder:printcolumn:name="Participants",type=integer,JSONPath=`.spec.participants.count` +// +kubebuilder:printcolumn:name="Recurring",type=boolean,JSONPath=`.spec.recurring.enabled` +type CertificationProfile struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CertificationProfileSpec `json:"spec"` + Status CertificationProfileStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true +type CertificationProfileList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CertificationProfile `json:"items"` +} diff --git a/api/signal/v1alpha1/certificationrun_types.go b/api/signal/v1alpha1/certificationrun_types.go new file mode 100644 index 0000000000..21a4be518c --- /dev/null +++ b/api/signal/v1alpha1/certificationrun_types.go @@ -0,0 +1,153 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +type RunPhase string + +const ( + RunPhasePending RunPhase = "Pending" + RunPhaseRunning RunPhase = "Running" + RunPhaseSucceeded RunPhase = "Succeeded" + RunPhaseFailed RunPhase = "Failed" + RunPhaseInconclusive RunPhase = "Inconclusive" + RunPhaseCancelled RunPhase = "Canceled" +) + +func (p RunPhase) Terminal() bool { + switch p { + case RunPhaseSucceeded, RunPhaseFailed, RunPhaseInconclusive, RunPhaseCancelled: + return true + default: + return false + } +} + +type Participant struct { + // +kubebuilder:validation:Minimum=0 + Index int32 `json:"index"` + NodeName string `json:"nodeName"` + NodeUID types.UID `json:"nodeUID"` + Capacity corev1.ResourceList `json:"capacity,omitempty"` + Allocatable corev1.ResourceList `json:"allocatable,omitempty"` +} + +type BackendResourceReference struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Namespace string `json:"namespace,omitempty"` + Name string `json:"name"` + UID types.UID `json:"uid"` + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=64 + ParticipantIndexes []int32 `json:"participantIndexes,omitempty"` +} + +type PinnedProfile struct { + Name string `json:"name"` + UID types.UID `json:"uid"` + Generation int64 `json:"generation"` +} + +type ScriptSnapshot struct { + ConfigMapName string `json:"configMapName"` + Key string `json:"key"` + Digest string `json:"digest"` +} + +// +kubebuilder:validation:XValidation:rule="self.profile == oldSelf.profile",message="profile is immutable" +// +kubebuilder:validation:XValidation:rule="self.participants == oldSelf.participants",message="participants are immutable" +// +kubebuilder:validation:XValidation:rule="self.script == oldSelf.script",message="script is immutable" +// +kubebuilder:validation:XValidation:rule="self.workflow == oldSelf.workflow",message="workflow is immutable" +// +kubebuilder:validation:XValidation:rule="self.backend == oldSelf.backend",message="backend is immutable" +// +kubebuilder:validation:XValidation:rule="self.runnerImage == oldSelf.runnerImage",message="runnerImage is immutable" +// +kubebuilder:validation:XValidation:rule="self.parameters == oldSelf.parameters",message="parameters are immutable" +// +kubebuilder:validation:XValidation:rule="self.evidencePath == oldSelf.evidencePath",message="evidencePath is immutable" +// +kubebuilder:validation:XValidation:rule="self.deadline == oldSelf.deadline",message="deadline is immutable" +// +kubebuilder:validation:XValidation:rule="self.maxAttempts == oldSelf.maxAttempts",message="maxAttempts is immutable" +// +kubebuilder:validation:XValidation:rule="self.tolerations == oldSelf.tolerations",message="tolerations are immutable" +// +kubebuilder:validation:XValidation:rule="self.requestNodeName == oldSelf.requestNodeName",message="requestNodeName is immutable" +type CertificationRunSpec struct { + Profile PinnedProfile `json:"profile"` + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=64 + Participants []Participant `json:"participants"` + Script ScriptSnapshot `json:"script"` + Workflow WorkflowReference `json:"workflow"` + Backend ExecutionBackend `json:"backend"` + RunnerImage string `json:"runnerImage"` + // +kubebuilder:validation:Enum=FullCertification;QuickCertification;Preflight;TargetedVerification;Diagnostic + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="purpose is immutable" + Purpose CertificationPurpose `json:"purpose,omitempty"` + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="stagePlan is immutable" + StagePlan []CertificationStage `json:"stagePlan,omitempty"` + Parameters map[string]string `json:"parameters,omitempty"` + EvidencePath string `json:"evidencePath"` + Deadline metav1.Duration `json:"deadline"` + MaxAttempts int32 `json:"maxAttempts"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + RequestNodeName string `json:"requestNodeName"` + Cancel bool `json:"cancel,omitempty"` +} + +type WorkflowStatus struct { + Name string `json:"name,omitempty"` + UID string `json:"uid,omitempty"` + Attempt int32 `json:"attempt,omitempty"` + Phase string `json:"phase,omitempty"` + Outputs map[string]string `json:"outputs,omitempty"` + // +kubebuilder:validation:MaxItems=64 + BackendResources []BackendResourceReference `json:"backendResources,omitempty"` +} + +type EvidenceStatus struct { + URI string `json:"uri,omitempty"` + Digest string `json:"digest,omitempty"` +} + +type CleanupStatus struct { + ReservationsReleased bool `json:"reservationsReleased,omitempty"` + SnapshotDeleted bool `json:"snapshotDeleted,omitempty"` +} + +type CertificationRunStatus struct { + Phase RunPhase `json:"phase,omitempty"` + ExecutionState string `json:"executionState,omitempty"` + Verdict string `json:"verdict,omitempty"` + Reason string `json:"reason,omitempty"` + Workflow WorkflowStatus `json:"workflow,omitempty"` + Evidence EvidenceStatus `json:"evidence,omitempty"` + Cleanup CleanupStatus `json:"cleanup,omitempty"` + StartedAt *metav1.Time `json:"startedAt,omitempty"` + FinishedAt *metav1.Time `json:"finishedAt,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Namespaced,shortName=certrun +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Profile",type=string,JSONPath=`.spec.profile.name` +// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` +// +kubebuilder:printcolumn:name="Verdict",type=string,JSONPath=`.status.verdict` +// +kubebuilder:printcolumn:name="Workflow",type=string,JSONPath=`.status.workflow.name` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` +type CertificationRun struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CertificationRunSpec `json:"spec"` + Status CertificationRunStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true +type CertificationRunList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CertificationRun `json:"items"` +} diff --git a/api/signal/v1alpha1/groupversion_info.go b/api/signal/v1alpha1/groupversion_info.go new file mode 100644 index 0000000000..0ce40be47d --- /dev/null +++ b/api/signal/v1alpha1/groupversion_info.go @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +// +kubebuilder:object:generate=true +// +groupName=signal.unbounded-cloud.io +package v1alpha1 + +//go:generate go tool controller-gen object:headerFile=../../../hack/boilerplate.go.txt paths=. +//go:generate go tool controller-gen crd:headerFile=../../../hack/boilerplate.yaml.txt paths=. output:crd:dir=../../../deploy/signal/crd diff --git a/api/signal/v1alpha1/register.go b/api/signal/v1alpha1/register.go new file mode 100644 index 0000000000..11130bcf55 --- /dev/null +++ b/api/signal/v1alpha1/register.go @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +const GroupName = "signal.unbounded-cloud.io" + +var ( + GroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha1"} + SchemeBuilder = runtime.NewSchemeBuilder(func(s *runtime.Scheme) error { + s.AddKnownTypes( + GroupVersion, + &CertificationProfile{}, + &CertificationProfileList{}, + &CertificationRun{}, + &CertificationRunList{}, + ) + metav1.AddToGroupVersion(s, GroupVersion) + + return nil + }) + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/api/signal/v1alpha1/zz_generated.deepcopy.go b/api/signal/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000000..6775dfde0e --- /dev/null +++ b/api/signal/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,571 @@ +//go:build !ignore_autogenerated + +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackendResourceReference) DeepCopyInto(out *BackendResourceReference) { + *out = *in + if in.ParticipantIndexes != nil { + in, out := &in.ParticipantIndexes, &out.ParticipantIndexes + *out = make([]int32, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackendResourceReference. +func (in *BackendResourceReference) DeepCopy() *BackendResourceReference { + if in == nil { + return nil + } + out := new(BackendResourceReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationProfile) DeepCopyInto(out *CertificationProfile) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationProfile. +func (in *CertificationProfile) DeepCopy() *CertificationProfile { + if in == nil { + return nil + } + out := new(CertificationProfile) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CertificationProfile) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationProfileList) DeepCopyInto(out *CertificationProfileList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CertificationProfile, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationProfileList. +func (in *CertificationProfileList) DeepCopy() *CertificationProfileList { + if in == nil { + return nil + } + out := new(CertificationProfileList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CertificationProfileList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationProfileSpec) DeepCopyInto(out *CertificationProfileSpec) { + *out = *in + in.Participants.DeepCopyInto(&out.Participants) + out.Script = in.Script + out.Workflow = in.Workflow + out.Backend = in.Backend + if in.StagePlan != nil { + in, out := &in.StagePlan, &out.StagePlan + *out = make([]CertificationStage, len(*in)) + copy(*out, *in) + } + if in.Parameters != nil { + in, out := &in.Parameters, &out.Parameters + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + out.Deadline = in.Deadline + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]v1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Recurring != nil { + in, out := &in.Recurring, &out.Recurring + *out = new(RecurringPolicy) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationProfileSpec. +func (in *CertificationProfileSpec) DeepCopy() *CertificationProfileSpec { + if in == nil { + return nil + } + out := new(CertificationProfileSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationProfileStatus) DeepCopyInto(out *CertificationProfileStatus) { + *out = *in + if in.Nodes != nil { + in, out := &in.Nodes, &out.Nodes + *out = make([]RecurringNodeStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationProfileStatus. +func (in *CertificationProfileStatus) DeepCopy() *CertificationProfileStatus { + if in == nil { + return nil + } + out := new(CertificationProfileStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationRun) DeepCopyInto(out *CertificationRun) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationRun. +func (in *CertificationRun) DeepCopy() *CertificationRun { + if in == nil { + return nil + } + out := new(CertificationRun) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CertificationRun) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationRunList) DeepCopyInto(out *CertificationRunList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CertificationRun, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationRunList. +func (in *CertificationRunList) DeepCopy() *CertificationRunList { + if in == nil { + return nil + } + out := new(CertificationRunList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CertificationRunList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationRunSpec) DeepCopyInto(out *CertificationRunSpec) { + *out = *in + out.Profile = in.Profile + if in.Participants != nil { + in, out := &in.Participants, &out.Participants + *out = make([]Participant, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + out.Script = in.Script + out.Workflow = in.Workflow + out.Backend = in.Backend + if in.StagePlan != nil { + in, out := &in.StagePlan, &out.StagePlan + *out = make([]CertificationStage, len(*in)) + copy(*out, *in) + } + if in.Parameters != nil { + in, out := &in.Parameters, &out.Parameters + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + out.Deadline = in.Deadline + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]v1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationRunSpec. +func (in *CertificationRunSpec) DeepCopy() *CertificationRunSpec { + if in == nil { + return nil + } + out := new(CertificationRunSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationRunStatus) DeepCopyInto(out *CertificationRunStatus) { + *out = *in + in.Workflow.DeepCopyInto(&out.Workflow) + out.Evidence = in.Evidence + out.Cleanup = in.Cleanup + if in.StartedAt != nil { + in, out := &in.StartedAt, &out.StartedAt + *out = (*in).DeepCopy() + } + if in.FinishedAt != nil { + in, out := &in.FinishedAt, &out.FinishedAt + *out = (*in).DeepCopy() + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationRunStatus. +func (in *CertificationRunStatus) DeepCopy() *CertificationRunStatus { + if in == nil { + return nil + } + out := new(CertificationRunStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationStage) DeepCopyInto(out *CertificationStage) { + *out = *in + out.ExpectedDuration = in.ExpectedDuration +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationStage. +func (in *CertificationStage) DeepCopy() *CertificationStage { + if in == nil { + return nil + } + out := new(CertificationStage) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CleanupStatus) DeepCopyInto(out *CleanupStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CleanupStatus. +func (in *CleanupStatus) DeepCopy() *CleanupStatus { + if in == nil { + return nil + } + out := new(CleanupStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EvidenceStatus) DeepCopyInto(out *EvidenceStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EvidenceStatus. +func (in *EvidenceStatus) DeepCopy() *EvidenceStatus { + if in == nil { + return nil + } + out := new(EvidenceStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExecutionBackend) DeepCopyInto(out *ExecutionBackend) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExecutionBackend. +func (in *ExecutionBackend) DeepCopy() *ExecutionBackend { + if in == nil { + return nil + } + out := new(ExecutionBackend) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Participant) DeepCopyInto(out *Participant) { + *out = *in + if in.Capacity != nil { + in, out := &in.Capacity, &out.Capacity + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + if in.Allocatable != nil { + in, out := &in.Allocatable, &out.Allocatable + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Participant. +func (in *Participant) DeepCopy() *Participant { + if in == nil { + return nil + } + out := new(Participant) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ParticipantPolicy) DeepCopyInto(out *ParticipantPolicy) { + *out = *in + in.Selector.DeepCopyInto(&out.Selector) + if in.RequiredResources != nil { + in, out := &in.RequiredResources, &out.RequiredResources + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ParticipantPolicy. +func (in *ParticipantPolicy) DeepCopy() *ParticipantPolicy { + if in == nil { + return nil + } + out := new(ParticipantPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PinnedProfile) DeepCopyInto(out *PinnedProfile) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PinnedProfile. +func (in *PinnedProfile) DeepCopy() *PinnedProfile { + if in == nil { + return nil + } + out := new(PinnedProfile) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RecurringNodeStatus) DeepCopyInto(out *RecurringNodeStatus) { + *out = *in + if in.IdleSince != nil { + in, out := &in.IdleSince, &out.IdleSince + *out = (*in).DeepCopy() + } + if in.LastEvaluatedAt != nil { + in, out := &in.LastEvaluatedAt, &out.LastEvaluatedAt + *out = (*in).DeepCopy() + } + if in.LastSuccessfulAt != nil { + in, out := &in.LastSuccessfulAt, &out.LastSuccessfulAt + *out = (*in).DeepCopy() + } + if in.NextDueAt != nil { + in, out := &in.NextDueAt, &out.NextDueAt + *out = (*in).DeepCopy() + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RecurringNodeStatus. +func (in *RecurringNodeStatus) DeepCopy() *RecurringNodeStatus { + if in == nil { + return nil + } + out := new(RecurringNodeStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RecurringPolicy) DeepCopyInto(out *RecurringPolicy) { + *out = *in + if in.NodeNames != nil { + in, out := &in.NodeNames, &out.NodeNames + *out = make([]string, len(*in)) + copy(*out, *in) + } + out.MinIdleDuration = in.MinIdleDuration + out.ValidFor = in.ValidFor + if in.NotBefore != nil { + in, out := &in.NotBefore, &out.NotBefore + *out = (*in).DeepCopy() + } + in.ExpiresAt.DeepCopyInto(&out.ExpiresAt) + out.EvaluationInterval = in.EvaluationInterval + out.FailureBackoff = in.FailureBackoff + if in.ExclusiveProfiles != nil { + in, out := &in.ExclusiveProfiles, &out.ExclusiveProfiles + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RecurringPolicy. +func (in *RecurringPolicy) DeepCopy() *RecurringPolicy { + if in == nil { + return nil + } + out := new(RecurringPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ScriptReference) DeepCopyInto(out *ScriptReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScriptReference. +func (in *ScriptReference) DeepCopy() *ScriptReference { + if in == nil { + return nil + } + out := new(ScriptReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ScriptSnapshot) DeepCopyInto(out *ScriptSnapshot) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScriptSnapshot. +func (in *ScriptSnapshot) DeepCopy() *ScriptSnapshot { + if in == nil { + return nil + } + out := new(ScriptSnapshot) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkflowReference) DeepCopyInto(out *WorkflowReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkflowReference. +func (in *WorkflowReference) DeepCopy() *WorkflowReference { + if in == nil { + return nil + } + out := new(WorkflowReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkflowStatus) DeepCopyInto(out *WorkflowStatus) { + *out = *in + if in.Outputs != nil { + in, out := &in.Outputs, &out.Outputs + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.BackendResources != nil { + in, out := &in.BackendResources, &out.BackendResources + *out = make([]BackendResourceReference, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkflowStatus. +func (in *WorkflowStatus) DeepCopy() *WorkflowStatus { + if in == nil { + return nil + } + out := new(WorkflowStatus) + in.DeepCopyInto(out) + return out +} diff --git a/cmd/project-signal-controller/main.go b/cmd/project-signal-controller/main.go new file mode 100644 index 0000000000..9002e90ddb --- /dev/null +++ b/cmd/project-signal-controller/main.go @@ -0,0 +1,160 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "errors" + "fmt" + "os" + osSignal "os/signal" + "syscall" + + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + machinav1 "github.com/Azure/unbounded/api/machina/v1alpha3" + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" + projectsignal "github.com/Azure/unbounded/internal/signal" + "github.com/Azure/unbounded/internal/version" +) + +const controllerName = "project-signal-controller" + +type config struct { + namespace string + scriptNamespace string + metricsAddress string + probeAddress string + leaderElect bool + maxConcurrentRuns int +} + +func main() { + var cfg config + + cmd := &cobra.Command{ + Use: controllerName, + Short: "Project Signal node certification controller", + RunE: func(cmd *cobra.Command, _ []string) error { + ctx, cancel := osSignal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + return run(ctx, cfg) + }, + Version: version.Version + " (commit: " + version.GitCommit + ")", + } + + cmd.Flags().StringVar(&cfg.namespace, "namespace", "project-signal", "Namespace for CertificationRuns and Workflows") + cmd.Flags().StringVar(&cfg.scriptNamespace, "script-namespace", "project-signal", "Namespace containing mutable source script ConfigMaps") + cmd.Flags().StringVar(&cfg.metricsAddress, "metrics-bind-address", ":8080", "Address for metrics") + cmd.Flags().StringVar(&cfg.probeAddress, "health-probe-bind-address", ":8081", "Address for health probes") + cmd.Flags().BoolVar(&cfg.leaderElect, "leader-elect", true, "Enable leader election") + cmd.Flags().IntVar(&cfg.maxConcurrentRuns, "max-concurrent-runs", 4, "Maximum concurrent CertificationRun reconciles") + cmd.CompletionOptions.DisableDefaultCmd = true + + if err := cmd.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } +} + +func run(ctx context.Context, cfg config) error { + ctrl.SetLogger(zap.New()) + + if cfg.namespace == "" || cfg.scriptNamespace == "" { + return errors.New("namespace and script namespace are required") + } + + scheme := runtime.NewScheme() + if err := clientgoscheme.AddToScheme(scheme); err != nil { + return err + } + + if err := signalv1.AddToScheme(scheme); err != nil { + return err + } + + if err := machinav1.AddToScheme(scheme); err != nil { + return err + } + + watchedNamespaces := map[string]cache.Config{cfg.namespace: {}} + watchedNamespaces[cfg.scriptNamespace] = cache.Config{} + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + Cache: cache.Options{DefaultNamespaces: watchedNamespaces}, + Metrics: metricsserver.Options{BindAddress: cfg.metricsAddress}, + HealthProbeBindAddress: cfg.probeAddress, + LeaderElection: cfg.leaderElect, + LeaderElectionID: controllerName, + LeaderElectionNamespace: cfg.namespace, + LeaderElectionReleaseOnCancel: true, + }) + if err != nil { + return fmt.Errorf("create manager: %w", err) + } + + evidence, err := projectsignal.NewKubernetesEvidenceStore(mgr.GetClient(), cfg.namespace) + if err != nil { + return err + } + + if _, err := mgr.GetRESTMapper().RESTMapping(schema.GroupKind{Group: "argoproj.io", Kind: "Workflow"}, "v1alpha1"); err != nil { + return fmt.Errorf("argo Workflows is required but Workflow.argoproj.io/v1alpha1 is unavailable: %w", err) + } + + admission := &projectsignal.AdmissionReconciler{ + Client: mgr.GetClient(), + APIReader: mgr.GetAPIReader(), + Namespace: cfg.namespace, + ScriptNamespace: cfg.scriptNamespace, + Recorder: mgr.GetEventRecorder(controllerName), + } + if err := admission.SetupWithManager(mgr); err != nil { + return fmt.Errorf("register admission reconciler: %w", err) + } + + runs := &projectsignal.RunReconciler{ + Client: mgr.GetClient(), + APIReader: mgr.GetAPIReader(), + Namespace: cfg.namespace, + Evidence: evidence, + MaxConcurrentRuns: cfg.maxConcurrentRuns, + Recorder: mgr.GetEventRecorder(controllerName), + } + if err := runs.SetupWithManager(mgr); err != nil { + return fmt.Errorf("register run reconciler: %w", err) + } + + recurring := &projectsignal.RecurringReconciler{ + Client: mgr.GetClient(), + Reader: mgr.GetAPIReader(), + Namespace: cfg.namespace, + Admission: admission, + } + if err := recurring.SetupWithManager(mgr); err != nil { + return fmt.Errorf("register recurring reconciler: %w", err) + } + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + return err + } + + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + return err + } + + ctrl.Log.Info("starting Project Signal controller", "version", version.String(), "namespace", cfg.namespace) + + return mgr.Start(ctx) +} diff --git a/deploy/signal/00-namespace.yaml.tmpl b/deploy/signal/00-namespace.yaml.tmpl new file mode 100644 index 0000000000..b47ae5cc04 --- /dev/null +++ b/deploy/signal/00-namespace.yaml.tmpl @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: {{ default "project-signal" .Namespace }} + labels: + app.kubernetes.io/part-of: project-signal diff --git a/deploy/signal/01-serviceaccounts.yaml.tmpl b/deploy/signal/01-serviceaccounts.yaml.tmpl new file mode 100644 index 0000000000..e0efbbe38e --- /dev/null +++ b/deploy/signal/01-serviceaccounts.yaml.tmpl @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: project-signal-controller + namespace: {{ default "project-signal" .Namespace }} + labels: + app.kubernetes.io/name: project-signal-controller +automountServiceAccountToken: true diff --git a/deploy/signal/02-rbac.yaml.tmpl b/deploy/signal/02-rbac.yaml.tmpl new file mode 100644 index 0000000000..1b2bc29d89 --- /dev/null +++ b/deploy/signal/02-rbac.yaml.tmpl @@ -0,0 +1,107 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: project-signal-controller +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list"] + - apiGroups: ["signal.unbounded-cloud.io"] + resources: ["certificationprofiles"] + verbs: ["get", "list", "watch"] + - apiGroups: ["signal.unbounded-cloud.io"] + resources: ["certificationprofiles/status"] + verbs: ["get", "update", "patch"] + - apiGroups: ["signal.unbounded-cloud.io"] + resources: ["certificationruns"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["signal.unbounded-cloud.io"] + resources: ["certificationruns/status"] + verbs: ["get", "update", "patch"] + - apiGroups: ["signal.unbounded-cloud.io"] + resources: ["certificationruns/finalizers"] + verbs: ["update"] + - apiGroups: ["unbounded-cloud.io"] + resources: ["machines", "machineoperations"] + verbs: ["get", "list"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: project-signal-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: project-signal-controller +subjects: + - kind: ServiceAccount + name: project-signal-controller + namespace: {{ default "project-signal" .Namespace }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: project-signal-controller + namespace: {{ default "project-signal" .Namespace }} +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] + - apiGroups: ["events.k8s.io"] + resources: ["events"] + verbs: ["create", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["argoproj.io"] + resources: ["workflows"] + verbs: ["get", "list", "watch", "create", "delete"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: project-signal-controller + namespace: {{ default "project-signal" .Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: project-signal-controller +subjects: + - kind: ServiceAccount + name: project-signal-controller + namespace: {{ default "project-signal" .Namespace }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: project-signal-script-reader + namespace: {{ default (default "project-signal" .Namespace) .ScriptNamespace }} +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: project-signal-script-reader + namespace: {{ default (default "project-signal" .Namespace) .ScriptNamespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: project-signal-script-reader +subjects: + - kind: ServiceAccount + name: project-signal-controller + namespace: {{ default "project-signal" .Namespace }} diff --git a/deploy/signal/06-deployment.yaml.tmpl b/deploy/signal/06-deployment.yaml.tmpl new file mode 100644 index 0000000000..d58ea202ba --- /dev/null +++ b/deploy/signal/06-deployment.yaml.tmpl @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: project-signal-controller + namespace: {{ default "project-signal" .Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: project-signal-controller + template: + metadata: + labels: + app.kubernetes.io/name: project-signal-controller + spec: + serviceAccountName: project-signal-controller + containers: + - name: controller + image: {{ default "example.invalid/project-signal-controller:replace-me" .ControllerImage }} + imagePullPolicy: IfNotPresent + args: + - --namespace={{ default "project-signal" .Namespace }} + - --script-namespace={{ default "project-signal" .Namespace }} + - --leader-elect=true + ports: + - name: metrics + containerPort: 8080 + - name: health + containerPort: 8081 + resources: + requests: + cpu: 50m + memory: 96Mi + limits: + cpu: 500m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + capabilities: + drop: [ALL] + livenessProbe: + httpGet: + path: /healthz + port: health + readinessProbe: + httpGet: + path: /readyz + port: health diff --git a/deploy/signal/README.md b/deploy/signal/README.md new file mode 100644 index 0000000000..68544d4ced --- /dev/null +++ b/deploy/signal/README.md @@ -0,0 +1,197 @@ +# Project Signal controller + +Project Signal is a Kubernetes controller for admitting approved idle Nodes to +operator-supplied certification workflows. The upstream integration provides +the API, controller, CRDs, RBAC, and controller Deployment. It intentionally +does not publish certification scripts, runner images, CertificationProfiles, +or WorkflowTemplates. + +## Architecture + +```text +approved Node or recurring policy + | + v + CertificationProfile + | + v + reserve Nodes and snapshot the selected ConfigMap key + | + v + immutable CertificationRun input + SHA-256 digest + | + v + operator-supplied Argo WorkflowTemplate + | + v + typed result -> retained evidence -> Node projection -> cleanup +``` + +The controller snapshots one selected script key from a source ConfigMap into +a run-owned immutable ConfigMap. Updating the source affects only later runs. +The controller does not contain or publish certification script bodies. + +## Prerequisites + +- Kubernetes with the `Node`, `Pod`, and `ConfigMap` APIs. +- Argo Workflows `argoproj.io/v1alpha1` Workflow API. +- A controller image built from `cmd/project-signal-controller`. +- An operator-owned script ConfigMap and execution image. +- An operator-owned WorkflowTemplate implementing the interface below. +- Explicitly reviewed Node selectors, resource requests, tolerations, policy + values, and cleanup behavior. + +Backends such as Jobs, MPIJobs, or other workload APIs are operator choices. +Install their controllers and grant their runner permissions separately. +Project Signal does not install them or grant generic workload-runner RBAC. + +## WorkflowTemplate interface + +The referenced WorkflowTemplate must accept these string parameters: + +| Parameter | Meaning | +| --- | --- | +| `run-name` | CertificationRun name | +| `run-uid` | CertificationRun UID | +| `participants` | Comma-separated admitted Node names retained for compatibility | +| `participants-json` | Ordered JSON array of participant indexes, Node names, immutable UIDs, capacity, and allocatable resources | +| `participant-count` | Number of atomically admitted participants | +| `request-node-name` | Node that triggered the grouped run | +| `script-configmap` | Immutable run-owned ConfigMap | +| `script-key` | Selected ConfigMap key | +| `script-digest` | SHA-256 of the selected script bytes | +| `backend-api-version` | Backend API version from the profile | +| `backend-kind` | Backend kind from the profile | +| `queue` | Optional backend queue | +| `runner-image` | Digest-pinned execution image | +| `tolerations` | JSON-encoded toleration list | +| `script-environment` | JSON-encoded, validated environment map | +| `mpi-payload-mode` | Optional operator-defined execution mode | +| `expected-gpu-product-regex` | Optional operator policy | +| `expected-gpu-count` | Optional operator policy | +| `observed-gpu-capacity` | Capacity copied from the admitted Node | +| `observed-gpu-allocatable` | Allocatable count copied from the admitted Node | +| `gpu-resource-count` | Optional operator policy | +| `privileged` | Optional operator policy | +| `attempt` | One-based attempt number | + +The Workflow must publish only these output parameters: + +| Output | Limit | Meaning | +| --- | ---: | --- | +| `verdict` | 32 bytes | `Pass`, `Fail`, or `Inconclusive` | +| `reason` | 4 KiB | Bounded diagnostic reason | +| `measurements` | 16 KiB | Bounded measurement summary | +| `backend-resources` | 16 KiB | Optional JSON array of backend workload identities and covered participant indexes; required for a multi-Node `Pass` | + +Unexpected, duplicate, malformed, or oversized outputs are rejected and the +run remains fail-closed. Each `backend-resources` entry must contain +`apiVersion`, `kind`, `name`, `uid`, and the zero-based `participantIndexes` +that the workload executed. `namespace` is optional. A multi-Node successful +result must cover every admitted participant. + +## Multi-Node execution + +Participant admission is deterministic and atomic. The requesting Node is +participant zero, additional eligible Nodes are selected by name, and a failed +reservation rolls back every reservation already acquired. Each participant is +bound by Node UID and an immutable capacity and allocatable snapshot in the +CertificationRun. + +The participant group is a certification boundary, not necessarily one +distributed process group. Operator WorkflowTemplates should create one +independently pinned backend workload per participant unless a test explicitly +requires a cross-Node world. The backend adapter owns launcher and worker +topology, rank environment, rendezvous or SSH configuration, workload RBAC, +deadline propagation, foreground cleanup, and result aggregation. + +The currently validated adapter contract uses ordinary single-Node workloads +for Node diagnostics and `kubeflow.org/v2beta1` MPIJobs for tests that require +an OpenMPI launcher and worker. Kubeflow Trainer v2 TrainJobs can be supplied by +operators through the same generic interface, but require a separately managed, +version-pinned MPI TrainingRuntime and are not installed by Project Signal. + +## Render and install + +Render the controller manifests: + +```bash +make signal-manifests \ + SIGNAL_CONTROLLER_IMAGE= +``` + +Generated output is written to `deploy/signal/rendered/`. Validate it with the +target cluster before applying: + +```bash +kubectl apply --dry-run=server -f deploy/signal/rendered/crd/ +kubectl apply --dry-run=server -f deploy/signal/rendered/ +kubectl apply -f deploy/signal/rendered/crd/ +kubectl apply -f deploy/signal/rendered/ +``` + +The upstream manifests install no profiles. Operators must create a +CertificationProfile that references their own script ConfigMap, +WorkflowTemplate, backend, and digest-pinned runner image. + +## Results and evidence + +Workflow infrastructure state and certification verdicts are separate: + +```text +executionState=Completed, verdict=Pass|Fail, reason= +executionState=Error, verdict=Inconclusive, reason= +``` + +Ambiguous or incomplete workflow execution remains `Inconclusive`. The +controller does not infer a successful result from partial evidence. + +Cluster-local evidence consists of CertificationRun status, the referenced +Workflow and backend status, Events, pod logs, termination messages, and the +SHA-256 digest of the controller's terminal evidence document. + +## Node metadata + +Controller-owned labels: + +| Key | Values | +| --- | --- | +| `signal.unbounded-cloud.io/state` | `pending`, `testing`, `passed`, `failed`, `inconclusive` | +| `signal.unbounded-cloud.io/eligibility` | `blocked`, `production` | +| `signal.unbounded-cloud.io/profile` | bounded profile identifier | + +Pending, testing, failed, and inconclusive states use +`eligibility=blocked`. A successful run projects `passed/production`. Older +runs cannot overwrite a newer terminal projection. + +## Operations + +Request a run by applying the stable trigger label: + +```bash +kubectl label node \ + signal.unbounded-cloud.io/certification-request= +``` + +Cancel an active run: + +```bash +kubectl -n project-signal patch certificationrun \ + --type=merge -p '{"spec":{"cancel":true}}' +``` + +Before removing the controller, disable recurring policies and wait for active +runs to become terminal. Argo Workflows and any workload backend may be shared +cluster services and must not be removed as part of Project Signal cleanup. + +## Non-goals + +This upstream integration does not provide: + +- certification scripts or proprietary thresholds; +- a runner image; +- bundled CertificationProfiles or WorkflowTemplates; +- workload-backend installation or runner RBAC; +- destructive stress, reboot, cordon, drain, or remediation; +- an external evidence service; +- authoritative recovery of ambiguous external mutations. diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml new file mode 100644 index 0000000000..ab5b7120fe --- /dev/null +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml @@ -0,0 +1,391 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: certificationprofiles.signal.unbounded-cloud.io +spec: + group: signal.unbounded-cloud.io + names: + kind: CertificationProfile + listKind: CertificationProfileList + plural: certificationprofiles + shortNames: + - certprofile + singular: certificationprofile + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.backend.kind + name: Backend + type: string + - jsonPath: .spec.participants.count + name: Participants + type: integer + - jsonPath: .spec.recurring.enabled + name: Recurring + type: boolean + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + backend: + properties: + apiVersion: + type: string + kind: + type: string + queue: + type: string + required: + - apiVersion + - kind + type: object + deadline: + type: string + evidencePath: + type: string + maxAttempts: + format: int32 + type: integer + parameters: + additionalProperties: + type: string + type: object + participants: + properties: + count: + format: int32 + maximum: 64 + minimum: 1 + type: integer + requiredResources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: ResourceList is a set of (resource name, quantity) + pairs. + type: object + selector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - count + - selector + type: object + purpose: + enum: + - FullCertification + - QuickCertification + - Preflight + - TargetedVerification + - Diagnostic + type: string + recurring: + properties: + enabled: + type: boolean + evaluationInterval: + type: string + exclusiveProfiles: + items: + type: string + type: array + expiresAt: + format: date-time + type: string + failureBackoff: + type: string + failureDomainLabel: + type: string + maxConcurrent: + format: int32 + type: integer + maxConcurrentPerFailureDomain: + format: int32 + type: integer + maxConsecutiveFailures: + format: int32 + type: integer + minIdleDuration: + type: string + nodeNames: + items: + type: string + type: array + notBefore: + format: date-time + type: string + validFor: + type: string + required: + - enabled + - expiresAt + - failureBackoff + - maxConcurrent + - maxConsecutiveFailures + - minIdleDuration + - nodeNames + - validFor + type: object + runnerImage: + type: string + script: + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + stagePlan: + items: + properties: + category: + type: string + conditional: + type: boolean + description: + type: string + displayName: + type: string + expectedDuration: + type: string + id: + type: string + required: + type: boolean + required: + - category + - displayName + - id + - required + type: object + type: array + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + workflow: + properties: + name: + type: string + required: + - name + type: object + required: + - backend + - deadline + - evidencePath + - participants + - runnerImage + - script + - workflow + type: object + status: + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + nodes: + items: + properties: + consecutiveFailures: + format: int32 + type: integer + idleSince: + format: date-time + type: string + lastEvaluatedAt: + format: date-time + type: string + lastSuccessfulAt: + format: date-time + type: string + lastSuccessfulRun: + type: string + nextDueAt: + format: date-time + type: string + nodeName: + type: string + reason: + type: string + suspended: + type: boolean + required: + - nodeName + type: object + type: array + observedGeneration: + format: int64 + type: integer + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml new file mode 100644 index 0000000000..a95227c6d1 --- /dev/null +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml @@ -0,0 +1,425 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: certificationruns.signal.unbounded-cloud.io +spec: + group: signal.unbounded-cloud.io + names: + kind: CertificationRun + listKind: CertificationRunList + plural: certificationruns + shortNames: + - certrun + singular: certificationrun + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.profile.name + name: Profile + type: string + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.verdict + name: Verdict + type: string + - jsonPath: .status.workflow.name + name: Workflow + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + backend: + properties: + apiVersion: + type: string + kind: + type: string + queue: + type: string + required: + - apiVersion + - kind + type: object + cancel: + type: boolean + deadline: + type: string + evidencePath: + type: string + maxAttempts: + format: int32 + type: integer + parameters: + additionalProperties: + type: string + type: object + participants: + items: + properties: + allocatable: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: ResourceList is a set of (resource name, quantity) + pairs. + type: object + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: ResourceList is a set of (resource name, quantity) + pairs. + type: object + index: + format: int32 + minimum: 0 + type: integer + nodeName: + type: string + nodeUID: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + required: + - index + - nodeName + - nodeUID + type: object + maxItems: 64 + minItems: 1 + type: array + profile: + properties: + generation: + format: int64 + type: integer + name: + type: string + uid: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + required: + - generation + - name + - uid + type: object + purpose: + enum: + - FullCertification + - QuickCertification + - Preflight + - TargetedVerification + - Diagnostic + type: string + x-kubernetes-validations: + - message: purpose is immutable + rule: self == oldSelf + requestNodeName: + type: string + runnerImage: + type: string + script: + properties: + configMapName: + type: string + digest: + type: string + key: + type: string + required: + - configMapName + - digest + - key + type: object + stagePlan: + items: + properties: + category: + type: string + conditional: + type: boolean + description: + type: string + displayName: + type: string + expectedDuration: + type: string + id: + type: string + required: + type: boolean + required: + - category + - displayName + - id + - required + type: object + type: array + x-kubernetes-validations: + - message: stagePlan is immutable + rule: self == oldSelf + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + workflow: + properties: + name: + type: string + required: + - name + type: object + required: + - backend + - deadline + - evidencePath + - maxAttempts + - participants + - profile + - requestNodeName + - runnerImage + - script + - workflow + type: object + x-kubernetes-validations: + - message: profile is immutable + rule: self.profile == oldSelf.profile + - message: participants are immutable + rule: self.participants == oldSelf.participants + - message: script is immutable + rule: self.script == oldSelf.script + - message: workflow is immutable + rule: self.workflow == oldSelf.workflow + - message: backend is immutable + rule: self.backend == oldSelf.backend + - message: runnerImage is immutable + rule: self.runnerImage == oldSelf.runnerImage + - message: parameters are immutable + rule: self.parameters == oldSelf.parameters + - message: evidencePath is immutable + rule: self.evidencePath == oldSelf.evidencePath + - message: deadline is immutable + rule: self.deadline == oldSelf.deadline + - message: maxAttempts is immutable + rule: self.maxAttempts == oldSelf.maxAttempts + - message: tolerations are immutable + rule: self.tolerations == oldSelf.tolerations + - message: requestNodeName is immutable + rule: self.requestNodeName == oldSelf.requestNodeName + status: + properties: + cleanup: + properties: + reservationsReleased: + type: boolean + snapshotDeleted: + type: boolean + type: object + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + evidence: + properties: + digest: + type: string + uri: + type: string + type: object + executionState: + type: string + finishedAt: + format: date-time + type: string + phase: + type: string + reason: + type: string + startedAt: + format: date-time + type: string + verdict: + type: string + workflow: + properties: + attempt: + format: int32 + type: integer + backendResources: + items: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + participantIndexes: + items: + format: int32 + type: integer + maxItems: 64 + minItems: 1 + type: array + uid: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + required: + - apiVersion + - kind + - name + - uid + type: object + maxItems: 64 + type: array + name: + type: string + outputs: + additionalProperties: + type: string + type: object + phase: + type: string + uid: + type: string + type: object + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/signal/manifest_test.go b/deploy/signal/manifest_test.go new file mode 100644 index 0000000000..e0f97d395b --- /dev/null +++ b/deploy/signal/manifest_test.go @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signalmanifest_test + +import ( + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" +) + +func TestUpstreamManifestsContainOnlyControllerSurface(t *testing.T) { + t.Parallel() + + templates, err := filepath.Glob("*.yaml.tmpl") + if err != nil { + t.Fatal(err) + } + + sort.Strings(templates) + + want := []string{ + "00-namespace.yaml.tmpl", + "01-serviceaccounts.yaml.tmpl", + "02-rbac.yaml.tmpl", + "06-deployment.yaml.tmpl", + } + if !reflect.DeepEqual(templates, want) { + t.Fatalf("manifest templates = %v, want %v", templates, want) + } + + for _, path := range []string{"01-serviceaccounts.yaml.tmpl", "02-rbac.yaml.tmpl"} { + content, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + + manifest := string(content) + if strings.Contains(manifest, "project-signal-runner") || + strings.Contains(manifest, "kubeflow.org") { + t.Fatalf("%s contains bundled runner surface", path) + } + } +} + +func TestUpstreamDoesNotPublishCertificationPackage(t *testing.T) { + t.Parallel() + + for _, path := range []string{ + "../../project-signal-certification", + "../../project-signal-certification-runner", + } { + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("%s must not be published upstream: %v", path, err) + } + } +} diff --git a/deploy/signal/rendered/.gitignore b/deploy/signal/rendered/.gitignore new file mode 100644 index 0000000000..d6b7ef32c8 --- /dev/null +++ b/deploy/signal/rendered/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/images/project-signal-controller/Containerfile b/images/project-signal-controller/Containerfile new file mode 100644 index 0000000000..dd52f42b80 --- /dev/null +++ b/images/project-signal-controller/Containerfile @@ -0,0 +1,29 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +FROM docker.io/library/golang:1.27.0-trixie AS builder + +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates git make \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . + +ARG TARGETOS +ARG TARGETARCH +ARG VERSION=dev +ARG GIT_COMMIT= +ARG BUILD_TIME= + +RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 \ + make project-signal-controller-build VERSION=${VERSION} \ + ${GIT_COMMIT:+GIT_COMMIT=${GIT_COMMIT}} ${BUILD_TIME:+BUILD_TIME=${BUILD_TIME}} + +FROM gcr.io/distroless/static-debian12:nonroot + +COPY --from=builder /src/bin/project-signal-controller /project-signal-controller +COPY LICENSE NOTICE /licenses/ + +ENTRYPOINT ["/project-signal-controller"] diff --git a/internal/signal/controller.go b/internal/signal/controller.go new file mode 100644 index 0000000000..d91434d75a --- /dev/null +++ b/internal/signal/controller.go @@ -0,0 +1,1575 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signal + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "reflect" + "regexp" + "sort" + "strings" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/rand" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/recorder" + + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" +) + +const ( + RequestLabel = "signal.unbounded-cloud.io/certification-request" + ReservationLabel = "signal.unbounded-cloud.io/reserved-by" + RunNodeLabel = "signal.unbounded-cloud.io/request-node" + RunProfileLabel = "signal.unbounded-cloud.io/profile" + RunTriggerAnnotation = "signal.unbounded-cloud.io/trigger" + RunFinalizer = "signal.unbounded-cloud.io/cleanup" + SnapshotRunLabel = "signal.unbounded-cloud.io/run" + DefaultMaxAttempts = int32(1) +) + +var runnerImageDigestPattern = regexp.MustCompile(`^[^@\s]+@sha256:[0-9a-f]{64}$`) + +var reservedScriptEnvironmentNames = map[string]struct{}{ + "BASH_ENV": {}, + "BASHOPTS": {}, + "CDPATH": {}, + "ENV": {}, + "GLOBIGNORE": {}, + "HOST_PROC_ROOT": {}, + "HOST_SYSFS_ROOT": {}, + "IFS": {}, + "LD_LIBRARY_PATH": {}, + "LD_PRELOAD": {}, + "PATH": {}, + "PROJECT_SIGNAL_PARAMETERS_JSON": {}, + "PROJECT_SIGNAL_RESULTS_DIR": {}, + "PYTHONHOME": {}, + "PYTHONPATH": {}, + "SHELLOPTS": {}, +} + +var workflowOutputLimits = map[string]int{ + "backend-resources": 16 * 1024, + "measurements": 16 * 1024, + "reason": 4 * 1024, + "verdict": 32, +} + +var workflowGVK = schema.GroupVersionKind{ + Group: "argoproj.io", + Version: "v1alpha1", + Kind: "Workflow", +} + +type AdmissionReconciler struct { + client.Client + APIReader client.Reader + Namespace string + ScriptNamespace string + Recorder recorder.EventRecorder + Now func() time.Time + admissionMu sync.Mutex +} + +type RunReconciler struct { + client.Client + APIReader client.Reader + Namespace string + Evidence EvidenceStore + Now func() time.Time + MaxConcurrentRuns int + WorkflowAPIVersion string + Recorder recorder.EventRecorder +} + +func (r *AdmissionReconciler) SetupWithManager(mgr ctrl.Manager) error { + if r.APIReader == nil { + r.APIReader = mgr.GetAPIReader() + } + + return ctrl.NewControllerManagedBy(mgr). + For(&corev1.Node{}, builder.WithPredicates(predicate.LabelChangedPredicate{})). + WithOptions(controller.Options{MaxConcurrentReconciles: 1}). + Complete(r) +} + +func (r *RunReconciler) SetupWithManager(mgr ctrl.Manager) error { + if r.APIReader == nil { + r.APIReader = mgr.GetAPIReader() + } + + workflow := &unstructured.Unstructured{} + workflow.SetGroupVersionKind(workflowGVK) + + return ctrl.NewControllerManagedBy(mgr). + For(&signalv1.CertificationRun{}). + Watches(workflow, handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &signalv1.CertificationRun{})). + WithOptions(controller.Options{MaxConcurrentReconciles: max(1, r.MaxConcurrentRuns)}). + Complete(r) +} + +func (r *AdmissionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var node corev1.Node + if err := r.Get(ctx, client.ObjectKey{Name: req.Name}, &node); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + profileName := node.Labels[RequestLabel] + if profileName == "" { + return ctrl.Result{}, nil + } + + return ctrl.Result{}, r.admitNode(ctx, &node, profileName) +} + +func (r *AdmissionReconciler) admitNode(ctx context.Context, node *corev1.Node, profileName string) error { + return r.admitNodeWithTrigger(ctx, node, profileName, "manual") +} + +func (r *AdmissionReconciler) admitNodeWithTrigger( + ctx context.Context, + node *corev1.Node, + profileName string, + trigger string, +) error { + r.admissionMu.Lock() + defer r.admissionMu.Unlock() + + return r.admitNodeWithTriggerLocked(ctx, node, profileName, trigger) +} + +func (r *AdmissionReconciler) admitNodeWithTriggerLocked( + ctx context.Context, + node *corev1.Node, + profileName string, + trigger string, +) error { + if runName := node.Annotations[ReservationLabel]; runName != "" { + var run signalv1.CertificationRun + + reader := r.APIReader + if reader == nil { + reader = r.Client + } + + err := reader.Get(ctx, client.ObjectKey{Namespace: r.Namespace, Name: runName}, &run) + if err == nil { + return nil + } + + if !apierrors.IsNotFound(err) { + return err + } + + var snapshot corev1.ConfigMap + + err = reader.Get( + ctx, + client.ObjectKey{ + Namespace: r.Namespace, + Name: runName + "-scripts", + }, + &snapshot, + ) + if err == nil { + if snapshot.Labels[SnapshotRunLabel] != runName { + return fmt.Errorf( + "orphaned snapshot %s is not bound to reservation %q", + snapshot.Name, + runName, + ) + } + + if err := r.Delete(ctx, &snapshot); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("delete orphaned script snapshot: %w", err) + } + } else if !apierrors.IsNotFound(err) { + return fmt.Errorf("get orphaned script snapshot: %w", err) + } + + if err := r.releaseOrphanedReservations(ctx, runName); err != nil { + return fmt.Errorf("release orphaned Project Signal reservations: %w", err) + } + } + + var profile signalv1.CertificationProfile + if err := r.Get(ctx, client.ObjectKey{Name: profileName}, &profile); err != nil { + return fmt.Errorf("get CertificationProfile %s: %w", profileName, err) + } + + if err := validateStagePlan(profile.Spec.StagePlan); err != nil { + return fmt.Errorf("profile %s stage plan: %w", profile.Name, err) + } + + if !runnerImageDigestPattern.MatchString(profile.Spec.RunnerImage) { + return fmt.Errorf("profile %s runner image must use an immutable sha256 digest", profile.Name) + } + + if _, err := scriptEnvironment(profile.Spec.Parameters); err != nil { + return fmt.Errorf("profile %s parameters: %w", profile.Name, err) + } + + participants, err := r.selectParticipants(ctx, node, &profile) + if err != nil { + return err + } + + runName := runName(node.Name, profile.Name, r.now()) + if err := r.reserveParticipants(ctx, participants, runName); err != nil { + return err + } + + releaseOnFailure := true + defer func() { + if releaseOnFailure { + if err := r.releaseParticipants(context.WithoutCancel(ctx), participants, runName); err != nil { + log.FromContext(ctx).Error(err, "failed to roll back participant reservations", "run", runName) + } + } + }() + + parameters := make(map[string]string, len(profile.Spec.Parameters)+2) + for key, value := range profile.Spec.Parameters { + parameters[key] = value + } + + if len(participants) == 1 { + var participantNode corev1.Node + + reader := r.APIReader + if reader == nil { + reader = r.Client + } + + if err := reader.Get(ctx, client.ObjectKey{Name: participants[0].NodeName}, &participantNode); err != nil { + return fmt.Errorf("read admitted participant inventory: %w", err) + } + + gpuResource := corev1.ResourceName("nvidia.com/gpu") + gpuCapacity := participantNode.Status.Capacity[gpuResource] + gpuAllocatable := participantNode.Status.Allocatable[gpuResource] + parameters["observed-gpu-capacity"] = gpuCapacity.String() + parameters["observed-gpu-allocatable"] = gpuAllocatable.String() + } + + snapshot, scriptDigest, err := r.createSnapshot(ctx, runName, &profile) + if err != nil { + return err + } + + maxAttempts := profile.Spec.MaxAttempts + if maxAttempts < 1 { + maxAttempts = DefaultMaxAttempts + } + + run := &signalv1.CertificationRun{ + TypeMeta: metav1.TypeMeta{APIVersion: signalv1.GroupVersion.String(), Kind: "CertificationRun"}, + ObjectMeta: metav1.ObjectMeta{ + Name: runName, + Namespace: r.Namespace, + Labels: map[string]string{ + RunNodeLabel: node.Name, + RunProfileLabel: profile.Name, + }, + Annotations: map[string]string{ + RunTriggerAnnotation: trigger, + }, + Finalizers: []string{RunFinalizer}, + }, + Spec: signalv1.CertificationRunSpec{ + Profile: signalv1.PinnedProfile{ + Name: profile.Name, + UID: profile.UID, + Generation: profile.Generation, + }, + Participants: participants, + Script: signalv1.ScriptSnapshot{ConfigMapName: snapshot.Name, Key: profile.Spec.Script.Key, Digest: scriptDigest}, + Workflow: profile.Spec.Workflow, + Backend: profile.Spec.Backend, + RunnerImage: profile.Spec.RunnerImage, + Purpose: profile.Spec.Purpose, + StagePlan: append([]signalv1.CertificationStage(nil), profile.Spec.StagePlan...), + Parameters: parameters, + EvidencePath: strings.Trim(profile.Spec.EvidencePath, "/") + "/" + runName + ".json", + Deadline: profile.Spec.Deadline, + MaxAttempts: maxAttempts, + Tolerations: profile.Spec.Tolerations, + RequestNodeName: node.Name, + }, + } + + if err := r.Create(ctx, run); err != nil { + deleteErr := r.Delete(context.WithoutCancel(ctx), snapshot) + + return errors.Join(fmt.Errorf("create CertificationRun: %w", err), client.IgnoreNotFound(deleteErr)) + } + + cleanupFailedAdmission := func(cause error) error { + deleteRunErr := r.Delete(context.WithoutCancel(ctx), run) + if deleteRunErr != nil && !apierrors.IsNotFound(deleteRunErr) { + // Keep the reservation if the run could not be made inert. This + // prevents another admission from overlapping the surviving run. + releaseOnFailure = false + } + + deleteSnapshotErr := r.Delete(context.WithoutCancel(ctx), snapshot) + + return errors.Join(cause, client.IgnoreNotFound(deleteRunErr), client.IgnoreNotFound(deleteSnapshotErr)) + } + + snapshot.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(run, signalv1.GroupVersion.WithKind("CertificationRun"))} + if err := r.Update(ctx, snapshot); err != nil { + return cleanupFailedAdmission(fmt.Errorf("bind script snapshot to run: %w", err)) + } + + if err := projectPending(ctx, r.Client, participants, run.Spec.Profile); err != nil { + return cleanupFailedAdmission(fmt.Errorf("project pending certification state: %w", err)) + } + + releaseOnFailure = false + + if r.Recorder != nil { + r.Recorder.Eventf(run, nil, corev1.EventTypeNormal, "CertificationAdmitted", "AdmitCertification", + "Created CertificationRun %s with immutable script digest %s", run.Name, scriptDigest) + } + + return nil +} + +func (r *AdmissionReconciler) selectParticipants( + ctx context.Context, + requestNode *corev1.Node, + profile *signalv1.CertificationProfile, +) ([]signalv1.Participant, error) { + count := profile.Spec.Participants.Count + if count < 1 { + return nil, fmt.Errorf("profile %s participant count must be positive", profile.Name) + } + + if count > 64 { + return nil, fmt.Errorf("profile %s participant count must not exceed 64", profile.Name) + } + + reader := r.APIReader + if reader == nil { + reader = r.Client + } + + var authoritativeRequest corev1.Node + if err := reader.Get(ctx, client.ObjectKey{Name: requestNode.Name}, &authoritativeRequest); err != nil { + return nil, fmt.Errorf("read request node %s: %w", requestNode.Name, err) + } + + if requestNode.UID != "" && authoritativeRequest.UID != requestNode.UID { + return nil, fmt.Errorf("request node %s identity changed", requestNode.Name) + } + + requestNode = &authoritativeRequest + + selector, err := metav1.LabelSelectorAsSelector(&profile.Spec.Participants.Selector) + if err != nil { + return nil, fmt.Errorf("profile %s participant selector: %w", profile.Name, err) + } + + if !selector.Matches(labels.Set(requestNode.Labels)) || !nodeEligible(requestNode, profile.Spec.Participants.RequiredResources) { + return nil, fmt.Errorf("request node %s does not satisfy profile %s", requestNode.Name, profile.Name) + } + + var nodes corev1.NodeList + + if err := reader.List(ctx, &nodes); err != nil { + return nil, fmt.Errorf("list participant nodes: %w", err) + } + + sort.Slice(nodes.Items, func(i, j int) bool { return nodes.Items[i].Name < nodes.Items[j].Name }) + + selected := []signalv1.Participant{participantSnapshot(requestNode, 0)} + + for i := range nodes.Items { + candidate := &nodes.Items[i] + + if int32(len(selected)) == count { + break + } + + if candidate.Name == requestNode.Name || + !selector.Matches(labels.Set(candidate.Labels)) || + !nodeEligible(candidate, profile.Spec.Participants.RequiredResources) || + candidate.Annotations[ReservationLabel] != "" { + continue + } + + selected = append(selected, participantSnapshot(candidate, int32(len(selected)))) + } + + if int32(len(selected)) != count { + return nil, fmt.Errorf("profile %s requires %d eligible unlocked participants, found %d", profile.Name, count, len(selected)) + } + + return selected, nil +} + +func (r *AdmissionReconciler) releaseOrphanedReservations(ctx context.Context, runName string) error { + reader := r.APIReader + if reader == nil { + reader = r.Client + } + + var nodes corev1.NodeList + if err := reader.List(ctx, &nodes); err != nil { + return fmt.Errorf("list Nodes reserved by %s: %w", runName, err) + } + + participants := make([]signalv1.Participant, 0) + + for index := range nodes.Items { + node := &nodes.Items[index] + if node.Annotations[ReservationLabel] == runName { + participants = append(participants, signalv1.Participant{ + NodeName: node.Name, + NodeUID: node.UID, + }) + } + } + + return r.releaseParticipants(ctx, participants, runName) +} + +func participantSnapshot(node *corev1.Node, index int32) signalv1.Participant { + return signalv1.Participant{ + Index: index, + NodeName: node.Name, + NodeUID: node.UID, + Capacity: node.Status.Capacity.DeepCopy(), + Allocatable: node.Status.Allocatable.DeepCopy(), + } +} + +func nodeEligible(node *corev1.Node, required corev1.ResourceList) bool { + if node.Spec.Unschedulable || !nodeReady(node) { + return false + } + + for name, quantity := range required { + available, ok := node.Status.Allocatable[name] + if !ok || available.Cmp(quantity) < 0 { + return false + } + } + + return true +} + +func nodeReady(node *corev1.Node) bool { + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady { + return condition.Status == corev1.ConditionTrue + } + } + + return false +} + +func (r *AdmissionReconciler) reserveParticipants(ctx context.Context, participants []signalv1.Participant, runName string) error { + reserved := make([]signalv1.Participant, 0, len(participants)) + for _, participant := range participants { + var node corev1.Node + if err := r.Get(ctx, client.ObjectKey{Name: participant.NodeName}, &node); err != nil { + return errors.Join( + fmt.Errorf("get participant %s: %w", participant.NodeName, err), + r.releaseParticipants(context.WithoutCancel(ctx), reserved, runName), + ) + } + + if node.UID != participant.NodeUID { + return errors.Join( + fmt.Errorf("participant %s identity changed", participant.NodeName), + r.releaseParticipants(context.WithoutCancel(ctx), reserved, runName), + ) + } + + if value := node.Annotations[ReservationLabel]; value != "" { + return errors.Join( + fmt.Errorf("participant %s is already reserved by %s", participant.NodeName, value), + r.releaseParticipants(context.WithoutCancel(ctx), reserved, runName), + ) + } + + before := node.DeepCopy() + if node.Annotations == nil { + node.Annotations = map[string]string{} + } + + node.Annotations[ReservationLabel] = runName + if err := r.Patch( + ctx, + &node, + client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{}), + ); err != nil { + return errors.Join( + fmt.Errorf("reserve participant %s: %w", participant.NodeName, err), + r.releaseParticipants(context.WithoutCancel(ctx), reserved, runName), + ) + } + + reserved = append(reserved, participant) + } + + return nil +} + +func (r *AdmissionReconciler) releaseParticipants(ctx context.Context, participants []signalv1.Participant, runName string) error { + var errs []error + + for _, participant := range participants { + var node corev1.Node + if err := r.Get(ctx, client.ObjectKey{Name: participant.NodeName}, &node); err != nil { + if !apierrors.IsNotFound(err) { + errs = append(errs, err) + } + + continue + } + + if participant.NodeUID != "" && node.UID != participant.NodeUID { + continue + } + + if node.Annotations[ReservationLabel] != runName { + continue + } + + before := node.DeepCopy() + delete(node.Annotations, ReservationLabel) + + if err := r.Patch(ctx, &node, client.MergeFrom(before)); err != nil { + errs = append(errs, err) + } + } + + return errors.Join(errs...) +} + +func (r *AdmissionReconciler) createSnapshot( + ctx context.Context, + runName string, + profile *signalv1.CertificationProfile, +) (*corev1.ConfigMap, string, error) { + var source corev1.ConfigMap + if err := r.Get(ctx, client.ObjectKey{Namespace: r.ScriptNamespace, Name: profile.Spec.Script.Name}, &source); err != nil { + return nil, "", fmt.Errorf("get source script ConfigMap: %w", err) + } + + content, ok := source.Data[profile.Spec.Script.Key] + if !ok { + return nil, "", fmt.Errorf("source script ConfigMap %s has no key %s", source.Name, profile.Spec.Script.Key) + } + + digest := digestString(content) + immutable := true + + snapshot := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: runName + "-scripts", + Namespace: r.Namespace, + Labels: map[string]string{SnapshotRunLabel: runName}, + }, + Immutable: &immutable, + Data: map[string]string{ + profile.Spec.Script.Key: content, + "sha256": digest, + }, + } + if err := r.Create(ctx, snapshot); err != nil { + return nil, "", fmt.Errorf("create immutable script snapshot: %w", err) + } + + return snapshot, digest, nil +} + +func (r *RunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var run signalv1.CertificationRun + if err := r.Get(ctx, req.NamespacedName, &run); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + if !run.DeletionTimestamp.IsZero() { + return r.reconcileDeleting(ctx, &run) + } + + if run.Status.Phase.Terminal() { + return r.cleanup(ctx, &run) + } + + if run.Status.FinishedAt != nil { + return r.finish( + ctx, + &run, + nil, + run.Status.ExecutionState, + run.Status.Verdict, + run.Status.Reason, + ) + } + + if run.Spec.Cancel { + return r.cancel(ctx, &run) + } + + if run.Status.Workflow.Name == "" { + return r.startAttempt(ctx, &run, 1) + } + + workflow := &unstructured.Unstructured{} + workflow.SetGroupVersionKind(workflowGVK) + + reader := r.APIReader + if reader == nil { + reader = r.Client + } + + if err := reader.Get(ctx, client.ObjectKey{Namespace: run.Namespace, Name: run.Status.Workflow.Name}, workflow); err != nil { + if apierrors.IsNotFound(err) { + return r.retryOrFinish(ctx, &run, "Error", "WorkflowMissing", "workflow disappeared before terminal status") + } + + return ctrl.Result{}, err + } + + if run.Status.Workflow.UID == "" || + string(workflow.GetUID()) != run.Status.Workflow.UID || + !metav1.IsControlledBy(workflow, &run) { + return ctrl.Result{}, fmt.Errorf( + "workflow %s identity or ownership changed; refusing concurrent retry", + workflow.GetName(), + ) + } + + phase, _, err := unstructured.NestedString(workflow.Object, "status", "phase") + if err != nil { + return ctrl.Result{}, fmt.Errorf("read Workflow phase: %w", err) + } + + switch phase { + case "", "Pending", "Running": + if err := projectTesting(ctx, r.Client, &run); err != nil { + return ctrl.Result{}, err + } + + if run.Status.Phase != signalv1.RunPhaseRunning || run.Status.Workflow.Phase != phase { + before := run.DeepCopy() + run.Status.Phase = signalv1.RunPhaseRunning + + run.Status.Workflow.Phase = phase + if run.Status.StartedAt == nil { + now := metav1.NewTime(r.now()) + run.Status.StartedAt = &now + } + + return ctrl.Result{}, r.Status().Patch(ctx, &run, client.MergeFrom(before)) + } + + return ctrl.Result{RequeueAfter: 15 * time.Second}, nil + case "Succeeded": + verdict := workflowParameter(workflow, "verdict") + reason := workflowParameter(workflow, "reason") + + if verdict == "" { + verdict = "Inconclusive" + reason = "workflow completed without a verdict" + } + + return r.finish(ctx, &run, workflow, "Completed", verdict, reason) + case "Failed", "Error": + return r.retryOrFinish(ctx, &run, phase, "Workflow"+phase, workflowMessage(workflow)) + default: + return ctrl.Result{}, fmt.Errorf("unsupported Workflow phase %q", phase) + } +} + +func (r *RunReconciler) startAttempt(ctx context.Context, run *signalv1.CertificationRun, attempt int32) (ctrl.Result, error) { + workflow, err := buildWorkflow(run, attempt) + if err != nil { + return ctrl.Result{}, err + } + + if err := r.Create(ctx, workflow); err != nil { + if !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, fmt.Errorf("create Workflow attempt %d: %w", attempt, err) + } + + existing := &unstructured.Unstructured{} + existing.SetGroupVersionKind(workflowGVK) + + if getErr := r.Get(ctx, client.ObjectKeyFromObject(workflow), existing); getErr != nil { + return ctrl.Result{}, fmt.Errorf("get existing Workflow attempt %d: %w", attempt, getErr) + } + + if !metav1.IsControlledBy(existing, run) { + return ctrl.Result{}, fmt.Errorf("workflow attempt %d already exists without matching CertificationRun ownership", attempt) + } + + workflow = existing + } + + before := run.DeepCopy() + run.Status.Phase = signalv1.RunPhaseRunning + run.Status.ExecutionState = "Running" + + run.Status.Workflow = signalv1.WorkflowStatus{ + Name: workflow.GetName(), + UID: string(workflow.GetUID()), + Attempt: attempt, + Phase: "Pending", + } + if run.Status.StartedAt == nil { + now := metav1.NewTime(r.now()) + run.Status.StartedAt = &now + } + + if err := r.Status().Patch(ctx, run, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, fmt.Errorf("record Workflow attempt: %w", err) + } + + if r.Recorder != nil { + r.Recorder.Eventf(run, nil, corev1.EventTypeNormal, "WorkflowStarted", "StartWorkflow", + "Created Workflow %s for attempt %d", workflow.GetName(), attempt) + } + + return ctrl.Result{}, nil +} + +func buildWorkflow(run *signalv1.CertificationRun, attempt int32) (*unstructured.Unstructured, error) { + participantNames := make([]string, 0, len(run.Spec.Participants)) + for _, participant := range run.Spec.Participants { + participantNames = append(participantNames, participant.NodeName) + } + + tolerations := uniqueTolerations(append([]corev1.Toleration{{ + Key: "nvidia.com/gpu", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }}, run.Spec.Tolerations...)) + + tolerationsJSON, err := json.Marshal(tolerations) + if err != nil { + return nil, fmt.Errorf("encode Workflow tolerations: %w", err) + } + + scriptEnvironment, err := scriptEnvironment(run.Spec.Parameters) + if err != nil { + return nil, err + } + + scriptEnvironmentJSON, err := json.Marshal(scriptEnvironment) + if err != nil { + return nil, fmt.Errorf("encode script environment: %w", err) + } + + participantsJSON, err := json.Marshal(run.Spec.Participants) + if err != nil { + return nil, fmt.Errorf("encode Workflow participants: %w", err) + } + + name := fmt.Sprintf("%s-a%d", run.Name, attempt) + workflow := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": workflowGVK.GroupVersion().String(), + "kind": workflowGVK.Kind, + "metadata": map[string]any{ + "name": name, + "namespace": run.Namespace, + "labels": map[string]any{ + SnapshotRunLabel: run.Name, + }, + }, + "spec": map[string]any{ + "activeDeadlineSeconds": int64(run.Spec.Deadline.Seconds()), + "ttlStrategy": map[string]any{ + "secondsAfterCompletion": int64(6 * time.Hour / time.Second), + "secondsAfterSuccess": int64(6 * time.Hour / time.Second), + "secondsAfterFailure": int64(6 * time.Hour / time.Second), + }, + "podGC": map[string]any{ + "strategy": "OnWorkflowCompletion", + "deleteDelayDuration": "2h", + }, + "workflowTemplateRef": map[string]any{ + "name": run.Spec.Workflow.Name, + }, + "arguments": map[string]any{ + "parameters": []any{ + map[string]any{"name": "run-name", "value": run.Name}, + map[string]any{"name": "run-uid", "value": string(run.UID)}, + map[string]any{"name": "participants", "value": strings.Join(participantNames, ",")}, + map[string]any{"name": "participants-json", "value": string(participantsJSON)}, + map[string]any{"name": "participant-count", "value": fmt.Sprintf("%d", len(run.Spec.Participants))}, + map[string]any{"name": "request-node-name", "value": run.Spec.RequestNodeName}, + map[string]any{"name": "script-configmap", "value": run.Spec.Script.ConfigMapName}, + map[string]any{"name": "script-key", "value": run.Spec.Script.Key}, + map[string]any{"name": "script-digest", "value": run.Spec.Script.Digest}, + map[string]any{"name": "backend-api-version", "value": run.Spec.Backend.APIVersion}, + map[string]any{"name": "backend-kind", "value": run.Spec.Backend.Kind}, + map[string]any{"name": "queue", "value": run.Spec.Backend.Queue}, + map[string]any{"name": "runner-image", "value": run.Spec.RunnerImage}, + map[string]any{"name": "tolerations", "value": string(tolerationsJSON)}, + map[string]any{"name": "script-environment", "value": string(scriptEnvironmentJSON)}, + map[string]any{"name": "mpi-payload-mode", "value": run.Spec.Parameters["mpi-payload-mode"]}, + map[string]any{"name": "expected-gpu-product-regex", "value": run.Spec.Parameters["expected-gpu-product-regex"]}, + map[string]any{"name": "expected-gpu-count", "value": run.Spec.Parameters["expected-gpu-count"]}, + map[string]any{"name": "observed-gpu-capacity", "value": run.Spec.Parameters["observed-gpu-capacity"]}, + map[string]any{"name": "observed-gpu-allocatable", "value": run.Spec.Parameters["observed-gpu-allocatable"]}, + map[string]any{"name": "gpu-resource-count", "value": run.Spec.Parameters["gpu-resource-count"]}, + map[string]any{"name": "privileged", "value": run.Spec.Parameters["privileged"]}, + map[string]any{"name": "attempt", "value": fmt.Sprintf("%d", attempt)}, + }, + }, + }, + }} + workflow.SetOwnerReferences([]metav1.OwnerReference{ + *metav1.NewControllerRef(run, signalv1.GroupVersion.WithKind("CertificationRun")), + }) + + return workflow, nil +} + +func validEnvironmentName(value string) bool { + if value == "" || (value[0] != '_' && (value[0] < 'A' || value[0] > 'Z')) { + return false + } + + for index := 1; index < len(value); index++ { + character := value[index] + if (character < 'A' || character > 'Z') && + (character < '0' || character > '9') && + character != '_' { + return false + } + } + + return true +} + +func scriptEnvironment(parameters map[string]string) (map[string]string, error) { + keys := make([]string, 0, len(parameters)) + for key := range parameters { + keys = append(keys, key) + } + + sort.Strings(keys) + + environment := make(map[string]string, len(parameters)) + parameterByEnvironment := make(map[string]string, len(parameters)) + + for _, key := range keys { + environmentName := strings.ToUpper(strings.ReplaceAll(key, "-", "_")) + if !validEnvironmentName(environmentName) { + return nil, fmt.Errorf("profile parameter %q cannot be mapped to a script environment variable", key) + } + + if strings.HasPrefix(environmentName, "PROJECT_SIGNAL_") { + return nil, fmt.Errorf("profile parameter %q maps to reserved environment variable %s", key, environmentName) + } + + if _, reserved := reservedScriptEnvironmentNames[environmentName]; reserved { + return nil, fmt.Errorf("profile parameter %q maps to reserved environment variable %s", key, environmentName) + } + + if previous, exists := parameterByEnvironment[environmentName]; exists { + return nil, fmt.Errorf("profile parameters %q and %q both map to %s", previous, key, environmentName) + } + + parameterByEnvironment[environmentName] = key + environment[environmentName] = parameters[key] + } + + return environment, nil +} + +func (r *RunReconciler) retryOrFinish( + ctx context.Context, + run *signalv1.CertificationRun, + executionState string, + reason string, + message string, +) (ctrl.Result, error) { + if run.Status.Workflow.Attempt < run.Spec.MaxAttempts { + return r.startAttempt(ctx, run, run.Status.Workflow.Attempt+1) + } + + return r.finish(ctx, run, nil, executionState, "Inconclusive", reason+": "+message) +} + +func (r *RunReconciler) finish( + ctx context.Context, + run *signalv1.CertificationRun, + workflow *unstructured.Unstructured, + executionState string, + verdict string, + reason string, +) (ctrl.Result, error) { + if run.Status.FinishedAt == nil { + before := run.DeepCopy() + run.Status.ExecutionState = executionState + run.Status.Verdict = verdict + finalReason := reason + + if workflow != nil { + outputs, err := workflowParameters(workflow) + if err != nil { + run.Status.Verdict = "Inconclusive" + finalReason = "workflow outputs rejected: " + err.Error() + } else { + run.Status.Workflow.Outputs = outputs + + backendResources, backendErr := backendResourceReferences( + outputs["backend-resources"], + len(run.Spec.Participants), + ) + if backendErr != nil { + run.Status.Verdict = "Inconclusive" + finalReason = "workflow backend resources rejected: " + backendErr.Error() + } else { + run.Status.Workflow.BackendResources = backendResources + if run.Status.Verdict == "Pass" && + len(run.Spec.Participants) > 1 && + len(backendResources) == 0 { + run.Status.Verdict = "Inconclusive" + finalReason = "workflow did not identify backend resources for all participants" + } + } + } + } + + run.Status.Reason = finalReason + if executionState == "Completed" { + run.Status.Workflow.Phase = "Succeeded" + } else { + run.Status.Workflow.Phase = executionState + } + + now := metav1.NewTime(r.now()) + run.Status.FinishedAt = &now + + if err := r.Status().Patch(ctx, run, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, err + } + } + + executionState = run.Status.ExecutionState + verdict = run.Status.Verdict + reason = run.Status.Reason + + evidence, err := r.evidenceDocument(ctx, run, executionState, verdict, reason) + if err != nil { + return ctrl.Result{}, err + } + + uri, digest, err := r.Evidence.Put(ctx, run.Spec.EvidencePath, evidence) + if err != nil { + return ctrl.Result{}, err + } + + before := run.DeepCopy() + run.Status.Evidence = signalv1.EvidenceStatus{URI: uri, Digest: digest} + + switch verdict { + case "Pass": + run.Status.Phase = signalv1.RunPhaseSucceeded + case "Fail", "Suspect": + run.Status.Phase = signalv1.RunPhaseFailed + default: + run.Status.Phase = signalv1.RunPhaseInconclusive + } + + if err := r.Status().Patch(ctx, run, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, err + } + + if err := projectTerminal(ctx, r.Client, run); err != nil { + return ctrl.Result{}, err + } + + if r.Recorder != nil { + r.Recorder.Eventf(run, nil, corev1.EventTypeNormal, "CertificationFinished", "FinishCertification", + "ExecutionState=%s Verdict=%s Reason=%s", executionState, verdict, reason) + } + + return ctrl.Result{RequeueAfter: time.Millisecond}, nil +} + +func uniqueTolerations(tolerations []corev1.Toleration) []corev1.Toleration { + unique := make([]corev1.Toleration, 0, len(tolerations)) + for _, toleration := range tolerations { + duplicate := false + + for _, existing := range unique { + if reflect.DeepEqual(existing, toleration) { + duplicate = true + + break + } + } + + if !duplicate { + unique = append(unique, toleration) + } + } + + return unique +} + +func (r *RunReconciler) evidenceDocument( + ctx context.Context, + run *signalv1.CertificationRun, + executionState string, + verdict string, + reason string, +) ([]byte, error) { + var snapshot corev1.ConfigMap + if err := r.Get(ctx, client.ObjectKey{Namespace: run.Namespace, Name: run.Spec.Script.ConfigMapName}, &snapshot); err != nil { + return nil, fmt.Errorf("read script snapshot for evidence: %w", err) + } + + doc := map[string]any{ + "apiVersion": "signal.unbounded-cloud.io/evidence/v1alpha1", + "runID": string(run.UID), + "runName": run.Name, + "profile": run.Spec.Profile, + "participants": run.Spec.Participants, + "workflow": run.Status.Workflow, + "workflowOutputs": run.Status.Workflow.Outputs, + "backend": run.Spec.Backend, + "runnerImage": run.Spec.RunnerImage, + "purpose": run.Spec.Purpose, + "stagePlan": run.Spec.StagePlan, + "parameters": run.Spec.Parameters, + "scriptSnapshot": run.Spec.Script, + "scriptContent": snapshot.Data[run.Spec.Script.Key], + "executionState": executionState, + "verdict": verdict, + "reason": reason, + "startedAt": run.Status.StartedAt, + "finishedAt": run.Status.FinishedAt.UTC().Format(time.RFC3339Nano), + } + + return json.MarshalIndent(doc, "", " ") +} + +func backendResourceReferences( + value string, + participantCount int, +) ([]signalv1.BackendResourceReference, error) { + if participantCount < 1 || participantCount > 64 { + return nil, fmt.Errorf("participant count must be between 1 and 64") + } + + if value == "" { + return nil, nil + } + + var references []signalv1.BackendResourceReference + + decoder := json.NewDecoder(strings.NewReader(value)) + decoder.DisallowUnknownFields() + + if err := decoder.Decode(&references); err != nil { + return nil, fmt.Errorf("decode backend-resources: %w", err) + } + + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return nil, fmt.Errorf("backend-resources contains trailing JSON values") + } + + if len(references) == 0 || len(references) > 64 { + return nil, fmt.Errorf("backend-resources must contain between 1 and 64 entries") + } + + identities := make(map[string]struct{}, len(references)) + covered := make([]bool, participantCount) + + for index := range references { + reference := &references[index] + if reference.APIVersion == "" || reference.Kind == "" || reference.Name == "" || reference.UID == "" { + return nil, fmt.Errorf("backend-resources[%d] has an incomplete identity", index) + } + + identity := strings.Join([]string{ + reference.APIVersion, + reference.Kind, + reference.Namespace, + reference.Name, + string(reference.UID), + }, "\x00") + if _, exists := identities[identity]; exists { + return nil, fmt.Errorf("backend-resources[%d] duplicates a workload identity", index) + } + + identities[identity] = struct{}{} + + if len(reference.ParticipantIndexes) == 0 { + return nil, fmt.Errorf("backend-resources[%d] does not identify a participant", index) + } + + seenIndexes := make(map[int32]struct{}, len(reference.ParticipantIndexes)) + for _, participantIndex := range reference.ParticipantIndexes { + if participantIndex < 0 || int(participantIndex) >= participantCount { + return nil, fmt.Errorf( + "backend-resources[%d] participant index %d is out of range", + index, + participantIndex, + ) + } + + if _, exists := seenIndexes[participantIndex]; exists { + return nil, fmt.Errorf( + "backend-resources[%d] repeats participant index %d", + index, + participantIndex, + ) + } + + seenIndexes[participantIndex] = struct{}{} + covered[participantIndex] = true + } + } + + if participantCount > 1 { + for index, isCovered := range covered { + if !isCovered { + return nil, fmt.Errorf("backend-resources does not cover participant index %d", index) + } + } + } + + return references, nil +} + +func workflowParameters(workflow *unstructured.Unstructured) (map[string]string, error) { + parameters, found, err := unstructured.NestedSlice(workflow.Object, "status", "outputs", "parameters") + if err != nil { + return nil, fmt.Errorf("read parameters: %w", err) + } + + if !found { + return nil, nil + } + + outputs := make(map[string]string, len(parameters)) + for _, raw := range parameters { + parameter, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("parameter entry is not an object") + } + + name, nameOK := parameter["name"].(string) + value, valueOK := parameter["value"].(string) + + if !nameOK || !valueOK { + return nil, fmt.Errorf("parameter name and value must be strings") + } + + limit, allowed := workflowOutputLimits[name] + if !allowed { + return nil, fmt.Errorf("unexpected parameter %q", name) + } + + if len(value) > limit { + return nil, fmt.Errorf("parameter %q exceeds %d bytes", name, limit) + } + + if _, duplicate := outputs[name]; duplicate { + return nil, fmt.Errorf("duplicate parameter %q", name) + } + + outputs[name] = value + } + + return outputs, nil +} + +func (r *RunReconciler) cancel(ctx context.Context, run *signalv1.CertificationRun) (ctrl.Result, error) { + stopped, err := r.ensureWorkflowStopped(ctx, run) + if err != nil { + return ctrl.Result{}, err + } + + if !stopped { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + + before := run.DeepCopy() + run.Status.Phase = signalv1.RunPhaseCancelled + run.Status.ExecutionState = "Interrupted" + run.Status.Verdict = "Inconclusive" + run.Status.Reason = "canceled by operator" + now := metav1.NewTime(r.now()) + + run.Status.FinishedAt = &now + if err := r.Status().Patch(ctx, run, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{RequeueAfter: time.Millisecond}, nil +} + +func (r *RunReconciler) cleanup(ctx context.Context, run *signalv1.CertificationRun) (ctrl.Result, error) { + if err := projectTerminal(ctx, r.Client, run); err != nil { + return ctrl.Result{}, err + } + + released, err := r.releaseRunReservations(ctx, run) + if err != nil { + return ctrl.Result{}, err + } + + deleted := false + + snapshot := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: run.Namespace, Name: run.Spec.Script.ConfigMapName}} + if err := r.Delete(ctx, snapshot); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + + deleted = true + + if err := r.clearRequest(ctx, run); err != nil { + return ctrl.Result{}, err + } + + statusBefore := run.DeepCopy() + + run.Status.Cleanup = signalv1.CleanupStatus{ReservationsReleased: released, SnapshotDeleted: deleted} + if err := r.Status().Patch(ctx, run, client.MergeFrom(statusBefore)); err != nil { + return ctrl.Result{}, err + } + + before := run.DeepCopy() + + run.Finalizers = removeString(run.Finalizers, RunFinalizer) + if err := r.Patch(ctx, run, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, err + } + + if r.Recorder != nil { + r.Recorder.Eventf(run, nil, corev1.EventTypeNormal, "CertificationCleaned", "CleanCertification", + "Released participant reservations, cleared the request label, and deleted the script snapshot") + } + + return ctrl.Result{}, nil +} + +func (r *RunReconciler) reconcileDeleting(ctx context.Context, run *signalv1.CertificationRun) (ctrl.Result, error) { + if !containsString(run.Finalizers, RunFinalizer) { + return ctrl.Result{}, nil + } + + stopped, err := r.ensureWorkflowStopped(ctx, run) + if err != nil { + return ctrl.Result{}, err + } + + if !stopped { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + + if _, err := r.releaseRunReservations(ctx, run); err != nil { + return ctrl.Result{}, err + } + + if err := r.clearRequest(ctx, run); err != nil { + return ctrl.Result{}, err + } + + before := run.DeepCopy() + run.Finalizers = removeString(run.Finalizers, RunFinalizer) + + return ctrl.Result{}, r.Patch(ctx, run, client.MergeFrom(before)) +} + +func (r *RunReconciler) releaseRunReservations(ctx context.Context, run *signalv1.CertificationRun) (bool, error) { + for _, participant := range run.Spec.Participants { + var node corev1.Node + if err := r.Get(ctx, client.ObjectKey{Name: participant.NodeName}, &node); err != nil { + if apierrors.IsNotFound(err) { + continue + } + + return false, err + } + + if participant.NodeUID != "" && node.UID != participant.NodeUID { + continue + } + + if node.Annotations[ReservationLabel] != run.Name { + continue + } + + before := node.DeepCopy() + delete(node.Annotations, ReservationLabel) + + if err := r.Patch(ctx, &node, client.MergeFrom(before)); err != nil { + return false, err + } + } + + return true, nil +} + +func (r *RunReconciler) clearRequest(ctx context.Context, run *signalv1.CertificationRun) error { + var node corev1.Node + if err := r.Get(ctx, client.ObjectKey{Name: run.Spec.RequestNodeName}, &node); err != nil { + return client.IgnoreNotFound(err) + } + + if requestNodeUID := participantNodeUID(run, run.Spec.RequestNodeName); requestNodeUID != "" && node.UID != requestNodeUID { + return nil + } + + if node.Labels[RequestLabel] != run.Spec.Profile.Name { + return nil + } + + before := node.DeepCopy() + delete(node.Labels, RequestLabel) + + return r.Patch(ctx, &node, client.MergeFrom(before)) +} + +func (r *RunReconciler) ensureWorkflowStopped(ctx context.Context, run *signalv1.CertificationRun) (bool, error) { + if run.Status.Workflow.Name == "" { + return true, nil + } + + workflow := &unstructured.Unstructured{} + workflow.SetGroupVersionKind(workflowGVK) + + key := client.ObjectKey{Namespace: run.Namespace, Name: run.Status.Workflow.Name} + reader := r.APIReader + + if reader == nil { + reader = r.Client + } + + if err := reader.Get(ctx, key, workflow); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + + return false, fmt.Errorf("get Workflow before deletion: %w", err) + } + + if run.Status.Workflow.UID == "" || + string(workflow.GetUID()) != run.Status.Workflow.UID || + !metav1.IsControlledBy(workflow, run) { + return false, fmt.Errorf("refusing to delete Workflow with changed identity or ownership") + } + + propagation := metav1.DeletePropagationForeground + uid := workflow.GetUID() + + if err := r.Delete( + ctx, + workflow, + client.PropagationPolicy(propagation), + client.Preconditions{UID: &uid}, + ); err != nil && !apierrors.IsNotFound(err) { + return false, fmt.Errorf("delete Workflow before releasing participants: %w", err) + } + + if err := reader.Get(ctx, key, workflow); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + + return false, fmt.Errorf("confirm Workflow deletion: %w", err) + } + + return false, nil +} + +func participantNodeUID(run *signalv1.CertificationRun, nodeName string) types.UID { + for _, participant := range run.Spec.Participants { + if participant.NodeName == nodeName { + return participant.NodeUID + } + } + + return "" +} + +func workflowParameter(workflow *unstructured.Unstructured, name string) string { + parameters, _, err := unstructured.NestedSlice(workflow.Object, "status", "outputs", "parameters") + if err == nil { + if value := parameterValue(parameters, name); value != "" { + return value + } + } + + nodes, _, err := unstructured.NestedMap(workflow.Object, "status", "nodes") + if err != nil { + return "" + } + + for _, raw := range nodes { + node, ok := raw.(map[string]any) + if !ok { + continue + } + + outputs, ok := node["outputs"].(map[string]any) + if !ok { + continue + } + + parameters, ok := outputs["parameters"].([]any) + if ok { + if value := parameterValue(parameters, name); value != "" { + return value + } + } + } + + return "" +} + +func parameterValue(parameters []any, name string) string { + for _, raw := range parameters { + parameter, ok := raw.(map[string]any) + if !ok || parameter["name"] != name { + continue + } + + value, ok := parameter["value"].(string) + if !ok { + return "" + } + + return value + } + + return "" +} + +func workflowMessage(workflow *unstructured.Unstructured) string { + message, _, err := unstructured.NestedString(workflow.Object, "status", "message") + if err != nil { + return "workflow message is malformed" + } + + if message == "" { + return "workflow did not report a message" + } + + return message +} + +func digestString(content string) string { + sum := sha256.Sum256([]byte(content)) + + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func runName(node, profile string, now time.Time) string { + base := strings.ToLower(node + "-" + profile) + base = strings.Map(func(r rune) rune { + if r >= 'a' && r <= 'z' || r >= '0' && r <= '9' || r == '-' { + return r + } + + return '-' + }, base) + + base = strings.Trim(base, "-") + if len(base) > 40 { + base = strings.TrimRight(base[:40], "-") + } + + return fmt.Sprintf("%s-%x-%s", base, now.Unix(), rand.String(5)) +} + +func (r *AdmissionReconciler) now() time.Time { + if r.Now != nil { + return r.Now() + } + + return time.Now() +} + +func (r *RunReconciler) now() time.Time { + if r.Now != nil { + return r.Now() + } + + return time.Now() +} + +func containsString(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + + return false +} + +func removeString(values []string, remove string) []string { + result := values[:0] + for _, value := range values { + if value != remove { + result = append(result, value) + } + } + + return result +} + +func validateStagePlan(plan []signalv1.CertificationStage) error { + seen := map[string]struct{}{} + + for i := range plan { + stage := &plan[i] + if stage.ID == "" { + return fmt.Errorf("stage %d has an empty ID", i) + } + + if stage.DisplayName == "" { + return fmt.Errorf("stage %q has an empty display name", stage.ID) + } + + if _, ok := seen[stage.ID]; ok { + return fmt.Errorf("duplicate stage ID %q", stage.ID) + } + + seen[stage.ID] = struct{}{} + } + + return nil +} + +func AddToScheme(scheme *runtime.Scheme) error { + return signalv1.AddToScheme(scheme) +} diff --git a/internal/signal/controller_test.go b/internal/signal/controller_test.go new file mode 100644 index 0000000000..0e25380400 --- /dev/null +++ b/internal/signal/controller_test.go @@ -0,0 +1,1418 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signal + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" +) + +func TestAdmissionCreatesImmutableSnapshotAndPreventsDuplicate(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + node := readyNode("gpu-0", "example-gpu") + node.Labels[RequestLabel] = "gpu-smoke" + profile := testProfile("gpu-smoke", "example-gpu") + + const payload = "test payload\n" + + source := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, + Data: map[string]string{"payload.txt": payload}, + } + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node, profile, source).Build() + now := time.Unix(100, 0) + r := &AdmissionReconciler{Client: kube, Namespace: "signal", ScriptNamespace: "signal", Now: func() time.Time { return now }} + + if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}); err != nil { + t.Fatalf("Reconcile: %v", err) + } + + if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}); err != nil { + t.Fatalf("duplicate Reconcile: %v", err) + } + + var runs signalv1.CertificationRunList + if err := kube.List(context.Background(), &runs); err != nil { + t.Fatalf("list runs: %v", err) + } + + if len(runs.Items) != 1 { + t.Fatalf("runs = %d, want 1", len(runs.Items)) + } + + run := runs.Items[0] + if run.Spec.Script.Digest != digestString(payload) { + t.Fatalf("script digest = %q", run.Spec.Script.Digest) + } + + if len(run.Spec.StagePlan) != 1 || run.Spec.StagePlan[0].ID != "workload" { + t.Fatalf("stage plan = %#v, want immutable profile plan", run.Spec.StagePlan) + } + + if run.Spec.Purpose != signalv1.PurposeQuickCertification { + t.Fatalf("purpose = %q, want %q", run.Spec.Purpose, signalv1.PurposeQuickCertification) + } + + if run.Spec.Parameters["observed-gpu-capacity"] != "8" || + run.Spec.Parameters["observed-gpu-allocatable"] != "8" { + t.Fatalf("observed GPU inventory = %#v", run.Spec.Parameters) + } + + if run.Annotations[RunTriggerAnnotation] != "manual" { + t.Fatalf("trigger = %q, want manual", run.Annotations[RunTriggerAnnotation]) + } + + var snapshot corev1.ConfigMap + if err := kube.Get(context.Background(), client.ObjectKey{Namespace: "signal", Name: run.Spec.Script.ConfigMapName}, &snapshot); err != nil { + t.Fatalf("get snapshot: %v", err) + } + + if snapshot.Immutable == nil || !*snapshot.Immutable { + t.Fatal("snapshot is mutable") + } + + if snapshot.Data["payload.txt"] != payload { + t.Fatalf("snapshot payload = %q", snapshot.Data["payload.txt"]) + } + + var locked corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &locked); err != nil { + t.Fatalf("get locked node: %v", err) + } + + if locked.Annotations[ReservationLabel] != run.Name { + t.Fatalf("reservation = %q, want %q", locked.Annotations[ReservationLabel], run.Name) + } +} + +func TestAdmissionRejectsMutableRunnerImageBeforeReservation(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + node := readyNode("gpu-0", "example-gpu") + node.Labels[RequestLabel] = "gpu-smoke" + profile := testProfile("gpu-smoke", "example-gpu") + profile.Spec.RunnerImage = "registry.example/runner:latest" + source := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, + Data: map[string]string{"payload.txt": "test payload\n"}, + } + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node, profile, source).Build() + reconciler := &AdmissionReconciler{Client: kube, Namespace: "signal", ScriptNamespace: "signal"} + + _, err := reconciler.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}, + ) + if err == nil || !strings.Contains(err.Error(), "immutable sha256 digest") { + t.Fatalf("Reconcile error = %v", err) + } + + var got corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &got); err != nil { + t.Fatal(err) + } + + if got.Annotations[ReservationLabel] != "" { + t.Fatalf("node was reserved after rejected profile: %#v", got.Annotations) + } +} + +func TestParticipantSelectionIsDeterministicAndSkipsLockedNodes(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + request := readyNode("gpu-0", "H200") + peer := readyNode("gpu-1", "H200") + locked := readyNode("gpu-2", "H200") + locked.Annotations = map[string]string{ReservationLabel: "other-run"} + wrong := readyNode("gpu-3", "A100") + profile := testProfile("h200-pair", "H200") + profile.Spec.Participants.Count = 2 + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(request, peer, locked, wrong).Build() + r := &AdmissionReconciler{Client: kube} + + participants, err := r.selectParticipants(context.Background(), request, profile) + if err != nil { + t.Fatalf("selectParticipants: %v", err) + } + + if len(participants) != 2 || + participants[0].Index != 0 || + participants[0].NodeName != request.Name || + participants[1].Index != 1 || + participants[1].NodeName != peer.Name { + t.Fatalf("participants = %#v", participants) + } + + gpu := corev1.ResourceName("nvidia.com/gpu") + capacity := participants[1].Capacity[gpu] + + allocatable := participants[1].Allocatable[gpu] + if capacity.Cmp(resource.MustParse("8")) != 0 || + allocatable.Cmp(resource.MustParse("8")) != 0 { + t.Fatalf("participant inventory = %#v", participants[1]) + } +} + +func TestParticipantSelectionRejectsInsufficientEligibleNodes(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + request := readyNode("gpu-0", "H200") + profile := testProfile("h200-pair", "H200") + profile.Spec.Participants.Count = 2 + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(request).Build() + r := &AdmissionReconciler{Client: kube} + + if _, err := r.selectParticipants(context.Background(), request, profile); err == nil || + !strings.Contains(err.Error(), "requires 2 eligible unlocked participants, found 1") { + t.Fatalf("selectParticipants error = %v", err) + } +} + +func TestMultiNodeAdmissionReservesEveryParticipant(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + request := readyNode("gpu-0", "H200") + request.Labels[RequestLabel] = "h200-pair" + peer := readyNode("gpu-1", "H200") + profile := testProfile("h200-pair", "H200") + profile.Spec.Participants.Count = 2 + source := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, + Data: map[string]string{"payload.txt": "test payload\n"}, + } + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(request, peer, profile, source).Build() + r := &AdmissionReconciler{ + Client: kube, + Namespace: "signal", + ScriptNamespace: "signal", + Now: func() time.Time { return time.Unix(100, 0) }, + } + + if _, err := r.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKey{Name: request.Name}}, + ); err != nil { + t.Fatalf("Reconcile: %v", err) + } + + var runs signalv1.CertificationRunList + if err := kube.List(context.Background(), &runs); err != nil { + t.Fatal(err) + } + + if len(runs.Items) != 1 || len(runs.Items[0].Spec.Participants) != 2 { + t.Fatalf("runs = %#v", runs.Items) + } + + for index, name := range []string{"gpu-0", "gpu-1"} { + participant := runs.Items[0].Spec.Participants[index] + if participant.Index != int32(index) || participant.NodeName != name { + t.Fatalf("participant %d = %#v", index, participant) + } + + var node corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: name}, &node); err != nil { + t.Fatal(err) + } + + if node.Annotations[ReservationLabel] != runs.Items[0].Name { + t.Fatalf("Node %s reservation = %q", name, node.Annotations[ReservationLabel]) + } + } +} + +func TestMultiNodeReservationRollsBackPartialFailure(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + first := readyNode("gpu-0", "H200") + second := readyNode("gpu-1", "H200") + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(first, second). + WithInterceptorFuncs(interceptor.Funcs{ + Patch: func( + ctx context.Context, + c client.WithWatch, + obj client.Object, + patch client.Patch, + opts ...client.PatchOption, + ) error { + node, ok := obj.(*corev1.Node) + if ok && node.Name == second.Name && node.Annotations[ReservationLabel] == "run-pair" { + return errors.New("reservation conflict") + } + + return c.Patch(ctx, obj, patch, opts...) + }, + }). + Build() + r := &AdmissionReconciler{Client: kube} + participants := []signalv1.Participant{ + participantSnapshot(first, 0), + participantSnapshot(second, 1), + } + + if err := r.reserveParticipants(context.Background(), participants, "run-pair"); err == nil { + t.Fatal("reserveParticipants error = nil") + } + + var recovered corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: first.Name}, &recovered); err != nil { + t.Fatal(err) + } + + if recovered.Annotations[ReservationLabel] != "" { + t.Fatalf("partial reservation was not rolled back: %#v", recovered.Annotations) + } +} + +func TestAdmissionFailureMakesCreatedRunInertAndReleasesReservation(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + failSnapshot bool + failProjection bool + }{ + {name: "snapshot binding", failSnapshot: true}, + {name: "pending projection", failProjection: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + node := readyNode("gpu-0", "example-gpu") + node.Labels[RequestLabel] = "gpu-smoke" + profile := testProfile("gpu-smoke", "example-gpu") + source := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, + Data: map[string]string{"payload.txt": "test payload\n"}, + } + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(node, profile, source). + WithInterceptorFuncs(interceptor.Funcs{ + Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + if snapshot, ok := obj.(*corev1.ConfigMap); ok && + tc.failSnapshot && snapshot.Name != source.Name { + return errors.New("snapshot update failed") + } + + return c.Update(ctx, obj, opts...) + }, + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if projected, ok := obj.(*corev1.Node); ok && + tc.failProjection && projected.Labels[StateLabel] == "pending" { + return errors.New("projection patch failed") + } + + return c.Patch(ctx, obj, patch, opts...) + }, + }). + Build() + r := &AdmissionReconciler{ + Client: kube, + Namespace: "signal", + ScriptNamespace: "signal", + Now: func() time.Time { return time.Unix(100, 0) }, + } + + if _, err := r.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}, + ); err == nil { + t.Fatal("Reconcile error = nil, want failed admission") + } + + var runs signalv1.CertificationRunList + if err := kube.List(context.Background(), &runs); err != nil { + t.Fatalf("list runs: %v", err) + } + + for i := range runs.Items { + if runs.Items[i].DeletionTimestamp.IsZero() { + t.Fatalf("run %s remained active after failed admission", runs.Items[i].Name) + } + } + + var cleaned corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &cleaned); err != nil { + t.Fatalf("get cleaned node: %v", err) + } + + if cleaned.Annotations[ReservationLabel] != "" { + t.Fatalf("reservation remained after inerting failed run: %q", cleaned.Annotations[ReservationLabel]) + } + }) + } +} + +func TestAdmissionRecoversOrphanedReservationAndSnapshot(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + node := readyNode("gpu-0", "example-gpu") + node.Labels[RequestLabel] = "gpu-smoke" + node.Annotations = map[string]string{ReservationLabel: "orphaned-run"} + peer := readyNode("gpu-1", "example-gpu") + peer.Annotations = map[string]string{ReservationLabel: "orphaned-run"} + profile := testProfile("gpu-smoke", "example-gpu") + profile.Spec.Participants.Count = 2 + source := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "scripts", Namespace: "signal"}, + Data: map[string]string{"payload.txt": "test payload\n"}, + } + immutable := true + orphanedSnapshot := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "orphaned-run-scripts", + Namespace: "signal", + Labels: map[string]string{SnapshotRunLabel: "orphaned-run"}, + }, + Immutable: &immutable, + Data: map[string]string{"payload.txt": "orphaned\n"}, + } + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(node, peer, profile, source, orphanedSnapshot). + Build() + r := &AdmissionReconciler{ + Client: kube, + APIReader: kube, + Namespace: "signal", + ScriptNamespace: "signal", + Now: func() time.Time { return time.Unix(100, 0) }, + } + + if _, err := r.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKey{Name: node.Name}}, + ); err != nil { + t.Fatalf("Reconcile: %v", err) + } + + if err := kube.Get( + context.Background(), + client.ObjectKeyFromObject(orphanedSnapshot), + &corev1.ConfigMap{}, + ); !apierrors.IsNotFound(err) { + t.Fatalf("orphaned snapshot get error = %v, want NotFound", err) + } + + var runs signalv1.CertificationRunList + if err := kube.List(context.Background(), &runs); err != nil { + t.Fatalf("list runs: %v", err) + } + + if len(runs.Items) != 1 { + t.Fatalf("runs = %d, want recovered admission", len(runs.Items)) + } + + for _, nodeName := range []string{node.Name, peer.Name} { + var recovered corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: nodeName}, &recovered); err != nil { + t.Fatalf("get recovered node %s: %v", nodeName, err) + } + + if recovered.Annotations[ReservationLabel] != runs.Items[0].Name { + t.Fatalf( + "Node %s reservation = %q, want %q", + nodeName, + recovered.Annotations[ReservationLabel], + runs.Items[0].Name, + ) + } + } +} + +func TestDeletingRunClearsReservationAndRequest(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + run := testRun("deleting-run") + run.Finalizers = []string{RunFinalizer} + node := readyNode(run.Spec.RequestNodeName, "A100") + node.Labels[RequestLabel] = run.Spec.Profile.Name + node.Annotations = map[string]string{ReservationLabel: run.Name} + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(run, node).Build() + reconciler := &RunReconciler{Client: kube} + + if _, err := reconciler.reconcileDeleting(context.Background(), run); err != nil { + t.Fatalf("reconcileDeleting: %v", err) + } + + var cleaned corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &cleaned); err != nil { + t.Fatalf("get cleaned node: %v", err) + } + + if cleaned.Annotations[ReservationLabel] != "" || cleaned.Labels[RequestLabel] != "" { + t.Fatalf("deletion cleanup left trigger metadata: labels=%v annotations=%v", cleaned.Labels, cleaned.Annotations) + } +} + +func TestCancellationWaitsForWorkflowDeletionBeforeReleasingNode(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + run := testRun("canceling-run") + run.Spec.Cancel = true + run.Status.Phase = signalv1.RunPhaseRunning + run.Status.Workflow = signalv1.WorkflowStatus{Name: "canceling-run-a1", Attempt: 1} + node := readyNode(run.Spec.RequestNodeName, "A100") + node.Annotations = map[string]string{ReservationLabel: run.Name} + workflow := workflowObject(run, "Running", "", "") + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationRun{}). + WithObjects(run, node, workflow). + WithInterceptorFuncs(interceptor.Funcs{ + Delete: func( + _ context.Context, + _ client.WithWatch, + _ client.Object, + _ ...client.DeleteOption, + ) error { + return nil + }, + }). + Build() + reconciler := &RunReconciler{Client: kube} + + result, err := reconciler.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}, + ) + if err != nil { + t.Fatalf("Reconcile: %v", err) + } + + if result.RequeueAfter == 0 { + t.Fatal("cancellation did not wait for Workflow deletion") + } + + var gotRun signalv1.CertificationRun + if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &gotRun); err != nil { + t.Fatal(err) + } + + if gotRun.Status.Phase.Terminal() { + t.Fatalf("run became terminal before Workflow deletion: %#v", gotRun.Status) + } + + var gotNode corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &gotNode); err != nil { + t.Fatal(err) + } + + if gotNode.Annotations[ReservationLabel] != run.Name { + t.Fatalf("reservation was released before Workflow deletion: %v", gotNode.Annotations) + } +} + +func TestWorkflowPinsAttemptInputs(t *testing.T) { + t.Parallel() + + run := testRun("run-1") + run.UID = types.UID("run-uid") + run.Spec.Parameters = map[string]string{ + "expected-gpu-count": "8", + "health-mode": "snapshot", + "mpi-payload-mode": "script", + } + run.Spec.Tolerations = []corev1.Toleration{{ + Key: "nvidia.com/gpu", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }} + + workflow, err := buildWorkflow(run, 2) + if err != nil { + t.Fatal(err) + } + + if workflow.GetName() != "run-1-a2" { + t.Fatalf("workflow name = %q", workflow.GetName()) + } + + if got, _, _ := unstructured.NestedInt64(workflow.Object, "spec", "ttlStrategy", "secondsAfterCompletion"); got != 21600 { + t.Fatalf("workflow completion TTL = %d, want 21600", got) + } + + if got, _, _ := unstructured.NestedString(workflow.Object, "spec", "podGC", "deleteDelayDuration"); got != "2h" { + t.Fatalf("workflow pod GC delay = %q, want 2h", got) + } + + if len(workflow.GetOwnerReferences()) != 1 || workflow.GetOwnerReferences()[0].UID != run.UID { + t.Fatalf("owner references = %#v", workflow.GetOwnerReferences()) + } + + parameters, found, err := unstructured.NestedSlice(workflow.Object, "spec", "arguments", "parameters") + if err != nil || !found { + t.Fatalf("workflow parameters: found=%t err=%v", found, err) + } + + values := map[string]string{} + + for _, raw := range parameters { + parameter := raw.(map[string]any) + values[parameter["name"].(string)] = parameter["value"].(string) + } + + if values["script-digest"] != run.Spec.Script.Digest || values["backend-kind"] != "MPIJob" || values["attempt"] != "2" { + t.Fatalf("workflow values = %#v", values) + } + + if values["participant-count"] != "1" || values["request-node-name"] != run.Spec.RequestNodeName { + t.Fatalf("workflow participant metadata = %#v", values) + } + + var participants []signalv1.Participant + if err := json.Unmarshal([]byte(values["participants-json"]), &participants); err != nil { + t.Fatalf("decode participants: %v", err) + } + + if len(participants) != 1 || + participants[0].NodeName != run.Spec.Participants[0].NodeName || + participants[0].NodeUID != run.Spec.Participants[0].NodeUID { + t.Fatalf("workflow participants = %#v", participants) + } + + if values["script-configmap"] != run.Spec.Script.ConfigMapName || values["script-configmap"] == "scripts" { + t.Fatalf("workflow must mount the immutable snapshot, values = %#v", values) + } + + if !strings.Contains(values["tolerations"], `"key":"nvidia.com/gpu"`) { + t.Fatalf("workflow tolerations = %q", values["tolerations"]) + } + + var environment map[string]string + if err := json.Unmarshal([]byte(values["script-environment"]), &environment); err != nil { + t.Fatalf("decode script environment: %v", err) + } + + if environment["EXPECTED_GPU_COUNT"] != "8" || + environment["HEALTH_MODE"] != "snapshot" || + values["mpi-payload-mode"] != "script" { + t.Fatalf("workflow script policy = %#v, parameters = %#v", environment, values) + } + + var tolerations []corev1.Toleration + if err := json.Unmarshal([]byte(values["tolerations"]), &tolerations); err != nil { + t.Fatalf("decode workflow tolerations: %v", err) + } + + if len(tolerations) != 1 { + t.Fatalf("workflow tolerations = %#v, want one deduplicated GPU toleration", tolerations) + } +} + +func TestBuildWorkflowRejectsParameterThatCannotBecomeEnvironmentVariable(t *testing.T) { + t.Parallel() + + run := testRun("run-invalid-parameter") + run.Spec.Parameters = map[string]string{"invalid.parameter": "value"} + + if _, err := buildWorkflow(run, 1); err == nil || + !strings.Contains(err.Error(), "cannot be mapped to a script environment variable") { + t.Fatalf("buildWorkflow error = %v", err) + } +} + +func TestScriptEnvironmentRejectsCollidingParameterNames(t *testing.T) { + t.Parallel() + + _, err := scriptEnvironment(map[string]string{ + "gpu-devices": "0,1", + "gpu_devices": "2,3", + }) + if err == nil || !strings.Contains(err.Error(), "both map to GPU_DEVICES") { + t.Fatalf("scriptEnvironment error = %v", err) + } +} + +func TestScriptEnvironmentRejectsReservedNames(t *testing.T) { + t.Parallel() + + for _, parameter := range []string{"path", "bash-env", "project-signal-run-uid"} { + _, err := scriptEnvironment(map[string]string{parameter: "value"}) + if err == nil || !strings.Contains(err.Error(), "reserved environment variable") { + t.Fatalf("scriptEnvironment(%q) error = %v", parameter, err) + } + } +} + +func TestWorkflowParametersRejectMalformedEvidence(t *testing.T) { + t.Parallel() + + tests := map[string][]any{ + "duplicate": { + map[string]any{"name": "verdict", "value": "Pass"}, + map[string]any{"name": "verdict", "value": "Fail"}, + }, + "unexpected": { + map[string]any{"name": "unbounded", "value": "value"}, + }, + "oversized": { + map[string]any{"name": "measurements", "value": strings.Repeat("x", 16*1024+1)}, + }, + "wrong type": { + map[string]any{"name": "verdict", "value": true}, + }, + } + + for name, parameters := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + workflow := &unstructured.Unstructured{Object: map[string]any{ + "status": map[string]any{ + "outputs": map[string]any{"parameters": parameters}, + }, + }} + + if _, err := workflowParameters(workflow); err == nil { + t.Fatal("workflowParameters succeeded") + } + }) + } +} + +func TestBackendResourceReferencesRequireCompleteMultiNodeCoverage(t *testing.T) { + t.Parallel() + + value := `[ + {"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","namespace":"signal","name":"node-0","uid":"uid-0","participantIndexes":[0]}, + {"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","namespace":"signal","name":"node-1","uid":"uid-1","participantIndexes":[1]} + ]` + + references, err := backendResourceReferences(value, 2) + if err != nil { + t.Fatalf("backendResourceReferences: %v", err) + } + + if len(references) != 2 || references[1].ParticipantIndexes[0] != 1 { + t.Fatalf("references = %#v", references) + } + + for name, malformed := range map[string]string{ + "missing participant": `[{"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","name":"node-0","uid":"uid-0","participantIndexes":[0]}]`, + "out of range": `[{"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","name":"node-0","uid":"uid-0","participantIndexes":[0,2]}]`, + "empty coverage": `[{"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","name":"node-0","uid":"uid-0"}]`, + "missing uid": `[{"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","name":"node-0","participantIndexes":[0,1]}]`, + "unknown field": `[{"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","name":"node-0","uid":"uid-0","participantIndexes":[0,1],"extra":true}]`, + "trailing value": `[{"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","name":"node-0","uid":"uid-0","participantIndexes":[0,1]}] {}`, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + if _, err := backendResourceReferences(malformed, 2); err == nil { + t.Fatalf("backendResourceReferences(%s) error = nil", malformed) + } + }) + } +} + +func TestMultiNodePassWithoutBackendIdentityBecomesInconclusive(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + run := testRun("run-multi") + run.Spec.Participants = []signalv1.Participant{ + participantSnapshot(readyNode("gpu-0", "H200"), 0), + participantSnapshot(readyNode("gpu-1", "H200"), 1), + } + run.Status = signalv1.CertificationRunStatus{ + Phase: signalv1.RunPhaseRunning, + Workflow: signalv1.WorkflowStatus{Name: "run-multi-a1", Attempt: 1}, + StartedAt: ptrTime(metav1.NewTime(time.Unix(100, 0))), + } + snapshot := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Script.ConfigMapName, Namespace: run.Namespace}, + Data: map[string]string{run.Spec.Script.Key: "test payload\n"}, + } + workflow := workflowObject(run, "Succeeded", "Pass", "Healthy") + store := &memoryEvidenceStore{} + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationRun{}). + WithObjects(run, snapshot, workflow). + Build() + r := &RunReconciler{Client: kube, Evidence: store, Now: func() time.Time { return time.Unix(200, 0) }} + + if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}); err != nil { + t.Fatalf("Reconcile: %v", err) + } + + var got signalv1.CertificationRun + if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { + t.Fatal(err) + } + + if got.Status.Phase != signalv1.RunPhaseInconclusive || + got.Status.Verdict != "Inconclusive" || + !strings.Contains(got.Status.Reason, "did not identify backend resources") { + t.Fatalf("status = %#v", got.Status) + } +} + +func TestMultiNodePassRetainsCoveredBackendIdentities(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + run := testRun("run-multi-pass") + run.Spec.Participants = []signalv1.Participant{ + participantSnapshot(readyNode("gpu-0", "H200"), 0), + participantSnapshot(readyNode("gpu-1", "H200"), 1), + } + run.Status = signalv1.CertificationRunStatus{ + Phase: signalv1.RunPhaseRunning, + Workflow: signalv1.WorkflowStatus{Name: "run-multi-pass-a1", Attempt: 1}, + StartedAt: ptrTime(metav1.NewTime(time.Unix(100, 0))), + } + snapshot := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Script.ConfigMapName, Namespace: run.Namespace}, + Data: map[string]string{run.Spec.Script.Key: "test payload\n"}, + } + workflow := workflowObject(run, "Succeeded", "Pass", "Healthy") + + outputs, _, err := unstructured.NestedSlice(workflow.Object, "status", "outputs", "parameters") + if err != nil { + t.Fatal(err) + } + + outputs = append(outputs, map[string]any{ + "name": "backend-resources", + "value": `[ + {"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","namespace":"signal","name":"node-0","uid":"uid-0","participantIndexes":[0]}, + {"apiVersion":"kubeflow.org/v2beta1","kind":"MPIJob","namespace":"signal","name":"node-1","uid":"uid-1","participantIndexes":[1]} + ]`, + }) + if err := unstructured.SetNestedSlice(workflow.Object, outputs, "status", "outputs", "parameters"); err != nil { + t.Fatal(err) + } + + store := &memoryEvidenceStore{} + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationRun{}). + WithObjects(run, snapshot, workflow). + Build() + r := &RunReconciler{Client: kube, Evidence: store, Now: func() time.Time { return time.Unix(200, 0) }} + + if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}); err != nil { + t.Fatalf("Reconcile: %v", err) + } + + var got signalv1.CertificationRun + if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { + t.Fatal(err) + } + + if got.Status.Phase != signalv1.RunPhaseSucceeded || + len(got.Status.Workflow.BackendResources) != 2 || + got.Status.Workflow.BackendResources[1].ParticipantIndexes[0] != 1 { + t.Fatalf("status = %#v", got.Status) + } +} + +func TestStartAttemptRecoversExistingOwnedWorkflow(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + run := testRun("run-recovered") + + workflow, err := buildWorkflow(run, 1) + if err != nil { + t.Fatal(err) + } + + workflow.SetUID("workflow-uid") + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationRun{}). + WithObjects(run, workflow). + Build() + r := &RunReconciler{Client: kube, Evidence: &memoryEvidenceStore{}} + + if _, err := r.startAttempt(context.Background(), run, 1); err != nil { + t.Fatalf("startAttempt: %v", err) + } + + var got signalv1.CertificationRun + if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { + t.Fatalf("get run: %v", err) + } + + if got.Status.Workflow.Name != workflow.GetName() || + got.Status.Workflow.UID != string(workflow.GetUID()) || + got.Status.Workflow.Attempt != 1 { + t.Fatalf("workflow status = %#v", got.Status.Workflow) + } +} + +func TestWorkflowParameterFallsBackToEntrypointNodeOutputs(t *testing.T) { + t.Parallel() + + workflow := &unstructured.Unstructured{Object: map[string]any{ + "status": map[string]any{ + "nodes": map[string]any{ + "root": map[string]any{ + "outputs": map[string]any{ + "parameters": []any{map[string]any{"name": "verdict", "value": "Pass"}}, + }, + }, + }, + }, + }} + + if got := workflowParameter(workflow, "verdict"); got != "Pass" { + t.Fatalf("verdict = %q", got) + } +} + +func TestValidateStagePlanRejectsInvalidAndDuplicateStages(t *testing.T) { + t.Parallel() + + if err := validateStagePlan([]signalv1.CertificationStage{{DisplayName: "Missing ID"}}); err == nil { + t.Fatal("expected empty stage ID to fail validation") + } + + if err := validateStagePlan([]signalv1.CertificationStage{ + {ID: "workload", DisplayName: "First"}, + {ID: "workload", DisplayName: "Duplicate"}, + }); err == nil { + t.Fatal("expected duplicate stage IDs to fail validation") + } +} + +func TestTerminalVerdictPersistsEvidenceAndCleanupReleasesNode(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + run := testRun("run-pass") + run.Status = signalv1.CertificationRunStatus{ + Phase: signalv1.RunPhaseRunning, + Workflow: signalv1.WorkflowStatus{Name: "run-pass-a1", Attempt: 1}, + StartedAt: ptrTime(metav1.NewTime(time.Unix(100, 0))), + } + node := readyNode("gpu-0", "A100") + node.Labels[RequestLabel] = run.Spec.Profile.Name + node.Annotations = map[string]string{ReservationLabel: run.Name} + snapshot := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Script.ConfigMapName, Namespace: run.Namespace}, + Data: map[string]string{run.Spec.Script.Key: "test payload\n"}, + } + workflow := workflowObject(run, "Succeeded", "Pass", "Healthy") + store := &memoryEvidenceStore{} + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationRun{}). + WithObjects(run, node, snapshot, workflow). + Build() + r := &RunReconciler{Client: kube, Evidence: store, Now: func() time.Time { return time.Unix(200, 0) }} + + if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}); err != nil { + t.Fatalf("terminal Reconcile: %v", err) + } + + if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}); err != nil { + t.Fatalf("cleanup Reconcile: %v", err) + } + + var got signalv1.CertificationRun + if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { + t.Fatalf("get run: %v", err) + } + + if got.Status.Phase != signalv1.RunPhaseSucceeded || + got.Status.Verdict != "Pass" || + got.Status.Workflow.Phase != "Succeeded" { + t.Fatalf("status = %#v", got.Status) + } + + if got.Status.Workflow.Outputs["measurements"] != "checked=14" { + t.Fatalf("retained workflow outputs = %#v", got.Status.Workflow.Outputs) + } + + if got.Status.Evidence.URI == "" || len(store.data) == 0 { + t.Fatal("evidence was not persisted") + } + + var evidence struct { + Workflow signalv1.WorkflowStatus `json:"workflow"` + } + if err := json.Unmarshal(store.data, &evidence); err != nil { + t.Fatalf("decode evidence: %v", err) + } + + if evidence.Workflow.Phase != "Succeeded" || + evidence.Workflow.Outputs["measurements"] != "checked=14" { + t.Fatalf("evidence workflow = %#v", evidence.Workflow) + } + + if !got.Status.Cleanup.ReservationsReleased || !got.Status.Cleanup.SnapshotDeleted { + t.Fatalf("cleanup status = %#v", got.Status.Cleanup) + } + + if containsString(got.Finalizers, RunFinalizer) { + t.Fatal("cleanup finalizer remains") + } + + var released corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &released); err != nil { + t.Fatalf("get released node: %v", err) + } + + if released.Annotations[ReservationLabel] != "" || released.Labels[RequestLabel] != "" { + t.Fatalf("node was not released: labels=%v annotations=%v", released.Labels, released.Annotations) + } + + if released.Labels[StateLabel] != "passed" || + released.Labels[EligibilityLabel] != "production" || + released.Annotations[LastSuccessfulRunAnnotation] != run.Name { + t.Fatalf("terminal projection was not retained: labels=%v annotations=%v", released.Labels, released.Annotations) + } +} + +func TestTerminalFinalizationRetriesWithIdenticalEvidence(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + run := testRun("run-retry-finalization") + run.Status = signalv1.CertificationRunStatus{ + Phase: signalv1.RunPhaseRunning, + Workflow: signalv1.WorkflowStatus{Name: "run-retry-finalization-a1", Attempt: 1}, + StartedAt: ptrTime(metav1.NewTime(time.Unix(100, 0))), + } + node := readyNode("gpu-0", "A100") + node.Annotations = map[string]string{ReservationLabel: run.Name} + snapshot := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Script.ConfigMapName, Namespace: run.Namespace}, + Data: map[string]string{run.Spec.Script.Key: "test payload\n"}, + } + workflow := workflowObject(run, "Succeeded", "Pass", "Healthy") + statusPatches := 0 + base := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationRun{}). + WithObjects(run, node, snapshot, workflow). + Build() + kube := interceptor.NewClient(base, interceptor.Funcs{ + SubResourcePatch: func( + ctx context.Context, + c client.Client, + subResource string, + obj client.Object, + patch client.Patch, + opts ...client.SubResourcePatchOption, + ) error { + if subResource == "status" { + statusPatches++ + if statusPatches == 2 { + return errors.New("status patch interrupted after evidence persistence") + } + } + + return c.SubResource(subResource).Patch(ctx, obj, patch, opts...) + }, + }) + store := &immutableMemoryEvidenceStore{} + r := &RunReconciler{ + Client: kube, + Evidence: store, + Now: func() time.Time { return time.Unix(200+int64(statusPatches), 0) }, + } + key := reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)} + + if _, err := r.Reconcile(context.Background(), key); err == nil { + t.Fatal("first Reconcile error = nil, want interrupted status patch") + } + + var interrupted signalv1.CertificationRun + if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &interrupted); err != nil { + t.Fatalf("get interrupted run: %v", err) + } + + beforeCancel := interrupted.DeepCopy() + interrupted.Spec.Cancel = true + + if err := kube.Patch(context.Background(), &interrupted, client.MergeFrom(beforeCancel)); err != nil { + t.Fatalf("request cancellation: %v", err) + } + + if _, err := r.Reconcile(context.Background(), key); err != nil { + t.Fatalf("retry Reconcile: %v", err) + } + + var got signalv1.CertificationRun + if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { + t.Fatalf("get run: %v", err) + } + + if got.Status.Phase != signalv1.RunPhaseSucceeded { + t.Fatalf("phase = %q, want Succeeded", got.Status.Phase) + } + + if store.puts != 2 { + t.Fatalf("evidence puts = %d, want 2 identical attempts", store.puts) + } +} + +func TestWorkflowReplacementIsRejected(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + run := testRun("run-workflow-replaced") + run.Status = signalv1.CertificationRunStatus{ + Phase: signalv1.RunPhaseRunning, + Workflow: signalv1.WorkflowStatus{ + Name: "run-workflow-replaced-a1", + UID: "original-workflow-uid", + Attempt: 1, + }, + } + replacement := workflowObject(run, "Succeeded", "Pass", "forged") + replacement.SetUID("replacement-workflow-uid") + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationRun{}). + WithObjects(run, replacement). + Build() + r := &RunReconciler{Client: kube, Evidence: &memoryEvidenceStore{}} + + if _, err := r.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}, + ); err == nil || !strings.Contains(err.Error(), "refusing concurrent retry") { + t.Fatalf("Reconcile error = %v, want identity rejection", err) + } + + var got signalv1.CertificationRun + if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { + t.Fatalf("get run: %v", err) + } + + if got.Status.Workflow.Attempt != 1 || + got.Status.Workflow.Name != "run-workflow-replaced-a1" { + t.Fatalf("replacement workflow triggered a retry: %#v", got.Status.Workflow) + } + + var workflows unstructured.UnstructuredList + workflows.SetGroupVersionKind(workflowGVK.GroupVersion().WithKind("WorkflowList")) + + if err := kube.List(context.Background(), &workflows, client.InNamespace(run.Namespace)); err != nil { + t.Fatalf("list Workflows: %v", err) + } + + if len(workflows.Items) != 1 { + t.Fatalf("workflows = %d, want only mismatched original name", len(workflows.Items)) + } +} + +func TestCancellationRefusesToDeleteReplacementWorkflow(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + run := testRun("run-cancel-replacement") + run.Spec.Cancel = true + run.Status = signalv1.CertificationRunStatus{ + Phase: signalv1.RunPhaseRunning, + Workflow: signalv1.WorkflowStatus{ + Name: "run-cancel-replacement-a1", + UID: "original-workflow-uid", + Attempt: 1, + }, + } + replacement := workflowObject(run, "Running", "", "") + replacement.SetUID("replacement-workflow-uid") + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationRun{}). + WithObjects(run, replacement). + Build() + r := &RunReconciler{Client: kube} + + if _, err := r.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}, + ); err == nil || !strings.Contains(err.Error(), "refusing to delete Workflow") { + t.Fatalf("Reconcile error = %v, want identity rejection", err) + } + + var remaining unstructured.Unstructured + remaining.SetGroupVersionKind(workflowGVK) + + if err := kube.Get(context.Background(), client.ObjectKeyFromObject(replacement), &remaining); err != nil { + t.Fatalf("replacement Workflow was deleted: %v", err) + } +} + +func TestWorkflowFailureRetriesWithoutSwitchingPinnedInputs(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + run := testRun("run-retry") + run.Spec.MaxAttempts = 2 + run.Status = signalv1.CertificationRunStatus{ + Phase: signalv1.RunPhaseRunning, + Workflow: signalv1.WorkflowStatus{Name: "run-retry-a1", Attempt: 1}, + } + workflow := workflowObject(run, "Failed", "", "") + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationRun{}). + WithObjects(run, workflow). + Build() + r := &RunReconciler{Client: kube, Evidence: &memoryEvidenceStore{}} + + if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}); err != nil { + t.Fatalf("Reconcile: %v", err) + } + + var got signalv1.CertificationRun + if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { + t.Fatalf("get run: %v", err) + } + + if got.Status.Workflow.Attempt != 2 || got.Status.Workflow.Name != "run-retry-a2" { + t.Fatalf("workflow status = %#v", got.Status.Workflow) + } + + var retry unstructured.Unstructured + retry.SetGroupVersionKind(workflowGVK) + + if err := kube.Get(context.Background(), client.ObjectKey{Namespace: run.Namespace, Name: "run-retry-a2"}, &retry); err != nil { + t.Fatalf("get retry workflow: %v", err) + } + + if value := workflowArgument(&retry, "script-digest"); value != run.Spec.Script.Digest { + t.Fatalf("retry script digest = %q", value) + } + + if value := workflowArgument(&retry, "backend-kind"); value != run.Spec.Backend.Kind { + t.Fatalf("retry backend = %q", value) + } +} + +func TestCanceledRunCleansUpDespiteCancelFlag(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + run := testRun("run-canceled") + run.Spec.Cancel = true + run.Status.Phase = signalv1.RunPhaseCancelled + node := readyNode("gpu-0", "A100") + node.Labels[RequestLabel] = run.Spec.Profile.Name + node.Annotations = map[string]string{ReservationLabel: run.Name} + snapshot := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: run.Spec.Script.ConfigMapName, Namespace: run.Namespace}, + Data: map[string]string{run.Spec.Script.Key: "test payload\n"}, + } + kube := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationRun{}). + WithObjects(run, node, snapshot). + Build() + r := &RunReconciler{Client: kube, Evidence: &memoryEvidenceStore{}} + + if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}); err != nil { + t.Fatalf("Reconcile: %v", err) + } + + var got signalv1.CertificationRun + if err := kube.Get(context.Background(), client.ObjectKeyFromObject(run), &got); err != nil { + t.Fatalf("get run: %v", err) + } + + if containsString(got.Finalizers, RunFinalizer) || !got.Status.Cleanup.ReservationsReleased { + t.Fatalf("run was not cleaned up: %#v", got) + } +} + +type memoryEvidenceStore struct { + data []byte +} + +func (s *memoryEvidenceStore) Put(_ context.Context, path string, data []byte) (string, string, error) { + s.data = append([]byte(nil), data...) + + return "memory://" + path, digestString(string(data)), nil +} + +type immutableMemoryEvidenceStore struct { + data []byte + puts int +} + +func (s *immutableMemoryEvidenceStore) Put(_ context.Context, path string, data []byte) (string, string, error) { + s.puts++ + if s.data != nil && string(s.data) != string(data) { + return "", "", errors.New("evidence changed across retry") + } + + s.data = append([]byte(nil), data...) + + return "memory://" + path, digestString(string(data)), nil +} + +func testScheme(t *testing.T) *runtime.Scheme { + t.Helper() + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + if err := signalv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + scheme.AddKnownTypeWithName(workflowGVK, &unstructured.Unstructured{}) + scheme.AddKnownTypeWithName(workflowGVK.GroupVersion().WithKind("WorkflowList"), &unstructured.UnstructuredList{}) + + return scheme +} + +func readyNode(name, sku string) *corev1.Node { + return &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(name + "-uid"), + Labels: map[string]string{ + "node.kubernetes.io/instance-type": sku, + }, + }, + Status: corev1.NodeStatus{ + Capacity: corev1.ResourceList{corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("8")}, + Allocatable: corev1.ResourceList{corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("8")}, + Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}, + }, + } +} + +func testProfile(name, sku string) *signalv1.CertificationProfile { + return &signalv1.CertificationProfile{ + ObjectMeta: metav1.ObjectMeta{Name: name, UID: types.UID(name + "-uid"), Generation: 3}, + Spec: signalv1.CertificationProfileSpec{ + Participants: signalv1.ParticipantPolicy{ + Count: 1, + Selector: metav1.LabelSelector{MatchLabels: map[string]string{ + "node.kubernetes.io/instance-type": sku, + }}, + RequiredResources: corev1.ResourceList{corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("1")}, + }, + Script: signalv1.ScriptReference{Name: "scripts", Key: "payload.txt"}, + Workflow: signalv1.WorkflowReference{Name: "gpu-cert"}, + Backend: signalv1.ExecutionBackend{APIVersion: "kubeflow.org/v2beta1", Kind: "MPIJob"}, + RunnerImage: "runner@sha256:" + strings.Repeat("a", 64), + Purpose: signalv1.PurposeQuickCertification, + StagePlan: []signalv1.CertificationStage{{ID: "workload", DisplayName: "Workload", Category: "compute", Required: true}}, + EvidencePath: "signal/" + sku, + Deadline: metav1.Duration{Duration: 10 * time.Minute}, + MaxAttempts: 2, + }, + } +} + +func testRun(name string) *signalv1.CertificationRun { + return &signalv1.CertificationRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "signal", + UID: types.UID(name + "-uid"), + Finalizers: []string{RunFinalizer}, + }, + Spec: signalv1.CertificationRunSpec{ + Profile: signalv1.PinnedProfile{Name: "gpu-profile", UID: "profile-uid", Generation: 3}, + Participants: []signalv1.Participant{{NodeName: "gpu-0", NodeUID: "gpu-0-uid"}}, + Script: signalv1.ScriptSnapshot{ConfigMapName: name + "-scripts", Key: "payload.txt", Digest: digestString("test payload\n")}, + Workflow: signalv1.WorkflowReference{Name: "gpu-cert"}, + Backend: signalv1.ExecutionBackend{APIVersion: "kubeflow.org/v2beta1", Kind: "MPIJob"}, + RunnerImage: "runner@sha256:" + strings.Repeat("a", 64), + EvidencePath: "signal/" + name + ".json", + Deadline: metav1.Duration{Duration: 10 * time.Minute}, + MaxAttempts: 2, + RequestNodeName: "gpu-0", + }, + } +} + +func workflowObject(run *signalv1.CertificationRun, phase, verdict, reason string) *unstructured.Unstructured { + const workflowUID = "workflow-uid" + + run.Status.Workflow.UID = workflowUID + workflow := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": workflowGVK.GroupVersion().String(), + "kind": workflowGVK.Kind, + "metadata": map[string]any{ + "name": run.Status.Workflow.Name, + "namespace": run.Namespace, + "uid": workflowUID, + "ownerReferences": []any{ + map[string]any{ + "apiVersion": signalv1.GroupVersion.String(), + "kind": "CertificationRun", + "name": run.Name, + "uid": string(run.UID), + "controller": true, + "blockOwnerDeletion": true, + }, + }, + }, + "status": map[string]any{ + "phase": phase, + "message": "test workflow message", + "outputs": map[string]any{"parameters": []any{ + map[string]any{"name": "verdict", "value": verdict}, + map[string]any{"name": "reason", "value": reason}, + map[string]any{"name": "measurements", "value": "checked=14"}, + }}, + }, + }} + + return workflow +} + +func workflowArgument(workflow *unstructured.Unstructured, name string) string { + parameters, _, _ := unstructured.NestedSlice(workflow.Object, "spec", "arguments", "parameters") + for _, raw := range parameters { + parameter := raw.(map[string]any) + if parameter["name"] == name { + return fmt.Sprint(parameter["value"]) + } + } + + return "" +} + +func ptrTime(value metav1.Time) *metav1.Time { + return &value +} diff --git a/internal/signal/evidence.go b/internal/signal/evidence.go new file mode 100644 index 0000000000..dec8e50152 --- /dev/null +++ b/internal/signal/evidence.go @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signal + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "net/url" + "strings" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type EvidenceStore interface { + Put(context.Context, string, []byte) (string, string, error) +} + +type KubernetesEvidenceStore struct { + kube client.Client + namespace string +} + +func NewKubernetesEvidenceStore(kube client.Client, namespace string) (*KubernetesEvidenceStore, error) { + if kube == nil { + return nil, fmt.Errorf("kubernetes client is required") + } + + if namespace == "" { + return nil, fmt.Errorf("evidence namespace is required") + } + + return &KubernetesEvidenceStore{kube: kube, namespace: namespace}, nil +} + +func (s *KubernetesEvidenceStore) Put(ctx context.Context, path string, data []byte) (string, string, error) { + name := strings.TrimSuffix(url.PathEscape(strings.TrimSuffix(path[strings.LastIndex(path, "/")+1:], ".json")), ".json") + if name == "" { + return "", "", fmt.Errorf("evidence path is required") + } + + sum := sha256.Sum256(data) + digest := "sha256:" + hex.EncodeToString(sum[:]) + + compressed, err := compressEvidence(data) + if err != nil { + return "", "", err + } + + immutable := true + + evidence := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name + "-evidence", + Namespace: s.namespace, + Labels: map[string]string{ + SnapshotRunLabel: name, + }, + Annotations: map[string]string{ + "signal.unbounded-cloud.io/evidence-digest": digest, + }, + }, + Immutable: &immutable, + BinaryData: map[string][]byte{ + "evidence.json.gz": compressed, + }, + } + if err := s.kube.Create(ctx, evidence); err != nil { + if !apierrors.IsAlreadyExists(err) { + return "", "", fmt.Errorf("create evidence ConfigMap: %w", err) + } + + var existing corev1.ConfigMap + if getErr := s.kube.Get(ctx, client.ObjectKeyFromObject(evidence), &existing); getErr != nil { + return "", "", fmt.Errorf("get existing evidence ConfigMap: %w", getErr) + } + + if existing.Annotations["signal.unbounded-cloud.io/evidence-digest"] != digest || + !bytes.Equal(existing.BinaryData["evidence.json.gz"], compressed) { + return "", "", fmt.Errorf("evidence ConfigMap %s already exists with different content", evidence.Name) + } + } + + return fmt.Sprintf("kubernetes://%s/configmaps/%s/evidence.json.gz", s.namespace, evidence.Name), digest, nil +} + +func compressEvidence(data []byte) ([]byte, error) { + var compressed bytes.Buffer + + writer := gzip.NewWriter(&compressed) + if _, err := writer.Write(data); err != nil { + return nil, fmt.Errorf("compress evidence: %w", err) + } + + if err := writer.Close(); err != nil { + return nil, fmt.Errorf("finish evidence compression: %w", err) + } + + return compressed.Bytes(), nil +} diff --git a/internal/signal/evidence_test.go b/internal/signal/evidence_test.go new file mode 100644 index 0000000000..b24f269dea --- /dev/null +++ b/internal/signal/evidence_test.go @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signal + +import ( + "bytes" + "compress/gzip" + "context" + "io" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestKubernetesEvidenceStoreReturnsQueryableRunURIAndDigest(t *testing.T) { + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + kube := fake.NewClientBuilder().WithScheme(scheme).Build() + + store, err := NewKubernetesEvidenceStore(kube, "project-signal") + if err != nil { + t.Fatal(err) + } + + data := []byte(`{"verdict":"Pass"}`) + + uri, digest, err := store.Put(context.Background(), "project-signal/example/run-a.json", data) + if err != nil { + t.Fatal(err) + } + + if uri != "kubernetes://project-signal/configmaps/run-a-evidence/evidence.json.gz" { + t.Fatalf("unexpected URI %q", uri) + } + + if !strings.HasPrefix(digest, "sha256:") || len(digest) != len("sha256:")+64 { + t.Fatalf("unexpected digest %q", digest) + } + + var evidence corev1.ConfigMap + if err := kube.Get( + context.Background(), + client.ObjectKey{Namespace: "project-signal", Name: "run-a-evidence"}, + &evidence, + ); err != nil { + t.Fatal(err) + } + + if evidence.Immutable == nil || !*evidence.Immutable { + t.Fatal("evidence ConfigMap is mutable") + } + + reader, err := gzip.NewReader(bytes.NewReader(evidence.BinaryData["evidence.json.gz"])) + if err != nil { + t.Fatal(err) + } + + got, err := io.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + + if err := reader.Close(); err != nil { + t.Fatal(err) + } + + if string(got) != string(data) { + t.Fatalf("persisted evidence = %q", got) + } +} diff --git a/internal/signal/policy.go b/internal/signal/policy.go new file mode 100644 index 0000000000..bdcd9d39a4 --- /dev/null +++ b/internal/signal/policy.go @@ -0,0 +1,718 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signal + +import ( + "context" + "fmt" + "slices" + "sort" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + + machinav1 "github.com/Azure/unbounded/api/machina/v1alpha3" + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" +) + +const defaultPolicyEvaluationInterval = time.Minute + +type RecurringReconciler struct { + client.Client + Reader client.Reader + Namespace string + Admission *AdmissionReconciler + Now func() time.Time +} + +func (r *RecurringReconciler) SetupWithManager(mgr ctrl.Manager) error { + if r.Reader == nil { + r.Reader = mgr.GetAPIReader() + } + + return ctrl.NewControllerManagedBy(mgr). + For(&signalv1.CertificationProfile{}). + WithOptions(controller.Options{MaxConcurrentReconciles: 1}). + Complete(r) +} + +func (r *RecurringReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var profile signalv1.CertificationProfile + if err := r.Get(ctx, req.NamespacedName, &profile); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + policy := profile.Spec.Recurring + if policy == nil || !policy.Enabled { + return ctrl.Result{}, nil + } + + now := r.now() + + interval := policy.EvaluationInterval.Duration + if interval <= 0 { + interval = defaultPolicyEvaluationInterval + } + + before := profile.DeepCopy() + profile.Status.ObservedGeneration = profile.Generation + + if err := validateRecurringPolicy(policy); err != nil { + setProfileCondition(&profile, now, metav1.ConditionFalse, "InvalidPolicy", err.Error()) + + return ctrl.Result{RequeueAfter: interval}, r.Status().Patch(ctx, &profile, client.MergeFrom(before)) + } + + if policyExpired(policy, now) { + for index := range profile.Status.Nodes { + profile.Status.Nodes[index].Suspended = true + profile.Status.Nodes[index].Reason = "PolicyExpired" + profile.Status.Nodes[index].NextDueAt = nil + } + + setProfileCondition(&profile, now, metav1.ConditionFalse, "Expired", + fmt.Sprintf("recurring admission expired at %s", policy.ExpiresAt.UTC().Format(time.RFC3339))) + + if err := r.Status().Patch(ctx, &profile, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{}, r.projectStatuses(ctx, &profile) + } + + if r.Admission == nil { + return ctrl.Result{}, fmt.Errorf("admission reconciler is required") + } + + r.Admission.admissionMu.Lock() + defer r.Admission.admissionMu.Unlock() + + var runs signalv1.CertificationRunList + if err := r.Reader.List(ctx, &runs, client.InNamespace(r.Namespace)); err != nil { + return ctrl.Result{}, fmt.Errorf("list CertificationRuns: %w", err) + } + + activeByProfile, activeByDomain := activeBudgets(runs.Items, &profile) + if policy.FailureDomainLabel != "" { + for index := range runs.Items { + run := &runs.Items[index] + if run.Spec.Profile.Name != profile.Name || run.Status.Phase.Terminal() { + continue + } + + for _, participant := range run.Spec.Participants { + var node corev1.Node + if err := r.Reader.Get(ctx, client.ObjectKey{Name: participant.NodeName}, &node); err != nil { + return ctrl.Result{}, fmt.Errorf( + "get active participant %s for failure-domain accounting: %w", + participant.NodeName, + err, + ) + } + + activeByDomain[node.Labels[policy.FailureDomainLabel]]++ + } + } + } + + statuses := make([]signalv1.RecurringNodeStatus, 0, len(policy.NodeNames)) + + for _, nodeName := range policy.NodeNames { + old := recurringNodeStatus(profile.Status.Nodes, nodeName) + + status, admitted, err := r.evaluateNode(ctx, &profile, nodeName, old, runs.Items, activeByProfile, activeByDomain, now) + if err != nil { + status.Reason = "EvaluationError: " + err.Error() + statuses = append(statuses, status) + + continue + } + + statuses = append(statuses, status) + + if admitted { + activeByProfile++ + + if policy.FailureDomainLabel != "" { + if err := r.accountAdmittedFailureDomains(ctx, nodeName, policy.FailureDomainLabel, activeByDomain); err != nil { + return ctrl.Result{}, err + } + } + } + } + + profile.Status.Nodes = statuses + setProfileCondition(&profile, now, metav1.ConditionTrue, "Evaluated", + fmt.Sprintf("evaluated %d explicitly opted-in nodes", len(statuses))) + + if err := r.Status().Patch(ctx, &profile, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, err + } + + if err := r.projectStatuses(ctx, &profile); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{RequeueAfter: interval}, nil +} + +func (r *RecurringReconciler) accountAdmittedFailureDomains( + ctx context.Context, + requestNodeName string, + failureDomainLabel string, + activeByDomain map[string]int32, +) error { + var requestNode corev1.Node + if err := r.Reader.Get(ctx, client.ObjectKey{Name: requestNodeName}, &requestNode); err != nil { + return fmt.Errorf("get admitted request Node %s for failure-domain accounting: %w", requestNodeName, err) + } + + runName := requestNode.Annotations[ReservationLabel] + if runName == "" { + return fmt.Errorf("admitted request Node %s has no reservation", requestNodeName) + } + + var run signalv1.CertificationRun + if err := r.Reader.Get(ctx, client.ObjectKey{Namespace: r.Namespace, Name: runName}, &run); err != nil { + return fmt.Errorf("get admitted run %s for failure-domain accounting: %w", runName, err) + } + + for _, participant := range run.Spec.Participants { + var node corev1.Node + if err := r.Reader.Get(ctx, client.ObjectKey{Name: participant.NodeName}, &node); err != nil { + return fmt.Errorf( + "get admitted participant %s for failure-domain accounting: %w", + participant.NodeName, + err, + ) + } + + if participant.NodeUID != "" && node.UID != participant.NodeUID { + return fmt.Errorf("admitted participant %s identity changed", participant.NodeName) + } + + activeByDomain[node.Labels[failureDomainLabel]]++ + } + + return nil +} + +func (r *RecurringReconciler) projectStatuses(ctx context.Context, profile *signalv1.CertificationProfile) error { + for _, status := range profile.Status.Nodes { + if err := projectRecurring(ctx, r.Client, profile, status); err != nil { + return err + } + } + + return nil +} + +func (r *RecurringReconciler) evaluateNode( + ctx context.Context, + profile *signalv1.CertificationProfile, + nodeName string, + old signalv1.RecurringNodeStatus, + runs []signalv1.CertificationRun, + activeByProfile int32, + activeByDomain map[string]int32, + now time.Time, +) (signalv1.RecurringNodeStatus, bool, error) { + status := old + status.NodeName = nodeName + evaluated := metav1.NewTime(now) + status.LastEvaluatedAt = &evaluated + status.Suspended = false + + var node corev1.Node + if err := r.Reader.Get(ctx, client.ObjectKey{Name: nodeName}, &node); err != nil { + status.IdleSince = nil + + return status, false, fmt.Errorf("get Node: %w", err) + } + + selector, err := metav1.LabelSelectorAsSelector(&profile.Spec.Participants.Selector) + if err != nil { + return status, false, fmt.Errorf("participant selector: %w", err) + } + + if !selector.Matches(labels.Set(node.Labels)) || !nodeEligible(&node, profile.Spec.Participants.RequiredResources) { + status.IdleSince = nil + status.Reason = "NodeIneligible" + + return status, false, nil + } + + busy, reason, err := r.nodeBusy(ctx, &node) + if err != nil { + status.IdleSince = nil + + return status, false, err + } + + if busy { + status.IdleSince = nil + status.Reason = reason + + return status, false, nil + } + + if status.IdleSince == nil { + idleSince := metav1.NewTime(now) + status.IdleSince = &idleSince + } + + nodeRuns := runsForNodeProfile(runs, nodeName, profile.Name) + lastSuccess, consecutiveFailures, latestFailure := runHistory(nodeRuns) + status.ConsecutiveFailures = consecutiveFailures + + if lastSuccess != nil { + status.LastSuccessfulRun = lastSuccess.Name + status.LastSuccessfulAt = lastSuccess.Status.FinishedAt + } + + due := dueAt(profile.Spec.Recurring, status.IdleSince, lastSuccess) + status.NextDueAt = ptrMetaTime(due) + + if suspended, reason := failureGate(profile.Spec.Recurring, consecutiveFailures, latestFailure, now); reason != "" { + status.Suspended = suspended + status.Reason = reason + + return status, false, nil + } + + if now.Before(due) { + status.Reason = "NotDue" + + return status, false, nil + } + + if activeRunForNode(runs, nodeName) || node.Annotations[ReservationLabel] != "" { + status.Reason = "RunActive" + + return status, false, nil + } + + if excludedRunActive(runs, nodeName, profile.Spec.Recurring.ExclusiveProfiles) { + status.Reason = "ExclusiveProfileActive" + + return status, false, nil + } + + if activeByProfile >= profile.Spec.Recurring.MaxConcurrent { + status.Reason = "ConcurrencyBudget" + + return status, false, nil + } + + domain := node.Labels[profile.Spec.Recurring.FailureDomainLabel] + if profile.Spec.Recurring.FailureDomainLabel != "" && + profile.Spec.Recurring.MaxConcurrentPerFailureDomain > 0 && + activeByDomain[domain] >= profile.Spec.Recurring.MaxConcurrentPerFailureDomain { + status.Reason = "FailureDomainBudget" + + return status, false, nil + } + + if profile.Spec.Recurring.FailureDomainLabel != "" && + profile.Spec.Recurring.MaxConcurrentPerFailureDomain > 0 { + participants, err := r.Admission.selectParticipants(ctx, &node, profile) + if err != nil { + return status, false, fmt.Errorf("select recurring participants: %w", err) + } + + fits, err := r.participantsFitFailureDomainBudget( + ctx, + participants, + profile.Spec.Recurring.FailureDomainLabel, + profile.Spec.Recurring.MaxConcurrentPerFailureDomain, + activeByDomain, + ) + if err != nil { + return status, false, err + } + + if !fits { + status.Reason = "FailureDomainBudget" + + return status, false, nil + } + } + + if err := r.Admission.admitNodeWithTriggerLocked(ctx, &node, profile.Name, "recurring"); err != nil { + if apierrors.IsAlreadyExists(err) || apierrors.IsConflict(err) { + status.Reason = "AdmissionRaced" + + return status, false, nil + } + + return status, false, fmt.Errorf("automatic admission: %w", err) + } + + status.Reason = "Admitted" + status.NextDueAt = nil + + return status, true, nil +} + +func (r *RecurringReconciler) participantsFitFailureDomainBudget( + ctx context.Context, + participants []signalv1.Participant, + failureDomainLabel string, + maxConcurrent int32, + activeByDomain map[string]int32, +) (bool, error) { + prospective := make(map[string]int32) + + for _, participant := range participants { + var node corev1.Node + if err := r.Reader.Get(ctx, client.ObjectKey{Name: participant.NodeName}, &node); err != nil { + return false, fmt.Errorf( + "get selected participant %s for failure-domain accounting: %w", + participant.NodeName, + err, + ) + } + + if participant.NodeUID != "" && node.UID != participant.NodeUID { + return false, fmt.Errorf("selected participant %s identity changed", participant.NodeName) + } + + domain := node.Labels[failureDomainLabel] + prospective[domain]++ + + if activeByDomain[domain]+prospective[domain] > maxConcurrent { + return false, nil + } + } + + return true, nil +} + +func (r *RecurringReconciler) nodeBusy(ctx context.Context, node *corev1.Node) (bool, string, error) { + var pods corev1.PodList + if err := r.Reader.List(ctx, &pods, client.MatchingFields{"spec.nodeName": node.Name}); err != nil { + return true, "WorkloadStateUnavailable", fmt.Errorf("list Node pods: %w", err) + } + + for index := range pods.Items { + pod := &pods.Items[index] + if pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { + continue + } + + if controlledByDaemonSet(pod) { + continue + } + + return true, "WorkloadActive", nil + } + + active, err := r.activeMachineOperation(ctx, node.Name) + if err != nil { + return true, "MaintenanceStateUnavailable", err + } + + if active { + return true, "MaintenanceOperationActive", nil + } + + return false, "", nil +} + +func (r *RecurringReconciler) activeMachineOperation(ctx context.Context, nodeName string) (bool, error) { + var machines machinav1.MachineList + if err := r.Reader.List(ctx, &machines); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + + return false, fmt.Errorf("list Machines: %w", err) + } + + machineNames := map[string]struct{}{} + + for index := range machines.Items { + machine := &machines.Items[index] + if machine.Spec.Kubernetes != nil && + machine.Spec.Kubernetes.NodeRef != nil && + machine.Spec.Kubernetes.NodeRef.Name == nodeName { + machineNames[machine.Name] = struct{}{} + } + } + + if len(machineNames) == 0 { + return false, nil + } + + var operations machinav1.MachineOperationList + if err := r.Reader.List(ctx, &operations); err != nil { + return false, fmt.Errorf("list MachineOperations: %w", err) + } + + for index := range operations.Items { + operation := &operations.Items[index] + if operation.Status.Phase != machinav1.OperationPhasePending && + operation.Status.Phase != machinav1.OperationPhaseInProgress { + continue + } + + if _, ok := machineNames[operation.Spec.MachineRef]; ok { + return true, nil + } + + if operation.Spec.MachineSelector != nil { + selector, err := metav1.LabelSelectorAsSelector(operation.Spec.MachineSelector) + if err != nil { + return false, fmt.Errorf("MachineOperation %s selector: %w", operation.Name, err) + } + + for machineIndex := range machines.Items { + machine := &machines.Items[machineIndex] + if _, ok := machineNames[machine.Name]; ok && selector.Matches(labels.Set(machine.Labels)) { + return true, nil + } + } + } + } + + return false, nil +} + +func controlledByDaemonSet(pod *corev1.Pod) bool { + for _, owner := range pod.OwnerReferences { + if owner.Controller != nil && *owner.Controller && owner.Kind == "DaemonSet" { + return true + } + } + + return false +} + +func validateRecurringPolicy(policy *signalv1.RecurringPolicy) error { + switch { + case len(policy.NodeNames) == 0: + return fmt.Errorf("nodeNames must explicitly opt in at least one Node") + case policy.MinIdleDuration.Duration < 15*time.Minute: + return fmt.Errorf("minIdleDuration must be at least 15 minutes") + case policy.ValidFor.Duration <= 0: + return fmt.Errorf("validFor must be positive") + case policy.ExpiresAt.IsZero(): + return fmt.Errorf("expiresAt is required") + case policy.MaxConcurrent < 1: + return fmt.Errorf("maxConcurrent must be positive") + case policy.MaxConsecutiveFailures < 1: + return fmt.Errorf("maxConsecutiveFailures must be positive") + } + + return nil +} + +func policyExpired(policy *signalv1.RecurringPolicy, now time.Time) bool { + return !now.Before(policy.ExpiresAt.Time) +} + +func failureGate( + policy *signalv1.RecurringPolicy, + consecutiveFailures int32, + latestFailure *time.Time, + now time.Time, +) (bool, string) { + if policy.MaxConsecutiveFailures > 0 && consecutiveFailures >= policy.MaxConsecutiveFailures { + return true, "CircuitOpen" + } + + if latestFailure != nil && now.Before(latestFailure.Add(policy.FailureBackoff.Duration)) { + return false, "FailureBackoff" + } + + return false, "" +} + +func dueAt( + policy *signalv1.RecurringPolicy, + idleSince *metav1.Time, + lastSuccess *signalv1.CertificationRun, +) time.Time { + due := idleSince.Add(policy.MinIdleDuration.Duration) + if policy.NotBefore != nil && due.Before(policy.NotBefore.Time) { + due = policy.NotBefore.Time + } + + if lastSuccess != nil && lastSuccess.Status.FinishedAt != nil { + freshUntil := lastSuccess.Status.FinishedAt.Add(policy.ValidFor.Duration) + if due.Before(freshUntil) { + due = freshUntil + } + } + + return due +} + +func activeBudgets(runs []signalv1.CertificationRun, profile *signalv1.CertificationProfile) (int32, map[string]int32) { + var active int32 + + domains := map[string]int32{} + + for index := range runs { + run := &runs[index] + if run.Spec.Profile.Name != profile.Name || run.Status.Phase.Terminal() { + continue + } + + active++ + } + + return active, domains +} + +func activeRunForNode(runs []signalv1.CertificationRun, nodeName string) bool { + for index := range runs { + run := &runs[index] + if run.Status.Phase.Terminal() { + continue + } + + for _, participant := range run.Spec.Participants { + if participant.NodeName == nodeName { + return true + } + } + } + + return false +} + +func excludedRunActive(runs []signalv1.CertificationRun, nodeName string, profiles []string) bool { + for index := range runs { + run := &runs[index] + if run.Status.Phase.Terminal() || !slices.Contains(profiles, run.Spec.Profile.Name) { + continue + } + + for _, participant := range run.Spec.Participants { + if participant.NodeName == nodeName { + return true + } + } + } + + return false +} + +func runsForNodeProfile( + runs []signalv1.CertificationRun, + nodeName string, + profileName string, +) []signalv1.CertificationRun { + result := make([]signalv1.CertificationRun, 0) + + for index := range runs { + run := runs[index] + if run.Spec.Profile.Name != profileName || !run.Status.Phase.Terminal() { + continue + } + + for _, participant := range run.Spec.Participants { + if participant.NodeName == nodeName { + result = append(result, run) + + break + } + } + } + + sort.Slice(result, func(i, j int) bool { + return completionTime(&result[i]).After(completionTime(&result[j])) + }) + + return result +} + +func runHistory( + runs []signalv1.CertificationRun, +) (*signalv1.CertificationRun, int32, *time.Time) { + var ( + lastSuccess *signalv1.CertificationRun + latestFailure *time.Time + consecutiveFailure int32 + ) + + for index := range runs { + run := &runs[index] + if run.Status.Phase == signalv1.RunPhaseSucceeded { + lastSuccess = run + + break + } + + if run.Status.Phase != signalv1.RunPhaseFailed && run.Status.Phase != signalv1.RunPhaseInconclusive { + continue + } + + consecutiveFailure++ + + if latestFailure == nil { + value := completionTime(run) + latestFailure = &value + } + } + + return lastSuccess, consecutiveFailure, latestFailure +} + +func completionTime(run *signalv1.CertificationRun) time.Time { + if run.Status.FinishedAt != nil { + return run.Status.FinishedAt.Time + } + + return run.CreationTimestamp.Time +} + +func recurringNodeStatus(statuses []signalv1.RecurringNodeStatus, nodeName string) signalv1.RecurringNodeStatus { + for _, status := range statuses { + if status.NodeName == nodeName { + return status + } + } + + return signalv1.RecurringNodeStatus{NodeName: nodeName} +} + +func ptrMetaTime(value time.Time) *metav1.Time { + result := metav1.NewTime(value) + + return &result +} + +func setProfileCondition( + profile *signalv1.CertificationProfile, + now time.Time, + status metav1.ConditionStatus, + reason string, + message string, +) { + apimeta.SetStatusCondition(&profile.Status.Conditions, metav1.Condition{ + Type: "RecurringAdmissionReady", + Status: status, + ObservedGeneration: profile.Generation, + LastTransitionTime: metav1.NewTime(now), + Reason: reason, + Message: message, + }) +} + +func (r *RecurringReconciler) now() time.Time { + if r.Now != nil { + return r.Now() + } + + return time.Now() +} diff --git a/internal/signal/policy_test.go b/internal/signal/policy_test.go new file mode 100644 index 0000000000..f856e4dc76 --- /dev/null +++ b/internal/signal/policy_test.go @@ -0,0 +1,323 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signal + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" +) + +func TestRecurringRunBudgetsUseAuthoritativeReader(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + if err := signalv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + now := time.Date(2026, 10, 10, 20, 0, 0, 0, time.UTC) + node := readyNode("gpu-0", "wrong-gpu") + profile := testProfile("gpu-smoke", "expected-gpu") + profile.Spec.Recurring = &signalv1.RecurringPolicy{ + Enabled: true, + NodeNames: []string{node.Name}, + MinIdleDuration: metav1.Duration{Duration: 15 * time.Minute}, + ValidFor: metav1.Duration{Duration: time.Hour}, + ExpiresAt: metav1.NewTime(now.Add(time.Hour)), + FailureBackoff: metav1.Duration{Duration: time.Minute}, + MaxConsecutiveFailures: 3, + MaxConcurrent: 1, + } + base := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationProfile{}). + WithObjects(node, profile). + Build() + cached := interceptor.NewClient(base, interceptor.Funcs{ + List: func( + context.Context, + client.WithWatch, + client.ObjectList, + ...client.ListOption, + ) error { + return errors.New("cached list must not define admission budgets") + }, + }) + r := &RecurringReconciler{ + Client: cached, + Reader: base, + Namespace: "signal", + Admission: &AdmissionReconciler{Client: cached}, + Now: func() time.Time { return now }, + } + + if _, err := r.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKey{Name: profile.Name}}, + ); err != nil { + t.Fatalf("Reconcile: %v", err) + } +} + +func TestRecurringFailureDomainAccountingFailsClosed(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + if err := signalv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + now := time.Date(2026, 10, 10, 20, 0, 0, 0, time.UTC) + profile := testProfile("gpu-smoke", "expected-gpu") + profile.Spec.Recurring = &signalv1.RecurringPolicy{ + Enabled: true, + NodeNames: []string{"gpu-0"}, + MinIdleDuration: metav1.Duration{Duration: 15 * time.Minute}, + ValidFor: metav1.Duration{Duration: time.Hour}, + ExpiresAt: metav1.NewTime(now.Add(time.Hour)), + FailureBackoff: metav1.Duration{Duration: time.Minute}, + MaxConsecutiveFailures: 3, + MaxConcurrent: 1, + FailureDomainLabel: "topology.kubernetes.io/zone", + MaxConcurrentPerFailureDomain: 1, + } + active := testRun("active") + active.Spec.Profile.Name = profile.Name + active.Spec.Participants = []signalv1.Participant{{NodeName: "missing-node", NodeUID: "missing-uid"}} + active.Status.Phase = signalv1.RunPhaseRunning + base := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&signalv1.CertificationProfile{}). + WithObjects(profile, active). + Build() + r := &RecurringReconciler{ + Client: base, + Reader: base, + Namespace: "signal", + Admission: &AdmissionReconciler{Client: base}, + Now: func() time.Time { return now }, + } + + if _, err := r.Reconcile( + context.Background(), + reconcile.Request{NamespacedName: client.ObjectKey{Name: profile.Name}}, + ); err == nil || !strings.Contains(err.Error(), "failure-domain accounting") { + t.Fatalf("Reconcile error = %v, want fail-closed accounting", err) + } +} + +func TestRecurringMultiNodeFailureDomainBudgetAccountsWholeGroup(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + if err := signalv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + first := readyNode("gpu-0", "H200") + first.Labels["topology.kubernetes.io/zone"] = "zone-a" + second := readyNode("gpu-1", "H200") + second.Labels["topology.kubernetes.io/zone"] = "zone-a" + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(first, second).Build() + r := &RecurringReconciler{Reader: kube} + participants := []signalv1.Participant{ + participantSnapshot(first, 0), + participantSnapshot(second, 1), + } + + fits, err := r.participantsFitFailureDomainBudget( + context.Background(), + participants, + "topology.kubernetes.io/zone", + 1, + map[string]int32{}, + ) + if err != nil { + t.Fatal(err) + } + + if fits { + t.Fatal("two participants in one domain exceeded the per-domain budget") + } + + second.Labels["topology.kubernetes.io/zone"] = "zone-b" + if err := kube.Update(context.Background(), second); err != nil { + t.Fatal(err) + } + + fits, err = r.participantsFitFailureDomainBudget( + context.Background(), + participants, + "topology.kubernetes.io/zone", + 1, + map[string]int32{}, + ) + if err != nil { + t.Fatal(err) + } + + if !fits { + t.Fatal("participants in distinct domains should fit the budget") + } +} + +func TestRecurringDueUsesContinuousIdleAndFreshness(t *testing.T) { + t.Parallel() + + start := time.Date(2026, 10, 10, 20, 0, 0, 0, time.UTC) + idleSince := metav1.NewTime(start) + policy := &signalv1.RecurringPolicy{ + MinIdleDuration: metav1.Duration{Duration: 15 * time.Minute}, + ValidFor: metav1.Duration{Duration: time.Hour}, + } + + if got := dueAt(policy, &idleSince, nil); !got.Equal(start.Add(15 * time.Minute)) { + t.Fatalf("initial due = %s", got) + } + + success := testRun("success") + success.Status.Phase = signalv1.RunPhaseSucceeded + success.Status.FinishedAt = ptrTime(metav1.NewTime(start.Add(10 * time.Minute))) + + if got := dueAt(policy, &idleSince, success); !got.Equal(start.Add(70 * time.Minute)) { + t.Fatalf("freshness due = %s", got) + } +} + +func TestRecurringIdleStateSurvivesRestart(t *testing.T) { + t.Parallel() + + idleSince := metav1.NewTime(time.Date(2026, 10, 10, 20, 0, 0, 0, time.UTC)) + statuses := []signalv1.RecurringNodeStatus{{NodeName: "gpu-0", IdleSince: &idleSince, Reason: "NotDue"}} + + got := recurringNodeStatus(statuses, "gpu-0") + if got.IdleSince == nil || !got.IdleSince.Equal(&idleSince) { + t.Fatalf("idleSince was not recovered: %#v", got) + } +} + +func TestRecurringExpiryFailsClosed(t *testing.T) { + t.Parallel() + + expiry := metav1.NewTime(time.Date(2026, 10, 12, 14, 0, 0, 0, time.UTC)) + policy := &signalv1.RecurringPolicy{ExpiresAt: expiry} + + if policyExpired(policy, expiry.Add(-time.Nanosecond)) { + t.Fatal("policy expired early") + } + + if !policyExpired(policy, expiry.Time) { + t.Fatal("policy remained active at its expiry") + } +} + +func TestRecurringConcurrencyAndDuplicatePrevention(t *testing.T) { + t.Parallel() + + active := *testRun("active") + active.Status.Phase = signalv1.RunPhaseRunning + active.Spec.Profile.Name = "short" + profile := testProfile("short", "A100") + + count, _ := activeBudgets([]signalv1.CertificationRun{active}, profile) + if count != 1 || !activeRunForNode([]signalv1.CertificationRun{active}, "gpu-0") { + t.Fatalf("active budget=%d duplicate=%t", count, activeRunForNode([]signalv1.CertificationRun{active}, "gpu-0")) + } +} + +func TestRecurringShortLongMutualExclusion(t *testing.T) { + t.Parallel() + + longRun := *testRun("long") + longRun.Spec.Profile.Name = "gpu-long" + longRun.Status.Phase = signalv1.RunPhaseRunning + + if !excludedRunActive( + []signalv1.CertificationRun{longRun}, + "gpu-0", + []string{"gpu-long"}, + ) { + t.Fatal("active long run did not exclude the short profile") + } +} + +func TestRecurringFailureBackoffAndCircuitBreaker(t *testing.T) { + t.Parallel() + + now := time.Date(2026, 10, 10, 20, 0, 0, 0, time.UTC) + failed := *testRun("failed") + failed.Status.Phase = signalv1.RunPhaseFailed + failed.Status.FinishedAt = ptrTime(metav1.NewTime(now.Add(-5 * time.Minute))) + inconclusive := *testRun("inconclusive") + inconclusive.Status.Phase = signalv1.RunPhaseInconclusive + inconclusive.Status.FinishedAt = ptrTime(metav1.NewTime(now.Add(-10 * time.Minute))) + runs := []signalv1.CertificationRun{failed, inconclusive} + + _, count, latest := runHistory(runs) + policy := &signalv1.RecurringPolicy{ + FailureBackoff: metav1.Duration{Duration: 30 * time.Minute}, + MaxConsecutiveFailures: 3, + } + + suspended, reason := failureGate(policy, count, latest, now) + if suspended || reason != "FailureBackoff" { + t.Fatalf("backoff gate = (%t, %q)", suspended, reason) + } + + policy.MaxConsecutiveFailures = 2 + + suspended, reason = failureGate(policy, count, latest, now) + if !suspended || reason != "CircuitOpen" { + t.Fatalf("circuit gate = (%t, %q)", suspended, reason) + } +} + +func TestIdleDetectionIgnoresOnlyDaemonSetsAndTerminalPods(t *testing.T) { + t.Parallel() + + controller := true + daemon := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{OwnerReferences: []metav1.OwnerReference{{ + APIVersion: appsv1.SchemeGroupVersion.String(), + Kind: "DaemonSet", + Controller: &controller, + }}}, + } + deployment := daemon.DeepCopy() + deployment.OwnerReferences[0].Kind = "ReplicaSet" + + if !controlledByDaemonSet(daemon) { + t.Fatal("DaemonSet pod was treated as workload activity") + } + + if controlledByDaemonSet(deployment) { + t.Fatal("non-DaemonSet pod was treated as disruptable") + } +} diff --git a/internal/signal/projection.go b/internal/signal/projection.go new file mode 100644 index 0000000000..4ed10d4a30 --- /dev/null +++ b/internal/signal/projection.go @@ -0,0 +1,249 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signal + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "strconv" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" +) + +const ( + StateLabel = "signal.unbounded-cloud.io/state" + EligibilityLabel = "signal.unbounded-cloud.io/eligibility" + ProfileLabel = "signal.unbounded-cloud.io/profile" + + ProfileNameAnnotation = "signal.unbounded-cloud.io/profile-name" + ProfileGenerationAnnotation = "signal.unbounded-cloud.io/profile-generation" + LastRunAnnotation = "signal.unbounded-cloud.io/last-run" + LastRunAtAnnotation = "signal.unbounded-cloud.io/last-run-at" + LastSuccessfulRunAnnotation = "signal.unbounded-cloud.io/last-successful-run" + LastSuccessfulAtAnnotation = "signal.unbounded-cloud.io/last-successful-at" + EvidenceDigestAnnotation = "signal.unbounded-cloud.io/evidence-digest" + ScriptDigestAnnotation = "signal.unbounded-cloud.io/script-digest" + NextDueAtAnnotation = "signal.unbounded-cloud.io/next-due-at" + RecurringReasonAnnotation = "signal.unbounded-cloud.io/recurring-reason" + RecurringSuspendedAnnotation = "signal.unbounded-cloud.io/recurring-suspended" + + maxProjectionAnnotationValue = 1024 +) + +func projectPending( + ctx context.Context, + kube client.Client, + participants []signalv1.Participant, + profile signalv1.PinnedProfile, +) error { + return projectParticipants(ctx, kube, participants, func(node *corev1.Node) error { + setProfileProjection(node, profile) + node.Labels[StateLabel] = "pending" + node.Labels[EligibilityLabel] = "blocked" + + return nil + }) +} + +func projectTesting(ctx context.Context, kube client.Client, run *signalv1.CertificationRun) error { + return projectParticipants(ctx, kube, run.Spec.Participants, func(node *corev1.Node) error { + setProfileProjection(node, run.Spec.Profile) + node.Labels[StateLabel] = "testing" + node.Labels[EligibilityLabel] = "blocked" + + return nil + }) +} + +func projectTerminal(ctx context.Context, kube client.Client, run *signalv1.CertificationRun) error { + if run.Status.FinishedAt == nil { + return nil + } + + finishedAt := run.Status.FinishedAt.UTC().Format(time.RFC3339) + + return projectParticipants(ctx, kube, run.Spec.Participants, func(node *corev1.Node) error { + if projectionTimeAfter(node.Annotations[LastRunAtAnnotation], run.Status.FinishedAt.Time) { + return nil + } + + setProfileProjection(node, run.Spec.Profile) + node.Annotations[LastRunAnnotation] = boundedAnnotationValue(run.Name) + node.Annotations[LastRunAtAnnotation] = finishedAt + node.Annotations[EvidenceDigestAnnotation] = boundedAnnotationValue(run.Status.Evidence.Digest) + node.Annotations[ScriptDigestAnnotation] = boundedAnnotationValue(run.Spec.Script.Digest) + + switch run.Status.Phase { + case signalv1.RunPhaseSucceeded: + node.Labels[StateLabel] = "passed" + node.Labels[EligibilityLabel] = "production" + + if !projectionTimeAfter(node.Annotations[LastSuccessfulAtAnnotation], run.Status.FinishedAt.Time) { + node.Annotations[LastSuccessfulRunAnnotation] = boundedAnnotationValue(run.Name) + node.Annotations[LastSuccessfulAtAnnotation] = finishedAt + } + case signalv1.RunPhaseFailed: + node.Labels[StateLabel] = "failed" + node.Labels[EligibilityLabel] = "blocked" + default: + node.Labels[StateLabel] = "inconclusive" + node.Labels[EligibilityLabel] = "blocked" + } + + return nil + }) +} + +func projectRecurring( + ctx context.Context, + kube client.Client, + profile *signalv1.CertificationProfile, + status signalv1.RecurringNodeStatus, +) error { + participant := signalv1.Participant{NodeName: status.NodeName} + pinned := signalv1.PinnedProfile{Name: profile.Name, UID: profile.UID, Generation: profile.Generation} + + return projectParticipants(ctx, kube, []signalv1.Participant{participant}, func(node *corev1.Node) error { + selectedProfile := node.Annotations[ProfileNameAnnotation] + + active := node.Labels[StateLabel] == "pending" || node.Labels[StateLabel] == "testing" + if active && selectedProfile != profile.Name { + return nil + } + + if status.LastSuccessfulAt != nil && + !projectionTimeAfter(node.Annotations[LastSuccessfulAtAnnotation], status.LastSuccessfulAt.Time) { + setProfileProjection(node, pinned) + } else { + return nil + } + + setOrDeleteAnnotation(node, NextDueAtAnnotation, formatMetaTime(status.NextDueAt)) + setOrDeleteAnnotation(node, RecurringReasonAnnotation, status.Reason) + node.Annotations[RecurringSuspendedAnnotation] = strconv.FormatBool(status.Suspended) + + if status.LastSuccessfulAt != nil && + !projectionTimeAfter(node.Annotations[LastSuccessfulAtAnnotation], status.LastSuccessfulAt.Time) { + node.Annotations[LastSuccessfulRunAnnotation] = boundedAnnotationValue(status.LastSuccessfulRun) + node.Annotations[LastSuccessfulAtAnnotation] = status.LastSuccessfulAt.UTC().Format(time.RFC3339) + + if node.Labels[StateLabel] == "" { + node.Labels[StateLabel] = "passed" + node.Labels[EligibilityLabel] = "production" + } + } + + return nil + }) +} + +func projectParticipants( + ctx context.Context, + kube client.Client, + participants []signalv1.Participant, + mutate func(*corev1.Node) error, +) error { + for _, participant := range participants { + if participant.NodeName == "" { + continue + } + + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + var node corev1.Node + if err := kube.Get(ctx, client.ObjectKey{Name: participant.NodeName}, &node); err != nil { + return client.IgnoreNotFound(err) + } + + if participant.NodeUID != "" && node.UID != participant.NodeUID { + return nil + } + + before := node.DeepCopy() + ensureProjectionMaps(&node) + + if err := mutate(&node); err != nil { + return err + } + + return kube.Patch(ctx, &node, client.MergeFrom(before)) + }); err != nil { + return fmt.Errorf("project Node %s status: %w", participant.NodeName, err) + } + } + + return nil +} + +func setProfileProjection(node *corev1.Node, profile signalv1.PinnedProfile) { + ensureProjectionMaps(node) + node.Labels[ProfileLabel] = projectionLabelValue(profile.Name) + node.Annotations[ProfileNameAnnotation] = boundedAnnotationValue(profile.Name) + node.Annotations[ProfileGenerationAnnotation] = strconv.FormatInt(profile.Generation, 10) +} + +func ensureProjectionMaps(node *corev1.Node) { + if node.Labels == nil { + node.Labels = map[string]string{} + } + + if node.Annotations == nil { + node.Annotations = map[string]string{} + } +} + +func projectionLabelValue(value string) string { + if len(validation.IsValidLabelValue(value)) == 0 { + return value + } + + sum := sha256.Sum256([]byte(value)) + + return "sha256-" + hex.EncodeToString(sum[:8]) +} + +func boundedAnnotationValue(value string) string { + if len(value) <= maxProjectionAnnotationValue { + return value + } + + sum := sha256.Sum256([]byte(value)) + + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func projectionTimeAfter(value string, candidate time.Time) bool { + existing, err := time.Parse(time.RFC3339, value) + if err != nil { + return false + } + + return existing.After(candidate) +} + +func setOrDeleteAnnotation(node *corev1.Node, key, value string) { + if value == "" { + delete(node.Annotations, key) + + return + } + + node.Annotations[key] = boundedAnnotationValue(value) +} + +func formatMetaTime(value *metav1.Time) string { + if value == nil { + return "" + } + + return value.UTC().Format(time.RFC3339) +} diff --git a/internal/signal/projection_test.go b/internal/signal/projection_test.go new file mode 100644 index 0000000000..6af9abcb6b --- /dev/null +++ b/internal/signal/projection_test.go @@ -0,0 +1,227 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signal + +import ( + "context" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" +) + +func TestNodeProjectionTransitionsAndRetainsTerminalSummary(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + node := readyNode("gpu-0", "A100") + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node).Build() + run := testRun("run-pass") + + if err := projectPending(context.Background(), kube, run.Spec.Participants, run.Spec.Profile); err != nil { + t.Fatal(err) + } + + assertNodeProjection(t, kube, node.Name, "pending", "blocked") + + if err := projectTesting(context.Background(), kube, run); err != nil { + t.Fatal(err) + } + + assertNodeProjection(t, kube, node.Name, "testing", "blocked") + + run.Status.Phase = signalv1.RunPhaseSucceeded + run.Status.ExecutionState = "Completed" + run.Status.Verdict = "Pass" + run.Status.Reason = "Healthy" + run.Status.Evidence.Digest = "sha256:evidence" + run.Status.FinishedAt = ptrTime(metav1.NewTime(time.Date(2026, 10, 11, 5, 0, 0, 0, time.UTC))) + + if err := projectTerminal(context.Background(), kube, run); err != nil { + t.Fatal(err) + } + + var got corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &got); err != nil { + t.Fatal(err) + } + + if got.Labels[StateLabel] != "passed" || got.Labels[EligibilityLabel] != "production" { + t.Fatalf("terminal labels = %v", got.Labels) + } + + if got.Annotations[LastRunAnnotation] != run.Name || + got.Annotations[LastSuccessfulRunAnnotation] != run.Name || + got.Annotations[EvidenceDigestAnnotation] != run.Status.Evidence.Digest || + got.Annotations[ScriptDigestAnnotation] != run.Spec.Script.Digest { + t.Fatalf("terminal annotations = %v", got.Annotations) + } +} + +func TestNodeProjectionFailureAndInconclusiveBlockEligibility(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + phase signalv1.RunPhase + state string + }{ + {name: "failed", phase: signalv1.RunPhaseFailed, state: "failed"}, + {name: "inconclusive", phase: signalv1.RunPhaseInconclusive, state: "inconclusive"}, + } { + t.Run(tc.name, func(t *testing.T) { + scheme := testScheme(t) + node := readyNode("gpu-0", "A100") + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node).Build() + run := testRun("run-" + tc.name) + run.Status.Phase = tc.phase + run.Status.FinishedAt = ptrTime(metav1.Now()) + + if err := projectTerminal(context.Background(), kube, run); err != nil { + t.Fatal(err) + } + + assertNodeProjection(t, kube, node.Name, tc.state, "blocked") + }) + } +} + +func TestNodeProjectionDoesNotRegressAfterRestartOrOlderRetry(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + node := readyNode("gpu-0", "A100") + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node).Build() + newer := testRun("newer") + newer.Status.Phase = signalv1.RunPhaseSucceeded + newer.Status.FinishedAt = ptrTime(metav1.NewTime(time.Date(2026, 10, 11, 6, 0, 0, 0, time.UTC))) + newer.Status.Evidence.Digest = "sha256:newer" + + if err := projectTerminal(context.Background(), kube, newer); err != nil { + t.Fatal(err) + } + + older := testRun("older") + older.Status.Phase = signalv1.RunPhaseFailed + older.Status.FinishedAt = ptrTime(metav1.NewTime(time.Date(2026, 10, 11, 5, 0, 0, 0, time.UTC))) + older.Status.Evidence.Digest = "sha256:older" + + if err := projectTerminal(context.Background(), kube, older); err != nil { + t.Fatal(err) + } + + var got corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &got); err != nil { + t.Fatal(err) + } + + if got.Labels[StateLabel] != "passed" || + got.Annotations[LastRunAnnotation] != newer.Name || + got.Annotations[EvidenceDigestAnnotation] != "sha256:newer" { + t.Fatalf("projection regressed: labels=%v annotations=%v", got.Labels, got.Annotations) + } +} + +func TestNodeProjectionDoesNotCertifyReplacementNode(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + replacement := readyNode("gpu-0", "A100") + replacement.UID = "replacement-uid" + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(replacement).Build() + run := testRun("run-pass") + run.Status.Phase = signalv1.RunPhaseSucceeded + run.Status.Verdict = "Pass" + run.Status.FinishedAt = ptrTime(metav1.Now()) + + if err := projectTerminal(context.Background(), kube, run); err != nil { + t.Fatal(err) + } + + var got corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: replacement.Name}, &got); err != nil { + t.Fatal(err) + } + + if got.Labels[StateLabel] != "" || got.Labels[EligibilityLabel] != "" { + t.Fatalf("replacement Node was certified: labels=%v", got.Labels) + } +} + +func TestProjectionValuesAreBoundedAndValid(t *testing.T) { + t.Parallel() + + longValue := strings.Repeat("Invalid/Profile:", 200) + + label := projectionLabelValue(longValue) + if len(label) > 63 { + t.Fatalf("label value is too long: %d", len(label)) + } + + if bounded := boundedAnnotationValue(longValue); len(bounded) > len("sha256:")+64 { + t.Fatalf("annotation value is unbounded: %d", len(bounded)) + } + + if got := projectionLabelValue("gpu-smoke-example"); got != "gpu-smoke-example" { + t.Fatalf("valid profile label changed to %q", got) + } +} + +func TestRecurringProjectionPublishesNextDueAndSuspension(t *testing.T) { + t.Parallel() + + scheme := testScheme(t) + node := readyNode("gpu-0", "A100") + profile := testProfile("gpu-smoke-example", "example-gpu") + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(node).Build() + nextDue := metav1.NewTime(time.Date(2026, 10, 11, 7, 0, 0, 0, time.UTC)) + status := signalv1.RecurringNodeStatus{ + NodeName: node.Name, + NextDueAt: &nextDue, + LastSuccessfulRun: "run-pass", + LastSuccessfulAt: ptrTime(metav1.NewTime(time.Date(2026, 10, 11, 6, 0, 0, 0, time.UTC))), + Suspended: true, + Reason: "CircuitOpen", + } + + if err := projectRecurring(context.Background(), kube, profile, status); err != nil { + t.Fatal(err) + } + + var got corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: node.Name}, &got); err != nil { + t.Fatal(err) + } + + if got.Annotations[NextDueAtAnnotation] != "2026-10-11T07:00:00Z" || + got.Annotations[RecurringReasonAnnotation] != "CircuitOpen" || + got.Annotations[RecurringSuspendedAnnotation] != "true" { + t.Fatalf("recurring annotations = %v", got.Annotations) + } +} + +func assertNodeProjection( + t *testing.T, + kube client.Client, + nodeName string, + state string, + eligibility string, +) { + t.Helper() + + var node corev1.Node + if err := kube.Get(context.Background(), client.ObjectKey{Name: nodeName}, &node); err != nil { + t.Fatal(err) + } + + if node.Labels[StateLabel] != state || node.Labels[EligibilityLabel] != eligibility { + t.Fatalf("projection = state %q eligibility %q", node.Labels[StateLabel], node.Labels[EligibilityLabel]) + } +}