diff --git a/.dockerignore b/.dockerignore index 9bbe7a5f07..5d20bc8b6b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,6 +6,7 @@ __debug* tmp/ .tmp/ dist/ +deploy/signal/rendered/ # Docker .dockerignore diff --git a/Makefile b/Makefile index 91b0b53a74..48ff025ebd 100644 --- a/Makefile +++ b/Makefile @@ -92,6 +92,17 @@ MACHINE_OPS_CONTROLLER_NAME ?= machine-ops-controller MACHINE_OPS_PROVIDER ?= MACHINE_OPS_SITE ?= +PROJECT_SIGNAL_CONTROLLER_BIN=bin/project-signal-controller +PROJECT_SIGNAL_CONTROLLER_CMD=./cmd/project-signal-controller +SIGNAL_CONTROLLER_IMAGE ?= example.invalid/project-signal-controller:replace-me +PROJECT_SIGNAL_ADAPTER_BIN=bin/project-signal-adapter +PROJECT_SIGNAL_ADAPTER_CMD=./cmd/project-signal-adapter +SIGNAL_ADAPTER_IMAGE ?= example.invalid/project-signal-adapter@sha256:0000000000000000000000000000000000000000000000000000000000000000 +SIGNAL_ADAPTER_BUILD_IMAGE ?= example.invalid/project-signal-adapter:replace-me +SIGNAL_NAMESPACE ?= project-signal +SIGNAL_MANIFEST_TEMPLATES_DIR := deploy/signal +SIGNAL_MANIFEST_RENDERED_DIR := deploy/signal/rendered + METALMAN_BIN=bin/metalman METALMAN_CMD=./cmd/metalman NETBOOT_IMAGE ?= $(CONTAINER_REGISTRY)/netboot:$(VERSION_TAG) @@ -233,14 +244,14 @@ NET_FRONTEND_CACHE_FILE := $(NET_FRONTEND_DIST_DIR)/.frontend-build-key # Frontend build toggle (dev builds produce unminified output with sourcemaps). REACT_DEV ?= false -.PHONY: all help fmt lint lint-actions test build vulncheck check-deps kubectl-unbounded kubectl-unbounded-build install-tools install-protoc install-helm generate kubectl-unbounded forge relctl relctl-build agent-artifacts-builder agent-artifacts-builder-build orcadev unbounded-agent machina machina-build machina-oci machina-oci-push machina-manifests machine-ops-controller machine-ops-controller-build machine-ops-controller-oci machine-ops-controller-oci-push machine-ops-manifests metalman metalman-build metalman-oci metalman-oci-push unbounded-operator unbounded-operator-build unbounded-operator-manifests playpen-manifests e2e-gantry e2e-playpen e2e-racer gomod docs-serve unbounded-net-controller unbounded-net-controller-build unbounded-net-node unbounded-net-node-build unbounded-net-routeplan-debug unping unping-build unroute unroute-build license-check notice notice-check gantry gantry-build gantry-manifests inventory-manifests +.PHONY: all help fmt lint lint-actions test build vulncheck check-deps kubectl-unbounded kubectl-unbounded-build install-tools install-protoc install-helm generate kubectl-unbounded forge relctl relctl-build agent-artifacts-builder agent-artifacts-builder-build orcadev unbounded-agent machina machina-build machina-oci machina-oci-push machina-manifests machine-ops-controller machine-ops-controller-build machine-ops-controller-oci machine-ops-controller-oci-push machine-ops-manifests project-signal-controller project-signal-controller-build project-signal-adapter project-signal-adapter-build signal-manifests image-project-signal-controller-local image-project-signal-adapter-local metalman metalman-build metalman-oci metalman-oci-push unbounded-operator unbounded-operator-build unbounded-operator-manifests playpen-manifests e2e-gantry e2e-playpen e2e-racer gomod docs-serve unbounded-net-controller unbounded-net-controller-build unbounded-net-node unbounded-net-node-build unbounded-net-routeplan-debug unping unping-build unroute unroute-build license-check notice notice-check gantry gantry-build gantry-manifests inventory-manifests .PHONY: net-frontend net-frontend-clean net-ebpf-build net-ebpf-generate net-ebpf-verify net-manifests gantry-chart-lint gantry-chart-package release-bom release-manifests unbounded-operator-release-manifest .PHONY: image-machina-local image-token-refresher-local image-machine-ops-controller-local image-metalman-local image-unbounded-operator-local image-unbounded-operator-push image-playpen-local image-net-controller-local image-net-node-local image-gantry-local image-gantry-push images-local .PHONY: image-net-controller-push image-net-node-push images-net-all images-net-all-push ##@ General -all: kubectl-unbounded forge relctl machina machine-ops-controller token-refresher unbounded-operator unbounded-net-controller unbounded-net-node unbounded-ipam unbounded-net-routeplan-debug unping unroute gantry ## Build all binaries (default) +all: kubectl-unbounded forge relctl machina machine-ops-controller project-signal-controller project-signal-adapter token-refresher unbounded-operator unbounded-net-controller unbounded-net-node unbounded-ipam unbounded-net-routeplan-debug unping unroute gantry ## Build all binaries (default) help: ## Show this help @echo "" @@ -292,6 +303,9 @@ help: ## Show this help @echo " unbounded-agent Build unbounded-agent (linux)" @echo " machina | machina-build Build machina controller (with/without lint/test)" @echo " machine-ops-controller Build machine-ops-controller" + @echo " project-signal-controller Build Project Signal controller" + @echo " project-signal-adapter Build Project Signal participant adapter" + @echo " signal-manifests Render Project Signal manifests" @echo " metalman | metalman-build Build metalman controller (with/without lint/test)" @echo " unbounded-operator | unbounded-operator-build Build the top-level Site operator" @echo " unbounded-net-controller Build net controller" @@ -313,6 +327,8 @@ help: ## Show this help @echo " image-machina-local Build machina image with \$$(CONTAINER_ENGINE)" @echo " image-token-refresher-local Build token-refresher image" @echo " image-machine-ops-controller-local Build machine-ops-controller image" + @echo " image-project-signal-controller-local Build Project Signal controller image" + @echo " image-project-signal-adapter-local Build Project Signal adapter image" @echo " image-metalman-local Build metalman image" @echo " image-unbounded-operator-local Build unbounded-operator image" @echo " image-unbounded-operator-push Build and push unbounded-operator image" @@ -547,13 +563,13 @@ lint-actions: ## Run actionlint over .github/workflows ifdef CI # In CI each job is independent; skip chained prerequisites. -test: machina-manifests token-refresher-manifests machine-ops-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Run all tests with race detector +test: machina-manifests token-refresher-manifests machine-ops-manifests signal-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Run all tests with race detector $(GOTEST) -race ./... else # Locally, chain test -> lint for convenience. -test: lint machina-manifests token-refresher-manifests machine-ops-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Run all tests (implies lint) +test: lint machina-manifests token-refresher-manifests machine-ops-manifests signal-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Run all tests (implies lint) $(GOTEST) ./... endif @@ -661,13 +677,13 @@ image-racer-object-local: ## Build the racer-object container image locally (sin -f ./images/racer-object/Containerfile . $(call trivy-maybe,$(RACER_OBJECT_IMAGE)) -build: machina-manifests token-refresher-manifests machine-ops-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Build all Go packages +build: machina-manifests token-refresher-manifests machine-ops-manifests signal-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Build all Go packages $(GOBUILD) ./... generate: install-protoc ## Run go generate for API types (deepcopy, CRDs) and protobuf PATH="$(PROTOC_DIR)/bin:$$PATH" $(GOCMD) generate $(GO_PACKAGES) -vulncheck: machina-manifests token-refresher-manifests machine-ops-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Run govulncheck; fails only on reachable vulnerabilities that have a final-release fix +vulncheck: machina-manifests token-refresher-manifests machine-ops-manifests signal-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Run govulncheck; fails only on reachable vulnerabilities that have a final-release fix @# The JSON stream is the documented programmatic interface. The gate owns @# the verdict, so govulncheck is not asked for one: in JSON mode it exits 0 @# whether or not it found anything, and a non-zero exit here means the scan @@ -1085,6 +1101,36 @@ machine-ops-controller-oci: image-machine-ops-controller-local ## Alias for imag machine-ops-controller-oci-push: machine-ops-controller-oci ## Build and push the machine-ops-controller image $(CONTAINER_ENGINE) push $(MACHINE_OPS_CONTROLLER_IMAGE) +project-signal-controller-build: signal-manifests ## Build Project Signal controller without lint/test + @mkdir -p bin + $(GOBUILD) -trimpath -ldflags '$(STAMP_LDFLAGS)' -o $(PROJECT_SIGNAL_CONTROLLER_BIN) $(PROJECT_SIGNAL_CONTROLLER_CMD) + +project-signal-controller: test project-signal-controller-build ## Build Project Signal controller (implies test) + +project-signal-adapter-build: signal-manifests ## Build Project Signal adapter without lint/test + @mkdir -p bin + $(GOBUILD) -trimpath -ldflags '$(STAMP_LDFLAGS)' -o $(PROJECT_SIGNAL_ADAPTER_BIN) $(PROJECT_SIGNAL_ADAPTER_CMD) + +project-signal-adapter: test project-signal-adapter-build ## Build Project Signal adapter (implies test) + +image-project-signal-controller-local: ## Build Project Signal controller image locally + $(CONTAINER_ENGINE) build \ + --build-arg VERSION=$(VERSION) \ + --build-arg GIT_COMMIT=$(GIT_COMMIT) \ + --build-arg BUILD_TIME=$(BUILD_TIME) \ + -t project-signal-controller:$(VERSION_TAG) -t $(SIGNAL_CONTROLLER_IMAGE) \ + -f ./images/project-signal-controller/Containerfile . + $(call trivy-maybe,$(SIGNAL_CONTROLLER_IMAGE)) + +image-project-signal-adapter-local: ## Build Project Signal adapter image locally + $(CONTAINER_ENGINE) build \ + --build-arg VERSION=$(VERSION) \ + --build-arg GIT_COMMIT=$(GIT_COMMIT) \ + --build-arg BUILD_TIME=$(BUILD_TIME) \ + -t project-signal-adapter:$(VERSION_TAG) -t $(SIGNAL_ADAPTER_BUILD_IMAGE) \ + -f ./images/project-signal-adapter/Containerfile . + $(call trivy-maybe,$(SIGNAL_ADAPTER_BUILD_IMAGE)) + MACHINA_NAMESPACE ?= $(UNBOUNDED_NAMESPACE) MACHINA_API_SERVER_ENDPOINT ?= MACHINA_MANIFEST_TEMPLATES_DIR := deploy/machina @@ -1157,6 +1203,19 @@ machine-ops-manifests: ## Render machine-ops-controller manifests into deploy/ma --set APIServerEndpoint=$(MACHINE_OPS_API_SERVER_ENDPOINT) @echo "Rendered machine-ops manifests into $(MACHINE_OPS_MANIFEST_RENDERED_DIR) (image: $(MACHINE_OPS_CONTROLLER_IMAGE))" +signal-manifests: ## Render Project Signal manifests into deploy/signal/rendered + @mkdir -p $(SIGNAL_MANIFEST_RENDERED_DIR)/crd + @find $(SIGNAL_MANIFEST_RENDERED_DIR) -mindepth 1 -not -name .gitignore -delete + $(GOCMD) run ./hack/cmd/render-manifests \ + --templates-dir $(SIGNAL_MANIFEST_TEMPLATES_DIR) \ + --output-dir $(SIGNAL_MANIFEST_RENDERED_DIR) \ + --set Namespace=$(SIGNAL_NAMESPACE) \ + --set ControllerImage=$(SIGNAL_CONTROLLER_IMAGE) \ + --set AdapterImage=$(SIGNAL_ADAPTER_IMAGE) + @mkdir -p $(SIGNAL_MANIFEST_RENDERED_DIR)/crd + @cp $(SIGNAL_MANIFEST_TEMPLATES_DIR)/crd/*.yaml $(SIGNAL_MANIFEST_RENDERED_DIR)/crd/ + @echo "Rendered Project Signal manifests into $(SIGNAL_MANIFEST_RENDERED_DIR) (image: $(SIGNAL_CONTROLLER_IMAGE))" + playpen-manifests: ## Render playpen operator and runner manifests into deploy/playpen/rendered @mkdir -p $(PLAYPEN_MANIFEST_RENDERED_DIR) @find $(PLAYPEN_MANIFEST_RENDERED_DIR) -mindepth 1 -not -name .gitignore -delete diff --git a/api/signal/v1alpha1/certificationprofile_types.go b/api/signal/v1alpha1/certificationprofile_types.go new file mode 100644 index 0000000000..d3d5a06da1 --- /dev/null +++ b/api/signal/v1alpha1/certificationprofile_types.go @@ -0,0 +1,129 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type ScriptReference struct { + Name string `json:"name"` + Key string `json:"key"` + // Digest is the approved algorithm-qualified SHA-256 digest of the script bytes. + // +kubebuilder:validation:Pattern=`^sha256:[0-9a-f]{64}$` + Digest string `json:"digest"` +} + +type CheckID string + +type ParticipantRoleConstraint struct { + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` + Name string `json:"name"` + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=64 + Count int32 `json:"count"` + // Selector further constrains participants assigned to this role. + Selector metav1.LabelSelector `json:"selector,omitempty"` +} + +type ParticipantTopologyConstraint struct { + // Role names the participant role whose selected Nodes are evaluated. + Role string `json:"role"` + // LabelKey is the Node label used to identify topology domains. + LabelKey string `json:"labelKey"` + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=64 + MinDistinctValues int32 `json:"minDistinctValues"` +} + +type CheckParticipantPolicy struct { + // +kubebuilder:validation:MaxItems=16 + Roles []string `json:"roles,omitempty"` +} + +// +kubebuilder:validation:XValidation:rule="duration(self.backendActiveDeadline) > duration('0s')",message="backendActiveDeadline must be positive" +// +kubebuilder:validation:XValidation:rule="!has(self.scriptEnvironment) || self.scriptEnvironment.all(k, size(k) <= 253 && size(self.scriptEnvironment[k]) <= 4096)",message="scriptEnvironment names must not exceed 253 characters and values must not exceed 4096 characters" +type CertificationCheck struct { + // ID and Order are derived from the check's position and are never serialized. + ID CheckID `json:"-"` + Order int32 `json:"-"` + Script ScriptReference `json:"script"` + RunnerImage string `json:"runnerImage"` + Backend ExecutionBackend `json:"backend"` + WorkloadPolicy WorkloadPolicy `json:"workloadPolicy,omitempty"` + // +kubebuilder:validation:Minimum=0 + GPUCount int32 `json:"gpuCount,omitempty"` + // +kubebuilder:validation:MaxProperties=32 + ScriptEnvironment map[string]string `json:"scriptEnvironment,omitempty"` + BackendActiveDeadline metav1.Duration `json:"backendActiveDeadline"` + ParticipantPolicy CheckParticipantPolicy `json:"participantPolicy,omitempty"` +} + +// +kubebuilder:validation:Enum=Pod;MPIJob +type ExecutionBackend string + +const ( + ExecutionBackendPod ExecutionBackend = "Pod" + ExecutionBackendMPIJob ExecutionBackend = "MPIJob" +) + +type ParticipantPolicy struct { + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=64 + Count int32 `json:"count"` + + Selector metav1.LabelSelector `json:"selector"` + + MinimumAllocatable corev1.ResourceList `json:"minimumAllocatable,omitempty"` +} + +type WorkloadHostPath struct { + MountPath string `json:"mountPath"` + Path string `json:"path"` +} + +type WorkloadPolicy struct { + Resources corev1.ResourceRequirements `json:"resources,omitempty"` + Privileged bool `json:"privileged,omitempty"` + Capabilities []corev1.Capability `json:"capabilities,omitempty"` + HostNetwork bool `json:"hostNetwork,omitempty"` + WorkingDir string `json:"workingDir,omitempty"` + // +kubebuilder:validation:MaxItems=16 + HostPaths []WorkloadHostPath `json:"hostPaths,omitempty"` +} + +// +kubebuilder:validation:XValidation:rule="self.checks.all(c, duration(self.deadline) - duration(c.backendActiveDeadline) >= duration('60s'))",message="every backendActiveDeadline must leave at least 60 seconds before deadline" +// +kubebuilder:validation:XValidation:rule="self.checks.map(c, duration(c.backendActiveDeadline) + duration('60s')).sum() <= duration(self.deadline)",message="deadline must cover every sequential backendActiveDeadline plus 60 seconds of evidence time per check" +type CertificationProfileSpec struct { + Participants ParticipantPolicy `json:"participants"` + // +kubebuilder:validation:MaxItems=16 + ParticipantRoles []ParticipantRoleConstraint `json:"participantRoles,omitempty"` + // +kubebuilder:validation:MaxItems=16 + TopologyConstraints []ParticipantTopologyConstraint `json:"topologyConstraints,omitempty"` + // +kubebuilder:validation:MinItems=15 + // +kubebuilder:validation:MaxItems=15 + Checks []CertificationCheck `json:"checks"` + Deadline metav1.Duration `json:"deadline"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster,shortName=certprofile +// +kubebuilder:printcolumn:name="Checks",type=integer,JSONPath=`.spec.checks.size()` +// +kubebuilder:printcolumn:name="Participants",type=integer,JSONPath=`.spec.participants.count` +type CertificationProfile struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CertificationProfileSpec `json:"spec"` +} + +// +kubebuilder:object:root=true +type CertificationProfileList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CertificationProfile `json:"items"` +} diff --git a/api/signal/v1alpha1/certificationrun_types.go b/api/signal/v1alpha1/certificationrun_types.go new file mode 100644 index 0000000000..c9dfc817e8 --- /dev/null +++ b/api/signal/v1alpha1/certificationrun_types.go @@ -0,0 +1,211 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +// +kubebuilder:validation:Enum=Pending;Running;Succeeded;Failed;Inconclusive;Canceled +type RunPhase string + +const ( + ReservationNodeUIDLabel = "signal.unbounded-cloud.io/reserved-node-uid" + RunConditionCordoned = "Cordoned" + RunConditionCleanup = "WorkloadCleanupComplete" + RunConditionReleased = "Released" + + RunPhasePending RunPhase = "Pending" + RunPhaseRunning RunPhase = "Running" + RunPhaseSucceeded RunPhase = "Succeeded" + RunPhaseFailed RunPhase = "Failed" + RunPhaseInconclusive RunPhase = "Inconclusive" + RunPhaseCancelled RunPhase = "Canceled" +) + +func (p RunPhase) Terminal() bool { + switch p { + case RunPhaseSucceeded, RunPhaseFailed, RunPhaseInconclusive, RunPhaseCancelled: + return true + default: + return false + } +} + +type Participant struct { + // Index is derived from the participant's position and is never serialized. + Index int32 `json:"-"` + NodeName string `json:"nodeName"` + NodeUID types.UID `json:"nodeUID"` + Allocatable corev1.ResourceList `json:"allocatable,omitempty"` + Role string `json:"role,omitempty"` + // +kubebuilder:validation:MaxProperties=16 + TopologyValues map[string]string `json:"topologyValues,omitempty"` +} + +type BackendResourceReference struct { + Name string `json:"name"` + UID types.UID `json:"uid"` + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=64 + ParticipantIndexes []int32 `json:"participantIndexes"` +} + +type PinnedProfile struct { + Name string `json:"name"` + UID types.UID `json:"uid"` + Generation int64 `json:"generation"` +} + +type ScriptBundleReference struct { + Name string `json:"name"` + UID types.UID `json:"uid"` +} + +type ScriptSnapshot struct { + Key string `json:"key"` + // Digest is the algorithm-qualified SHA-256 digest of the selected script bytes. + // +kubebuilder:validation:Pattern=`^sha256:[0-9a-f]{64}$` + Digest string `json:"digest"` +} + +// +kubebuilder:validation:XValidation:rule="duration(self.backendActiveDeadline) > duration('0s')",message="backendActiveDeadline must be positive" +// +kubebuilder:validation:XValidation:rule="!has(self.scriptEnvironment) || self.scriptEnvironment.all(k, size(k) <= 253 && size(self.scriptEnvironment[k]) <= 4096)",message="scriptEnvironment names must not exceed 253 characters and values must not exceed 4096 characters" +type CertificationCheckSnapshot struct { + // ID and Order are derived from the check's position and are never serialized. + ID CheckID `json:"-"` + Order int32 `json:"-"` + Script ScriptSnapshot `json:"script"` + RunnerImage string `json:"runnerImage"` + Backend ExecutionBackend `json:"backend"` + WorkloadPolicy WorkloadPolicy `json:"workloadPolicy,omitempty"` + GPUCount int32 `json:"gpuCount,omitempty"` + // +kubebuilder:validation:MaxProperties=32 + ScriptEnvironment map[string]string `json:"scriptEnvironment,omitempty"` + BackendActiveDeadline metav1.Duration `json:"backendActiveDeadline"` + ParticipantPolicy CheckParticipantPolicy `json:"participantPolicy,omitempty"` +} + +// +kubebuilder:validation:XValidation:rule="self.profile == oldSelf.profile",message="profile is immutable" +// +kubebuilder:validation:XValidation:rule="self.participants == oldSelf.participants",message="participants are immutable" +// +kubebuilder:validation:XValidation:rule="self.scriptBundle == oldSelf.scriptBundle",message="scriptBundle is immutable" +// +kubebuilder:validation:XValidation:rule="self.checks == oldSelf.checks",message="checks are immutable" +// +kubebuilder:validation:XValidation:rule="self.deadline == oldSelf.deadline",message="deadline is immutable" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.cancel) || !oldSelf.cancel || (has(self.cancel) && self.cancel)",message="cancel cannot be cleared" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.release) || !oldSelf.release || (has(self.release) && self.release)",message="release cannot be cleared" +// +kubebuilder:validation:XValidation:rule="self.checks.all(c, duration(self.deadline) - duration(c.backendActiveDeadline) >= duration('60s'))",message="every backendActiveDeadline must leave at least 60 seconds before deadline" +// +kubebuilder:validation:XValidation:rule="self.checks.map(c, duration(c.backendActiveDeadline) + duration('60s')).sum() <= duration(self.deadline)",message="deadline must cover every sequential backendActiveDeadline plus 60 seconds of evidence time per check" +type CertificationRunSpec struct { + Profile PinnedProfile `json:"profile"` + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=64 + Participants []Participant `json:"participants"` + ScriptBundle ScriptBundleReference `json:"scriptBundle"` + // +kubebuilder:validation:MinItems=15 + // +kubebuilder:validation:MaxItems=15 + Checks []CertificationCheckSnapshot `json:"checks"` + Deadline metav1.Duration `json:"deadline"` + Cancel bool `json:"cancel,omitempty"` + Release bool `json:"release,omitempty"` +} + +type WorkflowStatus struct { + Name string `json:"name,omitempty"` + UID string `json:"uid,omitempty"` + Phase string `json:"phase,omitempty"` +} + +type EvidenceStatus struct { + URI string `json:"uri,omitempty"` + // Digest is the algorithm-qualified SHA-256 digest of the evidence bytes. + // +kubebuilder:validation:Pattern=`^sha256:[0-9a-f]{64}$` + Digest string `json:"digest,omitempty"` +} + +// +kubebuilder:validation:Enum=Pending;Running;Succeeded;Failed;Inconclusive;Skipped +type CheckPhase string + +const ( + CheckPhasePending CheckPhase = "Pending" + CheckPhaseRunning CheckPhase = "Running" + CheckPhaseSucceeded CheckPhase = "Succeeded" + CheckPhaseFailed CheckPhase = "Failed" + CheckPhaseInconclusive CheckPhase = "Inconclusive" + CheckPhaseSkipped CheckPhase = "Skipped" +) + +type BackendEvidenceStatus struct { + Complete bool `json:"complete,omitempty"` + // Digest identifies the complete backend-evidence v1 document retained in + // the durable evidence artifact. + // +kubebuilder:validation:Pattern=`^sha256:[0-9a-f]{64}$` + Digest string `json:"digest,omitempty"` + // Content is transient input for the durable evidence artifact. + Content string `json:"-"` +} + +type CheckCleanupStatus struct { + Complete bool `json:"-"` + Reason string `json:"-"` +} + +type CheckStatus struct { + // Internal fields are derived or transient and are never persisted in CR status. + ID CheckID `json:"-"` + Order int32 `json:"-"` + WorkflowNodeName string `json:"-"` + Phase CheckPhase `json:"phase"` + Verdict string `json:"-"` + // +kubebuilder:validation:MaxLength=1024 + Reason string `json:"reason,omitempty"` + // +kubebuilder:validation:MaxLength=16384 + Measurements string `json:"measurements,omitempty"` + // +kubebuilder:validation:MaxItems=64 + BackendResources []BackendResourceReference `json:"backendResources,omitempty"` + BackendEvidence BackendEvidenceStatus `json:"backendEvidence,omitempty"` + StartedAt *metav1.Time `json:"startedAt,omitempty"` + FinishedAt *metav1.Time `json:"finishedAt,omitempty"` + Cleanup CheckCleanupStatus `json:"-"` +} + +type CertificationRunStatus struct { + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + Phase RunPhase `json:"phase,omitempty"` + // +kubebuilder:validation:MaxLength=1024 + Reason string `json:"reason,omitempty"` + Workflow WorkflowStatus `json:"workflow,omitempty"` + Evidence EvidenceStatus `json:"evidence,omitempty"` + // +kubebuilder:validation:MaxItems=15 + Checks []CheckStatus `json:"checks,omitempty"` + StartedAt *metav1.Time `json:"startedAt,omitempty"` + FinishedAt *metav1.Time `json:"finishedAt,omitempty"` + // +listType=map + // +listMapKey=type + // +kubebuilder:validation:MaxItems=8 + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Namespaced,shortName=certrun +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Profile",type=string,JSONPath=`.spec.profile.name` +// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` +// +kubebuilder:printcolumn:name="Workflow",type=string,JSONPath=`.status.workflow.name` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` +type CertificationRun struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CertificationRunSpec `json:"spec"` + Status CertificationRunStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true +type CertificationRunList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CertificationRun `json:"items"` +} diff --git a/api/signal/v1alpha1/digest.go b/api/signal/v1alpha1/digest.go new file mode 100644 index 0000000000..981dc0cba4 --- /dev/null +++ b/api/signal/v1alpha1/digest.go @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import "regexp" + +// SHA256DigestPattern is the canonical Project Signal digest representation. +const SHA256DigestPattern = `^sha256:[0-9a-f]{64}$` + +var sha256DigestPattern = regexp.MustCompile(SHA256DigestPattern) + +// IsSHA256Digest reports whether value is an algorithm-qualified SHA-256 digest. +func IsSHA256Digest(value string) bool { + return sha256DigestPattern.MatchString(value) +} diff --git a/api/signal/v1alpha1/groupversion_info.go b/api/signal/v1alpha1/groupversion_info.go new file mode 100644 index 0000000000..0ce40be47d --- /dev/null +++ b/api/signal/v1alpha1/groupversion_info.go @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +// +kubebuilder:object:generate=true +// +groupName=signal.unbounded-cloud.io +package v1alpha1 + +//go:generate go tool controller-gen object:headerFile=../../../hack/boilerplate.go.txt paths=. +//go:generate go tool controller-gen crd:headerFile=../../../hack/boilerplate.yaml.txt paths=. output:crd:dir=../../../deploy/signal/crd diff --git a/api/signal/v1alpha1/register.go b/api/signal/v1alpha1/register.go new file mode 100644 index 0000000000..11130bcf55 --- /dev/null +++ b/api/signal/v1alpha1/register.go @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +const GroupName = "signal.unbounded-cloud.io" + +var ( + GroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha1"} + SchemeBuilder = runtime.NewSchemeBuilder(func(s *runtime.Scheme) error { + s.AddKnownTypes( + GroupVersion, + &CertificationProfile{}, + &CertificationProfileList{}, + &CertificationRun{}, + &CertificationRunList{}, + ) + metav1.AddToGroupVersion(s, GroupVersion) + + return nil + }) + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/api/signal/v1alpha1/runtime_contract.go b/api/signal/v1alpha1/runtime_contract.go new file mode 100644 index 0000000000..1cb10c4648 --- /dev/null +++ b/api/signal/v1alpha1/runtime_contract.go @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + "errors" + "fmt" + "time" +) + +const MinimumBackendDeadlineReserve = time.Minute + +func ValidateExecutionDeadlines(backendDeadline, outerDeadline time.Duration) error { + if backendDeadline <= 0 { + return errors.New("backend active deadline must be positive") + } + + if backendDeadline%time.Second != 0 || outerDeadline%time.Second != 0 { + return errors.New("backend and outer deadlines must use whole seconds") + } + + if outerDeadline-backendDeadline < MinimumBackendDeadlineReserve { + return fmt.Errorf( + "backend active deadline must leave at least %s before the outer deadline", + MinimumBackendDeadlineReserve, + ) + } + + return nil +} + +func ValidateSuiteDeadlines(backendDeadlines []time.Duration, suiteDeadline time.Duration) error { + if suiteDeadline <= 0 || suiteDeadline%time.Second != 0 { + return errors.New("suite deadline must be a positive whole number of seconds") + } + + var required time.Duration + + for _, backendDeadline := range backendDeadlines { + if backendDeadline <= 0 || backendDeadline%time.Second != 0 { + return errors.New("backend active deadlines must be positive whole numbers of seconds") + } + + if backendDeadline > time.Duration(1<<63-1)-required-MinimumBackendDeadlineReserve { + return errors.New("combined check deadlines overflow") + } + + required += backendDeadline + MinimumBackendDeadlineReserve + } + + if required > suiteDeadline { + return fmt.Errorf( + "combined backend deadlines and evidence reserves require %s, suite deadline is %s", + required, + suiteDeadline, + ) + } + + return nil +} diff --git a/api/signal/v1alpha1/zz_generated.deepcopy.go b/api/signal/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000000..2dbb4f175e --- /dev/null +++ b/api/signal/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,606 @@ +//go:build !ignore_autogenerated + +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackendEvidenceStatus) DeepCopyInto(out *BackendEvidenceStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackendEvidenceStatus. +func (in *BackendEvidenceStatus) DeepCopy() *BackendEvidenceStatus { + if in == nil { + return nil + } + out := new(BackendEvidenceStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackendResourceReference) DeepCopyInto(out *BackendResourceReference) { + *out = *in + if in.ParticipantIndexes != nil { + in, out := &in.ParticipantIndexes, &out.ParticipantIndexes + *out = make([]int32, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackendResourceReference. +func (in *BackendResourceReference) DeepCopy() *BackendResourceReference { + if in == nil { + return nil + } + out := new(BackendResourceReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationCheck) DeepCopyInto(out *CertificationCheck) { + *out = *in + out.Script = in.Script + in.WorkloadPolicy.DeepCopyInto(&out.WorkloadPolicy) + if in.ScriptEnvironment != nil { + in, out := &in.ScriptEnvironment, &out.ScriptEnvironment + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + out.BackendActiveDeadline = in.BackendActiveDeadline + in.ParticipantPolicy.DeepCopyInto(&out.ParticipantPolicy) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationCheck. +func (in *CertificationCheck) DeepCopy() *CertificationCheck { + if in == nil { + return nil + } + out := new(CertificationCheck) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationCheckSnapshot) DeepCopyInto(out *CertificationCheckSnapshot) { + *out = *in + out.Script = in.Script + in.WorkloadPolicy.DeepCopyInto(&out.WorkloadPolicy) + if in.ScriptEnvironment != nil { + in, out := &in.ScriptEnvironment, &out.ScriptEnvironment + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + out.BackendActiveDeadline = in.BackendActiveDeadline + in.ParticipantPolicy.DeepCopyInto(&out.ParticipantPolicy) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationCheckSnapshot. +func (in *CertificationCheckSnapshot) DeepCopy() *CertificationCheckSnapshot { + if in == nil { + return nil + } + out := new(CertificationCheckSnapshot) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationProfile) DeepCopyInto(out *CertificationProfile) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationProfile. +func (in *CertificationProfile) DeepCopy() *CertificationProfile { + if in == nil { + return nil + } + out := new(CertificationProfile) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CertificationProfile) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationProfileList) DeepCopyInto(out *CertificationProfileList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CertificationProfile, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationProfileList. +func (in *CertificationProfileList) DeepCopy() *CertificationProfileList { + if in == nil { + return nil + } + out := new(CertificationProfileList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CertificationProfileList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationProfileSpec) DeepCopyInto(out *CertificationProfileSpec) { + *out = *in + in.Participants.DeepCopyInto(&out.Participants) + if in.ParticipantRoles != nil { + in, out := &in.ParticipantRoles, &out.ParticipantRoles + *out = make([]ParticipantRoleConstraint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.TopologyConstraints != nil { + in, out := &in.TopologyConstraints, &out.TopologyConstraints + *out = make([]ParticipantTopologyConstraint, len(*in)) + copy(*out, *in) + } + if in.Checks != nil { + in, out := &in.Checks, &out.Checks + *out = make([]CertificationCheck, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + out.Deadline = in.Deadline +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationProfileSpec. +func (in *CertificationProfileSpec) DeepCopy() *CertificationProfileSpec { + if in == nil { + return nil + } + out := new(CertificationProfileSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationRun) DeepCopyInto(out *CertificationRun) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationRun. +func (in *CertificationRun) DeepCopy() *CertificationRun { + if in == nil { + return nil + } + out := new(CertificationRun) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CertificationRun) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationRunList) DeepCopyInto(out *CertificationRunList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CertificationRun, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationRunList. +func (in *CertificationRunList) DeepCopy() *CertificationRunList { + if in == nil { + return nil + } + out := new(CertificationRunList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CertificationRunList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationRunSpec) DeepCopyInto(out *CertificationRunSpec) { + *out = *in + out.Profile = in.Profile + if in.Participants != nil { + in, out := &in.Participants, &out.Participants + *out = make([]Participant, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + out.ScriptBundle = in.ScriptBundle + if in.Checks != nil { + in, out := &in.Checks, &out.Checks + *out = make([]CertificationCheckSnapshot, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + out.Deadline = in.Deadline +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationRunSpec. +func (in *CertificationRunSpec) DeepCopy() *CertificationRunSpec { + if in == nil { + return nil + } + out := new(CertificationRunSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertificationRunStatus) DeepCopyInto(out *CertificationRunStatus) { + *out = *in + out.Workflow = in.Workflow + out.Evidence = in.Evidence + if in.Checks != nil { + in, out := &in.Checks, &out.Checks + *out = make([]CheckStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.StartedAt != nil { + in, out := &in.StartedAt, &out.StartedAt + *out = (*in).DeepCopy() + } + if in.FinishedAt != nil { + in, out := &in.FinishedAt, &out.FinishedAt + *out = (*in).DeepCopy() + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertificationRunStatus. +func (in *CertificationRunStatus) DeepCopy() *CertificationRunStatus { + if in == nil { + return nil + } + out := new(CertificationRunStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CheckCleanupStatus) DeepCopyInto(out *CheckCleanupStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CheckCleanupStatus. +func (in *CheckCleanupStatus) DeepCopy() *CheckCleanupStatus { + if in == nil { + return nil + } + out := new(CheckCleanupStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CheckParticipantPolicy) DeepCopyInto(out *CheckParticipantPolicy) { + *out = *in + if in.Roles != nil { + in, out := &in.Roles, &out.Roles + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CheckParticipantPolicy. +func (in *CheckParticipantPolicy) DeepCopy() *CheckParticipantPolicy { + if in == nil { + return nil + } + out := new(CheckParticipantPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CheckStatus) DeepCopyInto(out *CheckStatus) { + *out = *in + if in.BackendResources != nil { + in, out := &in.BackendResources, &out.BackendResources + *out = make([]BackendResourceReference, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + out.BackendEvidence = in.BackendEvidence + if in.StartedAt != nil { + in, out := &in.StartedAt, &out.StartedAt + *out = (*in).DeepCopy() + } + if in.FinishedAt != nil { + in, out := &in.FinishedAt, &out.FinishedAt + *out = (*in).DeepCopy() + } + out.Cleanup = in.Cleanup +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CheckStatus. +func (in *CheckStatus) DeepCopy() *CheckStatus { + if in == nil { + return nil + } + out := new(CheckStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EvidenceStatus) DeepCopyInto(out *EvidenceStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EvidenceStatus. +func (in *EvidenceStatus) DeepCopy() *EvidenceStatus { + if in == nil { + return nil + } + out := new(EvidenceStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Participant) DeepCopyInto(out *Participant) { + *out = *in + if in.Allocatable != nil { + in, out := &in.Allocatable, &out.Allocatable + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + if in.TopologyValues != nil { + in, out := &in.TopologyValues, &out.TopologyValues + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Participant. +func (in *Participant) DeepCopy() *Participant { + if in == nil { + return nil + } + out := new(Participant) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ParticipantPolicy) DeepCopyInto(out *ParticipantPolicy) { + *out = *in + in.Selector.DeepCopyInto(&out.Selector) + if in.MinimumAllocatable != nil { + in, out := &in.MinimumAllocatable, &out.MinimumAllocatable + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ParticipantPolicy. +func (in *ParticipantPolicy) DeepCopy() *ParticipantPolicy { + if in == nil { + return nil + } + out := new(ParticipantPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ParticipantRoleConstraint) DeepCopyInto(out *ParticipantRoleConstraint) { + *out = *in + in.Selector.DeepCopyInto(&out.Selector) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ParticipantRoleConstraint. +func (in *ParticipantRoleConstraint) DeepCopy() *ParticipantRoleConstraint { + if in == nil { + return nil + } + out := new(ParticipantRoleConstraint) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ParticipantTopologyConstraint) DeepCopyInto(out *ParticipantTopologyConstraint) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ParticipantTopologyConstraint. +func (in *ParticipantTopologyConstraint) DeepCopy() *ParticipantTopologyConstraint { + if in == nil { + return nil + } + out := new(ParticipantTopologyConstraint) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PinnedProfile) DeepCopyInto(out *PinnedProfile) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PinnedProfile. +func (in *PinnedProfile) DeepCopy() *PinnedProfile { + if in == nil { + return nil + } + out := new(PinnedProfile) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ScriptBundleReference) DeepCopyInto(out *ScriptBundleReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScriptBundleReference. +func (in *ScriptBundleReference) DeepCopy() *ScriptBundleReference { + if in == nil { + return nil + } + out := new(ScriptBundleReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ScriptReference) DeepCopyInto(out *ScriptReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScriptReference. +func (in *ScriptReference) DeepCopy() *ScriptReference { + if in == nil { + return nil + } + out := new(ScriptReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ScriptSnapshot) DeepCopyInto(out *ScriptSnapshot) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScriptSnapshot. +func (in *ScriptSnapshot) DeepCopy() *ScriptSnapshot { + if in == nil { + return nil + } + out := new(ScriptSnapshot) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkflowStatus) DeepCopyInto(out *WorkflowStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkflowStatus. +func (in *WorkflowStatus) DeepCopy() *WorkflowStatus { + if in == nil { + return nil + } + out := new(WorkflowStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkloadHostPath) DeepCopyInto(out *WorkloadHostPath) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadHostPath. +func (in *WorkloadHostPath) DeepCopy() *WorkloadHostPath { + if in == nil { + return nil + } + out := new(WorkloadHostPath) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkloadPolicy) DeepCopyInto(out *WorkloadPolicy) { + *out = *in + in.Resources.DeepCopyInto(&out.Resources) + if in.Capabilities != nil { + in, out := &in.Capabilities, &out.Capabilities + *out = make([]v1.Capability, len(*in)) + copy(*out, *in) + } + if in.HostPaths != nil { + in, out := &in.HostPaths, &out.HostPaths + *out = make([]WorkloadHostPath, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadPolicy. +func (in *WorkloadPolicy) DeepCopy() *WorkloadPolicy { + if in == nil { + return nil + } + out := new(WorkloadPolicy) + in.DeepCopyInto(out) + return out +} diff --git a/cmd/project-signal-adapter/main.go b/cmd/project-signal-adapter/main.go new file mode 100644 index 0000000000..4b4b66ccb5 --- /dev/null +++ b/cmd/project-signal-adapter/main.go @@ -0,0 +1,192 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + osSignal "os/signal" + "path/filepath" + "syscall" + + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + "github.com/Azure/unbounded/internal/signaladapter" + "github.com/Azure/unbounded/internal/version" +) + +const adapterName = "project-signal-adapter" + +type config struct { + signaladapter.Config + outputDirectory string +} + +func main() { + var cfg config + + command := &cobra.Command{ + Use: adapterName, + Short: "Run participant-aware Project Signal backend workloads", + Version: version.Version + " (commit: " + version.GitCommit + ")", + RunE: func(cmd *cobra.Command, _ []string) error { + ctx, cancel := osSignal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + return run(ctx, cfg) + }, + } + + flags := command.Flags() + flags.StringVar(&cfg.Namespace, "namespace", "", "Namespace for backend resources") + flags.StringVar(&cfg.WorkflowName, "workflow-name", "", "Owning Argo Workflow name") + flags.StringVar(&cfg.RunName, "run-name", "", "CertificationRun name") + flags.Var(uidValue{target: &cfg.RunUID}, "run-uid", "CertificationRun UID") + flags.StringVar(&cfg.ParticipantsJSON, "participants-json", "", "Ordered participant snapshot JSON") + flags.IntVar(&cfg.ParticipantCount, "participant-count", 0, "Declared participant count") + flags.StringVar(&cfg.ScriptConfigMap, "script-configmap", "", "Immutable script ConfigMap") + flags.StringVar(&cfg.ScriptKey, "script-key", "", "Script ConfigMap key") + flags.StringVar( + &cfg.ScriptDigest, + "script-digest", + "", + "algorithm-qualified SHA-256 digest of the script bytes (sha256:<64 lowercase hex>)", + ) + flags.StringVar(&cfg.RunnerImage, "runner-image", "", "Digest-pinned runner image") + flags.StringVar(&cfg.BackendKind, "backend-kind", "", "Backend kind") + flags.Int64Var( + &cfg.BackendDeadlineSeconds, + "backend-active-deadline-seconds", + 0, + "Active deadline for participant backend workloads", + ) + flags.StringVar(&cfg.ScriptEnvironmentJSON, "script-environment", "{}", "JSON-encoded script environment") + flags.StringVar(&cfg.WorkloadPolicyJSON, "workload-policy", "{}", "JSON-encoded participant workload policy") + flags.StringVar(&cfg.GPUResourceCount, "gpu-resource-count", "0", "GPU resource limit per participant") + flags.StringVar(&cfg.outputDirectory, "output-directory", "/outputs", "Argo output directory") + + command.CompletionOptions.DisableDefaultCmd = true + + if err := command.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } +} + +func run(ctx context.Context, cfg config) error { + restConfig, err := rest.InClusterConfig() + if err != nil { + return fmt.Errorf("load in-cluster Kubernetes configuration: %w", err) + } + + dynamicClient, err := dynamic.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("create dynamic Kubernetes client: %w", err) + } + + kubernetesClient, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("create Kubernetes client: %w", err) + } + + runner := &signaladapter.Runner{ + Dynamic: dynamicClient, + Kubernetes: kubernetesClient, + PersistBackendEvidence: func(content string) error { + if err := signaladapter.ValidateBackendEvidence(content); err != nil { + return err + } + + if err := os.MkdirAll(cfg.outputDirectory, 0o750); err != nil { + return fmt.Errorf("create output directory: %w", err) + } + + return writeOutput(cfg.outputDirectory, "backend-evidence", content) + }, + } + + output, err := runner.Execute(ctx, cfg.Config) + if err != nil { + return err + } + + return writeOutputs(cfg.outputDirectory, output) +} + +func writeOutputs(directory string, output signaladapter.AggregateOutput) error { + if directory == "" { + return errors.New("output-directory is required") + } + + if err := signaladapter.ValidateBackendEvidence(output.BackendEvidence); err != nil { + return err + } + + if err := os.MkdirAll(directory, 0o750); err != nil { + return fmt.Errorf("create output directory: %w", err) + } + + outputs := map[string]string{ + "verdict": output.Verdict, + "reason": output.Reason, + "measurements": output.Measurements, + "backend-resources": output.BackendJSON, + "backend-evidence": output.BackendEvidence, + } + for name, value := range outputs { + if err := writeOutput(directory, name, value); err != nil { + return err + } + } + + evidence, err := json.Marshal(output) + if err != nil { + return fmt.Errorf("encode adapter result: %w", err) + } + + fmt.Println(string(evidence)) + + return nil +} + +func writeOutput(directory, name, value string) error { + if err := os.WriteFile(filepath.Join(directory, name), []byte(value), 0o600); err != nil { + return fmt.Errorf("write %s output: %w", name, err) + } + + return nil +} + +type uidValue struct { + target *types.UID +} + +func (v uidValue) String() string { + if v.target == nil { + return "" + } + + return string(*v.target) +} + +func (v uidValue) Set(value string) error { + if v.target == nil { + return errors.New("UID target is nil") + } + + *v.target = types.UID(value) + + return nil +} + +func (uidValue) Type() string { + return "uid" +} diff --git a/cmd/project-signal-controller/main.go b/cmd/project-signal-controller/main.go new file mode 100644 index 0000000000..b8afa3bf80 --- /dev/null +++ b/cmd/project-signal-controller/main.go @@ -0,0 +1,144 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "errors" + "fmt" + "os" + osSignal "os/signal" + "syscall" + + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" + projectsignal "github.com/Azure/unbounded/internal/signal" + "github.com/Azure/unbounded/internal/version" +) + +const controllerName = "project-signal-controller" + +type config struct { + namespace string + scriptNamespace string + metricsAddress string + probeAddress string + leaderElect bool + maxConcurrentRuns int +} + +func main() { + var cfg config + + cmd := &cobra.Command{ + Use: controllerName, + Short: "Project Signal node certification controller", + RunE: func(cmd *cobra.Command, _ []string) error { + ctx, cancel := osSignal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + return run(ctx, cfg) + }, + Version: version.Version + " (commit: " + version.GitCommit + ")", + } + + cmd.Flags().StringVar(&cfg.namespace, "namespace", "project-signal", "Namespace for CertificationRuns and Workflows") + cmd.Flags().StringVar(&cfg.scriptNamespace, "script-namespace", "project-signal", "Namespace containing mutable source script ConfigMaps") + cmd.Flags().StringVar(&cfg.metricsAddress, "metrics-bind-address", ":8080", "Address for metrics") + cmd.Flags().StringVar(&cfg.probeAddress, "health-probe-bind-address", ":8081", "Address for health probes") + cmd.Flags().BoolVar(&cfg.leaderElect, "leader-elect", true, "Enable leader election") + cmd.Flags().IntVar(&cfg.maxConcurrentRuns, "max-concurrent-runs", 4, "Maximum concurrent CertificationRun reconciles") + cmd.CompletionOptions.DisableDefaultCmd = true + + if err := cmd.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } +} + +func run(ctx context.Context, cfg config) error { + ctrl.SetLogger(zap.New()) + + if cfg.namespace == "" || cfg.scriptNamespace == "" { + return errors.New("namespace and script namespace are required") + } + + scheme := runtime.NewScheme() + if err := clientgoscheme.AddToScheme(scheme); err != nil { + return err + } + + if err := signalv1.AddToScheme(scheme); err != nil { + return err + } + + watchedNamespaces := map[string]cache.Config{cfg.namespace: {}} + watchedNamespaces[cfg.scriptNamespace] = cache.Config{} + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + Cache: cache.Options{DefaultNamespaces: watchedNamespaces}, + Metrics: metricsserver.Options{BindAddress: cfg.metricsAddress}, + HealthProbeBindAddress: cfg.probeAddress, + LeaderElection: cfg.leaderElect, + LeaderElectionID: controllerName, + LeaderElectionNamespace: cfg.namespace, + LeaderElectionReleaseOnCancel: true, + }) + if err != nil { + return fmt.Errorf("create manager: %w", err) + } + + evidence, err := projectsignal.NewKubernetesEvidenceStore(mgr.GetClient(), cfg.namespace) + if err != nil { + return err + } + + if _, err := mgr.GetRESTMapper().RESTMapping(schema.GroupKind{Group: "argoproj.io", Kind: "Workflow"}, "v1alpha1"); err != nil { + return fmt.Errorf("argo Workflows is required but Workflow.argoproj.io/v1alpha1 is unavailable: %w", err) + } + + admission := &projectsignal.AdmissionReconciler{ + Client: mgr.GetClient(), + APIReader: mgr.GetAPIReader(), + Namespace: cfg.namespace, + ScriptNamespace: cfg.scriptNamespace, + Recorder: mgr.GetEventRecorder(controllerName), + } + if err := admission.SetupWithManager(mgr); err != nil { + return fmt.Errorf("register admission reconciler: %w", err) + } + + runs := &projectsignal.RunReconciler{ + Client: mgr.GetClient(), + APIReader: mgr.GetAPIReader(), + Evidence: evidence, + MaxConcurrentRuns: cfg.maxConcurrentRuns, + Recorder: mgr.GetEventRecorder(controllerName), + } + if err := runs.SetupWithManager(mgr); err != nil { + return fmt.Errorf("register run reconciler: %w", err) + } + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + return err + } + + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + return err + } + + ctrl.Log.Info("starting Project Signal controller", "version", version.String(), "namespace", cfg.namespace) + + return mgr.Start(ctx) +} diff --git a/deploy/signal/00-namespace.yaml.tmpl b/deploy/signal/00-namespace.yaml.tmpl new file mode 100644 index 0000000000..b47ae5cc04 --- /dev/null +++ b/deploy/signal/00-namespace.yaml.tmpl @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: {{ default "project-signal" .Namespace }} + labels: + app.kubernetes.io/part-of: project-signal diff --git a/deploy/signal/01-serviceaccounts.yaml.tmpl b/deploy/signal/01-serviceaccounts.yaml.tmpl new file mode 100644 index 0000000000..e0efbbe38e --- /dev/null +++ b/deploy/signal/01-serviceaccounts.yaml.tmpl @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: project-signal-controller + namespace: {{ default "project-signal" .Namespace }} + labels: + app.kubernetes.io/name: project-signal-controller +automountServiceAccountToken: true diff --git a/deploy/signal/02-rbac.yaml.tmpl b/deploy/signal/02-rbac.yaml.tmpl new file mode 100644 index 0000000000..e834a22f7c --- /dev/null +++ b/deploy/signal/02-rbac.yaml.tmpl @@ -0,0 +1,116 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: project-signal-controller +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["signal.unbounded-cloud.io"] + resources: ["certificationprofiles"] + verbs: ["get", "list", "watch"] + - apiGroups: ["signal.unbounded-cloud.io"] + resources: ["certificationruns"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["signal.unbounded-cloud.io"] + resources: ["certificationruns/status"] + verbs: ["get", "update", "patch"] + - apiGroups: ["signal.unbounded-cloud.io"] + resources: ["certificationruns/finalizers"] + verbs: ["update"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: project-signal-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: project-signal-controller +subjects: + - kind: ServiceAccount + name: project-signal-controller + namespace: {{ default "project-signal" .Namespace }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: project-signal-controller + namespace: {{ default "project-signal" .Namespace }} +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "patch"] + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "list", "patch"] + - apiGroups: ["kubeflow.org"] + resources: ["mpijobs"] + verbs: ["get", "list", "patch"] + - apiGroups: ["batch"] + resources: ["jobs/finalizers"] + verbs: ["update"] + - apiGroups: ["kubeflow.org"] + resources: ["mpijobs/finalizers"] + verbs: ["update"] + - apiGroups: ["argoproj.io"] + resources: ["workflows/finalizers"] + verbs: ["update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] + - apiGroups: ["events.k8s.io"] + resources: ["events"] + verbs: ["create", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["argoproj.io"] + resources: ["workflows"] + verbs: ["get", "list", "watch", "create", "delete"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: project-signal-controller + namespace: {{ default "project-signal" .Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: project-signal-controller +subjects: + - kind: ServiceAccount + name: project-signal-controller + namespace: {{ default "project-signal" .Namespace }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: project-signal-script-reader + namespace: {{ default (default "project-signal" .Namespace) .ScriptNamespace }} +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: project-signal-script-reader + namespace: {{ default (default "project-signal" .Namespace) .ScriptNamespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: project-signal-script-reader +subjects: + - kind: ServiceAccount + name: project-signal-controller + namespace: {{ default "project-signal" .Namespace }} diff --git a/deploy/signal/03-adapter-serviceaccount-rbac.yaml.tmpl b/deploy/signal/03-adapter-serviceaccount-rbac.yaml.tmpl new file mode 100644 index 0000000000..7a32f5ca0e --- /dev/null +++ b/deploy/signal/03-adapter-serviceaccount-rbac.yaml.tmpl @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: project-signal-adapter + namespace: {{ default "project-signal" .Namespace }} + labels: + app.kubernetes.io/name: project-signal-adapter +automountServiceAccountToken: true + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: project-signal-adapter + namespace: {{ default "project-signal" .Namespace }} +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get"] + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "list"] + - apiGroups: ["kubeflow.org"] + resources: ["mpijobs"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["argoproj.io"] + resources: ["workflows"] + verbs: ["get"] + - apiGroups: ["argoproj.io"] + resources: ["workflowtaskresults"] + verbs: ["create", "patch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: project-signal-adapter + namespace: {{ default "project-signal" .Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: project-signal-adapter +subjects: + - kind: ServiceAccount + name: project-signal-adapter + namespace: {{ default "project-signal" .Namespace }} diff --git a/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl b/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl new file mode 100644 index 0000000000..23a536e86c --- /dev/null +++ b/deploy/signal/04-adapter-workflowtemplate.yaml.tmpl @@ -0,0 +1,94 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: project-signal-participant-adapter + namespace: {{ default "project-signal" .Namespace }} +spec: + serviceAccountName: project-signal-adapter + templates: + - name: certification + inputs: + parameters: + - name: run-name + - name: run-uid + - name: participants-json + - name: participant-count + - name: script-configmap + - name: script-key + - name: script-digest + - name: backend-kind + - name: backend-active-deadline-seconds + - name: runner-image + - name: script-environment + - name: workload-policy + - name: gpu-resource-count + tolerations: + - key: node.kubernetes.io/unschedulable + operator: Exists + effect: NoSchedule + container: + image: {{ default "example.invalid/project-signal-adapter@sha256:0000000000000000000000000000000000000000000000000000000000000000" .AdapterImage }} + imagePullPolicy: IfNotPresent + args: + - --namespace + - "$(POD_NAMESPACE)" + - --workflow-name + - "$(WORKFLOW_NAME)" + - --run-name + - "{{`{{inputs.parameters.run-name}}`}}" + - --run-uid + - "{{`{{inputs.parameters.run-uid}}`}}" + - --participants-json + - "{{`{{inputs.parameters.participants-json}}`}}" + - --participant-count + - "{{`{{inputs.parameters.participant-count}}`}}" + - --script-configmap + - "{{`{{inputs.parameters.script-configmap}}`}}" + - --script-key + - "{{`{{inputs.parameters.script-key}}`}}" + - --script-digest + - "{{`{{inputs.parameters.script-digest}}`}}" + - --runner-image + - "{{`{{inputs.parameters.runner-image}}`}}" + - --backend-kind + - "{{`{{inputs.parameters.backend-kind}}`}}" + - --backend-active-deadline-seconds + - "{{`{{inputs.parameters.backend-active-deadline-seconds}}`}}" + - --script-environment + - "{{`{{inputs.parameters.script-environment}}`}}" + - --workload-policy + - "{{`{{inputs.parameters.workload-policy}}`}}" + - --gpu-resource-count + - "{{`{{inputs.parameters.gpu-resource-count}}`}}" + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: WORKFLOW_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['workflows.argoproj.io/workflow'] + volumeMounts: + - name: outputs + mountPath: /outputs + volumes: + - name: outputs + emptyDir: {} + outputs: + parameters: + - name: verdict + valueFrom: + path: /outputs/verdict + - name: reason + valueFrom: + path: /outputs/reason + - name: measurements + valueFrom: + path: /outputs/measurements + - name: backend-resources + valueFrom: + path: /outputs/backend-resources + - name: backend-evidence + valueFrom: + path: /outputs/backend-evidence diff --git a/deploy/signal/06-deployment.yaml.tmpl b/deploy/signal/06-deployment.yaml.tmpl new file mode 100644 index 0000000000..5763f36013 --- /dev/null +++ b/deploy/signal/06-deployment.yaml.tmpl @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: project-signal-controller + namespace: {{ default "project-signal" .Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: project-signal-controller + template: + metadata: + labels: + app.kubernetes.io/name: project-signal-controller + spec: + serviceAccountName: project-signal-controller + containers: + - name: controller + image: {{ default "example.invalid/project-signal-controller:replace-me" .ControllerImage }} + imagePullPolicy: IfNotPresent + args: + - --namespace={{ default "project-signal" .Namespace }} + - --script-namespace={{ default (default "project-signal" .Namespace) .ScriptNamespace }} + - --leader-elect=true + ports: + - name: metrics + containerPort: 8080 + - name: health + containerPort: 8081 + resources: + requests: + cpu: 50m + memory: 96Mi + limits: + cpu: 500m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + capabilities: + drop: [ALL] + livenessProbe: + httpGet: + path: /healthz + port: health + readinessProbe: + httpGet: + path: /readyz + port: health diff --git a/deploy/signal/README.md b/deploy/signal/README.md new file mode 100644 index 0000000000..bc808d10c7 --- /dev/null +++ b/deploy/signal/README.md @@ -0,0 +1,252 @@ +# Project Signal controller + +Project Signal performs one certification run for a newly joined Node or a +small, explicitly selected group of Nodes. The controller snapshots the +approved profile and scripts, reserves and cordons every participant, runs 15 +ordered checks through Argo Workflows, persists one immutable evidence +document, and cleans up all temporary workloads. + +Project Signal does not integrate with Kueue. Queue admission and repeatable +fleet scheduling are intentionally deferred. + +## Components + +- `CertificationProfile` is cluster-scoped operator policy. +- `CertificationRun` is a namespaced, immutable execution snapshot. +- `project-signal-controller` admits Nodes, owns reservations and cordons, + creates the Workflow, persists status and evidence, and performs teardown. +- `project-signal-participant-adapter` creates Pod or MPIJob backends and + captures typed evidence. Ordinary Pods and completed MPI resources remain + until controller teardown; the MPI Operator removes still-running workers + after launcher completion so later sequential checks can use the GPUs. + On setup, observation, or wall-clock deadline failures, the adapter first + persists the available evidence and then quiesces uncertain backends. +- `project-signal-participant-adapter` is a fixed WorkflowTemplate used by + every check. Profiles cannot replace it or pass arbitrary Workflow wiring. + +Argo Workflows is required. The Kubeflow MPI Operator is required only when a +profile contains an `MPIJob` check. + +## First deployment + +Render the manifests with immutable controller and adapter images: + +```bash +make signal-manifests \ + SIGNAL_CONTROLLER_IMAGE= \ + SIGNAL_ADAPTER_IMAGE=@sha256: +``` + +Install the rendered CRDs and namespaced resources: + +```bash +kubectl apply -f deploy/signal/rendered/crd/ +kubectl apply -f deploy/signal/rendered/ +``` + +The install creates: + +- the `project-signal` namespace; +- controller and adapter service accounts and RBAC; +- the fixed participant adapter WorkflowTemplate; +- the controller Deployment. + +The controller requires cluster permission to get, list, watch, and patch +Nodes because it owns participant reservation markers and cordons. + +## CertificationProfile contract + +A Profile contains only operator choices that legitimately vary between +certification suites: + +| Field | Meaning | +|---|---| +| `spec.participants.count` | Total Nodes selected for a Run | +| `spec.participants.selector` | Labels every selected Node must match | +| `spec.participants.minimumAllocatable` | Admission-time allocatable resource floor | +| `spec.participantRoles` | Optional named role counts and role-specific selectors | +| `spec.topologyConstraints` | Optional minimum topology diversity for a role | +| `spec.checks` | Exactly 15 ordered checks; position defines R01 through R15 | +| `spec.deadline` | Whole-suite deadline | + +If roles are configured, their counts must sum to the total participant count. +The requesting Node is participant zero. The remaining participants are +selected deterministically from currently ready, schedulable, unreserved +Nodes. + +Each check contains: + +| Field | Meaning | +|---|---| +| `script` | Source ConfigMap name, key, and approved SHA-256 digest | +| `runnerImage` | Digest-pinned check image | +| `backend` | `Pod` or `MPIJob` | +| `backendActiveDeadline` | Backend execution deadline | +| `gpuCount` | GPU request; must be positive for MPIJob | +| `scriptEnvironment` | At most 32 bounded, non-reserved environment values | +| `participantPolicy.roles` | Optional included roles; omission means all participants | +| `workloadPolicy` | Narrow container resources and host access policy | + +Workload policy is intentionally constrained: + +- the only added Linux capability is `IPC_LOCK`; +- host sources must be under `/proc`, `/sys`, or + `/var/lib/project-signal`; +- host mounts are read-only and their volume names and `Directory` type are + controller-owned; +- scheduling tolerations are controller-owned; +- arbitrary PodSpec, Workflow, retry, TTL, queue, and cleanup settings are not + accepted. + +Profile edits affect only future Runs. + +## Starting one certification + +Add the request label to a schedulable Node: + +```bash +kubectl label node \ + signal.unbounded-cloud.io/certification-request= +``` + +Admission then: + +1. validates the Profile and all 15 checks; +2. selects the requesting Node plus any additional participants; +3. atomically records the Run reservation name, participant Node UID, and + `spec.unschedulable=true` on each participant; +4. copies all verified scripts into one immutable Run-owned ConfigMap; +5. creates one immutable `CertificationRun`; +6. projects the pending certification state and marks admission ready. + +Ordinary workloads cannot schedule onto cordoned participants. Signal backend +Pods receive only the fixed GPU toleration and the fixed +`node.kubernetes.io/unschedulable` toleration needed to execute on the Nodes +reserved by this Run. Required Node affinity pins every backend to the admitted +Node name and UID. + +## CertificationRun contract + +The Run spec snapshots: + +- Profile name, UID, and generation; +- ordered participants with Node name, Node UID, admission-time allocatable + resources, optional role, and referenced topology values; +- the immutable script bundle name and UID; +- 15 ordered check snapshots; +- the suite deadline; +- monotonic `cancel` and `release` actions. + +Participant and check identity is positional. Participant zero is the +requesting Node. Check positions zero through fourteen are R01 through R15. +The controller derives these identities after every API read rather than +serializing duplicate index and ID fields. + +Run status contains: + +- `observedGeneration`; +- phase and bounded reason; +- Workflow name, UID, and observed phase; +- evidence URI and SHA-256 digest; +- positional check phases, bounded reasons and measurements, backend + identities, and backend-evidence digest/completeness; +- start and finish timestamps; +- `Cordoned`, `WorkloadCleanupComplete`, and `Released` conditions. + +Detailed Pod and MPIJob evidence is retained in the durable evidence document, +not duplicated into CR status. + +## Terminal behavior + +The controller never releases a participant before evidence is durable and +all Workflow and backend resources are gone. + +- `Succeeded`: cleanup completes and participants are released automatically. +- `Failed`, `Inconclusive`, or `Canceled`: cleanup completes, but participants + remain reserved and cordoned. +- Set `spec.release: true` to release a non-successful Run after inspection. +- Deleting a Run requests teardown. Its finalizer stops and removes owned + workloads, verifies participant identity and reservation ownership, removes + owned cordons and reservations, deletes the script bundle, and then permits + deletion. + +Release uncordons a Node only when all of these still match: + +- the snapshotted Node UID; +- `signal.unbounded-cloud.io/reserved-by=`; +- `signal.unbounded-cloud.io/reserved-node-uid=`. + +Identity replacement or ownership loss is skipped during release. Signal +never uncordons or removes markers from a Node it cannot prove it still owns, +but it continues releasing the remaining exact identity matches. + +Cancel a live Run: + +```bash +kubectl patch certificationrun -n project-signal \ + --type=merge -p '{"spec":{"cancel":true}}' +``` + +Release a retained terminal Run: + +```bash +kubectl patch certificationrun -n project-signal \ + --type=merge -p '{"spec":{"release":true}}' +``` + +## Evidence + +The evidence document includes: + +- Run, Profile, participant, script-bundle, and Workflow identities; +- the immutable check snapshots and script contents; +- aggregate execution state, verdict, reason, phase, and timestamps; +- each positional check's verdict and timing; +- typed Pod or MPIJob evidence, including Pods, containers, states, bounded + logs, and capture errors; +- backend identities and participant positions; +- a per-check cleanup disposition showing that deletion is deferred to the + controller. + +The adapter persists typed backend evidence and leaves completed resources in +place. MPI worker evidence is captured when still present; a worker may already +be absent because `cleanPodPolicy: Running` releases its GPU after launcher +completion. If an uncertain backend cannot be quiesced, the adapter task fails +and later checks are skipped rather than overlapping execution. The controller +deletes the Workflow and all remaining owned backends only after the final +evidence document is durable. +`WorkloadCleanupComplete=True` is the authoritative record that this +controller-owned cleanup finished. + +## Operations + +Inspect Runs and lifecycle conditions: + +```bash +kubectl get certificationruns -n project-signal +kubectl describe certificationrun -n project-signal +``` + +Inspect the immutable script bundle before terminal cleanup: + +```bash +kubectl get configmap -n project-signal -scripts -o yaml +``` + +Controller restarts are safe: + +- Workflow creation intent is recorded before creation; +- a missing Workflow after durable intent becomes Inconclusive and is not + recreated; +- positional identities are re-derived after every Run read; +- persisted terminal evidence can reconstruct compact Run status; +- owned premature uncordons are repaired while a Run is active or retained. + +## Non-goals + +- Kueue or other queue integration; +- recurring certification or automatic retry; +- arbitrary WorkflowTemplate, PodSpec, toleration, capability, or host-path + pass-through; +- general-purpose workload scheduling; +- live Node draining or eviction of workloads that existed before the cordon. diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml new file mode 100644 index 0000000000..4c18a27065 --- /dev/null +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationprofiles.yaml @@ -0,0 +1,380 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: certificationprofiles.signal.unbounded-cloud.io +spec: + group: signal.unbounded-cloud.io + names: + kind: CertificationProfile + listKind: CertificationProfileList + plural: certificationprofiles + shortNames: + - certprofile + singular: certificationprofile + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.checks.size() + name: Checks + type: integer + - jsonPath: .spec.participants.count + name: Participants + type: integer + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + checks: + items: + properties: + backend: + enum: + - Pod + - MPIJob + type: string + backendActiveDeadline: + type: string + gpuCount: + format: int32 + minimum: 0 + type: integer + participantPolicy: + properties: + roles: + items: + type: string + maxItems: 16 + type: array + type: object + runnerImage: + type: string + script: + properties: + digest: + description: Digest is the approved algorithm-qualified + SHA-256 digest of the script bytes. + pattern: ^sha256:[0-9a-f]{64}$ + type: string + key: + type: string + name: + type: string + required: + - digest + - key + - name + type: object + scriptEnvironment: + additionalProperties: + type: string + maxProperties: 32 + type: object + workloadPolicy: + properties: + capabilities: + items: + description: Capability represent POSIX capabilities type + type: string + type: array + hostNetwork: + type: boolean + hostPaths: + items: + properties: + mountPath: + type: string + path: + type: string + required: + - mountPath + - path + type: object + maxItems: 16 + type: array + privileged: + type: boolean + resources: + description: ResourceRequirements describes the compute + resource requirements. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in + PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + workingDir: + type: string + type: object + required: + - backend + - backendActiveDeadline + - runnerImage + - script + type: object + x-kubernetes-validations: + - message: backendActiveDeadline must be positive + rule: duration(self.backendActiveDeadline) > duration('0s') + - message: scriptEnvironment names must not exceed 253 characters + and values must not exceed 4096 characters + rule: '!has(self.scriptEnvironment) || self.scriptEnvironment.all(k, + size(k) <= 253 && size(self.scriptEnvironment[k]) <= 4096)' + maxItems: 15 + minItems: 15 + type: array + deadline: + type: string + participantRoles: + items: + properties: + count: + format: int32 + maximum: 64 + minimum: 1 + type: integer + name: + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + selector: + description: Selector further constrains participants assigned + to this role. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - count + - name + type: object + maxItems: 16 + type: array + participants: + properties: + count: + format: int32 + maximum: 64 + minimum: 1 + type: integer + minimumAllocatable: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: ResourceList is a set of (resource name, quantity) + pairs. + type: object + selector: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - count + - selector + type: object + topologyConstraints: + items: + properties: + labelKey: + description: LabelKey is the Node label used to identify topology + domains. + type: string + minDistinctValues: + format: int32 + maximum: 64 + minimum: 1 + type: integer + role: + description: Role names the participant role whose selected + Nodes are evaluated. + type: string + required: + - labelKey + - minDistinctValues + - role + type: object + maxItems: 16 + type: array + required: + - checks + - deadline + - participants + type: object + x-kubernetes-validations: + - message: every backendActiveDeadline must leave at least 60 seconds + before deadline + rule: self.checks.all(c, duration(self.deadline) - duration(c.backendActiveDeadline) + >= duration('60s')) + - message: deadline must cover every sequential backendActiveDeadline + plus 60 seconds of evidence time per check + rule: self.checks.map(c, duration(c.backendActiveDeadline) + duration('60s')).sum() + <= duration(self.deadline) + required: + - spec + type: object + served: true + storage: true + subresources: {} diff --git a/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml new file mode 100644 index 0000000000..2d61c5a6e9 --- /dev/null +++ b/deploy/signal/crd/signal.unbounded-cloud.io_certificationruns.yaml @@ -0,0 +1,481 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: certificationruns.signal.unbounded-cloud.io +spec: + group: signal.unbounded-cloud.io + names: + kind: CertificationRun + listKind: CertificationRunList + plural: certificationruns + shortNames: + - certrun + singular: certificationrun + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.profile.name + name: Profile + type: string + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.workflow.name + name: Workflow + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + cancel: + type: boolean + checks: + items: + properties: + backend: + enum: + - Pod + - MPIJob + type: string + backendActiveDeadline: + type: string + gpuCount: + format: int32 + type: integer + participantPolicy: + properties: + roles: + items: + type: string + maxItems: 16 + type: array + type: object + runnerImage: + type: string + script: + properties: + digest: + description: Digest is the algorithm-qualified SHA-256 digest + of the selected script bytes. + pattern: ^sha256:[0-9a-f]{64}$ + type: string + key: + type: string + required: + - digest + - key + type: object + scriptEnvironment: + additionalProperties: + type: string + maxProperties: 32 + type: object + workloadPolicy: + properties: + capabilities: + items: + description: Capability represent POSIX capabilities type + type: string + type: array + hostNetwork: + type: boolean + hostPaths: + items: + properties: + mountPath: + type: string + path: + type: string + required: + - mountPath + - path + type: object + maxItems: 16 + type: array + privileged: + type: boolean + resources: + description: ResourceRequirements describes the compute + resource requirements. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in + PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + workingDir: + type: string + type: object + required: + - backend + - backendActiveDeadline + - runnerImage + - script + type: object + x-kubernetes-validations: + - message: backendActiveDeadline must be positive + rule: duration(self.backendActiveDeadline) > duration('0s') + - message: scriptEnvironment names must not exceed 253 characters + and values must not exceed 4096 characters + rule: '!has(self.scriptEnvironment) || self.scriptEnvironment.all(k, + size(k) <= 253 && size(self.scriptEnvironment[k]) <= 4096)' + maxItems: 15 + minItems: 15 + type: array + deadline: + type: string + participants: + items: + properties: + allocatable: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: ResourceList is a set of (resource name, quantity) + pairs. + type: object + nodeName: + type: string + nodeUID: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + role: + type: string + topologyValues: + additionalProperties: + type: string + maxProperties: 16 + type: object + required: + - nodeName + - nodeUID + type: object + maxItems: 64 + minItems: 1 + type: array + profile: + properties: + generation: + format: int64 + type: integer + name: + type: string + uid: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + required: + - generation + - name + - uid + type: object + release: + type: boolean + scriptBundle: + properties: + name: + type: string + uid: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + required: + - name + - uid + type: object + required: + - checks + - deadline + - participants + - profile + - scriptBundle + type: object + x-kubernetes-validations: + - message: profile is immutable + rule: self.profile == oldSelf.profile + - message: participants are immutable + rule: self.participants == oldSelf.participants + - message: scriptBundle is immutable + rule: self.scriptBundle == oldSelf.scriptBundle + - message: checks are immutable + rule: self.checks == oldSelf.checks + - message: deadline is immutable + rule: self.deadline == oldSelf.deadline + - message: cancel cannot be cleared + rule: '!has(oldSelf.cancel) || !oldSelf.cancel || (has(self.cancel) + && self.cancel)' + - message: release cannot be cleared + rule: '!has(oldSelf.release) || !oldSelf.release || (has(self.release) + && self.release)' + - message: every backendActiveDeadline must leave at least 60 seconds + before deadline + rule: self.checks.all(c, duration(self.deadline) - duration(c.backendActiveDeadline) + >= duration('60s')) + - message: deadline must cover every sequential backendActiveDeadline + plus 60 seconds of evidence time per check + rule: self.checks.map(c, duration(c.backendActiveDeadline) + duration('60s')).sum() + <= duration(self.deadline) + status: + properties: + checks: + items: + properties: + backendEvidence: + properties: + complete: + type: boolean + digest: + description: |- + Digest identifies the complete backend-evidence v1 document retained in + the durable evidence artifact. + pattern: ^sha256:[0-9a-f]{64}$ + type: string + type: object + backendResources: + items: + properties: + name: + type: string + participantIndexes: + items: + format: int32 + type: integer + maxItems: 64 + minItems: 1 + type: array + uid: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + required: + - name + - participantIndexes + - uid + type: object + maxItems: 64 + type: array + finishedAt: + format: date-time + type: string + measurements: + maxLength: 16384 + type: string + phase: + enum: + - Pending + - Running + - Succeeded + - Failed + - Inconclusive + - Skipped + type: string + reason: + maxLength: 1024 + type: string + startedAt: + format: date-time + type: string + required: + - phase + type: object + maxItems: 15 + type: array + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + evidence: + properties: + digest: + description: Digest is the algorithm-qualified SHA-256 digest + of the evidence bytes. + pattern: ^sha256:[0-9a-f]{64}$ + type: string + uri: + type: string + type: object + finishedAt: + format: date-time + type: string + observedGeneration: + format: int64 + type: integer + phase: + enum: + - Pending + - Running + - Succeeded + - Failed + - Inconclusive + - Canceled + type: string + reason: + maxLength: 1024 + type: string + startedAt: + format: date-time + type: string + workflow: + properties: + name: + type: string + phase: + type: string + uid: + type: string + type: object + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/signal/rendered/.gitignore b/deploy/signal/rendered/.gitignore new file mode 100644 index 0000000000..d6b7ef32c8 --- /dev/null +++ b/deploy/signal/rendered/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/images/project-signal-adapter/Containerfile b/images/project-signal-adapter/Containerfile new file mode 100644 index 0000000000..2317fc7945 --- /dev/null +++ b/images/project-signal-adapter/Containerfile @@ -0,0 +1,29 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +FROM docker.io/library/golang:1.27.0-trixie AS builder + +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates git make \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . + +ARG TARGETOS +ARG TARGETARCH +ARG VERSION=dev +ARG GIT_COMMIT= +ARG BUILD_TIME= + +RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 \ + make project-signal-adapter-build VERSION=${VERSION} \ + ${GIT_COMMIT:+GIT_COMMIT=${GIT_COMMIT}} ${BUILD_TIME:+BUILD_TIME=${BUILD_TIME}} + +FROM gcr.io/distroless/static-debian12:nonroot + +COPY --from=builder /src/bin/project-signal-adapter /project-signal-adapter +COPY LICENSE NOTICE /licenses/ + +ENTRYPOINT ["/project-signal-adapter"] diff --git a/images/project-signal-controller/Containerfile b/images/project-signal-controller/Containerfile new file mode 100644 index 0000000000..dd52f42b80 --- /dev/null +++ b/images/project-signal-controller/Containerfile @@ -0,0 +1,29 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +FROM docker.io/library/golang:1.27.0-trixie AS builder + +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates git make \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . + +ARG TARGETOS +ARG TARGETARCH +ARG VERSION=dev +ARG GIT_COMMIT= +ARG BUILD_TIME= + +RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 \ + make project-signal-controller-build VERSION=${VERSION} \ + ${GIT_COMMIT:+GIT_COMMIT=${GIT_COMMIT}} ${BUILD_TIME:+BUILD_TIME=${BUILD_TIME}} + +FROM gcr.io/distroless/static-debian12:nonroot + +COPY --from=builder /src/bin/project-signal-controller /project-signal-controller +COPY LICENSE NOTICE /licenses/ + +ENTRYPOINT ["/project-signal-controller"] diff --git a/internal/signal/controller.go b/internal/signal/controller.go new file mode 100644 index 0000000000..0b673c0214 --- /dev/null +++ b/internal/signal/controller.go @@ -0,0 +1,3641 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signal + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "maps" + "reflect" + "regexp" + "slices" + "sort" + "strconv" + "strings" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/rand" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/recorder" + + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" + "github.com/Azure/unbounded/internal/signaladapter" + "github.com/Azure/unbounded/internal/signalworkload" +) + +const ( + RequestLabel = "signal.unbounded-cloud.io/certification-request" + ReservationLabel = "signal.unbounded-cloud.io/reserved-by" + RunNodeLabel = "signal.unbounded-cloud.io/request-node" + RunProfileLabel = "signal.unbounded-cloud.io/profile" + RunTriggerAnnotation = "signal.unbounded-cloud.io/trigger" + RunAdmissionAnnotation = "signal.unbounded-cloud.io/admission" + RunFinalizer = "signal.unbounded-cloud.io/cleanup" + SnapshotRunLabel = "signal.unbounded-cloud.io/run" + runAdmissionReady = "ready" + runAdmissionPending = "pending" +) + +var runnerImageDigestPattern = regexp.MustCompile(`^[^@\s]+@sha256:[0-9a-f]{64}$`) + +var reservedScriptEnvironmentNames = map[string]struct{}{ + "BASH_ENV": {}, + "BASHOPTS": {}, + "CDPATH": {}, + "ENV": {}, + "GLOBIGNORE": {}, + "HOST_PROC_ROOT": {}, + "HOST_SYSFS_ROOT": {}, + "IFS": {}, + "LD_LIBRARY_PATH": {}, + "LD_PRELOAD": {}, + "PATH": {}, + "PROJECT_SIGNAL_PARAMETERS_JSON": {}, + "PROJECT_SIGNAL_RESULTS_DIR": {}, + "PYTHONHOME": {}, + "PYTHONPATH": {}, + "SHELLOPTS": {}, +} + +var workflowOutputLimits = map[string]int{ + "backend-resources": signaladapter.MaxBackendOutputBytes, + "backend-evidence": signaladapter.MaxBackendEvidenceBytes, + "measurements": signaladapter.MaxMeasurementsBytes, + "reason": signaladapter.MaxReasonBytes, + "verdict": 32, +} + +const ( + groupedCheckCount = 15 + adapterServiceAccountName = "project-signal-adapter" + adapterWorkflowTemplateName = "project-signal-participant-adapter" + maximumWorkflowOutputBytes = 768 * 1024 + maximumStatusTextRunes = 1024 +) + +var workflowGVK = schema.GroupVersionKind{ + Group: "argoproj.io", + Version: "v1alpha1", + Kind: "Workflow", +} + +type AdmissionReconciler struct { + client.Client + APIReader client.Reader + Namespace string + ScriptNamespace string + Recorder recorder.EventRecorder + Now func() time.Time + admissionMu sync.Mutex +} + +type RunReconciler struct { + client.Client + APIReader client.Reader + Evidence EvidenceStore + Now func() time.Time + MaxConcurrentRuns int + Recorder recorder.EventRecorder +} + +func (r *AdmissionReconciler) SetupWithManager(mgr ctrl.Manager) error { + if r.APIReader == nil { + r.APIReader = mgr.GetAPIReader() + } + + return ctrl.NewControllerManagedBy(mgr). + For(&corev1.Node{}, builder.WithPredicates(predicate.LabelChangedPredicate{})). + WithOptions(controller.Options{MaxConcurrentReconciles: 1}). + Complete(r) +} + +func (r *RunReconciler) SetupWithManager(mgr ctrl.Manager) error { + if r.APIReader == nil { + r.APIReader = mgr.GetAPIReader() + } + + workflow := &unstructured.Unstructured{} + workflow.SetGroupVersionKind(workflowGVK) + + return ctrl.NewControllerManagedBy(mgr). + For(&signalv1.CertificationRun{}). + Watches(workflow, handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &signalv1.CertificationRun{})). + WithOptions(controller.Options{MaxConcurrentReconciles: max(1, r.MaxConcurrentRuns)}). + Complete(r) +} + +func (r *AdmissionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var node corev1.Node + if err := r.Get(ctx, client.ObjectKey{Name: req.Name}, &node); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + profileName := node.Labels[RequestLabel] + if profileName == "" { + runName := node.Annotations[ReservationLabel] + if runName == "" || node.Labels[signalv1.ReservationNodeUIDLabel] != string(node.UID) { + return ctrl.Result{}, nil + } + + r.admissionMu.Lock() + defer r.admissionMu.Unlock() + + _, err := r.resumeOrReleaseReservation(ctx, runName) + + return ctrl.Result{}, err + } + + return ctrl.Result{}, r.admitNode(ctx, &node, profileName) +} + +func (r *AdmissionReconciler) admitNode(ctx context.Context, node *corev1.Node, profileName string) error { + r.admissionMu.Lock() + defer r.admissionMu.Unlock() + + if runName := node.Annotations[ReservationLabel]; runName != "" { + handled, err := r.resumeOrReleaseReservation(ctx, runName) + if err != nil || handled { + return err + } + } + + var profile signalv1.CertificationProfile + if err := r.Get(ctx, client.ObjectKey{Name: profileName}, &profile); err != nil { + return fmt.Errorf("get CertificationProfile %s: %w", profileName, err) + } + + checks := profileChecks(&profile) + if err := validateCheckPlan(checks, profile.Spec.Deadline.Duration); err != nil { + return fmt.Errorf("profile %s checks: %w", profile.Name, err) + } + + if err := validateParticipantConstraints(&profile, checks); err != nil { + return fmt.Errorf("profile %s participant constraints: %w", profile.Name, err) + } + + for index := range checks { + check := &checks[index] + if err := signalv1.ValidateExecutionDeadlines( + check.BackendActiveDeadline.Duration, + profile.Spec.Deadline.Duration, + ); err != nil { + return fmt.Errorf("profile %s check %s runtime contract: %w", profile.Name, check.ID, err) + } + + if !runnerImageDigestPattern.MatchString(check.RunnerImage) { + return fmt.Errorf("profile %s check %s runner image must use an immutable sha256 digest", profile.Name, check.ID) + } + + if err := validateScriptEnvironment(check.ScriptEnvironment); err != nil { + return fmt.Errorf("profile %s check %s script environment: %w", profile.Name, check.ID, err) + } + + if err := signalworkload.Validate(check.WorkloadPolicy); err != nil { + return fmt.Errorf("profile %s check %s workload policy: %w", profile.Name, check.ID, err) + } + } + + participants, err := r.selectParticipants(ctx, node, &profile) + if err != nil { + return err + } + + for index := range checks { + checkParticipants, _, err := participantsForCheck(participants, checks[index].ParticipantPolicy) + if err != nil { + return fmt.Errorf("check %s participants: %w", checks[index].ID, err) + } + + if len(checkParticipants) > signaladapter.MaxCheckParticipants { + return fmt.Errorf( + "check %s selects %d participants, maximum is %d", + checks[index].ID, + len(checkParticipants), + signaladapter.MaxCheckParticipants, + ) + } + + minimumEvidenceBytes := signaladapter.MinimumBackendEvidenceBytes(len(checkParticipants)) + if signaladapter.MaxBackendEvidenceBytes < minimumEvidenceBytes { + return fmt.Errorf( + "check %s fixed backend evidence limit %d cannot represent %d participants; requires at least %d bytes", + checks[index].ID, + signaladapter.MaxBackendEvidenceBytes, + len(checkParticipants), + minimumEvidenceBytes, + ) + } + + minimumReasonBytes := signaladapter.MinimumPartialReasonBytes() + if signaladapter.MaxReasonBytes < minimumReasonBytes { + return fmt.Errorf( + "check %s fixed reason limit %d cannot represent mandatory failure output; requires at least %d bytes", + checks[index].ID, + signaladapter.MaxReasonBytes, + minimumReasonBytes, + ) + } + + minimumMeasurementsBytes := signaladapter.MinimumPartialMeasurementsBytes(len(checkParticipants)) + if signaladapter.MaxMeasurementsBytes < minimumMeasurementsBytes { + return fmt.Errorf( + "check %s fixed measurements limit %d cannot represent %d participants; requires at least %d bytes", + checks[index].ID, + signaladapter.MaxMeasurementsBytes, + len(checkParticipants), + minimumMeasurementsBytes, + ) + } + + minimumBackendResourcesBytes := signaladapter.MinimumPartialBackendResourcesBytes(len(checkParticipants)) + if signaladapter.MaxBackendOutputBytes < minimumBackendResourcesBytes { + return fmt.Errorf( + "check %s fixed backend resource limit %d cannot represent %d participants; requires at least %d bytes", + checks[index].ID, + signaladapter.MaxBackendOutputBytes, + len(checkParticipants), + minimumBackendResourcesBytes, + ) + } + + if _, err := signalworkload.Encode(checks[index].WorkloadPolicy); err != nil { + return fmt.Errorf("check %s workload policy: %w", checks[index].ID, err) + } + + if err := validateGPUCount(checks[index].GPUCount, checks[index].Backend); err != nil { + return fmt.Errorf("check %s GPU count: %w", checks[index].ID, err) + } + } + + runName := runName(node.Name, profile.Name, r.now()) + if err := r.reserveParticipants(ctx, participants, runName); err != nil { + return err + } + + releaseOnFailure := true + defer func() { + if releaseOnFailure { + if err := r.releaseParticipants(context.WithoutCancel(ctx), participants, runName); err != nil { + log.FromContext(ctx).Error(err, "failed to roll back participant reservations", "run", runName) + } + } + }() + + snapshot, checkSnapshots, err := r.createSnapshot(ctx, runName, checks) + if err != nil { + return err + } + + run := &signalv1.CertificationRun{ + TypeMeta: metav1.TypeMeta{APIVersion: signalv1.GroupVersion.String(), Kind: "CertificationRun"}, + ObjectMeta: metav1.ObjectMeta{ + Name: runName, + Namespace: r.Namespace, + Labels: map[string]string{ + RunNodeLabel: node.Name, + RunProfileLabel: profile.Name, + }, + Annotations: map[string]string{ + RunTriggerAnnotation: "manual", + RunAdmissionAnnotation: runAdmissionPending, + }, + Finalizers: []string{RunFinalizer}, + }, + Spec: signalv1.CertificationRunSpec{ + Profile: signalv1.PinnedProfile{ + Name: profile.Name, + UID: profile.UID, + Generation: profile.Generation, + }, + Participants: participants, + ScriptBundle: signalv1.ScriptBundleReference{ + Name: snapshot.Name, + UID: snapshot.UID, + }, + Checks: checkSnapshots, + Deadline: profile.Spec.Deadline, + }, + } + if _, err := buildWorkflow(run); err != nil { + deleteErr := r.Delete(context.WithoutCancel(ctx), snapshot) + + return errors.Join(fmt.Errorf("validate executable plan: %w", err), client.IgnoreNotFound(deleteErr)) + } + + if err := r.Create(ctx, run); err != nil { + deleteErr := r.Delete(context.WithoutCancel(ctx), snapshot) + + return errors.Join(fmt.Errorf("create CertificationRun: %w", err), client.IgnoreNotFound(deleteErr)) + } + + cleanupFailedAdmission := func(cause error) error { + releaseOnFailure = false + deleteRunErr := r.Delete(context.WithoutCancel(ctx), run) + + // The Run finalizer owns rollback once the Run exists, including its snapshot. + return errors.Join(cause, client.IgnoreNotFound(deleteRunErr)) + } + + snapshot.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(run, signalv1.GroupVersion.WithKind("CertificationRun"))} + if err := r.Update(ctx, snapshot); err != nil { + return cleanupFailedAdmission(fmt.Errorf("bind script snapshot to run: %w", err)) + } + + releaseOnFailure = false + + if err := r.completeRunAdmission(ctx, run); err != nil { + return err + } + + if r.Recorder != nil { + r.Recorder.Eventf(run, nil, corev1.EventTypeNormal, "CertificationAdmitted", "AdmitCertification", + "Created CertificationRun %s with %d immutable checks", run.Name, len(checkSnapshots)) + } + + return nil +} + +func (r *AdmissionReconciler) resumeOrReleaseReservation(ctx context.Context, runName string) (bool, error) { + var run signalv1.CertificationRun + + reader := r.APIReader + if reader == nil { + reader = r.Client + } + + err := reader.Get(ctx, client.ObjectKey{Namespace: r.Namespace, Name: runName}, &run) + if err == nil { + return true, r.completeRunAdmission(ctx, &run) + } + + if !apierrors.IsNotFound(err) { + return false, err + } + + var snapshot corev1.ConfigMap + + err = reader.Get( + ctx, + client.ObjectKey{ + Namespace: r.Namespace, + Name: runName + "-scripts", + }, + &snapshot, + ) + if err == nil { + if snapshot.Labels[SnapshotRunLabel] != runName { + return false, fmt.Errorf( + "orphaned snapshot %s is not bound to reservation %q", + snapshot.Name, + runName, + ) + } + + if err := r.Delete(ctx, &snapshot); err != nil && !apierrors.IsNotFound(err) { + return false, fmt.Errorf("delete orphaned script snapshot: %w", err) + } + } else if !apierrors.IsNotFound(err) { + return false, fmt.Errorf("get orphaned script snapshot: %w", err) + } + + if err := r.releaseOrphanedReservations(ctx, runName); err != nil { + return false, fmt.Errorf("release orphaned Project Signal reservations: %w", err) + } + + return false, nil +} + +func (r *AdmissionReconciler) completeRunAdmission( + ctx context.Context, + run *signalv1.CertificationRun, +) error { + if run.Annotations[RunAdmissionAnnotation] == runAdmissionReady { + return nil + } + + if err := projectPending(ctx, r.Client, run); err != nil { + return fmt.Errorf("project pending certification state: %w", err) + } + + before := run.DeepCopy() + if run.Annotations == nil { + run.Annotations = map[string]string{} + } + + run.Annotations[RunAdmissionAnnotation] = runAdmissionReady + + if err := r.Patch( + ctx, + run, + client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{}), + ); err != nil { + return fmt.Errorf("mark certification admission ready: %w", err) + } + + return nil +} + +func (r *AdmissionReconciler) selectParticipants( + ctx context.Context, + requestNode *corev1.Node, + profile *signalv1.CertificationProfile, +) ([]signalv1.Participant, error) { + count := profile.Spec.Participants.Count + if count < 1 { + return nil, fmt.Errorf("profile %s participant count must be positive", profile.Name) + } + + if count > 64 { + return nil, fmt.Errorf("profile %s participant count must not exceed 64", profile.Name) + } + + reader := r.APIReader + if reader == nil { + reader = r.Client + } + + var authoritativeRequest corev1.Node + if err := reader.Get(ctx, client.ObjectKey{Name: requestNode.Name}, &authoritativeRequest); err != nil { + return nil, fmt.Errorf("read request node %s: %w", requestNode.Name, err) + } + + if requestNode.UID != "" && authoritativeRequest.UID != requestNode.UID { + return nil, fmt.Errorf("request node %s identity changed", requestNode.Name) + } + + requestNode = &authoritativeRequest + + selector, err := metav1.LabelSelectorAsSelector(&profile.Spec.Participants.Selector) + if err != nil { + return nil, fmt.Errorf("profile %s participant selector: %w", profile.Name, err) + } + + if !selector.Matches(labels.Set(requestNode.Labels)) || + !nodeEligible(requestNode, profile.Spec.Participants.MinimumAllocatable) { + return nil, fmt.Errorf("request node %s does not satisfy profile %s", requestNode.Name, profile.Name) + } + + var nodes corev1.NodeList + + if err := reader.List(ctx, &nodes); err != nil { + return nil, fmt.Errorf("list participant nodes: %w", err) + } + + sort.Slice(nodes.Items, func(i, j int) bool { return nodes.Items[i].Name < nodes.Items[j].Name }) + + candidates := []corev1.Node{*requestNode.DeepCopy()} + + for i := range nodes.Items { + candidate := &nodes.Items[i] + + if candidate.Name == requestNode.Name || + !selector.Matches(labels.Set(candidate.Labels)) || + !nodeEligible(candidate, profile.Spec.Participants.MinimumAllocatable) || + candidate.Annotations[ReservationLabel] != "" || + candidate.Labels[signalv1.ReservationNodeUIDLabel] != "" { + continue + } + + candidates = append(candidates, *candidate.DeepCopy()) + } + + if int32(len(candidates)) < count { + return nil, fmt.Errorf( + "profile %s requires %d eligible unlocked participants, found %d", + profile.Name, + count, + len(candidates), + ) + } + + selected, err := selectParticipantPlan( + candidates, + requestNode.Name, + count, + profile.Spec.ParticipantRoles, + profile.Spec.TopologyConstraints, + ) + if err != nil { + return nil, fmt.Errorf("profile %s participant constraints: %w", profile.Name, err) + } + + return selected, nil +} + +func (r *AdmissionReconciler) releaseOrphanedReservations(ctx context.Context, runName string) error { + reader := r.APIReader + if reader == nil { + reader = r.Client + } + + var nodes corev1.NodeList + if err := reader.List(ctx, &nodes); err != nil { + return fmt.Errorf("list Nodes reserved by %s: %w", runName, err) + } + + participants := make([]signalv1.Participant, 0) + + for index := range nodes.Items { + node := &nodes.Items[index] + if node.Annotations[ReservationLabel] == runName { + participants = append(participants, signalv1.Participant{ + NodeName: node.Name, + NodeUID: node.UID, + }) + } + } + + return r.releaseParticipants(ctx, participants, runName) +} + +func participantSnapshot(node *corev1.Node, index int32) signalv1.Participant { + return signalv1.Participant{ + Index: index, + NodeName: node.Name, + NodeUID: node.UID, + Allocatable: node.Status.Allocatable.DeepCopy(), + } +} + +type participantRoleSlot struct { + name string +} + +func selectParticipantPlan( + candidates []corev1.Node, + requestNodeName string, + count int32, + roles []signalv1.ParticipantRoleConstraint, + topology []signalv1.ParticipantTopologyConstraint, +) ([]signalv1.Participant, error) { + assignments, err := solveParticipantRoles( + candidates, + requestNodeName, + count, + roles, + topology, + ) + if err != nil { + return nil, err + } + + selectedIndexes := make([]int, 0, count) + selected := make([]bool, len(candidates)) + + selectedIndexes = append(selectedIndexes, 0) + selected[0] = true + + for index := range candidates { + if selected[index] || len(assignments[index]) == 0 { + continue + } + + selectedIndexes = append(selectedIndexes, index) + selected[index] = true + } + + for index := range candidates { + if int32(len(selectedIndexes)) == count { + break + } + + if selected[index] { + continue + } + + selectedIndexes = append(selectedIndexes, index) + selected[index] = true + } + + if int32(len(selectedIndexes)) != count { + return nil, fmt.Errorf("participant plan selected %d participants, want %d", len(selectedIndexes), count) + } + + participants := make([]signalv1.Participant, 0, count) + for _, candidateIndex := range selectedIndexes { + participant := participantSnapshot(&candidates[candidateIndex], int32(len(participants))) + if len(assignments[candidateIndex]) > 0 { + participant.Role = assignments[candidateIndex][0] + } + + participant.TopologyValues = topologyValues(&candidates[candidateIndex], topology) + participants = append(participants, participant) + } + + return participants, nil +} + +func topologyValues( + node *corev1.Node, + constraints []signalv1.ParticipantTopologyConstraint, +) map[string]string { + values := make(map[string]string) + + for index := range constraints { + key := constraints[index].LabelKey + if value, exists := node.Labels[key]; exists { + values[key] = value + } + } + + if len(values) == 0 { + return nil + } + + return values +} + +func solveParticipantRoles( + candidates []corev1.Node, + requestNodeName string, + participantCount int32, + roles []signalv1.ParticipantRoleConstraint, + topology []signalv1.ParticipantTopologyConstraint, +) ([][]string, error) { + roleNames := make(map[string]struct{}, len(roles)) + roleSelectors := make(map[string]labels.Selector, len(roles)) + slots := make([]participantRoleSlot, 0, participantCount) + + for roleIndex := range roles { + role := &roles[roleIndex] + if _, duplicate := roleNames[role.Name]; duplicate { + return nil, fmt.Errorf("duplicate participant role %q", role.Name) + } + + selector, err := metav1.LabelSelectorAsSelector(&role.Selector) + if err != nil { + return nil, fmt.Errorf("role %s selector: %w", role.Name, err) + } + + roleNames[role.Name] = struct{}{} + roleSelectors[role.Name] = selector + + for range role.Count { + slots = append(slots, participantRoleSlot{name: role.Name}) + } + } + + if int32(len(slots)) > participantCount { + return nil, fmt.Errorf( + "participant roles require %d assignments but participant count is %d", + len(slots), + participantCount, + ) + } + + for index := range topology { + if _, exists := roleNames[topology[index].Role]; !exists { + return nil, fmt.Errorf("topology constraint references unknown role %q", topology[index].Role) + } + } + + assignments := make([][]string, len(candidates)) + assigned := make([]bool, len(candidates)) + slotAssignments := make([]int, len(slots)) + candidateOrders := make(map[string][]int, len(roles)) + + for role, selector := range roleSelectors { + candidateOrders[role] = roleCandidateOrder(candidates, selector, role, topology) + } + + const maximumSearchStates = 1_000_000 + + searchStates := 0 + searchExhausted := false + + var search func(int) bool + + search = func(slotIndex int) bool { + if searchExhausted { + return false + } + + searchStates++ + if searchStates > maximumSearchStates { + searchExhausted = true + + return false + } + + if slotIndex == len(slots) { + requestAssigned := requestNodeName == "" + assignedCount := int32(0) + + for index := range candidates { + if len(assignments[index]) == 0 { + continue + } + + assignedCount++ + + if candidates[index].Name == requestNodeName { + requestAssigned = true + } + } + + if !requestAssigned && assignedCount >= participantCount { + return false + } + + return participantTopologySatisfied(candidates, assignments, topology) + } + + slot := &slots[slotIndex] + + firstCandidate := 0 + if slotIndex > 0 && slots[slotIndex-1].name == slot.name { + firstCandidate = slotAssignments[slotIndex-1] + 1 + } + + order := candidateOrders[slot.name] + for candidatePosition := firstCandidate; candidatePosition < len(order); candidatePosition++ { + candidateIndex := order[candidatePosition] + if assigned[candidateIndex] { + continue + } + + assigned[candidateIndex] = true + assignments[candidateIndex] = []string{slot.name} + slotAssignments[slotIndex] = candidatePosition + + roleComplete := slotIndex+1 == len(slots) || slots[slotIndex+1].name != slot.name + if (!roleComplete || participantRoleTopologySatisfied( + candidates, + assignments, + slot.name, + topology, + )) && search(slotIndex+1) { + return true + } + + assignments[candidateIndex] = nil + assigned[candidateIndex] = false + } + + return false + } + + if !search(0) { + if searchExhausted { + return nil, fmt.Errorf("participant role search exceeded %d states", maximumSearchStates) + } + + return nil, errors.New("eligible participants cannot satisfy role and topology constraints") + } + + return assignments, nil +} + +func roleCandidateOrder( + candidates []corev1.Node, + selector labels.Selector, + role string, + topology []signalv1.ParticipantTopologyConstraint, +) []int { + constraints := make([]signalv1.ParticipantTopologyConstraint, 0) + + for index := range topology { + if topology[index].Role == role { + constraints = append(constraints, topology[index]) + } + } + + if len(constraints) == 0 { + order := make([]int, 0, len(candidates)) + for index := range candidates { + if selector.Matches(labels.Set(candidates[index].Labels)) { + order = append(order, index) + } + } + + return order + } + + groups := map[string][]int{} + + for index := range candidates { + if !selector.Matches(labels.Set(candidates[index].Labels)) { + continue + } + + values := make([]string, len(constraints)) + valid := true + + for constraintIndex := range constraints { + values[constraintIndex] = candidates[index].Labels[constraints[constraintIndex].LabelKey] + if values[constraintIndex] == "" { + valid = false + + break + } + } + + if valid { + key := strings.Join(values, "\x00") + groups[key] = append(groups[key], index) + } + } + + keys := slices.Sorted(maps.Keys(groups)) + order := make([]int, 0, len(candidates)) + + for offset := 0; ; offset++ { + added := false + + for _, key := range keys { + if offset >= len(groups[key]) { + continue + } + + order = append(order, groups[key][offset]) + added = true + } + + if !added { + return order + } + } +} + +func participantRoleTopologySatisfied( + candidates []corev1.Node, + assignments [][]string, + role string, + topology []signalv1.ParticipantTopologyConstraint, +) bool { + for index := range topology { + if topology[index].Role != role { + continue + } + + if !participantTopologyConstraintSatisfied(candidates, assignments, &topology[index]) { + return false + } + } + + return true +} + +func participantTopologySatisfied( + candidates []corev1.Node, + assignments [][]string, + topology []signalv1.ParticipantTopologyConstraint, +) bool { + for constraintIndex := range topology { + if !participantTopologyConstraintSatisfied( + candidates, + assignments, + &topology[constraintIndex], + ) { + return false + } + } + + return true +} + +func participantTopologyConstraintSatisfied( + candidates []corev1.Node, + assignments [][]string, + constraint *signalv1.ParticipantTopologyConstraint, +) bool { + values := map[string]struct{}{} + + for participantIndex := range candidates { + if !slices.Contains(assignments[participantIndex], constraint.Role) { + continue + } + + value := candidates[participantIndex].Labels[constraint.LabelKey] + if value == "" { + return false + } + + values[value] = struct{}{} + } + + return int32(len(values)) >= constraint.MinDistinctValues +} + +func nodeEligible(node *corev1.Node, required corev1.ResourceList) bool { + if node.Spec.Unschedulable || !nodeReady(node) { + return false + } + + for name, quantity := range required { + available, ok := node.Status.Allocatable[name] + if !ok || available.Cmp(quantity) < 0 { + return false + } + } + + return true +} + +func nodeReady(node *corev1.Node) bool { + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady { + return condition.Status == corev1.ConditionTrue + } + } + + return false +} + +func (r *AdmissionReconciler) reserveParticipants(ctx context.Context, participants []signalv1.Participant, runName string) error { + reserved := make([]signalv1.Participant, 0, len(participants)) + for _, participant := range participants { + var node corev1.Node + if err := r.Get(ctx, client.ObjectKey{Name: participant.NodeName}, &node); err != nil { + return errors.Join( + fmt.Errorf("get participant %s: %w", participant.NodeName, err), + r.releaseParticipants(context.WithoutCancel(ctx), reserved, runName), + ) + } + + if node.UID != participant.NodeUID { + return errors.Join( + fmt.Errorf("participant %s identity changed", participant.NodeName), + r.releaseParticipants(context.WithoutCancel(ctx), reserved, runName), + ) + } + + if node.Spec.Unschedulable { + return errors.Join( + fmt.Errorf("participant %s became unschedulable before reservation", participant.NodeName), + r.releaseParticipants(context.WithoutCancel(ctx), reserved, runName), + ) + } + + if value := node.Annotations[ReservationLabel]; value != "" { + return errors.Join( + fmt.Errorf("participant %s is already reserved by %s", participant.NodeName, value), + r.releaseParticipants(context.WithoutCancel(ctx), reserved, runName), + ) + } + + if value := node.Labels[signalv1.ReservationNodeUIDLabel]; value != "" { + return errors.Join( + fmt.Errorf("participant %s has stale reservation identity %s", participant.NodeName, value), + r.releaseParticipants(context.WithoutCancel(ctx), reserved, runName), + ) + } + + before := node.DeepCopy() + if node.Annotations == nil { + node.Annotations = map[string]string{} + } + + if node.Labels == nil { + node.Labels = map[string]string{} + } + + node.Annotations[ReservationLabel] = runName + node.Labels[signalv1.ReservationNodeUIDLabel] = string(participant.NodeUID) + node.Spec.Unschedulable = true + + if err := r.Patch( + ctx, + &node, + client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{}), + ); err != nil { + return errors.Join( + fmt.Errorf("reserve participant %s: %w", participant.NodeName, err), + r.releaseParticipants(context.WithoutCancel(ctx), reserved, runName), + ) + } + + reserved = append(reserved, participant) + } + + return nil +} + +func (r *AdmissionReconciler) releaseParticipants(ctx context.Context, participants []signalv1.Participant, runName string) error { + var errs []error + + for _, participant := range participants { + var node corev1.Node + if err := r.Get(ctx, client.ObjectKey{Name: participant.NodeName}, &node); err != nil { + if !apierrors.IsNotFound(err) { + errs = append(errs, err) + } + + continue + } + + if participant.NodeUID != "" && node.UID != participant.NodeUID { + errs = append(errs, fmt.Errorf( + "participant %s identity changed from %s to %s", + participant.NodeName, + participant.NodeUID, + node.UID, + )) + + continue + } + + if node.Annotations[ReservationLabel] != runName { + continue + } + + if value := node.Labels[signalv1.ReservationNodeUIDLabel]; value != string(participant.NodeUID) { + errs = append(errs, fmt.Errorf( + "participant %s reservation identity changed from %s to %s", + participant.NodeName, + participant.NodeUID, + value, + )) + + continue + } + + before := node.DeepCopy() + node.Spec.Unschedulable = false + delete(node.Annotations, ReservationLabel) + delete(node.Labels, signalv1.ReservationNodeUIDLabel) + + if err := r.Patch(ctx, &node, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil { + errs = append(errs, err) + } + } + + return errors.Join(errs...) +} + +func (r *AdmissionReconciler) createSnapshot( + ctx context.Context, + runName string, + checks []signalv1.CertificationCheck, +) (*corev1.ConfigMap, []signalv1.CertificationCheckSnapshot, error) { + immutable := true + + snapshot := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: runName + "-scripts", + Namespace: r.Namespace, + Labels: map[string]string{SnapshotRunLabel: runName}, + }, + Immutable: &immutable, + Data: map[string]string{}, + } + + checkSnapshots := make([]signalv1.CertificationCheckSnapshot, len(checks)) + for index := range checks { + check := &checks[index] + + var source corev1.ConfigMap + if err := r.Get( + ctx, + client.ObjectKey{Namespace: r.ScriptNamespace, Name: check.Script.Name}, + &source, + ); err != nil { + return nil, nil, fmt.Errorf("get source script ConfigMap for check %s: %w", check.ID, err) + } + + content, ok := source.Data[check.Script.Key] + if !ok { + return nil, nil, fmt.Errorf( + "source script ConfigMap %s has no key %s for check %s", + source.Name, + check.Script.Key, + check.ID, + ) + } + + digest := digestString(content) + if check.Script.Digest != "" && digest != check.Script.Digest { + return nil, nil, fmt.Errorf( + "source script digest for check %s is %s, want %s", + check.ID, + digest, + check.Script.Digest, + ) + } + + key := fmt.Sprintf("%02d-%s", check.Order, sanitizeTaskName(string(check.ID))) + + snapshot.Data[key] = content + checkSnapshots[index] = signalv1.CertificationCheckSnapshot{ + ID: check.ID, + Order: check.Order, + Script: signalv1.ScriptSnapshot{ + Key: key, + Digest: digest, + }, + RunnerImage: check.RunnerImage, + Backend: check.Backend, + WorkloadPolicy: *check.WorkloadPolicy.DeepCopy(), + GPUCount: check.GPUCount, + ScriptEnvironment: maps.Clone(check.ScriptEnvironment), + BackendActiveDeadline: check.BackendActiveDeadline, + ParticipantPolicy: check.ParticipantPolicy, + } + } + + if err := r.Create(ctx, snapshot); err != nil { + return nil, nil, fmt.Errorf("create immutable script snapshot: %w", err) + } + + return snapshot, checkSnapshots, nil +} + +func (r *RunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var run signalv1.CertificationRun + if err := r.Get(ctx, req.NamespacedName, &run); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + hydrateRunDerivedFields(&run) + + if !run.DeletionTimestamp.IsZero() { + return r.reconcileDeleting(ctx, &run) + } + + if run.Status.Phase.Terminal() { + if !containsString(run.Finalizers, RunFinalizer) { + return ctrl.Result{}, nil + } + + return r.cleanup(ctx, &run) + } + + if err := r.verifyScriptBundle(ctx, &run); err != nil { + return ctrl.Result{}, err + } + + if run.Status.Evidence.URI == "" { + result, recovered, err := r.resumeGroupedEvidence(ctx, &run) + if err != nil || recovered { + return result, err + } + } + + if run.Spec.Cancel { + return r.cancel(ctx, &run) + } + + participantsOwned, err := r.ensureParticipantsCordoned(ctx, &run) + if err != nil { + return ctrl.Result{}, err + } + + if !participantsOwned { + return r.finishParticipantOwnershipLoss(ctx, &run) + } + + if run.Annotations[RunAdmissionAnnotation] != runAdmissionReady { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + + if run.Status.Workflow.Name == "" { + return r.startWorkflow(ctx, &run) + } + + workflow := &unstructured.Unstructured{} + workflow.SetGroupVersionKind(workflowGVK) + + reader := r.APIReader + if reader == nil { + reader = r.Client + } + + if err := reader.Get(ctx, client.ObjectKey{Namespace: run.Namespace, Name: run.Status.Workflow.Name}, workflow); err != nil { + if apierrors.IsNotFound(err) { + workflow.SetName(run.Status.Workflow.Name) + workflow.Object["status"] = map[string]any{ + "phase": "Error", + "message": "workflow disappeared before terminal status", + } + + return r.finishGrouped(ctx, &run, workflow, "Error") + } + + return ctrl.Result{}, err + } + + if !metav1.IsControlledBy(workflow, &run) { + return ctrl.Result{}, fmt.Errorf( + "workflow %s identity or ownership changed; refusing concurrent retry", + workflow.GetName(), + ) + } + + if run.Status.Workflow.UID == "" { + return r.recordWorkflowStarted(ctx, &run, workflow) + } + + if string(workflow.GetUID()) != run.Status.Workflow.UID { + return ctrl.Result{}, fmt.Errorf( + "workflow %s identity or ownership changed; refusing concurrent retry", + workflow.GetName(), + ) + } + + phase, _, err := unstructured.NestedString(workflow.Object, "status", "phase") + if err != nil { + return ctrl.Result{}, fmt.Errorf("read Workflow phase: %w", err) + } + + switch phase { + case "", "Pending", "Running": + if err := projectTesting(ctx, r.Client, &run); err != nil { + return ctrl.Result{}, err + } + + checkStatuses := runningCheckStatuses(workflow, &run) + + if run.Status.Phase != signalv1.RunPhaseRunning || + run.Status.Workflow.Phase != phase || + !reflect.DeepEqual(run.Status.Checks, checkStatuses) { + before := run.DeepCopy() + run.Status.Phase = signalv1.RunPhaseRunning + + run.Status.Workflow.Phase = phase + + run.Status.Checks = checkStatuses + if run.Status.StartedAt == nil { + now := metav1.NewTime(r.now()) + run.Status.StartedAt = &now + } + + return ctrl.Result{}, r.Status().Patch(ctx, &run, client.MergeFrom(before)) + } + + return ctrl.Result{RequeueAfter: 15 * time.Second}, nil + case "Succeeded": + return r.finishGrouped(ctx, &run, workflow, phase) + case "Failed", "Error": + return r.finishGrouped(ctx, &run, workflow, phase) + default: + return ctrl.Result{}, fmt.Errorf("unsupported Workflow phase %q", phase) + } +} + +func (r *RunReconciler) verifyScriptBundle(ctx context.Context, run *signalv1.CertificationRun) error { + reader := r.APIReader + if reader == nil { + reader = r.Client + } + + var snapshot corev1.ConfigMap + if err := reader.Get( + ctx, + client.ObjectKey{Namespace: run.Namespace, Name: run.Spec.ScriptBundle.Name}, + &snapshot, + ); err != nil { + return fmt.Errorf("read immutable script bundle: %w", err) + } + + if snapshot.UID != run.Spec.ScriptBundle.UID || snapshot.Labels[SnapshotRunLabel] != run.Name { + return errors.New("script bundle identity does not match the run") + } + + if snapshot.Immutable == nil || !*snapshot.Immutable { + return errors.New("script bundle is not immutable") + } + + for index := range run.Spec.Checks { + check := &run.Spec.Checks[index] + content, found := snapshot.Data[check.Script.Key] + + if !found { + return fmt.Errorf("script bundle is missing check %s key %s", check.ID, check.Script.Key) + } + + if digestString(content) != check.Script.Digest { + return fmt.Errorf("script bundle digest does not match check %s", check.ID) + } + } + + return nil +} + +func hydrateRunDerivedFields(run *signalv1.CertificationRun) { + for index := range run.Spec.Participants { + run.Spec.Participants[index].Index = int32(index) + } + + for index := range run.Spec.Checks { + run.Spec.Checks[index].ID = signalv1.CheckID(fmt.Sprintf("R%02d", index+1)) + run.Spec.Checks[index].Order = int32(index) + } + + if len(run.Status.Checks) == len(run.Spec.Checks) { + for index := range run.Status.Checks { + run.Status.Checks[index].ID = run.Spec.Checks[index].ID + run.Status.Checks[index].Order = run.Spec.Checks[index].Order + } + } +} + +func (r *RunReconciler) ensureParticipantsCordoned( + ctx context.Context, + run *signalv1.CertificationRun, +) (bool, error) { + allOwned := true + + for _, participant := range run.Spec.Participants { + var node corev1.Node + if err := r.Get(ctx, client.ObjectKey{Name: participant.NodeName}, &node); err != nil { + if apierrors.IsNotFound(err) { + allOwned = false + + continue + } + + return false, fmt.Errorf("get cordoned participant %s: %w", participant.NodeName, err) + } + + if node.UID != participant.NodeUID { + allOwned = false + + continue + } + + if node.Annotations[ReservationLabel] != run.Name || + node.Labels[signalv1.ReservationNodeUIDLabel] != string(participant.NodeUID) { + allOwned = false + + continue + } + + if node.Spec.Unschedulable { + continue + } + + before := node.DeepCopy() + node.Spec.Unschedulable = true + + if err := r.Patch( + ctx, + &node, + client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{}), + ); err != nil { + return false, fmt.Errorf("restore participant %s cordon: %w", participant.NodeName, err) + } + } + + status := metav1.ConditionTrue + reason := "ParticipantsCordoned" + message := "All participants are reserved and cordoned by this Run" + + if !allOwned { + status = metav1.ConditionFalse + reason = "ParticipantOwnershipLost" + message = "One or more snapshotted participants are missing, replaced, or no longer owned by this Run" + } + + if err := r.patchRunCondition( + ctx, + run, + signalv1.RunConditionCordoned, + status, + reason, + message, + ); err != nil { + return false, err + } + + return allOwned, nil +} + +func (r *RunReconciler) finishParticipantOwnershipLoss( + ctx context.Context, + run *signalv1.CertificationRun, +) (ctrl.Result, error) { + workflow, err := r.workflowForCancellation(ctx, run) + if err != nil { + return ctrl.Result{}, err + } + + if workflow == nil { + workflow = &unstructured.Unstructured{} + workflow.SetGroupVersionKind(workflowGVK) + workflow.SetNamespace(run.Namespace) + workflow.SetName(fmt.Sprintf("%s-a1", run.Name)) + } else { + workflow = workflow.DeepCopy() + } + + if err := unstructured.SetNestedField(workflow.Object, "Error", "status", "phase"); err != nil { + return ctrl.Result{}, err + } + + if err := unstructured.SetNestedField( + workflow.Object, + "participant identity or reservation ownership changed", + "status", + "message", + ); err != nil { + return ctrl.Result{}, err + } + + return r.finishGrouped(ctx, run, workflow, "Error") +} + +func (r *RunReconciler) patchRunCondition( + ctx context.Context, + run *signalv1.CertificationRun, + conditionType string, + status metav1.ConditionStatus, + reason string, + message string, +) error { + before := run.DeepCopy() + run.Status.ObservedGeneration = run.Generation + apimeta.SetStatusCondition(&run.Status.Conditions, metav1.Condition{ + Type: conditionType, + Status: status, + ObservedGeneration: run.Generation, + Reason: reason, + Message: message, + }) + + if reflect.DeepEqual(before.Status, run.Status) { + return nil + } + + return r.Status().Patch(ctx, run, client.MergeFrom(before)) +} + +func (r *RunReconciler) startWorkflow(ctx context.Context, run *signalv1.CertificationRun) (ctrl.Result, error) { + workflow, err := buildWorkflow(run) + if err != nil { + before := run.DeepCopy() + now := metav1.NewTime(r.now()) + run.Status.Phase = signalv1.RunPhaseInconclusive + run.Status.Reason = boundedStatusText("InvalidExecutionPlan: " + err.Error()) + run.Status.FinishedAt = &now + + return ctrl.Result{RequeueAfter: time.Millisecond}, r.Status().Patch(ctx, run, client.MergeFrom(before)) + } + + before := run.DeepCopy() + run.Status.Workflow = signalv1.WorkflowStatus{ + Name: workflow.GetName(), + Phase: "Creating", + } + run.Status.Checks = initialCheckStatuses(run.Spec.Checks) + + if err := r.Status().Patch(ctx, run, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, fmt.Errorf("record Workflow creation intent: %w", err) + } + + if err := r.Create(ctx, workflow); err != nil { + if !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, fmt.Errorf("create Workflow: %w", err) + } + + existing := &unstructured.Unstructured{} + existing.SetGroupVersionKind(workflowGVK) + + if getErr := r.Get(ctx, client.ObjectKeyFromObject(workflow), existing); getErr != nil { + return ctrl.Result{}, fmt.Errorf("get existing Workflow: %w", getErr) + } + + if !metav1.IsControlledBy(existing, run) { + return ctrl.Result{}, errors.New("workflow already exists without matching CertificationRun ownership") + } + + workflow = existing + } + + return r.recordWorkflowStarted(ctx, run, workflow) +} + +func (r *RunReconciler) recordWorkflowStarted( + ctx context.Context, + run *signalv1.CertificationRun, + workflow *unstructured.Unstructured, +) (ctrl.Result, error) { + before := run.DeepCopy() + run.Status.Phase = signalv1.RunPhaseRunning + + run.Status.Workflow = signalv1.WorkflowStatus{ + Name: workflow.GetName(), + UID: string(workflow.GetUID()), + Phase: "Pending", + } + run.Status.Checks = initialCheckStatuses(run.Spec.Checks) + + if run.Status.StartedAt == nil { + now := metav1.NewTime(r.now()) + run.Status.StartedAt = &now + } + + if err := r.Status().Patch(ctx, run, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, fmt.Errorf("record Workflow identity: %w", err) + } + + if r.Recorder != nil { + r.Recorder.Eventf(run, nil, corev1.EventTypeNormal, "WorkflowStarted", "StartWorkflow", + "Created Workflow %s", workflow.GetName()) + } + + return ctrl.Result{}, nil +} + +func buildWorkflow(run *signalv1.CertificationRun) (*unstructured.Unstructured, error) { + checks := run.Spec.Checks + if err := validateRunCheckPlan(checks, run.Spec.Deadline.Duration); err != nil { + return nil, err + } + + name := run.Name + "-a1" + tasks := make([]any, 0, len(checks)) + + for index := range checks { + check := &checks[index] + + checkParticipants, _, participantErr := participantsForCheck( + run.Spec.Participants, + check.ParticipantPolicy, + ) + if participantErr != nil { + return nil, fmt.Errorf("check %s participants: %w", check.ID, participantErr) + } + + participantsJSON, marshalErr := json.Marshal(checkParticipants) + if marshalErr != nil { + return nil, fmt.Errorf("encode check %s participants: %w", check.ID, marshalErr) + } + + environmentJSON, marshalErr := json.Marshal(check.ScriptEnvironment) + if marshalErr != nil { + return nil, fmt.Errorf("encode check %s script environment: %w", check.ID, marshalErr) + } + + workloadPolicyJSON, encodeErr := signalworkload.Encode(check.WorkloadPolicy) + if encodeErr != nil { + return nil, fmt.Errorf("encode check %s workload policy: %w", check.ID, encodeErr) + } + + taskName := checkTaskName(check) + task := map[string]any{ + "name": taskName, + "templateRef": map[string]any{ + "name": adapterWorkflowTemplateName, + "template": "certification", + }, + "arguments": map[string]any{ + "parameters": checkWorkflowParameters( + run, + check, + checkParticipants, + string(participantsJSON), + string(environmentJSON), + workloadPolicyJSON, + strconv.FormatInt(int64(check.GPUCount), 10), + ), + }, + } + + if index > 0 { + previous := checkTaskName(&checks[index-1]) + task["depends"] = previous + ".Succeeded" + } + + tasks = append(tasks, task) + } + + workflow := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": workflowGVK.GroupVersion().String(), + "kind": workflowGVK.Kind, + "metadata": map[string]any{ + "name": name, + "namespace": run.Namespace, + "labels": map[string]any{ + SnapshotRunLabel: run.Name, + }, + }, + "spec": map[string]any{ + "activeDeadlineSeconds": int64(run.Spec.Deadline.Seconds()), + "serviceAccountName": adapterServiceAccountName, + "entrypoint": "certification-suite", + "templates": []any{ + map[string]any{ + "name": "certification-suite", + "dag": map[string]any{ + "failFast": false, + "tasks": tasks, + }, + }, + }, + }, + }} + workflow.SetOwnerReferences([]metav1.OwnerReference{ + *metav1.NewControllerRef(run, signalv1.GroupVersion.WithKind("CertificationRun")), + }) + + return workflow, nil +} + +func checkWorkflowParameters( + run *signalv1.CertificationRun, + check *signalv1.CertificationCheckSnapshot, + participants []signalv1.Participant, + participantsJSON string, + scriptEnvironmentJSON string, + workloadPolicyJSON string, + gpuCount string, +) []any { + return []any{ + map[string]any{"name": "run-name", "value": checkRunName(run.Name, check.ID)}, + map[string]any{"name": "run-uid", "value": string(run.UID)}, + map[string]any{"name": "participants-json", "value": participantsJSON}, + map[string]any{"name": "participant-count", "value": fmt.Sprintf("%d", len(participants))}, + map[string]any{"name": "script-configmap", "value": run.Spec.ScriptBundle.Name}, + map[string]any{"name": "script-key", "value": check.Script.Key}, + map[string]any{"name": "script-digest", "value": check.Script.Digest}, + map[string]any{"name": "backend-kind", "value": string(check.Backend)}, + map[string]any{ + "name": "backend-active-deadline-seconds", + "value": fmt.Sprintf("%d", int64(check.BackendActiveDeadline.Seconds())), + }, + map[string]any{"name": "runner-image", "value": check.RunnerImage}, + map[string]any{"name": "script-environment", "value": scriptEnvironmentJSON}, + map[string]any{"name": "workload-policy", "value": workloadPolicyJSON}, + map[string]any{"name": "gpu-resource-count", "value": gpuCount}, + } +} + +func validateGPUCount(gpuCount int32, backend signalv1.ExecutionBackend) error { + if backend != signalv1.ExecutionBackendPod && backend != signalv1.ExecutionBackendMPIJob { + return fmt.Errorf("unsupported backend %q", backend) + } + + if gpuCount < 0 || (backend == signalv1.ExecutionBackendMPIJob && gpuCount == 0) { + return errors.New("must be nonnegative and positive for MPIJob") + } + + return nil +} + +func validEnvironmentName(value string) bool { + if value == "" || (value[0] != '_' && (value[0] < 'A' || value[0] > 'Z')) { + return false + } + + for index := 1; index < len(value); index++ { + character := value[index] + if (character < 'A' || character > 'Z') && + (character < '0' || character > '9') && + character != '_' { + return false + } + } + + return true +} + +func validateScriptEnvironment(environment map[string]string) error { + if len(environment) > 32 { + return errors.New("must not contain more than 32 entries") + } + + keys := make([]string, 0, len(environment)) + for key := range environment { + keys = append(keys, key) + } + + sort.Strings(keys) + + for _, name := range keys { + if len(name) > 253 { + return fmt.Errorf("name %q exceeds 253 bytes", name) + } + + if !validEnvironmentName(name) { + return fmt.Errorf("name %q is invalid", name) + } + + if strings.HasPrefix(name, "PROJECT_SIGNAL_") { + return fmt.Errorf("name %q is reserved", name) + } + + if _, reserved := reservedScriptEnvironmentNames[name]; reserved { + return fmt.Errorf("name %q is reserved", name) + } + + if len(environment[name]) > 4096 { + return fmt.Errorf("value for %q exceeds 4096 bytes", name) + } + } + + return nil +} + +func (r *RunReconciler) finishGrouped( + ctx context.Context, + run *signalv1.CertificationRun, + workflow *unstructured.Unstructured, + workflowPhase string, +) (ctrl.Result, error) { + checks, verdict, reason := terminalCheckStatuses(workflow, run) + executionState := "Completed" + + if workflowPhase != "Succeeded" { + executionState = workflowPhase + verdict = "Inconclusive" + + if reason == "" { + reason = "Workflow" + workflowPhase + ": " + workflowMessage(workflow) + } + } + + evidenceRun := run.DeepCopy() + evidenceRun.Status.Checks = checks + evidenceRun.Status.Reason = reason + evidenceRun.Status.Phase = runPhaseForVerdict(verdict) + evidenceRun.Status.Workflow.Phase = workflowPhase + + finishedAt := groupedFinishedAt(workflow, run) + evidenceRun.Status.FinishedAt = &finishedAt + + evidence, err := r.evidenceDocument( + ctx, + evidenceRun, + executionState, + verdict, + reason, + ) + if err != nil { + return ctrl.Result{}, err + } + + uri, digest, err := r.Evidence.Put(ctx, evidencePath(run), evidence) + if err != nil { + return ctrl.Result{}, err + } + + before := run.DeepCopy() + run.Status = evidenceRun.Status + run.Status.Checks = compactCheckStatuses(checks) + run.Status.Evidence = signalv1.EvidenceStatus{URI: uri, Digest: digest} + run.Status.Phase = runPhaseForVerdict(verdict) + run.Status.Reason = boundedStatusText(reason) + + if err := r.Status().Patch(ctx, run, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, err + } + + if err := projectTerminal(ctx, r.Client, run); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{RequeueAfter: time.Millisecond}, nil +} + +func (r *RunReconciler) resumeGroupedEvidence( + ctx context.Context, + run *signalv1.CertificationRun, +) (ctrl.Result, bool, error) { + data, uri, digest, found, err := r.Evidence.Get(ctx, evidencePath(run)) + if err != nil || !found { + return ctrl.Result{}, false, err + } + + var document struct { + RunID string `json:"runID"` + RunName string `json:"runName"` + Workflow signalv1.WorkflowStatus `json:"workflow"` + CheckStatus []evidenceCheckStatus `json:"checkStatus"` + ExecutionState string `json:"executionState"` + Verdict string `json:"verdict"` + Reason string `json:"reason"` + Phase signalv1.RunPhase `json:"phase"` + FinishedAt string `json:"finishedAt"` + } + if err := json.Unmarshal(data, &document); err != nil { + return ctrl.Result{}, false, fmt.Errorf("decode persisted grouped evidence: %w", err) + } + + if document.RunName != run.Name || document.RunID == "" || document.RunID != string(run.UID) { + return ctrl.Result{}, false, fmt.Errorf("persisted grouped evidence identity does not match run %s", run.Name) + } + + if document.Workflow.Name != "" { + if document.Workflow.Name != fmt.Sprintf("%s-a1", run.Name) || + (run.Status.Workflow.UID != "" && document.Workflow.UID != run.Status.Workflow.UID) { + return ctrl.Result{}, false, errors.New("persisted evidence Workflow identity does not match run") + } + + neverObserved := document.Workflow.UID == "" && + ((document.Phase == signalv1.RunPhaseInconclusive && document.Workflow.Phase == "Error") || + (document.Phase == signalv1.RunPhaseCancelled && document.Workflow.Phase == "Canceled")) + if document.Workflow.UID == "" && !neverObserved { + return ctrl.Result{}, false, errors.New("persisted evidence is missing the observed Workflow UID") + } + } + + if len(document.CheckStatus) != len(run.Spec.Checks) { + return ctrl.Result{}, false, fmt.Errorf( + "persisted grouped evidence contains %d check statuses, want %d", + len(document.CheckStatus), + len(run.Spec.Checks), + ) + } + + for index := range run.Spec.Checks { + if document.CheckStatus[index].ID != run.Spec.Checks[index].ID || + document.CheckStatus[index].Order != run.Spec.Checks[index].Order { + return ctrl.Result{}, false, fmt.Errorf( + "persisted grouped evidence check %d identity does not match run", + index, + ) + } + } + + finishedAt, err := time.Parse(time.RFC3339Nano, document.FinishedAt) + if err != nil { + return ctrl.Result{}, false, fmt.Errorf("parse persisted grouped evidence finishedAt: %w", err) + } + + before := run.DeepCopy() + run.Status.Checks = compactEvidenceCheckStatuses(document.CheckStatus) + run.Status.Reason = boundedStatusText(document.Reason) + run.Status.Workflow = document.Workflow + run.Status.Evidence = signalv1.EvidenceStatus{URI: uri, Digest: digest} + + if document.Phase == "" { + run.Status.Phase = runPhaseForVerdict(document.Verdict) + } else { + if !document.Phase.Terminal() { + return ctrl.Result{}, false, fmt.Errorf("persisted evidence contains nonterminal phase %q", document.Phase) + } + + run.Status.Phase = document.Phase + } + + value := metav1.NewTime(finishedAt) + run.Status.FinishedAt = &value + + if err := r.Status().Patch(ctx, run, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, true, err + } + + if err := projectTerminal(ctx, r.Client, run); err != nil { + return ctrl.Result{}, true, err + } + + return ctrl.Result{RequeueAfter: time.Millisecond}, true, nil +} + +func runPhaseForVerdict(verdict string) signalv1.RunPhase { + switch verdict { + case "Pass": + return signalv1.RunPhaseSucceeded + case "Fail": + return signalv1.RunPhaseFailed + default: + return signalv1.RunPhaseInconclusive + } +} + +func groupedFinishedAt( + workflow *unstructured.Unstructured, + run *signalv1.CertificationRun, +) metav1.Time { + if value, found, err := unstructured.NestedString(workflow.Object, "status", "finishedAt"); err == nil && found { + if parsed, parseErr := time.Parse(time.RFC3339, value); parseErr == nil { + return metav1.NewTime(parsed) + } + } + + nodes, found, err := unstructured.NestedMap(workflow.Object, "status", "nodes") + if err == nil && found { + var latest time.Time + + for index := range run.Spec.Checks { + node, count := exactCheckNode(nodes, workflow.GetName(), &run.Spec.Checks[index]) + if count != 1 { + continue + } + + parsed, parseErr := time.Parse(time.RFC3339, nodeString(node, "finishedAt")) + if parseErr == nil && parsed.After(latest) { + latest = parsed + } + } + + if !latest.IsZero() { + return metav1.NewTime(latest) + } + } + + if run.Status.StartedAt != nil { + return *run.Status.StartedAt.DeepCopy() + } + + if !run.CreationTimestamp.IsZero() { + return run.CreationTimestamp + } + + return metav1.NewTime(time.Unix(0, 0).UTC()) +} + +func initialCheckStatuses(checks []signalv1.CertificationCheckSnapshot) []signalv1.CheckStatus { + statuses := make([]signalv1.CheckStatus, len(checks)) + for index := range checks { + statuses[index] = signalv1.CheckStatus{ + ID: checks[index].ID, + Order: checks[index].Order, + Phase: signalv1.CheckPhasePending, + } + } + + return statuses +} + +func compactCheckStatuses(statuses []signalv1.CheckStatus) []signalv1.CheckStatus { + compacted := make([]signalv1.CheckStatus, len(statuses)) + for index := range statuses { + compacted[index] = *statuses[index].DeepCopy() + compacted[index].BackendEvidence.Content = "" + compacted[index].Reason = boundedStatusText(compacted[index].Reason) + } + + return compacted +} + +func boundedStatusText(value string) string { + runes := []rune(value) + if len(runes) <= maximumStatusTextRunes { + return value + } + + return string(runes[:maximumStatusTextRunes]) +} + +func contentDigest(content string) string { + sum := sha256.Sum256([]byte(content)) + + return fmt.Sprintf("sha256:%x", sum) +} + +func evidencePath(run *signalv1.CertificationRun) string { + return run.Name + ".json" +} + +func runningCheckStatuses( + workflow *unstructured.Unstructured, + run *signalv1.CertificationRun, +) []signalv1.CheckStatus { + statuses := initialCheckStatuses(run.Spec.Checks) + + nodes, found, err := unstructured.NestedMap(workflow.Object, "status", "nodes") + if err != nil || !found { + return statuses + } + + for index := range run.Spec.Checks { + node, count := exactCheckNode(nodes, workflow.GetName(), &run.Spec.Checks[index]) + if count != 1 { + continue + } + + statuses[index].WorkflowNodeName = nodeString(node, "name") + phase := nodeString(node, "phase") + + switch phase { + case "Running": + statuses[index].Phase = signalv1.CheckPhaseRunning + case "Succeeded": + statuses[index].Phase = signalv1.CheckPhaseSucceeded + case "Failed", "Error": + statuses[index].Phase = signalv1.CheckPhaseInconclusive + case "Omitted", "Skipped": + statuses[index].Phase = signalv1.CheckPhaseSkipped + } + + setCheckTimes(&statuses[index], node) + } + + return statuses +} + +func terminalCheckStatuses( + workflow *unstructured.Unstructured, + run *signalv1.CertificationRun, +) ([]signalv1.CheckStatus, string, string) { + statuses := initialCheckStatuses(run.Spec.Checks) + + nodes, found, err := unstructured.NestedMap(workflow.Object, "status", "nodes") + if err != nil || !found { + for index := range statuses { + statuses[index].Phase = signalv1.CheckPhaseInconclusive + statuses[index].Verdict = "Inconclusive" + statuses[index].Reason = "WorkflowNodesUnavailable" + } + + return statuses, "Inconclusive", "workflow task nodes are missing or malformed" + } + + aggregateVerdict := "Pass" + aggregateReason := "AllChecksPassed" + + for index := range run.Spec.Checks { + check := &run.Spec.Checks[index] + status := &statuses[index] + node, count := exactCheckNode(nodes, workflow.GetName(), check) + + if count != 1 { + status.Phase = signalv1.CheckPhaseInconclusive + status.Verdict = "Inconclusive" + status.Reason = "WorkflowTaskIdentityInvalid" + aggregateVerdict = "Inconclusive" + aggregateReason = "workflow did not contain exactly one task node for every check" + + continue + } + + status.WorkflowNodeName = nodeString(node, "name") + setCheckTimes(status, node) + + phase := nodeString(node, "phase") + switch phase { + case "Omitted", "Skipped": + status.Phase = signalv1.CheckPhaseSkipped + status.Verdict = "Inconclusive" + status.Reason = "NotRun" + + aggregateVerdict = "Inconclusive" + aggregateReason = "one or more checks were not run" + + continue + case "Failed", "Error": + status.Phase = signalv1.CheckPhaseInconclusive + status.Verdict = "Inconclusive" + status.Reason = "WorkflowTask" + phase + aggregateVerdict = "Inconclusive" + aggregateReason = "one or more check tasks failed at the workflow boundary" + + continue + case "Succeeded": + default: + status.Phase = signalv1.CheckPhaseInconclusive + status.Verdict = "Inconclusive" + status.Reason = "WorkflowTaskIncomplete" + aggregateVerdict = "Inconclusive" + aggregateReason = "one or more check tasks were incomplete" + + continue + } + + outputs, outputErr := nodeWorkflowParameters(node) + if outputErr != nil { + status.Phase = signalv1.CheckPhaseInconclusive + status.Verdict = "Inconclusive" + status.Reason = "OutputsRejected: " + outputErr.Error() + aggregateVerdict = "Inconclusive" + aggregateReason = "one or more check outputs were malformed or incomplete" + + continue + } + + status.Verdict = outputs["verdict"] + status.Reason = outputs["reason"] + status.Measurements = outputs["measurements"] + allowPartialBackendEvidence := status.Verdict == "Inconclusive" && + status.Reason == "BackendSetupFailed" + + _, globalIndexes, participantErr := participantsForCheck( + run.Spec.Participants, + check.ParticipantPolicy, + ) + if participantErr != nil { + status.Phase = signalv1.CheckPhaseInconclusive + status.Verdict = "Inconclusive" + status.Reason = "ParticipantPolicyInvalid: " + participantErr.Error() + aggregateVerdict = "Inconclusive" + aggregateReason = "one or more checks had invalid participant policy" + + continue + } + + normalizedEvidence, complete, evidenceErr := groupedBackendEvidenceForParticipants( + outputs["backend-evidence"], + globalIndexes, + outputs["backend-resources"], + check.Backend, + run.Namespace, + allowPartialBackendEvidence, + ) + if evidenceErr != nil { + status.Phase = signalv1.CheckPhaseInconclusive + status.Verdict = "Inconclusive" + status.Reason = "BackendEvidenceRejected: " + evidenceErr.Error() + aggregateVerdict = "Inconclusive" + aggregateReason = "one or more checks had invalid backend evidence" + + continue + } + + status.BackendEvidence = signalv1.BackendEvidenceStatus{ + Complete: complete, + Digest: contentDigest(normalizedEvidence), + Content: normalizedEvidence, + } + + resources, resourceErr := backendResourceReferencesForParticipants( + outputs["backend-resources"], + globalIndexes, + run.Namespace, + allowPartialBackendEvidence, + ) + if resourceErr != nil { + status.Phase = signalv1.CheckPhaseInconclusive + status.Verdict = "Inconclusive" + status.Reason = "BackendResourcesRejected: " + resourceErr.Error() + aggregateVerdict = "Inconclusive" + aggregateReason = "one or more checks had invalid backend identities" + + continue + } + + status.BackendResources = resources + status.Cleanup.Complete = false + status.Cleanup.Reason = "DeferredToController" + + if !complete || len(resources) == 0 { + status.Phase = signalv1.CheckPhaseInconclusive + status.Verdict = "Inconclusive" + status.Reason = "EvidenceOrCleanupIncomplete" + aggregateVerdict = "Inconclusive" + aggregateReason = "one or more checks lacked complete evidence or backend coverage" + + continue + } + + switch status.Verdict { + case "Pass": + status.Phase = signalv1.CheckPhaseSucceeded + case "Fail": + status.Phase = signalv1.CheckPhaseFailed + + if aggregateVerdict == "Pass" { + aggregateVerdict = "Fail" + aggregateReason = "OneOrMoreChecksFailed" + } + case "Inconclusive": + status.Phase = signalv1.CheckPhaseInconclusive + aggregateVerdict = "Inconclusive" + aggregateReason = "one or more checks were inconclusive" + default: + status.Phase = signalv1.CheckPhaseInconclusive + status.Verdict = "Inconclusive" + status.Reason = "InvalidVerdict" + aggregateVerdict = "Inconclusive" + aggregateReason = "one or more checks returned an invalid verdict" + } + } + + return statuses, aggregateVerdict, aggregateReason +} + +func exactCheckNode( + nodes map[string]any, + workflowName string, + check *signalv1.CertificationCheckSnapshot, +) (map[string]any, int) { + expected := workflowName + "." + checkTaskName(check) + + var match map[string]any + + count := 0 + + for _, raw := range nodes { + node, ok := raw.(map[string]any) + if !ok { + continue + } + + name, ok := node["name"].(string) + if ok && name == expected { + match = node + count++ + } + } + + return match, count +} + +func nodeWorkflowParameters(node map[string]any) (map[string]string, error) { + outputs, ok := node["outputs"].(map[string]any) + if !ok { + return nil, errors.New("task outputs are missing or malformed") + } + + parameters, ok := outputs["parameters"].([]any) + if !ok { + return nil, errors.New("task output parameters are missing or malformed") + } + + limits := map[string]int{ + "verdict": workflowOutputLimits["verdict"], + "reason": workflowOutputLimits["reason"], + "measurements": workflowOutputLimits["measurements"], + "backend-resources": workflowOutputLimits["backend-resources"], + "backend-evidence": workflowOutputLimits["backend-evidence"], + } + + decoded, err := decodeWorkflowParametersWithLimits(parameters, limits) + if err != nil { + return nil, err + } + + for _, required := range []string{ + "verdict", + "reason", + "measurements", + "backend-resources", + "backend-evidence", + } { + if _, exists := decoded[required]; !exists { + return nil, fmt.Errorf("required parameter %q is missing", required) + } + } + + return decoded, nil +} + +func setCheckTimes(status *signalv1.CheckStatus, node map[string]any) { + started := nodeString(node, "startedAt") + finished := nodeString(node, "finishedAt") + + if parsed, err := time.Parse(time.RFC3339, started); err == nil { + value := metav1.NewTime(parsed) + status.StartedAt = &value + } + + if parsed, err := time.Parse(time.RFC3339, finished); err == nil { + value := metav1.NewTime(parsed) + status.FinishedAt = &value + } +} + +func nodeString(node map[string]any, key string) string { + value, ok := node[key].(string) + if !ok { + return "" + } + + return value +} + +type evidenceBackendStatus struct { + Complete bool `json:"complete"` + Digest string `json:"digest,omitempty"` + Document json.RawMessage `json:"document,omitempty"` +} + +type evidenceCleanupStatus struct { + Complete bool `json:"complete"` + Reason string `json:"reason,omitempty"` +} + +type evidenceCheckStatus struct { + ID signalv1.CheckID `json:"id"` + Order int32 `json:"order"` + WorkflowNodeName string `json:"workflowNodeName,omitempty"` + Phase signalv1.CheckPhase `json:"phase"` + Verdict string `json:"verdict"` + Reason string `json:"reason,omitempty"` + Measurements string `json:"measurements,omitempty"` + BackendResources []signalv1.BackendResourceReference `json:"backendResources,omitempty"` + BackendEvidence evidenceBackendStatus `json:"backendEvidence"` + Cleanup evidenceCleanupStatus `json:"cleanup"` + StartedAt *metav1.Time `json:"startedAt,omitempty"` + FinishedAt *metav1.Time `json:"finishedAt,omitempty"` +} + +func evidenceCheckStatuses(statuses []signalv1.CheckStatus) ([]evidenceCheckStatus, error) { + result := make([]evidenceCheckStatus, len(statuses)) + + for index := range statuses { + status := &statuses[index] + + var document json.RawMessage + + if status.BackendEvidence.Content != "" { + document = json.RawMessage(status.BackendEvidence.Content) + if !json.Valid(document) { + return nil, fmt.Errorf("check %s backend evidence is not valid JSON", status.ID) + } + } + + result[index] = evidenceCheckStatus{ + ID: status.ID, + Order: status.Order, + WorkflowNodeName: status.WorkflowNodeName, + Phase: status.Phase, + Verdict: status.Verdict, + Reason: status.Reason, + Measurements: status.Measurements, + BackendResources: status.BackendResources, + BackendEvidence: evidenceBackendStatus{ + Complete: status.BackendEvidence.Complete, + Digest: status.BackendEvidence.Digest, + Document: document, + }, + Cleanup: evidenceCleanupStatus{ + Complete: status.Cleanup.Complete, + Reason: status.Cleanup.Reason, + }, + StartedAt: status.StartedAt, + FinishedAt: status.FinishedAt, + } + } + + return result, nil +} + +func compactEvidenceCheckStatuses(statuses []evidenceCheckStatus) []signalv1.CheckStatus { + result := make([]signalv1.CheckStatus, len(statuses)) + for index := range statuses { + status := &statuses[index] + result[index] = signalv1.CheckStatus{ + ID: status.ID, + Order: status.Order, + WorkflowNodeName: status.WorkflowNodeName, + Phase: status.Phase, + Verdict: status.Verdict, + Reason: boundedStatusText(status.Reason), + Measurements: status.Measurements, + BackendResources: status.BackendResources, + BackendEvidence: signalv1.BackendEvidenceStatus{ + Complete: status.BackendEvidence.Complete, + Digest: status.BackendEvidence.Digest, + }, + StartedAt: status.StartedAt, + FinishedAt: status.FinishedAt, + Cleanup: signalv1.CheckCleanupStatus{ + Complete: status.Cleanup.Complete, + Reason: status.Cleanup.Reason, + }, + } + } + + return result +} + +func (r *RunReconciler) evidenceDocument( + ctx context.Context, + run *signalv1.CertificationRun, + executionState string, + verdict string, + reason string, +) ([]byte, error) { + checks := run.Spec.Checks + if len(checks) == 0 { + return nil, errors.New("run has no certification checks") + } + + var snapshot corev1.ConfigMap + if err := r.Get( + ctx, + client.ObjectKey{Namespace: run.Namespace, Name: run.Spec.ScriptBundle.Name}, + &snapshot, + ); err != nil { + return nil, fmt.Errorf("read script snapshot for evidence: %w", err) + } + + if snapshot.UID != run.Spec.ScriptBundle.UID || snapshot.Labels[SnapshotRunLabel] != run.Name { + return nil, errors.New("script bundle identity does not match the run") + } + + scriptContents := make(map[string]string, len(checks)) + for index := range checks { + content, exists := snapshot.Data[checks[index].Script.Key] + if !exists { + return nil, fmt.Errorf("script snapshot is missing check %s key %s", checks[index].ID, checks[index].Script.Key) + } + + scriptContents[string(checks[index].ID)] = content + } + + checkStatuses, err := evidenceCheckStatuses(run.Status.Checks) + if err != nil { + return nil, err + } + + doc := map[string]any{ + "apiVersion": "signal.unbounded-cloud.io/evidence/v1alpha1", + "runID": string(run.UID), + "runName": run.Name, + "profile": run.Spec.Profile, + "participants": run.Spec.Participants, + "scriptBundle": run.Spec.ScriptBundle, + "workflow": run.Status.Workflow, + "checks": checks, + "checkStatus": checkStatuses, + "scriptContents": scriptContents, + "executionState": executionState, + "verdict": verdict, + "reason": reason, + "phase": run.Status.Phase, + "startedAt": run.Status.StartedAt, + "finishedAt": run.Status.FinishedAt.UTC().Format(time.RFC3339Nano), + } + + return json.MarshalIndent(doc, "", " ") +} + +func backendResourceReferences( + value string, + participantCount int, + _ string, + allowPartial bool, +) ([]signalv1.BackendResourceReference, error) { + if participantCount < 1 || participantCount > 64 { + return nil, fmt.Errorf("participant count must be between 1 and 64") + } + + if value == "" { + return nil, nil + } + + var references []signalv1.BackendResourceReference + + decoder := json.NewDecoder(strings.NewReader(value)) + decoder.DisallowUnknownFields() + + if err := decoder.Decode(&references); err != nil { + return nil, fmt.Errorf("decode backend-resources: %w", err) + } + + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return nil, fmt.Errorf("backend-resources contains trailing JSON values") + } + + if len(references) == 0 || len(references) > 64 { + return nil, fmt.Errorf("backend-resources must contain between 1 and 64 entries") + } + + identities := make(map[string]struct{}, len(references)) + covered := make([]bool, participantCount) + + for index := range references { + reference := &references[index] + if reference.Name == "" || reference.UID == "" { + return nil, fmt.Errorf("backend-resources[%d] has an incomplete identity", index) + } + + identity := strings.Join([]string{ + reference.Name, + string(reference.UID), + }, "\x00") + if _, exists := identities[identity]; exists { + return nil, fmt.Errorf("backend-resources[%d] duplicates a workload identity", index) + } + + identities[identity] = struct{}{} + + if len(reference.ParticipantIndexes) == 0 { + return nil, fmt.Errorf("backend-resources[%d] does not identify a participant", index) + } + + seenIndexes := make(map[int32]struct{}, len(reference.ParticipantIndexes)) + for _, participantIndex := range reference.ParticipantIndexes { + if participantIndex < 0 || int(participantIndex) >= participantCount { + return nil, fmt.Errorf( + "backend-resources[%d] participant index %d is out of range", + index, + participantIndex, + ) + } + + if _, exists := seenIndexes[participantIndex]; exists { + return nil, fmt.Errorf( + "backend-resources[%d] repeats participant index %d", + index, + participantIndex, + ) + } + + seenIndexes[participantIndex] = struct{}{} + covered[participantIndex] = true + } + } + + if !allowPartial && participantCount > 1 { + for index, isCovered := range covered { + if !isCovered { + return nil, fmt.Errorf("backend-resources does not cover participant index %d", index) + } + } + } + + return references, nil +} + +func backendResourceReferencesForParticipants( + value string, + globalIndexes []int32, + namespace string, + allowPartial bool, +) ([]signalv1.BackendResourceReference, error) { + references, err := backendResourceReferences(value, len(globalIndexes), namespace, allowPartial) + if err != nil { + return nil, err + } + + for referenceIndex := range references { + for participantIndex := range references[referenceIndex].ParticipantIndexes { + localIndex := references[referenceIndex].ParticipantIndexes[participantIndex] + references[referenceIndex].ParticipantIndexes[participantIndex] = globalIndexes[localIndex] + } + } + + return references, nil +} + +func decodeWorkflowParametersWithLimits(parameters []any, limits map[string]int) (map[string]string, error) { + outputs := make(map[string]string, len(parameters)) + for _, raw := range parameters { + parameter, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("parameter entry is not an object") + } + + name, nameOK := parameter["name"].(string) + value, valueOK := parameter["value"].(string) + + if !nameOK || !valueOK { + return nil, fmt.Errorf("parameter name and value must be strings") + } + + limit, allowed := limits[name] + if !allowed { + return nil, fmt.Errorf("unexpected parameter %q", name) + } + + if len(value) > limit { + return nil, fmt.Errorf("parameter %q exceeds %d bytes", name, limit) + } + + if _, duplicate := outputs[name]; duplicate { + return nil, fmt.Errorf("duplicate parameter %q", name) + } + + outputs[name] = value + } + + return outputs, nil +} + +func groupedBackendEvidenceComplete( + content string, + participantCount int, + references []signalv1.BackendResourceReference, + backend signalv1.ExecutionBackend, + namespace string, + allowPartial bool, +) (bool, error) { + if err := signaladapter.ValidateBackendEvidence(content); err != nil { + return false, err + } + + var evidence signaladapter.BackendEvidence + if err := json.Unmarshal([]byte(content), &evidence); err != nil { + return false, err + } + + if len(evidence.Participants) > participantCount || (!allowPartial && len(evidence.Participants) != participantCount) { + return false, fmt.Errorf( + "backend-evidence contains %d participants, want at most %d", + len(evidence.Participants), + participantCount, + ) + } + + seen := make([]bool, participantCount) + complete := !allowPartial && evidence.Complete && evidence.CaptureError == "" + podUIDs := map[types.UID]struct{}{} + + expectedAPIVersion, expectedKind, err := backendIdentity(backend) + if err != nil { + return false, err + } + + for index := range evidence.Participants { + participant := &evidence.Participants[index] + if participant.ParticipantIndex < 0 || participant.ParticipantIndex >= participantCount { + return false, fmt.Errorf( + "backend-evidence participant index %d is out of range", + participant.ParticipantIndex, + ) + } + + if seen[participant.ParticipantIndex] { + return false, fmt.Errorf( + "backend-evidence repeats participant index %d", + participant.ParticipantIndex, + ) + } + + seen[participant.ParticipantIndex] = true + + if participant.Resource.APIVersion == "" || + participant.Resource.Kind == "" || + participant.Resource.Name == "" || + participant.Resource.UID == "" { + return false, fmt.Errorf( + "backend-evidence participant %d has incomplete resource identity", + participant.ParticipantIndex, + ) + } + + matches := 0 + + for _, reference := range references { + if !slices.Contains(reference.ParticipantIndexes, int32(participant.ParticipantIndex)) { + continue + } + + if reference.Name != participant.Resource.Name || + reference.UID != participant.Resource.UID { + return false, fmt.Errorf("backend-evidence participant %d resource differs from backend-resources", participant.ParticipantIndex) + } + + matches++ + } + + if matches != 1 || participant.Resource.APIVersion != expectedAPIVersion || + participant.Resource.Kind != expectedKind || participant.Resource.Namespace != namespace { + return false, fmt.Errorf("backend-evidence participant %d does not match the declared backend", participant.ParticipantIndex) + } + + complete = complete && participant.CaptureError == "" && len(participant.Pods) > 0 + launcher := false + + for _, pod := range participant.Pods { + if pod.UID == "" || pod.Name == "" { + return false, errors.New("backend-evidence Pod identity is incomplete") + } + + if _, duplicate := podUIDs[pod.UID]; duplicate { + return false, errors.New("backend-evidence repeats a Pod UID") + } + + podUIDs[pod.UID] = struct{}{} + if backend == signalv1.ExecutionBackendPod && (pod.UID != participant.Resource.UID || + pod.Name != participant.Resource.Name || pod.Role != "participant" || len(participant.Pods) != 1) { + return false, errors.New("backend-evidence Pod differs from its backend resource") + } + + launcher = launcher || pod.Role == "launcher" + + complete = complete && pod.Phase != "" && len(pod.Containers) > 0 + for _, container := range pod.Containers { + complete = complete && container.Name != "" && container.State.State != "" && + container.CurrentLog.Status == "captured" && container.CurrentLog.Error == "" && + (container.PreviousLog.Status == "captured" || container.PreviousLog.Status == "not-applicable") && + container.PreviousLog.Error == "" + if container.RestartCount > 0 || container.LastState.State == "terminated" { + complete = complete && container.PreviousLog.Status == "captured" + } + } + } + + if backend == signalv1.ExecutionBackendMPIJob { + complete = complete && launcher && participant.LauncherJob != nil && + participant.LauncherJob.Name != "" && participant.LauncherJob.UID != "" + } + } + + return complete, nil +} + +func backendIdentity(backend signalv1.ExecutionBackend) (string, string, error) { + switch backend { + case signalv1.ExecutionBackendPod: + return "v1", "Pod", nil + case signalv1.ExecutionBackendMPIJob: + return "kubeflow.org/v2beta1", "MPIJob", nil + default: + return "", "", fmt.Errorf("unsupported backend %q", backend) + } +} + +func groupedBackendEvidenceForParticipants( + content string, + globalIndexes []int32, + resources string, + backend signalv1.ExecutionBackend, + namespace string, + allowPartial bool, +) (string, bool, error) { + references, err := backendResourceReferences(resources, len(globalIndexes), namespace, allowPartial) + if err != nil { + return "", false, err + } + + complete, err := groupedBackendEvidenceComplete( + content, + len(globalIndexes), + references, + backend, + namespace, + allowPartial, + ) + if err != nil { + return "", false, err + } + + var evidence map[string]json.RawMessage + if err := json.Unmarshal([]byte(content), &evidence); err != nil { + return "", false, fmt.Errorf("decode backend-evidence for participant mapping: %w", err) + } + + var participants []map[string]json.RawMessage + if err := json.Unmarshal(evidence["participants"], &participants); err != nil { + return "", false, fmt.Errorf("decode backend-evidence participants for mapping: %w", err) + } + + for index := range participants { + var localIndex int + if err := json.Unmarshal(participants[index]["participantIndex"], &localIndex); err != nil { + return "", false, fmt.Errorf("decode backend-evidence participant index: %w", err) + } + + mapped, err := json.Marshal(globalIndexes[localIndex]) + if err != nil { + return "", false, fmt.Errorf("encode mapped backend-evidence participant index: %w", err) + } + + participants[index]["participantIndex"] = mapped + } + + mappedParticipants, err := json.Marshal(participants) + if err != nil { + return "", false, fmt.Errorf("encode mapped backend-evidence participants: %w", err) + } + + evidence["participants"] = mappedParticipants + + normalized, err := json.Marshal(evidence) + if err != nil { + return "", false, fmt.Errorf("encode mapped backend-evidence: %w", err) + } + + return string(normalized), complete, nil +} + +func (r *RunReconciler) cancel(ctx context.Context, run *signalv1.CertificationRun) (ctrl.Result, error) { + evidenceRun := run.DeepCopy() + evidenceRun.Status.Phase = signalv1.RunPhaseCancelled + evidenceRun.Status.Reason = "canceled by operator" + + if !run.DeletionTimestamp.IsZero() { + evidenceRun.Status.Reason = "canceled by deletion" + } + + evidenceRun.Status.Workflow.Phase = "Canceled" + + statuses := initialCheckStatuses(run.Spec.Checks) + + workflow, err := r.workflowForCancellation(ctx, run) + if err != nil { + return ctrl.Result{}, err + } + + if workflow != nil { + evidenceRun.Status.Workflow.Name = workflow.GetName() + evidenceRun.Status.Workflow.UID = string(workflow.GetUID()) + + if phase, found, phaseErr := unstructured.NestedString(workflow.Object, "status", "phase"); phaseErr != nil { + return ctrl.Result{}, fmt.Errorf("read Workflow phase during cancellation: %w", phaseErr) + } else if found { + evidenceRun.Status.Workflow.Phase = phase + } + + completed, _, _ := terminalCheckStatuses(workflow, run) + + nodes, found, nodesErr := unstructured.NestedMap(workflow.Object, "status", "nodes") + if nodesErr != nil { + return ctrl.Result{}, fmt.Errorf("read Workflow nodes during cancellation: %w", nodesErr) + } + + if found { + for index := range run.Spec.Checks { + node, count := exactCheckNode(nodes, workflow.GetName(), &run.Spec.Checks[index]) + + if count == 1 && nodeString(node, "phase") == "Succeeded" { + statuses[index] = completed[index] + } + } + } + } + + for index := range statuses { + if statuses[index].Phase == signalv1.CheckPhaseSucceeded || + statuses[index].Phase == signalv1.CheckPhaseFailed || + statuses[index].Phase == signalv1.CheckPhaseInconclusive { + continue + } + + statuses[index].Phase = signalv1.CheckPhaseSkipped + statuses[index].Verdict = "Inconclusive" + statuses[index].Reason = "Canceled" + } + + evidenceRun.Status.Checks = statuses + + now := metav1.NewTime(r.now()) + evidenceRun.Status.FinishedAt = &now + + evidence, err := r.evidenceDocument( + ctx, + evidenceRun, + "Interrupted", + "Inconclusive", + evidenceRun.Status.Reason, + ) + if err != nil { + return ctrl.Result{}, err + } + + uri, digest, err := r.Evidence.Put(ctx, evidencePath(run), evidence) + if err != nil { + return ctrl.Result{}, err + } + + before := run.DeepCopy() + run.Status = evidenceRun.Status + run.Status.Evidence = signalv1.EvidenceStatus{URI: uri, Digest: digest} + + if err := r.Status().Patch(ctx, run, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{RequeueAfter: time.Millisecond}, nil +} + +func (r *RunReconciler) workflowForCancellation( + ctx context.Context, + run *signalv1.CertificationRun, +) (*unstructured.Unstructured, error) { + name := run.Status.Workflow.Name + if name == "" { + name = fmt.Sprintf("%s-a1", run.Name) + } + + workflow := &unstructured.Unstructured{} + workflow.SetGroupVersionKind(workflowGVK) + + reader := r.APIReader + if reader == nil { + reader = r.Client + } + + err := reader.Get( + ctx, + client.ObjectKey{Namespace: run.Namespace, Name: name}, + workflow, + ) + if apierrors.IsNotFound(err) { + return nil, nil + } + + if err != nil { + return nil, err + } + + if !metav1.IsControlledBy(workflow, run) || + (run.Status.Workflow.UID != "" && string(workflow.GetUID()) != run.Status.Workflow.UID) { + return nil, errors.New("workflow identity changed during cancellation") + } + + return workflow, nil +} + +func (r *RunReconciler) cleanup(ctx context.Context, run *signalv1.CertificationRun) (ctrl.Result, error) { + holdParticipants := run.Status.Phase != signalv1.RunPhaseSucceeded && !run.Spec.Release + if holdParticipants { + if _, err := r.ensureParticipantsCordoned(ctx, run); err != nil { + return ctrl.Result{}, err + } + } + + stopped, err := r.ensureWorkflowStopped(ctx, run) + if err != nil { + return ctrl.Result{}, err + } + + if !stopped { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + + if err := projectTerminal(ctx, r.Client, run); err != nil { + return ctrl.Result{}, err + } + + if err := r.deleteScriptBundle(ctx, run); err != nil { + return ctrl.Result{}, err + } + + if err := r.patchRunCondition( + ctx, + run, + signalv1.RunConditionCleanup, + metav1.ConditionTrue, + "WorkloadsDeleted", + "Workflow, backend descendants, and script bundle are deleted", + ); err != nil { + return ctrl.Result{}, err + } + + if holdParticipants { + if err := r.patchRunCondition( + ctx, + run, + signalv1.RunConditionReleased, + metav1.ConditionFalse, + "AwaitingExplicitRelease", + "Participants remain reserved and cordoned until spec.release is true", + ); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } + + if err := r.clearRequest(ctx, run); err != nil { + return ctrl.Result{}, err + } + + if _, err := r.releaseRunReservations(ctx, run); err != nil { + return ctrl.Result{}, err + } + + if err := r.patchRunCondition( + ctx, + run, + signalv1.RunConditionReleased, + metav1.ConditionTrue, + "ParticipantsReleased", + "Participant reservations and Signal-owned cordons are released", + ); err != nil { + return ctrl.Result{}, err + } + + if err := r.patchRunCondition( + ctx, + run, + signalv1.RunConditionCordoned, + metav1.ConditionFalse, + "ParticipantsReleased", + "Signal no longer owns participant cordons", + ); err != nil { + return ctrl.Result{}, err + } + + before := run.DeepCopy() + + run.Finalizers = removeString(run.Finalizers, RunFinalizer) + if err := r.Patch(ctx, run, client.MergeFrom(before)); err != nil { + return ctrl.Result{}, err + } + + if r.Recorder != nil { + r.Recorder.Eventf(run, nil, corev1.EventTypeNormal, "CertificationCleaned", "CleanCertification", + "Deleted certification workloads and released participant reservations and cordons") + } + + return ctrl.Result{}, nil +} + +func (r *RunReconciler) deleteScriptBundle(ctx context.Context, run *signalv1.CertificationRun) error { + snapshot := &corev1.ConfigMap{} + + reader := r.APIReader + if reader == nil { + reader = r.Client + } + + err := reader.Get( + ctx, + client.ObjectKey{Namespace: run.Namespace, Name: run.Spec.ScriptBundle.Name}, + snapshot, + ) + if apierrors.IsNotFound(err) { + return nil + } + + if err != nil { + return err + } + + if snapshot.UID != run.Spec.ScriptBundle.UID || snapshot.Labels[SnapshotRunLabel] != run.Name { + return errors.New("refusing to delete script bundle with changed identity") + } + + uid := snapshot.UID + + return client.IgnoreNotFound(r.Delete(ctx, snapshot, client.Preconditions{UID: &uid})) +} + +func (r *RunReconciler) reconcileDeleting(ctx context.Context, run *signalv1.CertificationRun) (ctrl.Result, error) { + if !containsString(run.Finalizers, RunFinalizer) { + return ctrl.Result{}, nil + } + + if !run.Status.Phase.Terminal() && run.Status.Evidence.URI == "" { + result, recovered, err := r.resumeGroupedEvidence(ctx, run) + if err != nil || recovered { + return result, err + } + } + + if !run.Status.Phase.Terminal() && run.Status.Workflow.Name != "" { + return r.cancel(ctx, run) + } + + stopped, err := r.ensureWorkflowStopped(ctx, run) + if err != nil { + return ctrl.Result{}, err + } + + if !stopped { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + + if err := r.clearRequest(ctx, run); err != nil { + return ctrl.Result{}, err + } + + if _, err := r.releaseRunReservations(ctx, run); err != nil { + return ctrl.Result{}, err + } + + if err := r.deleteScriptBundle(ctx, run); err != nil { + return ctrl.Result{}, err + } + + before := run.DeepCopy() + run.Finalizers = removeString(run.Finalizers, RunFinalizer) + + return ctrl.Result{}, r.Patch(ctx, run, client.MergeFrom(before)) +} + +func (r *RunReconciler) releaseRunReservations(ctx context.Context, run *signalv1.CertificationRun) (bool, error) { + for _, participant := range run.Spec.Participants { + var node corev1.Node + if err := r.Get(ctx, client.ObjectKey{Name: participant.NodeName}, &node); err != nil { + if apierrors.IsNotFound(err) { + continue + } + + return false, err + } + + if participant.NodeUID != "" && node.UID != participant.NodeUID { + continue + } + + if node.Annotations[ReservationLabel] != run.Name { + continue + } + + if node.Labels[signalv1.ReservationNodeUIDLabel] != string(participant.NodeUID) { + continue + } + + before := node.DeepCopy() + node.Spec.Unschedulable = false + delete(node.Annotations, ReservationLabel) + delete(node.Labels, signalv1.ReservationNodeUIDLabel) + + if err := r.Patch(ctx, &node, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil { + return false, err + } + } + + return true, nil +} + +func (r *RunReconciler) clearRequest(ctx context.Context, run *signalv1.CertificationRun) error { + if len(run.Spec.Participants) == 0 { + return errors.New("run has no requesting participant") + } + + requestNode := run.Spec.Participants[0] + + var node corev1.Node + if err := r.Get(ctx, client.ObjectKey{Name: requestNode.NodeName}, &node); err != nil { + return client.IgnoreNotFound(err) + } + + if node.UID != requestNode.NodeUID { + return nil + } + + if node.Labels[RequestLabel] != run.Spec.Profile.Name || node.Annotations[ReservationLabel] != run.Name { + return nil + } + + before := node.DeepCopy() + delete(node.Labels, RequestLabel) + + return r.Patch(ctx, &node, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})) +} + +func (r *RunReconciler) ensureWorkflowStopped(ctx context.Context, run *signalv1.CertificationRun) (bool, error) { + name := run.Status.Workflow.Name + if name == "" { + name = fmt.Sprintf("%s-a1", run.Name) + } + + workflow := &unstructured.Unstructured{} + workflow.SetGroupVersionKind(workflowGVK) + + key := client.ObjectKey{Namespace: run.Namespace, Name: name} + reader := r.APIReader + + if reader == nil { + reader = r.Client + } + + if err := reader.Get(ctx, key, workflow); err != nil { + if apierrors.IsNotFound(err) { + return r.runWorkloadsGone(ctx, reader, run) + } + + return false, fmt.Errorf("get Workflow before deletion: %w", err) + } + + if (run.Status.Workflow.UID != "" && string(workflow.GetUID()) != run.Status.Workflow.UID) || + !metav1.IsControlledBy(workflow, run) { + return false, fmt.Errorf("refusing to delete Workflow with changed identity or ownership") + } + + if err := r.fenceWorkflowDependents(ctx, reader, workflow, run); err != nil { + return false, err + } + + propagation := metav1.DeletePropagationForeground + uid := workflow.GetUID() + + if err := r.Delete( + ctx, + workflow, + client.PropagationPolicy(propagation), + client.Preconditions{UID: &uid}, + ); err != nil && !apierrors.IsNotFound(err) { + return false, fmt.Errorf("delete Workflow before releasing participants: %w", err) + } + + if err := reader.Get(ctx, key, workflow); err != nil { + if apierrors.IsNotFound(err) { + return r.runWorkloadsGone(ctx, reader, run) + } + + return false, fmt.Errorf("confirm Workflow deletion: %w", err) + } + + return false, nil +} + +func (r *RunReconciler) runWorkloadsGone(ctx context.Context, reader client.Reader, run *signalv1.CertificationRun) (bool, error) { + for _, check := range run.Spec.Checks { + for _, kind := range []schema.GroupVersionKind{ + corev1.SchemeGroupVersion.WithKind("Pod"), + {Group: "batch", Version: "v1", Kind: "Job"}, + {Group: "kubeflow.org", Version: "v2beta1", Kind: "MPIJob"}, + } { + if kind.Kind != "Pod" && check.Backend != signalv1.ExecutionBackendMPIJob { + continue + } + + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(kind.GroupVersion().WithKind(kind.Kind + "List")) + + if err := reader.List(ctx, list, client.InNamespace(run.Namespace), + client.MatchingLabels{SnapshotRunLabel: checkRunName(run.Name, check.ID)}); err != nil { + return false, fmt.Errorf("confirm %s cleanup: %w", kind.Kind, err) + } + + if len(list.Items) > 0 { + return false, nil + } + } + } + + return true, nil +} + +func (r *RunReconciler) fenceWorkflowDependents( + ctx context.Context, + reader client.Reader, + workflow *unstructured.Unstructured, + run *signalv1.CertificationRun, +) error { + owners := map[types.UID]struct{}{workflow.GetUID(): {}} + kinds := []schema.GroupVersionKind{} + + for _, check := range run.Spec.Checks { + if check.Backend == signalv1.ExecutionBackendMPIJob { + kinds = append(kinds, schema.GroupVersionKind{Group: "kubeflow.org", Version: "v2beta1", Kind: "MPIJob"}) + + break + } + } + + kinds = append( + kinds, + schema.GroupVersionKind{Group: "batch", Version: "v1", Kind: "Job"}, + corev1.SchemeGroupVersion.WithKind("Pod"), + ) + + // Fence every level before deleting the Workflow so foreground GC also waits + // for existing backend descendants whose original owner reference did not block. + for _, kind := range kinds { + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(kind.GroupVersion().WithKind(kind.Kind + "List")) + + if err := reader.List(ctx, list, client.InNamespace(run.Namespace)); err != nil { + return fmt.Errorf("list %s before Workflow cleanup: %w", kind.Kind, err) + } + + for index := range list.Items { + object := &list.Items[index] + references := object.GetOwnerReferences() + owned := false + changed := false + + for ownerIndex := range references { + if _, found := owners[references[ownerIndex].UID]; !found { + continue + } + + owned = true + + if references[ownerIndex].BlockOwnerDeletion == nil || !*references[ownerIndex].BlockOwnerDeletion { + block := true + references[ownerIndex].BlockOwnerDeletion = &block + changed = true + } + } + + if !owned { + continue + } + + owners[object.GetUID()] = struct{}{} + if changed { + before := object.DeepCopy() + object.SetOwnerReferences(references) + + if err := r.Patch(ctx, object, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil { + return fmt.Errorf("fence %s %s before Workflow cleanup: %w", kind.Kind, object.GetName(), err) + } + } + } + } + + return nil +} + +func workflowMessage(workflow *unstructured.Unstructured) string { + message, _, err := unstructured.NestedString(workflow.Object, "status", "message") + if err != nil { + return "workflow message is malformed" + } + + if message == "" { + return "workflow did not report a message" + } + + return message +} + +func digestString(content string) string { + return digestBytes([]byte(content)) +} + +func digestBytes(content []byte) string { + sum := sha256.Sum256(content) + + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func runName(node, profile string, now time.Time) string { + base := strings.ToLower(node + "-" + profile) + base = strings.Map(func(r rune) rune { + if r >= 'a' && r <= 'z' || r >= '0' && r <= '9' || r == '-' { + return r + } + + return '-' + }, base) + + base = strings.Trim(base, "-") + if len(base) > 40 { + base = strings.TrimRight(base[:40], "-") + } + + return fmt.Sprintf("%s-%x-%s", base, now.Unix(), rand.String(5)) +} + +func (r *AdmissionReconciler) now() time.Time { + if r.Now != nil { + return r.Now() + } + + return time.Now() +} + +func (r *RunReconciler) now() time.Time { + if r.Now != nil { + return r.Now() + } + + return time.Now() +} + +func containsString(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + + return false +} + +func removeString(values []string, remove string) []string { + result := values[:0] + for _, value := range values { + if value != remove { + result = append(result, value) + } + } + + return result +} + +func profileChecks(profile *signalv1.CertificationProfile) []signalv1.CertificationCheck { + checks := make([]signalv1.CertificationCheck, len(profile.Spec.Checks)) + for index := range profile.Spec.Checks { + checks[index] = profile.Spec.Checks[index] + checks[index].ID = signalv1.CheckID(fmt.Sprintf("R%02d", index+1)) + checks[index].Order = int32(index) + checks[index].WorkloadPolicy = *profile.Spec.Checks[index].WorkloadPolicy.DeepCopy() + checks[index].ScriptEnvironment = maps.Clone(profile.Spec.Checks[index].ScriptEnvironment) + checks[index].ParticipantPolicy.Roles = slices.Clone(profile.Spec.Checks[index].ParticipantPolicy.Roles) + } + + return checks +} + +func validateCheckPlan(checks []signalv1.CertificationCheck, deadline time.Duration) error { + if len(checks) != groupedCheckCount { + return fmt.Errorf("must contain exactly %d checks", groupedCheckCount) + } + + for index := range checks { + check := &checks[index] + if err := validateCheckParticipantPolicy(check.ID, check.ParticipantPolicy); err != nil { + return err + } + + deadlines := make([]time.Duration, len(checks)) + for index := range checks { + deadlines[index] = checks[index].BackendActiveDeadline.Duration + } + + if err := signalv1.ValidateSuiteDeadlines(deadlines, deadline); err != nil { + return err + } + } + + if fixedWorkflowOutputBytes() > maximumWorkflowOutputBytes { + return fmt.Errorf( + "fixed aggregate check output budget is %d bytes, maximum is %d", + fixedWorkflowOutputBytes(), + maximumWorkflowOutputBytes, + ) + } + + return nil +} + +func validateRunCheckPlan(checks []signalv1.CertificationCheckSnapshot, deadline time.Duration) error { + if len(checks) != groupedCheckCount { + return fmt.Errorf("run must contain exactly %d checks", groupedCheckCount) + } + + for index := range checks { + check := &checks[index] + check.ID = signalv1.CheckID(fmt.Sprintf("R%02d", index+1)) + check.Order = int32(index) + } + + deadlines := make([]time.Duration, len(checks)) + for index := range checks { + deadlines[index] = checks[index].BackendActiveDeadline.Duration + } + + if err := signalv1.ValidateSuiteDeadlines(deadlines, deadline); err != nil { + return err + } + + if fixedWorkflowOutputBytes() > maximumWorkflowOutputBytes { + return fmt.Errorf( + "fixed run aggregate check output budget is %d bytes, maximum is %d", + fixedWorkflowOutputBytes(), + maximumWorkflowOutputBytes, + ) + } + + return nil +} + +func fixedWorkflowOutputBytes() int64 { + perCheck := int64( + workflowOutputLimits["verdict"] + + workflowOutputLimits["reason"] + + workflowOutputLimits["measurements"] + + workflowOutputLimits["backend-resources"] + + workflowOutputLimits["backend-evidence"], + ) + + return groupedCheckCount * perCheck +} + +func validateCheckParticipantPolicy(id signalv1.CheckID, policy signalv1.CheckParticipantPolicy) error { + roles := make(map[string]struct{}, len(policy.Roles)) + for _, role := range policy.Roles { + if role == "" { + return fmt.Errorf("check %s contains an empty participant role", id) + } + + if _, duplicate := roles[role]; duplicate { + return fmt.Errorf("check %s repeats participant role %q", id, role) + } + + roles[role] = struct{}{} + } + + return nil +} + +func validateParticipantConstraints( + profile *signalv1.CertificationProfile, + checks []signalv1.CertificationCheck, +) error { + roles := make(map[string]struct{}, len(profile.Spec.ParticipantRoles)) + + var assigned int32 + + for index := range profile.Spec.ParticipantRoles { + role := &profile.Spec.ParticipantRoles[index] + if role.Name == "" || role.Count < 1 { + return fmt.Errorf("participant role %d has invalid name or count", index) + } + + if _, duplicate := roles[role.Name]; duplicate { + return fmt.Errorf("duplicate participant role %q", role.Name) + } + + if _, err := metav1.LabelSelectorAsSelector(&role.Selector); err != nil { + return fmt.Errorf("participant role %s selector: %w", role.Name, err) + } + + roles[role.Name] = struct{}{} + assigned += role.Count + } + + if len(profile.Spec.ParticipantRoles) > 0 && assigned != profile.Spec.Participants.Count { + return fmt.Errorf( + "participant roles require exactly %d assignments, got %d", + profile.Spec.Participants.Count, + assigned, + ) + } + + for index := range profile.Spec.TopologyConstraints { + constraint := &profile.Spec.TopologyConstraints[index] + if _, exists := roles[constraint.Role]; !exists { + return fmt.Errorf("topology constraint references unknown role %q", constraint.Role) + } + + if constraint.LabelKey == "" || constraint.MinDistinctValues < 1 { + return fmt.Errorf("topology constraint %d has invalid label key or minimum", index) + } + } + + for index := range checks { + for _, role := range checks[index].ParticipantPolicy.Roles { + if _, exists := roles[role]; !exists { + return fmt.Errorf("check %s references unknown participant role %q", checks[index].ID, role) + } + } + } + + return nil +} + +func participantsForCheck( + participants []signalv1.Participant, + policy signalv1.CheckParticipantPolicy, +) ([]signalv1.Participant, []int32, error) { + included := make(map[string]struct{}, len(policy.Roles)) + for _, role := range policy.Roles { + included[role] = struct{}{} + } + + selected := make([]signalv1.Participant, 0, len(participants)) + + globalIndexes := make([]int32, 0, len(participants)) + for index := range participants { + participant := participants[index] + _, include := included[participant.Role] + include = len(included) == 0 || include + + if !include { + continue + } + + globalIndexes = append(globalIndexes, int32(index)) + participant.Index = int32(len(selected)) + selected = append(selected, participant) + } + + if len(selected) == 0 { + return nil, nil, errors.New("participant policy selected no participants") + } + + return selected, globalIndexes, nil +} + +func sanitizeTaskName(value string) string { + value = strings.ToLower(value) + value = strings.Map(func(character rune) rune { + if character >= 'a' && character <= 'z' || + character >= '0' && character <= '9' || + character == '-' { + return character + } + + return '-' + }, value) + + value = strings.Trim(value, "-") + if value == "" { + value = "check" + } + + if len(value) > 48 { + value = strings.TrimRight(value[:48], "-") + } + + return value +} + +func checkTaskName(check *signalv1.CertificationCheckSnapshot) string { + return fmt.Sprintf("check-%02d-%s", check.Order, sanitizeTaskName(string(check.ID))) +} + +func checkRunName(runName string, checkID signalv1.CheckID) string { + suffix := sanitizeTaskName(string(checkID)) + + candidate := runName + "-" + suffix + if len(candidate) <= 63 { + return candidate + } + + sum := sha256.Sum256([]byte(candidate)) + hash := hex.EncodeToString(sum[:4]) + prefixLength := 63 - len(hash) - 1 + + return strings.TrimRight(candidate[:prefixLength], "-") + "-" + hash +} + +func AddToScheme(scheme *runtime.Scheme) error { + return signalv1.AddToScheme(scheme) +} diff --git a/internal/signal/evidence.go b/internal/signal/evidence.go new file mode 100644 index 0000000000..b7a17ee56f --- /dev/null +++ b/internal/signal/evidence.go @@ -0,0 +1,303 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signal + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "maps" + "net/url" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type EvidenceStore interface { + Put(context.Context, string, []byte) (string, string, error) + Get(context.Context, string) ([]byte, string, string, bool, error) +} + +type KubernetesEvidenceStore struct { + kube client.Client + namespace string +} + +const maximumEvidenceChunkBytes = 700 * 1024 + +type evidenceChunkManifest struct { + Version string `json:"version"` + Generation string `json:"generation"` + Chunks int `json:"chunks"` + CompressedDigest string `json:"compressedDigest"` +} + +func NewKubernetesEvidenceStore(kube client.Client, namespace string) (*KubernetesEvidenceStore, error) { + if kube == nil { + return nil, fmt.Errorf("kubernetes client is required") + } + + if namespace == "" { + return nil, fmt.Errorf("evidence namespace is required") + } + + return &KubernetesEvidenceStore{kube: kube, namespace: namespace}, nil +} + +func (s *KubernetesEvidenceStore) Put(ctx context.Context, path string, data []byte) (string, string, error) { + name := evidenceName(path) + if name == "" { + return "", "", fmt.Errorf("evidence path is required") + } + + digest := digestBytes(data) + generation := strings.TrimPrefix(digest, "sha256:") + + compressed, err := compressEvidence(data) + if err != nil { + return "", "", err + } + + immutable := true + + evidence := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name + "-evidence", + Namespace: s.namespace, + Labels: map[string]string{ + SnapshotRunLabel: name, + }, + Annotations: map[string]string{ + "signal.unbounded-cloud.io/evidence-digest": digest, + }, + }, + Immutable: &immutable, + BinaryData: map[string][]byte{}, + } + uriKey := "evidence.json.gz" + + if len(compressed) <= maximumEvidenceChunkBytes { + evidence.BinaryData["evidence.json.gz"] = compressed + } else { + uriKey = "manifest.json" + + chunkCount := (len(compressed) + maximumEvidenceChunkBytes - 1) / maximumEvidenceChunkBytes + for index := range chunkCount { + start := index * maximumEvidenceChunkBytes + end := min(start+maximumEvidenceChunkBytes, len(compressed)) + + chunk := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-evidence-%s-%03d", name, generation, index), + Namespace: s.namespace, + Labels: map[string]string{ + SnapshotRunLabel: name, + }, + Annotations: map[string]string{ + "signal.unbounded-cloud.io/evidence-digest": digest, + "signal.unbounded-cloud.io/chunk-index": strconv.Itoa(index), + }, + }, + Immutable: &immutable, + BinaryData: map[string][]byte{ + "chunk": compressed[start:end], + }, + } + if err := s.createOrVerifyEvidenceConfigMap(ctx, chunk); err != nil { + return "", "", err + } + } + + manifest, err := json.Marshal(evidenceChunkManifest{ + Version: "v2", + Generation: generation, + Chunks: chunkCount, + CompressedDigest: digestBytes(compressed), + }) + if err != nil { + return "", "", fmt.Errorf("encode evidence chunk manifest: %w", err) + } + + evidence.BinaryData["manifest.json"] = manifest + } + + if err := s.createOrVerifyEvidenceConfigMap(ctx, evidence); err != nil { + return "", "", err + } + + return evidenceURI(s.namespace, evidence.Name, uriKey), digest, nil +} + +func (s *KubernetesEvidenceStore) Get( + ctx context.Context, + path string, +) ([]byte, string, string, bool, error) { + name := evidenceName(path) + if name == "" { + return nil, "", "", false, fmt.Errorf("evidence path is required") + } + + var evidence corev1.ConfigMap + if err := s.kube.Get( + ctx, + client.ObjectKey{Namespace: s.namespace, Name: name + "-evidence"}, + &evidence, + ); err != nil { + if apierrors.IsNotFound(err) { + return nil, "", "", false, nil + } + + return nil, "", "", false, fmt.Errorf("get evidence ConfigMap: %w", err) + } + + if evidence.Immutable == nil || !*evidence.Immutable { + return nil, "", "", false, fmt.Errorf("evidence ConfigMap %s is mutable", evidence.Name) + } + + compressed, exists := evidence.BinaryData["evidence.json.gz"] + + uriKey := "evidence.json.gz" + if !exists { + uriKey = "manifest.json" + + var manifest evidenceChunkManifest + if err := json.Unmarshal(evidence.BinaryData["manifest.json"], &manifest); err != nil { + return nil, "", "", false, fmt.Errorf("decode evidence ConfigMap %s chunk manifest: %w", evidence.Name, err) + } + + if manifest.Version != "v2" || manifest.Generation == "" || manifest.Chunks < 1 { + return nil, "", "", false, fmt.Errorf("evidence ConfigMap %s has an invalid chunk manifest", evidence.Name) + } + + if "sha256:"+manifest.Generation != evidence.Annotations["signal.unbounded-cloud.io/evidence-digest"] { + return nil, "", "", false, fmt.Errorf("evidence ConfigMap %s chunk generation does not match its digest", evidence.Name) + } + + compressed = make([]byte, 0, manifest.Chunks*maximumEvidenceChunkBytes) + for index := range manifest.Chunks { + var chunk corev1.ConfigMap + if err := s.kube.Get( + ctx, + client.ObjectKey{ + Namespace: s.namespace, + Name: fmt.Sprintf("%s-%s-%03d", evidence.Name, manifest.Generation, index), + }, + &chunk, + ); err != nil { + return nil, "", "", false, fmt.Errorf("get evidence chunk %d: %w", index, err) + } + + if chunk.Immutable == nil || !*chunk.Immutable { + return nil, "", "", false, fmt.Errorf("evidence chunk %s is mutable", chunk.Name) + } + + if chunk.Annotations["signal.unbounded-cloud.io/evidence-digest"] != + evidence.Annotations["signal.unbounded-cloud.io/evidence-digest"] || + chunk.Annotations["signal.unbounded-cloud.io/chunk-index"] != strconv.Itoa(index) { + return nil, "", "", false, fmt.Errorf("evidence chunk %s identity does not match its manifest", chunk.Name) + } + + compressed = append(compressed, chunk.BinaryData["chunk"]...) + } + + if digestBytes(compressed) != manifest.CompressedDigest { + return nil, "", "", false, fmt.Errorf("evidence ConfigMap %s chunks do not match their manifest", evidence.Name) + } + } + + data, err := decompressEvidence(compressed) + if err != nil { + return nil, "", "", false, fmt.Errorf("decompress evidence ConfigMap %s: %w", evidence.Name, err) + } + + digest := digestBytes(data) + if evidence.Annotations["signal.unbounded-cloud.io/evidence-digest"] != digest { + return nil, "", "", false, fmt.Errorf("evidence ConfigMap %s digest does not match its content", evidence.Name) + } + + uri := evidenceURI(s.namespace, evidence.Name, uriKey) + + return data, uri, digest, true, nil +} + +func evidenceURI(namespace, name, key string) string { + return fmt.Sprintf("kubernetes://%s/configmaps/%s/%s", namespace, name, key) +} + +func (s *KubernetesEvidenceStore) createOrVerifyEvidenceConfigMap( + ctx context.Context, + evidence *corev1.ConfigMap, +) error { + if err := s.kube.Create(ctx, evidence); err != nil { + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("create evidence ConfigMap %s: %w", evidence.Name, err) + } + + var existing corev1.ConfigMap + if getErr := s.kube.Get(ctx, client.ObjectKeyFromObject(evidence), &existing); getErr != nil { + return fmt.Errorf("get existing evidence ConfigMap %s: %w", evidence.Name, getErr) + } + + if existing.Immutable == nil || !*existing.Immutable { + return fmt.Errorf("evidence ConfigMap %s already exists but is mutable", evidence.Name) + } + + if !maps.Equal(existing.Labels, evidence.Labels) || + !maps.Equal(existing.Annotations, evidence.Annotations) || + !maps.EqualFunc(existing.BinaryData, evidence.BinaryData, bytes.Equal) { + return fmt.Errorf("evidence ConfigMap %s already exists with different content", evidence.Name) + } + } + + return nil +} + +func evidenceName(path string) string { + return strings.TrimSuffix( + url.PathEscape(strings.TrimSuffix(path[strings.LastIndex(path, "/")+1:], ".json")), + ".json", + ) +} + +func compressEvidence(data []byte) ([]byte, error) { + var compressed bytes.Buffer + + writer := gzip.NewWriter(&compressed) + if _, err := writer.Write(data); err != nil { + return nil, fmt.Errorf("compress evidence: %w", err) + } + + if err := writer.Close(); err != nil { + return nil, fmt.Errorf("finish evidence compression: %w", err) + } + + return compressed.Bytes(), nil +} + +func decompressEvidence(compressed []byte) ([]byte, error) { + reader, err := gzip.NewReader(bytes.NewReader(compressed)) + if err != nil { + return nil, err + } + + data, readErr := io.ReadAll(reader) + closeErr := reader.Close() + + if readErr != nil { + return nil, readErr + } + + if closeErr != nil { + return nil, closeErr + } + + return data, nil +} diff --git a/internal/signal/projection.go b/internal/signal/projection.go new file mode 100644 index 0000000000..022a7a9e1b --- /dev/null +++ b/internal/signal/projection.go @@ -0,0 +1,220 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signal + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "strconv" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" +) + +const ( + StateLabel = "signal.unbounded-cloud.io/state" + EligibilityLabel = "signal.unbounded-cloud.io/eligibility" + ProfileLabel = "signal.unbounded-cloud.io/profile" + + ProfileNameAnnotation = "signal.unbounded-cloud.io/profile-name" + ProfileGenerationAnnotation = "signal.unbounded-cloud.io/profile-generation" + LastRunAnnotation = "signal.unbounded-cloud.io/last-run" + LastRunAtAnnotation = "signal.unbounded-cloud.io/last-run-at" + LastSuccessfulRunAnnotation = "signal.unbounded-cloud.io/last-successful-run" + LastSuccessfulAtAnnotation = "signal.unbounded-cloud.io/last-successful-at" + EvidenceDigestAnnotation = "signal.unbounded-cloud.io/evidence-digest" + ScriptDigestAnnotation = "signal.unbounded-cloud.io/script-digest" + + maxProjectionAnnotationValue = 1024 +) + +func projectPending( + ctx context.Context, + kube client.Client, + run *signalv1.CertificationRun, +) error { + return projectParticipants(ctx, kube, run.Spec.Participants, true, func(node *corev1.Node) error { + if node.Annotations[ReservationLabel] != run.Name { + return fmt.Errorf("reservation is not owned by Run %s", run.Name) + } + + setProfileProjection(node, run.Spec.Profile) + node.Labels[StateLabel] = "pending" + node.Labels[EligibilityLabel] = "blocked" + + return nil + }) +} + +func projectTesting(ctx context.Context, kube client.Client, run *signalv1.CertificationRun) error { + return projectParticipants(ctx, kube, run.Spec.Participants, false, func(node *corev1.Node) error { + if node.Annotations[ReservationLabel] != run.Name { + return nil + } + + setProfileProjection(node, run.Spec.Profile) + node.Labels[StateLabel] = "testing" + node.Labels[EligibilityLabel] = "blocked" + + return nil + }) +} + +func projectTerminal(ctx context.Context, kube client.Client, run *signalv1.CertificationRun) error { + if run.Status.FinishedAt == nil { + return nil + } + + finishedAt := run.Status.FinishedAt.UTC().Format(time.RFC3339) + + return projectParticipants(ctx, kube, run.Spec.Participants, false, func(node *corev1.Node) error { + if node.Annotations[ReservationLabel] != run.Name { + return nil + } + + if projectionTimeAfter(node.Annotations[LastRunAtAnnotation], run.Status.FinishedAt.Time) { + return nil + } + + setProfileProjection(node, run.Spec.Profile) + node.Annotations[LastRunAnnotation] = boundedAnnotationValue(run.Name) + node.Annotations[LastRunAtAnnotation] = finishedAt + node.Annotations[EvidenceDigestAnnotation] = boundedAnnotationValue(run.Status.Evidence.Digest) + node.Annotations[ScriptDigestAnnotation] = boundedAnnotationValue(runScriptDigest(run)) + + switch run.Status.Phase { + case signalv1.RunPhaseSucceeded: + node.Labels[StateLabel] = "passed" + node.Labels[EligibilityLabel] = "production" + + if !projectionTimeAfter(node.Annotations[LastSuccessfulAtAnnotation], run.Status.FinishedAt.Time) { + node.Annotations[LastSuccessfulRunAnnotation] = boundedAnnotationValue(run.Name) + node.Annotations[LastSuccessfulAtAnnotation] = finishedAt + } + case signalv1.RunPhaseFailed: + node.Labels[StateLabel] = "failed" + node.Labels[EligibilityLabel] = "blocked" + default: + node.Labels[StateLabel] = "inconclusive" + node.Labels[EligibilityLabel] = "blocked" + } + + return nil + }) +} + +func runScriptDigest(run *signalv1.CertificationRun) string { + hash := sha256.New() + for index := range run.Spec.Checks { + _, _ = hash.Write([]byte(run.Spec.Checks[index].ID)) + _, _ = hash.Write([]byte{0}) + _, _ = hash.Write([]byte(run.Spec.Checks[index].Script.Digest)) + _, _ = hash.Write([]byte{0}) + } + + return "sha256:" + hex.EncodeToString(hash.Sum(nil)) +} + +func projectParticipants( + ctx context.Context, + kube client.Client, + participants []signalv1.Participant, + strict bool, + mutate func(*corev1.Node) error, +) error { + for _, participant := range participants { + if participant.NodeName == "" { + continue + } + + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + var node corev1.Node + if err := kube.Get(ctx, client.ObjectKey{Name: participant.NodeName}, &node); err != nil { + if !strict { + return client.IgnoreNotFound(err) + } + + return err + } + + if participant.NodeUID != "" && node.UID != participant.NodeUID { + if strict { + return fmt.Errorf( + "node UID changed from %s to %s", + participant.NodeUID, + node.UID, + ) + } + + return nil + } + + before := node.DeepCopy() + ensureProjectionMaps(&node) + + if err := mutate(&node); err != nil { + return err + } + + return kube.Patch(ctx, &node, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})) + }); err != nil { + return fmt.Errorf("project Node %s status: %w", participant.NodeName, err) + } + } + + return nil +} + +func setProfileProjection(node *corev1.Node, profile signalv1.PinnedProfile) { + ensureProjectionMaps(node) + node.Labels[ProfileLabel] = projectionLabelValue(profile.Name) + node.Annotations[ProfileNameAnnotation] = boundedAnnotationValue(profile.Name) + node.Annotations[ProfileGenerationAnnotation] = strconv.FormatInt(profile.Generation, 10) +} + +func ensureProjectionMaps(node *corev1.Node) { + if node.Labels == nil { + node.Labels = map[string]string{} + } + + if node.Annotations == nil { + node.Annotations = map[string]string{} + } +} + +func projectionLabelValue(value string) string { + if len(validation.IsValidLabelValue(value)) == 0 { + return value + } + + sum := sha256.Sum256([]byte(value)) + + return "sha256-" + hex.EncodeToString(sum[:8]) +} + +func boundedAnnotationValue(value string) string { + if len(value) <= maxProjectionAnnotationValue { + return value + } + + sum := sha256.Sum256([]byte(value)) + + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func projectionTimeAfter(value string, candidate time.Time) bool { + existing, err := time.Parse(time.RFC3339, value) + if err != nil { + return false + } + + return existing.After(candidate) +} diff --git a/internal/signaladapter/adapter.go b/internal/signaladapter/adapter.go new file mode 100644 index 0000000000..715051a1d3 --- /dev/null +++ b/internal/signaladapter/adapter.go @@ -0,0 +1,1351 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signaladapter + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "regexp" + "slices" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" + + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" + "github.com/Azure/unbounded/internal/signalworkload" +) + +const ( + MaxParticipants = 64 + MaxCheckParticipants = 8 + MaxReasonBytes = 512 + MaxMeasurementsBytes = 16 * 1024 + MaxBackendOutputBytes = 8 * 1024 + MaxBackendEvidenceBytes = 24 * 1024 + maxMPIWorkerReadyWait = int64(30) + maxMPIWorkerSlots = int64(1<<31 - 1) + mpiBackoffLimit = int64(0) + mpiHostfilePath = "/tmp/project-signal-hostfile" + mpiKnownHostsPath = "/tmp/project-signal-known-hosts" + resultPrefix = "PROJECT_SIGNAL_RESULT_V2 " +) + +var runnerImagePattern = regexp.MustCompile(`^[^@\s]+@sha256:[0-9a-f]{64}$`) + +var reservedEnvironmentNames = map[string]struct{}{ + "BASH_ENV": {}, + "BASHOPTS": {}, + "CDPATH": {}, + "ENV": {}, + "GLOBIGNORE": {}, + "HOST_PROC_ROOT": {}, + "HOST_SYSFS_ROOT": {}, + "IFS": {}, + "LD_LIBRARY_PATH": {}, + "LD_PRELOAD": {}, + "PATH": {}, + "PYTHONHOME": {}, + "PYTHONPATH": {}, + "SHELLOPTS": {}, +} + +type Config struct { + Namespace string + WorkflowName string + RunName string + RunUID types.UID + ParticipantsJSON string + ParticipantCount int + ScriptConfigMap string + ScriptKey string + ScriptDigest string + RunnerImage string + BackendKind string + BackendDeadlineSeconds int64 + ScriptEnvironmentJSON string + WorkloadPolicyJSON string + GPUResourceCount string +} + +type Result struct { + ParticipantIndex int `json:"participantIndex"` + ExecutionState string `json:"executionState"` + Verdict string `json:"verdict"` + Reason string `json:"reason"` + Measurements string `json:"measurements"` + ResultLog *ResultLogMetadata `json:"resultLog,omitempty"` + ExitCode int `json:"-"` +} + +type ResultLogSourceMetadata struct { + Status string `json:"status"` + Bytes int `json:"bytes,omitempty"` +} + +type ResultLogMetadata struct { + Current ResultLogSourceMetadata `json:"current"` + Previous ResultLogSourceMetadata `json:"previous"` + SelectedSource string `json:"selectedSource,omitempty"` +} + +type mpiWorkerHost struct { + Name string + Slots int64 +} + +type AggregateOutput struct { + ExecutionState string `json:"executionState"` + Verdict string `json:"verdict"` + Reason string `json:"reason"` + Measurements string `json:"measurements"` + BackendResources []signalv1.BackendResourceReference `json:"-"` + BackendJSON string `json:"backend-resources"` + BackendEvidence string `json:"backend-evidence"` +} + +func ParseParticipants( + participantsJSON string, + participantCount int, +) ([]signalv1.Participant, error) { + if participantCount < 1 || participantCount > MaxCheckParticipants { + return nil, fmt.Errorf("participant-count must be between 1 and %d", MaxCheckParticipants) + } + + if strings.TrimSpace(participantsJSON) == "" { + return nil, errors.New("participants-json is required") + } + + var participants []signalv1.Participant + + decoder := json.NewDecoder(strings.NewReader(participantsJSON)) + decoder.DisallowUnknownFields() + + if err := decoder.Decode(&participants); err != nil { + return nil, fmt.Errorf("decode participants-json: %w", err) + } + + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + if err == nil { + return nil, errors.New("participants-json contains trailing values") + } + + return nil, fmt.Errorf("decode trailing participants-json value: %w", err) + } + + if len(participants) != participantCount { + return nil, fmt.Errorf( + "participant-count %d does not match participants-json length %d", + participantCount, + len(participants), + ) + } + + names := make(map[string]struct{}, len(participants)) + uids := make(map[types.UID]struct{}, len(participants)) + + for index := range participants { + participant := &participants[index] + participant.Index = int32(index) + + if errors := validation.IsDNS1123Subdomain(participant.NodeName); len(errors) > 0 { + return nil, fmt.Errorf("participants-json[%d].nodeName is invalid: %s", index, strings.Join(errors, ", ")) + } + + if _, duplicate := names[participant.NodeName]; duplicate { + return nil, fmt.Errorf("participants-json[%d].nodeName duplicates %q", index, participant.NodeName) + } + + names[participant.NodeName] = struct{}{} + + if strings.TrimSpace(participantsJSON) == "" { + continue + } + + if participant.NodeUID == "" { + return nil, fmt.Errorf("participants-json[%d].nodeUID is required", index) + } + + if _, duplicate := uids[participant.NodeUID]; duplicate { + return nil, fmt.Errorf("participants-json[%d].nodeUID duplicates %q", index, participant.NodeUID) + } + + uids[participant.NodeUID] = struct{}{} + } + + return participants, nil +} + +func ValidateConfig(config Config) ([]signalv1.Participant, []corev1.Toleration, []corev1.EnvVar, error) { + for name, value := range map[string]string{ + "namespace": config.Namespace, + "workflow-name": config.WorkflowName, + "run-name": config.RunName, + "run-uid": string(config.RunUID), + "script-configmap": config.ScriptConfigMap, + "script-key": config.ScriptKey, + "script-digest": config.ScriptDigest, + "runner-image": config.RunnerImage, + "backend-kind": config.BackendKind, + } { + if value == "" { + return nil, nil, nil, fmt.Errorf("%s is required", name) + } + } + + for name, value := range map[string]string{ + "namespace": config.Namespace, + "workflow-name": config.WorkflowName, + "run-name": config.RunName, + "script-configmap": config.ScriptConfigMap, + } { + if errors := validation.IsDNS1123Subdomain(value); len(errors) > 0 { + return nil, nil, nil, fmt.Errorf("%s is invalid: %s", name, strings.Join(errors, ", ")) + } + } + + if errors := validation.IsConfigMapKey(config.ScriptKey); len(errors) > 0 { + return nil, nil, nil, fmt.Errorf("script-key is invalid: %s", strings.Join(errors, ", ")) + } + + if !runnerImagePattern.MatchString(config.RunnerImage) { + return nil, nil, nil, errors.New("runner-image must be an immutable sha256 digest reference") + } + + if !signalv1.IsSHA256Digest(config.ScriptDigest) { + return nil, nil, nil, errors.New( + "script-digest must be an algorithm-qualified SHA-256 digest (sha256:<64 lowercase hexadecimal characters>)", + ) + } + + if config.BackendDeadlineSeconds <= 0 { + return nil, nil, nil, errors.New("backend-active-deadline-seconds must be positive") + } + + switch signalv1.ExecutionBackend(config.BackendKind) { + case signalv1.ExecutionBackendPod: + case signalv1.ExecutionBackendMPIJob: + if config.BackendDeadlineSeconds < 2 { + return nil, nil, nil, errors.New( + "backend-active-deadline-seconds must be at least 2 for MPIJob worker readiness", + ) + } + default: + return nil, nil, nil, fmt.Errorf("unsupported backend %q", config.BackendKind) + } + + if _, err := signalworkload.Decode(config.WorkloadPolicyJSON); err != nil { + return nil, nil, nil, err + } + + participants, err := ParseParticipants( + config.ParticipantsJSON, + config.ParticipantCount, + ) + if err != nil { + return nil, nil, nil, err + } + + minimumEvidenceBytes := MinimumBackendEvidenceBytes(len(participants)) + if minimumEvidenceBytes > MaxBackendEvidenceBytes { + return nil, nil, nil, fmt.Errorf( + "fixed backend evidence limit %d cannot represent %d participants; requires at least %d bytes", + MaxBackendEvidenceBytes, + len(participants), + minimumEvidenceBytes, + ) + } + + tolerations := []corev1.Toleration{{ + Key: "nvidia.com/gpu", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, { + Key: corev1.TaintNodeUnschedulable, + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }} + + var environment map[string]string + if err := decodeJSONObject(config.ScriptEnvironmentJSON, &environment); err != nil { + return nil, nil, nil, fmt.Errorf("decode script-environment: %w", err) + } + + if len(environment) > 32 { + return nil, nil, nil, errors.New("script-environment must not contain more than 32 entries") + } + + environmentNames := make([]string, 0, len(environment)) + for name := range environment { + environmentNames = append(environmentNames, name) + } + + slices.Sort(environmentNames) + + env := make([]corev1.EnvVar, 0, len(environmentNames)+4) + for _, name := range environmentNames { + if len(name) > 253 { + return nil, nil, nil, fmt.Errorf("script-environment name %q exceeds 253 bytes", name) + } + + if errors := validation.IsEnvVarName(name); len(errors) > 0 { + return nil, nil, nil, fmt.Errorf("script-environment contains invalid name %q", name) + } + + if strings.HasPrefix(name, "PROJECT_SIGNAL_") { + return nil, nil, nil, fmt.Errorf("script-environment contains reserved name %q", name) + } + + if _, reserved := reservedEnvironmentNames[name]; reserved { + return nil, nil, nil, fmt.Errorf("script-environment contains reserved name %q", name) + } + + if len(environment[name]) > 4096 { + return nil, nil, nil, fmt.Errorf("script-environment value for %q exceeds 4096 bytes", name) + } + + env = append(env, corev1.EnvVar{Name: name, Value: environment[name]}) + } + + return participants, tolerations, env, nil +} + +func RenderResources( + config Config, + participants []signalv1.Participant, + tolerations []corev1.Toleration, + environment []corev1.EnvVar, + workflow metav1.OwnerReference, +) ([]*unstructured.Unstructured, error) { + resources := make([]*unstructured.Unstructured, 0, len(participants)) + + gpuCount, err := strconv.ParseInt(config.GPUResourceCount, 10, 64) + if err != nil || gpuCount < 0 { + return nil, fmt.Errorf("gpu-resource-count must be a nonnegative integer") + } + + if config.BackendKind == "MPIJob" && gpuCount < 1 { + return nil, errors.New("gpu-resource-count must be positive for MPIJob execution") + } + + policy, err := signalworkload.Decode(config.WorkloadPolicyJSON) + if err != nil { + return nil, err + } + + gpuResource := corev1.ResourceName("nvidia.com/gpu") + if _, exists := policy.Resources.Requests[gpuResource]; exists { + return nil, errors.New("workload policy must not set nvidia.com/gpu requests; use gpu-resource-count") + } + + if _, exists := policy.Resources.Limits[gpuResource]; exists { + return nil, errors.New("workload policy must not set nvidia.com/gpu limits; use gpu-resource-count") + } + + for _, participant := range participants { + var object any + + switch config.BackendKind { + case "Pod": + object = ordinaryPod( + config, + participant, + tolerations, + environment, + policy, + workflow, + config.BackendDeadlineSeconds, + gpuCount, + ) + case "MPIJob": + object, err = mpiJob( + config, + participant, + tolerations, + environment, + policy, + workflow, + config.BackendDeadlineSeconds, + gpuCount, + ) + if err != nil { + return nil, fmt.Errorf("render MPIJob for participant %d: %w", participant.Index, err) + } + default: + return nil, fmt.Errorf("unsupported backend kind %q", config.BackendKind) + } + + content, err := runtime.DefaultUnstructuredConverter.ToUnstructured(object) + if err != nil { + return nil, fmt.Errorf("convert %s for participant %d: %w", config.BackendKind, participant.Index, err) + } + + resources = append(resources, &unstructured.Unstructured{Object: content}) + } + + return resources, nil +} + +func BackendResource(resource *unstructured.Unstructured, participantIndex int32) (signalv1.BackendResourceReference, error) { + label := resource.GetLabels()["signal.unbounded-cloud.io/participant-index"] + if label != strconv.FormatInt(int64(participantIndex), 10) { + return signalv1.BackendResourceReference{}, fmt.Errorf( + "backend resource participant label %q does not match index %d", + label, + participantIndex, + ) + } + + return backendResourceIdentity(resource, participantIndex) +} + +func backendResourceIdentity( + resource *unstructured.Unstructured, + participantIndex int32, +) (signalv1.BackendResourceReference, error) { + if resource.GetAPIVersion() == "" || + resource.GetKind() == "" || + resource.GetName() == "" || + resource.GetUID() == "" { + return signalv1.BackendResourceReference{}, errors.New("backend resource has an incomplete identity") + } + + return signalv1.BackendResourceReference{ + Name: resource.GetName(), + UID: resource.GetUID(), + ParticipantIndexes: []int32{participantIndex}, + }, nil +} + +func MinimumPartialReasonBytes() int { + return len("BackendSetupFailed") +} + +func MinimumPartialMeasurementsBytes(participantCount int) int { + results := make([]Result, participantCount) + for index := range results { + results[index] = Result{ + ParticipantIndex: index, + ExecutionState: "Error", + Verdict: "Inconclusive", + Reason: "BackendSetupFailed", + } + } + + content, err := json.Marshal(map[string]any{ + "participantCount": participantCount, + "createdBackends": participantCount, + "error": strings.Repeat("\x00", maxEvidenceTextBytes), + "results": results, + }) + if err != nil { + return MaxMeasurementsBytes + 1 + } + + return len(content) + 64 +} + +func MinimumPartialBackendResourcesBytes(participantCount int) int { + references := make([]signalv1.BackendResourceReference, participantCount) + for index := range references { + references[index] = signalv1.BackendResourceReference{ + Name: strings.Repeat("a", 63), + UID: types.UID(strings.Repeat("a", 128)), + ParticipantIndexes: []int32{int32(index)}, + } + } + + content, err := json.Marshal(references) + if err != nil { + return MaxBackendOutputBytes + 1 + } + + return len(content) + 64 +} + +func Aggregate( + participantCount int, + backendResources []signalv1.BackendResourceReference, + results []Result, +) (AggregateOutput, error) { + if participantCount < 1 || participantCount > MaxParticipants { + return AggregateOutput{}, fmt.Errorf("participant-count must be between 1 and %d", MaxParticipants) + } + + if err := validateBackendCoverage(participantCount, backendResources); err != nil { + return AggregateOutput{}, err + } + + if len(results) != participantCount { + return AggregateOutput{}, fmt.Errorf("results must contain exactly %d entries", participantCount) + } + + ordered := make([]Result, participantCount) + seen := make([]bool, participantCount) + executionFailed := false + executionFailureReason := "" + diagnosticFailed := false + allPassed := true + + for index, result := range results { + if result.ParticipantIndex < 0 || result.ParticipantIndex >= participantCount { + return AggregateOutput{}, fmt.Errorf("results[%d].participantIndex is out of range", index) + } + + if seen[result.ParticipantIndex] { + return AggregateOutput{}, fmt.Errorf("results repeats participant index %d", result.ParticipantIndex) + } + + seen[result.ParticipantIndex] = true + + if result.ExecutionState == "" || result.Reason == "" { + return AggregateOutput{}, fmt.Errorf("results[%d] requires executionState and reason", index) + } + + if len(result.Reason) > MaxReasonBytes { + return AggregateOutput{}, fmt.Errorf("results[%d].reason exceeds %d bytes", index, MaxReasonBytes) + } + + switch result.Verdict { + case "Pass": + case "Fail": + diagnosticFailed = true + allPassed = false + case "Inconclusive": + allPassed = false + default: + return AggregateOutput{}, fmt.Errorf("results[%d].verdict is invalid", index) + } + + if result.ExecutionState != "Succeeded" { + executionFailed = true + + if executionFailureReason == "" { + executionFailureReason = result.Reason + } + } + + if len(result.Measurements) > MaxMeasurementsBytes { + return AggregateOutput{}, fmt.Errorf("results[%d].measurements exceeds %d bytes", index, MaxMeasurementsBytes) + } + + ordered[result.ParticipantIndex] = result + } + + for index, present := range seen { + if !present { + return AggregateOutput{}, fmt.Errorf("results does not cover participant index %d", index) + } + } + + backendJSON, err := json.Marshal(backendResources) + if err != nil { + return AggregateOutput{}, fmt.Errorf("encode backend-resources: %w", err) + } + + if len(backendJSON) > MaxBackendOutputBytes { + return AggregateOutput{}, fmt.Errorf("backend-resources exceeds %d bytes", MaxBackendOutputBytes) + } + + measurementsJSON, err := json.Marshal(map[string]any{ + "participantCount": participantCount, + "results": ordered, + }) + if err != nil { + return AggregateOutput{}, fmt.Errorf("encode measurements: %w", err) + } + + if len(measurementsJSON) > MaxMeasurementsBytes { + return AggregateOutput{}, fmt.Errorf("aggregate measurements exceeds %d bytes", MaxMeasurementsBytes) + } + + output := AggregateOutput{ + ExecutionState: "Succeeded", + Verdict: "Inconclusive", + Reason: "ParticipantCertificationInconclusive", + Measurements: string(measurementsJSON), + BackendResources: backendResources, + BackendJSON: string(backendJSON), + } + + switch { + case executionFailed: + output.ExecutionState = "Error" + output.Reason = executionFailureReason + case diagnosticFailed: + output.Verdict = "Fail" + output.Reason = "ParticipantCertificationFailed" + case allPassed: + output.Verdict = "Pass" + output.Reason = "AllParticipantsPassed" + } + + return output, nil +} + +func ParseResultLog(logContent string, participantIndex int) (Result, error) { + lines := strings.Split(logContent, "\n") + for index := len(lines) - 1; index >= 0; index-- { + line := lines[index] + if !strings.HasPrefix(line, resultPrefix) { + continue + } + + fields := strings.Fields(strings.TrimPrefix(line, resultPrefix)) + if len(fields) != 4 { + return Result{}, errors.New("result record must contain verdict, reason, measurements, and exit code") + } + + values := make([]string, 3) + + for fieldIndex, field := range fields[:3] { + if field == "-" { + continue + } + + decoded, err := base64.StdEncoding.DecodeString(field) + if err != nil { + return Result{}, fmt.Errorf("decode result field %d: %w", fieldIndex, err) + } + + values[fieldIndex] = string(decoded) + } + + exitCode, err := strconv.Atoi(fields[3]) + if err != nil || exitCode < 0 || exitCode > 255 { + return Result{}, errors.New("result exit code must be an integer between 0 and 255") + } + + result := Result{ + ParticipantIndex: participantIndex, + ExecutionState: "Succeeded", + Verdict: values[0], + Reason: values[1], + Measurements: values[2], + ExitCode: exitCode, + } + if err := validateResult(result); err != nil { + return Result{}, err + } + + return result, nil + } + + return Result{}, errors.New("workload log does not contain a Project Signal result record") +} + +func validateResult(result Result) error { + if result.Reason == "" { + return errors.New("result reason must not be empty") + } + + if len(result.Reason) > MaxReasonBytes { + return fmt.Errorf("result reason exceeds %d bytes", MaxReasonBytes) + } + + if len(result.Measurements) > MaxMeasurementsBytes { + return fmt.Errorf("result measurements exceeds %d bytes", MaxMeasurementsBytes) + } + + var expectedExitCode int + + switch result.Verdict { + case "Pass": + expectedExitCode = 0 + case "Fail": + expectedExitCode = 1 + case "Inconclusive": + expectedExitCode = 2 + default: + return fmt.Errorf("result verdict %q is invalid", result.Verdict) + } + + if result.ExitCode != expectedExitCode { + return fmt.Errorf( + "result verdict %s requires exit code %d, got %d", + result.Verdict, + expectedExitCode, + result.ExitCode, + ) + } + + return nil +} + +func resultCommand(arguments ...string) []string { + return resultCommandWithPrelude("", arguments...) +} + +func resultCommandWithPrelude(prelude string, arguments ...string) []string { + if strings.TrimSpace(prelude) == "" { + prelude = ":" + } + + quotedArguments := make([]string, len(arguments)) + for index, argument := range arguments { + quotedArguments[index] = shellQuote(argument) + } + + return []string{ + "/bin/bash", + "-c", + fmt.Sprintf(` +set -u +mkdir -p /results +chmod 0755 /results +rm -f /results/verdict /results/reason /results/measurements.txt +printf 'PROJECT_SIGNAL_RESULTS_V1 prepared\n' +set +e +script_path="/scripts/${PROJECT_SIGNAL_SCRIPT_KEY}" +expected_digest="${PROJECT_SIGNAL_SCRIPT_DIGEST#sha256:}" +actual_digest="" +verification_reason="" +if command -v sha256sum >/dev/null 2>&1; then + actual_digest="$(sha256sum "${script_path}" | cut -d ' ' -f 1)" +elif command -v shasum >/dev/null 2>&1; then + actual_digest="$(shasum -a 256 "${script_path}" | cut -d ' ' -f 1)" +else + verification_reason="ScriptDigestVerificationUnavailable" +fi +if [[ -z "${verification_reason}" && "${actual_digest}" != "${expected_digest}" ]]; then + verification_reason="ScriptDigestMismatch" +fi +if [[ -n "${verification_reason}" ]]; then + printf 'Inconclusive' > /results/verdict + printf '%%s' "${verification_reason}" > /results/reason + status=125 +else + %s + prelude_status=$? + if [[ "${prelude_status}" -eq 0 ]]; then + /bin/bash "${script_path}" %s + status=$? + else + status="${prelude_status}" + fi +fi +set -e +canonicalize_result_file() { + if [[ -f "$1" ]]; then + local value + value="$(<"$1")" + printf '%%s' "${value}" > "$1" + fi +} +canonicalize_result_file /results/verdict +canonicalize_result_file /results/reason +canonicalize_result_file /results/measurements.txt +encode_result_file() { + if [[ -f "$1" ]]; then + base64 < "$1" | tr -d '\n' + else + printf '%%s' - + fi +} +verdict="$(encode_result_file /results/verdict)" +reason="$(encode_result_file /results/reason)" +measurements="$(encode_result_file /results/measurements.txt)" +[[ -n "${verdict}" ]] || verdict=- +[[ -n "${reason}" ]] || reason=- +[[ -n "${measurements}" ]] || measurements=- +printf 'PROJECT_SIGNAL_RESULTS_V1 verdict=%%s reason=%%s measurements=%%s\n' \ + "$([[ -f /results/verdict ]] && printf present || printf missing)" \ + "$([[ -f /results/reason ]] && printf present || printf missing)" \ + "$([[ -f /results/measurements.txt ]] && printf present || printf missing)" +printf '%s%%s %%s %%s %%d\n' "${verdict}" "${reason}" "${measurements}" "$status" +exit "$status" +`, prelude, strings.Join(quotedArguments, " "), resultPrefix), + } +} + +func mpiWorkerReadinessTimeout(backendDeadlineSeconds int64) int64 { + timeout := backendDeadlineSeconds - 1 + if timeout > maxMPIWorkerReadyWait { + timeout = maxMPIWorkerReadyWait + } + + return timeout +} + +func mpiWorkerHosts(namespace, jobName string, workerCount, slots int64) ([]mpiWorkerHost, error) { + if workerCount < 1 || workerCount > MaxParticipants { + return nil, fmt.Errorf("MPI worker count must be between 1 and %d", MaxParticipants) + } + + if slots < 1 || slots > maxMPIWorkerSlots { + return nil, fmt.Errorf("MPI worker slots must be between 1 and %d", maxMPIWorkerSlots) + } + + for field, value := range map[string]string{ + "namespace": namespace, + "job name": jobName, + } { + if errors := validation.IsDNS1123Label(value); len(errors) > 0 { + return nil, fmt.Errorf("MPI %s %q is invalid: %s", field, value, strings.Join(errors, ", ")) + } + } + + hosts := make([]mpiWorkerHost, workerCount) + for index := range workerCount { + workerName := fmt.Sprintf("%s-worker-%d", jobName, index) + if errors := validation.IsDNS1123Label(workerName); len(errors) > 0 { + return nil, fmt.Errorf( + "generated MPI worker name %q is invalid: %s", + workerName, + strings.Join(errors, ", "), + ) + } + + name := fmt.Sprintf( + "%s.%s.%s.svc", + workerName, + jobName, + namespace, + ) + if errors := validation.IsDNS1123Subdomain(name); len(errors) > 0 { + return nil, fmt.Errorf("generated MPI worker host %q is invalid: %s", name, strings.Join(errors, ", ")) + } + + hosts[index] = mpiWorkerHost{Name: name, Slots: slots} + } + + return hosts, nil +} + +func mpiLauncherPrelude(hostfile string, hosts []mpiWorkerHost, timeoutSeconds int64) string { + hostfileLines := make([]string, len(hosts)) + + hostArguments := make([]string, len(hosts)) + for index, host := range hosts { + hostfileLines[index] = fmt.Sprintf("%s slots=%d", host.Name, host.Slots) + hostArguments[index] = shellQuote(host.Name) + } + + return fmt.Sprintf( + ` +umask 077 +printf '%%s\n' %s > %s +export OMPI_MCA_orte_default_hostfile=%s +wait_for_mpi_workers() { + local hostfile=%s + local timeout_seconds=%d + local deadline=$((SECONDS + timeout_seconds)) + local host all_ready last_unready remaining + local -a hosts=(%s) + while (( SECONDS < deadline )); do + all_ready=1 + for host in "${hosts[@]}"; do + remaining=$((deadline - SECONDS)) + if (( remaining < 1 )) || ! timeout "${remaining}s" ssh \ + -o BatchMode=yes \ + -o ConnectTimeout=2 \ + -o ConnectionAttempts=1 \ + -o StrictHostKeyChecking=accept-new \ + -o UserKnownHostsFile=%s \ + -- "${host}" /bin/true /dev/null; then + all_ready=0 + last_unready="${host}" + break + fi + done + if (( all_ready == 1 )); then + return 0 + fi + sleep 1 + done + printf 'Project Signal MPI readiness timed out after %%ds waiting for every worker in %%s to resolve and accept SSH/22; last unready host=%%s\n' \ + "${timeout_seconds}" "${hostfile}" "${last_unready:-unknown}" >&2 + return 124 +} +wait_for_mpi_workers +`, + strings.Join(quotedShellArguments(hostfileLines), " "), + shellQuote(hostfile), + shellQuote(hostfile), + shellQuote(hostfile), + timeoutSeconds, + strings.Join(hostArguments, " "), + shellQuote(mpiKnownHostsPath), + ) +} + +func ordinaryPod( + config Config, + participant signalv1.Participant, + tolerations []corev1.Toleration, + environment []corev1.EnvVar, + policy signalv1.WorkloadPolicy, + workflow metav1.OwnerReference, + backendDeadlineSeconds int64, + gpuCount int64, +) *corev1.Pod { + labels := resourceLabels(config, participant) + + env := workloadEnvironment(config, participant, environment) + container := workloadContainer( + config, + policy, + corev1.Container{ + Name: "main", + Image: config.RunnerImage, + Command: resultCommand(), + Env: env, + }, + gpuCount, + ) + + pod := &corev1.Pod{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Pod"}, + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName(config.RunName, participant.Index), + Namespace: config.Namespace, + Labels: labels, + OwnerReferences: []metav1.OwnerReference{workflow}, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + Affinity: participantNodeAffinity(participant.NodeName, participant.NodeUID), + Tolerations: tolerations, + ActiveDeadlineSeconds: &backendDeadlineSeconds, + AutomountServiceAccountToken: ptr(false), + Containers: []corev1.Container{container}, + Volumes: workloadVolumes(config.ScriptConfigMap, policy), + }, + } + applyPodPolicy(&pod.Spec, policy) + + return pod +} + +func mpiJob( + config Config, + participant signalv1.Participant, + tolerations []corev1.Toleration, + environment []corev1.EnvVar, + policy signalv1.WorkloadPolicy, + workflow metav1.OwnerReference, + backendDeadlineSeconds int64, + gpuCount int64, +) (*unstructured.Unstructured, error) { + labels := resourceLabels(config, participant) + + jobName := resourceNameWithLimit(config.RunName, participant.Index, 63-len("-worker-0")) + launcherEnv := workloadEnvironment(config, participant, environment) + workerEnv := workloadEnvironment(config, participant, environment) + readinessTimeout := mpiWorkerReadinessTimeout(backendDeadlineSeconds) + + workerHosts, err := mpiWorkerHosts(config.Namespace, jobName, 1, gpuCount) + if err != nil { + return nil, err + } + + worker := workloadContainer( + config, + policy, + corev1.Container{ + Name: "worker", + Image: config.RunnerImage, + Command: resultCommand("mpi-worker"), + Env: workerEnv, + }, + gpuCount, + ) + + object := map[string]any{ + "apiVersion": "kubeflow.org/v2beta1", + "kind": "MPIJob", + "metadata": map[string]any{ + "name": jobName, + "namespace": config.Namespace, + "labels": stringMapAny(labels), + "ownerReferences": []any{ownerReferenceMap(workflow)}, + }, + "spec": map[string]any{ + "launcherCreationPolicy": "WaitForWorkersReady", + "mpiImplementation": "OpenMPI", + "runLauncherAsWorker": false, + "slotsPerWorker": gpuCount, + "runPolicy": map[string]any{ + "activeDeadlineSeconds": backendDeadlineSeconds, + "backoffLimit": mpiBackoffLimit, + "cleanPodPolicy": "Running", + }, + "mpiReplicaSpecs": map[string]any{ + "Launcher": map[string]any{ + "replicas": int64(1), + "restartPolicy": string(corev1.RestartPolicyNever), + "template": podTemplate( + labels, + "", + "", + tolerations, + corev1.Container{ + Name: "launcher", + Image: config.RunnerImage, + Command: resultCommandWithPrelude( + mpiLauncherPrelude(mpiHostfilePath, workerHosts, readinessTimeout), + "mpi-launcher", + strconv.FormatInt(gpuCount, 10), + ), + Env: launcherEnv, + VolumeMounts: resultVolumeMounts(), + }, + config.ScriptConfigMap, + ), + }, + "Worker": map[string]any{ + "replicas": int64(1), + "restartPolicy": string(corev1.RestartPolicyNever), + "template": podTemplate( + labels, + participant.NodeName, + participant.NodeUID, + tolerations, + worker, + config.ScriptConfigMap, + policy, + ), + }, + }, + }, + } + + return &unstructured.Unstructured{Object: object}, nil +} + +func podTemplate( + labels map[string]string, + nodeName string, + nodeUID types.UID, + tolerations []corev1.Toleration, + container corev1.Container, + scriptConfigMap string, + policies ...signalv1.WorkloadPolicy, +) map[string]any { + policy := signalv1.WorkloadPolicy{} + if len(policies) > 0 { + policy = policies[0] + } + + template := corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + Affinity: participantNodeAffinity(nodeName, nodeUID), + Tolerations: tolerations, + AutomountServiceAccountToken: ptr(false), + Containers: []corev1.Container{container}, + Volumes: workloadVolumes(scriptConfigMap, policy), + }, + } + applyPodPolicy(&template.Spec, policy) + + content, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&template) + if err != nil { + panic(err) + } + + return content +} + +func participantNodeAffinity(nodeName string, nodeUID types.UID) *corev1.Affinity { + if nodeName == "" { + return nil + } + + return &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{ + MatchFields: []corev1.NodeSelectorRequirement{{ + Key: "metadata.name", + Operator: corev1.NodeSelectorOpIn, + Values: []string{nodeName}, + }}, + MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: signalv1.ReservationNodeUIDLabel, + Operator: corev1.NodeSelectorOpIn, + Values: []string{string(nodeUID)}, + }}, + }}, + }, + }, + } +} + +func applyPodPolicy(spec *corev1.PodSpec, policy signalv1.WorkloadPolicy) { + if !policy.HostNetwork { + return + } + + spec.HostNetwork = true + spec.DNSPolicy = corev1.DNSClusterFirstWithHostNet +} + +func workloadEnvironment( + config Config, + participant signalv1.Participant, + environment []corev1.EnvVar, +) []corev1.EnvVar { + result := slices.Clone(environment) + + result = append( + result, + corev1.EnvVar{Name: "PROJECT_SIGNAL_RUN_NAME", Value: config.RunName}, + corev1.EnvVar{Name: "PROJECT_SIGNAL_RUN_UID", Value: string(config.RunUID)}, + corev1.EnvVar{Name: "PROJECT_SIGNAL_PARTICIPANT_INDEX", Value: strconv.FormatInt(int64(participant.Index), 10)}, + corev1.EnvVar{Name: "PROJECT_SIGNAL_NODE_NAME", Value: participant.NodeName}, + corev1.EnvVar{Name: "PROJECT_SIGNAL_NODE_UID", Value: string(participant.NodeUID)}, + corev1.EnvVar{Name: "PROJECT_SIGNAL_SCRIPT_KEY", Value: config.ScriptKey}, + corev1.EnvVar{Name: "PROJECT_SIGNAL_SCRIPT_DIGEST", Value: config.ScriptDigest}, + corev1.EnvVar{Name: "PROJECT_SIGNAL_RESULTS_DIR", Value: "/results"}, + ) + + return result +} + +func resultVolumes(scriptConfigMap string) []corev1.Volume { + return []corev1.Volume{ + { + Name: "scripts", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: scriptConfigMap}, + DefaultMode: ptr(int32(0o555)), + }, + }, + }, + {Name: "results", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + } +} + +func workloadContainer( + config Config, + policy signalv1.WorkloadPolicy, + container corev1.Container, + gpuCount int64, +) corev1.Container { + container.Resources = *policy.Resources.DeepCopy() + if gpuCount > 0 { + if container.Resources.Requests == nil { + container.Resources.Requests = corev1.ResourceList{} + } + + if container.Resources.Limits == nil { + container.Resources.Limits = corev1.ResourceList{} + } + + gpu := *resourceQuantity(gpuCount) + container.Resources.Requests[corev1.ResourceName("nvidia.com/gpu")] = gpu + container.Resources.Limits[corev1.ResourceName("nvidia.com/gpu")] = gpu + } + + if policy.Privileged || len(policy.Capabilities) > 0 { + container.SecurityContext = &corev1.SecurityContext{ + Privileged: ptr(policy.Privileged), + } + if len(policy.Capabilities) > 0 { + container.SecurityContext.Capabilities = &corev1.Capabilities{ + Add: slices.Clone(policy.Capabilities), + } + } + } + + container.WorkingDir = policy.WorkingDir + container.VolumeMounts = append(resultVolumeMounts(), workloadVolumeMounts(policy)...) + + return container +} + +func workloadVolumes(scriptConfigMap string, policy signalv1.WorkloadPolicy) []corev1.Volume { + volumes := resultVolumes(scriptConfigMap) + + for index := range policy.HostPaths { + hostPath := &policy.HostPaths[index] + hostPathType := corev1.HostPathDirectory + volumes = append(volumes, corev1.Volume{ + Name: signalworkload.HostPathVolumeName(index), + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: hostPath.Path, + Type: &hostPathType, + }, + }, + }) + } + + return volumes +} + +func workloadVolumeMounts(policy signalv1.WorkloadPolicy) []corev1.VolumeMount { + mounts := make([]corev1.VolumeMount, 0, len(policy.HostPaths)) + for index := range policy.HostPaths { + hostPath := &policy.HostPaths[index] + mounts = append(mounts, corev1.VolumeMount{ + Name: signalworkload.HostPathVolumeName(index), + MountPath: hostPath.MountPath, + ReadOnly: true, + }) + } + + return mounts +} + +func resultVolumeMounts() []corev1.VolumeMount { + return []corev1.VolumeMount{ + {Name: "scripts", MountPath: "/scripts", ReadOnly: true}, + {Name: "results", MountPath: "/results"}, + } +} + +func validateBackendCoverage( + participantCount int, + resources []signalv1.BackendResourceReference, +) error { + if len(resources) != participantCount { + return fmt.Errorf("backend-resources must contain exactly %d entries", participantCount) + } + + identities := make(map[string]struct{}, len(resources)) + covered := make([]bool, participantCount) + + for index, resource := range resources { + if resource.Name == "" || resource.UID == "" { + return fmt.Errorf("backend-resources[%d] has an incomplete identity", index) + } + + identity := strings.Join([]string{ + resource.Name, + string(resource.UID), + }, "\x00") + if _, duplicate := identities[identity]; duplicate { + return fmt.Errorf("backend-resources[%d] duplicates a workload identity", index) + } + + identities[identity] = struct{}{} + + if len(resource.ParticipantIndexes) != 1 { + return fmt.Errorf("backend-resources[%d] must identify exactly one participant", index) + } + + participantIndex := resource.ParticipantIndexes[0] + if participantIndex < 0 || participantIndex >= int32(participantCount) { + return fmt.Errorf("backend-resources[%d] participant index %d is out of range", index, participantIndex) + } + + if covered[participantIndex] { + return fmt.Errorf("backend-resources[%d] repeats participant index %d", index, participantIndex) + } + + covered[participantIndex] = true + } + + for index, present := range covered { + if !present { + return fmt.Errorf("backend-resources does not cover participant index %d", index) + } + } + + return nil +} + +func decodeJSONObject[T any](content string, destination *T) error { + if strings.TrimSpace(content) == "" { + content = "[]" + if _, ok := any(destination).(*map[string]string); ok { + content = "{}" + } + } + + decoder := json.NewDecoder(strings.NewReader(content)) + decoder.DisallowUnknownFields() + + if err := decoder.Decode(destination); err != nil { + return err + } + + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + if err == nil { + return errors.New("contains trailing JSON values") + } + + return err + } + + return nil +} + +func resourceLabels(config Config, participant signalv1.Participant) map[string]string { + return map[string]string{ + "app.kubernetes.io/name": "project-signal-adapter", + "signal.unbounded-cloud.io/run": config.RunName, + "signal.unbounded-cloud.io/participant-index": strconv.FormatInt(int64(participant.Index), 10), + } +} + +func resourceName(runName string, participantIndex int32) string { + return resourceNameWithLimit(runName, participantIndex, 63) +} + +func resourceNameWithLimit(runName string, participantIndex int32, maximumLength int) string { + suffix := fmt.Sprintf("-p%d", participantIndex) + + maximumRunLength := maximumLength - len(suffix) + if len(runName) > maximumRunLength { + digest := fmt.Sprintf("%x", sha256.Sum256([]byte(runName)))[:8] + prefixLength := maximumRunLength - len(digest) - 1 + runName = strings.TrimRight(runName[:prefixLength], "-.") + "-" + digest + } + + return runName + suffix +} + +func stringMapAny(values map[string]string) map[string]any { + result := make(map[string]any, len(values)) + for key, value := range values { + result[key] = value + } + + return result +} + +func ownerReferenceMap(reference metav1.OwnerReference) map[string]any { + return map[string]any{ + "apiVersion": reference.APIVersion, + "kind": reference.Kind, + "name": reference.Name, + "uid": string(reference.UID), + "controller": reference.Controller, + } +} + +func shellQuote(value string) string { + return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'" +} + +func quotedShellArguments(values []string) []string { + quoted := make([]string, len(values)) + for index, value := range values { + quoted[index] = shellQuote(value) + } + + return quoted +} + +func ptr[T any](value T) *T { + return &value +} diff --git a/internal/signaladapter/backend_evidence.go b/internal/signaladapter/backend_evidence.go new file mode 100644 index 0000000000..e356ef5a69 --- /dev/null +++ b/internal/signaladapter/backend_evidence.go @@ -0,0 +1,828 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signaladapter + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "regexp" + "slices" + "strconv" + "strings" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" +) + +const ( + backendEvidenceVersion = "v1" + maxEvidenceLogBytes = 512 + maxEvidenceTextBytes = 512 + resultsStatusPrefix = "PROJECT_SIGNAL_RESULTS_V1 " +) + +var ( + sensitiveAssignmentPattern = regexp.MustCompile( + `(?i)((?:token|password|passwd|secret|credential|api[_-]?key)\s*[=:]\s*)([^\s]+)`, + ) + bearerPattern = regexp.MustCompile(`(?i)(authorization:\s*bearer\s+)([^\s]+)`) + urlUserPattern = regexp.MustCompile(`(https?://)[^/@\s]+@`) + resultsFinalPattern = regexp.MustCompile( + `^verdict=(?:present|missing) reason=(?:present|missing) measurements=(?:present|missing)$`, + ) +) + +type BackendEvidence struct { + Version string `json:"version"` + Complete bool `json:"complete"` + Participants []ParticipantBackendEvidence `json:"participants"` + CaptureError string `json:"captureError,omitempty"` +} + +type ParticipantBackendEvidence struct { + ParticipantIndex int `json:"participantIndex"` + Resource BackendObjectEvidence `json:"resource"` + LauncherJob *JobEvidence `json:"launcherJob,omitempty"` + Pods []PodEvidence `json:"pods"` + CaptureError string `json:"captureError,omitempty"` +} + +type BackendObjectEvidence struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Namespace string `json:"namespace,omitempty"` + Name string `json:"name"` + UID types.UID `json:"uid"` + Phase string `json:"phase,omitempty"` + Reason string `json:"reason,omitempty"` + Message string `json:"message,omitempty"` + Conditions []ConditionEvidence `json:"conditions,omitempty"` + ReplicaStatuses map[string]ReplicaStatusEvidence `json:"replicaStatuses,omitempty"` +} + +type ReplicaStatusEvidence struct { + Active int64 `json:"active,omitempty"` + Succeeded int64 `json:"succeeded,omitempty"` + Failed int64 `json:"failed,omitempty"` +} + +type ConditionEvidence struct { + Type string `json:"type"` + Status string `json:"status"` + Reason string `json:"reason,omitempty"` + Message string `json:"message,omitempty"` +} + +type JobEvidence struct { + Name string `json:"name"` + UID types.UID `json:"uid"` + Conditions []ConditionEvidence `json:"conditions,omitempty"` +} + +type PodEvidence struct { + Name string `json:"name"` + UID types.UID `json:"uid"` + Role string `json:"role"` + Phase corev1.PodPhase `json:"phase,omitempty"` + Reason string `json:"reason,omitempty"` + Message string `json:"message,omitempty"` + Containers []ContainerEvidence `json:"containers"` + Conditions []ConditionEvidence `json:"conditions,omitempty"` +} + +type ContainerEvidence struct { + Name string `json:"name"` + RestartCount int32 `json:"restartCount"` + State ContainerStateEvidence `json:"state"` + LastState ContainerStateEvidence `json:"lastState"` + CurrentLog CapturedLogEvidence `json:"currentLog"` + PreviousLog CapturedLogEvidence `json:"previousLog"` +} + +type ContainerStateEvidence struct { + State string `json:"state,omitempty"` + ExitCode int32 `json:"exitCode,omitempty"` + Signal int32 `json:"signal,omitempty"` + Reason string `json:"reason,omitempty"` + Message string `json:"message,omitempty"` + FinishedAt string `json:"finishedAt,omitempty"` +} + +type CapturedLogEvidence struct { + Status string `json:"status"` + Bytes int `json:"bytes,omitempty"` + CapturedBytes int `json:"capturedBytes,omitempty"` + Truncated bool `json:"truncated,omitempty"` + Content string `json:"content,omitempty"` + ResultMarkerStatus string `json:"resultMarkerStatus,omitempty"` + ResultDirectoryStatus string `json:"resultDirectoryStatus,omitempty"` + Error string `json:"error,omitempty"` +} + +func MinimumBackendEvidenceBytes(participantCount int) int { + evidence := BackendEvidence{ + Version: backendEvidenceVersion, + Complete: false, + Participants: make([]ParticipantBackendEvidence, participantCount), + } + + for index := range evidence.Participants { + evidence.Participants[index] = ParticipantBackendEvidence{ + ParticipantIndex: index, + Resource: BackendObjectEvidence{ + APIVersion: strings.Repeat("a", 32), + Kind: strings.Repeat("a", 16), + Namespace: strings.Repeat("a", 63), + Name: strings.Repeat("a", 63), + UID: types.UID(strings.Repeat("a", 128)), + }, + CaptureError: strings.Repeat("a", maxEvidenceTextBytes), + } + } + + content, err := json.Marshal(evidence) + if err != nil { + return MaxBackendEvidenceBytes + 1 + } + + return len(content) + 64 +} + +func (r *Runner) captureBackendEvidence( + ctx context.Context, + config Config, + backends []createdResource, +) (string, bool, error) { + evidence := BackendEvidence{ + Version: backendEvidenceVersion, + Complete: true, + Participants: make([]ParticipantBackendEvidence, len(backends)), + } + + for index := range backends { + participant, complete := r.captureParticipantEvidence(ctx, config, backends[index]) + evidence.Participants[index] = participant + evidence.Complete = evidence.Complete && complete + } + + content, complete, err := marshalBackendEvidence(evidence, MaxBackendEvidenceBytes) + if err != nil { + return "", false, err + } + + return string(content), complete, nil +} + +func marshalBackendEvidence(evidence BackendEvidence, limit int) ([]byte, bool, error) { + if limit == 0 { + limit = MaxBackendEvidenceBytes + } + + if limit < 1 || limit > MaxBackendEvidenceBytes { + return nil, false, fmt.Errorf("backend evidence limit must be between 1 and %d bytes", MaxBackendEvidenceBytes) + } + + content, err := json.Marshal(evidence) + if err != nil { + return nil, false, fmt.Errorf("encode backend evidence: %w", err) + } + + if len(content) <= limit { + return content, evidence.Complete, nil + } + + evidence.Complete = false + for index := range evidence.Participants { + participant := &evidence.Participants[index] + + participant.CaptureError = "container log content omitted because backend evidence exceeded the size limit" + for podIndex := range participant.Pods { + pod := &participant.Pods[podIndex] + for containerIndex := range pod.Containers { + container := &pod.Containers[containerIndex] + omitLogContent(&container.CurrentLog) + omitLogContent(&container.PreviousLog) + } + } + } + + content, err = json.Marshal(evidence) + if err != nil { + return nil, false, fmt.Errorf("encode bounded backend evidence: %w", err) + } + + if len(content) <= limit { + return content, false, nil + } + + for index := range evidence.Participants { + participant := &evidence.Participants[index] + participant.Resource = BackendObjectEvidence{ + APIVersion: participant.Resource.APIVersion, + Kind: participant.Resource.Kind, + Namespace: participant.Resource.Namespace, + Name: participant.Resource.Name, + UID: participant.Resource.UID, + } + participant.LauncherJob = nil + participant.Pods = nil + participant.CaptureError = "backend evidence projection reduced because it exceeded the size limit" + } + + content, err = json.Marshal(evidence) + if err != nil { + return nil, false, fmt.Errorf("encode reduced backend evidence: %w", err) + } + + if len(content) > limit { + return nil, false, fmt.Errorf("reduced backend evidence exceeds %d bytes", limit) + } + + return content, false, nil +} + +func omitLogContent(logEvidence *CapturedLogEvidence) { + if logEvidence.Content == "" { + return + } + + logEvidence.Status = "unavailable" + logEvidence.Content = "" + logEvidence.CapturedBytes = 0 + logEvidence.Error = "content omitted because backend evidence exceeded the size limit" +} + +func markBackendEvidenceIncomplete(content, captureError string, limit int) (string, error) { + var evidence BackendEvidence + + if err := json.Unmarshal([]byte(content), &evidence); err != nil { + return "", fmt.Errorf("decode backend evidence after persistence failure: %w", err) + } + + evidence.Complete = false + evidence.CaptureError = boundedRedactedText(captureError, maxEvidenceTextBytes) + + bounded, _, err := marshalBackendEvidence(evidence, limit) + if err != nil { + return "", err + } + + return string(bounded), nil +} + +func (r *Runner) captureParticipantEvidence( + ctx context.Context, + config Config, + backend createdResource, +) (ParticipantBackendEvidence, bool) { + participant := ParticipantBackendEvidence{ + ParticipantIndex: int(backend.participantIndex), + Resource: BackendObjectEvidence{ + APIVersion: backend.resource.GetAPIVersion(), + Kind: backend.resource.GetKind(), + Namespace: backend.resource.GetNamespace(), + Name: backend.resource.GetName(), + UID: backend.resource.GetUID(), + }, + } + complete := true + + object, err := r.Dynamic.Resource(backend.gvr).Namespace(config.Namespace). + Get(ctx, backend.resource.GetName(), metav1.GetOptions{}) + if err != nil { + participant.CaptureError = boundedRedactedText("get backend object: "+err.Error(), maxEvidenceTextBytes) + complete = false + } else if object.GetUID() != backend.resource.GetUID() { + participant.CaptureError = "backend object UID changed before evidence capture" + complete = false + } else { + projected, projectErr := projectBackendObject(object) + if projectErr != nil { + participant.CaptureError = joinCaptureError( + participant.CaptureError, + "project backend object: "+projectErr.Error(), + ) + complete = false + } else { + participant.Resource = projected + } + } + + switch backend.resource.GetKind() { + case "Pod": + pod, podErr := r.Kubernetes.CoreV1().Pods(config.Namespace). + Get(ctx, backend.resource.GetName(), metav1.GetOptions{}) + if podErr != nil { + participant.CaptureError = joinCaptureError(participant.CaptureError, "get participant Pod: "+podErr.Error()) + + return participant, false + } + + if pod.UID != backend.resource.GetUID() { + participant.CaptureError = joinCaptureError(participant.CaptureError, "participant Pod UID changed before evidence capture") + + return participant, false + } + + podEvidence, podComplete := r.capturePodEvidence(ctx, config.Namespace, pod, "participant") + participant.Pods = []PodEvidence{podEvidence} + complete = complete && podComplete + case "MPIJob": + jobs, jobErr := r.Kubernetes.BatchV1().Jobs(config.Namespace).List(ctx, metav1.ListOptions{}) + if jobErr != nil { + participant.CaptureError = joinCaptureError(participant.CaptureError, "list launcher Jobs: "+jobErr.Error()) + complete = false + } + + jobUIDs := map[types.UID]struct{}{} + + if jobErr == nil { + for index := range jobs.Items { + job := &jobs.Items[index] + if !ownedBy(job.OwnerReferences, backend.resource.GetUID()) { + continue + } + + jobUIDs[job.UID] = struct{}{} + projected := projectJob(job) + participant.LauncherJob = &projected + } + } + + pods, podErr := r.Kubernetes.CoreV1().Pods(config.Namespace).List(ctx, metav1.ListOptions{ + LabelSelector: participantLabelSelector(config.RunName, backend.participantIndex), + }) + if podErr != nil { + participant.CaptureError = joinCaptureError(participant.CaptureError, "list MPI Pods: "+podErr.Error()) + + return participant, false + } + + for index := range pods.Items { + pod := &pods.Items[index] + + role := mpiPodRole(pod, backend.resource.GetUID(), jobUIDs) + if role == "" { + continue + } + + podEvidence, podComplete := r.capturePodEvidence(ctx, config.Namespace, pod, role) + participant.Pods = append(participant.Pods, podEvidence) + complete = complete && podComplete + } + + slices.SortFunc(participant.Pods, func(left, right PodEvidence) int { + return strings.Compare(left.Name, right.Name) + }) + + if participant.LauncherJob == nil { + participant.CaptureError = joinCaptureError(participant.CaptureError, "MPI launcher Job was not found") + complete = false + } + + if !hasPodRole(participant.Pods, "launcher") { + participant.CaptureError = joinCaptureError( + participant.CaptureError, + "MPI launcher Pod was not found", + ) + complete = false + } + default: + participant.CaptureError = joinCaptureError( + participant.CaptureError, + fmt.Sprintf("unsupported backend kind %q", backend.resource.GetKind()), + ) + complete = false + } + + return participant, complete +} + +func (r *Runner) capturePodEvidence( + ctx context.Context, + namespace string, + pod *corev1.Pod, + role string, +) (PodEvidence, bool) { + evidence := PodEvidence{ + Name: pod.Name, + UID: pod.UID, + Role: role, + Phase: pod.Status.Phase, + Reason: boundedRedactedText(pod.Status.Reason, maxEvidenceTextBytes), + Message: boundedRedactedText(pod.Status.Message, maxEvidenceTextBytes), + Conditions: projectPodConditions(pod.Status.Conditions), + } + complete := true + + statuses := append(slices.Clone(pod.Status.InitContainerStatuses), pod.Status.ContainerStatuses...) + for _, status := range statuses { + if !capturedContainer(role, status.Name) { + continue + } + + container := ContainerEvidence{ + Name: status.Name, + RestartCount: status.RestartCount, + State: projectContainerState(status.State), + LastState: projectContainerState(status.LastTerminationState), + } + + current, currentErr := r.readPodLog(ctx, namespace, pod.Name, status.Name, false) + + container.CurrentLog = capturedLog(current, currentErr) + if currentErr != nil { + complete = false + } + + if status.RestartCount > 0 || status.LastTerminationState.Terminated != nil { + previous, previousErr := r.readPodLog(ctx, namespace, pod.Name, status.Name, true) + + container.PreviousLog = capturedLog(previous, previousErr) + if previousErr != nil { + complete = false + } + } else { + container.PreviousLog = CapturedLogEvidence{Status: "not-applicable"} + } + + evidence.Containers = append(evidence.Containers, container) + } + + if len(evidence.Containers) == 0 { + complete = false + } + + return evidence, complete +} + +func projectBackendObject(object *unstructured.Unstructured) (BackendObjectEvidence, error) { + evidence := BackendObjectEvidence{ + APIVersion: object.GetAPIVersion(), + Kind: object.GetKind(), + Namespace: object.GetNamespace(), + Name: object.GetName(), + UID: object.GetUID(), + } + + var err error + + evidence.Phase, _, err = unstructured.NestedString(object.Object, "status", "phase") + if err != nil { + return evidence, fmt.Errorf("read status.phase: %w", err) + } + + evidence.Reason, _, err = unstructured.NestedString(object.Object, "status", "reason") + if err != nil { + return evidence, fmt.Errorf("read status.reason: %w", err) + } + + evidence.Message, _, err = unstructured.NestedString(object.Object, "status", "message") + if err != nil { + return evidence, fmt.Errorf("read status.message: %w", err) + } + + evidence.Reason = boundedRedactedText(evidence.Reason, maxEvidenceTextBytes) + evidence.Message = boundedRedactedText(evidence.Message, maxEvidenceTextBytes) + + evidence.Conditions, err = projectUnstructuredConditions(object) + if err != nil { + return evidence, err + } + + statuses, found, err := unstructured.NestedMap(object.Object, "status", "replicaStatuses") + if err != nil { + return evidence, fmt.Errorf("read status.replicaStatuses: %w", err) + } + + if found { + evidence.ReplicaStatuses = make(map[string]ReplicaStatusEvidence, len(statuses)) + for name := range statuses { + active, countErr := nestedReplicaCount(statuses, name, "active") + if countErr != nil { + return evidence, countErr + } + + succeeded, countErr := nestedReplicaCount(statuses, name, "succeeded") + if countErr != nil { + return evidence, countErr + } + + failed, countErr := nestedReplicaCount(statuses, name, "failed") + if countErr != nil { + return evidence, countErr + } + + evidence.ReplicaStatuses[name] = ReplicaStatusEvidence{ + Active: active, + Succeeded: succeeded, + Failed: failed, + } + } + } + + return evidence, nil +} + +func nestedReplicaCount(statuses map[string]any, replica, field string) (int64, error) { + value, _, err := unstructured.NestedInt64(statuses, replica, field) + if err != nil { + return 0, fmt.Errorf("read status.replicaStatuses.%s.%s: %w", replica, field, err) + } + + return value, nil +} + +func projectUnstructuredConditions(object *unstructured.Unstructured) ([]ConditionEvidence, error) { + conditions, found, err := unstructured.NestedSlice(object.Object, "status", "conditions") + if err != nil { + return nil, fmt.Errorf("read status.conditions: %w", err) + } + + if !found { + return nil, nil + } + + projected := make([]ConditionEvidence, 0, len(conditions)) + for _, raw := range conditions { + condition, ok := raw.(map[string]any) + if !ok { + continue + } + + projected = append(projected, ConditionEvidence{ + Type: stringValue(condition["type"]), + Status: stringValue(condition["status"]), + Reason: boundedRedactedText(stringValue(condition["reason"]), maxEvidenceTextBytes), + Message: boundedRedactedText(stringValue(condition["message"]), maxEvidenceTextBytes), + }) + } + + return projected, nil +} + +func projectJob(job *batchv1.Job) JobEvidence { + conditions := make([]ConditionEvidence, 0, len(job.Status.Conditions)) + for _, condition := range job.Status.Conditions { + conditions = append(conditions, ConditionEvidence{ + Type: string(condition.Type), + Status: string(condition.Status), + Reason: boundedRedactedText(condition.Reason, maxEvidenceTextBytes), + Message: boundedRedactedText(condition.Message, maxEvidenceTextBytes), + }) + } + + return JobEvidence{Name: job.Name, UID: job.UID, Conditions: conditions} +} + +func projectPodConditions(conditions []corev1.PodCondition) []ConditionEvidence { + projected := make([]ConditionEvidence, 0, len(conditions)) + for _, condition := range conditions { + projected = append(projected, ConditionEvidence{ + Type: string(condition.Type), + Status: string(condition.Status), + Reason: boundedRedactedText(condition.Reason, maxEvidenceTextBytes), + Message: boundedRedactedText(condition.Message, maxEvidenceTextBytes), + }) + } + + return projected +} + +func projectContainerState(state corev1.ContainerState) ContainerStateEvidence { + switch { + case state.Terminated != nil: + return ContainerStateEvidence{ + State: "terminated", + ExitCode: state.Terminated.ExitCode, + Signal: state.Terminated.Signal, + Reason: boundedRedactedText(state.Terminated.Reason, maxEvidenceTextBytes), + Message: boundedRedactedText(state.Terminated.Message, maxEvidenceTextBytes), + FinishedAt: state.Terminated.FinishedAt.UTC().Format("2006-01-02T15:04:05.999999999Z07:00"), + } + case state.Waiting != nil: + return ContainerStateEvidence{ + State: "waiting", + Reason: boundedRedactedText(state.Waiting.Reason, maxEvidenceTextBytes), + Message: boundedRedactedText(state.Waiting.Message, maxEvidenceTextBytes), + } + case state.Running != nil: + return ContainerStateEvidence{State: "running"} + default: + return ContainerStateEvidence{} + } +} + +func capturedLog(content string, err error) CapturedLogEvidence { + if err != nil { + return CapturedLogEvidence{ + Status: "unavailable", + Error: boundedRedactedText(err.Error(), maxEvidenceTextBytes), + } + } + + redacted := redactLog(content) + + truncated := len(redacted) > maxEvidenceLogBytes + if truncated { + redacted = redacted[len(redacted)-maxEvidenceLogBytes:] + } + + return CapturedLogEvidence{ + Status: "captured", + Bytes: len(content), + CapturedBytes: len(redacted), + Truncated: truncated, + Content: redacted, + ResultMarkerStatus: resultLogSourceMetadata(content, nil).Status, + ResultDirectoryStatus: resultDirectoryStatus(content), + } +} + +func redactLog(content string) string { + lines := strings.Split(content, "\n") + begin := strings.Index(content, "BEGIN ") + end := strings.Index(content, "END ") + inPrivateKey := end >= 0 && (begin < 0 || end < begin) && strings.Contains(content[end:], "PRIVATE KEY") + + for index, line := range lines { + if strings.Contains(line, "BEGIN ") && strings.Contains(line, "PRIVATE KEY") { + inPrivateKey = true + lines[index] = "[REDACTED PRIVATE KEY]" + + continue + } + + if inPrivateKey { + lines[index] = "[REDACTED PRIVATE KEY]" + + if strings.Contains(line, "END ") && strings.Contains(line, "PRIVATE KEY") { + inPrivateKey = false + } + + continue + } + + if strings.HasPrefix(line, resultPrefix) { + lines[index] = resultPrefix + "[REDACTED]" + + continue + } + + line = bearerPattern.ReplaceAllString(line, `${1}[REDACTED]`) + line = sensitiveAssignmentPattern.ReplaceAllString(line, `${1}[REDACTED]`) + lines[index] = urlUserPattern.ReplaceAllString(line, `${1}[REDACTED]@`) + } + + return strings.Join(lines, "\n") +} + +func resultDirectoryStatus(content string) string { + status := "missing" + + for _, line := range strings.Split(content, "\n") { + if strings.HasPrefix(line, resultsStatusPrefix) { + value := strings.TrimPrefix(line, resultsStatusPrefix) + switch { + case value == "prepared", resultsFinalPattern.MatchString(value): + status = value + default: + status = "invalid" + } + } + } + + return status +} + +func boundedRedactedText(value string, limit int) string { + value = redactLog(value) + if len(value) <= limit { + return value + } + + return value[:limit] +} + +func participantLabelSelector(runName string, participantIndex int32) string { + return strings.Join([]string{ + "signal.unbounded-cloud.io/run=" + runName, + "signal.unbounded-cloud.io/participant-index=" + strconv.FormatInt(int64(participantIndex), 10), + }, ",") +} + +func mpiPodRole(pod *corev1.Pod, mpiJobUID types.UID, jobUIDs map[types.UID]struct{}) string { + if ownedBy(pod.OwnerReferences, mpiJobUID) && hasContainer(pod, "worker") { + return "worker" + } + + for _, reference := range pod.OwnerReferences { + if _, exists := jobUIDs[reference.UID]; exists && hasContainer(pod, "launcher") { + return "launcher" + } + } + + return "" +} + +func hasContainer(pod *corev1.Pod, name string) bool { + return slices.ContainsFunc(pod.Spec.Containers, func(container corev1.Container) bool { + return container.Name == name + }) +} + +func capturedContainer(role, name string) bool { + switch role { + case "participant": + return name == "main" + case "launcher": + return name == "launcher" + case "worker": + return name == "worker" + default: + return false + } +} + +func hasPodRole(pods []PodEvidence, role string) bool { + return slices.ContainsFunc(pods, func(pod PodEvidence) bool { + return pod.Role == role + }) +} + +func joinCaptureError(current, next string) string { + next = boundedRedactedText(next, maxEvidenceTextBytes) + if current == "" { + return next + } + + return boundedRedactedText(current+"; "+next, maxEvidenceTextBytes) +} + +func stringValue(value any) string { + text, ok := value.(string) + if !ok { + return "" + } + + return text +} + +func recordEvidenceCaptureFailure(output *AggregateOutput) { + output.ExecutionState = "Error" + output.Verdict = "Inconclusive" + + if output.Reason == "" || output.Reason == "AllParticipantsPassed" || + output.Reason == "ParticipantCertificationFailed" || + output.Reason == "ParticipantCertificationInconclusive" { + output.Reason = "BackendEvidenceCaptureFailed" + } +} + +func ValidateBackendEvidence(content string) error { + if content == "" { + return errors.New("backend evidence is required") + } + + if len(content) > MaxBackendEvidenceBytes { + return fmt.Errorf("backend evidence exceeds %d bytes", MaxBackendEvidenceBytes) + } + + var evidence BackendEvidence + + decoder := json.NewDecoder(strings.NewReader(content)) + decoder.DisallowUnknownFields() + + if err := decoder.Decode(&evidence); err != nil { + return fmt.Errorf("decode backend evidence: %w", err) + } + + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + if err == nil { + return errors.New("backend evidence contains trailing JSON values") + } + + return fmt.Errorf("decode trailing backend evidence value: %w", err) + } + + if evidence.Version != backendEvidenceVersion { + return fmt.Errorf("backend evidence version %q is invalid", evidence.Version) + } + + if len(evidence.Participants) < 1 || len(evidence.Participants) > MaxParticipants { + return fmt.Errorf( + "backend evidence must contain between 1 and %d participants", + MaxParticipants, + ) + } + + return nil +} diff --git a/internal/signaladapter/quantity.go b/internal/signaladapter/quantity.go new file mode 100644 index 0000000000..1dc68523ca --- /dev/null +++ b/internal/signaladapter/quantity.go @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signaladapter + +import ( + "strconv" + + "k8s.io/apimachinery/pkg/api/resource" +) + +func resourceQuantity(value int64) *resource.Quantity { + quantity := resource.MustParse(strconv.FormatInt(value, 10)) + + return &quantity +} diff --git a/internal/signaladapter/runner.go b/internal/signaladapter/runner.go new file mode 100644 index 0000000000..e48b6a2b6d --- /dev/null +++ b/internal/signaladapter/runner.go @@ -0,0 +1,801 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signaladapter + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "strings" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" +) + +var ( + workflowGVR = schema.GroupVersionResource{Group: "argoproj.io", Version: "v1alpha1", Resource: "workflows"} + mpiJobGVR = schema.GroupVersionResource{Group: "kubeflow.org", Version: "v2beta1", Resource: "mpijobs"} + podGVR = schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"} +) + +type Runner struct { + Dynamic dynamic.Interface + Kubernetes kubernetes.Interface + PollInterval time.Duration + CaptureWait time.Duration + ReadPodLog func(context.Context, string, string, string, bool) (string, error) + PersistBackendEvidence func(string) error +} + +type createdResource struct { + gvr schema.GroupVersionResource + resource *unstructured.Unstructured + participantIndex int32 +} + +const maxResultLogBytes = 64 * 1024 + +type resultLogSource struct { + content string + err error + metadata ResultLogSourceMetadata +} + +type resultLogSnapshot struct { + current resultLogSource + previous resultLogSource +} + +func (r *Runner) Execute(ctx context.Context, config Config) (AggregateOutput, error) { + if r.Dynamic == nil || r.Kubernetes == nil { + return AggregateOutput{}, errors.New("kubernetes clients are required") + } + + participants, tolerations, environment, err := ValidateConfig(config) + if err != nil { + return AggregateOutput{}, err + } + + workflow, err := r.Dynamic.Resource(workflowGVR).Namespace(config.Namespace). + Get(ctx, config.WorkflowName, metav1.GetOptions{}) + if err != nil { + return AggregateOutput{}, fmt.Errorf("get owning Workflow: %w", err) + } + + activeDeadlineSeconds, found, err := unstructured.NestedInt64( + workflow.Object, + "spec", + "activeDeadlineSeconds", + ) + if err != nil { + return AggregateOutput{}, fmt.Errorf("read Workflow active deadline: %w", err) + } + + if !found || activeDeadlineSeconds < 1 { + return AggregateOutput{}, errors.New("owning Workflow requires a positive activeDeadlineSeconds") + } + + if err := signalv1.ValidateExecutionDeadlines( + time.Duration(config.BackendDeadlineSeconds)*time.Second, + time.Duration(activeDeadlineSeconds)*time.Second, + ); err != nil { + return AggregateOutput{}, fmt.Errorf("validate backend and Workflow deadlines: %w", err) + } + + executionContext, cancelExecution := context.WithTimeout( + ctx, + time.Duration(config.BackendDeadlineSeconds)*time.Second, + ) + defer cancelExecution() + + controller := true + owner := metav1.OwnerReference{ + APIVersion: workflow.GetAPIVersion(), + Kind: workflow.GetKind(), + Name: workflow.GetName(), + UID: workflow.GetUID(), + Controller: &controller, + } + + resources, err := RenderResources( + config, + participants, + tolerations, + environment, + owner, + ) + if err != nil { + return AggregateOutput{}, err + } + + created := make([]createdResource, 0, len(resources)) + references := make([]signalv1.BackendResourceReference, 0, len(resources)) + finishBackendFailure := func(cause error) (AggregateOutput, error) { + return r.backendFailureOutput(config, len(participants), created, references, cause) + } + + for index, resource := range resources { + gvr := podGVR + if resource.GetKind() == "MPIJob" { + gvr = mpiJobGVR + } + + createdObject, createErr := r.Dynamic.Resource(gvr).Namespace(config.Namespace). + Create(executionContext, resource, metav1.CreateOptions{}) + if createErr != nil { + if len(created) > 0 { + return finishBackendFailure(fmt.Errorf("create participant %d %s: %w", index, resource.GetKind(), createErr)) + } + + return AggregateOutput{}, fmt.Errorf("create participant %d %s: %w", index, resource.GetKind(), createErr) + } + + created = append(created, createdResource{ + gvr: gvr, + resource: createdObject, + participantIndex: int32(index), + }) + + reference, referenceErr := BackendResource(createdObject, int32(index)) + if referenceErr != nil { + fallbackReference, identityErr := backendResourceIdentity(createdObject, int32(index)) + if identityErr != nil { + return AggregateOutput{}, errors.Join( + fmt.Errorf("record participant %d backend identity: %w", index, referenceErr), + identityErr, + ) + } + + references = append(references, fallbackReference) + + return finishBackendFailure(fmt.Errorf("record participant %d backend identity: %w", index, referenceErr)) + } + + references = append(references, reference) + } + + results := make([]Result, len(created)) + + var waitGroup sync.WaitGroup + + for index := range created { + waitGroup.Add(1) + + go func() { + defer waitGroup.Done() + + results[index] = r.waitForResult(executionContext, config, created[index]) + }() + } + + waitGroup.Wait() + + output, err := Aggregate(len(participants), references, results) + if err != nil { + return finishBackendFailure(fmt.Errorf("aggregate participant results: %w", err)) + } + + captureContext, cancelCapture := context.WithTimeout(context.Background(), r.captureWait()) + defer cancelCapture() + + var evidenceComplete bool + + output.BackendEvidence, evidenceComplete, err = r.captureBackendEvidence( + captureContext, + config, + created, + ) + if err != nil { + return finishBackendFailure(fmt.Errorf("capture backend evidence: %w", err)) + } + + if !evidenceComplete { + recordEvidenceCaptureFailure(&output) + } + + if err := r.persistBackendEvidence(&output, MaxBackendEvidenceBytes); err != nil { + return finishBackendFailure(fmt.Errorf("persist backend evidence: %w", err)) + } + + if resultsRequireQuiescence(results) { + if err := r.quiesceBackends(created); err != nil { + return AggregateOutput{}, fmt.Errorf("quiesce uncertain backends: %w", err) + } + } + + return output, nil +} + +func (r *Runner) backendFailureOutput( + config Config, + participantCount int, + created []createdResource, + references []signalv1.BackendResourceReference, + cause error, +) (AggregateOutput, error) { + captureContext, cancelCapture := context.WithTimeout(context.Background(), r.captureWait()) + defer cancelCapture() + + evidence, _, err := r.captureBackendEvidence( + captureContext, + config, + created, + ) + if err != nil { + return AggregateOutput{}, fmt.Errorf("capture partial backend evidence: %w", err) + } + + backendJSON, err := json.Marshal(references) + if err != nil { + return AggregateOutput{}, fmt.Errorf("encode partial backend resources: %w", err) + } + + reason := boundedRedactedText(cause.Error(), maxEvidenceTextBytes) + + results := make([]Result, participantCount) + for index := range results { + results[index] = Result{ + ParticipantIndex: index, + ExecutionState: "Error", + Verdict: "Inconclusive", + Reason: "BackendSetupFailed", + } + } + + measurements, err := json.Marshal(map[string]any{ + "participantCount": participantCount, + "createdBackends": len(created), + "error": reason, + "results": results, + }) + if err != nil { + return AggregateOutput{}, fmt.Errorf("encode partial backend measurements: %w", err) + } + + output := AggregateOutput{ + ExecutionState: "Error", + Verdict: "Inconclusive", + Reason: "BackendSetupFailed", + Measurements: string(measurements), + BackendResources: references, + BackendJSON: string(backendJSON), + BackendEvidence: evidence, + } + + if err := r.persistBackendEvidence(&output, MaxBackendEvidenceBytes); err != nil { + return AggregateOutput{}, err + } + + if err := r.quiesceBackends(created); err != nil { + return AggregateOutput{}, fmt.Errorf("quiesce partial backends: %w", err) + } + + return output, nil +} + +func (r *Runner) persistBackendEvidence(output *AggregateOutput, limit int) error { + if r.PersistBackendEvidence == nil { + return nil + } + + if err := r.PersistBackendEvidence(output.BackendEvidence); err != nil { + var markErr error + + output.BackendEvidence, markErr = markBackendEvidenceIncomplete( + output.BackendEvidence, + "persist backend evidence: "+err.Error(), + limit, + ) + if markErr != nil { + return errors.Join( + fmt.Errorf("persist backend evidence: %w", err), + markErr, + ) + } + + recordEvidenceCaptureFailure(output) + + return fmt.Errorf("persist backend evidence: %w", err) + } + + return nil +} + +func resultsRequireQuiescence(results []Result) bool { + for _, result := range results { + if result.Reason == "BackendDeadlineExceeded" || + strings.HasPrefix(result.Reason, "BackendObservationFailed:") || + strings.HasPrefix(result.Reason, "BackendStatusInvalid:") { + return true + } + } + + return false +} + +func (r *Runner) quiesceBackends(backends []createdResource) error { + ctx, cancel := context.WithTimeout(context.Background(), r.captureWait()) + defer cancel() + + propagation := metav1.DeletePropagationForeground + + for _, backend := range backends { + uid := backend.resource.GetUID() + err := r.Dynamic.Resource(backend.gvr).Namespace(backend.resource.GetNamespace()). + Delete(ctx, backend.resource.GetName(), metav1.DeleteOptions{ + Preconditions: &metav1.Preconditions{UID: &uid}, + PropagationPolicy: &propagation, + }) + + if err != nil && !apierrors.IsNotFound(err) { + return err + } + } + + ticker := time.NewTicker(r.pollInterval()) + defer ticker.Stop() + + for _, backend := range backends { + for { + object, err := r.Dynamic.Resource(backend.gvr).Namespace(backend.resource.GetNamespace()). + Get(ctx, backend.resource.GetName(), metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + break + } + + if err != nil { + return err + } + + if object.GetUID() != backend.resource.GetUID() { + return fmt.Errorf("backend %s/%s identity changed during quiescence", object.GetKind(), object.GetName()) + } + + select { + case <-ctx.Done(): + return fmt.Errorf("wait for backend %s/%s quiescence: %w", object.GetKind(), object.GetName(), ctx.Err()) + case <-ticker.C: + } + } + } + + return nil +} + +func (r *Runner) waitForResult(ctx context.Context, config Config, backend createdResource) Result { + result := Result{ + ParticipantIndex: int(backend.participantIndex), + ExecutionState: "Error", + Verdict: "Inconclusive", + Reason: "BackendExecutionFailed", + } + + ticker := time.NewTicker(r.pollInterval()) + defer ticker.Stop() + + var capturedResult *Result + + for { + object, err := r.Dynamic.Resource(backend.gvr).Namespace(config.Namespace). + Get(ctx, backend.resource.GetName(), metav1.GetOptions{}) + if err != nil { + result.Reason = "BackendObservationFailed: " + err.Error() + + return result + } + + terminal, succeeded, stateErr := backendState(object) + if stateErr != nil { + result.Reason = "BackendStatusInvalid: " + stateErr.Error() + + return result + } + + if backend.resource.GetKind() == "MPIJob" && capturedResult == nil { + current := r.resultLogs(ctx, config, backend, false).current + if current.err == nil { + parsed, parseErr := ParseResultLog(current.content, int(backend.participantIndex)) + if parseErr == nil { + parsed.ResultLog = &ResultLogMetadata{ + Current: current.metadata, + Previous: ResultLogSourceMetadata{Status: "not-read"}, + SelectedSource: "current", + } + capturedResult = &parsed + } + } + } + + if terminal { + logs := r.resultLogs(ctx, config, backend, backend.resource.GetKind() == "MPIJob") + + completed, hardFailure := backendCompletionFromLogs( + logs, + int(backend.participantIndex), + succeeded, + ) + if completed.ExecutionState == "Succeeded" || hardFailure || capturedResult == nil { + return completed + } + + capturedResult.ResultLog = &ResultLogMetadata{ + Current: capturedResult.ResultLog.Current, + Previous: logs.previous.metadata, + SelectedSource: "captured-current", + } + + return completedResult(*capturedResult, succeeded) + } + + select { + case <-ctx.Done(): + result.Reason = "BackendDeadlineExceeded" + + return result + case <-ticker.C: + } + } +} + +func backendCompletionFromLogs( + logs resultLogSnapshot, + participantIndex int, + backendSucceeded bool, +) (Result, bool) { + result := Result{ + ParticipantIndex: participantIndex, + ExecutionState: "Error", + Verdict: "Inconclusive", + Reason: "BackendExecutionFailed", + ResultLog: &ResultLogMetadata{ + Current: logs.current.metadata, + Previous: logs.previous.metadata, + }, + } + + current, currentErr := parseResultLogSource(logs.current, participantIndex) + previous, previousErr := parseResultLogSource(logs.previous, participantIndex) + + if currentErr == nil { + if previousErr == nil && !sameResult(current, previous) { + result.Reason = "BackendResultInvalid: current and previous launcher logs contain conflicting results" + + return result, true + } + + current.ResultLog = result.ResultLog + current.ResultLog.SelectedSource = "current" + + return completedResult(current, backendSucceeded), false + } + + if logs.current.metadata.Status == "invalid" { + if backendSucceeded { + result.Reason = "BackendResultInvalid: " + currentErr.Error() + } + + return result, true + } + + if previousErr == nil { + previous.ResultLog = result.ResultLog + previous.ResultLog.SelectedSource = "previous" + + return completedResult(previous, backendSucceeded), false + } + + hardFailure := logs.previous.metadata.Status == "invalid" + + if backendSucceeded { + switch { + case hardFailure: + result.Reason = "BackendResultInvalid: " + previousErr.Error() + case logs.current.err != nil: + result.Reason = "BackendResultUnavailable: " + logs.current.err.Error() + default: + result.Reason = "BackendResultInvalid: " + currentErr.Error() + } + } + + return result, hardFailure +} + +func parseResultLogSource(source resultLogSource, participantIndex int) (Result, error) { + if source.err != nil { + return Result{}, source.err + } + + return ParseResultLog(source.content, participantIndex) +} + +func sameResult(left, right Result) bool { + return left.Verdict == right.Verdict && + left.Reason == right.Reason && + left.Measurements == right.Measurements && + left.ExitCode == right.ExitCode +} + +func completedResult(result Result, backendSucceeded bool) Result { + if (result.ExitCode == 0) != backendSucceeded { + return Result{ + ParticipantIndex: result.ParticipantIndex, + ExecutionState: "Error", + Verdict: "Inconclusive", + Reason: "BackendResultInvalid: result exit code contradicts backend terminal state", + ResultLog: result.ResultLog, + } + } + + return result +} + +func backendState(resource *unstructured.Unstructured) (bool, bool, error) { + switch resource.GetKind() { + case "Pod": + phase, found, err := unstructured.NestedString(resource.Object, "status", "phase") + if err != nil { + return false, false, err + } + + if !found { + return false, false, nil + } + + switch corev1.PodPhase(phase) { + case corev1.PodSucceeded: + return true, true, nil + case corev1.PodFailed: + return true, false, nil + default: + return false, false, nil + } + case "MPIJob": + succeeded, _, err := unstructured.NestedInt64( + resource.Object, + "status", + "replicaStatuses", + "Launcher", + "succeeded", + ) + if err != nil { + return false, false, err + } + + if succeeded > 0 { + return true, true, nil + } + + failed, _, err := unstructured.NestedInt64( + resource.Object, + "status", + "replicaStatuses", + "Launcher", + "failed", + ) + if err != nil { + return false, false, err + } + + if failed > 0 { + return true, false, nil + } + + return false, false, nil + default: + return false, false, fmt.Errorf("unsupported backend kind %q", resource.GetKind()) + } +} + +func (r *Runner) resultLogs( + ctx context.Context, + config Config, + backend createdResource, + includePrevious bool, +) resultLogSnapshot { + podName := backend.resource.GetName() + containerName := "main" + + if backend.resource.GetKind() == "MPIJob" { + jobs, err := r.Kubernetes.BatchV1().Jobs(config.Namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + source := resultLogSourceFromError(fmt.Errorf("list MPIJob launcher Jobs: %w", err)) + + return resultLogSnapshot{ + current: source, + previous: resultLogSourceMetadataOnly("not-read"), + } + } + + jobUIDs := make(map[types.UID]struct{}) + + for index := range jobs.Items { + job := &jobs.Items[index] + if ownedBy(job.OwnerReferences, backend.resource.GetUID()) { + jobUIDs[job.UID] = struct{}{} + } + } + + pods, err := r.Kubernetes.CoreV1().Pods(config.Namespace).List(ctx, metav1.ListOptions{ + LabelSelector: participantLabelSelector(config.RunName, backend.participantIndex), + }) + if err != nil { + source := resultLogSourceFromError(fmt.Errorf("list MPIJob Pods: %w", err)) + + return resultLogSnapshot{ + current: source, + previous: resultLogSourceMetadataOnly("not-read"), + } + } + + podName = "" + + for index := range pods.Items { + pod := &pods.Items[index] + if mpiPodRole(pod, backend.resource.GetUID(), jobUIDs) == "launcher" { + podName = pod.Name + containerName = "launcher" + + break + } + } + + if podName == "" { + source := resultLogSourceFromError(errors.New("MPIJob launcher Pod was not found")) + + return resultLogSnapshot{ + current: source, + previous: resultLogSourceMetadataOnly("not-read"), + } + } + } + + currentContent, currentErr := r.readPodLog(ctx, config.Namespace, podName, containerName, false) + snapshot := resultLogSnapshot{ + current: resultLogSourceWithContent(currentContent, currentErr), + previous: resultLogSourceMetadataOnly("not-read"), + } + + if includePrevious { + previousContent, previousErr := r.readPodLog(ctx, config.Namespace, podName, containerName, true) + snapshot.previous = resultLogSourceWithContent(previousContent, previousErr) + } + + return snapshot +} + +func (r *Runner) readPodLog( + ctx context.Context, + namespace, podName, containerName string, + previous bool, +) (string, error) { + if r.ReadPodLog != nil { + return r.ReadPodLog(ctx, namespace, podName, containerName, previous) + } + + tailLines := int64(64) + + stream, err := r.Kubernetes.CoreV1().Pods(namespace). + GetLogs(podName, &corev1.PodLogOptions{Container: containerName, Previous: previous, TailLines: &tailLines}). + Stream(ctx) + if err != nil { + return "", fmt.Errorf("open result log: %w", err) + } + + tail := &resultLogTail{} + + _, err = io.Copy(tail, stream) + if err != nil { + closeErr := stream.Close() + if closeErr != nil { + return "", errors.Join( + fmt.Errorf("read result log: %w", err), + fmt.Errorf("close result log: %w", closeErr), + ) + } + + return "", fmt.Errorf("read result log: %w", err) + } + + if err := stream.Close(); err != nil { + return "", fmt.Errorf("close result log: %w", err) + } + + return string(tail.content), nil +} + +type resultLogTail struct { + content []byte +} + +func (tail *resultLogTail) Write(content []byte) (int, error) { + count := len(content) + if len(content) >= maxResultLogBytes { + tail.content = append(tail.content[:0], content[len(content)-maxResultLogBytes:]...) + + return count, nil + } + + if excess := len(tail.content) + len(content) - maxResultLogBytes; excess > 0 { + tail.content = tail.content[excess:] + } + + tail.content = append(tail.content, content...) + + return count, nil +} + +func resultLogSourceWithContent(content string, err error) resultLogSource { + return resultLogSource{ + content: content, + err: err, + metadata: resultLogSourceMetadata(content, err), + } +} + +func resultLogSourceFromError(err error) resultLogSource { + return resultLogSourceWithContent("", err) +} + +func resultLogSourceMetadataOnly(status string) resultLogSource { + return resultLogSource{metadata: ResultLogSourceMetadata{Status: status}} +} + +func resultLogSourceMetadata(content string, err error) ResultLogSourceMetadata { + if err != nil { + return ResultLogSourceMetadata{Status: "unavailable"} + } + + status := "missing" + + if strings.Contains(content, resultPrefix) { + if _, parseErr := ParseResultLog(content, 0); parseErr != nil { + status = "invalid" + } else { + status = "valid" + } + } + + return ResultLogSourceMetadata{ + Status: status, + Bytes: len(content), + } +} + +func (r *Runner) pollInterval() time.Duration { + if r.PollInterval > 0 { + return r.PollInterval + } + + return 2 * time.Second +} + +func (r *Runner) captureWait() time.Duration { + if r.CaptureWait > 0 { + return r.CaptureWait + } + + return 30 * time.Second +} + +func ownedBy(references []metav1.OwnerReference, uid types.UID) bool { + for _, reference := range references { + if reference.UID == uid { + return true + } + } + + return false +} diff --git a/internal/signalworkload/policy.go b/internal/signalworkload/policy.go new file mode 100644 index 0000000000..c29d0b2076 --- /dev/null +++ b/internal/signalworkload/policy.go @@ -0,0 +1,188 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package signalworkload + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "path" + "regexp" + "strings" + + corev1 "k8s.io/api/core/v1" + + signalv1 "github.com/Azure/unbounded/api/signal/v1alpha1" +) + +const MaxPolicyBytes = 16 * 1024 + +var ( + capabilityPattern = regexp.MustCompile(`^[A-Z][A-Z0-9_]*$`) + allowedCapabilities = map[corev1.Capability]struct{}{ + "IPC_LOCK": {}, + } + allowedHostPathRoots = []string{"/proc", "/sys", "/var/lib/project-signal"} +) + +func HostPathVolumeName(index int) string { + return fmt.Sprintf("host-%02d", index) +} + +func Encode(policy signalv1.WorkloadPolicy) (string, error) { + if err := Validate(policy); err != nil { + return "", err + } + + content, err := json.Marshal(policy) + if err != nil { + return "", fmt.Errorf("encode workload policy: %w", err) + } + + if len(content) > MaxPolicyBytes { + return "", fmt.Errorf("workload policy exceeds %d bytes", MaxPolicyBytes) + } + + return string(content), nil +} + +func Decode(content string) (signalv1.WorkloadPolicy, error) { + var policy signalv1.WorkloadPolicy + + if strings.TrimSpace(content) == "" { + content = "{}" + } + + decoder := json.NewDecoder(strings.NewReader(content)) + decoder.DisallowUnknownFields() + + if err := decoder.Decode(&policy); err != nil { + return policy, fmt.Errorf("decode workload policy: %w", err) + } + + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + if err == nil { + return policy, errors.New("workload policy contains trailing JSON values") + } + + return policy, fmt.Errorf("decode trailing workload policy value: %w", err) + } + + if err := Validate(policy); err != nil { + return policy, err + } + + return policy, nil +} + +func Validate(policy signalv1.WorkloadPolicy) error { + if err := validateResources(policy.Resources); err != nil { + return err + } + + if err := validateCapabilities(policy.Capabilities); err != nil { + return err + } + + if policy.WorkingDir != "" { + if err := validateAbsolutePath("workingDir", policy.WorkingDir); err != nil { + return err + } + } + + if len(policy.HostPaths) > 16 { + return errors.New("workload policy must not contain more than 16 host paths") + } + + mountPaths := map[string]struct{}{"/scripts": {}, "/results": {}} + + for index := range policy.HostPaths { + hostPath := &policy.HostPaths[index] + if err := validateAbsolutePath(fmt.Sprintf("workload policy host path %d mountPath", index), hostPath.MountPath); err != nil { + return err + } + + if _, exists := mountPaths[hostPath.MountPath]; exists { + return fmt.Errorf( + "workload policy host path %d duplicates reserved or existing mountPath %q", + index, + hostPath.MountPath, + ) + } + + mountPaths[hostPath.MountPath] = struct{}{} + if err := validateAbsolutePath(fmt.Sprintf("workload policy host path %d path", index), hostPath.Path); err != nil { + return err + } + + if !allowedHostPath(hostPath.Path) { + return fmt.Errorf("workload policy host path %d source %q is not allowlisted", index, hostPath.Path) + } + } + + return nil +} + +func validateResources(resources corev1.ResourceRequirements) error { + for name, quantity := range resources.Requests { + if quantity.Sign() < 0 { + return fmt.Errorf("workload policy resource request %q must not be negative", name) + } + } + + for name, quantity := range resources.Limits { + if quantity.Sign() < 0 { + return fmt.Errorf("workload policy resource limit %q must not be negative", name) + } + + request, exists := resources.Requests[name] + if exists && quantity.Cmp(request) < 0 { + return fmt.Errorf("workload policy resource limit %q must not be less than its request", name) + } + } + + return nil +} + +func validateCapabilities(capabilities []corev1.Capability) error { + seen := map[corev1.Capability]struct{}{} + + for _, capability := range capabilities { + if !capabilityPattern.MatchString(string(capability)) { + return fmt.Errorf("workload policy capability %q is invalid", capability) + } + + if _, allowed := allowedCapabilities[capability]; !allowed { + return fmt.Errorf("workload policy capability %q is not allowlisted", capability) + } + + if _, exists := seen[capability]; exists { + return fmt.Errorf("workload policy capability %q is repeated", capability) + } + + seen[capability] = struct{}{} + } + + return nil +} + +func allowedHostPath(value string) bool { + for _, root := range allowedHostPathRoots { + if value == root || strings.HasPrefix(value, root+"/") { + return true + } + } + + return false +} + +func validateAbsolutePath(name, value string) error { + if value == "" || !strings.HasPrefix(value, "/") || path.Clean(value) != value { + return fmt.Errorf("%s must be a clean absolute path", name) + } + + return nil +}