Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,13 @@ setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist

.PHONY: test-e2e
test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind.
KIND_CLUSTER=$(KIND_CLUSTER) go test -tags e2e ./test/e2e/ -v -ginkgo.v
KIND_CLUSTER=$(KIND_CLUSTER) go test -tags e2e ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter='!perf'
$(MAKE) cleanup-test-e2e

.PHONY: test-perf
test-perf: setup-test-e2e manifests generate fmt vet ## Run the workload-sync benchmark (Kind, PERF_WORKLOADS replicas).
KIND_CLUSTER=$(KIND_CLUSTER) \
go test -tags e2e ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=perf -timeout 40m
$(MAKE) cleanup-test-e2e

.PHONY: cleanup-test-e2e
Expand Down
2 changes: 1 addition & 1 deletion config/samples/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ metadata:
labels:
app.kubernetes.io/managed-by: nebula
spec:
replicas: 1
replicas: 30
selector:
matchLabels:
app: gpu-workload-sample
Expand Down
18 changes: 18 additions & 0 deletions test/e2e/e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,13 @@ var _ = Describe("Manager", Ordered, func() {
_, _ = utils.Run(exec.Command("kubectl", "delete", "nodepool", fakePoolName, "--ignore-not-found=true"))
_, _ = utils.Run(exec.Command("kubectl", "delete", "ns", fakeWorkloadNS, "--ignore-not-found=true"))

By("cleaning up the sync-benchmark batch, pool, and namespace")
// The benchmark deletes its own batch, so this only covers the spec failing
// part-way — a leftover Pod here would keep a claim alive and block the drain
// below. Pods go with the namespace; the pool is cluster-scoped.
_, _ = utils.Run(exec.Command("kubectl", "delete", "ns", perfWorkloadNS, "--ignore-not-found=true"))
_, _ = utils.Run(exec.Command("kubectl", "delete", "nodepool", perfPoolName, "--ignore-not-found=true"))

By("waiting for NodeClaims to drain while the manager can still terminate instances")
drained := waitForNodeClaimsGone(2 * time.Minute)

Expand Down Expand Up @@ -428,6 +435,17 @@ spec:
_, _ = utils.Run(exec.Command("kubectl", "delete", "-f", manifestFile, "--ignore-not-found=true"))
})

It("should sync a batch of workloads within the time budget", Label("perf"), func() {
// A benchmark, not a latency SLO: it scales one Deployment to N replicas and
// reports how long the whole sync path takes per workload, asserting only a
// loose ceiling so it catches a stalled path without flaking on a busy node.
//
// The perf label keeps it OUT of `make test-e2e` (which filters '!perf') and
// is what `make test-perf` selects, since the batch is slow enough that it
// does not belong in every e2e run. See perf_test.go.
benchmarkWorkloadSync()
})

// +kubebuilder:scaffold:e2e-webhooks-checks

// TODO: Customize the e2e test suite with scenarios specific to your project.
Expand Down
Loading
Loading