Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 19 additions & 12 deletions cli/cmd/deploy_components.go
Original file line number Diff line number Diff line change
Expand Up @@ -882,9 +882,9 @@ spec:
// allows the resource creation call through even if webhook TLS is still
// not verifiable. BOTH configs must be bypassed — the validating webhook
// blocks the CREATE exactly like the mutating one, so patching only the
// mutating config leaves the failure in place. cert-manager reconciles
// the webhook configs back to its desired state shortly after.
// mutating config leaves the failure in place.
dl.Println(" ⚠ Webhook TLS not verifiable — temporarily setting failurePolicy: Ignore")
defer restoreCertMgrWebhookPolicy(gCtx, certMgrWebhookConfigs)
for _, kind := range certMgrWebhookConfigs {
runExecFn(gCtx, "kubectl", "patch",
kind, "cert-manager-webhook",
Expand All @@ -894,19 +894,10 @@ spec:
time.Sleep(2 * time.Second) // let the API server pick up the patch

applyErr := runExecStdinFn(gCtx, "kubectl", []string{"apply", "-f", "-"}, clusterIssuer)

// Always restore failurePolicy: Fail regardless of outcome.
for _, kind := range certMgrWebhookConfigs {
runExecFn(gCtx, "kubectl", "patch",
kind, "cert-manager-webhook",
"--type=json", "-p",
`[{"op":"replace","path":"/webhooks/0/failurePolicy","value":"Fail"}]`)
}

if applyErr != nil {
return fmt.Errorf("failed to apply cert-manager ClusterIssuer: %w", applyErr)
}
dl.Println(" - ClusterIssuer created. Webhook policy restored.")
dl.Println(" - ClusterIssuer created. Restoring the webhook policy.")
return nil
})
}
Expand Down Expand Up @@ -1020,6 +1011,22 @@ spec:
return nil
}

// restoreCertMgrWebhookPolicy puts failurePolicy: Fail back on the cert-manager
// webhook configs, on a context ctx's cancellation does not reach. When a
// sibling branch of the deploy group failed, ctx was already done, the restore
// patch never ran, and the webhook stayed fail-open for good: later deploys
// skip an installed cert-manager, and cert-manager does not put the policy back.
func restoreCertMgrWebhookPolicy(ctx context.Context, configs []string) {
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
defer cancel()
for _, kind := range configs {
if err := runExecFn(ctx, "kubectl", "patch", kind, "cert-manager-webhook", "--type=json", "-p",
`[{"op":"replace","path":"/webhooks/0/failurePolicy","value":"Fail"}]`); err != nil {
dl.Printf(" ⚠ Could not restore failurePolicy: Fail on %s cert-manager-webhook: %v\n", kind, err)
}
}
}

// deployGroupB deploys Group B components (Core Infrastructure) in parallel:
// Kafka, Monitoring, PostgreSQL, followed by sequential readiness waits,
// entity operator wait, Kafka Connect, and CDC connectors.
Expand Down
33 changes: 33 additions & 0 deletions cli/cmd/deploy_components_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -980,3 +980,36 @@ func sliceContainsPrefix(values []string, prefix string) bool {
}
return false
}

// A sibling branch that fails cancels the deploy group's context while the
// ClusterIssuer goes in past the cert-manager webhook. The restore ran on that
// context, so its patches never ran and the webhook stayed fail-open.
func TestRestoreCertMgrWebhookPolicyOutlivesCancel(t *testing.T) {
origExec, origDL := runExecFn, dl
t.Cleanup(func() { runExecFn, dl = origExec, origDL })
dl = &DashboardController{}
var patched []string
runExecFn = func(ctx context.Context, name string, args ...string) error {
if err := ctx.Err(); err != nil {
return err
}
if _, ok := ctx.Deadline(); !ok {
t.Errorf("restore %v runs with no deadline", args)
}
patched = append(patched, strings.Join(args, " "))
return nil
}

gCtx, cancel := context.WithCancel(context.Background())
cancel()
restoreCertMgrWebhookPolicy(gCtx, []string{"mutatingwebhookconfiguration", "validatingwebhookconfiguration"})

if len(patched) != 2 {
t.Fatalf("patched %d webhook configs, want both: %q", len(patched), patched)
}
for i, kind := range []string{"mutatingwebhookconfiguration", "validatingwebhookconfiguration"} {
if !strings.HasPrefix(patched[i], "patch "+kind+" cert-manager-webhook") || !strings.Contains(patched[i], `"value":"Fail"`) {
t.Errorf("patch %d = %q, want failurePolicy Fail on %s", i, patched[i], kind)
}
}
}
Loading