From 508725fef5e88160104d221b7d0ea425bf5b537f Mon Sep 17 00:00:00 2001 From: Prawal Pokharel <57414987+prawalpokharel@users.noreply.github.com> Date: Mon, 16 Nov 2026 23:26:48 -0600 Subject: [PATCH 1/2] test(managed): add cases for a create retried in the same second Add a case to the modern and legacy reconciler tests in which the external resource doesn't exist and the last create result isn't from an earlier second. The cases record the result a minute ahead, so they can't cross into a later second while they run. The reconciler should requeue without writing external-create-pending or calling Create. These cases fail without the fix in the next commit. The create annotations have one second resolution, so a pending annotation written in the same second as the last result is not newer than it, and ExternalCreateIncomplete can't tell an interrupted retry from the attempt that finished before it. Signed-off-by: Prawal Pokharel <57414987+prawalpokharel@users.noreply.github.com> --- .../managed/reconciler_legacy_test.go | 47 +++++++++++++++++++ .../managed/reconciler_modern_test.go | 47 +++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/pkg/reconciler/managed/reconciler_legacy_test.go b/pkg/reconciler/managed/reconciler_legacy_test.go index ae5a471c3..2ffc6ab3a 100644 --- a/pkg/reconciler/managed/reconciler_legacy_test.go +++ b/pkg/reconciler/managed/reconciler_legacy_test.go @@ -852,6 +852,53 @@ func TestReconciler(t *testing.T) { }, want: want{result: reconcile.Result{Requeue: true}}, }, + "CreateRetryNotAfterLastResult": { + reason: "We should requeue, without updating the managed resource or calling Create, until the current second is later than the one in which the last create result was recorded.", + args: args{ + m: &fake.Manager{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { + mg := asLegacyManaged(obj, 42) + + // A result from the current second is handled the + // same way. One from the future makes this case + // independent of the wall clock. + meta.SetExternalCreatePending(mg, time.Now().Add(time.Minute)) + meta.SetExternalCreateFailed(mg, time.Now().Add(time.Minute)) + + return nil + }), + // Neither should be called in this scenario. + MockUpdate: test.NewMockUpdateFn(errBoom), + MockStatusUpdate: test.NewMockSubResourceUpdateFn(errBoom), + }, + Scheme: fake.SchemeWith(&fake.LegacyManaged{}), + }, + mg: resource.ManagedKind(fake.GVK(&fake.LegacyManaged{})), + o: []ReconcilerOption{ + WithInitializers(), + WithReferenceResolver(ReferenceResolverFn(func(_ context.Context, _ resource.Managed) error { return nil })), + WithExternalConnector(ExternalConnectorFn(func(_ context.Context, _ resource.Managed) (ExternalClient, error) { + c := &ExternalClientFns{ + ObserveFn: func(_ context.Context, _ resource.Managed) (ExternalObservation, error) { + return ExternalObservation{ResourceExists: false}, nil + }, + CreateFn: func(_ context.Context, _ resource.Managed) (ExternalCreation, error) { + // Create should not be called in this scenario. + return ExternalCreation{}, errBoom + }, + DisconnectFn: func(_ context.Context) error { + return nil + }, + } + + return c, nil + })), + WithFinalizer(resource.FinalizerFns{AddFinalizerFn: func(_ context.Context, _ resource.Object) error { return nil }}), + }, + }, + want: want{result: reconcile.Result{Requeue: true}}, + }, "UpdateCriticalAnnotationsError": { reason: "Errors updating critical annotations after creation should trigger a requeue after a short wait.", args: args{ diff --git a/pkg/reconciler/managed/reconciler_modern_test.go b/pkg/reconciler/managed/reconciler_modern_test.go index c0903fd0c..b00669b0c 100644 --- a/pkg/reconciler/managed/reconciler_modern_test.go +++ b/pkg/reconciler/managed/reconciler_modern_test.go @@ -1008,6 +1008,53 @@ func TestModernReconciler(t *testing.T) { }, want: want{result: reconcile.Result{Requeue: true}}, }, + "CreateRetryNotAfterLastResult": { + reason: "We should requeue, without updating the managed resource or calling Create, until the current second is later than the one in which the last create result was recorded.", + args: args{ + m: &fake.Manager{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { + mg := asModernManaged(obj, 42) + + // A result from the current second is handled the + // same way. One from the future makes this case + // independent of the wall clock. + meta.SetExternalCreatePending(mg, time.Now().Add(time.Minute)) + meta.SetExternalCreateFailed(mg, time.Now().Add(time.Minute)) + + return nil + }), + // Neither should be called in this scenario. + MockUpdate: test.NewMockUpdateFn(errBoom), + MockStatusUpdate: test.NewMockSubResourceUpdateFn(errBoom), + }, + Scheme: fake.SchemeWith(&fake.ModernManaged{}), + }, + mg: resource.ManagedKind(fake.GVK(&fake.ModernManaged{})), + o: []ReconcilerOption{ + WithInitializers(), + WithReferenceResolver(ReferenceResolverFn(func(_ context.Context, _ resource.Managed) error { return nil })), + WithExternalConnector(ExternalConnectorFn(func(_ context.Context, _ resource.Managed) (ExternalClient, error) { + c := &ExternalClientFns{ + ObserveFn: func(_ context.Context, _ resource.Managed) (ExternalObservation, error) { + return ExternalObservation{ResourceExists: false}, nil + }, + CreateFn: func(_ context.Context, _ resource.Managed) (ExternalCreation, error) { + // Create should not be called in this scenario. + return ExternalCreation{}, errBoom + }, + DisconnectFn: func(_ context.Context) error { + return nil + }, + } + + return c, nil + })), + WithFinalizer(resource.FinalizerFns{AddFinalizerFn: func(_ context.Context, _ resource.Object) error { return nil }}), + }, + }, + want: want{result: reconcile.Result{Requeue: true}}, + }, "UpdateCriticalAnnotationsError": { reason: "Errors updating critical annotations after creation should trigger a requeue after a short wait.", args: args{ From 85ddbb117642462d7d23ea3cafa407cefff9f940 Mon Sep 17 00:00:00 2001 From: Prawal Pokharel <57414987+prawalpokharel@users.noreply.github.com> Date: Tue, 17 Nov 2026 00:03:13 -0600 Subject: [PATCH 2/2] fix(managed): wait for the next second before retrying a create The external-create-pending, -succeeded and -failed annotations have one second resolution. When a create fails and is retried within the same second, the new pending annotation is equal to the failed one. If that retry creates the external resource but its result is never recorded, because the provider is restarted or the annotation update fails, ExternalCreateIncomplete reports the create as complete. If the API assigned the external name, the next reconcile then creates a second external resource. Requeue instead of writing the pending annotation until the current second is later than that of the last create result. The requeue goes through the rate limiter, as the creation grace period's does, so a create that keeps failing still backs off. Reconcilers built with WithDeterministicExternalName(true) skip the wait. They find the external resource by name, and an incomplete create doesn't pause them. Signed-off-by: Prawal Pokharel <57414987+prawalpokharel@users.noreply.github.com> --- pkg/reconciler/managed/reconciler.go | 22 ++++ .../managed/reconciler_legacy_test.go | 30 +++++ .../managed/reconciler_modern_test.go | 104 ++++++++++++++++++ 3 files changed, 156 insertions(+) diff --git a/pkg/reconciler/managed/reconciler.go b/pkg/reconciler/managed/reconciler.go index d27c9d393..c6bd53fac 100644 --- a/pkg/reconciler/managed/reconciler.go +++ b/pkg/reconciler/managed/reconciler.go @@ -954,6 +954,18 @@ func (r *Reconciler) effectivePollInterval(o metav1.Object) time.Duration { return r.pollInterval } +// afterLastCreateResult returns true if the supplied time is in a later second +// than the supplied resource's most recent create result, successful or failed, +// or if the resource has no create result. +func afterLastCreateResult(o metav1.Object, t time.Time) bool { + last := meta.GetExternalCreateFailed(o) + if s := meta.GetExternalCreateSucceeded(o); s.After(last) { + last = s + } + + return t.Truncate(time.Second).After(last.Truncate(time.Second)) +} + // NewReconciler returns a Reconciler that reconciles managed resources of the // supplied ManagedKind with resources in an external system such as a cloud // provider API. It panics if asked to reconcile a managed resource kind that is @@ -1477,6 +1489,16 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (resu } if !observation.ResourceExists && policy.ShouldCreate() { + // The create annotations have one second resolution. Unless we can + // find the external resource by name, wait until a pending annotation + // would be newer than the last create result. Otherwise an interrupted + // attempt would look complete and we'd create the resource again. + if !r.deterministicExternalName && !afterLastCreateResult(managed, time.Now()) { + log.Debug("Waiting to retry creation of external resource") + + return reconcile.Result{Requeue: true}, nil + } + // We write this annotation for two reasons. Firstly, it helps // us to detect the case in which we fail to persist critical // information (like the external name) that may be set by the diff --git a/pkg/reconciler/managed/reconciler_legacy_test.go b/pkg/reconciler/managed/reconciler_legacy_test.go index 2ffc6ab3a..eb448a212 100644 --- a/pkg/reconciler/managed/reconciler_legacy_test.go +++ b/pkg/reconciler/managed/reconciler_legacy_test.go @@ -899,6 +899,36 @@ func TestReconciler(t *testing.T) { }, want: want{result: reconcile.Result{Requeue: true}}, }, + "CreateRetryNotAfterLastResultDeterministicName": { + reason: "We should not wait for the second of the last create result to pass when the external name is deterministic, because we can find what we created by name.", + args: args{ + m: &fake.Manager{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { + mg := asLegacyManaged(obj, 42) + meta.SetExternalCreatePending(mg, time.Now().Add(time.Minute)) + meta.SetExternalCreateFailed(mg, time.Now().Add(time.Minute)) + + return nil + }), + // We get as far as updating our status after Create. + MockUpdate: test.NewMockUpdateFn(nil), + MockStatusUpdate: test.NewMockSubResourceUpdateFn(errBoom), + }, + Scheme: fake.SchemeWith(&fake.LegacyManaged{}), + }, + mg: resource.ManagedKind(fake.GVK(&fake.LegacyManaged{})), + o: []ReconcilerOption{ + WithInitializers(), + WithReferenceResolver(ReferenceResolverFn(func(_ context.Context, _ resource.Managed) error { return nil })), + WithExternalConnector(&NopConnector{}), + WithCriticalAnnotationUpdater(CriticalAnnotationUpdateFn(func(_ context.Context, _ client.Object) error { return nil })), + WithFinalizer(resource.FinalizerFns{AddFinalizerFn: func(_ context.Context, _ resource.Object) error { return nil }}), + WithDeterministicExternalName(true), + }, + }, + want: want{result: reconcile.Result{Requeue: true}, err: errors.Wrap(errBoom, errUpdateManagedStatus)}, + }, "UpdateCriticalAnnotationsError": { reason: "Errors updating critical annotations after creation should trigger a requeue after a short wait.", args: args{ diff --git a/pkg/reconciler/managed/reconciler_modern_test.go b/pkg/reconciler/managed/reconciler_modern_test.go index b00669b0c..d2b86140f 100644 --- a/pkg/reconciler/managed/reconciler_modern_test.go +++ b/pkg/reconciler/managed/reconciler_modern_test.go @@ -1055,6 +1055,36 @@ func TestModernReconciler(t *testing.T) { }, want: want{result: reconcile.Result{Requeue: true}}, }, + "CreateRetryNotAfterLastResultDeterministicName": { + reason: "We should not wait for the second of the last create result to pass when the external name is deterministic, because we can find what we created by name.", + args: args{ + m: &fake.Manager{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { + mg := asModernManaged(obj, 42) + meta.SetExternalCreatePending(mg, time.Now().Add(time.Minute)) + meta.SetExternalCreateFailed(mg, time.Now().Add(time.Minute)) + + return nil + }), + // We get as far as updating our status after Create. + MockUpdate: test.NewMockUpdateFn(nil), + MockStatusUpdate: test.NewMockSubResourceUpdateFn(errBoom), + }, + Scheme: fake.SchemeWith(&fake.ModernManaged{}), + }, + mg: resource.ManagedKind(fake.GVK(&fake.ModernManaged{})), + o: []ReconcilerOption{ + WithInitializers(), + WithReferenceResolver(ReferenceResolverFn(func(_ context.Context, _ resource.Managed) error { return nil })), + WithExternalConnector(&NopConnector{}), + WithCriticalAnnotationUpdater(CriticalAnnotationUpdateFn(func(_ context.Context, _ client.Object) error { return nil })), + WithFinalizer(resource.FinalizerFns{AddFinalizerFn: func(_ context.Context, _ resource.Object) error { return nil }}), + WithDeterministicExternalName(true), + }, + }, + want: want{result: reconcile.Result{Requeue: true}, err: errors.Wrap(errBoom, errUpdateManagedStatus)}, + }, "UpdateCriticalAnnotationsError": { reason: "Errors updating critical annotations after creation should trigger a requeue after a short wait.", args: args{ @@ -3171,6 +3201,80 @@ func TestEffectivePollInterval(t *testing.T) { } } +func TestAfterLastCreateResult(t *testing.T) { + second := time.Date(2026, time.November, 16, 12, 0, 30, 0, time.UTC) + + cases := map[string]struct { + reason string + failed time.Time + succeeded time.Time + now time.Time + want bool + }{ + "NoCreateResult": { + reason: "A resource that has never recorded a create result can record a pending create at any time.", + now: second.Add(400 * time.Millisecond), + want: true, + }, + "FailedDuringCurrentSecond": { + reason: "A pending create recorded in the second of a failed create would not be newer than it.", + failed: second.Add(200 * time.Millisecond), + now: second.Add(400 * time.Millisecond), + want: false, + }, + "SucceededDuringCurrentSecond": { + reason: "The most recent result counts, here a success after an older failure.", + failed: second.Add(-time.Minute), + succeeded: second, + now: second.Add(900 * time.Millisecond), + want: false, + }, + "FailedDuringPreviousSecond": { + reason: "A pending create recorded in a later second than the last result is newer than it.", + failed: second.Add(-time.Millisecond), + now: second, + want: true, + }, + "FailedInTheFuture": { + reason: "A result recorded by a clock that is ahead of ours is not yet in the past.", + failed: second.Add(time.Hour), + now: second, + want: false, + }, + "SucceededInTheFuture": { + reason: "A success recorded in the future counts, even when the failure before it is in an earlier second.", + failed: second, + succeeded: second.Add(2 * time.Second), + now: second.Add(1500 * time.Millisecond), + want: false, + }, + "OtherTimeZone": { + reason: "Results are compared as instants, whatever time zone they were recorded in.", + failed: second.In(time.FixedZone("UTC+0530", 5*60*60+30*60)), + now: second.Add(500 * time.Millisecond), + want: false, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + mg := &fake.ModernManaged{} + if !tc.failed.IsZero() { + meta.SetExternalCreateFailed(mg, tc.failed) + } + + if !tc.succeeded.IsZero() { + meta.SetExternalCreateSucceeded(mg, tc.succeeded) + } + + got := afterLastCreateResult(mg, tc.now) + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("\n%s\nafterLastCreateResult(...): -want, +got:\n%s", tc.reason, diff) + } + }) + } +} + func TestReconcilePollIntervalAnnotation(t *testing.T) { cases := map[string]struct { reason string