diff --git a/pkg/reconciler/managed/reconciler.go b/pkg/reconciler/managed/reconciler.go index d27c9d393..c6bd53fac 100644 --- a/pkg/reconciler/managed/reconciler.go +++ b/pkg/reconciler/managed/reconciler.go @@ -954,6 +954,18 @@ func (r *Reconciler) effectivePollInterval(o metav1.Object) time.Duration { return r.pollInterval } +// afterLastCreateResult returns true if the supplied time is in a later second +// than the supplied resource's most recent create result, successful or failed, +// or if the resource has no create result. +func afterLastCreateResult(o metav1.Object, t time.Time) bool { + last := meta.GetExternalCreateFailed(o) + if s := meta.GetExternalCreateSucceeded(o); s.After(last) { + last = s + } + + return t.Truncate(time.Second).After(last.Truncate(time.Second)) +} + // NewReconciler returns a Reconciler that reconciles managed resources of the // supplied ManagedKind with resources in an external system such as a cloud // provider API. It panics if asked to reconcile a managed resource kind that is @@ -1477,6 +1489,16 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (resu } if !observation.ResourceExists && policy.ShouldCreate() { + // The create annotations have one second resolution. Unless we can + // find the external resource by name, wait until a pending annotation + // would be newer than the last create result. Otherwise an interrupted + // attempt would look complete and we'd create the resource again. + if !r.deterministicExternalName && !afterLastCreateResult(managed, time.Now()) { + log.Debug("Waiting to retry creation of external resource") + + return reconcile.Result{Requeue: true}, nil + } + // We write this annotation for two reasons. Firstly, it helps // us to detect the case in which we fail to persist critical // information (like the external name) that may be set by the diff --git a/pkg/reconciler/managed/reconciler_legacy_test.go b/pkg/reconciler/managed/reconciler_legacy_test.go index ae5a471c3..eb448a212 100644 --- a/pkg/reconciler/managed/reconciler_legacy_test.go +++ b/pkg/reconciler/managed/reconciler_legacy_test.go @@ -852,6 +852,83 @@ func TestReconciler(t *testing.T) { }, want: want{result: reconcile.Result{Requeue: true}}, }, + "CreateRetryNotAfterLastResult": { + reason: "We should requeue, without updating the managed resource or calling Create, until the current second is later than the one in which the last create result was recorded.", + args: args{ + m: &fake.Manager{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { + mg := asLegacyManaged(obj, 42) + + // A result from the current second is handled the + // same way. One from the future makes this case + // independent of the wall clock. + meta.SetExternalCreatePending(mg, time.Now().Add(time.Minute)) + meta.SetExternalCreateFailed(mg, time.Now().Add(time.Minute)) + + return nil + }), + // Neither should be called in this scenario. + MockUpdate: test.NewMockUpdateFn(errBoom), + MockStatusUpdate: test.NewMockSubResourceUpdateFn(errBoom), + }, + Scheme: fake.SchemeWith(&fake.LegacyManaged{}), + }, + mg: resource.ManagedKind(fake.GVK(&fake.LegacyManaged{})), + o: []ReconcilerOption{ + WithInitializers(), + WithReferenceResolver(ReferenceResolverFn(func(_ context.Context, _ resource.Managed) error { return nil })), + WithExternalConnector(ExternalConnectorFn(func(_ context.Context, _ resource.Managed) (ExternalClient, error) { + c := &ExternalClientFns{ + ObserveFn: func(_ context.Context, _ resource.Managed) (ExternalObservation, error) { + return ExternalObservation{ResourceExists: false}, nil + }, + CreateFn: func(_ context.Context, _ resource.Managed) (ExternalCreation, error) { + // Create should not be called in this scenario. + return ExternalCreation{}, errBoom + }, + DisconnectFn: func(_ context.Context) error { + return nil + }, + } + + return c, nil + })), + WithFinalizer(resource.FinalizerFns{AddFinalizerFn: func(_ context.Context, _ resource.Object) error { return nil }}), + }, + }, + want: want{result: reconcile.Result{Requeue: true}}, + }, + "CreateRetryNotAfterLastResultDeterministicName": { + reason: "We should not wait for the second of the last create result to pass when the external name is deterministic, because we can find what we created by name.", + args: args{ + m: &fake.Manager{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { + mg := asLegacyManaged(obj, 42) + meta.SetExternalCreatePending(mg, time.Now().Add(time.Minute)) + meta.SetExternalCreateFailed(mg, time.Now().Add(time.Minute)) + + return nil + }), + // We get as far as updating our status after Create. + MockUpdate: test.NewMockUpdateFn(nil), + MockStatusUpdate: test.NewMockSubResourceUpdateFn(errBoom), + }, + Scheme: fake.SchemeWith(&fake.LegacyManaged{}), + }, + mg: resource.ManagedKind(fake.GVK(&fake.LegacyManaged{})), + o: []ReconcilerOption{ + WithInitializers(), + WithReferenceResolver(ReferenceResolverFn(func(_ context.Context, _ resource.Managed) error { return nil })), + WithExternalConnector(&NopConnector{}), + WithCriticalAnnotationUpdater(CriticalAnnotationUpdateFn(func(_ context.Context, _ client.Object) error { return nil })), + WithFinalizer(resource.FinalizerFns{AddFinalizerFn: func(_ context.Context, _ resource.Object) error { return nil }}), + WithDeterministicExternalName(true), + }, + }, + want: want{result: reconcile.Result{Requeue: true}, err: errors.Wrap(errBoom, errUpdateManagedStatus)}, + }, "UpdateCriticalAnnotationsError": { reason: "Errors updating critical annotations after creation should trigger a requeue after a short wait.", args: args{ diff --git a/pkg/reconciler/managed/reconciler_modern_test.go b/pkg/reconciler/managed/reconciler_modern_test.go index c0903fd0c..d2b86140f 100644 --- a/pkg/reconciler/managed/reconciler_modern_test.go +++ b/pkg/reconciler/managed/reconciler_modern_test.go @@ -1008,6 +1008,83 @@ func TestModernReconciler(t *testing.T) { }, want: want{result: reconcile.Result{Requeue: true}}, }, + "CreateRetryNotAfterLastResult": { + reason: "We should requeue, without updating the managed resource or calling Create, until the current second is later than the one in which the last create result was recorded.", + args: args{ + m: &fake.Manager{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { + mg := asModernManaged(obj, 42) + + // A result from the current second is handled the + // same way. One from the future makes this case + // independent of the wall clock. + meta.SetExternalCreatePending(mg, time.Now().Add(time.Minute)) + meta.SetExternalCreateFailed(mg, time.Now().Add(time.Minute)) + + return nil + }), + // Neither should be called in this scenario. + MockUpdate: test.NewMockUpdateFn(errBoom), + MockStatusUpdate: test.NewMockSubResourceUpdateFn(errBoom), + }, + Scheme: fake.SchemeWith(&fake.ModernManaged{}), + }, + mg: resource.ManagedKind(fake.GVK(&fake.ModernManaged{})), + o: []ReconcilerOption{ + WithInitializers(), + WithReferenceResolver(ReferenceResolverFn(func(_ context.Context, _ resource.Managed) error { return nil })), + WithExternalConnector(ExternalConnectorFn(func(_ context.Context, _ resource.Managed) (ExternalClient, error) { + c := &ExternalClientFns{ + ObserveFn: func(_ context.Context, _ resource.Managed) (ExternalObservation, error) { + return ExternalObservation{ResourceExists: false}, nil + }, + CreateFn: func(_ context.Context, _ resource.Managed) (ExternalCreation, error) { + // Create should not be called in this scenario. + return ExternalCreation{}, errBoom + }, + DisconnectFn: func(_ context.Context) error { + return nil + }, + } + + return c, nil + })), + WithFinalizer(resource.FinalizerFns{AddFinalizerFn: func(_ context.Context, _ resource.Object) error { return nil }}), + }, + }, + want: want{result: reconcile.Result{Requeue: true}}, + }, + "CreateRetryNotAfterLastResultDeterministicName": { + reason: "We should not wait for the second of the last create result to pass when the external name is deterministic, because we can find what we created by name.", + args: args{ + m: &fake.Manager{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { + mg := asModernManaged(obj, 42) + meta.SetExternalCreatePending(mg, time.Now().Add(time.Minute)) + meta.SetExternalCreateFailed(mg, time.Now().Add(time.Minute)) + + return nil + }), + // We get as far as updating our status after Create. + MockUpdate: test.NewMockUpdateFn(nil), + MockStatusUpdate: test.NewMockSubResourceUpdateFn(errBoom), + }, + Scheme: fake.SchemeWith(&fake.ModernManaged{}), + }, + mg: resource.ManagedKind(fake.GVK(&fake.ModernManaged{})), + o: []ReconcilerOption{ + WithInitializers(), + WithReferenceResolver(ReferenceResolverFn(func(_ context.Context, _ resource.Managed) error { return nil })), + WithExternalConnector(&NopConnector{}), + WithCriticalAnnotationUpdater(CriticalAnnotationUpdateFn(func(_ context.Context, _ client.Object) error { return nil })), + WithFinalizer(resource.FinalizerFns{AddFinalizerFn: func(_ context.Context, _ resource.Object) error { return nil }}), + WithDeterministicExternalName(true), + }, + }, + want: want{result: reconcile.Result{Requeue: true}, err: errors.Wrap(errBoom, errUpdateManagedStatus)}, + }, "UpdateCriticalAnnotationsError": { reason: "Errors updating critical annotations after creation should trigger a requeue after a short wait.", args: args{ @@ -3124,6 +3201,80 @@ func TestEffectivePollInterval(t *testing.T) { } } +func TestAfterLastCreateResult(t *testing.T) { + second := time.Date(2026, time.November, 16, 12, 0, 30, 0, time.UTC) + + cases := map[string]struct { + reason string + failed time.Time + succeeded time.Time + now time.Time + want bool + }{ + "NoCreateResult": { + reason: "A resource that has never recorded a create result can record a pending create at any time.", + now: second.Add(400 * time.Millisecond), + want: true, + }, + "FailedDuringCurrentSecond": { + reason: "A pending create recorded in the second of a failed create would not be newer than it.", + failed: second.Add(200 * time.Millisecond), + now: second.Add(400 * time.Millisecond), + want: false, + }, + "SucceededDuringCurrentSecond": { + reason: "The most recent result counts, here a success after an older failure.", + failed: second.Add(-time.Minute), + succeeded: second, + now: second.Add(900 * time.Millisecond), + want: false, + }, + "FailedDuringPreviousSecond": { + reason: "A pending create recorded in a later second than the last result is newer than it.", + failed: second.Add(-time.Millisecond), + now: second, + want: true, + }, + "FailedInTheFuture": { + reason: "A result recorded by a clock that is ahead of ours is not yet in the past.", + failed: second.Add(time.Hour), + now: second, + want: false, + }, + "SucceededInTheFuture": { + reason: "A success recorded in the future counts, even when the failure before it is in an earlier second.", + failed: second, + succeeded: second.Add(2 * time.Second), + now: second.Add(1500 * time.Millisecond), + want: false, + }, + "OtherTimeZone": { + reason: "Results are compared as instants, whatever time zone they were recorded in.", + failed: second.In(time.FixedZone("UTC+0530", 5*60*60+30*60)), + now: second.Add(500 * time.Millisecond), + want: false, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + mg := &fake.ModernManaged{} + if !tc.failed.IsZero() { + meta.SetExternalCreateFailed(mg, tc.failed) + } + + if !tc.succeeded.IsZero() { + meta.SetExternalCreateSucceeded(mg, tc.succeeded) + } + + got := afterLastCreateResult(mg, tc.now) + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("\n%s\nafterLastCreateResult(...): -want, +got:\n%s", tc.reason, diff) + } + }) + } +} + func TestReconcilePollIntervalAnnotation(t *testing.T) { cases := map[string]struct { reason string