Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions pkg/reconciler/managed/reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -954,6 +954,18 @@ func (r *Reconciler) effectivePollInterval(o metav1.Object) time.Duration {
return r.pollInterval
}

// afterLastCreateResult returns true if the supplied time is in a later second
// than the supplied resource's most recent create result, successful or failed,
// or if the resource has no create result.
func afterLastCreateResult(o metav1.Object, t time.Time) bool {
last := meta.GetExternalCreateFailed(o)
if s := meta.GetExternalCreateSucceeded(o); s.After(last) {
last = s
}

return t.Truncate(time.Second).After(last.Truncate(time.Second))
}

// NewReconciler returns a Reconciler that reconciles managed resources of the
// supplied ManagedKind with resources in an external system such as a cloud
// provider API. It panics if asked to reconcile a managed resource kind that is
Expand Down Expand Up @@ -1477,6 +1489,16 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (resu
}

if !observation.ResourceExists && policy.ShouldCreate() {
// The create annotations have one second resolution. Unless we can
// find the external resource by name, wait until a pending annotation
// would be newer than the last create result. Otherwise an interrupted
// attempt would look complete and we'd create the resource again.
if !r.deterministicExternalName && !afterLastCreateResult(managed, time.Now()) {
log.Debug("Waiting to retry creation of external resource")

return reconcile.Result{Requeue: true}, nil
}

// We write this annotation for two reasons. Firstly, it helps
// us to detect the case in which we fail to persist critical
// information (like the external name) that may be set by the
Expand Down
77 changes: 77 additions & 0 deletions pkg/reconciler/managed/reconciler_legacy_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -852,6 +852,83 @@ func TestReconciler(t *testing.T) {
},
want: want{result: reconcile.Result{Requeue: true}},
},
"CreateRetryNotAfterLastResult": {
reason: "We should requeue, without updating the managed resource or calling Create, until the current second is later than the one in which the last create result was recorded.",
args: args{
m: &fake.Manager{
Client: &test.MockClient{
MockGet: test.NewMockGetFn(nil, func(obj client.Object) error {
mg := asLegacyManaged(obj, 42)

// A result from the current second is handled the
// same way. One from the future makes this case
// independent of the wall clock.
meta.SetExternalCreatePending(mg, time.Now().Add(time.Minute))
meta.SetExternalCreateFailed(mg, time.Now().Add(time.Minute))

return nil
}),
// Neither should be called in this scenario.
MockUpdate: test.NewMockUpdateFn(errBoom),
MockStatusUpdate: test.NewMockSubResourceUpdateFn(errBoom),
},
Scheme: fake.SchemeWith(&fake.LegacyManaged{}),
},
mg: resource.ManagedKind(fake.GVK(&fake.LegacyManaged{})),
o: []ReconcilerOption{
WithInitializers(),
WithReferenceResolver(ReferenceResolverFn(func(_ context.Context, _ resource.Managed) error { return nil })),
WithExternalConnector(ExternalConnectorFn(func(_ context.Context, _ resource.Managed) (ExternalClient, error) {
c := &ExternalClientFns{
ObserveFn: func(_ context.Context, _ resource.Managed) (ExternalObservation, error) {
return ExternalObservation{ResourceExists: false}, nil
},
CreateFn: func(_ context.Context, _ resource.Managed) (ExternalCreation, error) {
// Create should not be called in this scenario.
return ExternalCreation{}, errBoom
},
DisconnectFn: func(_ context.Context) error {
return nil
},
}

return c, nil
})),
WithFinalizer(resource.FinalizerFns{AddFinalizerFn: func(_ context.Context, _ resource.Object) error { return nil }}),
},
},
want: want{result: reconcile.Result{Requeue: true}},
},
"CreateRetryNotAfterLastResultDeterministicName": {
reason: "We should not wait for the second of the last create result to pass when the external name is deterministic, because we can find what we created by name.",
args: args{
m: &fake.Manager{
Client: &test.MockClient{
MockGet: test.NewMockGetFn(nil, func(obj client.Object) error {
mg := asLegacyManaged(obj, 42)
meta.SetExternalCreatePending(mg, time.Now().Add(time.Minute))
meta.SetExternalCreateFailed(mg, time.Now().Add(time.Minute))

return nil
}),
// We get as far as updating our status after Create.
MockUpdate: test.NewMockUpdateFn(nil),
MockStatusUpdate: test.NewMockSubResourceUpdateFn(errBoom),
},
Scheme: fake.SchemeWith(&fake.LegacyManaged{}),
},
mg: resource.ManagedKind(fake.GVK(&fake.LegacyManaged{})),
o: []ReconcilerOption{
WithInitializers(),
WithReferenceResolver(ReferenceResolverFn(func(_ context.Context, _ resource.Managed) error { return nil })),
WithExternalConnector(&NopConnector{}),
WithCriticalAnnotationUpdater(CriticalAnnotationUpdateFn(func(_ context.Context, _ client.Object) error { return nil })),
WithFinalizer(resource.FinalizerFns{AddFinalizerFn: func(_ context.Context, _ resource.Object) error { return nil }}),
WithDeterministicExternalName(true),
},
},
want: want{result: reconcile.Result{Requeue: true}, err: errors.Wrap(errBoom, errUpdateManagedStatus)},
},
"UpdateCriticalAnnotationsError": {
reason: "Errors updating critical annotations after creation should trigger a requeue after a short wait.",
args: args{
Expand Down
151 changes: 151 additions & 0 deletions pkg/reconciler/managed/reconciler_modern_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1008,6 +1008,83 @@ func TestModernReconciler(t *testing.T) {
},
want: want{result: reconcile.Result{Requeue: true}},
},
"CreateRetryNotAfterLastResult": {
reason: "We should requeue, without updating the managed resource or calling Create, until the current second is later than the one in which the last create result was recorded.",
args: args{
m: &fake.Manager{
Client: &test.MockClient{
MockGet: test.NewMockGetFn(nil, func(obj client.Object) error {
mg := asModernManaged(obj, 42)

// A result from the current second is handled the
// same way. One from the future makes this case
// independent of the wall clock.
meta.SetExternalCreatePending(mg, time.Now().Add(time.Minute))
meta.SetExternalCreateFailed(mg, time.Now().Add(time.Minute))

return nil
}),
// Neither should be called in this scenario.
MockUpdate: test.NewMockUpdateFn(errBoom),
MockStatusUpdate: test.NewMockSubResourceUpdateFn(errBoom),
},
Scheme: fake.SchemeWith(&fake.ModernManaged{}),
},
mg: resource.ManagedKind(fake.GVK(&fake.ModernManaged{})),
o: []ReconcilerOption{
WithInitializers(),
WithReferenceResolver(ReferenceResolverFn(func(_ context.Context, _ resource.Managed) error { return nil })),
WithExternalConnector(ExternalConnectorFn(func(_ context.Context, _ resource.Managed) (ExternalClient, error) {
c := &ExternalClientFns{
ObserveFn: func(_ context.Context, _ resource.Managed) (ExternalObservation, error) {
return ExternalObservation{ResourceExists: false}, nil
},
CreateFn: func(_ context.Context, _ resource.Managed) (ExternalCreation, error) {
// Create should not be called in this scenario.
return ExternalCreation{}, errBoom
},
DisconnectFn: func(_ context.Context) error {
return nil
},
}

return c, nil
})),
WithFinalizer(resource.FinalizerFns{AddFinalizerFn: func(_ context.Context, _ resource.Object) error { return nil }}),
},
},
want: want{result: reconcile.Result{Requeue: true}},
},
"CreateRetryNotAfterLastResultDeterministicName": {
reason: "We should not wait for the second of the last create result to pass when the external name is deterministic, because we can find what we created by name.",
args: args{
m: &fake.Manager{
Client: &test.MockClient{
MockGet: test.NewMockGetFn(nil, func(obj client.Object) error {
mg := asModernManaged(obj, 42)
meta.SetExternalCreatePending(mg, time.Now().Add(time.Minute))
meta.SetExternalCreateFailed(mg, time.Now().Add(time.Minute))

return nil
}),
// We get as far as updating our status after Create.
MockUpdate: test.NewMockUpdateFn(nil),
MockStatusUpdate: test.NewMockSubResourceUpdateFn(errBoom),
},
Scheme: fake.SchemeWith(&fake.ModernManaged{}),
},
mg: resource.ManagedKind(fake.GVK(&fake.ModernManaged{})),
o: []ReconcilerOption{
WithInitializers(),
WithReferenceResolver(ReferenceResolverFn(func(_ context.Context, _ resource.Managed) error { return nil })),
WithExternalConnector(&NopConnector{}),
WithCriticalAnnotationUpdater(CriticalAnnotationUpdateFn(func(_ context.Context, _ client.Object) error { return nil })),
WithFinalizer(resource.FinalizerFns{AddFinalizerFn: func(_ context.Context, _ resource.Object) error { return nil }}),
WithDeterministicExternalName(true),
},
},
want: want{result: reconcile.Result{Requeue: true}, err: errors.Wrap(errBoom, errUpdateManagedStatus)},
},
"UpdateCriticalAnnotationsError": {
reason: "Errors updating critical annotations after creation should trigger a requeue after a short wait.",
args: args{
Expand Down Expand Up @@ -3124,6 +3201,80 @@ func TestEffectivePollInterval(t *testing.T) {
}
}

func TestAfterLastCreateResult(t *testing.T) {
second := time.Date(2026, time.November, 16, 12, 0, 30, 0, time.UTC)

cases := map[string]struct {
reason string
failed time.Time
succeeded time.Time
now time.Time
want bool
}{
"NoCreateResult": {
reason: "A resource that has never recorded a create result can record a pending create at any time.",
now: second.Add(400 * time.Millisecond),
want: true,
},
"FailedDuringCurrentSecond": {
reason: "A pending create recorded in the second of a failed create would not be newer than it.",
failed: second.Add(200 * time.Millisecond),
now: second.Add(400 * time.Millisecond),
want: false,
},
"SucceededDuringCurrentSecond": {
reason: "The most recent result counts, here a success after an older failure.",
failed: second.Add(-time.Minute),
succeeded: second,
now: second.Add(900 * time.Millisecond),
want: false,
},
"FailedDuringPreviousSecond": {
reason: "A pending create recorded in a later second than the last result is newer than it.",
failed: second.Add(-time.Millisecond),
now: second,
want: true,
},
"FailedInTheFuture": {
reason: "A result recorded by a clock that is ahead of ours is not yet in the past.",
failed: second.Add(time.Hour),
now: second,
want: false,
},
"SucceededInTheFuture": {
reason: "A success recorded in the future counts, even when the failure before it is in an earlier second.",
failed: second,
succeeded: second.Add(2 * time.Second),
now: second.Add(1500 * time.Millisecond),
want: false,
},
"OtherTimeZone": {
reason: "Results are compared as instants, whatever time zone they were recorded in.",
failed: second.In(time.FixedZone("UTC+0530", 5*60*60+30*60)),
now: second.Add(500 * time.Millisecond),
want: false,
},
}

for name, tc := range cases {
t.Run(name, func(t *testing.T) {
mg := &fake.ModernManaged{}
if !tc.failed.IsZero() {
meta.SetExternalCreateFailed(mg, tc.failed)
}

if !tc.succeeded.IsZero() {
meta.SetExternalCreateSucceeded(mg, tc.succeeded)
}

got := afterLastCreateResult(mg, tc.now)
if diff := cmp.Diff(tc.want, got); diff != "" {
t.Errorf("\n%s\nafterLastCreateResult(...): -want, +got:\n%s", tc.reason, diff)
}
})
}
}

func TestReconcilePollIntervalAnnotation(t *testing.T) {
cases := map[string]struct {
reason string
Expand Down