diff --git a/pkg/driver/docker/build.go b/pkg/driver/docker/build.go index 4750d5522..85bb9cfec 100644 --- a/pkg/driver/docker/build.go +++ b/pkg/driver/docker/build.go @@ -17,6 +17,7 @@ import ( "github.com/devsy-org/devsy/pkg/driver" "github.com/devsy-org/devsy/pkg/log" "github.com/devsy-org/devsy/pkg/provider" + "k8s.io/apimachinery/pkg/util/wait" ) func (d *dockerDriver) BuildDevContainer( @@ -405,13 +406,48 @@ func (d *dockerDriver) executeBuild( buildOptions *build.BuildOptions, ) error { log.Infof("build with %s", strategy.name()) + + var lastErr error + err := wait.ExponentialBackoffWithContext( + ctx, + buildBackoff, + func(ctx context.Context) (bool, error) { + output, buildErr := d.runBuildAttempt(ctx, strategy, req, buildOptions) + lastErr = classifyBuildError(buildErr, output) + if lastErr == nil { + return true, nil + } + if !errors.Is(lastErr, errTransientBuild) { + return false, lastErr + } + log.Warnf("transient build failure, retrying: %v", lastErr) + return false, nil + }, + ) + if wait.Interrupted(err) { + return lastErr + } + return err +} + +// runBuildAttempt runs the build once and returns the tail of the streamed +// output alongside the error for transient-failure classification. +func (d *dockerDriver) runBuildAttempt( + ctx context.Context, + strategy buildStrategy, + req driver.BuildRequest, + buildOptions *build.BuildOptions, +) (string, error) { writer := log.Writer(log.LevelInfo) defer func() { _ = writer.Close() }() - if err := strategy.build(ctx, writer, req.Options.Platform, buildOptions); err != nil { - return fmt.Errorf("%s build: %w", strategy.name(), err) + outputTail := &tailBuffer{limit: maxStderrDiagnostics} + multiWriter := io.MultiWriter(writer, outputTail) + + if err := strategy.build(ctx, multiWriter, req.Options.Platform, buildOptions); err != nil { + return outputTail.String(), fmt.Errorf("%s build: %w", strategy.name(), err) } - return nil + return "", nil } func (d *dockerDriver) createBuildInfo( diff --git a/pkg/driver/docker/build_retry.go b/pkg/driver/docker/build_retry.go new file mode 100644 index 000000000..fdb516fce --- /dev/null +++ b/pkg/driver/docker/build_retry.go @@ -0,0 +1,45 @@ +package docker + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "k8s.io/apimachinery/pkg/util/wait" +) + +var buildBackoff = wait.Backoff{ + Duration: 2 * time.Second, + Factor: 2.0, + Steps: 3, +} + +// errTransientBuild marks a build failure that is safe to retry. +var errTransientBuild = errors.New("transient build failure") + +// transientBuildPatterns are lowercase substrings matched against the build +// error and output. Add a pattern when a new transient failure is observed. +var transientBuildPatterns = []string{ + "incompleteread", +} + +// classifyBuildError wraps err with errTransientBuild when it matches a +// known-transient signature. Context cancellation is never transient. +func classifyBuildError(err error, output string) error { + if err == nil { + return nil + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + + haystack := strings.ToLower(err.Error() + "\n" + output) + for _, pattern := range transientBuildPatterns { + if strings.Contains(haystack, pattern) { + return fmt.Errorf("%w: %w", errTransientBuild, err) + } + } + return err +} diff --git a/pkg/driver/docker/build_retry_test.go b/pkg/driver/docker/build_retry_test.go new file mode 100644 index 000000000..c3d97f65a --- /dev/null +++ b/pkg/driver/docker/build_retry_test.go @@ -0,0 +1,64 @@ +package docker + +import ( + "context" + "errors" + "fmt" + "testing" +) + +func TestClassifyBuildError(t *testing.T) { + t.Parallel() + + // The opaque error the internal buildkit strategy returns; the transient + // signal lives only in the streamed output. + solveErr := errors.New( + "internal buildkit build: build: failed to solve: process " + + `"/bin/sh -c ./devcontainer-features-install.sh" did not complete successfully: exit code: 1`, + ) + incompleteReadOutput := "http.client.IncompleteRead: IncompleteRead(13384521 bytes read, 1002976 more expected)" + + tests := []struct { + name string + err error + output string + want bool + }{ + {name: "nil error", err: nil, want: false}, + { + name: "incomplete read in output", + err: solveErr, + output: incompleteReadOutput, + want: true, + }, + { + name: "genuine build failure", + err: solveErr, + output: "npm ERR! missing script: build", + want: false, + }, + { + name: "context canceled is not transient", + err: fmt.Errorf("build: %w", context.Canceled), + want: false, + }, + { + name: "deadline exceeded is not transient", + err: fmt.Errorf("build: %w", context.DeadlineExceeded), + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + classified := classifyBuildError(tt.err, tt.output) + if got := errors.Is(classified, errTransientBuild); got != tt.want { + t.Errorf("transient = %v, want %v", got, tt.want) + } + if tt.err != nil && !errors.Is(classified, tt.err) { + t.Errorf("classifyBuildError() dropped the original error") + } + }) + } +}