From 29e71a98e97a30ca1765b22dbed7ee5087dbaa83 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Mon, 27 Jul 2026 10:18:18 -0500 Subject: [PATCH 1/2] fix(docker): retry devcontainer builds on transient failures Feature installs that download release binaries inside the build (via nanolayer) can fail with a truncated download (http.client.IncompleteRead), failing the whole devcontainer build. Since the build is BuildKit-cached, retrying only replays the failed step. executeBuild now retries with exponential backoff, tee-ing the build output into a bounded buffer so the transient signal can be detected from the log stream. The text match is confined to classifyBuildError, which wraps matches in a named errTransientBuild sentinel so callers decide via errors.Is. --- pkg/driver/docker/build.go | 42 ++++++++++++++++-- pkg/driver/docker/build_retry.go | 45 +++++++++++++++++++ pkg/driver/docker/build_retry_test.go | 64 +++++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 3 deletions(-) create mode 100644 pkg/driver/docker/build_retry.go create mode 100644 pkg/driver/docker/build_retry_test.go diff --git a/pkg/driver/docker/build.go b/pkg/driver/docker/build.go index 4750d5522..85bb9cfec 100644 --- a/pkg/driver/docker/build.go +++ b/pkg/driver/docker/build.go @@ -17,6 +17,7 @@ import ( "github.com/devsy-org/devsy/pkg/driver" "github.com/devsy-org/devsy/pkg/log" "github.com/devsy-org/devsy/pkg/provider" + "k8s.io/apimachinery/pkg/util/wait" ) func (d *dockerDriver) BuildDevContainer( @@ -405,13 +406,48 @@ func (d *dockerDriver) executeBuild( buildOptions *build.BuildOptions, ) error { log.Infof("build with %s", strategy.name()) + + var lastErr error + err := wait.ExponentialBackoffWithContext( + ctx, + buildBackoff, + func(ctx context.Context) (bool, error) { + output, buildErr := d.runBuildAttempt(ctx, strategy, req, buildOptions) + lastErr = classifyBuildError(buildErr, output) + if lastErr == nil { + return true, nil + } + if !errors.Is(lastErr, errTransientBuild) { + return false, lastErr + } + log.Warnf("transient build failure, retrying: %v", lastErr) + return false, nil + }, + ) + if wait.Interrupted(err) { + return lastErr + } + return err +} + +// runBuildAttempt runs the build once and returns the tail of the streamed +// output alongside the error for transient-failure classification. +func (d *dockerDriver) runBuildAttempt( + ctx context.Context, + strategy buildStrategy, + req driver.BuildRequest, + buildOptions *build.BuildOptions, +) (string, error) { writer := log.Writer(log.LevelInfo) defer func() { _ = writer.Close() }() - if err := strategy.build(ctx, writer, req.Options.Platform, buildOptions); err != nil { - return fmt.Errorf("%s build: %w", strategy.name(), err) + outputTail := &tailBuffer{limit: maxStderrDiagnostics} + multiWriter := io.MultiWriter(writer, outputTail) + + if err := strategy.build(ctx, multiWriter, req.Options.Platform, buildOptions); err != nil { + return outputTail.String(), fmt.Errorf("%s build: %w", strategy.name(), err) } - return nil + return "", nil } func (d *dockerDriver) createBuildInfo( diff --git a/pkg/driver/docker/build_retry.go b/pkg/driver/docker/build_retry.go new file mode 100644 index 000000000..fdb516fce --- /dev/null +++ b/pkg/driver/docker/build_retry.go @@ -0,0 +1,45 @@ +package docker + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "k8s.io/apimachinery/pkg/util/wait" +) + +var buildBackoff = wait.Backoff{ + Duration: 2 * time.Second, + Factor: 2.0, + Steps: 3, +} + +// errTransientBuild marks a build failure that is safe to retry. +var errTransientBuild = errors.New("transient build failure") + +// transientBuildPatterns are lowercase substrings matched against the build +// error and output. Add a pattern when a new transient failure is observed. +var transientBuildPatterns = []string{ + "incompleteread", +} + +// classifyBuildError wraps err with errTransientBuild when it matches a +// known-transient signature. Context cancellation is never transient. +func classifyBuildError(err error, output string) error { + if err == nil { + return nil + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + + haystack := strings.ToLower(err.Error() + "\n" + output) + for _, pattern := range transientBuildPatterns { + if strings.Contains(haystack, pattern) { + return fmt.Errorf("%w: %w", errTransientBuild, err) + } + } + return err +} diff --git a/pkg/driver/docker/build_retry_test.go b/pkg/driver/docker/build_retry_test.go new file mode 100644 index 000000000..919b47074 --- /dev/null +++ b/pkg/driver/docker/build_retry_test.go @@ -0,0 +1,64 @@ +package docker + +import ( + "context" + "errors" + "fmt" + "testing" +) + +func TestClassifyBuildError(t *testing.T) { + t.Parallel() + + // The opaque error the internal buildkit strategy returns; the transient + // signal lives only in the streamed output. + solveErr := errors.New( + "internal buildkit build: build: failed to solve: process " + + `"/bin/sh -c ./devcontainer-features-install.sh" did not complete successfully: exit code: 1`, + ) + incompleteReadOutput := "http.client.IncompleteRead: IncompleteRead(13384521 bytes read, 1002976 more expected)" + + tests := []struct { + name string + err error + output string + want bool + }{ + {name: "nil error", err: nil, want: false}, + { + name: "incomplete read in output", + err: solveErr, + output: incompleteReadOutput, + want: true, + }, + { + name: "genuine build failure", + err: solveErr, + output: "npm ERR! missing script: build", + want: false, + }, + { + name: "context canceled is not transient", + err: fmt.Errorf("build: %w", context.Canceled), + want: false, + }, + { + name: "deadline exceeded is not transient", + err: fmt.Errorf("build: %w", context.DeadlineExceeded), + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + classified := classifyBuildError(tt.err, tt.output) + if got := errors.Is(classified, errTransientBuild); got != tt.want { + t.Errorf("errors.Is(classifyBuildError(), errTransientBuild) = %v, want %v", got, tt.want) + } + if tt.err != nil && !errors.Is(classified, tt.err) { + t.Errorf("classifyBuildError() dropped the original error") + } + }) + } +} From 257d73d3dfeaec0bf5dd9dc461c8181a70de88b4 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Mon, 27 Jul 2026 10:28:38 -0500 Subject: [PATCH 2/2] test(docker): shorten assertion to satisfy golines --- pkg/driver/docker/build_retry_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/driver/docker/build_retry_test.go b/pkg/driver/docker/build_retry_test.go index 919b47074..c3d97f65a 100644 --- a/pkg/driver/docker/build_retry_test.go +++ b/pkg/driver/docker/build_retry_test.go @@ -54,7 +54,7 @@ func TestClassifyBuildError(t *testing.T) { t.Parallel() classified := classifyBuildError(tt.err, tt.output) if got := errors.Is(classified, errTransientBuild); got != tt.want { - t.Errorf("errors.Is(classifyBuildError(), errTransientBuild) = %v, want %v", got, tt.want) + t.Errorf("transient = %v, want %v", got, tt.want) } if tt.err != nil && !errors.Is(classified, tt.err) { t.Errorf("classifyBuildError() dropped the original error")