diff --git a/backend/pkg/rustic/run.go b/backend/pkg/rustic/run.go index 5073f55d98..3ee4781acb 100644 --- a/backend/pkg/rustic/run.go +++ b/backend/pkg/rustic/run.go @@ -33,6 +33,15 @@ func Run(ctx context.Context, dockerClient *client.Client, password string, comm ) allMounts := append([]mount.Mount{}, mounts...) allMounts = append(allMounts, mount.Mount{Type: mount.TypeVolume, Source: CacheVolume, Target: "/cache"}) + if bundle := trustBundle(ctx); len(bundle) > 0 { + trustVolume, cleanup, stageErr := stageTrustBundle(ctx, dockerClient, bundle) + if stageErr != nil { + return "", fmt.Errorf("failed to stage CA certificates for Rustic container: %w", stageErr) + } + // Deferred before the Rustic container's removal so it runs after it. + defer cleanup() + allMounts = append(allMounts, mount.Mount{Type: mount.TypeVolume, Source: trustVolume, Target: trustBundleDir, ReadOnly: true}) + } hostConfig := volumes.HostConfig(DefaultImage, nil, allMounts) hostConfig.AutoRemove = false hostConfig.NetworkMode = networkMode @@ -63,14 +72,6 @@ func Run(ctx context.Context, dockerClient *client.Client, password string, comm time.Sleep(5 * time.Second) } }() - if bundle := trustBundle(ctx); len(bundle) > 0 { - copyCtx, copyCancel := context.WithTimeout(ctx, timeouts.DefaultDockerAPI) - copyErr := docker.CopyToContainer(copyCtx, dockerClient, created.ID, "/", map[string][]byte{trustBundleFile: bundle}) - copyCancel() - if copyErr != nil { - return "", fmt.Errorf("failed to stage CA certificates for Rustic container: %w", copyErr) - } - } if _, containerStartErr := dockerClient.ContainerStart(ctx, created.ID, client.ContainerStartOptions{}); containerStartErr != nil { return "", fmt.Errorf("failed to start Rustic container: %w", containerStartErr) } diff --git a/backend/pkg/rustic/trust.go b/backend/pkg/rustic/trust.go index 5fafe40f74..85017e7346 100644 --- a/backend/pkg/rustic/trust.go +++ b/backend/pkg/rustic/trust.go @@ -4,14 +4,27 @@ import ( "bytes" "context" "encoding/pem" + "log/slog" "os" "path/filepath" + "time" + + cerrdefs "github.com/containerd/errdefs" + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/api/types/mount" + "github.com/moby/moby/client" "kit.getarcane.app/acfs" + "kit.getarcane.app/docker" + + "getarcane.dev/arcane/v3/pkg/timeouts" + "getarcane.dev/arcane/v3/pkg/volumes" ) const ( - trustBundleFile = "etc/ssl/certs/ca-certificates.crt" + trustBundleDir = "/etc/ssl/certs" + trustBundleName = "ca-certificates.crt" + stagerRemoveRetries = 12 ) // Same lookup order as Go's crypto/x509 on Linux. @@ -81,3 +94,64 @@ func trustBundle(ctx context.Context) []byte { } return bundle.Bytes() } + +// stageTrustBundle copies the bundle into a volume through a stager that mounts +// only that volume; copying into the Rustic container fails when it has file +// mounts under read-only mounts. Run cleanup after Rustic exits: removing the +// stager right after the copy fails on ZFS while its dataset is busy. +func stageTrustBundle(ctx context.Context, dockerClient *client.Client, bundle []byte) (string, func(), error) { + created, err := dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Labels: volumes.Labels()}) + if err != nil { + return "", nil, err + } + stagerID := "" + cleanup := func() { + cleanupCtx := context.WithoutCancel(ctx) + if stagerID != "" { + removeStager(cleanupCtx, dockerClient, stagerID) + } + removeCtx, cancel := context.WithTimeout(cleanupCtx, timeouts.DefaultDockerAPI) + defer cancel() + if _, removeErr := dockerClient.VolumeRemove(removeCtx, created.Volume.Name, client.VolumeRemoveOptions{Force: true}); removeErr != nil && !cerrdefs.IsNotFound(removeErr) { + slog.WarnContext(cleanupCtx, "failed to remove Rustic CA certificate volume", "volumeName", created.Volume.Name, "error", removeErr) + } + } + stager, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{ + Config: &container.Config{ + Image: DefaultImage, + Labels: volumes.Labels(), + }, + HostConfig: &container.HostConfig{ + Mounts: []mount.Mount{{Type: mount.TypeVolume, Source: created.Volume.Name, Target: trustBundleDir}}, + }, + }) + if err != nil { + cleanup() + return "", nil, err + } + stagerID = stager.ID + copyCtx, copyCancel := context.WithTimeout(ctx, timeouts.DefaultDockerAPI) + err = docker.CopyToContainer(copyCtx, dockerClient, stager.ID, trustBundleDir, map[string][]byte{trustBundleName: bundle}) + copyCancel() + if err != nil { + cleanup() + return "", nil, err + } + return created.Volume.Name, cleanup, nil +} + +func removeStager(ctx context.Context, dockerClient *client.Client, containerID string) { + for attempt := 1; ; attempt++ { + attemptCtx, cancel := context.WithTimeout(ctx, timeouts.DefaultDockerAPI) + _, err := dockerClient.ContainerRemove(attemptCtx, containerID, volumes.RemoveOptions()) + cancel() + if err == nil || cerrdefs.IsNotFound(err) { + return + } + if attempt == stagerRemoveRetries { + slog.WarnContext(ctx, "failed to remove Rustic CA certificate stager", "containerId", containerID, "error", err) + return + } + time.Sleep(5 * time.Second) + } +}