From ed8ad8c095e900d9777bd6ad16538b9a9f2d1168 Mon Sep 17 00:00:00 2001 From: Neureka Date: Wed, 18 Nov 2026 11:06:57 -0800 Subject: [PATCH] fix: stage Rustic CA bundle in a volume instead of copying into the container Copying the bundle into the stopped Rustic container makes the daemon create every mount target first. System backups mount the database copies as files under the read-only /data mount, so that fails with "openat data/arcane.db: read-only file system". Stage the bundle in a volume through a stager that mounts only that volume, and remove the stager after Rustic exits, retrying, so ZFS no longer fails with a busy dataset. Co-Authored-By: Claude Opus 5.5 --- backend/pkg/rustic/run.go | 17 +++++---- backend/pkg/rustic/trust.go | 76 ++++++++++++++++++++++++++++++++++++- 2 files changed, 84 insertions(+), 9 deletions(-) diff --git a/backend/pkg/rustic/run.go b/backend/pkg/rustic/run.go index 5073f55d98..3ee4781acb 100644 --- a/backend/pkg/rustic/run.go +++ b/backend/pkg/rustic/run.go @@ -33,6 +33,15 @@ func Run(ctx context.Context, dockerClient *client.Client, password string, comm ) allMounts := append([]mount.Mount{}, mounts...) allMounts = append(allMounts, mount.Mount{Type: mount.TypeVolume, Source: CacheVolume, Target: "/cache"}) + if bundle := trustBundle(ctx); len(bundle) > 0 { + trustVolume, cleanup, stageErr := stageTrustBundle(ctx, dockerClient, bundle) + if stageErr != nil { + return "", fmt.Errorf("failed to stage CA certificates for Rustic container: %w", stageErr) + } + // Deferred before the Rustic container's removal so it runs after it. + defer cleanup() + allMounts = append(allMounts, mount.Mount{Type: mount.TypeVolume, Source: trustVolume, Target: trustBundleDir, ReadOnly: true}) + } hostConfig := volumes.HostConfig(DefaultImage, nil, allMounts) hostConfig.AutoRemove = false hostConfig.NetworkMode = networkMode @@ -63,14 +72,6 @@ func Run(ctx context.Context, dockerClient *client.Client, password string, comm time.Sleep(5 * time.Second) } }() - if bundle := trustBundle(ctx); len(bundle) > 0 { - copyCtx, copyCancel := context.WithTimeout(ctx, timeouts.DefaultDockerAPI) - copyErr := docker.CopyToContainer(copyCtx, dockerClient, created.ID, "/", map[string][]byte{trustBundleFile: bundle}) - copyCancel() - if copyErr != nil { - return "", fmt.Errorf("failed to stage CA certificates for Rustic container: %w", copyErr) - } - } if _, containerStartErr := dockerClient.ContainerStart(ctx, created.ID, client.ContainerStartOptions{}); containerStartErr != nil { return "", fmt.Errorf("failed to start Rustic container: %w", containerStartErr) } diff --git a/backend/pkg/rustic/trust.go b/backend/pkg/rustic/trust.go index 5fafe40f74..85017e7346 100644 --- a/backend/pkg/rustic/trust.go +++ b/backend/pkg/rustic/trust.go @@ -4,14 +4,27 @@ import ( "bytes" "context" "encoding/pem" + "log/slog" "os" "path/filepath" + "time" + + cerrdefs "github.com/containerd/errdefs" + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/api/types/mount" + "github.com/moby/moby/client" "kit.getarcane.app/acfs" + "kit.getarcane.app/docker" + + "getarcane.dev/arcane/v3/pkg/timeouts" + "getarcane.dev/arcane/v3/pkg/volumes" ) const ( - trustBundleFile = "etc/ssl/certs/ca-certificates.crt" + trustBundleDir = "/etc/ssl/certs" + trustBundleName = "ca-certificates.crt" + stagerRemoveRetries = 12 ) // Same lookup order as Go's crypto/x509 on Linux. @@ -81,3 +94,64 @@ func trustBundle(ctx context.Context) []byte { } return bundle.Bytes() } + +// stageTrustBundle copies the bundle into a volume through a stager that mounts +// only that volume; copying into the Rustic container fails when it has file +// mounts under read-only mounts. Run cleanup after Rustic exits: removing the +// stager right after the copy fails on ZFS while its dataset is busy. +func stageTrustBundle(ctx context.Context, dockerClient *client.Client, bundle []byte) (string, func(), error) { + created, err := dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Labels: volumes.Labels()}) + if err != nil { + return "", nil, err + } + stagerID := "" + cleanup := func() { + cleanupCtx := context.WithoutCancel(ctx) + if stagerID != "" { + removeStager(cleanupCtx, dockerClient, stagerID) + } + removeCtx, cancel := context.WithTimeout(cleanupCtx, timeouts.DefaultDockerAPI) + defer cancel() + if _, removeErr := dockerClient.VolumeRemove(removeCtx, created.Volume.Name, client.VolumeRemoveOptions{Force: true}); removeErr != nil && !cerrdefs.IsNotFound(removeErr) { + slog.WarnContext(cleanupCtx, "failed to remove Rustic CA certificate volume", "volumeName", created.Volume.Name, "error", removeErr) + } + } + stager, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{ + Config: &container.Config{ + Image: DefaultImage, + Labels: volumes.Labels(), + }, + HostConfig: &container.HostConfig{ + Mounts: []mount.Mount{{Type: mount.TypeVolume, Source: created.Volume.Name, Target: trustBundleDir}}, + }, + }) + if err != nil { + cleanup() + return "", nil, err + } + stagerID = stager.ID + copyCtx, copyCancel := context.WithTimeout(ctx, timeouts.DefaultDockerAPI) + err = docker.CopyToContainer(copyCtx, dockerClient, stager.ID, trustBundleDir, map[string][]byte{trustBundleName: bundle}) + copyCancel() + if err != nil { + cleanup() + return "", nil, err + } + return created.Volume.Name, cleanup, nil +} + +func removeStager(ctx context.Context, dockerClient *client.Client, containerID string) { + for attempt := 1; ; attempt++ { + attemptCtx, cancel := context.WithTimeout(ctx, timeouts.DefaultDockerAPI) + _, err := dockerClient.ContainerRemove(attemptCtx, containerID, volumes.RemoveOptions()) + cancel() + if err == nil || cerrdefs.IsNotFound(err) { + return + } + if attempt == stagerRemoveRetries { + slog.WarnContext(ctx, "failed to remove Rustic CA certificate stager", "containerId", containerID, "error", err) + return + } + time.Sleep(5 * time.Second) + } +}