From 398763b6fcc269dab80af5cbfce0a5aff50d0836 Mon Sep 17 00:00:00 2001 From: Andres Contreras Ballesteros Date: Fri, 9 Oct 2026 10:51:46 -0400 Subject: [PATCH 1/2] test(storage): add renewable acceptance harness --- .github/workflows/ci.yml | 8 + .gitignore | 1 + CONTRIBUTING.md | 2 + Makefile | 4 + README.md | 1 + command/app.go | 5 + doc/storage-acceptance.md | 56 ++++++ e2e/acceptance_cleanup_test.go | 101 ++++++++++ e2e/acceptance_test.go | 306 +++++++++++++++++++++++++++++++ e2e/credentials_test.go | 77 ++++++++ e2e/live_config.go | 44 +++++ e2e/live_config_test.go | 100 ++++++++++ e2e/main_test.go | 33 +++- e2e/util_test.go | 38 +++- scripts/oidc_credentials.py | 91 +++++++++ scripts/test_oidc_credentials.py | 82 +++++++++ storage/s3.go | 11 +- storage/s3_test.go | 8 + storage/storage.go | 2 + 19 files changed, 964 insertions(+), 6 deletions(-) create mode 100644 doc/storage-acceptance.md create mode 100644 e2e/acceptance_cleanup_test.go create mode 100644 e2e/acceptance_test.go create mode 100644 e2e/live_config.go create mode 100644 e2e/live_config_test.go create mode 100644 scripts/oidc_credentials.py create mode 100644 scripts/test_oidc_credentials.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aefdd5ccf..51939fd3b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,3 +95,11 @@ jobs: - run: make bootstrap - run: make check + + credential-helper: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - run: python3 -m unittest discover -s scripts -p 'test_*.py' diff --git a/.gitignore b/.gitignore index 6ff61b596..e25e2ff72 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ .DS_Store .env +__pycache__/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index eb367f9c8..d8b957e2e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,6 +38,8 @@ We strongly encourage you to write tests for your proposed changes. You can run make test ``` +For the bounded live CLI suite, see [storage acceptance setup](doc/storage-acceptance.md). Live tests require explicit configuration and use renewable profile credentials; ordinary test runs use fake storage. + ### Running static code analysis tools Here are the list of tools that are used to check the sanity of the code at compile time: diff --git a/Makefile b/Makefile index cd5ebae82..7e550c168 100644 --- a/Makefile +++ b/Makefile @@ -34,6 +34,10 @@ test_with_race: test_without_race: @S5CMD_BUILD_BINARY_WITHOUT_RACE_FLAG=1 go test -mod=readonly -count=1 ./... +.PHONY: acceptance +acceptance: + @S5CMD_TEST_MODE=live go test -mod=readonly ./e2e -run '^TestAcceptanceStorage$$' -count=1 -v -timeout=20m + ##@ Bootstrap # See following issues for why errors are ignored with `-e` flag: # * https://github.com/golang/go/issues/61857 diff --git a/README.md b/README.md index 6ff780aa0..c795ff949 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ storage services and local filesystems. - [S3 Transfer Acceleration](https://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html) support - Google Cloud Storage (and any other S3 API compatible service) support - Structured logging for querying command outputs +- Virtual-host addressing for custom S3 endpoints with `--use-virtual-host-style` - Shell auto-completion - S3 ListObjects API backward compatibility diff --git a/command/app.go b/command/app.go index 8c49cf4b3..5f5b6f8d2 100644 --- a/command/app.go +++ b/command/app.go @@ -78,6 +78,10 @@ var app = &cli.App{ Name: "use-list-objects-v1", Usage: "use ListObjectsV1 API for services that don't support ListObjectsV2", }, + &cli.BoolFlag{ + Name: "use-virtual-host-style", + Usage: "use bucket subdomains for a custom S3 endpoint", + }, &cli.StringFlag{ Name: "request-payer", Usage: "who pays for request (access requester pays buckets)", @@ -186,6 +190,7 @@ func NewStorageOpts(c *cli.Context) storage.Options { NoVerifySSL: c.Bool("no-verify-ssl"), RequestPayer: c.String("request-payer"), UseListObjectsV1: c.Bool("use-list-objects-v1"), + UseVirtualHostStyle: c.Bool("use-virtual-host-style"), Profile: c.String("profile"), CredentialFile: c.String("credentials-file"), LogLevel: log.LevelFromString(c.String("log")), diff --git a/doc/storage-acceptance.md b/doc/storage-acceptance.md new file mode 100644 index 000000000..a0a63ef6f --- /dev/null +++ b/doc/storage-acceptance.md @@ -0,0 +1,56 @@ +# Storage acceptance tests + +`make acceptance` builds the current checkout and runs a bounded CLI suite against an explicitly configured storage service. It requires live configuration and never falls back to fake storage. Ordinary `make test` runs the fake-storage regressions without requesting live credentials. + +The suite creates one private bucket named `s5cmd-acceptance-`. It checks multipart upload, download integrity, metadata, server-side copy, `pipe`, `sync --delete`, ListObjects v1/v2 pagination across 1,001 objects, and wildcard deletion. It uses at most eight fixture workers, two transfer workers, a 15-minute suite deadline, and two-minute command deadlines. Cleanup has a separate three-minute deadline and removes unfinished multipart uploads, object versions, delete markers, and the bucket. Cleanup failures fail the suite. + +## Local configuration + +Use a dedicated test organization and a restricted AWS profile. Supply actual configuration privately; do not commit it or copy it into PR descriptions. + +| Variable | Meaning | +| --- | --- | +| `S5CMD_TEST_MODE` | `live` | +| `S5CMD_TEST_ENDPOINT_URL` | HTTPS storage origin | +| `S5CMD_REGION` | Storage region or availability zone | +| `S5CMD_IS_VIRTUAL_HOST` | `true` | +| `S5CMD_I_KNOW_WHAT_IM_DOING` | `1` to authorize disposable resources | +| `AWS_PROFILE` | Named profile used by fixtures and CLI commands | +| `AWS_CONFIG_FILE` | Private AWS configuration file | +| `AWS_SHARED_CREDENTIALS_FILE` | Private credentials file, empty for process credentials | +| `S5CMD_TEST_RESOURCE_FILE` | Optional new private file recording the exact bucket for recovery | + +Unset static AWS credential variables and the legacy `S5CMD_ACCESS_KEY_ID` / `S5CMD_SECRET_ACCESS_KEY` variables. Use a renewable `credential_process` profile; local bootstrap credentials remain local. The fixture client uses the same protected process executor and credential-provider chain as the CLI. Storage TLS verification remains enabled. The suite passes region and endpoint explicitly and uses `--use-virtual-host-style` for custom endpoints. + +Run: + +```sh +make acceptance +``` + +The exact test selector is enforced in live mode, so a typo or a subset cannot produce a successful empty live run. Use fresh resource-ledger paths on reruns. The suite reports stage names and selected public error codes; it suppresses command output and raw credential/service diagnostics. Do not enable shell tracing, print environment variables, upload credential caches, or publish resource ledgers. + +For a local MinIO rehearsal, retain the existing external-test configuration (`S5CMD_TEST_ENDPOINT_URL`, `S5CMD_REGION`, `S5CMD_IS_VIRTUAL_HOST=false`, both legacy test keys, and explicit consent) and run `go test ./e2e -run '^TestAcceptanceStorage$' -count=1 -timeout=20m` without selecting live mode. This uses synthetic local credentials and path-style addressing. + +## GitHub OIDC process helper + +`scripts/oidc_credentials.py` is a fallback for a protected GitHub Actions job when a reviewed renewable login action is unavailable. Install a pinned CWIC version supporting `cwic auth accesskey oidc`, then set `CWIC_API_URL`, `CWIC_CACHE_DIR`, and `S5CMD_TEST_ORG` from protected configuration. Use a fresh private cache directory and private AWS config/credentials files. Configure the selected profile with: + +```ini +[profile acceptance] +credential_process = python3 /absolute/path/to/scripts/oidc_credentials.py +``` + +GitHub supplies `ACTIONS_ID_TOKEN_REQUEST_URL` and `ACTIONS_ID_TOKEN_REQUEST_TOKEN` to the job. CWIC calls the helper's token mode on credential exchange and renewal; each token invocation requests a fresh GitHub JWT with audience `https://coreweave.com/iam`. JWTs go only to CWIC; storage credentials go only to the SDK. The helper suppresses CWIC stderr and returns a fixed error on failed login or renewal. It has no API-token fallback. + +Run helper regressions with `python3 -m unittest discover -s scripts -p 'test_*.py'`. Normal PR CI runs these synthetic tests without OIDC permission or live configuration. + +Storage federation must trust the actual subject issued for this repository's protected environment, including any subject customization. Grant the credential-exchange permission separately from storage permissions on the acceptance bucket prefix. Configure environment reviewers and release-PR verification before granting the execution job `id-token: write`; this harness PR does not enable privileged live workflows or publication. GitHub App authentication for release-bot event delivery is separate from storage federation. + +## Cleanup and diagnosis + +On ordinary failure, deadline expiry, or a termination signal, the live runner cancels commands and attempts scoped cleanup before exiting. Forced termination or loss of the runner can interrupt cleanup. Keep the private resource ledger available to the trusted cleanup step; recover only its exact bucket with fresh short-lived credentials. Never sweep all buckets or infer ownership from an account inventory. A ledger remains if cleanup fails and is removed on success. + +A failed stage, authentication error, or cleanup error blocks the acceptance result. Check private endpoint/region configuration and WIF permissions first; inspect protected service-side diagnostics if needed. Do not expose JWTs, keys, signed requests, or raw HTTP responses in public Actions logs. + +Credential renewal, failed renewal, provider precedence, helper sanitization, and retry behavior have synthetic regressions in `storage` and `e2e`; use those to inject failures without intentionally disrupting a live service. Local validation does not verify GitHub-issued subjects or protected-environment enforcement. Release workflow activation and its required gate must validate those separately. diff --git a/e2e/acceptance_cleanup_test.go b/e2e/acceptance_cleanup_test.go new file mode 100644 index 000000000..cfa7d0e0f --- /dev/null +++ b/e2e/acceptance_cleanup_test.go @@ -0,0 +1,101 @@ +package e2e + +import ( + "context" + "errors" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3iface" +) + +type cleanupFixture struct { + s3iface.S3API + fail string + bucket string + aborted, deleted, batches int + removed bool +} + +func (f *cleanupFixture) check(bucket *string, stage string) error { + if aws.StringValue(bucket) != f.bucket { + return errors.New("cleanup escaped its bucket") + } + if f.fail == stage { + return errors.New("synthetic-private-value") + } + return nil +} +func (f *cleanupFixture) HeadBucketWithContext(_ aws.Context, in *s3.HeadBucketInput, _ ...request.Option) (*s3.HeadBucketOutput, error) { + if f.fail == "absent" { + return nil, awserr.NewRequestFailure(awserr.New("NoSuchBucket", "missing", nil), 404, "synthetic") + } + return &s3.HeadBucketOutput{}, f.check(in.Bucket, "head") +} +func (f *cleanupFixture) ListMultipartUploadsPagesWithContext(_ aws.Context, in *s3.ListMultipartUploadsInput, page func(*s3.ListMultipartUploadsOutput, bool) bool, _ ...request.Option) error { + if err := f.check(in.Bucket, "uploads"); err != nil { + return err + } + page(&s3.ListMultipartUploadsOutput{Uploads: []*s3.MultipartUpload{{Key: aws.String("unfinished"), UploadId: aws.String("upload")}}}, true) + return nil +} +func (f *cleanupFixture) AbortMultipartUploadWithContext(_ aws.Context, in *s3.AbortMultipartUploadInput, _ ...request.Option) (*s3.AbortMultipartUploadOutput, error) { + f.aborted++ + return &s3.AbortMultipartUploadOutput{}, f.check(in.Bucket, "abort") +} +func (f *cleanupFixture) ListObjectVersionsPagesWithContext(_ aws.Context, in *s3.ListObjectVersionsInput, page func(*s3.ListObjectVersionsOutput, bool) bool, _ ...request.Option) error { + if err := f.check(in.Bucket, "versions"); err != nil { + return err + } + output := &s3.ListObjectVersionsOutput{} + for n := 0; n < 1001; n++ { + output.Versions = append(output.Versions, &s3.ObjectVersion{Key: aws.String("object"), VersionId: aws.String("version")}) + } + page(output, false) + page(&s3.ListObjectVersionsOutput{DeleteMarkers: []*s3.DeleteMarkerEntry{{Key: aws.String("deleted"), VersionId: aws.String("marker")}}}, true) + return nil +} +func (f *cleanupFixture) DeleteObjectsWithContext(_ aws.Context, in *s3.DeleteObjectsInput, _ ...request.Option) (*s3.DeleteObjectsOutput, error) { + f.batches++ + f.deleted += len(in.Delete.Objects) + if f.fail == "partial delete" { + return &s3.DeleteObjectsOutput{Errors: []*s3.Error{{Code: aws.String("AccessDenied")}}}, nil + } + return &s3.DeleteObjectsOutput{}, f.check(in.Bucket, "delete") +} +func (f *cleanupFixture) DeleteBucketWithContext(_ aws.Context, in *s3.DeleteBucketInput, _ ...request.Option) (*s3.DeleteBucketOutput, error) { + if err := f.check(in.Bucket, "bucket"); err != nil { + return nil, err + } + f.removed = true + return &s3.DeleteBucketOutput{}, nil +} +func TestAcceptanceCleanup(t *testing.T) { + const bucket = "s5cmd-acceptance-0123456789abcdef01234567" + for _, stage := range []string{"success", "absent", "head", "uploads", "abort", "versions", "delete", "partial delete", "bucket"} { + t.Run(stage, func(t *testing.T) { + fixture := &cleanupFixture{fail: stage, bucket: bucket} + err := cleanAcceptanceBucket(context.Background(), fixture, bucket) + if stage == "success" { + if err != nil || !fixture.removed || fixture.aborted != 1 || fixture.deleted != 1002 || fixture.batches != 2 { + t.Fatal("cleanup omitted resources or exceeded batch size") + } + } else if stage == "absent" { + if err != nil || fixture.removed { + t.Fatal("absent bucket cleanup changed") + } + } else if err == nil { + t.Fatal("cleanup failure must fail the suite") + } + if err != nil && err.Error() == "synthetic-private-value" { + t.Fatal("cleanup exposed service diagnostics") + } + }) + } + if err := cleanAcceptanceBucket(context.Background(), &cleanupFixture{}, "another-run"); err == nil { + t.Fatal("invalid cleanup scope accepted") + } +} diff --git a/e2e/acceptance_test.go b/e2e/acceptance_test.go new file mode 100644 index 000000000..738f67e12 --- /dev/null +++ b/e2e/acceptance_test.go @@ -0,0 +1,306 @@ +package e2e + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3iface" +) + +func isMissingBucket(err error) bool { + var failure awserr.RequestFailure + return errors.As(err, &failure) && failure.StatusCode() == 404 +} + +// TestAcceptanceStorage is the complete live suite. It uses one disposable +// bucket and never lists or modifies resources belonging to another run. +func TestAcceptanceStorage(t *testing.T) { + if !isEndpointFromEnv() { + t.Skip("external storage is not selected") + } + ctx, cancel := context.WithTimeout(acceptanceContext, 15*time.Minute) + defer cancel() + client, command := setup(t) + if os.Getenv("S5CMD_TEST_MODE") == "live" { + if _, err := client.Config.Credentials.GetWithContext(ctx); err != nil { + t.Fatal("live fixture credential preflight failed") + } + expiration, err := client.Config.Credentials.ExpiresAt() + if err != nil || !expiration.After(time.Now()) { + t.Fatal("live credentials must have a future expiration") + } + } + var suffix [12]byte + if _, err := rand.Read(suffix[:]); err != nil { + t.Fatal("resource identifier generation failed") + } + bucket := "s5cmd-acceptance-" + hex.EncodeToString(suffix[:]) + uri := "s3://" + bucket + "/" + ledger := os.Getenv("S5CMD_TEST_RESOURCE_FILE") + if ledger != "" { + file, err := os.OpenFile(ledger, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + t.Fatal("resource ledger creation failed") + } + _, writeErr := io.WriteString(file, bucket+"\n") + closeErr := file.Close() + if writeErr != nil || closeErr != nil { + t.Fatal("resource ledger write failed") + } + } + // Register cleanup before creating the bucket: a lost response can hide a + // successful creation. Cleanup uses its own deadline after suite cancellation. + t.Cleanup(func() { + cleanupCtx, stop := context.WithTimeout(context.Background(), 3*time.Minute) + defer stop() + if err := cleanAcceptanceBucket(cleanupCtx, client, bucket); err != nil { + t.Error("acceptance resource cleanup failed") + } else if ledger != "" { + if err := os.Remove(ledger); err != nil { + t.Error("resource ledger cleanup failed") + } + } + }) + input := &s3.CreateBucketInput{Bucket: aws.String(bucket)} + region := aws.StringValue(client.Config.Region) + if region != "us-east-1" { + input.CreateBucketConfiguration = &s3.CreateBucketConfiguration{LocationConstraint: aws.String(region)} + } + if _, err := client.CreateBucketWithContext(ctx, input); err != nil { + var failure awserr.RequestFailure + if errors.As(err, &failure) { + t.Fatalf("acceptance bucket creation failed (HTTP %d, %s)", failure.StatusCode(), safeStorageErrorCode(failure.Code())) + } + t.Fatal("acceptance bucket creation failed") + } + readyCtx, readyCancel := context.WithTimeout(ctx, 3*time.Minute) + defer readyCancel() + for { + _, err := client.HeadBucketWithContext(readyCtx, &s3.HeadBucketInput{Bucket: aws.String(bucket)}) + if err == nil { + break + } + select { + case <-readyCtx.Done(): + t.Fatal("acceptance bucket readiness failed") + case <-time.After(2 * time.Second): + } + } + t.Log("disposable bucket ready") + dir := t.TempDir() + run := func(stage string, stdin io.Reader, args ...string) []byte { + t.Helper() + spec := command(args...) + cmdCtx, stop := context.WithTimeout(ctx, 2*time.Minute) + defer stop() + cmd := exec.CommandContext(cmdCtx, spec.Command[0], spec.Command[1:]...) + cmd.Env, cmd.Dir, cmd.Stdin = spec.Env, spec.Dir, stdin + var out bytes.Buffer + cmd.Stdout = &out + // Endpoint URLs, service error bodies and signed headers must not become + // CI diagnostics. A stage name is enough to identify the failing operation. + cmd.Stderr = io.Discard + if err := cmd.Run(); err != nil { + t.Fatalf("acceptance stage failed: %s", stage) + } + return out.Bytes() + } + payload := bytes.Repeat([]byte("acceptance-payload\n"), 1024*1024) + source := filepath.Join(dir, "source") + if err := os.WriteFile(source, payload, 0600); err != nil { + t.Fatal("payload creation failed") + } + run("multipart upload", nil, "cp", "--part-size", "5", "--concurrency", "2", "--metadata", "acceptance=verified", source, uri+"source") + head, err := client.HeadObjectWithContext(ctx, &s3.HeadObjectInput{Bucket: aws.String(bucket), Key: aws.String("source")}) + metadataOK := false + if err == nil { + for key, value := range head.Metadata { + if strings.EqualFold(key, "acceptance") && aws.StringValue(value) == "verified" { + metadataOK = true + } + } + } + if !metadataOK { + t.Fatal("object metadata validation failed") + } + download := filepath.Join(dir, "download") + run("download", nil, "cp", uri+"source", download) + got, err := os.ReadFile(download) + if err != nil || !bytes.Equal(got, payload) { + t.Fatal("download integrity validation failed") + } + run("server-side copy", nil, "cp", "--part-size", "5", "--concurrency", "2", uri+"source", uri+"copy") + // cat validates the server-side copy without writing object contents to logs. + if !bytes.Equal(run("copied content", nil, "cat", uri+"copy"), payload) { + t.Fatal("copy integrity validation failed") + } + pipe := []byte("synthetic pipe content\n") + run("pipe upload", bytes.NewReader(pipe), "pipe", uri+"pipe") + if !bytes.Equal(run("pipe content", nil, "cat", uri+"pipe"), pipe) { + t.Fatal("pipe integrity validation failed") + } + syncDir := filepath.Join(dir, "sync") + if err := os.Mkdir(syncDir, 0700); err != nil { + t.Fatal("sync fixture creation failed") + } + if err := os.WriteFile(filepath.Join(syncDir, "present"), pipe, 0600); err != nil { + t.Fatal("sync fixture creation failed") + } + putAcceptanceObject(t, ctx, client, bucket, "sync/stale", pipe) + run("sync and delete", nil, "sync", "--delete", syncDir+string(os.PathSeparator), uri+"sync/") + listed, err := client.ListObjectsV2WithContext(ctx, &s3.ListObjectsV2Input{Bucket: aws.String(bucket), Prefix: aws.String("sync/")}) + if err != nil || len(listed.Contents) != 1 || aws.StringValue(listed.Contents[0].Key) != "sync/present" { + t.Fatal("sync/delete validation failed") + } + // Exceed the service's default 1,000-object page size, using bounded workers + // and one-byte objects rather than large uploads. + const count = 1001 + var wg sync.WaitGroup + tasks := make(chan int) + failures := make(chan struct{}, count) + for worker := 0; worker < 8; worker++ { + wg.Add(1) + go func() { + defer wg.Done() + for n := range tasks { + _, err := client.PutObjectWithContext(ctx, &s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String(fmt.Sprintf("pages/%04d", n)), Body: bytes.NewReader([]byte("x"))}) + if err != nil { + failures <- struct{}{} + } + } + }() + } + for n := 0; n < count; n++ { + tasks <- n + } + close(tasks) + wg.Wait() + if len(failures) != 0 { + t.Fatal("pagination fixture creation failed") + } + for _, variant := range []string{"v2", "v1"} { + args := []string{"ls", uri + "pages/*"} + if variant == "v1" { + args = append([]string{"--use-list-objects-v1"}, args...) + } + output := run("paginated listing "+variant, nil, args...) + if len(strings.Split(strings.TrimSpace(string(output)), "\n")) != count { + t.Fatal("paginated CLI listing lost objects") + } + } + run("wildcard deletion", nil, "rm", uri+"pages/*") + remaining, err := client.ListObjectsV2WithContext(ctx, &s3.ListObjectsV2Input{Bucket: aws.String(bucket), Prefix: aws.String("pages/")}) + if err != nil || len(remaining.Contents) != 0 { + t.Fatal("wildcard deletion validation failed") + } + // The application must stop promptly when its parent cancels a command. + cancelCtx, stop := context.WithCancel(ctx) + spec := command("run") + blocked := exec.CommandContext(cancelCtx, spec.Command[0], spec.Command[1:]...) + blocked.Env = spec.Env + stdin, err := blocked.StdinPipe() + if err != nil { + t.Fatal("cancellation fixture failed") + } + defer stdin.Close() + if err := blocked.Start(); err != nil { + t.Fatal("cancellation command failed to start") + } + stop() + if err := blocked.Wait(); err == nil { + t.Fatal("cancelled command succeeded") + } + t.Log("upload, integrity, metadata, copy, pipe, sync/delete, pagination and cancellation passed") + acceptanceCompleted = true +} + +func safeStorageErrorCode(code string) string { + switch code { + case "AccessDenied", "Forbidden", "InvalidAccessKeyId", "InvalidToken", "ExpiredToken", "SignatureDoesNotMatch", "InvalidRegion", "InvalidRequest", "InvalidArgument", "AuthorizationHeaderMalformed", "InvalidLocationConstraint", "NoSuchBucket", "NotImplemented": + return code + default: + return "storage request failed" + } +} + +func putAcceptanceObject(t *testing.T, ctx context.Context, client *s3.S3, bucket, key string, body []byte) { + t.Helper() + _, err := client.PutObjectWithContext(ctx, &s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String(key), Body: bytes.NewReader(body)}) + if err != nil { + t.Fatal("acceptance fixture upload failed") + } +} + +// cleanAcceptanceBucket removes multipart uploads, versions, and objects only +// inside the exact bucket allocated by this run. No account-wide sweep occurs. +func cleanAcceptanceBucket(ctx context.Context, client s3iface.S3API, bucket string) error { + if !strings.HasPrefix(bucket, "s5cmd-acceptance-") || len(bucket) != len("s5cmd-acceptance-")+24 { + return fmt.Errorf("invalid cleanup scope") + } + if _, err := hex.DecodeString(strings.TrimPrefix(bucket, "s5cmd-acceptance-")); err != nil { + return fmt.Errorf("invalid cleanup scope") + } + _, err := client.HeadBucketWithContext(ctx, &s3.HeadBucketInput{Bucket: aws.String(bucket)}) + if isMissingBucket(err) { + return nil + } + if err != nil { + return fmt.Errorf("cleanup bucket lookup failed") + } + var uploads []*s3.MultipartUpload + err = client.ListMultipartUploadsPagesWithContext(ctx, &s3.ListMultipartUploadsInput{Bucket: aws.String(bucket)}, func(page *s3.ListMultipartUploadsOutput, _ bool) bool { + uploads = append(uploads, page.Uploads...) + return len(uploads) <= 100 + }) + if err != nil || len(uploads) > 100 { + return fmt.Errorf("cleanup upload enumeration failed") + } + for _, upload := range uploads { + if _, err := client.AbortMultipartUploadWithContext(ctx, &s3.AbortMultipartUploadInput{Bucket: aws.String(bucket), Key: upload.Key, UploadId: upload.UploadId}); err != nil { + return fmt.Errorf("cleanup multipart abort failed") + } + } + var objects []*s3.ObjectIdentifier + err = client.ListObjectVersionsPagesWithContext(ctx, &s3.ListObjectVersionsInput{Bucket: aws.String(bucket)}, func(page *s3.ListObjectVersionsOutput, _ bool) bool { + for _, version := range page.Versions { + objects = append(objects, &s3.ObjectIdentifier{Key: version.Key, VersionId: version.VersionId}) + } + for _, marker := range page.DeleteMarkers { + objects = append(objects, &s3.ObjectIdentifier{Key: marker.Key, VersionId: marker.VersionId}) + } + return len(objects) <= 3000 + }) + if err != nil || len(objects) > 3000 { + return fmt.Errorf("cleanup version enumeration failed") + } + for len(objects) > 0 { + n := len(objects) + if n > 1000 { + n = 1000 + } + out, err := client.DeleteObjectsWithContext(ctx, &s3.DeleteObjectsInput{Bucket: aws.String(bucket), Delete: &s3.Delete{Objects: objects[:n], Quiet: aws.Bool(true)}}) + if err != nil || len(out.Errors) > 0 { + return fmt.Errorf("cleanup object deletion failed") + } + objects = objects[n:] + } + if _, err := client.DeleteBucketWithContext(ctx, &s3.DeleteBucketInput{Bucket: aws.String(bucket)}); err != nil { + return fmt.Errorf("cleanup bucket deletion failed") + } + return nil +} diff --git a/e2e/credentials_test.go b/e2e/credentials_test.go index 2bb72e807..b3c154d52 100644 --- a/e2e/credentials_test.go +++ b/e2e/credentials_test.go @@ -15,6 +15,7 @@ import ( "testing" "time" + "github.com/peak/s5cmd/v2/storage" "gotest.tools/v3/assert" ) @@ -59,6 +60,32 @@ func TestCredentialProcessCLI(t *testing.T) { output, err := build.CombinedOutput() assert.NilError(t, err, "build credential process: %s", output) + t.Run("explicit virtual-host style for custom endpoints", func(t *testing.T) { + for _, force := range []bool{false, true} { + t.Run(fmt.Sprint(force), func(t *testing.T) { + hosts := make(chan string, 1) + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hosts <- r.Host + _, _ = io.WriteString(w, `testbucketfalse`) + })) + defer proxy.Close() + args := []string{"ls", "s3://testbucket/"} + if force { + args = append([]string{"--use-virtual-host-style"}, args...) + } + cmd, _ := credentialProcessCommand(t, helper, "success", "http://objects.example.invalid", args...) + cmd.Env = append(cmd.Env, "HTTP_PROXY="+proxy.URL, "NO_PROXY=") + _, err := cmd.CombinedOutput() + assert.NilError(t, err) + want := "objects.example.invalid" + if force { + want = "testbucket." + want + } + assert.Equal(t, <-hosts, want) + }) + } + }) + t.Run("helper cannot consume command input", func(t *testing.T) { requests := make(chan struct{}, 1) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -199,3 +226,53 @@ func TestCredentialProcessCLI(t *testing.T) { } }) } + +func TestLiveFixtureRenewableCredentials(t *testing.T) { + dir := t.TempDir() + helper := filepath.Join(dir, "credential-process.exe") + build := exec.Command("go", "build", "-o", helper, "../internal/testdata/credential-process/main.go") + output, err := build.CombinedOutput() + assert.NilError(t, err, "build credential process: %s", output) + for _, mode := range []string{"refresh", "renew-malformed"} { + t.Run(mode, func(t *testing.T) { + state := filepath.Join(t.TempDir(), "state") + config := filepath.Join(t.TempDir(), "config") + command := fmt.Sprintf("exec %q %s %q", helper, mode, state) + if runtime.GOOS == "windows" { + command = fmt.Sprintf("call \"%s\" %s \"%s\"", helper, mode, state) + } + assert.NilError(t, os.WriteFile(config, []byte("[profile acceptance]\ncredential_process = "+command+"\n"), 0600)) + credentials := filepath.Join(t.TempDir(), "credentials") + assert.NilError(t, os.WriteFile(credentials, nil, 0600)) + for _, value := range os.Environ() { + key := strings.SplitN(value, "=", 2)[0] + if strings.HasPrefix(key, "AWS_") { + t.Setenv(key, "") + } + } + for key, value := range map[string]string{ + "S5CMD_TEST_MODE": "live", "S5CMD_TEST_ENDPOINT_URL": "https://storage.example.invalid", "S5CMD_REGION": "us-east-1", + "S5CMD_IS_VIRTUAL_HOST": "true", "S5CMD_I_KNOW_WHAT_IM_DOING": "1", "S5CMD_ACCESS_KEY_ID": "", "S5CMD_SECRET_ACCESS_KEY": "", + "AWS_PROFILE": "acceptance", "AWS_CONFIG_FILE": config, "AWS_SHARED_CREDENTIALS_FILE": credentials, + } { + t.Setenv(key, value) + } + client := s3client(t, storage.Options{}, nil) + first, err := client.Config.Credentials.Get() + assert.NilError(t, err) + assert.Equal(t, first.ProviderName, "ProcessProvider") + time.Sleep(1100 * time.Millisecond) + second, err := client.Config.Credentials.Get() + if mode == "refresh" { + assert.NilError(t, err) + assert.Assert(t, first.AccessKeyID != second.AccessKeyID) + } else { + assert.Assert(t, err != nil) + assert.Assert(t, !strings.Contains(err.Error(), "synthetic-private-")) + } + data, err := os.ReadFile(state) + assert.NilError(t, err) + assert.Equal(t, string(data), "2") + }) + } +} diff --git a/e2e/live_config.go b/e2e/live_config.go new file mode 100644 index 000000000..6dc12414c --- /dev/null +++ b/e2e/live_config.go @@ -0,0 +1,44 @@ +package e2e + +import ( + "errors" + "net/url" + "strings" +) + +type liveConfig struct{ endpoint, region, profile string } + +// A partial live configuration must never select the local fake server. +func readLiveConfig(getenv func(string) string) (*liveConfig, error) { + mode := getenv("S5CMD_TEST_MODE") + if mode == "" || mode == "fake" { + return nil, nil + } + if mode != "live" { + return nil, errors.New("unsupported S5CMD_TEST_MODE") + } + for _, key := range []string{"S5CMD_TEST_ENDPOINT_URL", "S5CMD_REGION", "AWS_PROFILE"} { + if strings.TrimSpace(getenv(key)) == "" { + return nil, errors.New("missing required live configuration: " + key) + } + } + if getenv("S5CMD_I_KNOW_WHAT_IM_DOING") != "1" { + return nil, errors.New("live tests require explicit consent") + } + if getenv("S5CMD_IS_VIRTUAL_HOST") != "true" { + return nil, errors.New("live tests require virtual-host addressing") + } + u, err := url.Parse(getenv("S5CMD_TEST_ENDPOINT_URL")) + if err != nil || u.Scheme != "https" || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Path != "" && u.Path != "/") { + return nil, errors.New("live endpoint must be an HTTPS origin without credentials") + } + for _, key := range []string{"AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY", "AWS_SECRET_ACCESS_KEY", "AWS_SECRET_KEY", "AWS_SESSION_TOKEN", "S5CMD_ACCESS_KEY_ID", "S5CMD_SECRET_ACCESS_KEY"} { + if getenv(key) != "" { + return nil, errors.New("live tests require profile credentials without static environment credentials") + } + } + if value := strings.ToLower(getenv("AWS_SDK_LOAD_CONFIG")); value != "" && value != "1" && value != "true" { + return nil, errors.New("live tests require shared AWS configuration") + } + return &liveConfig{endpoint: u.String(), region: getenv("S5CMD_REGION"), profile: getenv("AWS_PROFILE")}, nil +} diff --git a/e2e/live_config_test.go b/e2e/live_config_test.go new file mode 100644 index 000000000..77dfabef0 --- /dev/null +++ b/e2e/live_config_test.go @@ -0,0 +1,100 @@ +package e2e + +import ( + "os" + "os/exec" + "strings" + "testing" +) + +func TestLiveConfiguration(t *testing.T) { + valid := map[string]string{ + "S5CMD_TEST_MODE": "live", "S5CMD_TEST_ENDPOINT_URL": "https://storage.example.invalid", + "S5CMD_REGION": "test-region", "S5CMD_IS_VIRTUAL_HOST": "true", "AWS_PROFILE": "acceptance", + "S5CMD_I_KNOW_WHAT_IM_DOING": "1", + } + for _, name := range []string{"valid", "missing endpoint", "missing region", "missing profile", "missing consent", "invalid addressing", "plain HTTP", "URL credentials", "URL query", "static fallback", "alternate static fallback", "unknown mode"} { + t.Run(name, func(t *testing.T) { + env := make(map[string]string) + for key, value := range valid { + env[key] = value + } + switch name { + case "missing endpoint": + delete(env, s5cmdTestEndpointEnv) + case "missing region": + delete(env, s5cmdTestRegionEnv) + case "missing profile": + delete(env, "AWS_PROFILE") + case "missing consent": + delete(env, s5cmdTestIKnowWhatImDoingEnv) + case "invalid addressing": + env[s5cmdTestIsVirtualHost] = "false" + case "plain HTTP": + env[s5cmdTestEndpointEnv] = "http://storage.example.invalid" + case "URL credentials": + env[s5cmdTestEndpointEnv] = "https://user:synthetic-private-value@storage.example.invalid" + case "URL query": + env[s5cmdTestEndpointEnv] += "?token=synthetic-private-value" + case "static fallback": + env["AWS_ACCESS_KEY_ID"] = "synthetic-private-value" + case "alternate static fallback": + env["AWS_ACCESS_KEY"] = "synthetic-private-value" + env["AWS_SECRET_KEY"] = "synthetic-private-value" + case "unknown mode": + env["S5CMD_TEST_MODE"] = "typo" + } + cfg, err := readLiveConfig(func(key string) string { return env[key] }) + if name == "valid" { + if err != nil || cfg == nil { + t.Fatal("valid live configuration rejected") + } + return + } + if err == nil { + t.Fatal("invalid live configuration accepted") + } + if strings.Contains(err.Error(), "synthetic-private-value") { + t.Fatal("configuration error disclosed a value") + } + }) + } + cfg, err := readLiveConfig(func(string) string { return "" }) + if err != nil || cfg != nil { + t.Fatal("default fake mode changed") + } +} + +func TestLiveStartupRejectsIncompleteRuns(t *testing.T) { + env := make([]string, 0, len(os.Environ())) + for _, value := range os.Environ() { + if !strings.HasPrefix(value, "AWS_") && !strings.HasPrefix(value, "S5CMD_") { + env = append(env, value) + } + } + for _, name := range []string{"missing config", "empty test selection", "skipped suite", "zero repetitions", "list tests"} { + t.Run(name, func(t *testing.T) { + args := []string{"-test.run=^TestDoesNotExist$"} + switch name { + case "skipped suite": + args = []string{"-test.run=^TestAcceptanceStorage$", "-test.skip=TestAcceptanceStorage"} + case "zero repetitions": + args = []string{"-test.run=^TestAcceptanceStorage$", "-test.count=0"} + case "list tests": + args = []string{"-test.run=^TestAcceptanceStorage$", "-test.list=."} + } + cmd := exec.Command(os.Args[0], args...) + cmd.Env = append(append([]string{}, env...), "S5CMD_TEST_MODE=live") + if name != "missing config" { + cmd.Env = append(cmd.Env, "S5CMD_TEST_ENDPOINT_URL=https://storage.example.invalid", "S5CMD_REGION=test-region", "S5CMD_IS_VIRTUAL_HOST=true", "S5CMD_I_KNOW_WHAT_IM_DOING=1", "AWS_PROFILE=acceptance") + } + output, err := cmd.CombinedOutput() + if err == nil { + t.Fatal("incomplete live run succeeded") + } + if !strings.Contains(string(output), "live") { + t.Fatal("live configuration diagnostic missing") + } + }) + } +} diff --git a/e2e/main_test.go b/e2e/main_test.go index e5e48d6b8..0741b2d88 100644 --- a/e2e/main_test.go +++ b/e2e/main_test.go @@ -1,16 +1,45 @@ package e2e import ( + "context" "flag" + "fmt" "os" + "os/signal" + "syscall" "testing" ) +var acceptanceContext = context.Background() +var acceptanceCompleted bool + func TestMain(m *testing.M) { + os.Exit(runTests(m)) +} + +func runTests(m *testing.M) int { flag.Parse() + cfg, err := readLiveConfig(os.Getenv) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + if cfg != nil && flag.Lookup("test.run").Value.String() != "^TestAcceptanceStorage$" { + fmt.Fprintln(os.Stderr, "live tests require the complete TestAcceptanceStorage suite") + return 1 + } + if cfg != nil { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + acceptanceContext = ctx + } cleanup := goBuildS5cmd() + defer cleanup() code := m.Run() - cleanup() - os.Exit(code) + if cfg != nil && code == 0 && !acceptanceCompleted { + fmt.Fprintln(os.Stderr, "live acceptance suite did not complete") + return 1 + } + return code } diff --git a/e2e/util_test.go b/e2e/util_test.go index a450e1cc5..efc0f8bf0 100644 --- a/e2e/util_test.go +++ b/e2e/util_test.go @@ -4,6 +4,7 @@ package e2e import ( "bytes" + "context" jsonpkg "encoding/json" "errors" "flag" @@ -23,6 +24,7 @@ import ( "testing" "time" + "github.com/peak/s5cmd/v2/log" "github.com/peak/s5cmd/v2/storage" "github.com/peak/s5cmd/v2/strutil" @@ -168,7 +170,7 @@ func setup(t *testing.T, options ...option) (*s3.S3, func(...string) icmd.Cmd) { region := "" if opts.region != "" { - region = opts.accessKeyID + region = opts.region } var cfg *credentialCfg @@ -183,7 +185,7 @@ func setup(t *testing.T, options ...option) (*s3.S3, func(...string) icmd.Cmd) { client := s3client(t, storage.Options{ Endpoint: endpoint, - NoVerifySSL: true, + NoVerifySSL: os.Getenv("S5CMD_TEST_MODE") != "live", }, cfg) return client, s5cmd(workdir, endpoint) @@ -221,6 +223,28 @@ func server(t *testing.T, testdir *fs.Dir, opts *setupOpts) string { func s3client(t *testing.T, options storage.Options, creds *credentialCfg) *s3.S3 { t.Helper() + if os.Getenv("S5CMD_TEST_MODE") == "live" { + cfg, err := readLiveConfig(os.Getenv) + if err != nil { + t.Fatal(err) + } + ctx, stop := storage.WithCredentialProcessContext(context.Background()) + t.Cleanup(stop) + options.Profile = cfg.profile + options.Endpoint = cfg.endpoint + options.NoVerifySSL = false + options.UseVirtualHostStyle = true + options.LogLevel = log.LevelError + options.MaxRetries = 3 + options.SetRegion(cfg.region) + sess, err := storage.NewSession(ctx, options) + if err != nil { + t.Fatal("live fixture session initialization failed") + } + fixture := sess.Copy(aws.NewConfig()) + fixture.Config.Retryer = newSlowDownRetryer(3) + return s3.New(fixture) + } awsLogLevel := aws.LogOff if *flagTestLogLevel == "debug" { @@ -305,6 +329,9 @@ func (c *slowDownRetryer) ShouldRetry(req *request.Request) bool { } func isEndpointFromEnv() bool { + if os.Getenv("S5CMD_TEST_MODE") == "live" { + return true + } return os.Getenv(s5cmdTestIDEnv) != "" && os.Getenv(s5cmdTestSecretEnv) != "" && os.Getenv(s5cmdTestEndpointEnv) != "" && @@ -346,6 +373,13 @@ func s5cmd(workdir, endpoint string) func(args ...string) icmd.Cmd { }..., ) } + if os.Getenv("S5CMD_TEST_MODE") == "live" { + cmd.Command = append(cmd.Command[:1], append([]string{"--use-virtual-host-style", "--profile", os.Getenv("AWS_PROFILE")}, cmd.Command[1:]...)...) + cmd.Env = append(env, "AWS_REGION="+os.Getenv(s5cmdTestRegionEnv)) + cmd.Dir = workdir + cmd.Timeout = 2 * time.Minute + return cmd + } env = append( env, diff --git a/scripts/oidc_credentials.py b/scripts/oidc_credentials.py new file mode 100644 index 000000000..c061387e5 --- /dev/null +++ b/scripts/oidc_credentials.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Renewable AWS credential_process for a protected GitHub Actions job.""" + +import datetime +import json +import os +from pathlib import Path +import subprocess +import sys +import urllib.parse +import urllib.request + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise ValueError('OIDC redirects are forbidden') + + +def oidc_token(): + url = urllib.parse.urlsplit(os.environ['ACTIONS_ID_TOKEN_REQUEST_URL']) + if (url.scheme != 'https' or url.username or url.password or url.fragment + or not url.hostname or not url.hostname.endswith('.actions.githubusercontent.com') + or url.port not in (None, 443)): + raise ValueError('invalid GitHub OIDC endpoint') + query = urllib.parse.parse_qsl(url.query, keep_blank_values=True) + query = [(key, value) for key, value in query if key != 'audience'] + query.append(('audience', 'https://coreweave.com/iam')) + target = urllib.parse.urlunsplit(url._replace(query=urllib.parse.urlencode(query))) + request = urllib.request.Request(target, headers={ + 'Authorization': 'Bearer ' + os.environ['ACTIONS_ID_TOKEN_REQUEST_TOKEN']}) + with urllib.request.build_opener(NoRedirect()).open(request, timeout=20) as response: + data = response.read(64 * 1024 + 1) + if len(data) > 64 * 1024: + raise ValueError('oversized OIDC response') + token = json.loads(data)['value'] + if not isinstance(token, str) or not token.strip(): + raise ValueError('empty OIDC token') + return token + + +def credentials(): + # Require explicit configuration; a wrong deployment must fail closed. + for key in ['CWIC_API_URL', 'CWIC_CACHE_DIR', 'S5CMD_TEST_ORG']: + if not os.environ.get(key): + raise ValueError('missing credential configuration') + endpoint = urllib.parse.urlsplit(os.environ['CWIC_API_URL']) + if endpoint.scheme != 'https' or not endpoint.hostname or endpoint.username or endpoint.password or endpoint.query or endpoint.fragment: + raise ValueError('invalid API endpoint') + cache = Path(os.environ['CWIC_CACHE_DIR']) + cache.mkdir(mode=0o700, parents=True, exist_ok=True) + if cache.is_symlink() or (cache.stat().st_mode & 0o077): + raise ValueError('credential cache must be private') + # CWIC caches storage credentials, then invokes --token again when they + # approach expiry. Each invocation obtains a fresh GitHub OIDC token. + cmd = ['cwic', 'auth', 'accesskey', 'oidc', '--storage', 'disk', + '--org-id', os.environ['S5CMD_TEST_ORG'], '--', + sys.executable, str(Path(__file__).resolve()), '--token'] + result = subprocess.run(cmd, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, timeout=50, check=True) + if len(result.stdout) > 8192: + raise ValueError('oversized credential response') + data = json.loads(result.stdout) + if data.get('Version') != 1 or not data.get('AccessKeyId') or not data.get('SecretAccessKey'): + raise ValueError('invalid credential response') + expiration = datetime.datetime.fromisoformat(data['Expiration'].replace('Z', '+00:00')) + if expiration <= datetime.datetime.now(datetime.timezone.utc): + raise ValueError('expired credential response') + return data + + +def main(args): + # Restrict all files created by the helper and CWIC in this process tree. + os.umask(0o077) + try: + if args == ['--token']: + value = {'apiVersion': 'client.authentication.k8s.io/v1', 'kind': 'ExecCredential', + 'status': {'token': oidc_token()}} + elif not args: + value = credentials() + else: + raise ValueError('unsupported arguments') + sys.stdout.write(json.dumps(value) + '\n') + return 0 + except Exception: + # Never print URL, HTTP body, JWT, CWIC output, or provider exceptions. + sys.stderr.write('temporary credential exchange failed\n') + return 1 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/scripts/test_oidc_credentials.py b/scripts/test_oidc_credentials.py new file mode 100644 index 000000000..212aa2e1e --- /dev/null +++ b/scripts/test_oidc_credentials.py @@ -0,0 +1,82 @@ +import contextlib +import datetime +import io +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +import oidc_credentials as helper + + +class OIDCCredentialsTests(unittest.TestCase): + def test_failed_exchange_is_sanitized(self): + stdout, stderr = io.StringIO(), io.StringIO() + with patch.object(helper, 'credentials', side_effect=RuntimeError('synthetic-private-jwt')), contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + self.assertEqual(helper.main([]), 1) + self.assertEqual(stdout.getvalue(), '') + self.assertNotIn('synthetic-private', stderr.getvalue()) + + def test_token_is_requested_on_every_call(self): + env = {'ACTIONS_ID_TOKEN_REQUEST_URL': 'https://tokens.actions.githubusercontent.com/id?api-version=2', + 'ACTIONS_ID_TOKEN_REQUEST_TOKEN': 'synthetic-request-token'} + class Response(io.BytesIO): + pass + class Opener: + def open(self, request, timeout): + self_request.append(request) + return Response(json.dumps({'value': 'synthetic-fresh-token'}).encode()) + self_request = [] + with patch.dict(os.environ, env, clear=True), patch.object(helper.urllib.request, 'build_opener', return_value=Opener()): + self.assertEqual(helper.oidc_token(), 'synthetic-fresh-token') + self.assertEqual(helper.oidc_token(), 'synthetic-fresh-token') + self.assertEqual(len(self_request), 2) + self.assertIn('audience=https%3A%2F%2Fcoreweave.com%2Fiam', self_request[0].full_url) + + def test_token_endpoint_rejects_unsafe_urls(self): + for url in ['http://tokens.actions.githubusercontent.com/id', 'https://example.invalid/id', + 'https://tokens.actions.githubusercontent.com@evil.invalid/id']: + with self.subTest(url=url), patch.dict(os.environ, {'ACTIONS_ID_TOKEN_REQUEST_URL': url, 'ACTIONS_ID_TOKEN_REQUEST_TOKEN': 'synthetic-token'}, clear=True): + with self.assertRaises(ValueError): + helper.oidc_token() + + def test_token_failures_never_echo_response(self): + with patch.object(helper, 'oidc_token', side_effect=RuntimeError('synthetic-private-jwt')): + stdout, stderr = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + self.assertEqual(helper.main(['--token']), 1) + self.assertEqual(stdout.getvalue(), '') + self.assertNotIn('synthetic-private', stderr.getvalue()) + + def test_cwic_receives_a_token_command_and_only_expiring_credentials_pass(self): + expiry = (datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=5)).isoformat() + valid = {'Version': 1, 'AccessKeyId': 'synthetic-key', 'SecretAccessKey': 'synthetic-secret', 'Expiration': expiry} + for invalid in [None, 'missing expiry', 'expired', 'bad version', 'oversized']: + with self.subTest(case=invalid), tempfile.TemporaryDirectory() as directory: + env = {'CWIC_API_URL': 'https://api.example.invalid', 'CWIC_CACHE_DIR': str(Path(directory) / 'cache'), 'S5CMD_TEST_ORG': 'synthetic-org'} + data = dict(valid) + if invalid == 'missing expiry': + del data['Expiration'] + elif invalid == 'expired': + data['Expiration'] = '2000-01-01T00:00:00Z' + elif invalid == 'bad version': + data['Version'] = 2 + stdout = json.dumps(data).encode() if invalid != 'oversized' else b'x' * 8193 + result = subprocess.CompletedProcess([], 0, stdout=stdout) + with patch.dict(os.environ, env, clear=True), patch.object(helper.subprocess, 'run', return_value=result) as call: + if invalid is None: + self.assertEqual(helper.credentials(), valid) + argv = call.call_args.args[0] + self.assertIn('--token', argv) + self.assertIn('oidc', argv) + self.assertEqual(call.call_args.kwargs['stderr'], subprocess.DEVNULL) + else: + with self.assertRaises((ValueError, KeyError)): + helper.credentials() + + +if __name__ == '__main__': + unittest.main() diff --git a/storage/s3.go b/storage/s3.go index 6570b9643..6820193cb 100644 --- a/storage/s3.go +++ b/storage/s3.go @@ -104,7 +104,7 @@ func newS3Storage(ctx context.Context, opts Options) (*S3, error) { return nil, err } - awsSession, err := globalSessionCache.newSession(ctx, opts) + awsSession, err := NewSession(ctx, opts) if err != nil { return nil, err } @@ -121,6 +121,13 @@ func newS3Storage(ctx context.Context, opts Options) (*S3, error) { }, nil } +// NewSession configures an S3 session with the same profile selection, renewable +// credentials, endpoint, TLS, and addressing rules used by storage operations. +// Callers using credential processes should scope ctx with WithCredentialProcessContext. +func NewSession(ctx context.Context, opts Options) (*session.Session, error) { + return globalSessionCache.newSession(ctx, opts) +} + // Stat retrieves metadata from S3 object without returning the object itself. func (s *S3) Stat(ctx context.Context, url *url.URL) (*Object, error) { input := &s3.HeadObjectInput{ @@ -1255,7 +1262,7 @@ func (sc *SessionCache) newSession(ctx context.Context, opts Options) (*session. // use virtual-host-style if the endpoint is known to support it, // otherwise use the path-style approach. - isVirtualHostStyle := isVirtualHostStyle(endpointURL) + isVirtualHostStyle := opts.UseVirtualHostStyle || isVirtualHostStyle(endpointURL) useAccelerate := supportsTransferAcceleration(endpointURL) // AWS SDK handles transfer acceleration automatically. Setting the diff --git a/storage/s3_test.go b/storage/s3_test.go index 2cc7577e6..5e1385d13 100644 --- a/storage/s3_test.go +++ b/storage/s3_test.go @@ -105,6 +105,14 @@ func TestNewSessionPathStyle(t *testing.T) { } } +func TestNewSessionExplicitVirtualHostStyle(t *testing.T) { + opts := Options{Endpoint: "https://storage.example.invalid", UseVirtualHostStyle: true} + opts.SetRegion("us-east-1") + sess, err := NewSession(context.Background(), opts) + assert.NilError(t, err) + assert.Assert(t, !aws.BoolValue(sess.Config.S3ForcePathStyle)) +} + func TestNewSessionWithRegionSetViaEnv(t *testing.T) { globalSessionCache.clear() diff --git a/storage/storage.go b/storage/storage.go index 1a6171263..ed3128dfa 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -65,6 +65,7 @@ func NewRemoteClient(ctx context.Context, url *url.URL, opts Options) (*S3, erro DryRun: opts.DryRun, NoSignRequest: opts.NoSignRequest, UseListObjectsV1: opts.UseListObjectsV1, + UseVirtualHostStyle: opts.UseVirtualHostStyle, RequestPayer: opts.RequestPayer, Profile: opts.Profile, CredentialFile: opts.CredentialFile, @@ -91,6 +92,7 @@ type Options struct { DryRun bool NoSignRequest bool UseListObjectsV1 bool + UseVirtualHostStyle bool LogLevel log.LogLevel RequestPayer string Profile string From 1862e4df21db58feb08da30f09f5c8c37771e5f1 Mon Sep 17 00:00:00 2001 From: Andres Contreras Ballesteros Date: Fri, 9 Oct 2026 11:29:12 -0400 Subject: [PATCH 2/2] test(storage): report bucket creation failures safely --- e2e/acceptance_cleanup_test.go | 13 +++++++++++++ e2e/acceptance_test.go | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/e2e/acceptance_cleanup_test.go b/e2e/acceptance_cleanup_test.go index cfa7d0e0f..a3b8d3f73 100644 --- a/e2e/acceptance_cleanup_test.go +++ b/e2e/acceptance_cleanup_test.go @@ -12,6 +12,19 @@ import ( "github.com/aws/aws-sdk-go/service/s3/s3iface" ) +func TestSafeStorageErrorCode(t *testing.T) { + for _, code := range []string{"TooManyBuckets", "InvalidRegion", "AccessDenied"} { + if got := safeStorageErrorCode(code); got != code { + t.Fatalf("public storage code %q was suppressed", code) + } + } + for _, code := range []string{"synthetic-private-value", "TooManyBuckets synthetic-private-value", ""} { + if got := safeStorageErrorCode(code); got != "storage request failed" { + t.Fatal("unknown storage code was disclosed") + } + } +} + type cleanupFixture struct { s3iface.S3API fail string diff --git a/e2e/acceptance_test.go b/e2e/acceptance_test.go index 738f67e12..04e53766c 100644 --- a/e2e/acceptance_test.go +++ b/e2e/acceptance_test.go @@ -231,7 +231,7 @@ func TestAcceptanceStorage(t *testing.T) { func safeStorageErrorCode(code string) string { switch code { - case "AccessDenied", "Forbidden", "InvalidAccessKeyId", "InvalidToken", "ExpiredToken", "SignatureDoesNotMatch", "InvalidRegion", "InvalidRequest", "InvalidArgument", "AuthorizationHeaderMalformed", "InvalidLocationConstraint", "NoSuchBucket", "NotImplemented": + case "AccessDenied", "Forbidden", "InvalidAccessKeyId", "InvalidToken", "ExpiredToken", "SignatureDoesNotMatch", "InvalidRegion", "InvalidRequest", "InvalidArgument", "AuthorizationHeaderMalformed", "InvalidLocationConstraint", "NoSuchBucket", "TooManyBuckets", "NotImplemented": return code default: return "storage request failed"