diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 818cdf5..313eb18 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,53 @@ jobs: - name: Test run: go test -v -race ./... + - name: Test TinyGo transport policy + run: go test -tags=tinygo -run TestTinyGo ./... + + - name: Build WebAssembly + run: GOOS=js GOARCH=wasm go build ./... + + - name: Build WASI + run: GOOS=wasip1 GOARCH=wasm go build ./... + + tinygo: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + + - name: Set up Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + package-manager-cache: false + + - name: Install TinyGo and Wasmtime + working-directory: ${{ runner.temp }} + run: | + curl --fail --location --silent --show-error --output tinygo.tar.gz https://github.com/tinygo-org/tinygo/releases/download/v0.42.0/tinygo0.42.0.linux-amd64.tar.gz + echo 'b87688fa2e19cee7d813cad7fd7dadb71dff3198e47125aba66ba4af5e490438 tinygo.tar.gz' | sha256sum --check + tar -xzf tinygo.tar.gz + echo "$RUNNER_TEMP/tinygo/bin" >> "$GITHUB_PATH" + curl --fail --location --silent --show-error --output wasmtime.tar.xz https://github.com/bytecodealliance/wasmtime/releases/download/v44.0.1/wasmtime-v44.0.1-x86_64-linux.tar.xz + echo 'afd58715f105e3a7f454169daed22168c5736ec5f225fb04c4ac62c54c9508a3 wasmtime.tar.xz' | sha256sum --check + tar -xJf wasmtime.tar.xz + echo "$RUNNER_TEMP/wasmtime-v44.0.1-x86_64-linux" >> "$GITHUB_PATH" + + - name: Test TinyGo WebAssembly transport policy and sync + run: tinygo test -target=wasm -run '^TestTinyGo' -v . + + - name: Test TinyGo WASI transport policy and sync + run: tinygo test -target=wasip1 -run '^TestTinyGo' -v . + lint: runs-on: ubuntu-latest steps: diff --git a/README.md b/README.md index 9cf1330..2169b17 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,8 @@ If an advertised chunk disappears, the client refreshes the properties once. `Ne The lower-level `NewReader` and `NewRawReader` functions parse saved chunks or caller-managed streams without HTTP. +These readers are available under TinyGo. TinyGo HTTP transports cannot enforce the connection-time public-address policy, so `Client.Sync` returns `ErrUnprotectedTransport` unless `AllowPrivateAddresses` is set. For synchronization, supply a host-compatible `HTTPClient` and enforce address and redirect restrictions in the host. + ## Command Build the command locally: @@ -96,7 +98,7 @@ The command writes newline-delimited JSON. A sync begins with its mode, followed Private, loopback, CGNAT, and NAT64 addresses are refused by default. The public-address policy bypasses environment proxies and rejects an explicit proxy, custom `RoundTripper`, or custom dialer because those paths cannot be checked at connection time. Set `AllowPrivateAddresses` when using one of those transports and enforce its address policy separately. The command exposes the same opt-out as `--allow-private`. -The default transport applies 30-second connection and response-header timeouts. Body reads also have a 30-second idle timeout, configurable with `ClientOptions.ResponseIdleTimeout`. Parser limits, retry counts, and locally generated backoff bounds are available through the same options type. A valid server `Retry-After` delay is honored without shortening it. +Under standard Go, the default transport applies 30-second connection and response-header timeouts. Body reads also have a 30-second idle timeout, configurable with `ClientOptions.ResponseIdleTimeout`. Parser limits, retry counts, and locally generated backoff bounds are available through the same options type. A valid server `Retry-After` delay is honored without shortening it. ## Use with other git-pkgs packages diff --git a/benchmark_test.go b/benchmark_test.go index dd2a388..4d4e166 100644 --- a/benchmark_test.go +++ b/benchmark_test.go @@ -1,3 +1,5 @@ +//go:build !tinygo + package nexus import ( diff --git a/client.go b/client.go index ad5b17d..b2a2f9c 100644 --- a/client.go +++ b/client.go @@ -46,7 +46,7 @@ var ( // ErrUnsafeURL is returned when a URL violates the remote URL policy. ErrUnsafeURL = errors.New("nexus: remote URL rejected") // ErrUnprotectedTransport is returned when strict address policy cannot be - // enforced by a custom HTTP transport or proxy. + // enforced by the HTTP transport or proxy. ErrUnprotectedTransport = errors.New("nexus: HTTP transport cannot enforce public-address policy") ) @@ -207,51 +207,6 @@ func defaultUserAgent() string { return "git-pkgs-nexus/" + version } -func protectedTransport(base http.RoundTripper, policy addressPolicy) (http.RoundTripper, error) { - usingDefault := base == nil - defaultTransport, defaultIsHTTP := http.DefaultTransport.(*http.Transport) - if transport, ok := base.(*http.Transport); ok && defaultIsHTTP && transport == defaultTransport { - usingDefault = true - } - if base == nil { - base = http.DefaultTransport - } - if transport, ok := base.(*http.Transport); ok { - clone := transport.Clone() - if !policy.allowPrivate { - if !usingDefault && clone.Proxy != nil { - return nil, fmt.Errorf("%w: proxies require AllowPrivateAddresses", ErrUnprotectedTransport) - } - if !usingDefault && clone.DialContext != nil { - return nil, fmt.Errorf("%w: custom dialers require AllowPrivateAddresses", ErrUnprotectedTransport) - } - hasCustomTLSDialer := clone.DialTLSContext != nil - hasCustomTLSDialer = hasCustomTLSDialer || clone.DialTLS != nil //nolint:staticcheck // DialTLS remains supported and bypasses DialContext. - if hasCustomTLSDialer { - return nil, fmt.Errorf("%w: custom TLS dialers require AllowPrivateAddresses", ErrUnprotectedTransport) - } - clone.Proxy = nil - } - if clone.ResponseHeaderTimeout == 0 { - clone.ResponseHeaderTimeout = defaultHeaderTimeout - } - underlying := clone.DialContext - if !policy.allowPrivate || underlying == nil { - dialer := &net.Dialer{Timeout: defaultDialTimeout, KeepAlive: defaultKeepAlive} - underlying = dialer.DialContext - } - clone.DialContext = policy.dialContext(underlying) - base = clone - } else if !policy.allowPrivate { - return nil, fmt.Errorf("%w: custom RoundTripper requires AllowPrivateAddresses", ErrUnprotectedTransport) - } - return &checkingRoundTripper{ - base: base, - policy: policy, - responseIdleTimeout: policy.responseIdleTimeout, - }, nil -} - type checkingRoundTripper struct { base http.RoundTripper policy addressPolicy @@ -305,43 +260,6 @@ func (policy addressPolicy) checkHost(ctx context.Context, host string) error { return nil } -func (policy addressPolicy) dialContext(underlying func(context.Context, string, string) (net.Conn, error)) func(context.Context, string, string) (net.Conn, error) { - return func(ctx context.Context, network, address string) (net.Conn, error) { - if policy.allowPrivate { - return underlying(ctx, network, address) - } - host, port, err := net.SplitHostPort(address) - if err != nil { - return nil, err - } - if parsed := net.ParseIP(host); parsed != nil { - if err := checkPublicIP(parsed); err != nil { - return nil, err - } - return underlying(ctx, network, address) - } - addresses, err := net.DefaultResolver.LookupIPAddr(ctx, host) - if err != nil { - return nil, err - } - var lastErr error - for _, candidate := range addresses { - if err := checkPublicIP(candidate.IP); err != nil { - return nil, fmt.Errorf("nexus: host %s: %w", host, err) - } - connection, dialErr := underlying(ctx, network, net.JoinHostPort(candidate.IP.String(), port)) - if dialErr == nil { - return connection, nil - } - lastErr = dialErr - } - if lastErr != nil { - return nil, lastErr - } - return nil, fmt.Errorf("nexus: no addresses resolved for %s", host) - } -} - const ( idleBodyOpen uint32 = iota idleBodyClosed diff --git a/client_test.go b/client_test.go index 48fd239..7a414e1 100644 --- a/client_test.go +++ b/client_test.go @@ -8,7 +8,6 @@ import ( "net" "net/http" "net/http/httptest" - "net/url" "strings" "sync/atomic" "testing" @@ -52,46 +51,6 @@ func TestClientRejectsInvalidOptionsBeforeRequest(t *testing.T) { } } -func TestClientRejectsUnprotectedTransports(t *testing.T) { - proxyURL := &url.URL{Scheme: "http", Host: "proxy.example.test"} - tests := []struct { - name string - transport http.RoundTripper - }{ - { - name: "custom RoundTripper", - transport: roundTripFunc(func(*http.Request) (*http.Response, error) { - return nil, errors.New("transport should not be called") - }), - }, - { - name: "proxy", - transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}, - }, - { - name: "custom dialer", - transport: &http.Transport{DialContext: func(context.Context, string, string) (net.Conn, error) { - return nil, errors.New("dialer should not be called") - }}, - }, - { - name: "custom TLS dialer", - transport: &http.Transport{DialTLSContext: func(context.Context, string, string) (net.Conn, error) { - return nil, errors.New("dialer should not be called") - }}, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := NewClient(ClientOptions{HTTPClient: &http.Client{Transport: test.transport}}) - _, err := client.Sync(context.Background(), "https://8.8.8.8/repository", nil) - if !errors.Is(err, ErrUnprotectedTransport) { - t.Fatalf("Sync error = %v, want ErrUnprotectedTransport", err) - } - }) - } -} - func TestClientAllowsCustomTransportWithPrivateAddresses(t *testing.T) { var calls atomic.Int32 transport := roundTripFunc(func(request *http.Request) (*http.Response, error) { diff --git a/transport_std.go b/transport_std.go new file mode 100644 index 0000000..5f4db64 --- /dev/null +++ b/transport_std.go @@ -0,0 +1,92 @@ +//go:build !tinygo + +package nexus + +import ( + "context" + "fmt" + "net" + "net/http" +) + +func protectedTransport(base http.RoundTripper, policy addressPolicy) (http.RoundTripper, error) { + usingDefault := base == nil + defaultTransport, defaultIsHTTP := http.DefaultTransport.(*http.Transport) + if transport, ok := base.(*http.Transport); ok && defaultIsHTTP && transport == defaultTransport { + usingDefault = true + } + if base == nil { + base = http.DefaultTransport + } + if transport, ok := base.(*http.Transport); ok { + clone := transport.Clone() + if !policy.allowPrivate { + if !usingDefault && clone.Proxy != nil { + return nil, fmt.Errorf("%w: proxies require AllowPrivateAddresses", ErrUnprotectedTransport) + } + if !usingDefault && clone.DialContext != nil { + return nil, fmt.Errorf("%w: custom dialers require AllowPrivateAddresses", ErrUnprotectedTransport) + } + hasCustomTLSDialer := clone.DialTLSContext != nil + hasCustomTLSDialer = hasCustomTLSDialer || clone.DialTLS != nil //nolint:staticcheck // DialTLS remains supported and bypasses DialContext. + if hasCustomTLSDialer { + return nil, fmt.Errorf("%w: custom TLS dialers require AllowPrivateAddresses", ErrUnprotectedTransport) + } + clone.Proxy = nil + } + if clone.ResponseHeaderTimeout == 0 { + clone.ResponseHeaderTimeout = defaultHeaderTimeout + } + underlying := clone.DialContext + if !policy.allowPrivate || underlying == nil { + dialer := &net.Dialer{Timeout: defaultDialTimeout, KeepAlive: defaultKeepAlive} + underlying = dialer.DialContext + } + clone.DialContext = policy.dialContext(underlying) + base = clone + } else if !policy.allowPrivate { + return nil, fmt.Errorf("%w: custom RoundTripper requires AllowPrivateAddresses", ErrUnprotectedTransport) + } + return &checkingRoundTripper{ + base: base, + policy: policy, + responseIdleTimeout: policy.responseIdleTimeout, + }, nil +} + +func (policy addressPolicy) dialContext(underlying func(context.Context, string, string) (net.Conn, error)) func(context.Context, string, string) (net.Conn, error) { + return func(ctx context.Context, network, address string) (net.Conn, error) { + if policy.allowPrivate { + return underlying(ctx, network, address) + } + host, port, err := net.SplitHostPort(address) + if err != nil { + return nil, err + } + if parsed := net.ParseIP(host); parsed != nil { + if err := checkPublicIP(parsed); err != nil { + return nil, err + } + return underlying(ctx, network, address) + } + addresses, err := net.DefaultResolver.LookupIPAddr(ctx, host) + if err != nil { + return nil, err + } + var lastErr error + for _, candidate := range addresses { + if err := checkPublicIP(candidate.IP); err != nil { + return nil, fmt.Errorf("nexus: host %s: %w", host, err) + } + connection, dialErr := underlying(ctx, network, net.JoinHostPort(candidate.IP.String(), port)) + if dialErr == nil { + return connection, nil + } + lastErr = dialErr + } + if lastErr != nil { + return nil, lastErr + } + return nil, fmt.Errorf("nexus: no addresses resolved for %s", host) + } +} diff --git a/transport_std_test.go b/transport_std_test.go new file mode 100644 index 0000000..7e41af6 --- /dev/null +++ b/transport_std_test.go @@ -0,0 +1,52 @@ +//go:build !tinygo + +package nexus + +import ( + "context" + "errors" + "net" + "net/http" + "net/url" + "testing" +) + +func TestClientRejectsUnprotectedTransports(t *testing.T) { + proxyURL := &url.URL{Scheme: "http", Host: "proxy.example.test"} + tests := []struct { + name string + transport http.RoundTripper + }{ + { + name: "custom RoundTripper", + transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return nil, errors.New("transport should not be called") + }), + }, + { + name: "proxy", + transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}, + }, + { + name: "custom dialer", + transport: &http.Transport{DialContext: func(context.Context, string, string) (net.Conn, error) { + return nil, errors.New("dialer should not be called") + }}, + }, + { + name: "custom TLS dialer", + transport: &http.Transport{DialTLSContext: func(context.Context, string, string) (net.Conn, error) { + return nil, errors.New("dialer should not be called") + }}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client := NewClient(ClientOptions{HTTPClient: &http.Client{Transport: test.transport}}) + _, err := client.Sync(context.Background(), "https://8.8.8.8/repository", nil) + if !errors.Is(err, ErrUnprotectedTransport) { + t.Fatalf("Sync error = %v, want ErrUnprotectedTransport", err) + } + }) + } +} diff --git a/transport_tinygo.go b/transport_tinygo.go new file mode 100644 index 0000000..eabf451 --- /dev/null +++ b/transport_tinygo.go @@ -0,0 +1,23 @@ +//go:build tinygo + +package nexus + +import ( + "fmt" + "net/http" +) + +func protectedTransport(base http.RoundTripper, policy addressPolicy) (http.RoundTripper, error) { + // TinyGo transports bypass the dialer used for connection-time address checks. + if !policy.allowPrivate { + return nil, fmt.Errorf("%w: TinyGo requires AllowPrivateAddresses", ErrUnprotectedTransport) + } + if base == nil { + base = http.DefaultTransport + } + return &checkingRoundTripper{ + base: base, + policy: policy, + responseIdleTimeout: policy.responseIdleTimeout, + }, nil +} diff --git a/transport_tinygo_test.go b/transport_tinygo_test.go new file mode 100644 index 0000000..63e7dcf --- /dev/null +++ b/transport_tinygo_test.go @@ -0,0 +1,138 @@ +//go:build tinygo + +package nexus + +import ( + "bytes" + "context" + _ "embed" + "errors" + "io" + "net/http" + "strings" + "testing" + "time" +) + +//go:embed testdata/maven-index-exporter/nexus-maven-repository-index.gz +var tinyGoIndex []byte + +func TestTinyGoClientRequiresAddressOptOut(t *testing.T) { + transport := roundTripFunc(func(*http.Request) (*http.Response, error) { + t.Error("transport called without address-policy opt-out") + return nil, errors.New("unexpected request") + }) + original := http.DefaultTransport + http.DefaultTransport = transport + t.Cleanup(func() { http.DefaultTransport = original }) + + for _, httpClient := range []*http.Client{nil, {}, {Transport: transport}, {Transport: &http.Transport{}}} { + client := NewClient(ClientOptions{HTTPClient: httpClient}) + _, err := client.Sync(context.Background(), "https://repo.example.test", nil) + if !errors.Is(err, ErrUnprotectedTransport) { + t.Fatalf("Sync error = %v, want ErrUnprotectedTransport", err) + } + } +} + +func TestTinyGoClientRejectsUnsafeURLs(t *testing.T) { + transport := roundTripFunc(func(*http.Request) (*http.Response, error) { + t.Error("transport called for unsafe URL") + return nil, errors.New("unexpected request") + }) + client := NewClient(ClientOptions{ + HTTPClient: &http.Client{Transport: transport}, + AllowPrivateAddresses: true, + }) + for _, repository := range []string{ + "ftp://repo.example.test", "https://user:password@repo.example.test", "https://repo.example.test?query=value", + } { + _, err := client.Sync(context.Background(), repository, nil) + if !errors.Is(err, ErrUnsafeURL) { + t.Fatalf("Sync(%q) error = %v, want ErrUnsafeURL", repository, err) + } + } +} + +func TestTinyGoClientSync(t *testing.T) { + for _, useDefault := range []bool{false, true} { + t.Run(map[bool]string{false: "custom transport", true: "default transport"}[useDefault], func(t *testing.T) { + testTinyGoSync(t, useDefault) + }) + } +} + +func testTinyGoSync(t *testing.T, useDefault bool) { + t.Helper() + timestamp := time.UnixMilli(1635883823455).UTC() + properties := "nexus.index.id=test\nnexus.index.timestamp=" + timestamp.Format(indexTimestampForm) + "\n" + var requests []string + transport := roundTripFunc(func(request *http.Request) (*http.Response, error) { + requests = append(requests, request.URL.Path) + var body io.Reader + switch request.URL.Path { + case "/repository/" + propertiesPath: + body = strings.NewReader(properties) + case "/repository/" + fullChunkPath: + body = bytes.NewReader(tinyGoIndex) + default: + t.Fatalf("unexpected request: %s", request.URL) + } + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(body), Header: make(http.Header)}, nil + }) + options := ClientOptions{AllowPrivateAddresses: true} + if useDefault { + original := http.DefaultTransport + http.DefaultTransport = transport + t.Cleanup(func() { http.DefaultTransport = original }) + } else { + options.HTTPClient = &http.Client{Transport: transport} + } + sync, err := NewClient(options).Sync(context.Background(), "https://repo.example.test/repository", nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := sync.Close(); err != nil { + t.Error(err) + } + }) + if sync.Mode() != SyncFull { + t.Fatalf("Mode = %s, want full", sync.Mode()) + } + chunk, err := sync.NextChunk() + if err != nil { + t.Fatal(err) + } + var events []Event + for { + event, err := chunk.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatal(err) + } + events = append(events, event) + } + if len(events) != 7 { + t.Fatalf("event count = %d, want 7", len(events)) + } + first := events[0].Artifact + if first.GroupID != "al.aldi" || first.ArtifactID != "sprova4j" || first.Version != "0.1.0" { + t.Errorf("first artifact = %+v", first) + } + cursor, err := chunk.Checkpoint() + if err != nil { + t.Fatal(err) + } + if cursor.IndexID != "test" || !cursor.Timestamp.Equal(timestamp) { + t.Errorf("checkpoint = %+v", cursor) + } + if _, err := sync.NextChunk(); !errors.Is(err, io.EOF) { + t.Fatalf("NextChunk error = %v, want EOF", err) + } + if len(requests) != 3 || requests[2] != "/repository/"+propertiesPath { + t.Errorf("requests = %v, want properties, index, properties", requests) + } +}