Add AdditionalCACerts to Configuration.Build

ChangeLog

@@ -5,6 +5,7 @@ Version 17.34.0
     * Subinclude cycle detection (#3615)
     * Fix for `plz export --notrim` in some cases (#3621)
     * `plz init plugin go` now writes a filegroup for `go.mod` and sets `modfile` (#3587)
+    * New `AdditionalCACerts` config option to work with intercepting `HTTPProxy` (#3639)
 
 Version 17.33.0
 ---------------

docs/config.html

@@ -788,6 +788,13 @@ <h3 class="mt1 f6 lh-title" id="build.httpproxy">HttpProxy</h3>
         <p>{{ index .ConfigHelpText "build.httpproxy" }}</p>
       </div>
     </li>
+    <li>
+      <div>
+        <h3 class="mt1 f6 lh-title" id="build.additionalcacerts">AdditionalCACerts</h3>
+
+        <p>{{ index .ConfigHelpText "build.additionalcacerts" }}</p>
+      </div>
+    </li>
     <li>
       <div>
         <h3 class="mt1 f6 lh-title" id="build.hashcheckers">HashCheckers</h3>

src/build/build_step.go

@@ -41,6 +41,7 @@ var errStop = fmt.Errorf("stopping build")
 // httpClient is the shared http client that we use for fetching remote files.
 var httpClient *retryablehttp.Client
 var httpClientOnce sync.Once
+var httpClientErr error
 var httpClientLimiter chan struct{}
 
 var successfulRemoteTargetBuildDuration = metrics.NewHistogramVec(
@@ -1058,15 +1059,18 @@ func fetchRemoteFile(state *core.BuildState, target *core.BuildTarget) error {
 		httpClient = retryablehttp.NewClient()
 		httpClient.Logger = &cli.HTTPLogWrapper{Log: log}
 
+		transport := httpClient.HTTPClient.Transport.(*http.Transport)
 		if state.Config.Build.HTTPProxy != "" {
-			httpClient.HTTPClient.Transport = &http.Transport{
-				Proxy: http.ProxyURL(state.Config.Build.HTTPProxy.AsURL()),
-			}
+			transport.Proxy = http.ProxyURL(state.Config.Build.HTTPProxy.AsURL())
 		}
+		transport.TLSClientConfig, httpClientErr = state.Config.TLSConfig()
 
 		httpClient.HTTPClient.Timeout = time.Duration(state.Config.Build.Timeout)
 		httpClientLimiter = make(chan struct{}, state.Config.Build.ParallelDownloads)
 	})
+	if httpClientErr != nil {
+		return httpClientErr
+	}
 
 	if err := prepareDirectory(target.OutDir(), false); err != nil {
 		return err

src/build/remote_file_test.go

@@ -1,8 +1,13 @@
 package build
 
 import (
+	"encoding/pem"
 	"net"
 	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"sync"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -97,3 +102,39 @@ func TestBasicAuth(t *testing.T) {
 	err = fetchRemoteFile(state, target)
 	require.NoError(t, err)
 }
+
+func TestAdditionalCACerts(t *testing.T) {
+	s := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+		writer.Write([]byte("wibble"))
+	}))
+	defer s.Close()
+	// t.TempDir() is relative when run remotely (TMPDIR is .), and relative paths are resolved against RepoRoot.
+	certFile, err := filepath.Abs(filepath.Join(t.TempDir(), "ca.pem"))
+	require.NoError(t, err)
+	err = os.WriteFile(certFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: s.Certificate().Raw}), 0644)
+	require.NoError(t, err)
+
+	// The client is only configured once, so reset it to pick up config changes.
+	resetHTTPClient := func() {
+		httpClientOnce = sync.Once{}
+		httpClientErr = nil
+	}
+	resetHTTPClient()
+	t.Cleanup(resetHTTPClient)
+
+	state, target := newState("//pkg:additional_ca_certs_test")
+	target.IsRemoteFile = true
+	target.Sources = []core.BuildInput{core.URLLabel(s.URL + "/file")}
+	target.AddOutput("file")
+
+	err = fetchRemoteFile(state, target)
+	assert.ErrorContains(t, err, "certificate")
+
+	resetHTTPClient()
+	state.Config.Build.AdditionalCACerts = []string{certFile}
+	err = fetchRemoteFile(state, target)
+	require.NoError(t, err)
+	contents, err := os.ReadFile(filepath.Join(target.TmpDir(), "file"))
+	require.NoError(t, err)
+	assert.Equal(t, "wibble", string(contents))
+}

src/core/config.go

@@ -4,6 +4,8 @@ package core
 
 import (
 	"crypto/sha1"
+	"crypto/tls"
+	"crypto/x509"
 	"fmt"
 	"io"
 	iofs "io/fs"
@@ -520,6 +522,7 @@ type Configuration struct {
 		PassEnv              []string     `help:"A list of environment variables to pass from the current environment to build rules. For example\n\nPassEnv = HTTP_PROXY\n\nwould copy your HTTP_PROXY environment variable to the build env for any rules."`
 		PassUnsafeEnv        []string     `help:"Similar to PassEnv, a list of environment variables to pass from the current environment to build rules. Unlike PassEnv, the environment variable values are not used when calculating build target hashes."`
 		HTTPProxy            cli.URL      `help:"A URL to use as a proxy server for downloads. Only applies to internal ones - e.g. self-updates or remote_file rules."`
+		AdditionalCACerts    []string     `help:"A list of paths to PEM-encoded CA certificates to trust in addition to the system trust store when making HTTPS connections, including to HTTPProxy. Only applies to internal downloads - e.g. self-updates or remote_file rules. Relative paths are resolved against the repo root."`
 		HashCheckers         []string     `help:"Set of hash algos supported by the 'hashes' argument on build rules. Defaults to: sha1,sha256,blake3." options:"sha1,sha256,blake3,xxhash,crc32,crc64"`
 		HashFunction         string       `help:"The hash function to use internally for build actions." options:"sha1,sha256,blake3,xxhash,crc32,crc64"`
 		ExitOnError          bool         `help:"True to have build actions automatically fail on error (essentially passing -e to the shell they run in)." var:"EXIT_ON_ERROR"`
@@ -805,6 +808,32 @@ func (config *Configuration) EnsurePleaseLocation() {
 	}
 }
 
+// TLSConfig returns the TLS config to use for internal HTTPS downloads. It trusts the system cert
+// pool plus any certificates in Build.AdditionalCACerts, or is nil if there are none of the latter.
+func (config *Configuration) TLSConfig() (*tls.Config, error) {
+	if len(config.Build.AdditionalCACerts) == 0 {
+		return nil, nil
+	}
+	pool, err := x509.SystemCertPool()
+	if err != nil {
+		return nil, fmt.Errorf("failed to load system cert pool: %w", err)
+	}
+	for _, path := range config.Build.AdditionalCACerts {
+		path = fs.ExpandHomePath(path)
+		if !filepath.IsAbs(path) {
+			path = filepath.Join(RepoRoot, path)
+		}
+		pem, err := os.ReadFile(path)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read CA certificate: %w", err)
+		}
+		if !pool.AppendCertsFromPEM(pem) {
+			return nil, fmt.Errorf("no valid PEM-encoded certificates found in %s", path)
+		}
+	}
+	return &tls.Config{RootCAs: pool}, nil
+}
+
 // Path returns the slice of strings corresponding to the PATH env var.
 func (config *Configuration) Path() []string {
 	config.GetBuildEnv() // ensure it is initialised

src/core/config_test.go

@@ -3,6 +3,7 @@ package core
 import (
 	"bytes"
 	"os"
+	"path/filepath"
 	"reflect"
 	"strings"
 	"testing"
@@ -441,3 +442,22 @@ func TestPluginConfig(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, []string{"fooc"}, config.Plugin["foo"].ExtraValues["fooctool"])
 }
+
+func TestTLSConfig(t *testing.T) {
+	config := DefaultConfiguration()
+	tlsConfig, err := config.TLSConfig()
+	assert.NoError(t, err)
+	assert.Nil(t, tlsConfig)
+
+	config.Build.AdditionalCACerts = []string{"/does/not/exist.pem"}
+	_, err = config.TLSConfig()
+	assert.Error(t, err)
+
+	// t.TempDir() is relative when run remotely (TMPDIR is .), and relative paths are resolved against RepoRoot.
+	notPEM, err := filepath.Abs(filepath.Join(t.TempDir(), "not.pem"))
+	require.NoError(t, err)
+	require.NoError(t, os.WriteFile(notPEM, []byte("wibble"), 0644))
+	config.Build.AdditionalCACerts = []string{notPEM}
+	_, err = config.TLSConfig()
+	assert.ErrorContains(t, err, "no valid PEM-encoded certificates")
+}

src/update/update.go

@@ -57,6 +57,11 @@ func pleaseVersion() semver.Version {
 func CheckAndUpdate(config *core.Configuration, updatesEnabled, updateCommand, forceUpdate, verify, progress, prerelease bool) {
 	httpClient = retryablehttp.NewClient()
 	httpClient.Logger = &cli.HTTPLogWrapper{Log: log}
+	tlsConfig, err := config.TLSConfig()
+	if err != nil {
+		log.Fatalf("Failed to load additional CA certificates: %s", err)
+	}
+	httpClient.HTTPClient.Transport.(*http.Transport).TLSClientConfig = tlsConfig
 
 	if !shouldUpdate(config, updatesEnabled, updateCommand, prerelease) && !forceUpdate {
 		clean(config, updateCommand)