fix(cli): point python at the OS trust store in deploy images

Mono-RevId: f8534eae039f05e66719e51b5b42b3ea26bbcc80

Author: nicktrn

.changeset/node-extra-ca-certs-run-env.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+`NODE_EXTRA_CA_CERTS` set as an environment variable now reaches the task process at startup instead of being overwritten by the image's empty default.

.changeset/python-os-trust-store.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Python tasks and `pip` in deploy images now verify TLS against the image's trust store instead of a pinned certifi bundle, so they also refuse the compromised eMudhra intermediates.

packages/cli-v3/src/deploy/buildImage.test.ts

@@ -158,6 +158,50 @@ describe("generateContainerfile", () => {
     );
   });
 
+  const osTrustStoreEnv =
+    /^ENV SSL_CERT_FILE=\$\{SSL_CERT_FILE:-\/etc\/ssl\/certs\/ca-certificates\.crt\} REQUESTS_CA_BUNDLE=\$\{REQUESTS_CA_BUNDLE:-\/etc\/ssl\/certs\/ca-certificates\.crt\} PIP_CERT=\$\{PIP_CERT:-\/etc\/ssl\/certs\/ca-certificates\.crt\}$/gm;
+
+  it("keeps a trust bundle set by image instructions", async () => {
+    const containerfile = await generateContainerfile({
+      runtime: "node",
+      build: {},
+      image: { instructions: ["ENV REQUESTS_CA_BUNDLE=/app/certs/corp.pem"] },
+      indexScript: "index.js",
+      entrypoint: "entrypoint.js",
+    });
+
+    const instruction = containerfile.indexOf("ENV REQUESTS_CA_BUNDLE=/app/certs/corp.pem");
+    expect(instruction).toBeGreaterThan(-1);
+    expect(instruction).toBeLessThan(containerfile.search(osTrustStoreEnv));
+  });
+
+  it.each(["node", "bun"] as BuildRuntime[])(
+    "points certifi users at the OS trust store in the base and prebuilt build stages on %s",
+    async (runtime) => {
+      const containerfile = await generateContainerfile({
+        runtime,
+        build: {},
+        image: undefined,
+        indexScript: "index.js",
+        entrypoint: "entrypoint.js",
+      });
+
+      expect(containerfile.match(osTrustStoreEnv)).toHaveLength(2);
+    }
+  );
+
+  it("points certifi users at the OS trust store once when the build stage derives from base", async () => {
+    const containerfile = await generateContainerfile({
+      runtime: "node",
+      build: {},
+      image: { instructions: ["RUN echo hi"] },
+      indexScript: "index.js",
+      entrypoint: "entrypoint.js",
+    });
+
+    expect(containerfile.match(osTrustStoreEnv)).toHaveLength(1);
+  });
+
   const rejectSteps = (containerfile: string) =>
     Array.from(
       containerfile.matchAll(

packages/cli-v3/src/deploy/buildImage.ts

@@ -743,6 +743,8 @@ export const EMSIGN_COMPROMISE_DISTRUSTED_ROOTS = [
 
 export const EMSIGN_REJECT_DIR = "/usr/local/share/ca-certificates/emsign-block";
 
+const OS_TRUST_STORE = "/etc/ssl/certs/ca-certificates.crt";
+
 const rejectCompromisedIntermediates = `RUN mkdir -p ${EMSIGN_REJECT_DIR} && \\
 ${EMSIGN_COMPROMISE_REJECTED_INTERMEDIATES.map(
   ({ file, pemLines }) =>
@@ -756,7 +758,8 @@ RUN sed -i -E 's#^(mozilla/(${EMSIGN_COMPROMISE_DISTRUSTED_ROOTS.join(
   "|"
 )})\\.crt)$#!\\1#' /etc/ca-certificates.conf && \\
   update-ca-certificates --fresh
-ENV NODE_OPTIONS=--use-openssl-ca`;
+ENV NODE_OPTIONS=--use-openssl-ca
+ENV SSL_CERT_FILE=\${SSL_CERT_FILE:-${OS_TRUST_STORE}} REQUESTS_CA_BUNDLE=\${REQUESTS_CA_BUNDLE:-${OS_TRUST_STORE}} PIP_CERT=\${PIP_CERT:-${OS_TRUST_STORE}}`;
 
 export async function generateContainerfile(options: GenerateContainerfileOptions) {
   switch (options.runtime) {

packages/cli-v3/src/entryPoints/managed/env.test.ts

@@ -0,0 +1,30 @@
+import { describe, it, expect } from "vitest";
+import { RunnerEnv } from "./env.js";
+
+const requiredEnv = {
+  TRIGGER_CONTENT_HASH: "hash",
+  TRIGGER_PROJECT_ID: "proj_1",
+  TRIGGER_PROJECT_REF: "proj_ref",
+  TRIGGER_DEPLOYMENT_ID: "deploy_1",
+  TRIGGER_DEPLOYMENT_VERSION: "20261020.1",
+  TRIGGER_ENV_ID: "env_1",
+  OTEL_EXPORTER_OTLP_ENDPOINT: "https://example.com/otel",
+  TRIGGER_RUNNER_ID: "runner_1",
+  TRIGGER_POD_SCHEDULED_AT_MS: "1",
+  TRIGGER_DEQUEUED_AT_MS: "1",
+  TRIGGER_SUPERVISOR_API_PROTOCOL: "http",
+  TRIGGER_SUPERVISOR_API_DOMAIN: "localhost",
+  TRIGGER_SUPERVISOR_API_PORT: "8000",
+  TRIGGER_WORKER_INSTANCE_NAME: "worker",
+};
+
+describe("RunnerEnv.gatherProcessEnv", () => {
+  it.each(["", "/app/certs/corp-root.pem"])(
+    "leaves NODE_EXTRA_CA_CERTS=%j to the task run env",
+    (value) => {
+      const env = new RunnerEnv({ ...requiredEnv, NODE_EXTRA_CA_CERTS: value });
+
+      expect(env.gatherProcessEnv()).not.toHaveProperty("NODE_EXTRA_CA_CERTS");
+    }
+  );
+});

packages/cli-v3/src/entryPoints/managed/env.ts

@@ -86,9 +86,6 @@ export class RunnerEnv {
   get NODE_ENV() {
     return this.env.NODE_ENV;
   }
-  get NODE_EXTRA_CA_CERTS() {
-    return this.env.NODE_EXTRA_CA_CERTS;
-  }
   get UV_USE_IO_URING() {
     return this.env.UV_USE_IO_URING;
   }
@@ -243,7 +240,6 @@ export class RunnerEnv {
   gatherProcessEnv(): Record<string, string> {
     const $env = {
       NODE_ENV: this.NODE_ENV,
-      NODE_EXTRA_CA_CERTS: this.NODE_EXTRA_CA_CERTS,
       OTEL_EXPORTER_OTLP_ENDPOINT: this.OTEL_EXPORTER_OTLP_ENDPOINT,
       TRIGGER_OTEL_EXPORTER_OTLP_ENDPOINT: this.OTEL_EXPORTER_OTLP_ENDPOINT,
       UV_USE_IO_URING: this.UV_USE_IO_URING,

packages/cli-v3/src/executions/taskRunProcess.test.ts

@@ -1,4 +1,4 @@
-import { TaskRunProcess, type TaskRunProcessOptions } from "./taskRunProcess.js";
+import { TaskRunProcess, trustStoreEnv, type TaskRunProcessOptions } from "./taskRunProcess.js";
 import { describe, it, expect, vi } from "vitest";
 import { UncaughtExceptionError, UnexpectedExitError } from "@trigger.dev/core/v3/errors";
 import type {
@@ -58,6 +58,50 @@ function createExecution(runId: string, attemptNumber: number): TaskRunExecution
   } as unknown as TaskRunExecution;
 }
 
+describe("trustStoreEnv", () => {
+  const controllerEnv = {
+    SSL_CERT_FILE: "/etc/ssl/certs/ca-certificates.crt",
+    REQUESTS_CA_BUNDLE: "/etc/ssl/certs/ca-certificates.crt",
+    PIP_CERT: "/etc/ssl/certs/ca-certificates.crt",
+    PATH: "/usr/bin",
+  };
+
+  it("forwards the controller's trust store variables to the task process", () => {
+    expect(trustStoreEnv(controllerEnv, {})).toEqual({
+      SSL_CERT_FILE: "/etc/ssl/certs/ca-certificates.crt",
+      REQUESTS_CA_BUNDLE: "/etc/ssl/certs/ca-certificates.crt",
+      PIP_CERT: "/etc/ssl/certs/ca-certificates.crt",
+    });
+  });
+
+  it("lets the run's own variables win", () => {
+    expect(trustStoreEnv(controllerEnv, { REQUESTS_CA_BUNDLE: "/app/corp.pem" })).toEqual({
+      SSL_CERT_FILE: "/etc/ssl/certs/ca-certificates.crt",
+      REQUESTS_CA_BUNDLE: "/app/corp.pem",
+      PIP_CERT: "/etc/ssl/certs/ca-certificates.crt",
+    });
+  });
+
+  it("sets nothing when neither side defines them", () => {
+    expect(trustStoreEnv({ PATH: "/usr/bin" }, {})).toEqual({});
+  });
+
+  it("lets the run's NODE_EXTRA_CA_CERTS win over the image's empty value", () => {
+    expect(
+      trustStoreEnv(
+        { NODE_EXTRA_CA_CERTS: "" },
+        { NODE_EXTRA_CA_CERTS: "/app/certs/corp-root.pem" }
+      )
+    ).toEqual({ NODE_EXTRA_CA_CERTS: "/app/certs/corp-root.pem" });
+  });
+
+  it("forwards NODE_EXTRA_CA_CERTS baked into the image when the run sets none", () => {
+    expect(trustStoreEnv({ NODE_EXTRA_CA_CERTS: "/app/certs/corp-root.pem" }, {})).toEqual({
+      NODE_EXTRA_CA_CERTS: "/app/certs/corp-root.pem",
+    });
+  });
+});
+
 describe("TaskRunProcess", () => {
   describe("execute() on a dead child process", () => {
     it("should reject when child process has already exited and IPC send is skipped", async () => {

packages/cli-v3/src/executions/taskRunProcess.ts

@@ -141,6 +141,7 @@ export class TaskRunProcess {
 
     const fullEnv = {
       ...$env,
+      ...trustStoreEnv(process.env, $env),
       OTEL_IMPORT_HOOK_INCLUDES: workerManifest.otelImportHook?.include?.join(","),
       // TODO: this will probably need to use something different for bun (maybe --preload?)
       NODE_OPTIONS: execOptionsForRuntime(workerManifest.runtime, workerManifest, maxOldSpaceSize),
@@ -606,6 +607,27 @@ export class TaskRunProcess {
   }
 }
 
+const TRUST_STORE_ENV_KEYS = [
+  "NODE_EXTRA_CA_CERTS",
+  "SSL_CERT_FILE",
+  "REQUESTS_CA_BUNDLE",
+  "PIP_CERT",
+] as const;
+
+export function trustStoreEnv(
+  controllerEnv: NodeJS.ProcessEnv,
+  runEnv: Record<string, string | undefined>
+): Record<string, string> {
+  const env: Record<string, string> = {};
+  for (const key of TRUST_STORE_ENV_KEYS) {
+    const value = runEnv[key] ?? controllerEnv[key];
+    if (value) {
+      env[key] = value;
+    }
+  }
+  return env;
+}
+
 function executorArgs(workerManifest: WorkerManifest): string[] {
   return [];
 }