feat(trigger): reach AWS from trigger.dev runs through workload identity

.github/workflows/ci.yml

@@ -292,7 +292,7 @@ jobs:
           if [ -n "$TRIGGER_BRANCH" ]; then
             TARGET_ARGS+=(--branch "$TRIGGER_BRANCH")
           fi
-          bunx trigger.dev@4.8.2 deploy "${TARGET_ARGS[@]}" --skip-promotion
+          bunx trigger.dev@4.8.3 deploy "${TARGET_ARGS[@]}" --skip-promotion
 
       - name: Validate deployment version output
         env:
@@ -643,7 +643,7 @@ jobs:
           if [ -n "$TRIGGER_BRANCH" ]; then
             TARGET_ARGS+=(--branch "$TRIGGER_BRANCH")
           fi
-          bunx trigger.dev@4.8.2 promote "$VERSION" "${TARGET_ARGS[@]}"
+          bunx trigger.dev@4.8.3 promote "$VERSION" "${TARGET_ARGS[@]}"
 
   # Production only, once the new app serves traffic: submit the library and comparison URLs this
   # push changed to IndexNow. Best effort: nothing depends on it, and a failure never fails the run.

.github/workflows/trigger-dev.yml

@@ -41,7 +41,7 @@ jobs:
           set -euo pipefail
           : "${TRIGGER_ACCESS_TOKEN:?TRIGGER_ACCESS_TOKEN_PREVIEW or TRIGGER_ACCESS_TOKEN must be configured}"
           : "${TRIGGER_PROJECT_ID:?TRIGGER_PROJECT_ID must be configured}"
-          bunx trigger.dev@4.8.2 deploy --env preview --branch dev-sim --skip-promotion
+          bunx trigger.dev@4.8.3 deploy --env preview --branch dev-sim --skip-promotion
 
       - name: Promote current dev preview
         working-directory: ./apps/sim
@@ -61,4 +61,4 @@ jobs:
             echo "::notice::Skipping superseded dev task version $VERSION"
             exit 0
           fi
-          bunx trigger.dev@4.8.2 promote "$VERSION" --env preview --branch dev-sim
+          bunx trigger.dev@4.8.3 promote "$VERSION" --env preview --branch dev-sim

apps/sim/lib/core/config/appconfig.ts

@@ -8,7 +8,7 @@ import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
 import { toRecordOrNull } from '@sim/utils/object'
 import { LRUCache } from 'lru-cache'
-import { getAwsCredentialsFromEnv } from '@/lib/core/config/aws'
+import { getAwsCredentials } from '@/lib/core/config/aws'
 import { env } from '@/lib/core/config/env'
 
 const logger = createLogger('AppConfig')
@@ -121,7 +121,7 @@ function getClient(): AppConfigDataClient {
   if (!client) {
     client = new AppConfigDataClient({
       region: env.AWS_REGION,
-      credentials: getAwsCredentialsFromEnv(),
+      credentials: getAwsCredentials(),
     })
   }
   return client

apps/sim/lib/core/config/aws.ts

@@ -1,24 +1,126 @@
+import { createLogger } from '@sim/logger'
+import { type AWSCredentials, awsCredentialProvider } from '@trigger.dev/sdk/oidc/aws'
 import { env } from '@/lib/core/config/env'
 
-interface AwsCredentials {
-  accessKeyId: string
-  secretAccessKey: string
+const logger = createLogger('AwsCredentials')
+
+type AwsCredentialsProvider = () => Promise<AWSCredentials>
+
+/**
+ * Refresh workload credentials this long before they expire. It must exceed the AWS SDK's own
+ * five-minute refresh window, or a client keeps asking for credentials this module still
+ * considers fresh.
+ */
+const REFRESH_BEFORE_EXPIRY_MS = 10 * 60_000
+
+/**
+ * At the start of a run, refresh workload credentials that expire within this window. A token can
+ * only be minted while an attempt is executing, and work a run leaves behind (the hosted-key
+ * metrics flush) still calls AWS after the attempt ends.
+ */
+const RUN_START_MIN_REMAINING_MS = 30 * 60_000
+
+/**
+ * One AWS session for the whole worker process, not one per run. Every run in a deployment
+ * assumes the same role, and Sim's AWS clients are process-wide, so a run-named session would
+ * attribute later runs' calls to the run that minted it.
+ */
+const WORKLOAD_ROLE_SESSION_NAME = 'sim-trigger-worker'
+
+let workloadProvider: AwsCredentialsProvider | null = null
+let currentWorkloadCredentials: AWSCredentials | null = null
+let pendingWorkloadCredentials: Promise<AWSCredentials> | null = null
+
+function getWorkloadProvider(roleArn: string): AwsCredentialsProvider {
+  if (!workloadProvider) {
+    if (!env.AWS_REGION) {
+      throw new Error('TRIGGER_AWS_ROLE_ARN is set but AWS_REGION is not; set both.')
+    }
+    workloadProvider = awsCredentialProvider({
+      roleArn,
+      region: env.AWS_REGION,
+      roleSessionName: WORKLOAD_ROLE_SESSION_NAME,
+    })
+  }
+  return workloadProvider
+}
+
+/** Credentials this close to expiry are treated as expired, so a signed request never outlives them. */
+const EXPIRY_SKEW_MS = 60_000
+
+function remainingMs(credentials: AWSCredentials | null): number {
+  return credentials ? credentials.expiration.getTime() - Date.now() : 0
+}
+
+/**
+ * The cached workload credentials while they have more than `minRemainingMs` left, otherwise a
+ * fresh exchange. Concurrent callers share one exchange. When the exchange fails, still-valid
+ * cached credentials are served instead: outside an executing attempt no token can be minted, and
+ * work a run leaves behind (the metrics flush) must not lose credentials that have not expired.
+ */
+function resolveWorkloadCredentials(
+  roleArn: string,
+  minRemainingMs: number
+): Promise<AWSCredentials> {
+  const current = currentWorkloadCredentials
+  if (current && remainingMs(current) > minRemainingMs) {
+    return Promise.resolve(current)
+  }
+  if (!pendingWorkloadCredentials) {
+    pendingWorkloadCredentials = getWorkloadProvider(roleArn)()
+      .then((credentials) => {
+        currentWorkloadCredentials = credentials
+        return credentials
+      })
+      .catch((error: unknown) => {
+        const cached = currentWorkloadCredentials
+        if (!cached || remainingMs(cached) <= EXPIRY_SKEW_MS) throw error
+        logger.warn('Workload-identity exchange failed; serving cached credentials until expiry', {
+          error,
+          remainingMs: remainingMs(cached),
+        })
+        return cached
+      })
+      .finally(() => {
+        pendingWorkloadCredentials = null
+      })
+  }
+  return pendingWorkloadCredentials
 }
 
 /**
- * Explicit AWS credentials from the environment, or `undefined` to defer to the
- * default AWS provider chain (the ECS task role in our deployments).
+ * Credentials for Sim's own AWS clients: file storage (S3), AppConfig, email (SES), hosted-key
+ * metrics and hosted Bedrock. Never for a user's integration, which brings its own keys.
  *
- * Shared by every AWS SDK client (S3, AppConfig, …) so credential resolution is
- * identical everywhere: explicit keys when both `AWS_ACCESS_KEY_ID` and
- * `AWS_SECRET_ACCESS_KEY` are set (self-hosted, trigger.dev workers), otherwise
- * the instance/task role.
+ * With `TRIGGER_AWS_ROLE_ARN` set (deployed Trigger.dev runs), temporary credentials for that
+ * role, exchanged for the run's Trigger.dev workload-identity token. It takes precedence over any
+ * static keys in the environment, so a failed exchange surfaces instead of silently falling back
+ * to a stored key. Otherwise `undefined`: the AWS SDK's default chain, which reads
+ * `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN`, then shared profiles, then
+ * the ECS task role.
+ */
+export function getAwsCredentials(): AwsCredentialsProvider | undefined {
+  const roleArn = env.TRIGGER_AWS_ROLE_ARN
+  if (!roleArn) return undefined
+  return () => resolveWorkloadCredentials(roleArn, REFRESH_BEFORE_EXPIRY_MS)
+}
+
+/** Whether Sim's own AWS clients have credentials of their own rather than the default chain. */
+export function hasExplicitAwsCredentials(): boolean {
+  return Boolean(env.TRIGGER_AWS_ROLE_ARN || (env.AWS_ACCESS_KEY_ID && env.AWS_SECRET_ACCESS_KEY))
+}
+
+/**
+ * Refreshes workload credentials at the start of a Trigger.dev run, while a token can be minted,
+ * so they outlive the attempt. A no-op outside workload identity. Never throws: a failure is
+ * logged here, and the run's first AWS call retries the exchange and fails with the cause.
  */
-export function getAwsCredentialsFromEnv(): AwsCredentials | undefined {
-  return env.AWS_ACCESS_KEY_ID && env.AWS_SECRET_ACCESS_KEY
-    ? {
-        accessKeyId: env.AWS_ACCESS_KEY_ID,
-        secretAccessKey: env.AWS_SECRET_ACCESS_KEY,
-      }
-    : undefined
+export async function refreshAwsCredentialsForRun(): Promise<void> {
+  const roleArn = env.TRIGGER_AWS_ROLE_ARN
+  if (!roleArn) return
+  try {
+    await resolveWorkloadCredentials(roleArn, RUN_START_MIN_REMAINING_MS)
+  } catch (error) {
+    logger.error('Could not exchange the workload-identity token for AWS credentials', { error })
+  }
 }

apps/sim/lib/core/config/env.ts

@@ -364,6 +364,7 @@ export const env = createEnv({
     AWS_REGION:                            z.string().optional(),                  // AWS region for S3 buckets
     AWS_ACCESS_KEY_ID:                     z.string().optional(),                  // AWS access key ID
     AWS_SECRET_ACCESS_KEY:                 z.string().optional(),                  // AWS secret access key
+    TRIGGER_AWS_ROLE_ARN:                  z.string().optional(),                  // Role Trigger.dev runs assume through workload identity, instead of static keys
     S3_BUCKET_NAME:                        z.string().optional(),                  // S3 bucket for general file storage
     S3_KB_BUCKET_NAME:                     z.string().optional(),                  // S3 bucket for knowledge base files
     S3_EXECUTION_FILES_BUCKET_NAME:        z.string().optional(),                  // S3 bucket for workflow execution files

apps/sim/lib/core/config/trigger-attempt.ts

@@ -0,0 +1,13 @@
+import { refreshAwsCredentialsForRun } from '@/lib/core/config/aws'
+import { warmRedisConnection } from '@/lib/core/config/redis'
+
+/**
+ * The per-attempt setup the Trigger.dev `init` hook awaits before `run()`. Warms the shared Redis
+ * connection, and refreshes workload-identity AWS credentials while the attempt is executing and a
+ * token can still be minted, so work a run leaves behind (the hosted-key metrics flush) has
+ * credentials after the attempt ends. Never throws, like each step: a throw in `init` fails the
+ * attempt.
+ */
+export async function prepareTriggerAttempt(): Promise<void> {
+  await Promise.all([warmRedisConnection(), refreshAwsCredentialsForRun()])
+}

apps/sim/lib/messaging/email/providers/ses.ts

@@ -1,20 +1,22 @@
 import { SESv2Client, SendEmailCommand } from '@aws-sdk/client-sesv2'
 import nodemailer from 'nodemailer'
 import type SESTransport from 'nodemailer/lib/ses-transport'
+import { getAwsCredentials } from '@/lib/core/config/aws'
 import { env } from '@/lib/core/config/env'
 import { sendViaNodemailer } from '@/lib/messaging/email/providers/send-via-nodemailer'
 import type { MailProvider } from '@/lib/messaging/email/types'
 
 /**
  * AWS SES via nodemailer's SES transport using the AWS SDK v3 client.
- * Credentials resolve through the SDK's default provider chain (env vars,
- * shared config, ECS/EKS task role, EC2 instance profile, SSO).
+ * Credentials come from `getAwsCredentials()`: workload identity in Trigger.dev
+ * runs, otherwise the SDK's default provider chain (env vars, shared config,
+ * ECS/EKS task role, EC2 instance profile, SSO).
  */
 export function createSesProvider(): MailProvider | null {
   const region = env.AWS_SES_REGION
   if (!region) return null
 
-  const sesClient = new SESv2Client({ region })
+  const sesClient = new SESv2Client({ region, credentials: getAwsCredentials() })
   const sesOptions: SESTransport.Options = {
     SES: { sesClient, SendEmailCommand },
   }

apps/sim/lib/monitoring/metrics.ts

@@ -30,6 +30,8 @@ import {
 } from '@aws-sdk/client-cloudwatch'
 import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
+import { getAwsCredentials } from '@/lib/core/config/aws'
+import { isHosted } from '@/lib/core/config/env-flags'
 
 const logger = createLogger('HostedKeyMetrics')
 
@@ -44,10 +46,10 @@ type QueueReason = 'actor_requests' | 'dimension' | 'queue_position'
 /** `metering` marks a provider call that succeeded but could not be priced. */
 type FailureReason = 'rate_limited' | 'auth' | 'other' | 'metering'
 
-// Deployed envs (app + trigger worker) carry static AWS creds; local dev does
-// not. No creds → no-op, so recorders stay always-safe to call (same contract
-// as the previous no-op-meter behavior).
-const ENABLED = Boolean(process.env.AWS_ACCESS_KEY_ID)
+// Hosted keys exist only on Sim Cloud, whose app and workers can always reach
+// CloudWatch (task role or workload identity). Elsewhere recorders are no-ops,
+// so they stay always-safe to call.
+const ENABLED = isHosted
 
 const ENVIRONMENT =
   process.env.OTEL_DEPLOYMENT_ENVIRONMENT ||
@@ -63,7 +65,10 @@ let handlersRegistered = false
 
 function getClient(): CloudWatchClient {
   if (!client) {
-    client = new CloudWatchClient({ region: process.env.AWS_REGION || 'us-east-1' })
+    client = new CloudWatchClient({
+      region: process.env.AWS_REGION || 'us-east-1',
+      credentials: getAwsCredentials(),
+    })
   }
   return client
 }

apps/sim/lib/uploads/core/setup.server.ts

@@ -2,6 +2,7 @@ import { existsSync } from 'fs'
 import { mkdir } from 'fs/promises'
 import path, { join } from 'path'
 import { createLogger } from '@sim/logger'
+import { hasExplicitAwsCredentials } from '@/lib/core/config/aws'
 import { env } from '@/lib/core/config/env'
 import {
   getStorageProvider,
@@ -88,7 +89,7 @@ if (typeof process !== 'undefined') {
     if (!env.S3_BUCKET_NAME || !env.AWS_REGION) {
       logger.warn('S3 storage configuration is incomplete')
       logger.warn('Set S3_BUCKET_NAME and AWS_REGION for S3 storage')
-    } else if (!env.AWS_ACCESS_KEY_ID || !env.AWS_SECRET_ACCESS_KEY) {
+    } else if (!hasExplicitAwsCredentials()) {
       logger.warn('AWS credentials are not set in environment variables')
       logger.warn('Set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY for S3 storage')
     } else {

apps/sim/lib/uploads/providers/s3/client.ts

@@ -16,7 +16,7 @@ import {
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
 import { getErrorMessage } from '@sim/utils/errors'
 import { generateId } from '@sim/utils/id'
-import { getAwsCredentialsFromEnv } from '@/lib/core/config/aws'
+import { getAwsCredentials } from '@/lib/core/config/aws'
 import {
   assertKnownSizeWithinLimit,
   readNodeStreamToBufferWithLimit,
@@ -62,7 +62,7 @@ export function getS3Client(): S3Client {
     region,
     endpoint: S3_CONFIG.endpoint,
     forcePathStyle: S3_CONFIG.forcePathStyle,
-    credentials: getAwsCredentialsFromEnv(),
+    credentials: getAwsCredentials(),
   })
 
   return _s3Client

apps/sim/lib/uploads/providers/s3/presigned-upload.test.ts

@@ -4,17 +4,13 @@
  * missing from `headers`, or a metadata key both signed as a header and hoisted
  * into the query, makes the provider reject or strip the upload.
  */
-import { resetEnvMock, setEnv } from '@sim/testing/mocks/env.mock'
 import { setUploadsConfig, uploadsConfigMock } from '@sim/testing/mocks/uploads-config.mock'
-import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
 
 vi.mock('@/lib/uploads/config', () => uploadsConfigMock)
 
 import { getS3PresignedUploadUrl, resetS3ClientForTesting } from '@/lib/uploads/providers/s3/client'
 
-setEnv({ AWS_ACCESS_KEY_ID: 'test-access-key', AWS_SECRET_ACCESS_KEY: 'test-secret-key' })
-afterAll(resetEnvMock)
-
 /** Headers the browser sets itself and the uploader never supplies. */
 const TRANSPORT_HEADERS = new Set(['host', 'content-length'])
 
@@ -44,7 +40,12 @@ async function presign() {
 }
 
 describe('getS3PresignedUploadUrl', () => {
-  beforeEach(() => configure(undefined))
+  beforeEach(() => {
+    // The client resolves credentials through the SDK's default chain, which reads process.env.
+    vi.stubEnv('AWS_ACCESS_KEY_ID', 'test-access-key')
+    vi.stubEnv('AWS_SECRET_ACCESS_KEY', 'test-secret-key')
+    configure(undefined)
+  })
 
   it('signs metadata as uploader-sent headers for a custom S3-compatible endpoint', async () => {
     configure('https://s3.de.io.cloud.ovh.net')

apps/sim/package.json

@@ -177,8 +177,8 @@
     "@tiptap/starter-kit": "3.31.4",
     "@tiptap/suggestion": "3.31.4",
     "@tiptap/y-tiptap": "3.0.7",
-    "@trigger.dev/core": "4.8.2",
-    "@trigger.dev/sdk": "4.8.2",
+    "@trigger.dev/core": "4.8.3",
+    "@trigger.dev/sdk": "4.8.3",
     "@typescript/typescript6": "^6.0.2",
     "@xterm/addon-fit": "0.11.0",
     "@xterm/addon-unicode11": "0.9.0",
@@ -298,7 +298,7 @@
     "@tailwindcss/postcss": "^4.3.3",
     "@tailwindcss/typography": "0.5.20",
     "@testing-library/jest-dom": "^6.10.0",
-    "@trigger.dev/build": "4.8.2",
+    "@trigger.dev/build": "4.8.3",
     "@types/archiver": "8.0.0",
     "@types/busboy": "1.5.4",
     "@types/chrome": "0.3.6",

apps/sim/providers/bedrock/index.ts

@@ -15,6 +15,7 @@ import {
 import { createLogger } from '@sim/logger'
 import { getErrorMessage, toError } from '@sim/utils/errors'
 import { isRecordLike } from '@sim/utils/object'
+import { getAwsCredentials } from '@/lib/core/config/aws'
 import { validateAwsRegion } from '@/lib/core/security/input-validation'
 import type { IterationToolCall, NormalizedBlockOutput, StreamingExecution } from '@/executor/types'
 import { MAX_TOOL_ITERATIONS } from '@/providers'
@@ -158,12 +159,12 @@ export const bedrockProvider: ProviderConfig = {
       )
     }
 
-    const clientConfig: BedrockRuntimeClientConfig = { region }
-    if (request.bedrockAccessKeyId && request.bedrockSecretKey) {
-      clientConfig.credentials = {
-        accessKeyId: request.bedrockAccessKeyId,
-        secretAccessKey: request.bedrockSecretKey,
-      }
+    const clientConfig: BedrockRuntimeClientConfig = {
+      region,
+      credentials:
+        request.bedrockAccessKeyId && request.bedrockSecretKey
+          ? { accessKeyId: request.bedrockAccessKeyId, secretAccessKey: request.bedrockSecretKey }
+          : getAwsCredentials(),
     }
 
     // Key on the full credential (access key id + secret) so a corrected secret