fix(db): resolve sealing env on the migrator like the app and defer only on batch timeouts

Author: waleedlatif1

helm/sim/templates/_helpers.tpl

@@ -683,3 +683,50 @@ Validate Copilot configuration
   {{- end -}}
 {{- end -}}
 {{- end }}
+
+{{/*
+Inline env entries the app container applies over its Secret, in precedence order (a later entry
+for the same name wins): envDefaults the operator did not override, existingSecret-mode app.env
+values, then extraEnvVars. Shared with the migrations init container so both resolve a key alike.
+*/}}
+{{- define "sim.appEnvOverrides" -}}
+{{- /*
+  Skip envDefaults keys that the user has explicitly overridden in app.env
+  with a non-empty value. K8s `env` takes precedence over `envFrom`, so an
+  inline default would otherwise mask the user's Secret-bound value.
+  Also skip envDefaults entirely in existingSecret mode: the pre-existing
+  Secret is the source of truth, and inlining localhost URL defaults would
+  silently shadow keys (NEXT_PUBLIC_APP_URL, BETTER_AUTH_URL, etc.) the
+  user stored in their Secret but left empty in app.env.
+*/}}
+{{- $appEnv := .Values.app.env | default dict }}
+{{- $useExistingSecret := and .Values.app.secrets.existingSecret.enabled (not .Values.externalSecrets.enabled) }}
+{{- if not $useExistingSecret }}
+{{- range $key, $value := .Values.app.envDefaults | default dict }}
+{{- $override := index $appEnv $key }}
+{{- if and (ne (toString $value) "") (ne (toString $value) "<nil>") (or (not $override) (eq (toString $override) "") (eq (toString $override) "<nil>")) }}
+- name: {{ $key }}
+  value: {{ $value | quote }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- /*
+  In existingSecret mode the chart-managed Secret is not rendered, so
+  non-empty values in app.env would otherwise go nowhere — inline them
+  here so the user's overrides actually reach the pod. Skipped in ESO
+  mode (ESO is the source of truth; inlining would mask the synced value)
+  and in inline mode (values flow through the chart-managed Secret).
+*/}}
+{{- if and .Values.app.secrets.existingSecret.enabled (not .Values.externalSecrets.enabled) }}
+{{- $chartComputed := list "DATABASE_URL" "SOCKET_SERVER_URL" "OLLAMA_URL" "PII_URL" }}
+{{- range $key, $value := $appEnv }}
+{{- if and (ne (toString $value) "") (ne (toString $value) "<nil>") (not (has $key $chartComputed)) }}
+- name: {{ $key }}
+  value: {{ $value | quote }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- with .Values.extraEnvVars }}
+{{ toYaml . }}
+{{- end }}
+{{- end }}

helm/sim/templates/deployment-app.yaml

@@ -59,17 +59,23 @@ spec:
               bun run db:migrate
           {{- /*
             The account token backfill script migration seals with the app's key and honors its
-            ENCRYPT_ACCOUNT_TOKENS opt-out, read the way the app container reads them; without a
-            usable key it does nothing and stays pending.
+            ENCRYPT_ACCOUNT_TOKENS opt-out. Each resolves as it does on the app container: the last
+            matching entry of the app's inline overrides (envDefaults, existingSecret inlining,
+            extraEnvVars), else the app Secret. Without a usable key the step stays pending.
           */}}
           env:
-            {{- $useExistingSecret := and .Values.app.secrets.existingSecret.enabled (not .Values.externalSecrets.enabled) }}
+            {{- $appOverrides := include "sim.appEnvOverrides" . | fromYamlArray }}
             {{- range $key := list "ENCRYPTION_KEY" "ENCRYPT_ACCOUNT_TOKENS" }}
-            {{- $inline := index ($.Values.app.env | default dict) $key }}
-            - name: {{ $key }}
-            {{- if and $useExistingSecret (ne (toString $inline) "") (ne (toString $inline) "<nil>") }}
-              value: {{ $inline | quote }}
+            {{- $override := dict }}
+            {{- range $appOverrides }}
+            {{- if eq (toString .name) $key }}
+            {{- $override = . }}
+            {{- end }}
+            {{- end }}
+            {{- if $override }}
+            - {{ toYaml $override | nindent 14 | trim }}
             {{- else }}
+            - name: {{ $key }}
               valueFrom:
                 secretKeyRef:
                   name: {{ include "sim.appSecretName" $ }}
@@ -111,42 +117,6 @@ spec:
               value: {{ include "sim.ollamaUrl" . | quote }}
             - name: PII_URL
               value: {{ include "sim.piiUrl" . | quote }}
-            {{- /*
-              Skip envDefaults keys that the user has explicitly overridden in app.env
-              with a non-empty value. K8s `env` takes precedence over `envFrom`, so an
-              inline default would otherwise mask the user's Secret-bound value.
-              Also skip envDefaults entirely in existingSecret mode: the pre-existing
-              Secret is the source of truth, and inlining localhost URL defaults would
-              silently shadow keys (NEXT_PUBLIC_APP_URL, BETTER_AUTH_URL, etc.) the
-              user stored in their Secret but left empty in app.env.
-            */}}
-            {{- $appEnv := .Values.app.env | default dict }}
-            {{- $useExistingSecret := and .Values.app.secrets.existingSecret.enabled (not .Values.externalSecrets.enabled) }}
-            {{- if not $useExistingSecret }}
-            {{- range $key, $value := .Values.app.envDefaults | default dict }}
-            {{- $override := index $appEnv $key }}
-            {{- if and (ne (toString $value) "") (ne (toString $value) "<nil>") (or (not $override) (eq (toString $override) "") (eq (toString $override) "<nil>")) }}
-            - name: {{ $key }}
-              value: {{ $value | quote }}
-            {{- end }}
-            {{- end }}
-            {{- end }}
-            {{- /*
-              In existingSecret mode the chart-managed Secret is not rendered, so
-              non-empty values in app.env would otherwise go nowhere — inline them
-              here so the user's overrides actually reach the pod. Skipped in ESO
-              mode (ESO is the source of truth; inlining would mask the synced value)
-              and in inline mode (values flow through the chart-managed Secret).
-            */}}
-            {{- if and .Values.app.secrets.existingSecret.enabled (not .Values.externalSecrets.enabled) }}
-            {{- $chartComputed := list "DATABASE_URL" "SOCKET_SERVER_URL" "OLLAMA_URL" "PII_URL" }}
-            {{- range $key, $value := $appEnv }}
-            {{- if and (ne (toString $value) "") (ne (toString $value) "<nil>") (not (has $key $chartComputed)) }}
-            - name: {{ $key }}
-              value: {{ $value | quote }}
-            {{- end }}
-            {{- end }}
-            {{- end }}
             {{- if .Values.telemetry.enabled }}
             {{- $nodeEnv := default (default "production" (index (.Values.app.envDefaults | default dict) "NODE_ENV")) (index (.Values.app.env | default dict) "NODE_ENV") }}
             # OpenTelemetry configuration
@@ -159,9 +129,8 @@ spec:
             - name: OTEL_RESOURCE_ATTRIBUTES
               value: "service.name=sim-app,service.version={{ .Chart.AppVersion }},deployment.environment={{ $nodeEnv }}"
             {{- end }}
-            {{- with .Values.extraEnvVars }}
-            {{- toYaml . | nindent 12 }}
-            {{- end }}
+            {{- /* Operator overrides last, so they win over the defaults above and over envFrom. */}}
+            {{- include "sim.appEnvOverrides" . | trim | nindent 12 }}
           envFrom:
             # Bundled Redis URL. Listed FIRST so any operator-supplied REDIS_URL
             # in a later source overrides it — see templates/configmap-redis.yaml.

helm/sim/tests/secret-modes_test.yaml

@@ -179,3 +179,65 @@ tests:
             name: ENCRYPTION_KEY
             valueFrom:
               secretKeyRef: { name: my-existing-secret, key: ENCRYPTION_KEY, optional: true }
+
+  - it: an extraEnvVars sealing opt-out reaches the migrations init container
+    template: deployment-app.yaml
+    set:
+      app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+      app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+      app.env.INTERNAL_API_SECRET: x
+      app.env.CRON_SECRET: x
+      postgresql.auth.password: xxxxxxxx
+      extraEnvVars:
+        - name: ENCRYPT_ACCOUNT_TOKENS
+          value: "false"
+    asserts:
+      - contains:
+          path: spec.template.spec.initContainers[0].env
+          content: { name: ENCRYPT_ACCOUNT_TOKENS, value: "false" }
+      - notContains:
+          path: spec.template.spec.initContainers[0].env
+          content:
+            name: ENCRYPT_ACCOUNT_TOKENS
+            valueFrom:
+              secretKeyRef: { name: t-sim-app-secrets, key: ENCRYPT_ACCOUNT_TOKENS, optional: true }
+
+  - it: an envDefaults sealing opt-out reaches the migrations init container
+    template: deployment-app.yaml
+    set:
+      app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+      app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+      app.env.INTERNAL_API_SECRET: x
+      app.env.CRON_SECRET: x
+      app.envDefaults.ENCRYPT_ACCOUNT_TOKENS: "false"
+      postgresql.auth.password: xxxxxxxx
+    asserts:
+      - contains:
+          path: spec.template.spec.initContainers[0].env
+          content: { name: ENCRYPT_ACCOUNT_TOKENS, value: "false" }
+
+  - it: an extraEnvVars ENCRYPTION_KEY override reaches the migrations init container
+    template: deployment-app.yaml
+    set:
+      app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+      app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+      app.env.INTERNAL_API_SECRET: x
+      app.env.CRON_SECRET: x
+      postgresql.auth.password: xxxxxxxx
+      extraEnvVars:
+        - name: ENCRYPTION_KEY
+          valueFrom:
+            secretKeyRef: { name: vault-synced, key: encryption-key }
+    asserts:
+      - contains:
+          path: spec.template.spec.initContainers[0].env
+          content:
+            name: ENCRYPTION_KEY
+            valueFrom:
+              secretKeyRef: { name: vault-synced, key: encryption-key }
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: ENCRYPTION_KEY
+            valueFrom:
+              secretKeyRef: { name: vault-synced, key: encryption-key }

helm/sim/values.yaml

@@ -109,7 +109,10 @@ app:
     BETTER_AUTH_SECRET: ""  # REQUIRED - set via --set flag or external secret manager
     ENCRYPTION_KEY: ""      # REQUIRED - set via --set flag or external secret manager
     # A 64-hex ENCRYPTION_KEY also seals stored OAuth account tokens, and the migrations init
-    # container seals the ones stored before. Set ENCRYPT_ACCOUNT_TOKENS: "false" to opt out.
+    # container seals the ones stored before. To opt out, set ENCRYPT_ACCOUNT_TOKENS to "false"
+    # or "0" wherever the app reads its env: here (inline or existingSecret mode), as a key in the
+    # pre-created Secret, mapped in externalSecrets.remoteRefs.app (External Secrets mode), or in
+    # app.envDefaults / extraEnvVars. The init container resolves it the same way as the app.
     INTERNAL_API_SECRET: ""  # REQUIRED - set via --set flag or external secret manager, used for internal service-to-service authentication
 
     # Optional: Scheduled Jobs Authentication
@@ -2006,6 +2009,9 @@ externalSecrets:
       CRON_SECRET: ""
       # Path to API_ENCRYPTION_KEY in external store (optional)
       API_ENCRYPTION_KEY: ""
+      # Path to ENCRYPT_ACCOUNT_TOKENS in external store (optional; "false" or "0" opts out of
+      # account token sealing)
+      ENCRYPT_ACCOUNT_TOKENS: ""
       # Path to MSHIP_SYSPROMPT_OVERRIDE in external store (optional)
       MSHIP_SYSPROMPT_OVERRIDE: ""
       # Path to REDIS_URL in external store (optional)

packages/db/script-migrations/0037_seal_account_tokens.integration.ts

@@ -7,6 +7,7 @@ import { openText } from '@sim/db/sealed-text'
 import { readTestDatabaseUrl } from '@sim/db/testing/test-infrastructure'
 import { seal, sealingKeyId } from '@sim/security/encryption'
 import { createDeferred } from '@sim/testing/helpers/deferred'
+import { sleep } from '@sim/utils/helpers'
 import { generateId } from '@sim/utils/id'
 import postgres, { type Sql } from 'postgres'
 import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
@@ -212,8 +213,9 @@ describe('sealing stored account tokens in PostgreSQL', () => {
       await markSealedReaderRelease('2 days')
     })
 
-    afterEach(() => {
+    afterEach(async () => {
       vi.unstubAllEnvs()
+      await admin.unsafe(`DROP FUNCTION IF EXISTS "${schemaName}".on_account_update() CASCADE`)
     })
 
     /** When the database first ran the migrate step of a release that reads sealed tokens. */
@@ -331,5 +333,44 @@ describe('sealing stored account tokens in PostgreSQL', () => {
       }
       expect((await storedRows())[0].access_token).toBe('locked-access')
     }, 30_000)
+
+    /** Makes every account UPDATE in this schema run `body` first, as the batch's own statement. */
+    async function onAccountUpdate(body: string) {
+      await admin.unsafe(`CREATE FUNCTION "${schemaName}".on_account_update() RETURNS trigger
+        LANGUAGE plpgsql AS $$ BEGIN ${body}; RETURN NEW; END $$`)
+      await admin.unsafe(`CREATE TRIGGER on_account_update BEFORE UPDATE ON "${schemaName}".account
+        FOR EACH ROW EXECUTE FUNCTION "${schemaName}".on_account_update()`)
+    }
+
+    it('defers when a batch hits its statement timeout', async () => {
+      await insertAccount('slow-access', 'slow-refresh')
+      await onAccountUpdate(
+        `RAISE EXCEPTION 'canceling statement due to statement timeout' USING ERRCODE = '57014'`
+      )
+
+      await migrate()
+
+      expect(await isRecorded()).toBe(false)
+      expect((await storedRows())[0].access_token).toBe('slow-access')
+    })
+
+    it('fails the run when an operator cancels a batch', async () => {
+      await insertAccount('cancelled-access', 'cancelled-refresh')
+      await onAccountUpdate('PERFORM pg_sleep(20)')
+
+      const run = migrate()
+      for (let attempt = 0; attempt < 100; attempt++) {
+        const [cancelled] = await admin<{ cancelled: boolean }[]>`
+          SELECT pg_cancel_backend(pid) AS cancelled FROM pg_stat_activity
+          WHERE datname = current_database() AND wait_event = 'PgSleep'
+            AND query LIKE '%UPDATE account%'`
+        if (cancelled?.cancelled) break
+        await sleep(50)
+      }
+
+      await expect(run).rejects.toThrow('canceling statement due to user request')
+      expect(await isRecorded()).toBe(false)
+      expect((await storedRows())[0].access_token).toBe('cancelled-access')
+    }, 30_000)
   })
 })

packages/db/script-migrations/0037_seal_account_tokens.ts

@@ -9,7 +9,7 @@ import {
 } from '@sim/db/sealed-text'
 import { createLogger } from '@sim/logger'
 import { sealedKeyId, sealingKeyId } from '@sim/security/encryption'
-import { getPostgresErrorCode } from '@sim/utils/errors'
+import { getPostgresCancellationReason } from '@sim/utils/errors'
 import { sleep } from '@sim/utils/helpers'
 import type { Sql } from 'postgres'
 
@@ -152,8 +152,11 @@ const SEALED_READER_RELEASE_MIGRATION = '0036_archive_legacy_scheduled_jobs'
  */
 const SEALED_READER_ROLLOUT_GRACE = '1 hour'
 
-/** Lock and statement timeouts a batch sets on itself; either leaves the rest for the next run. */
-const BATCH_TIMEOUT_CODES = new Set(['55P03', '57014'])
+/**
+ * The lock and statement timeouts a batch sets on itself; either leaves the rest for the next run.
+ * An operator's cancel (`user_cancel`) is not one of them, so it fails the run visibly.
+ */
+const BATCH_TIMEOUTS = new Set(['lock_timeout', 'statement_timeout'])
 
 /**
  * Seals the account tokens stored before this deployment sealed its writes, during `migrate`.
@@ -189,9 +192,9 @@ export const sealAccountTokensMigration: ScriptMigration = {
       }
       logger.info('Account token backfill finished', result)
     } catch (error) {
-      const code = getPostgresErrorCode(error)
-      if (!code || !BATCH_TIMEOUT_CODES.has(code)) throw error
-      throw new ScriptMigrationDeferred(`an account token batch timed out (${code})`)
+      const reason = getPostgresCancellationReason(error)
+      if (!reason || !BATCH_TIMEOUTS.has(reason)) throw error
+      throw new ScriptMigrationDeferred(`an account token batch hit its ${reason}`)
     }
   },
 }

packages/db/script-migrations/index.ts

@@ -16,6 +16,7 @@ import { backfillPublicShareEntitiesMigration } from '@sim/db/script-migrations/
 import { validateFileWorkspaceBindingMigration } from '@sim/db/script-migrations/0035_validate_file_workspace_binding'
 import { archiveLegacyScheduledJobsMigration } from '@sim/db/script-migrations/0036_archive_legacy_scheduled_jobs'
 import { sealAccountTokensMigration } from '@sim/db/script-migrations/0037_seal_account_tokens'
+import { createLogger } from '@sim/logger'
 import type { Sql } from 'postgres'
 import { backfillTableOrderKeys } from './0001_backfill_table_order_keys'
 import { backfillPausedBillingAttribution } from './0002_backfill_paused_billing_attribution'
@@ -30,6 +31,9 @@ import { type ScriptMigration, ScriptMigrationDeferred } from './types'
 
 export type { ScriptMigration } from './types'
 
+/** The migrations container's progress log, which operators read on every upgrade. */
+const logger = createLogger('ScriptMigrations', { enabled: true, logLevel: 'INFO' })
+
 /**
  * Ordered, append-only registry of script migrations. An entry may be deleted
  * once a later migration supersedes it (accepting that deployments which
@@ -126,7 +130,7 @@ export async function runScriptMigrations(
 
   const pending = migrations.filter((migration) => !applied.has(migration.name))
   if (pending.length === 0) {
-    console.log('No pending script migrations.')
+    logger.info('No pending script migrations.')
     return
   }
 
@@ -138,14 +142,17 @@ export async function runScriptMigrations(
         )
       }
     }
-    console.log(`Applying script migration ${migration.name}...`)
+    logger.info(`Applying script migration ${migration.name}...`)
     const startedAt = Date.now()
     try {
       await migration.up(sql)
     } catch (error) {
       if (!(error instanceof ScriptMigrationDeferred)) throw error
-      console.log(
-        `Script migration ${migration.name} deferred (${error.message}); it runs again on the next migrate.`
+      logger.info(
+        `Script migration ${migration.name} deferred; it runs again on the next migrate.`,
+        {
+          reason: error.message,
+        }
       )
       continue
     }
@@ -154,6 +161,6 @@ export async function runScriptMigrations(
         await tx`INSERT INTO script_migrations (name) VALUES (${name}) ON CONFLICT (name) DO NOTHING`
       }
     })
-    console.log(`Script migration ${migration.name} applied in ${Date.now() - startedAt}ms.`)
+    logger.info(`Script migration ${migration.name} applied in ${Date.now() - startedAt}ms.`)
   }
 }