Skip to content

Commit 319c3dc

Browse files
TheodoreSpeaksTheodoreSpeaks
committed
fix(scripts): simplify staging workflow backfill
1 parent 25f8c71 commit 319c3dc

2 files changed

Lines changed: 149 additions & 8 deletions

File tree

‎apps/sim/scripts/backfill-table-workflow-deployments.test.ts‎

Lines changed: 71 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,46 @@
11
/**
22
* @vitest-environment node
33
*/
4-
import { beforeEach, describe, expect, it, vi } from 'vitest'
4+
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
55

6-
const { mockPerformFullDeploy } = vi.hoisted(() => ({
6+
const { mockLoadRuntimeSecrets, mockPerformFullDeploy } = vi.hoisted(() => ({
7+
mockLoadRuntimeSecrets: vi.fn(),
78
mockPerformFullDeploy: vi.fn(),
89
}))
910

11+
vi.mock('@sim/runtime-secrets', () => ({
12+
loadRuntimeSecrets: mockLoadRuntimeSecrets,
13+
}))
14+
1015
vi.mock('@/lib/workflows/orchestration/deploy', () => ({
1116
performFullDeploy: mockPerformFullDeploy,
1217
}))
1318

1419
import {
1520
backfillTableWorkflowDeployments,
1621
deployTableWorkflow,
22+
parseTableWorkflowDeploymentBackfillArgs,
23+
prepareTableWorkflowDeploymentBackfillEnvironment,
1724
TABLE_WORKFLOW_DEPLOYMENT_BATCH_SIZE,
1825
type TableWorkflowDeploymentCandidate,
1926
type TableWorkflowDeploymentStore,
2027
} from '@/scripts/backfill-table-workflow-deployments'
2128

29+
const ORIGINAL_ENV = {
30+
DATABASE_URL: process.env.DATABASE_URL,
31+
DATABASE_URL_WEB: process.env.DATABASE_URL_WEB,
32+
SIM_ENV_SECRET_ID: process.env.SIM_ENV_SECRET_ID,
33+
}
34+
35+
function restoreEnvironmentVariable(key: keyof typeof ORIGINAL_ENV): void {
36+
const value = ORIGINAL_ENV[key]
37+
if (value === undefined) {
38+
Reflect.deleteProperty(process.env, key)
39+
} else {
40+
process.env[key] = value
41+
}
42+
}
43+
2244
function candidate(workflowId: string): TableWorkflowDeploymentCandidate {
2345
return {
2446
workflowId,
@@ -43,6 +65,53 @@ describe('backfillTableWorkflowDeployments', () => {
4365
vi.clearAllMocks()
4466
})
4567

68+
afterEach(() => {
69+
restoreEnvironmentVariable('DATABASE_URL')
70+
restoreEnvironmentVariable('DATABASE_URL_WEB')
71+
restoreEnvironmentVariable('SIM_ENV_SECRET_ID')
72+
})
73+
74+
it('loads the staging runtime secret before database modules are needed', async () => {
75+
Reflect.deleteProperty(process.env, 'DATABASE_URL')
76+
Reflect.deleteProperty(process.env, 'DATABASE_URL_WEB')
77+
Reflect.deleteProperty(process.env, 'SIM_ENV_SECRET_ID')
78+
mockLoadRuntimeSecrets.mockImplementation(async () => {
79+
process.env.DATABASE_URL = 'postgres://staging/database'
80+
})
81+
82+
await prepareTableWorkflowDeploymentBackfillEnvironment(['--environment=staging'])
83+
84+
expect(process.env.SIM_ENV_SECRET_ID).toBe('/staging/sim/env-vars')
85+
expect(mockLoadRuntimeSecrets).toHaveBeenCalledTimes(1)
86+
})
87+
88+
it('keeps the existing local DATABASE_URL mode when no environment is requested', async () => {
89+
process.env.DATABASE_URL = 'postgres://local/database'
90+
91+
await prepareTableWorkflowDeploymentBackfillEnvironment([])
92+
93+
expect(mockLoadRuntimeSecrets).not.toHaveBeenCalled()
94+
expect(process.env.DATABASE_URL).toBe('postgres://local/database')
95+
})
96+
97+
it('rejects unsupported, unknown, duplicate, and locally configured staging arguments', async () => {
98+
expect(() => parseTableWorkflowDeploymentBackfillArgs(['--environment=production'])).toThrow(
99+
'Unsupported backfill environment: production'
100+
)
101+
expect(() => parseTableWorkflowDeploymentBackfillArgs(['--dry-run'])).toThrow(
102+
'Unknown argument: --dry-run'
103+
)
104+
expect(() =>
105+
parseTableWorkflowDeploymentBackfillArgs(['--environment=staging', '--environment=staging'])
106+
).toThrow('can only be provided once')
107+
108+
process.env.DATABASE_URL = 'postgres://local/database'
109+
await expect(
110+
prepareTableWorkflowDeploymentBackfillEnvironment(['--environment=staging'])
111+
).rejects.toThrow('local configuration cannot override staging')
112+
expect(mockLoadRuntimeSecrets).not.toHaveBeenCalled()
113+
})
114+
46115
it('deploys bounded keyset pages and verifies the final desired state', async () => {
47116
const listCandidates = vi
48117
.fn<TableWorkflowDeploymentStore['listCandidates']>()

‎apps/sim/scripts/backfill-table-workflow-deployments.ts‎

Lines changed: 78 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,22 +11,25 @@
1111
*
1212
* Usage:
1313
* DATABASE_URL=... bun run apps/sim/scripts/backfill-table-workflow-deployments.ts
14+
* AWS_PROFILE=sim-admin bun --no-env-file apps/sim/scripts/backfill-table-workflow-deployments.ts --environment=staging
1415
*/
1516

16-
import { db } from '@sim/db'
1717
import { createLogger } from '@sim/logger'
18+
import { loadRuntimeSecrets } from '@sim/runtime-secrets'
1819
import { getErrorMessage } from '@sim/utils/errors'
1920
import { sql } from 'drizzle-orm'
20-
import {
21-
type PerformFullDeployResult,
22-
performFullDeploy,
23-
} from '@/lib/workflows/orchestration/deploy'
21+
import type { PerformFullDeployResult } from '@/lib/workflows/orchestration/deploy'
2422

2523
const logger = createLogger('BackfillTableWorkflowDeployments')
2624

2725
export const TABLE_WORKFLOW_DEPLOYMENT_BATCH_SIZE = 25
2826
const BACKFILL_ACTOR_ID = 'table-workflow-deployment-backfill'
2927
const BACKFILL_OPERATION_VERSION = 'v2'
28+
const STAGING_RUNTIME_SECRET_ID = '/staging/sim/env-vars'
29+
30+
interface TableWorkflowDeploymentBackfillCliOptions {
31+
environment?: 'staging'
32+
}
3033

3134
export interface TableWorkflowDeploymentCandidate {
3235
workflowId: string
@@ -89,8 +92,69 @@ interface DeploymentStateRow extends Record<string, unknown> {
8992
is_deployed: boolean
9093
}
9194

95+
/** Parses the deliberately small CLI surface for the backfill. */
96+
export function parseTableWorkflowDeploymentBackfillArgs(
97+
args: readonly string[]
98+
): TableWorkflowDeploymentBackfillCliOptions {
99+
let environment: TableWorkflowDeploymentBackfillCliOptions['environment']
100+
101+
for (const arg of args) {
102+
if (!arg.startsWith('--environment=')) {
103+
throw new Error(`Unknown argument: ${arg}`)
104+
}
105+
if (environment) {
106+
throw new Error('The --environment argument can only be provided once')
107+
}
108+
109+
const requestedEnvironment = arg.slice('--environment='.length)
110+
if (requestedEnvironment !== 'staging') {
111+
throw new Error(`Unsupported backfill environment: ${requestedEnvironment || '(empty)'}`)
112+
}
113+
environment = requestedEnvironment
114+
}
115+
116+
return { environment }
117+
}
118+
119+
/** Loads staging configuration before modules that read database settings are imported. */
120+
export async function prepareTableWorkflowDeploymentBackfillEnvironment(
121+
args: readonly string[]
122+
): Promise<void> {
123+
const { environment } = parseTableWorkflowDeploymentBackfillArgs(args)
124+
if (!environment) return
125+
126+
const configuredSecretId = process.env.SIM_ENV_SECRET_ID
127+
if (configuredSecretId && configuredSecretId !== STAGING_RUNTIME_SECRET_ID) {
128+
throw new Error(
129+
`SIM_ENV_SECRET_ID is already set to ${configuredSecretId}; expected ${STAGING_RUNTIME_SECRET_ID}`
130+
)
131+
}
132+
133+
const configuredDatabaseVariables = ['DATABASE_URL', 'DATABASE_URL_WEB'].filter(
134+
(key) => key in process.env
135+
)
136+
if (configuredDatabaseVariables.length > 0) {
137+
throw new Error(
138+
`Unset ${configuredDatabaseVariables.join(', ')} before using --environment=staging so local configuration cannot override staging`
139+
)
140+
}
141+
142+
process.env.SIM_ENV_SECRET_ID = STAGING_RUNTIME_SECRET_ID
143+
await loadRuntimeSecrets()
144+
145+
if (!process.env.DATABASE_URL && !process.env.DATABASE_URL_WEB) {
146+
throw new Error(`${STAGING_RUNTIME_SECRET_ID} did not provide a database URL`)
147+
}
148+
}
149+
150+
async function getDatabase() {
151+
const { db } = await import('@sim/db')
152+
return db
153+
}
154+
92155
/** Ensures the table group references can be traversed without silently dropping corrupt data. */
93156
async function assertTableWorkflowIntegrity(): Promise<void> {
157+
const db = await getDatabase()
94158
const [invalidSchema] = await db.execute<InvalidTableSchemaRow>(sql`
95159
SELECT id AS table_id
96160
FROM user_table_definitions
@@ -200,6 +264,7 @@ export const postgresTableWorkflowDeploymentStore: TableWorkflowDeploymentStore
200264
assertIntegrity: assertTableWorkflowIntegrity,
201265

202266
async listCandidates(afterWorkflowId, limit) {
267+
const db = await getDatabase()
203268
const rows = await db.execute<CandidateRow>(sql`
204269
WITH referenced_workflow_ids AS (
205270
SELECT DISTINCT workflow_group.value->>'workflowId' AS workflow_id
@@ -238,6 +303,7 @@ export const postgresTableWorkflowDeploymentStore: TableWorkflowDeploymentStore
238303
},
239304

240305
async isDeployed(workflowId) {
306+
const db = await getDatabase()
241307
const [state] = await db.execute<DeploymentStateRow>(sql`
242308
SELECT
243309
workflow.is_deployed,
@@ -265,6 +331,7 @@ export const postgresTableWorkflowDeploymentStore: TableWorkflowDeploymentStore
265331
export async function deployTableWorkflow(
266332
candidate: TableWorkflowDeploymentCandidate
267333
): Promise<PerformFullDeployResult> {
334+
const { performFullDeploy } = await import('@/lib/workflows/orchestration/deploy')
268335
return performFullDeploy({
269336
workflowId: candidate.workflowId,
270337
userId: candidate.userId,
@@ -365,8 +432,13 @@ export async function runTableWorkflowDeploymentBackfill(): Promise<void> {
365432
logger.info('Table workflow deployment backfill completed', summary)
366433
}
367434

435+
async function main(): Promise<void> {
436+
await prepareTableWorkflowDeploymentBackfillEnvironment(process.argv.slice(2))
437+
await runTableWorkflowDeploymentBackfill()
438+
}
439+
368440
if ((import.meta as { main?: boolean }).main) {
369-
runTableWorkflowDeploymentBackfill()
441+
main()
370442
.then(() => process.exit(0))
371443
.catch((error: unknown) => {
372444
logger.error('Table workflow deployment backfill failed', {

0 commit comments

Comments
 (0)