|
| 1 | +// FOUR-STORE MATRIX — proves RoutingRunStore is correct across legacy + new + two gen-2 shards |
| 2 | +// (a, b) against REAL databases (makeNShardRunOpsPostgresTest). NEVER mocked. This is where the |
| 3 | +// §3.4 disjoint-sum fix is proven end-to-end: a double count here strands a blocked run forever. |
| 4 | +// |
| 5 | +// runOpsStore.mixedResidency.test.ts is the TWO-store invariant lock and stays byte-identical; this |
| 6 | +// file is the N-store extension and lives separately. |
| 7 | + |
| 8 | +import { makeNShardRunOpsPostgresTest } from "@internal/testcontainers"; |
| 9 | +import type { PrismaClient } from "@trigger.dev/database"; |
| 10 | +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; |
| 11 | +import { resolveShard } from "@trigger.dev/core/v3/isomorphic"; |
| 12 | +import { describe, expect } from "vitest"; |
| 13 | +import { PostgresRunStore } from "./PostgresRunStore.js"; |
| 14 | +import { RoutingRunStore } from "./runOpsStore.js"; |
| 15 | +import type { CreateRunInput } from "./types.js"; |
| 16 | + |
| 17 | +type AnyClient = PrismaClient | RunOpsPrismaClient; |
| 18 | + |
| 19 | +const matrixTest = makeNShardRunOpsPostgresTest(2); |
| 20 | + |
| 21 | +// A gen-2 id: 24-char base32hex core, the shard char at index 24, version "2" at index 25. |
| 22 | +// resolveShard(gen2("a", ...)) === "a". A bare cuid-length id classifies "legacy". |
| 23 | +function gen2(shardChar: string, seed: string): string { |
| 24 | + const core = (seed.replace(/[^0-9a-v]/g, "0") + "k".repeat(24)).slice(0, 24); |
| 25 | + return `${core}${shardChar}2`; |
| 26 | +} |
| 27 | +function cuid(seed: string): string { |
| 28 | + return (seed + "c".repeat(25)).slice(0, 25); // 25 chars → LEGACY |
| 29 | +} |
| 30 | + |
| 31 | +function makeStore(prisma: AnyClient, variant: "legacy" | "dedicated") { |
| 32 | + return new PostgresRunStore({ |
| 33 | + prisma: prisma as never, |
| 34 | + readOnlyPrisma: prisma as never, |
| 35 | + schemaVariant: variant, |
| 36 | + }); |
| 37 | +} |
| 38 | + |
| 39 | +// The real four-store split: legacy (full schema) + new + gen-2 a + gen-2 b (dedicated subset), |
| 40 | +// routed by the REAL core resolveShard. |
| 41 | +function makeMatrixRouter( |
| 42 | + legacyPrisma: PrismaClient, |
| 43 | + newPrisma: RunOpsPrismaClient, |
| 44 | + shardPrismas: RunOpsPrismaClient[] |
| 45 | +) { |
| 46 | + return new RoutingRunStore({ |
| 47 | + new: makeStore(newPrisma, "dedicated"), |
| 48 | + legacy: makeStore(legacyPrisma, "legacy"), |
| 49 | + shards: [ |
| 50 | + { key: "a", store: makeStore(shardPrismas[0]!, "dedicated") }, |
| 51 | + { key: "b", store: makeStore(shardPrismas[1]!, "dedicated") }, |
| 52 | + ], |
| 53 | + resolveShard, |
| 54 | + }); |
| 55 | +} |
| 56 | + |
| 57 | +async function seedLegacyEnv(prisma: PrismaClient, suffix: string) { |
| 58 | + const organization = await prisma.organization.create({ |
| 59 | + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, |
| 60 | + }); |
| 61 | + const project = await prisma.project.create({ |
| 62 | + data: { |
| 63 | + name: `Project ${suffix}`, |
| 64 | + slug: `project-${suffix}`, |
| 65 | + externalRef: `proj_${suffix}`, |
| 66 | + organizationId: organization.id, |
| 67 | + }, |
| 68 | + }); |
| 69 | + const environment = await prisma.runtimeEnvironment.create({ |
| 70 | + data: { |
| 71 | + type: "DEVELOPMENT", |
| 72 | + slug: "dev", |
| 73 | + projectId: project.id, |
| 74 | + organizationId: organization.id, |
| 75 | + apiKey: `tr_dev_${suffix}`, |
| 76 | + pkApiKey: `pk_dev_${suffix}`, |
| 77 | + shortcode: `short_${suffix}`, |
| 78 | + }, |
| 79 | + }); |
| 80 | + return { |
| 81 | + organizationId: organization.id, |
| 82 | + projectId: project.id, |
| 83 | + runtimeEnvironmentId: environment.id, |
| 84 | + environmentId: environment.id, |
| 85 | + }; |
| 86 | +} |
| 87 | + |
| 88 | +function buildRun(params: { |
| 89 | + runId: string; |
| 90 | + runtimeEnvironmentId: string; |
| 91 | + organizationId: string; |
| 92 | + projectId: string; |
| 93 | + createdAt?: Date; |
| 94 | +}): CreateRunInput { |
| 95 | + return { |
| 96 | + data: { |
| 97 | + id: params.runId, |
| 98 | + engine: "V2", |
| 99 | + status: "PENDING", |
| 100 | + friendlyId: `run_${params.runId}`, |
| 101 | + runtimeEnvironmentId: params.runtimeEnvironmentId, |
| 102 | + environmentType: "DEVELOPMENT", |
| 103 | + organizationId: params.organizationId, |
| 104 | + projectId: params.projectId, |
| 105 | + taskIdentifier: "my-task", |
| 106 | + payload: "{}", |
| 107 | + payloadType: "application/json", |
| 108 | + context: {}, |
| 109 | + traceContext: {}, |
| 110 | + traceId: `trace_${params.runId}`, |
| 111 | + spanId: `span_${params.runId}`, |
| 112 | + runTags: [], |
| 113 | + queue: "task/my-task", |
| 114 | + isTest: false, |
| 115 | + taskEventStore: "taskEvent", |
| 116 | + depth: 0, |
| 117 | + createdAt: params.createdAt ?? new Date("2024-01-01T00:00:00.000Z"), |
| 118 | + }, |
| 119 | + snapshot: { |
| 120 | + engine: "V2", |
| 121 | + executionStatus: "RUN_CREATED", |
| 122 | + description: "Run was created", |
| 123 | + runStatus: "PENDING", |
| 124 | + environmentId: params.runtimeEnvironmentId, |
| 125 | + environmentType: "DEVELOPMENT", |
| 126 | + projectId: params.projectId, |
| 127 | + organizationId: params.organizationId, |
| 128 | + }, |
| 129 | + }; |
| 130 | +} |
| 131 | + |
| 132 | +async function seedPendingWaitpoint( |
| 133 | + prisma: AnyClient, |
| 134 | + params: { id: string; projectId: string; environmentId: string } |
| 135 | +) { |
| 136 | + await (prisma as PrismaClient).waitpoint.create({ |
| 137 | + data: { |
| 138 | + id: params.id, |
| 139 | + friendlyId: `wp_${params.id}`, |
| 140 | + type: "MANUAL", |
| 141 | + status: "PENDING", |
| 142 | + idempotencyKey: `idem_${params.id}`, |
| 143 | + userProvidedIdempotencyKey: false, |
| 144 | + projectId: params.projectId, |
| 145 | + environmentId: params.environmentId, |
| 146 | + }, |
| 147 | + }); |
| 148 | +} |
| 149 | + |
| 150 | +describe("RoutingRunStore four-store matrix — disjoint sum on real databases", () => { |
| 151 | + matrixTest( |
| 152 | + "countPendingWaitpoints unions across a gen-2 shard and the gen-1 pair with no double count", |
| 153 | + async ({ legacyPrisma, newPrisma, shardPrismas }) => { |
| 154 | + const env = await seedLegacyEnv(legacyPrisma, "disjoint"); |
| 155 | + const router = makeMatrixRouter(legacyPrisma, newPrisma, shardPrismas); |
| 156 | + |
| 157 | + // The blocked run lives on shard a. |
| 158 | + const runId = gen2("a", "run"); |
| 159 | + // Its blocking waitpoints: a gen-2 waitpoint on shard b, a cuid on legacy, and a cuid MIRRORED |
| 160 | + // onto both gen-1 stores (the drain-mirror case that must count once). |
| 161 | + const wpB = gen2("b", "wpb"); |
| 162 | + const wpCuid = cuid("wpcuid"); |
| 163 | + const wpMirror = cuid("wpmirror"); |
| 164 | + |
| 165 | + const dedicatedEnv = { projectId: env.projectId, environmentId: env.environmentId }; |
| 166 | + await seedPendingWaitpoint(shardPrismas[1]!, { id: wpB, ...dedicatedEnv }); |
| 167 | + await seedPendingWaitpoint(legacyPrisma, { id: wpCuid, ...dedicatedEnv }); |
| 168 | + await seedPendingWaitpoint(legacyPrisma, { id: wpMirror, ...dedicatedEnv }); |
| 169 | + await seedPendingWaitpoint(newPrisma, { id: wpMirror, ...dedicatedEnv }); |
| 170 | + |
| 171 | + // b:wpB (1) + legacy:wpCuid (1) + wpMirror (once, though on both gen-1 stores) = 3. |
| 172 | + const count = await router.countPendingWaitpoints([wpB, wpCuid, wpMirror], undefined, runId); |
| 173 | + expect(count).toBe(3); |
| 174 | + } |
| 175 | + ); |
| 176 | + |
| 177 | + matrixTest( |
| 178 | + "a gen-2 waitpoint on the run's own shard contributes exactly once, not twice", |
| 179 | + async ({ legacyPrisma, newPrisma, shardPrismas }) => { |
| 180 | + const env = await seedLegacyEnv(legacyPrisma, "ownshard"); |
| 181 | + const router = makeMatrixRouter(legacyPrisma, newPrisma, shardPrismas); |
| 182 | + const runId = gen2("a", "run2"); |
| 183 | + const wpA = gen2("a", "wpa"); |
| 184 | + await seedPendingWaitpoint(shardPrismas[0]!, { |
| 185 | + id: wpA, |
| 186 | + projectId: env.projectId, |
| 187 | + environmentId: env.environmentId, |
| 188 | + }); |
| 189 | + // wpA lives on the run's own shard a → found by the presence query, never re-queried elsewhere. |
| 190 | + expect(await router.countPendingWaitpoints([wpA], undefined, runId)).toBe(1); |
| 191 | + } |
| 192 | + ); |
| 193 | +}); |
| 194 | + |
| 195 | +describe("RoutingRunStore four-store matrix — alias topology", () => { |
| 196 | + matrixTest( |
| 197 | + "an aliased gen-2 shard counts its database ONCE in a sum (declaration, not identity)", |
| 198 | + async ({ legacyPrisma, newPrisma, shardPrismas }) => { |
| 199 | + const env = await seedLegacyEnv(legacyPrisma, "alias"); |
| 200 | + // Shard "a" aliases "new" over the SAME database, but via a SEPARATE store object built over |
| 201 | + // the same client — exactly how the wiring layer will construct it. Identity dedupe would see |
| 202 | + // two objects and double-count; declaration dedupe counts the database once. |
| 203 | + const newStore = makeStore(newPrisma, "dedicated"); |
| 204 | + const aStoreSameDb = makeStore(newPrisma, "dedicated"); // distinct object, same DB |
| 205 | + const router = new RoutingRunStore({ |
| 206 | + new: newStore, |
| 207 | + legacy: makeStore(legacyPrisma, "legacy"), |
| 208 | + shards: [{ key: "a", store: aStoreSameDb, aliasOf: "new" }], |
| 209 | + resolveShard, |
| 210 | + }); |
| 211 | + |
| 212 | + const wp = cuid("aliaswp"); |
| 213 | + await seedPendingWaitpoint(newPrisma, { |
| 214 | + id: wp, |
| 215 | + projectId: env.projectId, |
| 216 | + environmentId: env.environmentId, |
| 217 | + }); |
| 218 | + // No runId → the id-less sum fans over DISTINCT databases. The aliased "a" must not add a |
| 219 | + // second leg over the "new" database, or the one pending waitpoint counts twice. |
| 220 | + expect(await router.countPendingWaitpoints([wp])).toBe(1); |
| 221 | + } |
| 222 | + ); |
| 223 | +}); |
| 224 | + |
| 225 | +describe("RoutingRunStore four-store matrix — mixed gen-1 and gen-2 reads", () => { |
| 226 | + matrixTest( |
| 227 | + "findRunsByIds hydrates a mixed id set across legacy, new and both gen-2 shards", |
| 228 | + async ({ legacyPrisma, newPrisma, shardPrismas }) => { |
| 229 | + const env = await seedLegacyEnv(legacyPrisma, "mixed"); |
| 230 | + const router = makeMatrixRouter(legacyPrisma, newPrisma, shardPrismas); |
| 231 | + |
| 232 | + const legacyId = cuid("mixleg"); |
| 233 | + // A v1 run-ops id (version "1") routes to "new"; gen-2 ids route to their shard char. |
| 234 | + const newId = ("mixnew".replace(/[^0-9a-v]/g, "0") + "k".repeat(24)).slice(0, 24) + "01"; |
| 235 | + const aId = gen2("a", "mixa"); |
| 236 | + const bId = gen2("b", "mixb"); |
| 237 | + const all = [legacyId, newId, aId, bId]; |
| 238 | + |
| 239 | + for (const runId of all) { |
| 240 | + await router.createRun(buildRun({ runId, ...env })); |
| 241 | + } |
| 242 | + |
| 243 | + const found = await router.findRunsByIds(all, { select: { id: true } }); |
| 244 | + expect(new Set([...found.keys()])).toEqual(new Set(all)); |
| 245 | + } |
| 246 | + ); |
| 247 | +}); |
| 248 | + |
| 249 | +describe("RoutingRunStore four-store matrix — cross-tree completion across gen-2 shards", () => { |
| 250 | + matrixTest( |
| 251 | + "a gen-2 waitpoint completes on its own shard even under the cross-tree legacy pin", |
| 252 | + async ({ legacyPrisma, newPrisma, shardPrismas }) => { |
| 253 | + const env = await seedLegacyEnv(legacyPrisma, "crosstree"); |
| 254 | + const router = makeMatrixRouter(legacyPrisma, newPrisma, shardPrismas); |
| 255 | + // The waitpoint is owned by a run on shard b; the blocked run is on shard a (cross-tree). |
| 256 | + const wpB = gen2("b", "ctwp"); |
| 257 | + await seedPendingWaitpoint(shardPrismas[1]!, { |
| 258 | + id: wpB, |
| 259 | + projectId: env.projectId, |
| 260 | + environmentId: env.environmentId, |
| 261 | + }); |
| 262 | + // isCrossTreeIdempotency pins gen-1 flows to legacy; a gen-2 id must OVERRIDE that pin, or the |
| 263 | + // completion write lands on legacy, matches zero rows, and strands the run. |
| 264 | + const store = await router.forWaitpointCompletion(wpB, { |
| 265 | + isCrossTreeIdempotency: true, |
| 266 | + } as never); |
| 267 | + // The returned store finds wpB on its primary — only shard b holds it, so the override worked. |
| 268 | + const found = await store.findWaitpoint({ where: { id: wpB } }, store.primaryReadClient); |
| 269 | + expect(found?.id).toBe(wpB); |
| 270 | + } |
| 271 | + ); |
| 272 | +}); |
| 273 | + |
| 274 | +describe("RoutingRunStore four-store matrix — pagination merge", () => { |
| 275 | + matrixTest( |
| 276 | + "findRuns merges an open-predicate page across all four stores in orderBy order", |
| 277 | + async ({ legacyPrisma, newPrisma, shardPrismas }) => { |
| 278 | + const env = await seedLegacyEnv(legacyPrisma, "paginate"); |
| 279 | + const router = makeMatrixRouter(legacyPrisma, newPrisma, shardPrismas); |
| 280 | + // One run per store, distinct createdAt so the global sort order is unambiguous. |
| 281 | + const rows = [ |
| 282 | + { id: cuid("pgleg"), at: new Date("2024-01-01T00:00:00Z") }, |
| 283 | + { |
| 284 | + id: ("pgnew".replace(/[^0-9a-v]/g, "0") + "k".repeat(24)).slice(0, 24) + "01", |
| 285 | + at: new Date("2024-01-02T00:00:00Z"), |
| 286 | + }, |
| 287 | + { id: gen2("a", "pga"), at: new Date("2024-01-03T00:00:00Z") }, |
| 288 | + { id: gen2("b", "pgb"), at: new Date("2024-01-04T00:00:00Z") }, |
| 289 | + ]; |
| 290 | + for (const r of rows) { |
| 291 | + await router.createRun(buildRun({ runId: r.id, ...env, createdAt: r.at })); |
| 292 | + } |
| 293 | + // Open predicate (no id set) → fan out + merge; take 2 skip 1 over createdAt desc. |
| 294 | + const page = (await router.findRuns({ |
| 295 | + where: { runtimeEnvironmentId: env.runtimeEnvironmentId }, |
| 296 | + select: { id: true, createdAt: true }, |
| 297 | + orderBy: { createdAt: "desc" }, |
| 298 | + take: 2, |
| 299 | + skip: 1, |
| 300 | + })) as Array<{ id: string }>; |
| 301 | + // Global desc order is b, a, new, legacy; skip 1 take 2 → [a, new]. |
| 302 | + expect(page.map((r) => r.id)).toEqual([rows[2]!.id, rows[1]!.id]); |
| 303 | + } |
| 304 | + ); |
| 305 | +}); |
0 commit comments