|
| 1 | +// #maybeInfraRetry must charge a blip-retry to the correct budget: replica reads to the replica |
| 2 | +// budget (so a replica retry storm can't drain the writer's), writer ops to the writer budget, and a |
| 3 | +// caller-transaction client is never retried. Selection is by the stored read-only object OR the |
| 4 | +// replica brand, so a routing-layer-forwarded replica wrapper (a fresh object, not the stored one) |
| 5 | +// still bills the replica budget. Deterministic one-shot fault injection, real testcontainer DB. |
| 6 | +import { postgresTest } from "@internal/testcontainers"; |
| 7 | +import type { PrismaClient } from "@trigger.dev/database"; |
| 8 | +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; |
| 9 | +import { expect } from "vitest"; |
| 10 | +import { PostgresRunStore } from "./PostgresRunStore.js"; |
| 11 | +import { setupSnapshotIdFixture } from "./testFixtures/snapshotIdFixture.js"; |
| 12 | +import { withOneShotBlip } from "./testFixtures/oneShotBlip.js"; |
| 13 | +import { markReadReplicaClient } from "./readReplicaClient.js"; |
| 14 | + |
| 15 | +const options = { enabled: true, maxAttempts: 12, backoffMinMs: 20, backoffMaxMs: 120 }; |
| 16 | + |
| 17 | +async function seedWaitpoint(prisma: PrismaClient, env: { id: string; projectId: string }) { |
| 18 | + const id = generateInternalId(); |
| 19 | + await prisma.waitpoint.create({ |
| 20 | + data: { |
| 21 | + id, |
| 22 | + friendlyId: `wp_${id}`, |
| 23 | + type: "MANUAL", |
| 24 | + status: "PENDING", |
| 25 | + idempotencyKey: `idem_${id}`, |
| 26 | + userProvidedIdempotencyKey: false, |
| 27 | + projectId: env.projectId, |
| 28 | + environmentId: env.id, |
| 29 | + }, |
| 30 | + }); |
| 31 | + return id; |
| 32 | +} |
| 33 | + |
| 34 | +postgresTest( |
| 35 | + "a branded replica wrapper (not the stored one) charges its retry to the replica budget", |
| 36 | + async ({ prisma }) => { |
| 37 | + const { env } = await setupSnapshotIdFixture(prisma); |
| 38 | + const wpId = await seedWaitpoint(prisma as PrismaClient, env); |
| 39 | + |
| 40 | + let writer = 0; |
| 41 | + let replica = 0; |
| 42 | + const store = new PostgresRunStore({ |
| 43 | + prisma: prisma as never, |
| 44 | + readOnlyPrisma: prisma as never, |
| 45 | + infraRetry: { options, onRetry: () => writer++ }, |
| 46 | + readInfraRetry: { options, onRetry: () => replica++ }, |
| 47 | + }); |
| 48 | + |
| 49 | + // A freshly wrapped, branded replica handle: a different object than the stored readOnlyPrisma, so |
| 50 | + // the old identity-only check would have billed the writer. The brand routes it to the replica. |
| 51 | + const branded = markReadReplicaClient(withOneShotBlip(prisma, "waitpoint", "findFirst")); |
| 52 | + const wp = await store.findWaitpoint({ where: { id: wpId } }, branded as never); |
| 53 | + |
| 54 | + expect(wp?.id).toBe(wpId); |
| 55 | + expect(replica).toBe(1); |
| 56 | + expect(writer).toBe(0); |
| 57 | + } |
| 58 | +); |
| 59 | + |
| 60 | +postgresTest( |
| 61 | + "an op through the stored writer charges its retry to the writer budget", |
| 62 | + async ({ prisma }) => { |
| 63 | + const { env } = await setupSnapshotIdFixture(prisma); |
| 64 | + const wpId = await seedWaitpoint(prisma as PrismaClient, env); |
| 65 | + |
| 66 | + let writer = 0; |
| 67 | + let replica = 0; |
| 68 | + const faultingWriter = withOneShotBlip(prisma, "waitpoint", "updateMany"); |
| 69 | + const store = new PostgresRunStore({ |
| 70 | + prisma: faultingWriter as never, |
| 71 | + readOnlyPrisma: prisma as never, // distinct stored replica so identity selection is meaningful |
| 72 | + infraRetry: { options, onRetry: () => writer++ }, |
| 73 | + readInfraRetry: { options, onRetry: () => replica++ }, |
| 74 | + }); |
| 75 | + |
| 76 | + // markWaitpointCompleted is the retried writer op (updateManyWaitpoints is not retried); it runs |
| 77 | + // waitpoint.updateMany on the writer, which the one-shot blip faults once, charging the writer budget. |
| 78 | + await store.markWaitpointCompleted(wpId, { |
| 79 | + output: { value: "{}", type: "application/json", isError: false }, |
| 80 | + }); |
| 81 | + |
| 82 | + expect(writer).toBe(1); |
| 83 | + expect(replica).toBe(0); |
| 84 | + } |
| 85 | +); |
| 86 | + |
| 87 | +postgresTest( |
| 88 | + "a read through the stored replica charges its retry to the replica budget", |
| 89 | + async ({ prisma }) => { |
| 90 | + const { env } = await setupSnapshotIdFixture(prisma); |
| 91 | + const wpId = await seedWaitpoint(prisma as PrismaClient, env); |
| 92 | + |
| 93 | + let writer = 0; |
| 94 | + let replica = 0; |
| 95 | + const faultingReplica = withOneShotBlip(prisma, "waitpoint", "findFirst"); |
| 96 | + const store = new PostgresRunStore({ |
| 97 | + prisma: prisma as never, |
| 98 | + readOnlyPrisma: faultingReplica as never, |
| 99 | + infraRetry: { options, onRetry: () => writer++ }, |
| 100 | + readInfraRetry: { options, onRetry: () => replica++ }, |
| 101 | + }); |
| 102 | + |
| 103 | + const wp = await store.findWaitpoint({ where: { id: wpId } }); // no client -> stored readOnlyPrisma |
| 104 | + |
| 105 | + expect(wp?.id).toBe(wpId); |
| 106 | + expect(replica).toBe(1); |
| 107 | + expect(writer).toBe(0); |
| 108 | + } |
| 109 | +); |
| 110 | + |
| 111 | +postgresTest( |
| 112 | + "a caller-transaction client is never retried (boundary preserved for both budgets)", |
| 113 | + async ({ prisma }) => { |
| 114 | + const { env } = await setupSnapshotIdFixture(prisma); |
| 115 | + const wpId = await seedWaitpoint(prisma as PrismaClient, env); |
| 116 | + |
| 117 | + let writer = 0; |
| 118 | + let replica = 0; |
| 119 | + const faulting = withOneShotBlip(prisma, "waitpoint", "findFirst"); |
| 120 | + const store = new PostgresRunStore({ |
| 121 | + prisma: prisma as never, |
| 122 | + readOnlyPrisma: prisma as never, |
| 123 | + infraRetry: { options, onRetry: () => writer++ }, |
| 124 | + readInfraRetry: { options, onRetry: () => replica++ }, |
| 125 | + }); |
| 126 | + |
| 127 | + // A tx client has no `$transaction`, so the op runs exactly once: the blip surfaces and neither |
| 128 | + // budget's onRetry fires. |
| 129 | + await expect( |
| 130 | + (faulting as any).$transaction((tx: any) => store.findWaitpoint({ where: { id: wpId } }, tx)) |
| 131 | + ).rejects.toThrow(); |
| 132 | + |
| 133 | + expect(writer).toBe(0); |
| 134 | + expect(replica).toBe(0); |
| 135 | + } |
| 136 | +); |
0 commit comments