Skip to content

Commit 9d4ee02

Browse files
ericallamericallam
committed
perf(run-engine,webapp): narrow the control-plane BackgroundWorkerTask read to the columns dequeue uses
The worker-version resolve path fetched every column of every BackgroundWorkerTask for a worker (include: { tasks: true }) just to match one task at dequeue, pulling large JSON columns this path never reads (payloadSchema, config, queueConfig, description). Replace it with an explicit select of only the columns dequeue uses (id, slug, machineConfig, retryConfig, maxDurationInSeconds) in both the passthrough resolver and the app resolver. The shared ResolvedWorkerVersion.tasks element type becomes a narrow ResolvedWorkerTask. No behavior change.
1 parent fe199f7 commit 9d4ee02

6 files changed

Lines changed: 136 additions & 17 deletions

File tree

‎apps/webapp/app/v3/runOpsMigration/controlPlaneCache.server.ts‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
import type {
22
BackgroundWorker,
3-
BackgroundWorkerTask,
43
Prisma,
54
RuntimeEnvironmentType,
65
TaskQueue,
@@ -49,10 +48,34 @@ export type ResolvedEnv = {
4948
concurrencyLimitBurstFactor: Prisma.Decimal;
5049
};
5150

51+
/**
52+
* The BackgroundWorkerTask columns the dequeue resolve path reads. Mirrors run-engine's
53+
* `ResolvedWorkerTask` exactly. The unread heavy JSON columns (`payloadSchema`, `config`,
54+
* `queueConfig`, `description`) are dropped so this hot control-plane read stops shipping
55+
* ~62KB/query (and each cached entry stays small); `machineConfig`/`retryConfig` are read
56+
* at dequeue and stay.
57+
*/
58+
export type ResolvedWorkerTask = {
59+
id: string;
60+
slug: string;
61+
machineConfig: Prisma.JsonValue | null;
62+
retryConfig: Prisma.JsonValue | null;
63+
maxDurationInSeconds: number | null;
64+
};
65+
66+
/** The `select` that yields a `ResolvedWorkerTask`. */
67+
export const resolvedWorkerTaskSelect = {
68+
id: true,
69+
slug: true,
70+
machineConfig: true,
71+
retryConfig: true,
72+
maxDurationInSeconds: true,
73+
} satisfies Prisma.BackgroundWorkerTaskSelect;
74+
5275
/** Mirrors `WorkerDeploymentWithWorkerTasks` in `dequeueSystem.ts` exactly. */
5376
export type ResolvedWorkerVersion = {
5477
worker: BackgroundWorker;
55-
tasks: BackgroundWorkerTask[];
78+
tasks: ResolvedWorkerTask[];
5679
queues: TaskQueue[];
5780
deployment: WorkerDeployment | null;
5881
};

‎apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.ts‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import {
1010
ControlPlaneCache,
1111
DEFAULT_CP_CACHE_MAX_ENTRIES,
1212
DEFAULT_CP_CACHE_TTL_MS,
13+
resolvedWorkerTaskSelect,
1314
type ResolvedAuthenticatedEnv,
1415
type ResolvedEnv,
1516
type ResolvedWorkerVersion,
@@ -389,7 +390,11 @@ export class ControlPlaneResolver {
389390
if (backgroundWorkerId) {
390391
const worker = await client.backgroundWorker.findFirst({
391392
where: { id: backgroundWorkerId },
392-
include: { deployment: true, tasks: true, queues: true },
393+
include: {
394+
deployment: true,
395+
tasks: { select: resolvedWorkerTaskSelect },
396+
queues: true,
397+
},
393398
});
394399

395400
if (!worker) {
@@ -411,7 +416,9 @@ export class ControlPlaneResolver {
411416
where: { environmentId, label: CURRENT_DEPLOYMENT_LABEL },
412417
include: {
413418
deployment: {
414-
include: { worker: { include: { tasks: true, queues: true } } },
419+
include: {
420+
worker: { include: { tasks: { select: resolvedWorkerTaskSelect }, queues: true } },
421+
},
415422
},
416423
},
417424
});
@@ -434,7 +441,9 @@ export class ControlPlaneResolver {
434441
const latestV2Deployment = await client.workerDeployment.findFirst({
435442
where: { environmentId, type: "MANAGED" },
436443
orderBy: [{ createdAt: "desc" }, { id: "desc" }],
437-
include: { worker: { include: { tasks: true, queues: true } } },
444+
include: {
445+
worker: { include: { tasks: { select: resolvedWorkerTaskSelect }, queues: true } },
446+
},
438447
});
439448

440449
if (!latestV2Deployment?.worker) {
@@ -455,7 +464,11 @@ export class ControlPlaneResolver {
455464
): Promise<ResolvedWorkerVersion | null> {
456465
const worker = await client.backgroundWorker.findFirst({
457466
where: { id: workerId },
458-
include: { deployment: true, tasks: true, queues: true },
467+
include: {
468+
deployment: true,
469+
tasks: { select: resolvedWorkerTaskSelect },
470+
queues: true,
471+
},
459472
});
460473

461474
if (!worker) {
@@ -471,7 +484,7 @@ export class ControlPlaneResolver {
471484
): Promise<ResolvedWorkerVersion | null> {
472485
const worker = await client.backgroundWorker.findFirst({
473486
where: { runtimeEnvironmentId: environmentId },
474-
include: { tasks: true, queues: true },
487+
include: { tasks: { select: resolvedWorkerTaskSelect }, queues: true },
475488
orderBy: [{ createdAt: "desc" }, { id: "desc" }],
476489
});
477490

‎internal-packages/run-engine/src/engine/controlPlaneResolver.ts‎

Lines changed: 29 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
import type {
22
BackgroundWorker,
3-
BackgroundWorkerTask,
43
Prisma,
54
PrismaClient,
65
RuntimeEnvironmentType,
@@ -51,10 +50,33 @@ export type ResolvedEngineEnv = {
5150
*/
5251
export type ResolvedAuthenticatedEnv = AuthenticatedEnvironment & { git: Prisma.JsonValue | null };
5352

53+
/**
54+
* The BackgroundWorkerTask columns the dequeue resolve path actually reads. The worker's
55+
* whole task set (~73 rows) is fetched for one matched task, so pulling the unread heavy
56+
* JSON columns (`payloadSchema`, `config`, `queueConfig`, `description`) shipped ~62KB/query
57+
* on the hottest control-plane read. `machineConfig`/`retryConfig` are read at dequeue and stay.
58+
*/
59+
export type ResolvedWorkerTask = {
60+
id: string;
61+
slug: string;
62+
machineConfig: Prisma.JsonValue | null;
63+
retryConfig: Prisma.JsonValue | null;
64+
maxDurationInSeconds: number | null;
65+
};
66+
67+
/** The `select` that yields a `ResolvedWorkerTask`. */
68+
export const resolvedWorkerTaskSelect = {
69+
id: true,
70+
slug: true,
71+
machineConfig: true,
72+
retryConfig: true,
73+
maxDurationInSeconds: true,
74+
} satisfies Prisma.BackgroundWorkerTaskSelect;
75+
5476
/** Identical to dequeue's `WorkerDeploymentWithWorkerTasks`. */
5577
export type ResolvedWorkerVersion = {
5678
worker: BackgroundWorker;
57-
tasks: BackgroundWorkerTask[];
79+
tasks: ResolvedWorkerTask[];
5880
queues: TaskQueue[];
5981
deployment: WorkerDeployment | null;
6082
};
@@ -208,7 +230,7 @@ export class PassthroughControlPlaneResolver implements ControlPlaneResolver {
208230
},
209231
include: {
210232
deployment: true,
211-
tasks: true,
233+
tasks: { select: resolvedWorkerTaskSelect },
212234
queues: true,
213235
},
214236
});
@@ -231,7 +253,7 @@ export class PassthroughControlPlaneResolver implements ControlPlaneResolver {
231253
runtimeEnvironmentId: environmentId,
232254
},
233255
include: {
234-
tasks: true,
256+
tasks: { select: resolvedWorkerTaskSelect },
235257
queues: true,
236258
},
237259
orderBy: [{ createdAt: "desc" }, { id: "desc" }],
@@ -251,7 +273,7 @@ export class PassthroughControlPlaneResolver implements ControlPlaneResolver {
251273
},
252274
include: {
253275
deployment: true,
254-
tasks: true,
276+
tasks: { select: resolvedWorkerTaskSelect },
255277
queues: true,
256278
},
257279
});
@@ -281,7 +303,7 @@ export class PassthroughControlPlaneResolver implements ControlPlaneResolver {
281303
include: {
282304
worker: {
283305
include: {
284-
tasks: true,
306+
tasks: { select: resolvedWorkerTaskSelect },
285307
queues: true,
286308
},
287309
},
@@ -314,7 +336,7 @@ export class PassthroughControlPlaneResolver implements ControlPlaneResolver {
314336
include: {
315337
worker: {
316338
include: {
317-
tasks: true,
339+
tasks: { select: resolvedWorkerTaskSelect },
318340
queues: true,
319341
},
320342
},

‎internal-packages/run-engine/src/engine/systems/dequeueSystem.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,14 @@ import { generateInternalId, getMaxDuration, SnapshotId } from "@trigger.dev/cor
66
import { placementTag } from "@trigger.dev/core/v3/serverOnly";
77
import type {
88
BackgroundWorker,
9-
BackgroundWorkerTask,
109
Prisma,
1110
PrismaClientOrTransaction,
1211
RuntimeEnvironmentType,
1312
TaskQueue,
1413
WorkerDeployment,
1514
} from "@trigger.dev/database";
1615
import type { BillingCache } from "../billingCache.js";
16+
import type { ResolvedWorkerTask } from "../controlPlaneResolver.js";
1717

1818
import { sendNotificationToWorker } from "../eventBus.js";
1919
import { getMachinePreset } from "../machinePresets.js";
@@ -88,14 +88,14 @@ type RunWithBackgroundWorkerTasksResult =
8888
run: RunWithDequeueScalars;
8989
environmentType: RuntimeEnvironmentType;
9090
worker: BackgroundWorker;
91-
task: BackgroundWorkerTask;
91+
task: ResolvedWorkerTask;
9292
queue: TaskQueue;
9393
deployment: WorkerDeployment | null;
9494
};
9595

9696
type WorkerDeploymentWithWorkerTasks = {
9797
worker: BackgroundWorker;
98-
tasks: BackgroundWorkerTask[];
98+
tasks: ResolvedWorkerTask[];
9999
queues: TaskQueue[];
100100
deployment: WorkerDeployment | null;
101101
};

‎internal-packages/run-engine/src/engine/tests/dequeueSystem.controlPlaneResolver.test.ts‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -487,6 +487,66 @@ describe("DequeueSystem controlPlaneResolver (latest-v2 fallback + workerId bran
487487
);
488488
});
489489

490+
describe("DequeueSystem controlPlaneResolver (worker-task read shape)", () => {
491+
/**
492+
* The worker's whole task set is fetched to match one task, so the resolver must ship only
493+
* the BackgroundWorkerTask columns the dequeue path reads (id/slug + machineConfig/retryConfig/
494+
* maxDurationInSeconds). The heavy JSON columns it never reads (payloadSchema/config/queueConfig/
495+
* description) must NOT come back, so the hot control-plane read stops shipping ~62KB/query.
496+
*/
497+
containerTest(
498+
"resolveWorkerVersion returns only the dequeue-read task columns and drops the heavy JSON columns",
499+
async ({ prisma }) => {
500+
const taskSlug = "test-task";
501+
const cp = await seedControlPlane(prisma as unknown as PrismaClient, "slim", taskSlug);
502+
503+
await prisma.backgroundWorkerTask.update({
504+
where: { id: cp.task.id },
505+
data: {
506+
machineConfig: { preset: "small-2x" },
507+
retryConfig: { maxAttempts: 5, factor: 2, minTimeoutInMs: 100, maxTimeoutInMs: 1000 },
508+
maxDurationInSeconds: 120,
509+
payloadSchema: { type: "object", properties: { message: { type: "string" } } },
510+
config: { type: "ai-sdk-chat" },
511+
queueConfig: { concurrencyLimit: 7 },
512+
description: "must not be shipped on the dequeue read",
513+
},
514+
});
515+
516+
const resolver = new PassthroughControlPlaneResolver({
517+
prisma: prisma as unknown as PrismaClient,
518+
});
519+
520+
const version = await resolver.resolveWorkerVersion({
521+
environmentId: cp.environment.id,
522+
type: "PRODUCTION",
523+
});
524+
525+
assertNonNullable(version);
526+
const task = version.tasks.find((t) => t.slug === taskSlug);
527+
assertNonNullable(task);
528+
529+
expect(task.id).toBe(cp.task.id);
530+
expect(task.slug).toBe(taskSlug);
531+
expect(task.machineConfig).toEqual({ preset: "small-2x" });
532+
expect(task.retryConfig).toMatchObject({ maxAttempts: 5 });
533+
expect(task.maxDurationInSeconds).toBe(120);
534+
535+
expect("payloadSchema" in task).toBe(false);
536+
expect("config" in task).toBe(false);
537+
expect("queueConfig" in task).toBe(false);
538+
expect("description" in task).toBe(false);
539+
expect(Object.keys(task).sort()).toEqual([
540+
"id",
541+
"machineConfig",
542+
"maxDurationInSeconds",
543+
"retryConfig",
544+
"slug",
545+
]);
546+
}
547+
);
548+
});
549+
490550
describe("DequeueSystem controlPlaneResolver (single-DB passthrough)", () => {
491551
containerTest(
492552
"default passthrough dequeue is byte-identical (resolves env + worker version end-to-end)",

‎internal-packages/run-engine/src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ export type {
1818
ResolvedEngineEnv,
1919
ResolvedAuthenticatedEnv,
2020
ResolvedWorkerVersion,
21+
ResolvedWorkerTask,
2122
} from "./engine/controlPlaneResolver.js";
2223

2324
// Batch Queue exports

0 commit comments

Comments
 (0)