Skip to content

Commit 8fbc76f

Browse files
1stvampTrigger.dev RepoOps
authored andcommitted
fix(webapp): report a template key that was not written as failed
The admin template-builds route now reports a deployment as failed, with a reason, when the template key could not be stored. Mono-RevId: 52d799b6e54eefe3ba3a9c2125092dbf697ee5ed
1 parent b88c7ff commit 8fbc76f

3 files changed

Lines changed: 239 additions & 29 deletions

File tree

‎apps/webapp/app/v3/services/templatePrebuild.server.test.ts‎

Lines changed: 137 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -277,10 +277,13 @@ describe("prebuild", () => {
277277
workloadType: "MICROVM",
278278
orgFlags: { [FEATURE_FLAG.computeTemplatePrebuildWait]: true },
279279
});
280+
const row = await createDeployment(prisma, project);
280281
const service = new TemplatePrebuildService(prisma, config({ url: server.url }));
281282
const result = await service.prebuild({
282-
...deployment,
283283
projectId: project.id,
284+
deploymentId: row.id,
285+
deploymentFriendlyId: row.friendlyId,
286+
imageReference: IMAGE,
284287
imageDigest: DIGEST,
285288
});
286289

@@ -291,7 +294,7 @@ describe("prebuild", () => {
291294
image: `${IMAGE}@${DIGEST}`,
292295
machine_configs: [{ cpus: 0.5, memory_mb: 512 }],
293296
no_snap: true,
294-
deployment_id: "deployment_abc",
297+
deployment_id: row.friendlyId,
295298
});
296299
} finally {
297300
await server.close();
@@ -385,20 +388,23 @@ describe("prebuild", () => {
385388
});
386389

387390
postgresTest("does not wait on a CLI that stops reading logs", async ({ prisma }) => {
388-
const server = await builder(reply(201, { template_source: "built" }));
391+
const server = await builder(reply(201, { template_id: TEMPLATE_KEY }));
389392
try {
390393
const project = await createProject(prisma, {
391394
workloadType: "MICROVM",
392395
orgFlags: { [FEATURE_FLAG.computeTemplatePrebuildWait]: true },
393396
});
397+
const row = await createDeployment(prisma, project);
394398
// A sink that never accepts a chunk, so each write stays pending.
395399
const writer = new WritableStream<string>({
396400
write: () => new Promise<void>(() => {}),
397401
}).getWriter();
398402
const service = new TemplatePrebuildService(prisma, config({ url: server.url }));
399403
const result = await service.prebuild({
400-
...deployment,
401404
projectId: project.id,
405+
deploymentId: row.id,
406+
deploymentFriendlyId: row.friendlyId,
407+
imageReference: IMAGE,
402408
imageDigest: DIGEST,
403409
writer,
404410
});
@@ -454,13 +460,17 @@ describe("prebuild", () => {
454460
workloadType: "MICROVM",
455461
orgFlags: { [FEATURE_FLAG.computeTemplatePrebuildWait]: true },
456462
});
463+
const row = await createDeployment(prisma, project);
457464
const service = new TemplatePrebuildService(prisma, config({ url: server.url }));
458465
const result = await service.prebuild({
459-
...deployment,
460466
projectId: project.id,
467+
deploymentId: row.id,
468+
deploymentFriendlyId: row.friendlyId,
469+
imageReference: IMAGE,
461470
imageDigest: DIGEST,
462471
});
463472
expect(result.outcome).toBe("already_there");
473+
expect(await storedTemplateKey(prisma, row.id)).toBe("a".repeat(64));
464474
} finally {
465475
await server.close();
466476
}
@@ -542,10 +552,11 @@ describe("prebuild", () => {
542552
imageReference: `${IMAGE}@sha256:${"b".repeat(64)}`,
543553
});
544554
const service = new TemplatePrebuildService(prisma, config({ url: server.url }));
545-
await service.build(`${IMAGE}@${DIGEST}`, "shadow", {
555+
const result = await service.build(`${IMAGE}@${DIGEST}`, "shadow", {
546556
id: row.id,
547557
friendlyId: row.friendlyId,
548558
});
559+
expect(result).toEqual({ outcome: "failed", reason: "not_stored" });
549560
expect(await storedTemplateKey(prisma, row.id)).toBeNull();
550561
} finally {
551562
await server.close();
@@ -591,7 +602,8 @@ describe("prebuild", () => {
591602
imageReference: IMAGE,
592603
imageDigest: DIGEST,
593604
});
594-
expect(result.outcome).toBe("built");
605+
// The key is not known to be stored yet, so the caller sees a timeout.
606+
expect(result.outcome).toBe("timeout");
595607
expect(Date.now() - startedAt).toBeLessThan(5_000);
596608

597609
release!();
@@ -628,7 +640,7 @@ describe("prebuild", () => {
628640
imageReference: IMAGE,
629641
imageDigest: DIGEST,
630642
});
631-
expect(result.outcome).toBe("built");
643+
expect(result.outcome).toBe("failed");
632644
}
633645
expect(await storedTemplateKey(prisma, fresh.id)).toBeNull();
634646
expect(await storedTemplateKey(prisma, existing.id)).toBe(TEMPLATE_KEY);
@@ -771,6 +783,123 @@ describe("requestBuilds", () => {
771783
}
772784
});
773785

786+
postgresTest(
787+
"reports a stored key the mirror already had as already_there",
788+
async ({ prisma }) => {
789+
const server = await builder(
790+
reply(200, { template_id: TEMPLATE_KEY, template_source: "mirror" })
791+
);
792+
try {
793+
const project = await createProject(prisma);
794+
const row = await createDeployment(prisma, project, {
795+
status: "DEPLOYED",
796+
imageReference: PINNED,
797+
});
798+
const service = new TemplatePrebuildService(prisma, config({ url: server.url }));
799+
800+
const [result] = await service.requestBuilds({ deploymentIds: [row.id] });
801+
802+
expect(result).toEqual({
803+
deploymentId: row.id,
804+
friendlyId: row.friendlyId,
805+
outcome: "already_there",
806+
});
807+
expect(await storedTemplateKey(prisma, row.id)).toBe(TEMPLATE_KEY);
808+
} finally {
809+
await server.close();
810+
}
811+
}
812+
);
813+
814+
postgresTest("reports an invalid template id as failed", async ({ prisma }) => {
815+
const server = await builder(reply(200, { template_id: "not-a-key" }));
816+
try {
817+
const project = await createProject(prisma);
818+
const row = await createDeployment(prisma, project, {
819+
status: "DEPLOYED",
820+
imageReference: PINNED,
821+
});
822+
const service = new TemplatePrebuildService(prisma, config({ url: server.url }));
823+
824+
const [result] = await service.requestBuilds({ deploymentIds: [row.id] });
825+
826+
expect(result).toEqual({
827+
deploymentId: row.id,
828+
friendlyId: row.friendlyId,
829+
outcome: "failed",
830+
reason: "invalid_template_id",
831+
});
832+
expect(await storedTemplateKey(prisma, row.id)).toBeNull();
833+
} finally {
834+
await server.close();
835+
}
836+
});
837+
838+
postgresTest("reports failed when the store guard matches no row", async ({ prisma }) => {
839+
const project = await createProject(prisma);
840+
const row = await createDeployment(prisma, project, {
841+
status: "DEPLOYED",
842+
imageReference: PINNED,
843+
});
844+
// The deployment's image changes while the builder works, so the guard rejects the write.
845+
const server = await builder((req, body, res) => {
846+
prisma.workerDeployment
847+
.update({
848+
where: { id: row.id },
849+
data: { imageReference: `${IMAGE}@sha256:${"b".repeat(64)}` },
850+
})
851+
.then(() => reply(200, { template_id: TEMPLATE_KEY })(req, body, res));
852+
});
853+
try {
854+
const service = new TemplatePrebuildService(prisma, config({ url: server.url }));
855+
856+
const [result] = await service.requestBuilds({ deploymentIds: [row.id] });
857+
858+
expect(result).toEqual({
859+
deploymentId: row.id,
860+
friendlyId: row.friendlyId,
861+
outcome: "failed",
862+
reason: "not_stored",
863+
});
864+
expect(await storedTemplateKey(prisma, row.id)).toBeNull();
865+
} finally {
866+
await server.close();
867+
}
868+
});
869+
870+
postgresTest("reports failed when the key write errors", async ({ prisma }) => {
871+
const server = await builder(reply(200, { template_id: TEMPLATE_KEY }));
872+
try {
873+
const project = await createProject(prisma);
874+
const row = await createDeployment(prisma, project, {
875+
status: "DEPLOYED",
876+
imageReference: PINNED,
877+
});
878+
879+
// A read-only transaction lets the lookup through but makes the update fail.
880+
const results = await $transaction(
881+
prisma,
882+
"readOnlyTemplateStore",
883+
async (tx) => {
884+
await tx.$executeRaw`SET TRANSACTION READ ONLY`;
885+
const service = new TemplatePrebuildService(tx, config({ url: server.url }));
886+
return service.requestBuilds({ deploymentIds: [row.id] });
887+
},
888+
{ timeout: 30_000 }
889+
);
890+
891+
expect(results?.[0]).toEqual({
892+
deploymentId: row.id,
893+
friendlyId: row.friendlyId,
894+
outcome: "failed",
895+
reason: "store_error",
896+
});
897+
expect(await storedTemplateKey(prisma, row.id)).toBeNull();
898+
} finally {
899+
await server.close();
900+
}
901+
});
902+
774903
postgresTest("skips every deployment when no builder is configured", async ({ prisma }) => {
775904
const project = await createProject(prisma);
776905
const row = await createDeployment(prisma, project, {

‎apps/webapp/app/v3/services/templatePrebuild.server.ts‎

Lines changed: 57 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,18 @@ export type TemplateBuildOutcome =
3737

3838
type TemplateBuildRequestSkip = "no_builder" | "has_key" | "no_pinned_image";
3939

40+
/** Why a 2xx from the builder still ended as `failed`: the key was not written. */
41+
export type TemplateKeyStoreFailure = "invalid_template_id" | "not_stored" | "store_error";
42+
43+
export type TemplateBuildResult =
44+
| { outcome: Exclude<TemplateBuildOutcome, "failed"> }
45+
| { outcome: "failed"; reason?: TemplateKeyStoreFailure };
46+
4047
export type TemplateBuildRequestResult = {
4148
deploymentId: string;
4249
friendlyId?: string;
4350
} & (
44-
| { outcome: TemplateBuildOutcome }
51+
| TemplateBuildResult
4552
| { outcome: "skipped"; reason: TemplateBuildRequestSkip }
4653
| { outcome: "not_found" }
4754
);
@@ -183,7 +190,7 @@ export class TemplatePrebuildService {
183190
`event: log\ndata: ${JSON.stringify({ message: "Building compute template..." })}\n\n`
184191
)
185192
.catch(() => {});
186-
return { mode, reason, outcome: await build };
193+
return { mode, reason, outcome: (await build).outcome };
187194
} catch (error) {
188195
logger.error("Template prebuild failed before calling the builder", {
189196
deploymentId: options.deploymentFriendlyId,
@@ -324,27 +331,28 @@ export class TemplatePrebuildService {
324331
if (!image) {
325332
return { ...base, outcome: "skipped", reason: "no_pinned_image" };
326333
}
327-
const outcome = await this.build(image, "shadow", {
334+
const result = await this.build(image, "shadow", {
328335
id: deployment.id,
329336
friendlyId: deployment.friendlyId,
330337
});
331-
return { ...base, outcome };
338+
return { ...base, ...result };
332339
}
333340

334341
/**
335342
* Call the builder once and store the template key on success. Resolves with
336-
* the outcome; never rejects.
343+
* the outcome, which is `failed` when the key was not written; never rejects.
337344
*/
338345
async build(
339346
image: string,
340347
mode: Exclude<TemplatePrebuildMode, "skip">,
341348
deployment: { id: string; friendlyId: string }
342-
): Promise<TemplateBuildOutcome> {
349+
): Promise<TemplateBuildResult> {
343350
const startedAt = performance.now();
344351
const body = templateCreateRequest(image, this.config.presets, deployment.friendlyId);
345352
const log = { deploymentId: deployment.friendlyId, image, mode };
346353

347354
let outcome: TemplateBuildOutcome;
355+
let reason: TemplateKeyStoreFailure | undefined;
348356
try {
349357
const response = await fetch(new URL("/templates", this.config.url).toString(), {
350358
method: "POST",
@@ -372,10 +380,22 @@ export class TemplatePrebuildService {
372380
squashfsMs: parsed?.squashfs_ms,
373381
});
374382
// The write keeps going if it outlasts the timeout, but wait mode stops waiting.
375-
await withinBudget(
376-
this.storeTemplateKey(deployment.id, image, parsed?.template_id, log),
377-
this.config.timeoutMs - (performance.now() - startedAt)
383+
const remaining = () => this.config.timeoutMs - (performance.now() - startedAt);
384+
const write = this.storeTemplateKey(deployment.id, image, parsed?.template_id, log);
385+
const cached = write.then((result) =>
386+
result === "stored"
387+
? this.cacheTemplateKey(deployment.id, parsed?.template_id as string, log)
388+
: undefined
378389
);
390+
const stored = await withinBudget(write, remaining());
391+
// The cache is best effort, so a slow cache never changes the outcome.
392+
await withinBudget(cached, remaining());
393+
if (stored === undefined) {
394+
outcome = "timeout";
395+
} else if (stored !== "stored") {
396+
outcome = "failed";
397+
reason = stored;
398+
}
379399
} else {
380400
outcome = response.status === 429 ? "rate_limited" : "failed";
381401
logger.error("Template prebuild failed", {
@@ -398,7 +418,7 @@ export class TemplatePrebuildService {
398418
const attributes = { outcome, mode };
399419
telemetry.calls.add(1, attributes);
400420
telemetry.duration.record(performance.now() - startedAt, attributes);
401-
return outcome;
421+
return outcome === "failed" ? { outcome, reason } : { outcome };
402422
}
403423

404424
// In shadow mode the deployment may already be DEPLOYED; then only store the key
@@ -408,14 +428,14 @@ export class TemplatePrebuildService {
408428
image: string,
409429
templateId: unknown,
410430
log: Record<string, unknown>
411-
): Promise<void> {
431+
): Promise<"stored" | TemplateKeyStoreFailure> {
412432
const templateKey = validTemplateKey(templateId);
413433
if (!templateKey) {
414434
logger.warn("Template prebuild returned no valid template id", {
415435
...log,
416436
templateId: typeof templateId === "string" ? templateId.slice(0, 100) : templateId,
417437
});
418-
return;
438+
return "invalid_template_id";
419439
}
420440
try {
421441
const { count } = await this.prisma.workerDeployment.updateMany({
@@ -424,8 +444,25 @@ export class TemplatePrebuildService {
424444
});
425445
if (count === 0) {
426446
logger.warn("Template prebuild did not store the template key", { ...log, templateKey });
427-
return;
447+
return "not_stored";
428448
}
449+
} catch (error) {
450+
logger.error("Template prebuild could not store the template key", {
451+
...log,
452+
error: error instanceof Error ? error.message : String(error),
453+
});
454+
return "store_error";
455+
}
456+
return "stored";
457+
}
458+
459+
// The database is the source of truth; the cache refills from it.
460+
private async cacheTemplateKey(
461+
deploymentId: string,
462+
templateKey: string,
463+
log: Record<string, unknown>
464+
): Promise<void> {
465+
try {
429466
const deployment = await this.prisma.workerDeployment.findFirst({
430467
where: { id: deploymentId },
431468
select: { workerId: true, environmentId: true },
@@ -438,23 +475,25 @@ export class TemplatePrebuildService {
438475
);
439476
}
440477
} catch (error) {
441-
logger.error("Template prebuild could not store the template key", {
478+
logger.error("Template prebuild could not cache the template key", {
442479
...log,
443480
error: error instanceof Error ? error.message : String(error),
444481
});
445482
}
446483
}
447484
}
448485

449-
async function withinBudget(work: Promise<void>, ms: number): Promise<void> {
486+
/** The work's result, or undefined when the budget runs out first. */
487+
async function withinBudget<T>(work: Promise<T>, ms: number): Promise<T | undefined> {
450488
let timer: NodeJS.Timeout | undefined;
451-
await Promise.race([
489+
const result = await Promise.race([
452490
work,
453-
new Promise<void>((resolve) => {
454-
timer = setTimeout(resolve, Math.max(0, ms));
491+
new Promise<undefined>((resolve) => {
492+
timer = setTimeout(() => resolve(undefined), Math.max(0, ms));
455493
}),
456494
]);
457495
clearTimeout(timer);
496+
return result;
458497
}
459498

460499
function parseResponse(text: string): TemplateCreateResponse | undefined {

0 commit comments

Comments
 (0)