From 76a183a4fcf04b41a3e88d7b9903259779dd8513 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:37:10 +0100 Subject: [PATCH 01/21] fix: retry failed Loom imports and temporary source failures --- .../__tests__/unit/loom-download-url.test.ts | 110 +++++++++++++ apps/web/__tests__/unit/loom-import.test.ts | 66 +++++++- .../unit/loom-retry-processing.test.ts | 140 ++++++++++++++++ .../unit/media-processing-workflows.test.ts | 23 +++ apps/web/actions/loom.ts | 150 +++++++++-------- apps/web/actions/video/retry-processing.ts | 34 +++- apps/web/lib/loom-download-url.ts | 151 ++++++++++++++++++ apps/web/workflows/import-loom-video.ts | 99 ++++-------- 8 files changed, 623 insertions(+), 150 deletions(-) create mode 100644 apps/web/__tests__/unit/loom-download-url.test.ts create mode 100644 apps/web/__tests__/unit/loom-retry-processing.test.ts create mode 100644 apps/web/lib/loom-download-url.ts diff --git a/apps/web/__tests__/unit/loom-download-url.test.ts b/apps/web/__tests__/unit/loom-download-url.test.ts new file mode 100644 index 00000000000..e75657f526d --- /dev/null +++ b/apps/web/__tests__/unit/loom-download-url.test.ts @@ -0,0 +1,110 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + getLoomDownloadUrl, + getReusableLoomDownloadUrl, + LoomDownloadTemporaryError, +} from "@/lib/loom-download-url"; + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("getLoomDownloadUrl", () => { + it("honors rate limits without making more endpoint requests", async () => { + const fetch = vi.fn().mockResolvedValue( + new Response(null, { + status: 429, + headers: { "Retry-After": "120" }, + }), + ); + vi.stubGlobal("fetch", fetch); + await expect(getLoomDownloadUrl("video")).rejects.toMatchObject({ + retryAfterMs: 120_000, + }); + expect(fetch).toHaveBeenCalledOnce(); + }); + + it("treats network and server failures as temporary", async () => { + const fetch = vi.fn().mockRejectedValueOnce(new Error("Timeout")); + vi.stubGlobal("fetch", fetch); + await expect(getLoomDownloadUrl("video")).rejects.toBeInstanceOf( + LoomDownloadTemporaryError, + ); + fetch.mockResolvedValueOnce(new Response(null, { status: 503 })); + await expect(getLoomDownloadUrl("video")).rejects.toMatchObject({ + retryAfterMs: 60_000, + }); + }); + + it("does not classify an unavailable source as a temporary failure", async () => { + const fetch = vi + .fn() + .mockResolvedValue(new Response(null, { status: 204 })); + vi.stubGlobal("fetch", fetch); + await expect(getLoomDownloadUrl("video")).resolves.toBeNull(); + expect(fetch).toHaveBeenCalledTimes(4); + }); + + it("uses an available stream when a later endpoint is rate limited", async () => { + const fetch = vi + .fn() + .mockResolvedValueOnce( + Response.json({ + url: "https://cdn.loom.com/video.m3u8?Signature=token", + }), + ) + .mockResolvedValueOnce(new Response(null, { status: 429 })); + vi.stubGlobal("fetch", fetch); + await expect(getLoomDownloadUrl("video")).resolves.toBe( + "https://cdn.loom.com/video.m3u8?Signature=token", + ); + expect(fetch).toHaveBeenCalledTimes(2); + }); + + it("prefers a direct file over an earlier stream", async () => { + const fetch = vi + .fn() + .mockResolvedValueOnce( + Response.json({ url: "https://cdn.loom.com/video.m3u8" }), + ) + .mockResolvedValueOnce( + Response.json({ url: "https://cdn.loom.com/original.mp4" }), + ); + vi.stubGlobal("fetch", fetch); + await expect(getLoomDownloadUrl("video")).resolves.toBe( + "https://cdn.loom.com/original.mp4", + ); + }); +}); + +describe("getReusableLoomDownloadUrl", () => { + it("reuses a source with enough signed URL lifetime", () => { + const expires = Math.floor(Date.now() / 1000) + 7200; + const url = `https://cdn.loom.com/video.m3u8?Expires=${expires}`; + expect(getReusableLoomDownloadUrl(url)).toBe(url); + const policy = Buffer.from( + JSON.stringify({ + Statement: [ + { Condition: { DateLessThan: { "AWS:EpochTime": expires } } }, + ], + }), + ) + .toString("base64") + .replace(/\+/g, "-") + .replace(/=/g, "_") + .replace(/\//g, "~"); + const policyUrl = `https://cdn.loom.com/video.m3u8?Policy=${policy}`; + expect(getReusableLoomDownloadUrl(policyUrl)).toBe(policyUrl); + }); + + it.each([ + undefined, + "https://cdn.loom.com/video.m3u8?Expires=1", + "https://cdn.loom.com/video.m3u8?Policy=invalid", + "https://other.example/video.mp4", + "http://cdn.loom.com/video.mp4", + "https://cdn.loom.com/video.m3u8?Signature=unknown", + ])("refreshes stale or unrecognized sources: %s", (url) => { + expect(getReusableLoomDownloadUrl(url)).toBeNull(); + }); +}); diff --git a/apps/web/__tests__/unit/loom-import.test.ts b/apps/web/__tests__/unit/loom-import.test.ts index fc12562264a..05c135249be 100644 --- a/apps/web/__tests__/unit/loom-import.test.ts +++ b/apps/web/__tests__/unit/loom-import.test.ts @@ -1,3 +1,4 @@ +import type { Organisation } from "@cap/web-domain"; import { Effect, Option } from "effect"; import { beforeEach, describe, expect, it, vi } from "vitest"; @@ -111,6 +112,9 @@ vi.mock("@cap/database/schema", () => ({ }, videoUploads: { videoId: "videoId", + phase: "phase", + rawFileKey: "rawFileKey", + updatedAt: "updatedAt", }, })); @@ -256,6 +260,66 @@ describe("importFromLoom", () => { vi.stubGlobal("fetch", vi.fn()); }); + it("restarts a failed import without creating another video", async () => { + whereMock + .mockResolvedValueOnce([ + { + videoId: "existing-video", + ownerId: "user-123", + bucketId: "bucket-1", + phase: "error", + rawFileKey: null, + uploadUpdatedAt: new Date("2026-09-01T00:00:00Z"), + }, + ]) + .mockResolvedValueOnce([{ affectedRows: 1 }]); + const { importFromLoom } = await import("@/actions/loom"); + await expect( + importFromLoom({ + loomUrl: "https://www.loom.com/share/loom-abc1234567", + orgId: "org-1" as Organisation.OrganisationId, + }), + ).resolves.toEqual({ success: true, videoId: "existing-video" }); + expect(mockDb.insert).not.toHaveBeenCalled(); + expect(startMock).toHaveBeenCalledWith(expect.anything(), [ + { + videoId: "existing-video", + userId: "user-123", + bucketId: "bucket-1", + loomVideoId: "loom-abc1234567", + rawFileKey: "user-123/existing-video/raw-upload.mp4", + reuseExistingRawUpload: true, + }, + ]); + }); + + it.each([ + { ownerId: "user-123", phase: "processing", affectedRows: 1 }, + { ownerId: "other-user", phase: "error", affectedRows: 1 }, + { ownerId: "user-123", phase: "error", affectedRows: 0 }, + ])("does not restart an unclaimed or foreign import: %j", async (row) => { + whereMock + .mockResolvedValueOnce([ + { + videoId: "existing-video", + bucketId: "bucket-1", + rawFileKey: null, + uploadUpdatedAt: new Date("2026-09-01T00:00:00Z"), + ...row, + }, + ]) + .mockResolvedValueOnce([{ affectedRows: row.affectedRows }]); + const { importFromLoom } = await import("@/actions/loom"); + await expect( + importFromLoom({ + loomUrl: "https://www.loom.com/share/loom-abc1234567", + orgId: "org-1" as Organisation.OrganisationId, + }), + ).resolves.toMatchObject({ success: false }); + expect(startMock).not.toHaveBeenCalled(); + expect(mockDb.insert).not.toHaveBeenCalled(); + }); + it("returns direct MP4 URLs for public Loom downloads", async () => { const fetchMock = vi.mocked(fetch); fetchMock.mockImplementation(async (input) => { @@ -493,7 +557,7 @@ describe("importFromLoom", () => { 3, expect.objectContaining({ id: "video-123", - orgId: "org-1", + orgId: "org-1" as Organisation.OrganisationId, source: "loom", sourceId: "loom-abc1234567", }), diff --git a/apps/web/__tests__/unit/loom-retry-processing.test.ts b/apps/web/__tests__/unit/loom-retry-processing.test.ts new file mode 100644 index 00000000000..ad0dd9322de --- /dev/null +++ b/apps/web/__tests__/unit/loom-retry-processing.test.ts @@ -0,0 +1,140 @@ +import type { Video } from "@cap/web-domain"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + where: vi.fn(), + set: vi.fn(), + start: vi.fn(), + currentUser: vi.fn(), + startProcessing: vi.fn(), + setError: vi.fn(), +})); + +vi.mock("@cap/database", () => ({ + db: () => ({ + select: () => ({ from: () => ({ where: mocks.where }) }), + update: () => ({ set: mocks.set }), + }), +})); +vi.mock("@cap/database/auth/session", () => ({ + getCurrentUser: mocks.currentUser, +})); +vi.mock("@cap/database/schema", () => ({ + videos: { id: "videoId" }, + videoUploads: { + videoId: "uploadVideoId", + phase: "phase", + updatedAt: "updatedAt", + }, + importedVideos: { + id: "importVideoId", + orgId: "importOrgId", + source: "source", + sourceId: "sourceId", + }, +})); +vi.mock("drizzle-orm", () => ({ + eq: (column: string, value: unknown) => ({ column, value }), + and: (...conditions: unknown[]) => ({ conditions }), +})); +vi.mock("workflow/api", () => ({ start: mocks.start })); +vi.mock("@/workflows/import-loom-video", () => ({ + importLoomVideoWorkflow: vi.fn(), +})); +vi.mock("@/lib/video-processing", () => ({ + startVideoProcessingWorkflow: mocks.startProcessing, + setVideoProcessingError: mocks.setError, +})); + +import { retryVideoProcessing } from "@/actions/video/retry-processing"; + +const videoId = "video-1" as Video.VideoId; +const video = { id: videoId, ownerId: "owner-1", orgId: "org-1", bucket: null }; +const updatedAt = new Date(); +const upload = { + phase: "error", + rawFileKey: null, + updatedAt, + processingProgress: 0, +}; + +describe("Loom processing retries", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.currentUser.mockResolvedValue({ id: "owner-1" }); + mocks.set.mockReturnValue({ where: mocks.where }); + mocks.start.mockResolvedValue({}); + }); + + it("recovers a source lookup failure that has no raw file key", async () => { + mocks.where + .mockResolvedValueOnce([video]) + .mockResolvedValueOnce([upload]) + .mockResolvedValueOnce([{ source: "loom", sourceId: "loom-1" }]) + .mockResolvedValueOnce([{ affectedRows: 1 }]); + await expect(retryVideoProcessing({ videoId })).resolves.toEqual({ + success: true, + status: "started", + }); + expect(mocks.where.mock.calls[2]?.[0]).toEqual({ + conditions: [ + { column: "importVideoId", value: videoId }, + { column: "importOrgId", value: "org-1" }, + ], + }); + expect(mocks.start).toHaveBeenCalledWith(expect.anything(), [ + expect.objectContaining({ + videoId, + rawFileKey: "owner-1/video-1/raw-upload.mp4", + loomVideoId: "loom-1", + reuseExistingRawUpload: true, + }), + ]); + expect(mocks.startProcessing).not.toHaveBeenCalled(); + }); + + it("does not dispatch after losing the processing claim", async () => { + mocks.where + .mockResolvedValueOnce([video]) + .mockResolvedValueOnce([upload]) + .mockResolvedValueOnce([{ source: "loom", sourceId: "loom-1" }]) + .mockResolvedValueOnce([{ affectedRows: 0 }]); + await expect(retryVideoProcessing({ videoId })).resolves.toEqual({ + success: true, + status: "already-processing", + }); + expect(mocks.start).not.toHaveBeenCalled(); + }); + + it("leaves a recently active import running", async () => { + mocks.where + .mockResolvedValueOnce([video]) + .mockResolvedValueOnce([{ ...upload, phase: "processing" }]) + .mockResolvedValueOnce([{ source: "loom", sourceId: "loom-1" }]); + await expect(retryVideoProcessing({ videoId })).resolves.toEqual({ + success: true, + status: "already-processing", + }); + expect(mocks.set).not.toHaveBeenCalled(); + expect(mocks.start).not.toHaveBeenCalled(); + }); + + it("requires a raw file for a regular upload", async () => { + mocks.where + .mockResolvedValueOnce([video]) + .mockResolvedValueOnce([upload]) + .mockResolvedValueOnce([]); + await expect(retryVideoProcessing({ videoId })).rejects.toThrow( + "No raw file key found for retry", + ); + expect(mocks.start).not.toHaveBeenCalled(); + }); + + it("rejects a different owner's video", async () => { + mocks.where.mockResolvedValueOnce([{ ...video, ownerId: "other-owner" }]); + await expect(retryVideoProcessing({ videoId })).rejects.toThrow( + "Unauthorized", + ); + expect(mocks.start).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/__tests__/unit/media-processing-workflows.test.ts b/apps/web/__tests__/unit/media-processing-workflows.test.ts index 57deff5af8c..c41116c02a9 100644 --- a/apps/web/__tests__/unit/media-processing-workflows.test.ts +++ b/apps/web/__tests__/unit/media-processing-workflows.test.ts @@ -213,6 +213,29 @@ describe("media processing workflows", () => { ); }); + it("uses a fresh preflight source without requesting it again", async () => { + const expires = Math.floor(Date.now() / 1000) + 7200; + const loomDownloadUrl = `https://cdn.loom.com/original.mp4?Expires=${expires}`; + mocks.rows = [[video], [pending], [], [metadata]]; + await importLoomVideoWorkflow({ ...payload, loomDownloadUrl }); + expect(mocks.fetch).toHaveBeenCalledOnce(); + expect(JSON.parse(mocks.fetch.mock.calls[0]?.[1].body).videoUrl).toBe( + loomDownloadUrl, + ); + }); + + it("refreshes an expired preflight source before dispatch", async () => { + mocks.rows = [[video], [pending], [], [metadata]]; + await importLoomVideoWorkflow({ + ...payload, + loomDownloadUrl: "https://cdn.loom.com/original.mp4?Expires=1", + }); + expect(mocks.fetch).toHaveBeenCalledTimes(2); + expect(JSON.parse(mocks.fetch.mock.calls[1]?.[1].body).videoUrl).toBe( + "https://cdn.loom.com/original.mp4", + ); + }); + it("retries a failed Loom worker from its preserved original", async () => { mocks.rows = [ [video], diff --git a/apps/web/actions/loom.ts b/apps/web/actions/loom.ts index b9e03dcecf9..1b136ab89be 100644 --- a/apps/web/actions/loom.ts +++ b/apps/web/actions/loom.ts @@ -1,6 +1,5 @@ "use server"; -import { randomUUID } from "node:crypto"; import { db } from "@cap/database"; import { getCurrentUser } from "@cap/database/auth/session"; import { nanoId } from "@cap/database/helpers"; @@ -36,16 +35,16 @@ import { requireOrganizationSettingsManager, } from "@/actions/organization/authorization"; import { requireSpaceManager } from "@/actions/organization/space-authorization"; +import { + getLoomDownloadUrl, + LoomDownloadTemporaryError, +} from "@/lib/loom-download-url"; import type { LoomImportDestination } from "@/lib/loom-import-destination"; import { provisionOrganizationInvitee } from "@/lib/organization-provisioning"; import { canManageOrganizationSettings } from "@/lib/permissions/roles"; import { runPromise } from "@/lib/server"; import { importLoomVideoWorkflow } from "@/workflows/import-loom-video"; -interface LoomUrlResponse { - url?: string; -} - type LoomDownloadMode = "direct-download" | "browser-conversion"; interface LoomDownloadResult { @@ -117,47 +116,6 @@ function extractLoomVideoId(url: string): string | null { } } -async function fetchLoomEndpoint( - videoId: string, - endpoint: string, - includeBody = true, -): Promise { - try { - const options: RequestInit = { method: "POST" }; - if (includeBody) { - options.headers = { - "Content-Type": "application/json", - Accept: "application/json", - }; - options.body = JSON.stringify({ - anonID: randomUUID(), - deviceID: null, - force_original: false, - password: null, - }); - } - - const response = await fetch( - `https://www.loom.com/api/campaigns/sessions/${videoId}/${endpoint}`, - options, - ); - - if (!response.ok || response.status === 204) { - return null; - } - - const text = await response.text(); - if (!text.trim()) { - return null; - } - - const data: LoomUrlResponse = JSON.parse(text); - return data.url ?? null; - } catch { - return null; - } -} - async function fetchVideoName(videoId: string): Promise { try { const response = await fetch("https://www.loom.com/graphql", { @@ -189,38 +147,11 @@ async function fetchVideoName(videoId: string): Promise { } } -function isStreamingUrl(url: string): boolean { - const path = (url.split("?")[0] ?? "").toLowerCase(); - return path.endsWith(".m3u8") || path.endsWith(".mpd"); -} - function isDirectMp4Url(url: string): boolean { const path = (url.split("?")[0] ?? "").toLowerCase(); return path.endsWith(".mp4"); } -async function getLoomDownloadUrl(loomVideoId: string): Promise { - const requestVariants: Array<{ endpoint: string; includeBody: boolean }> = [ - { endpoint: "transcoded-url", includeBody: true }, - { endpoint: "raw-url", includeBody: true }, - { endpoint: "transcoded-url", includeBody: false }, - { endpoint: "raw-url", includeBody: false }, - ]; - - let fallbackStreamingUrl: string | null = null; - - for (const { endpoint, includeBody } of requestVariants) { - const url = await fetchLoomEndpoint(loomVideoId, endpoint, includeBody); - if (!url) continue; - - if (!isStreamingUrl(url)) return url; - - if (!fallbackStreamingUrl) fallbackStreamingUrl = url; - } - - return fallbackStreamingUrl; -} - async function fetchLoomOEmbed( loomVideoId: string, ): Promise<{ duration?: number; width?: number; height?: number } | null> { @@ -286,11 +217,13 @@ export async function downloadLoomVideo( height: oembedMeta?.height, requiresProxy: false, }; - } catch { + } catch (error) { return { success: false, error: - "An unexpected error occurred. Please try again or check your internet connection.", + error instanceof LoomDownloadTemporaryError + ? error.message + : "An unexpected error occurred. Please try again or check your internet connection.", }; } } @@ -318,6 +251,11 @@ async function importLoomVideoForOwner({ const existing = await db() .select({ videoId: videos.id, + ownerId: videos.ownerId, + bucketId: videos.bucket, + phase: videoUploads.phase, + rawFileKey: videoUploads.rawFileKey, + uploadUpdatedAt: videoUploads.updatedAt, }) .from(importedVideos) .leftJoin( @@ -327,6 +265,7 @@ async function importLoomVideoForOwner({ eq(videos.orgId, importedVideos.orgId), ), ) + .leftJoin(videoUploads, eq(videoUploads.videoId, videos.id)) .where( and( eq(importedVideos.orgId, orgId), @@ -335,7 +274,55 @@ async function importLoomVideoForOwner({ ), ); - if (existing.some((row) => row.videoId !== null)) { + const existingVideo = existing.find((row) => row.videoId !== null); + if (existingVideo?.videoId) { + if ( + existingVideo.phase === "error" && + existingVideo.ownerId === ownerId && + existingVideo.uploadUpdatedAt + ) { + const rawFileKey = + existingVideo.rawFileKey ?? + `${ownerId}/${existingVideo.videoId}/raw-upload.mp4`; + const claim = await db() + .update(videoUploads) + .set({ + phase: "processing", + processingProgress: 0, + processingMessage: "Restarting Loom import...", + processingError: null, + rawFileKey, + updatedAt: new Date(), + }) + .where( + and( + eq(videoUploads.videoId, existingVideo.videoId), + eq(videoUploads.phase, "error"), + eq(videoUploads.updatedAt, existingVideo.uploadUpdatedAt), + ), + ); + const affectedRows = Array.isArray(claim) + ? (claim[0] as { affectedRows?: number } | undefined)?.affectedRows + : (claim as { affectedRows?: number }).affectedRows; + if (affectedRows !== 1) { + return { + success: false, + error: "This Loom import is already restarting.", + }; + } + await start(importLoomVideoWorkflow, [ + { + videoId: existingVideo.videoId, + userId: ownerId, + rawFileKey, + bucketId: existingVideo.bucketId, + loomVideoId, + reuseExistingRawUpload: true, + }, + ]); + revalidatePath("/dashboard/caps"); + return { success: true, videoId: existingVideo.videoId }; + } return { success: false, error: "This Loom video has already been imported.", @@ -354,7 +341,15 @@ async function importLoomVideoForOwner({ ); } - const downloadUrl = await getLoomDownloadUrl(loomVideoId); + let downloadUrl: string | null; + try { + downloadUrl = await getLoomDownloadUrl(loomVideoId); + } catch (error) { + if (error instanceof LoomDownloadTemporaryError) { + return { success: false, error: error.message }; + } + throw error; + } if (!downloadUrl) { return { success: false, @@ -462,6 +457,7 @@ async function importLoomVideoForOwner({ rawFileKey, bucketId: Option.getOrNull(writable.bucketId), loomVideoId, + loomDownloadUrl: downloadUrl, }, ]); diff --git a/apps/web/actions/video/retry-processing.ts b/apps/web/actions/video/retry-processing.ts index fa53dabec2b..c439ce84338 100644 --- a/apps/web/actions/video/retry-processing.ts +++ b/apps/web/actions/video/retry-processing.ts @@ -4,7 +4,7 @@ import { db } from "@cap/database"; import { getCurrentUser } from "@cap/database/auth/session"; import { importedVideos, videos, videoUploads } from "@cap/database/schema"; import type { Video } from "@cap/web-domain"; -import { eq } from "drizzle-orm"; +import { and, eq } from "drizzle-orm"; import { start } from "workflow/api"; import { setVideoProcessingError, @@ -63,7 +63,6 @@ export async function retryVideoProcessing({ .where(eq(videoUploads.videoId, videoId)); if (!upload) throw new Error("No upload record found"); - if (!upload.rawFileKey) throw new Error("No raw file key found for retry"); const [importedVideo] = await db() .select({ @@ -71,34 +70,54 @@ export async function retryVideoProcessing({ sourceId: importedVideos.sourceId, }) .from(importedVideos) - .where(eq(importedVideos.id, videoId)); + .where( + and( + eq(importedVideos.id, videoId), + eq(importedVideos.orgId, video.orgId), + ), + ); if (importedVideo?.source === "loom") { if (upload.phase === "processing" && !shouldForceRetryProcessing(upload)) { return { success: true, status: "already-processing" }; } - await db() + const rawFileKey = + upload.rawFileKey ?? `${video.ownerId}/${videoId}/raw-upload.mp4`; + const claim = await db() .update(videoUploads) .set({ phase: "processing", processingProgress: 0, processingMessage: "Retrying Loom import...", processingError: null, - rawFileKey: upload.rawFileKey, + rawFileKey, updatedAt: new Date(), }) - .where(eq(videoUploads.videoId, videoId)); + .where( + and( + eq(videoUploads.videoId, videoId), + eq(videoUploads.phase, upload.phase), + eq(videoUploads.updatedAt, upload.updatedAt), + ), + ); + const affectedRows = Array.isArray(claim) + ? (claim[0] as { affectedRows?: number } | undefined)?.affectedRows + : (claim as { affectedRows?: number }).affectedRows; + if (affectedRows !== 1) { + return { success: true, status: "already-processing" }; + } try { await start(importLoomVideoWorkflow, [ { videoId, userId: user.id, - rawFileKey: upload.rawFileKey, + rawFileKey, bucketId: video.bucket ?? null, loomDownloadUrl: "", loomVideoId: importedVideo.sourceId, + reuseExistingRawUpload: true, }, ]); } catch (error) { @@ -117,6 +136,7 @@ export async function retryVideoProcessing({ return { success: true, status: "started" }; } + if (!upload.rawFileKey) throw new Error("No raw file key found for retry"); const status = await startVideoProcessingWorkflow({ videoId, userId: user.id, diff --git a/apps/web/lib/loom-download-url.ts b/apps/web/lib/loom-download-url.ts new file mode 100644 index 00000000000..d03fa757c7d --- /dev/null +++ b/apps/web/lib/loom-download-url.ts @@ -0,0 +1,151 @@ +import { randomUUID } from "node:crypto"; + +export class LoomDownloadTemporaryError extends Error { + constructor(public readonly retryAfterMs: number) { + super("Loom is temporarily unavailable. Please retry the import later."); + } +} + +export function getReusableLoomDownloadUrl(value: string | undefined) { + if (!value) return null; + try { + const url = new URL(value); + if ( + url.protocol !== "https:" || + url.hostname !== "cdn.loom.com" || + url.username || + url.password + ) { + return null; + } + if (!url.search) return url.href; + let expiresAt = Number(url.searchParams.get("Expires")); + const encodedPolicy = url.searchParams.get("Policy"); + if (encodedPolicy) { + const policy: unknown = JSON.parse( + Buffer.from( + encodedPolicy + .replace(/-/g, "+") + .replace(/_/g, "=") + .replace(/~/g, "/"), + "base64", + ).toString("utf8"), + ); + if ( + typeof policy !== "object" || + policy === null || + !("Statement" in policy) || + !Array.isArray(policy.Statement) + ) { + return null; + } + const statement = policy.Statement[0] as + | { Condition?: { DateLessThan?: { "AWS:EpochTime"?: unknown } } } + | undefined; + expiresAt = Number(statement?.Condition?.DateLessThan?.["AWS:EpochTime"]); + } + return Number.isFinite(expiresAt) && + expiresAt * 1000 > Date.now() + 60 * 60 * 1000 + ? url.href + : null; + } catch { + return null; + } +} + +function getRetryAfterMs(value: string | null): number { + if (!value) return 60_000; + const seconds = Number(value); + const delay = Number.isFinite(seconds) + ? seconds * 1000 + : Date.parse(value) - Date.now(); + return Number.isFinite(delay) ? Math.max(30_000, delay) : 60_000; +} + +async function fetchEndpoint( + videoId: string, + endpoint: string, + includeBody: boolean, +): Promise { + let response: Response; + try { + response = await fetch( + `https://www.loom.com/api/campaigns/sessions/${encodeURIComponent(videoId)}/${endpoint}`, + { + method: "POST", + signal: AbortSignal.timeout(15_000), + ...(includeBody + ? { + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify({ + anonID: randomUUID(), + deviceID: null, + force_original: false, + password: null, + }), + } + : {}), + }, + ); + } catch { + throw new LoomDownloadTemporaryError(60_000); + } + if (response.status === 429 || response.status >= 500) { + await response.body?.cancel(); + throw new LoomDownloadTemporaryError( + getRetryAfterMs(response.headers.get("retry-after")), + ); + } + if (!response.ok || response.status === 204) return null; + try { + const text = await response.text(); + if (!text.trim()) return null; + const data: unknown = JSON.parse(text); + if ( + typeof data !== "object" || + data === null || + !("url" in data) || + typeof data.url !== "string" + ) { + return null; + } + const url = new URL(data.url); + return url.protocol === "https:" && !url.username && !url.password + ? url.href + : null; + } catch { + throw new LoomDownloadTemporaryError(60_000); + } +} + +export async function getLoomDownloadUrl( + videoId: string, +): Promise { + let streamingUrl: string | null = null; + for (const { endpoint, includeBody } of [ + { endpoint: "transcoded-url", includeBody: true }, + { endpoint: "raw-url", includeBody: true }, + { endpoint: "transcoded-url", includeBody: false }, + { endpoint: "raw-url", includeBody: false }, + ]) { + let url: string | null; + try { + url = await fetchEndpoint(videoId, endpoint, includeBody); + } catch (error) { + if (streamingUrl && error instanceof LoomDownloadTemporaryError) { + return streamingUrl; + } + throw error; + } + if (!url) continue; + const pathname = new URL(url).pathname.toLowerCase(); + if (!pathname.endsWith(".m3u8") && !pathname.endsWith(".mpd")) { + return url; + } + streamingUrl ??= url; + } + return streamingUrl; +} diff --git a/apps/web/workflows/import-loom-video.ts b/apps/web/workflows/import-loom-video.ts index e821e92d885..e970f73262e 100644 --- a/apps/web/workflows/import-loom-video.ts +++ b/apps/web/workflows/import-loom-video.ts @@ -1,4 +1,3 @@ -import { randomUUID } from "node:crypto"; import { db } from "@cap/database"; import { agentApiOperations, videos, videoUploads } from "@cap/database/schema"; import { serverEnv } from "@cap/env"; @@ -6,7 +5,12 @@ import { Storage } from "@cap/web-backend/src/Storage/index"; import { Video } from "@cap/web-domain"; import { eq } from "drizzle-orm"; import { Effect } from "effect"; -import { FatalError, sleep } from "workflow"; +import { FatalError, RetryableError, sleep } from "workflow"; +import { + getLoomDownloadUrl, + getReusableLoomDownloadUrl, + LoomDownloadTemporaryError, +} from "@/lib/loom-download-url"; import { createMediaServerCapacityError, isMediaServerCapacityError, @@ -44,72 +48,32 @@ function isStreamingUrl(url: string): boolean { return path.endsWith(".m3u8") || path.endsWith(".mpd"); } -async function fetchLoomCdnUrl( - videoId: string, - endpoint: string, - includeBody: boolean, -): Promise { +async function fetchFreshLoomDownloadUrl( + loomVideoId: string, + existingUrl?: string, +): Promise { + "use step"; + + const reusableUrl = getReusableLoomDownloadUrl(existingUrl); + if (reusableUrl) return reusableUrl; try { - const options: RequestInit = { - method: "POST", - signal: AbortSignal.timeout(15_000), - }; - if (includeBody) { - options.headers = { - "Content-Type": "application/json", - Accept: "application/json", - }; - options.body = JSON.stringify({ - anonID: randomUUID(), - deviceID: null, - force_original: false, - password: null, + const url = await getLoomDownloadUrl(loomVideoId); + if (url) return url; + } catch (error) { + if (error instanceof LoomDownloadTemporaryError) { + throw new RetryableError(error.message, { + retryAfter: error.retryAfterMs, }); } - - const response = await fetch( - `https://www.loom.com/api/campaigns/sessions/${videoId}/${endpoint}`, - options, - ); - - if (!response.ok || response.status === 204) return null; - - const text = await response.text(); - if (!text.trim()) return null; - - const data = JSON.parse(text) as { url?: string }; - return data.url ?? null; - } catch { - return null; + throw error; } -} - -async function fetchFreshLoomDownloadUrl(loomVideoId: string): Promise { - const requestVariants: Array<{ endpoint: string; includeBody: boolean }> = [ - { endpoint: "transcoded-url", includeBody: true }, - { endpoint: "raw-url", includeBody: true }, - { endpoint: "transcoded-url", includeBody: false }, - { endpoint: "raw-url", includeBody: false }, - ]; - - let fallbackStreamingUrl: string | null = null; - - for (const { endpoint, includeBody } of requestVariants) { - const url = await fetchLoomCdnUrl(loomVideoId, endpoint, includeBody); - if (!url) continue; - - if (!isStreamingUrl(url)) return url; - - if (!fallbackStreamingUrl) fallbackStreamingUrl = url; - } - - if (fallbackStreamingUrl) return fallbackStreamingUrl; - throw new FatalError( "Could not retrieve a download URL from Loom. The video may be private, password-protected, or the link may have expired.", ); } +fetchFreshLoomDownloadUrl.maxRetries = 8; + interface VideoProcessingResult { success: boolean; message: string; @@ -210,7 +174,15 @@ export async function importLoomVideoWorkflow( let capacityRetryCount = 0; while (true) { try { - await processVideoOnMediaServer(payload, processingInput); + const loomSourceUrl = processingInput.importFromLoom + ? await fetchFreshLoomDownloadUrl( + payload.loomVideoId, + processingAttempt === 0 && capacityRetryCount === 0 + ? payload.loomDownloadUrl + : undefined, + ) + : undefined; + await processVideoOnMediaServer(payload, loomSourceUrl); break; } catch (error) { if (!isMediaServerCapacityError(error)) throw error; @@ -374,11 +346,11 @@ async function startMediaServerProcessJob( async function processVideoOnMediaServer( payload: ImportLoomPayload, - processingInput: LoomProcessingInput, + loomSourceUrl: string | undefined, ): Promise { "use step"; - const { videoId, userId, rawFileKey, loomVideoId } = payload; + const { videoId, userId, rawFileKey } = payload; const mediaServerUrl = serverEnv().MEDIA_SERVER_URL; if (!mediaServerUrl) { @@ -388,9 +360,6 @@ async function processVideoOnMediaServer( const webhookBaseUrl = serverEnv().MEDIA_SERVER_WEBHOOK_URL || serverEnv().WEB_URL; - const loomSourceUrl = processingInput.importFromLoom - ? await fetchFreshLoomDownloadUrl(loomVideoId) - : undefined; const { rawVideoUrl, sourcePresignedUrl, From 09a10ac14881df103bcc2cc4de0382b83dbd7b49 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:53:17 +0100 Subject: [PATCH 02/21] fix: preserve Loom segment credentials and legacy sources --- .../__tests__/unit/loom-download-url.test.ts | 90 ++++++++++- apps/web/lib/loom-download-url.ts | 147 ++++++++++++++++-- 2 files changed, 226 insertions(+), 11 deletions(-) diff --git a/apps/web/__tests__/unit/loom-download-url.test.ts b/apps/web/__tests__/unit/loom-download-url.test.ts index e75657f526d..9227e712f45 100644 --- a/apps/web/__tests__/unit/loom-download-url.test.ts +++ b/apps/web/__tests__/unit/loom-download-url.test.ts @@ -42,7 +42,7 @@ describe("getLoomDownloadUrl", () => { .mockResolvedValue(new Response(null, { status: 204 })); vi.stubGlobal("fetch", fetch); await expect(getLoomDownloadUrl("video")).resolves.toBeNull(); - expect(fetch).toHaveBeenCalledTimes(4); + expect(fetch).toHaveBeenCalledTimes(5); }); it("uses an available stream when a later endpoint is rate limited", async () => { @@ -75,6 +75,94 @@ describe("getLoomDownloadUrl", () => { "https://cdn.loom.com/original.mp4", ); }); + + it.each(["m3u8", "mpd"])( + "uses segment credentials for a legacy %s stream", + async (extension) => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + Response.json({ + url: `https://cdn.loom.com/video.${extension}?Policy=manifest-policy&Signature=manifest-signature&Key-Pair-Id=manifest-key`, + part_credentials: { + Policy: "segment-policy", + Signature: "segment-signature", + "Key-Pair-Id": "segment-key", + }, + }), + ), + ); + const url = new URL((await getLoomDownloadUrl("video")) ?? ""); + expect(url.searchParams.get("Policy")).toBe("segment-policy"); + expect(url.searchParams.get("Signature")).toBe("segment-signature"); + expect(url.searchParams.get("Key-Pair-Id")).toBe("segment-key"); + }, + ); + + it("treats incomplete segment credentials as a temporary source failure", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + Response.json({ + url: "https://cdn.loom.com/video.m3u8?Policy=manifest-policy", + part_credentials: { Policy: "segment-policy" }, + }), + ), + ); + await expect(getLoomDownloadUrl("video")).rejects.toBeInstanceOf( + LoomDownloadTemporaryError, + ); + }); + + it("uses the public player source for older trimmed videos", async () => { + const fetch = vi + .fn() + .mockResolvedValueOnce(new Response(null, { status: 204 })) + .mockResolvedValueOnce(new Response(null, { status: 204 })) + .mockResolvedValueOnce(new Response(null, { status: 204 })) + .mockResolvedValueOnce(new Response(null, { status: 204 })) + .mockResolvedValueOnce( + Response.json({ + data: { + getVideo: { + id: "video", + downloadable: true, + download_enabled: true, + nullableRawCdnUrl: { url: "https://cdn.loom.com/video-trim.mp4" }, + }, + }, + }), + ); + vi.stubGlobal("fetch", fetch); + await expect(getLoomDownloadUrl("video")).resolves.toBe( + "https://cdn.loom.com/video-trim.mp4", + ); + }); + + it.each([ + { id: "video", downloadable: false, download_enabled: false }, + { id: "different-video", downloadable: true, download_enabled: true }, + ])( + "rejects a disabled or mismatched public player source: %j", + async (video) => { + const fetch = vi.fn().mockImplementation(async (url: string) => + url.endsWith("/graphql") + ? Response.json({ + data: { + getVideo: { + ...video, + nullableRawCdnUrl: { + url: "https://cdn.loom.com/video.mp4", + }, + }, + }, + }) + : new Response(null, { status: 204 }), + ); + vi.stubGlobal("fetch", fetch); + await expect(getLoomDownloadUrl("video")).resolves.toBeNull(); + }, + ); }); describe("getReusableLoomDownloadUrl", () => { diff --git a/apps/web/lib/loom-download-url.ts b/apps/web/lib/loom-download-url.ts index d03fa757c7d..134ff6b9c35 100644 --- a/apps/web/lib/loom-download-url.ts +++ b/apps/web/lib/loom-download-url.ts @@ -104,18 +104,145 @@ async function fetchEndpoint( const text = await response.text(); if (!text.trim()) return null; const data: unknown = JSON.parse(text); + return parseLoomDownloadResponse(data); + } catch { + throw new LoomDownloadTemporaryError(60_000); + } +} + +function parseLoomDownloadResponse(data: unknown): string | null { + if ( + typeof data !== "object" || + data === null || + !("url" in data) || + typeof data.url !== "string" + ) { + return null; + } + const url = new URL(data.url); + if (url.protocol !== "https:" || url.username || url.password) return null; + const pathname = url.pathname.toLowerCase(); + if ( + (pathname.endsWith(".m3u8") || pathname.endsWith(".mpd")) && + "part_credentials" in data && + data.part_credentials !== null + ) { + const credentials = data.part_credentials; if ( - typeof data !== "object" || - data === null || - !("url" in data) || - typeof data.url !== "string" + typeof credentials !== "object" || + !("Signature" in credentials) || + typeof credentials.Signature !== "string" || + !credentials.Signature || + !("Key-Pair-Id" in credentials) || + typeof credentials["Key-Pair-Id"] !== "string" || + !credentials["Key-Pair-Id"] ) { - return null; + throw new LoomDownloadTemporaryError(60_000); } - const url = new URL(data.url); - return url.protocol === "https:" && !url.username && !url.password - ? url.href - : null; + if ( + "Policy" in credentials && + typeof credentials.Policy === "string" && + credentials.Policy + ) { + url.searchParams.set("Policy", credentials.Policy); + url.searchParams.delete("Expires"); + } else if ( + "Expires" in credentials && + Number.isFinite(Number(credentials.Expires)) && + Number(credentials.Expires) > 0 + ) { + url.searchParams.set("Expires", String(credentials.Expires)); + url.searchParams.delete("Policy"); + } else { + throw new LoomDownloadTemporaryError(60_000); + } + url.searchParams.set("Signature", credentials.Signature); + url.searchParams.set("Key-Pair-Id", credentials["Key-Pair-Id"]); + } + return url.href; +} + +async function fetchPublicLoomPlaybackUrl( + videoId: string, +): Promise { + let response: Response; + try { + response = await fetch("https://www.loom.com/graphql", { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify({ + operationName: "GetVideoSource", + variables: { + videoId, + password: null, + acceptableMimes: ["DASH", "M3U8", "MP4", "WEBM"], + }, + query: + "query GetVideoSource($videoId: ID!, $password: String, $acceptableMimes: [CloudfrontVideoAcceptableMime]) { getVideo(id: $videoId, password: $password) { ... on RegularUserVideo { id downloadable download_enabled nullableRawCdnUrl(acceptableMimes: $acceptableMimes, password: $password) { url credentials { Policy Signature KeyPairId } } } __typename } }", + }), + signal: AbortSignal.timeout(15_000), + }); + } catch { + throw new LoomDownloadTemporaryError(60_000); + } + if (response.status === 429 || response.status >= 500) { + await response.body?.cancel(); + throw new LoomDownloadTemporaryError( + getRetryAfterMs(response.headers.get("retry-after")), + ); + } + if (!response.ok || response.status === 204) return null; + try { + const body: unknown = await response.json(); + if (typeof body !== "object" || body === null) return null; + if ( + "errors" in body && + Array.isArray(body.errors) && + body.errors.length > 0 + ) + throw new LoomDownloadTemporaryError(60_000); + if ( + !("data" in body) || + typeof body.data !== "object" || + body.data === null || + !("getVideo" in body.data) + ) + return null; + const video = body.data.getVideo; + if ( + typeof video !== "object" || + video === null || + !("id" in video) || + video.id !== videoId || + !("downloadable" in video) || + video.downloadable !== true || + !("download_enabled" in video) || + video.download_enabled !== true || + !("nullableRawCdnUrl" in video) + ) + return null; + const source = video.nullableRawCdnUrl; + if (typeof source !== "object" || source === null) return null; + const credentials = "credentials" in source ? source.credentials : null; + const partCredentials = + typeof credentials === "object" && + credentials !== null && + "Policy" in credentials && + "Signature" in credentials && + "KeyPairId" in credentials + ? { + Policy: credentials.Policy, + Signature: credentials.Signature, + "Key-Pair-Id": credentials.KeyPairId, + } + : null; + return parseLoomDownloadResponse({ + ...source, + part_credentials: partCredentials, + }); } catch { throw new LoomDownloadTemporaryError(60_000); } @@ -147,5 +274,5 @@ export async function getLoomDownloadUrl( } streamingUrl ??= url; } - return streamingUrl; + return streamingUrl ?? fetchPublicLoomPlaybackUrl(videoId); } From 6fe772dca3d83768b3dbe463dfe1af76efe0103d Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Mon, 21 Sep 2026 21:29:34 +0100 Subject: [PATCH 03/21] fix: preserve Loom workflow dispatch receipts --- .../__tests__/unit/loom-download-url.test.ts | 41 ++-- .../__tests__/unit/loom-import-start.test.ts | 227 ++++++++++++++++++ apps/web/__tests__/unit/loom-import.test.ts | 78 +++++- .../unit/loom-retry-processing.test.ts | 57 +++++ apps/web/actions/loom.ts | 67 +++++- apps/web/actions/video/retry-processing.ts | 52 ++-- apps/web/lib/loom-download-url.ts | 2 +- apps/web/lib/loom-import-start.ts | 149 ++++++++++++ packages/database/types/metadata.ts | 4 + 9 files changed, 625 insertions(+), 52 deletions(-) create mode 100644 apps/web/__tests__/unit/loom-import-start.test.ts create mode 100644 apps/web/lib/loom-import-start.ts diff --git a/apps/web/__tests__/unit/loom-download-url.test.ts b/apps/web/__tests__/unit/loom-download-url.test.ts index 9227e712f45..cca084b6e1f 100644 --- a/apps/web/__tests__/unit/loom-download-url.test.ts +++ b/apps/web/__tests__/unit/loom-download-url.test.ts @@ -166,28 +166,33 @@ describe("getLoomDownloadUrl", () => { }); describe("getReusableLoomDownloadUrl", () => { - it("reuses a source with enough signed URL lifetime", () => { - const expires = Math.floor(Date.now() / 1000) + 7200; - const url = `https://cdn.loom.com/video.m3u8?Expires=${expires}`; - expect(getReusableLoomDownloadUrl(url)).toBe(url); - const policy = Buffer.from( - JSON.stringify({ - Statement: [ - { Condition: { DateLessThan: { "AWS:EpochTime": expires } } }, - ], - }), - ) - .toString("base64") - .replace(/\+/g, "-") - .replace(/=/g, "_") - .replace(/\//g, "~"); - const policyUrl = `https://cdn.loom.com/video.m3u8?Policy=${policy}`; - expect(getReusableLoomDownloadUrl(policyUrl)).toBe(policyUrl); - }); + it.each(["cdn.loom.com", "luna.loom.com"])( + "reuses a source with enough signed URL lifetime on %s", + (hostname) => { + const expires = Math.floor(Date.now() / 1000) + 7200; + const url = `https://${hostname}/video.m3u8?Expires=${expires}`; + expect(getReusableLoomDownloadUrl(url)).toBe(url); + const policy = Buffer.from( + JSON.stringify({ + Statement: [ + { Condition: { DateLessThan: { "AWS:EpochTime": expires } } }, + ], + }), + ) + .toString("base64") + .replace(/\+/g, "-") + .replace(/=/g, "_") + .replace(/\//g, "~"); + const policyUrl = `https://${hostname}/video.m3u8?Policy=${policy}`; + expect(getReusableLoomDownloadUrl(policyUrl)).toBe(policyUrl); + }, + ); it.each([ undefined, "https://cdn.loom.com/video.m3u8?Expires=1", + "https://luna.loom.com/video.m3u8?Expires=1", + "https://luna.loom.com.other.example/video.mp4", "https://cdn.loom.com/video.m3u8?Policy=invalid", "https://other.example/video.mp4", "http://cdn.loom.com/video.mp4", diff --git a/apps/web/__tests__/unit/loom-import-start.test.ts b/apps/web/__tests__/unit/loom-import-start.test.ts new file mode 100644 index 00000000000..95e82c130a6 --- /dev/null +++ b/apps/web/__tests__/unit/loom-import-start.test.ts @@ -0,0 +1,227 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + where: vi.fn(), + set: vi.fn(), + queue: vi.fn(), + start: vi.fn(), + getRun: vi.fn(), + getWorld: vi.fn(), +})); + +vi.mock("@cap/database", () => ({ + db: () => ({ update: () => ({ set: mocks.set }) }), +})); +vi.mock("@cap/database/schema", () => ({ + videos: { id: "videoId", ownerId: "ownerId", metadata: "metadata" }, + videoUploads: { + videoId: "uploadVideoId", + phase: "phase", + updatedAt: "updatedAt", + }, +})); +vi.mock("@cap/web-domain", () => ({ + Video: { VideoId: { make: (id: string) => id } }, + User: { UserId: { make: (id: string) => id } }, +})); +vi.mock("drizzle-orm", () => ({ + eq: (column: string, value: unknown) => ({ column, value }), + and: (...conditions: unknown[]) => ({ conditions }), + sql: (strings: TemplateStringsArray, ...values: unknown[]) => ({ + strings, + values, + }), +})); +vi.mock("workflow/api", () => ({ start: mocks.start, getRun: mocks.getRun })); +vi.mock("workflow/runtime", () => ({ getWorld: mocks.getWorld })); +vi.mock("@/workflows/import-loom-video", () => ({ + importLoomVideoWorkflow: vi.fn(), +})); + +import { + isLoomImportRunning, + LoomImportStartError, + restoreLoomImportStartError, + startLoomImportWorkflow, +} from "@/lib/loom-import-start"; + +const payload = { + videoId: "video-1", + userId: "owner-1", + rawFileKey: "owner-1/video-1/raw-upload.mp4", + bucketId: null, + loomVideoId: "loom-1", +}; + +describe("Loom workflow startup receipts", () => { + beforeEach(() => { + vi.resetAllMocks(); + mocks.set.mockReturnValue({ where: mocks.where }); + mocks.where.mockResolvedValue([{ affectedRows: 1 }]); + mocks.queue.mockResolvedValue({ messageId: "message-1" }); + mocks.getWorld.mockReturnValue({ queue: mocks.queue }); + mocks.start.mockImplementation( + async ( + _workflow: unknown, + _args: unknown, + options: { + world: { + queue: ( + name: string, + message: { runId: string }, + ) => Promise; + }; + }, + ) => { + await options.world.queue("workflow-loom", { runId: "run-1" }); + }, + ); + }); + + it("records the run before dispatch and stores queue acceptance", async () => { + await startLoomImportWorkflow(payload); + expect( + mocks.set.mock.calls.map(([value]) => + JSON.parse(value.metadata.values[1]), + ), + ).toEqual([ + { runId: "run-1", dispatch: "pending" }, + { runId: "run-1", dispatch: "accepted" }, + ]); + expect(mocks.where.mock.invocationCallOrder[0]).toBeLessThan( + mocks.queue.mock.invocationCallOrder[0] ?? 0, + ); + }); + + it.each([ + "BadRequestError", + "UnauthorizedError", + "ForbiddenError", + "TooManyRequestsError", + ])("allows retry after an explicit queue rejection: %s", async (name) => { + mocks.queue.mockRejectedValueOnce( + Object.assign(new Error("Queue rejected"), { name }), + ); + await expect(startLoomImportWorkflow(payload)).rejects.toMatchObject({ + canRetry: true, + }); + expect( + JSON.parse(mocks.set.mock.calls.at(-1)?.[0].metadata.values[1]), + ).toEqual({ runId: "run-1", dispatch: "rejected" }); + }); + + it("preserves an uncertain queue acknowledgement without allowing another dispatch", async () => { + mocks.queue.mockRejectedValueOnce(new TypeError("Connection reset")); + await expect(startLoomImportWorkflow(payload)).rejects.toMatchObject({ + canRetry: false, + }); + expect( + JSON.parse(mocks.set.mock.calls.at(-1)?.[0].metadata.values[1]), + ).toEqual({ runId: "run-1", dispatch: "uncertain" }); + }); + + it("does not treat a run creation error as a rejected queue message", async () => { + mocks.start.mockImplementationOnce( + async ( + _workflow: unknown, + _args: unknown, + options: { + world: { + queue: ( + name: string, + message: { runId: string }, + ) => Promise; + }; + }, + ) => { + await options.world.queue("workflow-loom", { runId: "run-1" }); + throw Object.assign(new Error("Run creation rejected"), { + name: "BadRequestError", + }); + }, + ); + await expect(startLoomImportWorkflow(payload)).resolves.toBeUndefined(); + expect( + JSON.parse(mocks.set.mock.calls.at(-1)?.[0].metadata.values[1]).dispatch, + ).toBe("accepted"); + }); + + it("allows retry when startup fails before dispatch", async () => { + mocks.start.mockRejectedValueOnce(new Error("Workflow not registered")); + await expect(startLoomImportWorkflow(payload)).rejects.toBeInstanceOf( + LoomImportStartError, + ); + await expect(startLoomImportWorkflow(payload)).resolves.toBeUndefined(); + expect(mocks.queue).toHaveBeenCalledTimes(1); + }); + + it("does not send a job when its run receipt cannot be saved", async () => { + mocks.where.mockResolvedValueOnce([{ affectedRows: 0 }]); + await expect(startLoomImportWorkflow(payload)).rejects.toMatchObject({ + canRetry: true, + }); + expect(mocks.queue).not.toHaveBeenCalled(); + }); + + it("keeps the claim when a rejected dispatch receipt cannot be saved", async () => { + mocks.queue.mockRejectedValueOnce( + Object.assign(new Error("Queue rejected"), { name: "ForbiddenError" }), + ); + mocks.where + .mockResolvedValueOnce([{ affectedRows: 1 }]) + .mockRejectedValueOnce(new Error("Database unavailable")); + await expect(startLoomImportWorkflow(payload)).rejects.toMatchObject({ + canRetry: false, + }); + }); + + it.each(["pending", "running", "completed", "failed", "cancelled"] as const)( + "uses authoritative run status for %s", + async (status) => { + mocks.getRun.mockReturnValue({ status: Promise.resolve(status) }); + expect( + await isLoomImportRunning({ runId: "run-1", dispatch: "uncertain" }), + ).toBe(status === "pending" || status === "running"); + }, + ); + + it("holds startup while the run cannot be queried", async () => { + mocks.getRun.mockReturnValue({ + status: Promise.reject(new Error("Not found yet")), + }); + expect( + await isLoomImportRunning({ runId: "run-1", dispatch: "uncertain" }), + ).toBe(true); + }); + + it("allows rejected and legacy imports without an active run check", async () => { + expect( + await isLoomImportRunning({ runId: "run-1", dispatch: "rejected" }), + ).toBe(false); + expect(await isLoomImportRunning()).toBe(false); + expect(mocks.getRun).not.toHaveBeenCalled(); + }); + + it("only restores the same upload claim after dispatch rejection", async () => { + const claimedAt = new Date("2026-09-01T00:00:00Z"); + await restoreLoomImportStartError( + "video-1", + "processing", + claimedAt, + "Startup rejected", + ); + expect(mocks.set).toHaveBeenCalledWith( + expect.objectContaining({ + phase: "error", + processingError: "Startup rejected", + }), + ); + expect(mocks.where).toHaveBeenCalledWith({ + conditions: [ + { column: "uploadVideoId", value: "video-1" }, + { column: "phase", value: "processing" }, + { column: "updatedAt", value: claimedAt }, + ], + }); + }); +}); diff --git a/apps/web/__tests__/unit/loom-import.test.ts b/apps/web/__tests__/unit/loom-import.test.ts index 05c135249be..78a5f97a2ed 100644 --- a/apps/web/__tests__/unit/loom-import.test.ts +++ b/apps/web/__tests__/unit/loom-import.test.ts @@ -12,6 +12,8 @@ const headersMock = vi.hoisted(() => vi.fn()); const getOrganizationAccessMock = vi.hoisted(() => vi.fn()); const requireSpaceManagerMock = vi.hoisted(() => vi.fn()); const requireOrganizationSettingsManagerMock = vi.hoisted(() => vi.fn()); +const isLoomImportRunningMock = vi.hoisted(() => vi.fn()); +const restoreLoomImportStartErrorMock = vi.hoisted(() => vi.fn()); const mockDb = { select: vi.fn(() => mockDb), @@ -22,7 +24,7 @@ const mockDb = { innerJoin: vi.fn(() => mockDb), leftJoin: vi.fn(() => mockDb), where: whereMock, - set: vi.fn(() => mockDb), + set: vi.fn((_value: { updatedAt?: Date }) => mockDb), values: valuesMock, transaction: vi.fn((callback) => callback(mockDb)), }; @@ -205,6 +207,18 @@ vi.mock("@/workflows/import-loom-video", () => ({ importLoomVideoWorkflow: Symbol("importLoomVideoWorkflow"), })); +vi.mock("@/lib/loom-import-start", () => ({ + isLoomImportRunning: isLoomImportRunningMock, + restoreLoomImportStartError: restoreLoomImportStartErrorMock, + startLoomImportWorkflow: (payload: unknown) => + startMock(Symbol("workflow"), [payload]), + LoomImportStartError: class extends Error { + constructor(readonly canRetry: boolean) { + super("Loom import startup failed"); + } + }, +})); + import { getCurrentUser } from "@cap/database/auth/session"; const mockGetCurrentUser = getCurrentUser as ReturnType; @@ -235,6 +249,7 @@ describe("importFromLoom", () => { valuesMock.mockResolvedValue(undefined); whereMock.mockResolvedValue([]); startMock.mockResolvedValue(undefined); + isLoomImportRunningMock.mockResolvedValue(false); checkRateLimitMock.mockResolvedValue({ rateLimited: false }); getOrganizationAccessMock.mockResolvedValue({ id: "org-1", @@ -293,6 +308,67 @@ describe("importFromLoom", () => { ]); }); + it.each([true, false])( + "restores a rejected startup without reopening an uncertain dispatch: %s", + async (canRetry) => { + whereMock + .mockResolvedValueOnce([ + { + videoId: "existing-video", + ownerId: "user-123", + phase: "error", + bucketId: null, + rawFileKey: null, + uploadUpdatedAt: new Date(), + }, + ]) + .mockResolvedValueOnce([{ affectedRows: 1 }]); + const { LoomImportStartError } = await import("@/lib/loom-import-start"); + startMock.mockRejectedValueOnce( + new LoomImportStartError(canRetry, new Error()), + ); + const { importFromLoom } = await import("@/actions/loom"); + expect( + await importFromLoom({ + loomUrl: "https://www.loom.com/share/loom-abc1234567", + orgId: "org-1" as never, + }), + ).toMatchObject({ success: false, error: "Loom import startup failed" }); + if (canRetry) { + expect(restoreLoomImportStartErrorMock).toHaveBeenCalledWith( + "existing-video", + "processing", + mockDb.set.mock.calls[0]?.[0].updatedAt, + "Loom import startup failed", + ); + } else expect(restoreLoomImportStartErrorMock).not.toHaveBeenCalled(); + }, + ); + + it("holds a failed row while its recorded workflow is still active", async () => { + const receipt = { runId: "run-active", dispatch: "accepted" }; + whereMock.mockResolvedValueOnce([ + { + videoId: "existing-video", + ownerId: "user-123", + phase: "error", + uploadUpdatedAt: new Date(), + metadata: { loomImportRun: receipt }, + }, + ]); + isLoomImportRunningMock.mockResolvedValueOnce(true); + const { importFromLoom } = await import("@/actions/loom"); + expect( + await importFromLoom({ + loomUrl: "https://www.loom.com/share/loom-abc1234567", + orgId: "org-1" as never, + }), + ).toMatchObject({ success: false }); + expect(isLoomImportRunningMock).toHaveBeenCalledWith(receipt); + expect(mockDb.update).not.toHaveBeenCalled(); + expect(startMock).not.toHaveBeenCalled(); + }); + it.each([ { ownerId: "user-123", phase: "processing", affectedRows: 1 }, { ownerId: "other-user", phase: "error", affectedRows: 1 }, diff --git a/apps/web/__tests__/unit/loom-retry-processing.test.ts b/apps/web/__tests__/unit/loom-retry-processing.test.ts index ad0dd9322de..32f16f32cc4 100644 --- a/apps/web/__tests__/unit/loom-retry-processing.test.ts +++ b/apps/web/__tests__/unit/loom-retry-processing.test.ts @@ -8,6 +8,8 @@ const mocks = vi.hoisted(() => ({ currentUser: vi.fn(), startProcessing: vi.fn(), setError: vi.fn(), + isRunning: vi.fn(), + restoreStartError: vi.fn(), })); vi.mock("@cap/database", () => ({ @@ -41,6 +43,17 @@ vi.mock("workflow/api", () => ({ start: mocks.start })); vi.mock("@/workflows/import-loom-video", () => ({ importLoomVideoWorkflow: vi.fn(), })); +vi.mock("@/lib/loom-import-start", () => ({ + isLoomImportRunning: mocks.isRunning, + restoreLoomImportStartError: mocks.restoreStartError, + startLoomImportWorkflow: (payload: unknown) => + mocks.start(Symbol("workflow"), [payload]), + LoomImportStartError: class extends Error { + constructor(readonly canRetry: boolean) { + super("Loom import startup failed"); + } + }, +})); vi.mock("@/lib/video-processing", () => ({ startVideoProcessingWorkflow: mocks.startProcessing, setVideoProcessingError: mocks.setError, @@ -64,6 +77,50 @@ describe("Loom processing retries", () => { mocks.currentUser.mockResolvedValue({ id: "owner-1" }); mocks.set.mockReturnValue({ where: mocks.where }); mocks.start.mockResolvedValue({}); + mocks.isRunning.mockResolvedValue(false); + }); + + it.each([true, false])( + "only restores the processing claim after a definite startup rejection: %s", + async (canRetry) => { + mocks.where + .mockResolvedValueOnce([video]) + .mockResolvedValueOnce([upload]) + .mockResolvedValueOnce([{ source: "loom", sourceId: "loom-1" }]) + .mockResolvedValueOnce([{ affectedRows: 1 }]); + const { LoomImportStartError } = await import("@/lib/loom-import-start"); + mocks.start.mockRejectedValueOnce( + new LoomImportStartError(canRetry, new Error()), + ); + await expect(retryVideoProcessing({ videoId })).rejects.toThrow( + "Loom import startup failed", + ); + if (canRetry) + expect(mocks.restoreStartError).toHaveBeenCalledWith( + videoId, + "processing", + mocks.set.mock.calls[0]?.[0].updatedAt, + "Loom import startup failed", + ); + else expect(mocks.restoreStartError).not.toHaveBeenCalled(); + expect(mocks.setError).not.toHaveBeenCalled(); + }, + ); + + it("holds a stale processing row while its workflow is still active", async () => { + mocks.where + .mockResolvedValueOnce([video]) + .mockResolvedValueOnce([ + { ...upload, phase: "processing", updatedAt: new Date(0) }, + ]) + .mockResolvedValueOnce([{ source: "loom", sourceId: "loom-1" }]); + mocks.isRunning.mockResolvedValueOnce(true); + await expect(retryVideoProcessing({ videoId })).resolves.toEqual({ + success: true, + status: "already-processing", + }); + expect(mocks.set).not.toHaveBeenCalled(); + expect(mocks.start).not.toHaveBeenCalled(); }); it("recovers a source lookup failure that has no raw file key", async () => { diff --git a/apps/web/actions/loom.ts b/apps/web/actions/loom.ts index 1b136ab89be..eca761d416b 100644 --- a/apps/web/actions/loom.ts +++ b/apps/web/actions/loom.ts @@ -28,7 +28,6 @@ import { import { and, asc, eq, isNull } from "drizzle-orm"; import { Option } from "effect"; import { revalidatePath } from "next/cache"; -import { start } from "workflow/api"; import { getOrganizationAccess, requireOrganizationAccess, @@ -40,10 +39,15 @@ import { LoomDownloadTemporaryError, } from "@/lib/loom-download-url"; import type { LoomImportDestination } from "@/lib/loom-import-destination"; +import { + isLoomImportRunning, + LoomImportStartError, + restoreLoomImportStartError, + startLoomImportWorkflow, +} from "@/lib/loom-import-start"; import { provisionOrganizationInvitee } from "@/lib/organization-provisioning"; import { canManageOrganizationSettings } from "@/lib/permissions/roles"; import { runPromise } from "@/lib/server"; -import { importLoomVideoWorkflow } from "@/workflows/import-loom-video"; type LoomDownloadMode = "direct-download" | "browser-conversion"; @@ -253,6 +257,7 @@ async function importLoomVideoForOwner({ videoId: videos.id, ownerId: videos.ownerId, bucketId: videos.bucket, + metadata: videos.metadata, phase: videoUploads.phase, rawFileKey: videoUploads.rawFileKey, uploadUpdatedAt: videoUploads.updatedAt, @@ -281,9 +286,16 @@ async function importLoomVideoForOwner({ existingVideo.ownerId === ownerId && existingVideo.uploadUpdatedAt ) { + if (await isLoomImportRunning(existingVideo.metadata?.loomImportRun)) { + return { + success: false, + error: "This Loom import is already restarting.", + }; + } const rawFileKey = existingVideo.rawFileKey ?? `${ownerId}/${existingVideo.videoId}/raw-upload.mp4`; + const claimedAt = new Date(Math.floor(Date.now() / 1000) * 1000); const claim = await db() .update(videoUploads) .set({ @@ -292,7 +304,7 @@ async function importLoomVideoForOwner({ processingMessage: "Restarting Loom import...", processingError: null, rawFileKey, - updatedAt: new Date(), + updatedAt: claimedAt, }) .where( and( @@ -310,16 +322,32 @@ async function importLoomVideoForOwner({ error: "This Loom import is already restarting.", }; } - await start(importLoomVideoWorkflow, [ - { + try { + await startLoomImportWorkflow({ videoId: existingVideo.videoId, userId: ownerId, rawFileKey, bucketId: existingVideo.bucketId, loomVideoId, reuseExistingRawUpload: true, - }, - ]); + }); + } catch (error) { + if (error instanceof LoomImportStartError && error.canRetry) { + await restoreLoomImportStartError( + existingVideo.videoId, + "processing", + claimedAt, + error.message, + ); + } + return { + success: false, + error: + error instanceof Error + ? error.message + : "Loom import could not restart.", + }; + } revalidatePath("/dashboard/caps"); return { success: true, videoId: existingVideo.videoId }; } @@ -389,6 +417,7 @@ async function importLoomVideoForOwner({ videoName || `Loom Import - ${new Date().toLocaleDateString("en-US", { day: "numeric", month: "long", year: "numeric" })}`; + const claimedAt = new Date(Math.floor(Date.now() / 1000) * 1000); await db().transaction(async (tx) => { await tx.insert(videos).values({ id: videoId, @@ -410,6 +439,7 @@ async function importLoomVideoForOwner({ phase: "uploading", processingProgress: 0, processingMessage: "Importing from Loom...", + updatedAt: claimedAt, }); await tx.insert(importedVideos).values({ @@ -450,16 +480,31 @@ async function importLoomVideoForOwner({ .catch(() => {}); } - await start(importLoomVideoWorkflow, [ - { + try { + await startLoomImportWorkflow({ videoId, userId: ownerId, rawFileKey, bucketId: Option.getOrNull(writable.bucketId), loomVideoId, loomDownloadUrl: downloadUrl, - }, - ]); + }); + } catch (error) { + if (error instanceof LoomImportStartError && error.canRetry) { + await restoreLoomImportStartError( + videoId, + "uploading", + claimedAt, + error.message, + ); + } + return { + success: false, + videoId, + error: + error instanceof Error ? error.message : "Loom import could not start.", + }; + } revalidatePath("/dashboard/caps"); if (destination.folderId) revalidatePath("/dashboard/folder/[id]", "page"); diff --git a/apps/web/actions/video/retry-processing.ts b/apps/web/actions/video/retry-processing.ts index c439ce84338..2081ed8f81b 100644 --- a/apps/web/actions/video/retry-processing.ts +++ b/apps/web/actions/video/retry-processing.ts @@ -5,13 +5,16 @@ import { getCurrentUser } from "@cap/database/auth/session"; import { importedVideos, videos, videoUploads } from "@cap/database/schema"; import type { Video } from "@cap/web-domain"; import { and, eq } from "drizzle-orm"; -import { start } from "workflow/api"; import { - setVideoProcessingError, + isLoomImportRunning, + LoomImportStartError, + restoreLoomImportStartError, + startLoomImportWorkflow, +} from "@/lib/loom-import-start"; +import { startVideoProcessingWorkflow, type VideoProcessingStartStatus, } from "@/lib/video-processing"; -import { importLoomVideoWorkflow } from "@/workflows/import-loom-video"; const SECOND = 1000; const MINUTE = 60 * SECOND; @@ -78,12 +81,18 @@ export async function retryVideoProcessing({ ); if (importedVideo?.source === "loom") { - if (upload.phase === "processing" && !shouldForceRetryProcessing(upload)) { + if ( + (await isLoomImportRunning(video.metadata?.loomImportRun)) || + ((upload.phase === "processing" || + upload.phase === "generating_thumbnail") && + !shouldForceRetryProcessing(upload)) + ) { return { success: true, status: "already-processing" }; } const rawFileKey = upload.rawFileKey ?? `${video.ownerId}/${videoId}/raw-upload.mp4`; + const claimedAt = new Date(Math.floor(Date.now() / 1000) * 1000); const claim = await db() .update(videoUploads) .set({ @@ -92,7 +101,7 @@ export async function retryVideoProcessing({ processingMessage: "Retrying Loom import...", processingError: null, rawFileKey, - updatedAt: new Date(), + updatedAt: claimedAt, }) .where( and( @@ -109,27 +118,28 @@ export async function retryVideoProcessing({ } try { - await start(importLoomVideoWorkflow, [ - { - videoId, - userId: user.id, - rawFileKey, - bucketId: video.bucket ?? null, - loomDownloadUrl: "", - loomVideoId: importedVideo.sourceId, - reuseExistingRawUpload: true, - }, - ]); + await startLoomImportWorkflow({ + videoId, + userId: user.id, + rawFileKey, + bucketId: video.bucket ?? null, + loomDownloadUrl: "", + loomVideoId: importedVideo.sourceId, + reuseExistingRawUpload: true, + }); } catch (error) { const normalizedError = error instanceof Error ? error : new Error("Loom import could not restart"); - await setVideoProcessingError( - videoId, - "Loom import could not restart.", - normalizedError, - ); + if (error instanceof LoomImportStartError && error.canRetry) { + await restoreLoomImportStartError( + videoId, + "processing", + claimedAt, + error.message, + ); + } throw normalizedError; } diff --git a/apps/web/lib/loom-download-url.ts b/apps/web/lib/loom-download-url.ts index 134ff6b9c35..61727eb3d3b 100644 --- a/apps/web/lib/loom-download-url.ts +++ b/apps/web/lib/loom-download-url.ts @@ -12,7 +12,7 @@ export function getReusableLoomDownloadUrl(value: string | undefined) { const url = new URL(value); if ( url.protocol !== "https:" || - url.hostname !== "cdn.loom.com" || + !["cdn.loom.com", "luna.loom.com"].includes(url.hostname) || url.username || url.password ) { diff --git a/apps/web/lib/loom-import-start.ts b/apps/web/lib/loom-import-start.ts new file mode 100644 index 00000000000..952740de739 --- /dev/null +++ b/apps/web/lib/loom-import-start.ts @@ -0,0 +1,149 @@ +import { db } from "@cap/database"; +import { videos, videoUploads } from "@cap/database/schema"; +import type { VideoMetadata } from "@cap/database/types"; +import { User, Video } from "@cap/web-domain"; +import { and, eq, sql } from "drizzle-orm"; +import { getRun, start } from "workflow/api"; +import { getWorld } from "workflow/runtime"; +import { importLoomVideoWorkflow } from "@/workflows/import-loom-video"; + +type LoomImportRun = NonNullable; +type ImportPayload = Parameters[0]; + +export async function restoreLoomImportStartError( + videoId: string, + phase: "uploading" | "processing", + claimedAt: Date, + message: string, +) { + await db() + .update(videoUploads) + .set({ + phase: "error", + processingProgress: 0, + processingMessage: null, + processingError: message, + updatedAt: new Date(), + }) + .where( + and( + eq(videoUploads.videoId, Video.VideoId.make(videoId)), + eq(videoUploads.phase, phase), + eq(videoUploads.updatedAt, claimedAt), + ), + ); +} + +export class LoomImportStartError extends Error { + constructor( + readonly canRetry: boolean, + cause: unknown, + ) { + super( + canRetry + ? "Loom import could not start. Please try again." + : "We are confirming whether your Loom import started. Please check its progress before retrying.", + { cause }, + ); + this.name = "LoomImportStartError"; + } +} + +export async function isLoomImportRunning(receipt?: LoomImportRun) { + if (!receipt || receipt.dispatch === "rejected") return false; + try { + const status = await getRun(receipt.runId).status; + return ( + status !== "completed" && status !== "failed" && status !== "cancelled" + ); + } catch { + return true; + } +} + +async function saveReceipt(payload: ImportPayload, receipt: LoomImportRun) { + const result = await db() + .update(videos) + .set({ + metadata: sql`JSON_SET(COALESCE(${videos.metadata}, JSON_OBJECT()), '$.loomImportRun', JSON_EXTRACT(${JSON.stringify(receipt)}, '$'))`, + }) + .where( + and( + eq(videos.id, Video.VideoId.make(payload.videoId)), + eq(videos.ownerId, User.UserId.make(payload.userId)), + ), + ); + const affectedRows = Array.isArray(result) + ? (result[0] as { affectedRows?: number } | undefined)?.affectedRows + : (result as { affectedRows?: number }).affectedRows; + if (affectedRows !== 1) + throw new Error("Could not record Loom import startup"); +} + +function isQueueRejection(error: unknown) { + return ( + error instanceof Error && + [ + "BadRequestError", + "UnauthorizedError", + "ForbiddenError", + "TooManyRequestsError", + ].includes(error.name) + ); +} + +export async function startLoomImportWorkflow(payload: ImportPayload) { + let attempted = false; + let accepted = false; + let rejected = false; + let receipt: LoomImportRun | undefined; + try { + const world = getWorld(); + const observedWorld: ReturnType = { + ...world, + queue: async (name, message, options) => { + if (!("runId" in message)) + throw new Error("Missing Loom import run ID"); + receipt = { runId: message.runId, dispatch: "pending" }; + await saveReceipt(payload, receipt); + attempted = true; + try { + const result = await world.queue(name, message, options); + accepted = true; + return result; + } catch (error) { + rejected = isQueueRejection(error); + throw error; + } + }, + }; + await start(importLoomVideoWorkflow, [payload], { world: observedWorld }); + } catch (error) { + if (!accepted) { + let canRetry = !attempted || rejected; + if (receipt) { + try { + await saveReceipt(payload, { + ...receipt, + dispatch: canRetry ? "rejected" : "uncertain", + }); + } catch (receiptError) { + canRetry = false; + console.error( + "Could not record Loom import dispatch failure", + receiptError, + ); + } + } + throw new LoomImportStartError(canRetry, error); + } + console.error("Loom import queued despite a startup error", error); + } + if (receipt) { + await saveReceipt(payload, { ...receipt, dispatch: "accepted" }).catch( + (error: unknown) => { + console.error("Could not record accepted Loom import dispatch", error); + }, + ); + } +} diff --git a/packages/database/types/metadata.ts b/packages/database/types/metadata.ts index b8e4c1df93c..78280fef68b 100644 --- a/packages/database/types/metadata.ts +++ b/packages/database/types/metadata.ts @@ -6,6 +6,10 @@ * Video metadata structure */ export interface VideoMetadata { + loomImportRun?: { + runId: string; + dispatch: "pending" | "accepted" | "rejected" | "uncertain"; + }; editProcessing?: { token: string; startedAt: string; From 37311921c28acf0d9fbcfcb7936a6d7344c989d1 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Mon, 21 Sep 2026 21:47:50 +0100 Subject: [PATCH 04/21] fix: protect concurrent Loom import receipts --- apps/web/__tests__/unit/loom-import-start.test.ts | 10 ++++++++++ apps/web/lib/loom-import-start.ts | 3 +++ 2 files changed, 13 insertions(+) diff --git a/apps/web/__tests__/unit/loom-import-start.test.ts b/apps/web/__tests__/unit/loom-import-start.test.ts index 95e82c130a6..e5a644cdd53 100644 --- a/apps/web/__tests__/unit/loom-import-start.test.ts +++ b/apps/web/__tests__/unit/loom-import-start.test.ts @@ -93,6 +93,16 @@ describe("Loom workflow startup receipts", () => { ); }); + it("does not overwrite a newer import startup receipt", async () => { + mocks.where + .mockResolvedValueOnce([{ affectedRows: 1 }]) + .mockResolvedValueOnce([{ affectedRows: 0 }]); + await expect(startLoomImportWorkflow(payload)).resolves.toBeUndefined(); + const condition = mocks.where.mock.calls.at(-1)?.[0].conditions.at(-1); + expect(condition.values).toEqual(["metadata", "run-1"]); + expect(condition.strings.join("")).toContain("$.loomImportRun.runId"); + }); + it.each([ "BadRequestError", "UnauthorizedError", diff --git a/apps/web/lib/loom-import-start.ts b/apps/web/lib/loom-import-start.ts index 952740de739..3b3725288ef 100644 --- a/apps/web/lib/loom-import-start.ts +++ b/apps/web/lib/loom-import-start.ts @@ -71,6 +71,9 @@ async function saveReceipt(payload: ImportPayload, receipt: LoomImportRun) { and( eq(videos.id, Video.VideoId.make(payload.videoId)), eq(videos.ownerId, User.UserId.make(payload.userId)), + receipt.dispatch === "pending" + ? undefined + : sql`JSON_UNQUOTE(JSON_EXTRACT(${videos.metadata}, '$.loomImportRun.runId')) = ${receipt.runId}`, ), ); const affectedRows = Array.isArray(result) From 1cc0e437e4250266d9eb8f8b5de36ab78304b32a Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Mon, 21 Sep 2026 21:53:10 +0100 Subject: [PATCH 05/21] fix: retry unavailable public Loom sources --- .../__tests__/unit/loom-download-url.test.ts | 28 ++++++++++++++++++- apps/web/lib/loom-download-url.ts | 7 +++-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/apps/web/__tests__/unit/loom-download-url.test.ts b/apps/web/__tests__/unit/loom-download-url.test.ts index cca084b6e1f..a6786025939 100644 --- a/apps/web/__tests__/unit/loom-download-url.test.ts +++ b/apps/web/__tests__/unit/loom-download-url.test.ts @@ -36,7 +36,7 @@ describe("getLoomDownloadUrl", () => { }); }); - it("does not classify an unavailable source as a temporary failure", async () => { + it("does not retry a missing or inaccessible video", async () => { const fetch = vi .fn() .mockResolvedValue(new Response(null, { status: 204 })); @@ -139,6 +139,32 @@ describe("getLoomDownloadUrl", () => { ); }); + it.each([null, {}])( + "retries a public downloadable video while its source is unavailable: %j", + async (source) => { + vi.stubGlobal( + "fetch", + vi.fn(async (input) => + String(input) === "https://www.loom.com/graphql" + ? Response.json({ + data: { + getVideo: { + id: "video", + downloadable: true, + download_enabled: true, + nullableRawCdnUrl: source, + }, + }, + }) + : new Response(null, { status: 204 }), + ), + ); + await expect(getLoomDownloadUrl("video")).rejects.toBeInstanceOf( + LoomDownloadTemporaryError, + ); + }, + ); + it.each([ { id: "video", downloadable: false, download_enabled: false }, { id: "different-video", downloadable: true, download_enabled: true }, diff --git a/apps/web/lib/loom-download-url.ts b/apps/web/lib/loom-download-url.ts index 61727eb3d3b..1efde716b32 100644 --- a/apps/web/lib/loom-download-url.ts +++ b/apps/web/lib/loom-download-url.ts @@ -225,7 +225,8 @@ async function fetchPublicLoomPlaybackUrl( ) return null; const source = video.nullableRawCdnUrl; - if (typeof source !== "object" || source === null) return null; + if (typeof source !== "object" || source === null) + throw new LoomDownloadTemporaryError(60_000); const credentials = "credentials" in source ? source.credentials : null; const partCredentials = typeof credentials === "object" && @@ -239,10 +240,12 @@ async function fetchPublicLoomPlaybackUrl( "Key-Pair-Id": credentials.KeyPairId, } : null; - return parseLoomDownloadResponse({ + const url = parseLoomDownloadResponse({ ...source, part_credentials: partCredentials, }); + if (!url) throw new LoomDownloadTemporaryError(60_000); + return url; } catch { throw new LoomDownloadTemporaryError(60_000); } From d12a17dd288d990fd2816dc83875491087f0e432 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 21 Oct 2026 04:15:39 +0800 Subject: [PATCH 06/21] fix: run Loom failed-import restarts through the concierge lease and request bookkeeping --- apps/web/__tests__/unit/loom-import.test.ts | 239 ++++++++++++++++++++ apps/web/actions/loom.ts | 155 ++++++++----- 2 files changed, 341 insertions(+), 53 deletions(-) diff --git a/apps/web/__tests__/unit/loom-import.test.ts b/apps/web/__tests__/unit/loom-import.test.ts index fb81787235f..50f4e1e44b1 100644 --- a/apps/web/__tests__/unit/loom-import.test.ts +++ b/apps/web/__tests__/unit/loom-import.test.ts @@ -17,6 +17,12 @@ const restoreLoomImportStartErrorMock = vi.hoisted(() => vi.fn()); const defaultSharing = vi.hoisted(() => ({ getNewVideoPublic: vi.fn(), })); +const conciergeMocks = vi.hoisted(() => ({ + requireMigrationOperator: vi.fn(), + reserveConciergeImport: vi.fn(), + renewConciergeImport: vi.fn(), + releaseConciergeImport: vi.fn(), +})); const mockDb = { select: vi.fn(() => mockDb), @@ -63,6 +69,19 @@ vi.mock("@cap/database/schema", () => ({ source: "source", sourceId: "sourceId", }, + loomMigrationImports: { + videoId: "loomMigrationImportVideoId", + requestId: "loomMigrationImportRequestId", + }, + loomMigrationRequests: { + id: "loomMigrationRequestId", + organizationId: "loomMigrationRequestOrganizationId", + activeOrganizationId: "loomMigrationRequestActiveOrganizationId", + status: "loomMigrationRequestStatus", + activeImportLeaseToken: "loomMigrationRequestLeaseToken", + activeImportLeaseUntil: "loomMigrationRequestLeaseUntil", + queuedVideoCount: "loomMigrationRequestQueuedVideoCount", + }, organizationMembers: { id: "memberId", userId: "memberUserId", @@ -174,6 +193,11 @@ vi.mock("drizzle-orm", () => ({ and: vi.fn((...args: unknown[]) => args), eq: vi.fn((field: unknown, value: unknown) => ({ field, value })), isNull: vi.fn((field: unknown) => ({ field })), + isNotNull: vi.fn((field: unknown) => ({ field })), + sql: vi.fn((strings: TemplateStringsArray, ...values: unknown[]) => ({ + strings, + values, + })), })); vi.mock("next/cache", () => ({ @@ -211,6 +235,25 @@ vi.mock("@/workflows/import-loom-video", () => ({ importLoomVideoWorkflow: Symbol("importLoomVideoWorkflow"), })); +vi.mock("@/lib/loom-concierge", () => ({ + ...conciergeMocks, + toLoomMigrationResult: async (run: () => Promise) => { + const { LoomMigrationError } = await import("@/lib/loom-migration-state"); + try { + return { ok: true, value: await run() }; + } catch (error) { + if (error instanceof LoomMigrationError) { + return { ok: false, error: error.message }; + } + throw error; + } + }, +})); + +vi.mock("@/lib/org-pro", () => ({ + isOrganizationOwnerPro: vi.fn(async () => true), +})); + vi.mock("@/lib/loom-import-start", () => ({ isLoomImportRunning: isLoomImportRunningMock, restoreLoomImportStartError: restoreLoomImportStartErrorMock, @@ -1297,3 +1340,199 @@ describe("importFromLoom", () => { expect(revalidatePathMock).toHaveBeenCalledWith("/dashboard"); }); }); + +describe("importFromLoomCsvForConcierge", () => { + const requestId = "request12345678"; + const leaseToken = "lease-1"; + const failedImport = { + videoId: "existing-video", + ownerId: "user-123", + bucketId: "bucket-1", + phase: "error", + rawFileKey: null, + uploadUpdatedAt: new Date("2026-09-01T00:00:00Z"), + }; + const forUpdateMock = vi.fn(); + + function withReservation(reservation: unknown) { + forUpdateMock.mockResolvedValueOnce([reservation]); + return { limit: vi.fn(() => ({ for: forUpdateMock })) }; + } + + function queueConciergeRowLookups() { + whereMock + .mockReturnValueOnce( + withLimit([ + { + organizationId: "org-1", + status: "in_progress", + ownerId: "owner-1", + }, + ]), + ) + .mockReturnValueOnce( + withLimit([{ userId: "user-123", email: "member@example.com" }]), + ) + .mockResolvedValueOnce([failedImport]); + } + + async function importConciergeRow() { + const { importFromLoomCsvForConcierge } = await import("@/actions/loom"); + return importFromLoomCsvForConcierge({ + requestId, + rows: [ + { + rowNumber: 2, + loomUrl: "https://www.loom.com/share/loom-abc1234567", + userEmail: "member@example.com", + }, + ], + }); + } + + beforeEach(() => { + vi.clearAllMocks(); + whereMock.mockReset(); + valuesMock.mockReset(); + forUpdateMock.mockReset(); + mockDb.select.mockReturnValue(mockDb); + mockDb.insert.mockReturnValue(mockDb); + mockDb.update.mockReturnValue(mockDb); + mockDb.from.mockReturnValue(mockDb); + mockDb.innerJoin.mockReturnValue(mockDb); + mockDb.leftJoin.mockReturnValue(mockDb); + mockDb.set.mockReturnValue(mockDb); + mockDb.transaction.mockImplementation((callback) => callback(mockDb)); + valuesMock.mockResolvedValue(undefined); + whereMock.mockResolvedValue([]); + startMock.mockResolvedValue(undefined); + isLoomImportRunningMock.mockResolvedValue(false); + conciergeMocks.requireMigrationOperator.mockResolvedValue({ + id: "operator-1", + }); + conciergeMocks.reserveConciergeImport.mockResolvedValue(leaseToken); + conciergeMocks.renewConciergeImport.mockResolvedValue(undefined); + conciergeMocks.releaseConciergeImport.mockResolvedValue(undefined); + }); + + it("records a restarted failed import against the migration request under the lease", async () => { + queueConciergeRowLookups(); + whereMock + .mockReturnValueOnce( + withReservation({ + activeOrganizationId: "org-1", + leaseToken, + leaseUntil: new Date(Date.now() + 60_000), + }), + ) + .mockReturnValueOnce(withLimit([])) + .mockResolvedValueOnce([{ affectedRows: 1 }]); + + const result = await importConciergeRow(); + + expect(result).toMatchObject({ + ok: true, + value: { + success: true, + importedCount: 1, + results: [{ success: true, videoId: "existing-video" }], + }, + }); + expect(forUpdateMock).toHaveBeenCalledWith("update"); + expect(mockDb.set).toHaveBeenCalledWith( + expect.objectContaining({ phase: "processing" }), + ); + expect(valuesMock).toHaveBeenCalledWith({ + videoId: "existing-video", + requestId, + }); + expect(mockDb.set).toHaveBeenCalledWith( + expect.objectContaining({ queuedVideoCount: expect.anything() }), + ); + expect(startMock).toHaveBeenCalledWith(expect.anything(), [ + expect.objectContaining({ + videoId: "existing-video", + reuseExistingRawUpload: true, + }), + ]); + expect(conciergeMocks.releaseConciergeImport).toHaveBeenCalledWith( + requestId, + leaseToken, + ); + }); + + it("does not count a failed import twice when it already belongs to the request", async () => { + queueConciergeRowLookups(); + whereMock + .mockReturnValueOnce( + withReservation({ + activeOrganizationId: "org-1", + leaseToken, + leaseUntil: new Date(Date.now() + 60_000), + }), + ) + .mockReturnValueOnce(withLimit([{ requestId }])) + .mockResolvedValueOnce([{ affectedRows: 1 }]); + + const result = await importConciergeRow(); + + expect(result).toMatchObject({ ok: true, value: { importedCount: 1 } }); + expect(valuesMock).not.toHaveBeenCalled(); + expect(mockDb.set).not.toHaveBeenCalledWith( + expect.objectContaining({ queuedVideoCount: expect.anything() }), + ); + expect(startMock).toHaveBeenCalledTimes(1); + }); + + it("does not restart a failed import once the concierge lease has expired", async () => { + queueConciergeRowLookups(); + whereMock.mockReturnValueOnce( + withReservation({ + activeOrganizationId: "org-1", + leaseToken: "newer-lease", + leaseUntil: new Date(Date.now() + 60_000), + }), + ); + + const result = await importConciergeRow(); + + expect(result).toMatchObject({ + ok: true, + value: { success: false, importedCount: 0, failedCount: 1 }, + }); + expect(mockDb.update).not.toHaveBeenCalled(); + expect(valuesMock).not.toHaveBeenCalled(); + expect(startMock).not.toHaveBeenCalled(); + }); + + it("refuses to restart a failed import that belongs to another migration request", async () => { + queueConciergeRowLookups(); + whereMock + .mockReturnValueOnce( + withReservation({ + activeOrganizationId: "org-1", + leaseToken, + leaseUntil: new Date(Date.now() + 60_000), + }), + ) + .mockReturnValueOnce(withLimit([{ requestId: "otherrequest123" }])); + + const result = await importConciergeRow(); + + expect(result).toMatchObject({ + ok: true, + value: { + success: false, + results: [ + { + success: false, + error: "This Loom import belongs to another migration request.", + }, + ], + }, + }); + expect(mockDb.update).not.toHaveBeenCalled(); + expect(valuesMock).not.toHaveBeenCalled(); + expect(startMock).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/actions/loom.ts b/apps/web/actions/loom.ts index e81c3461274..b140e85c3de 100644 --- a/apps/web/actions/loom.ts +++ b/apps/web/actions/loom.ts @@ -255,6 +255,53 @@ export async function downloadLoomVideo( } } +type LoomImportTransaction = Parameters< + Parameters["transaction"]>[0] +>[0]; + +async function assertConciergeLease( + tx: LoomImportTransaction, + requestId: string, + leaseToken: string | undefined, +) { + const [reservation] = await tx + .select({ + activeOrganizationId: loomMigrationRequests.activeOrganizationId, + leaseToken: loomMigrationRequests.activeImportLeaseToken, + leaseUntil: loomMigrationRequests.activeImportLeaseUntil, + }) + .from(loomMigrationRequests) + .where(eq(loomMigrationRequests.id, requestId)) + .limit(1) + .for("update"); + if ( + !leaseToken || + !reservation?.activeOrganizationId || + reservation.leaseToken !== leaseToken || + !reservation.leaseUntil || + reservation.leaseUntil <= new Date() + ) { + throw new LoomMigrationError( + "The concierge import expired. Retry this video.", + ); + } +} + +async function recordConciergeImport( + tx: LoomImportTransaction, + videoId: Video.VideoId, + requestId: string, +) { + await tx.insert(loomMigrationImports).values({ videoId, requestId }); + await tx + .update(loomMigrationRequests) + .set({ + queuedVideoCount: sql`${loomMigrationRequests.queuedVideoCount} + 1`, + updatedAt: new Date(), + }) + .where(eq(loomMigrationRequests.id, requestId)); +} + async function importLoomVideoForOwner({ loomUrl, orgId, @@ -322,28 +369,60 @@ async function importLoomVideoForOwner({ const rawFileKey = existingVideo.rawFileKey ?? `${ownerId}/${existingVideo.videoId}/raw-upload.mp4`; + const failedVideoId = existingVideo.videoId; + const failedUpdatedAt = existingVideo.uploadUpdatedAt; const claimedAt = new Date(Math.floor(Date.now() / 1000) * 1000); - const claim = await db() - .update(videoUploads) - .set({ - phase: "processing", - processingProgress: 0, - processingMessage: "Restarting Loom import...", - processingError: null, - rawFileKey, - updatedAt: claimedAt, - }) - .where( - and( - eq(videoUploads.videoId, existingVideo.videoId), - eq(videoUploads.phase, "error"), - eq(videoUploads.updatedAt, existingVideo.uploadUpdatedAt), - ), - ); - const affectedRows = Array.isArray(claim) - ? (claim[0] as { affectedRows?: number } | undefined)?.affectedRows - : (claim as { affectedRows?: number }).affectedRows; - if (affectedRows !== 1) { + const claimOutcome = await db().transaction(async (tx) => { + let mappedRequestId: string | undefined; + if (migrationRequestId) { + await assertConciergeLease( + tx, + migrationRequestId, + migrationLeaseToken, + ); + const [mapped] = await tx + .select({ requestId: loomMigrationImports.requestId }) + .from(loomMigrationImports) + .where(eq(loomMigrationImports.videoId, failedVideoId)) + .limit(1); + mappedRequestId = mapped?.requestId; + if (mappedRequestId && mappedRequestId !== migrationRequestId) { + return "foreign" as const; + } + } + const claim = await tx + .update(videoUploads) + .set({ + phase: "processing", + processingProgress: 0, + processingMessage: "Restarting Loom import...", + processingError: null, + rawFileKey, + updatedAt: claimedAt, + }) + .where( + and( + eq(videoUploads.videoId, failedVideoId), + eq(videoUploads.phase, "error"), + eq(videoUploads.updatedAt, failedUpdatedAt), + ), + ); + const affectedRows = Array.isArray(claim) + ? (claim[0] as { affectedRows?: number } | undefined)?.affectedRows + : (claim as { affectedRows?: number }).affectedRows; + if (affectedRows !== 1) return "busy" as const; + if (migrationRequestId && !mappedRequestId) { + await recordConciergeImport(tx, failedVideoId, migrationRequestId); + } + return "claimed" as const; + }); + if (claimOutcome === "foreign") { + return { + success: false, + error: "This Loom import belongs to another migration request.", + }; + } + if (claimOutcome === "busy") { return { success: false, error: "This Loom import is already restarting.", @@ -447,27 +526,7 @@ async function importLoomVideoForOwner({ const claimedAt = new Date(Math.floor(Date.now() / 1000) * 1000); await db().transaction(async (tx) => { if (migrationRequestId) { - const [reservation] = await tx - .select({ - activeOrganizationId: loomMigrationRequests.activeOrganizationId, - leaseToken: loomMigrationRequests.activeImportLeaseToken, - leaseUntil: loomMigrationRequests.activeImportLeaseUntil, - }) - .from(loomMigrationRequests) - .where(eq(loomMigrationRequests.id, migrationRequestId)) - .limit(1) - .for("update"); - if ( - !migrationLeaseToken || - !reservation?.activeOrganizationId || - reservation.leaseToken !== migrationLeaseToken || - !reservation.leaseUntil || - reservation.leaseUntil <= new Date() - ) { - throw new LoomMigrationError( - "The concierge import expired. Retry this video.", - ); - } + await assertConciergeLease(tx, migrationRequestId, migrationLeaseToken); } await tx.insert(videos).values({ id: videoId, @@ -499,17 +558,7 @@ async function importLoomVideoForOwner({ sourceId: loomVideoId, }); if (migrationRequestId) { - await tx.insert(loomMigrationImports).values({ - videoId, - requestId: migrationRequestId, - }); - await tx - .update(loomMigrationRequests) - .set({ - queuedVideoCount: sql`${loomMigrationRequests.queuedVideoCount} + 1`, - updatedAt: new Date(), - }) - .where(eq(loomMigrationRequests.id, migrationRequestId)); + await recordConciergeImport(tx, videoId, migrationRequestId); } if (destination.spaceId === orgId) { From e3a4a95fdbb837c52dfcbda2444386b9da184fe1 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:55:23 +0800 Subject: [PATCH 07/21] fix: preserve active Loom workflows during automatic recovery --- .../unit/video-processing-recovery.test.ts | 161 +++++++++++++++++- apps/web/lib/video-processing-recovery.ts | 47 +++-- 2 files changed, 191 insertions(+), 17 deletions(-) diff --git a/apps/web/__tests__/unit/video-processing-recovery.test.ts b/apps/web/__tests__/unit/video-processing-recovery.test.ts index e55f634d71c..d61d53b28e5 100644 --- a/apps/web/__tests__/unit/video-processing-recovery.test.ts +++ b/apps/web/__tests__/unit/video-processing-recovery.test.ts @@ -1,8 +1,11 @@ +import type { VideoMetadata } from "@cap/database/types"; import { beforeEach, describe, expect, it, vi } from "vitest"; const mockDb = vi.fn(); const mockStart = vi.fn(); const mockSetVideoProcessingError = vi.fn(); +const mockIsLoomImportRunning = vi.fn(); +const mockStartLoomImportWorkflow = vi.fn(); vi.mock("@cap/database", () => ({ db: mockDb, @@ -21,6 +24,7 @@ vi.mock("@cap/database/schema", () => ({ orgId: "videos.orgId", bucket: "videos.bucket", source: "videos.source", + metadata: "videos.metadata", }, videoUploads: { videoId: "videoUploads.videoId", @@ -53,6 +57,19 @@ vi.mock("@/lib/video-processing", () => ({ setVideoProcessingError: mockSetVideoProcessingError, })); +vi.mock("@/lib/loom-import-start", () => ({ + isLoomImportRunning: mockIsLoomImportRunning, + startLoomImportWorkflow: mockStartLoomImportWorkflow, + LoomImportStartError: class extends Error { + constructor( + readonly canRetry: boolean, + cause: unknown, + ) { + super("Loom import startup failed", { cause }); + } + }, +})); + vi.mock("@/workflows/process-video", () => ({ processVideoWorkflow: vi.fn(), })); @@ -66,6 +83,8 @@ type Candidate = { userId: string; bucketId: string | null; rawFileKey: string | null; + uploadUpdatedAt: Date; + metadata?: VideoMetadata; }; function makeSelectChain(candidates: unknown[]) { @@ -105,12 +124,15 @@ const candidate: Candidate = { userId: "user-1", bucketId: null, rawFileKey: "user-1/video-1/raw.mp4", + uploadUpdatedAt: new Date("2026-10-21T16:00:00Z"), }; beforeEach(() => { - vi.clearAllMocks(); + vi.resetAllMocks(); mockStart.mockResolvedValue({ runId: "run-1" }); mockSetVideoProcessingError.mockResolvedValue(undefined); + mockIsLoomImportRunning.mockResolvedValue(false); + mockStartLoomImportWorkflow.mockResolvedValue(undefined); }); describe("recoverFailedVideoProcessing", () => { @@ -189,14 +211,14 @@ describe("recoverFailedVideoProcessing", () => { const result = await recoverFailedVideoProcessing(); - expect(mockStart).toHaveBeenCalledWith(expect.any(Function), [ + expect(mockStartLoomImportWorkflow).toHaveBeenCalledWith( expect.objectContaining({ videoId: "video-1", loomVideoId: "loom-video-1", rawFileKey: "user-1/video-1/raw.mp4", reuseExistingRawUpload: true, }), - ]); + ); expect(result.statuses).toEqual({ started: 1 }); }); @@ -216,7 +238,10 @@ describe("recoverFailedVideoProcessing", () => { .mockReturnValueOnce(makeSelectChain([])) .mockReturnValueOnce(makeUpdateChain(1)) .mockReturnValueOnce(restore); - mockStart.mockRejectedValueOnce(new Error("temporary start failure")); + const { LoomImportStartError } = await import("@/lib/loom-import-start"); + mockStartLoomImportWorkflow.mockRejectedValueOnce( + new LoomImportStartError(true, new Error("temporary start failure")), + ); const { recoverFailedVideoProcessing } = await import( "@/lib/video-processing-recovery" ); @@ -231,4 +256,132 @@ describe("recoverFailedVideoProcessing", () => { ); expect(result.statuses).toEqual({ "retry-scheduled": 1 }); }); + + it.each(["pending", "accepted", "uncertain"] as const)( + "preserves a slow source lookup with a protected %s receipt", + async (dispatch) => { + const receipt = { runId: "slow-source-run", dispatch }; + mockDb + .mockReturnValueOnce( + makeSelectChain([ + { + ...candidate, + rawFileKey: null, + loomVideoId: "loom-video-1", + uploadUpdatedAt: new Date(Date.now() - 16 * 60 * 1000), + metadata: { loomImportRun: receipt }, + }, + ]), + ) + .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValueOnce(makeUpdateChain(1)); + mockIsLoomImportRunning.mockResolvedValueOnce(true); + const { recoverFailedVideoProcessing } = await import( + "@/lib/video-processing-recovery" + ); + + const result = await recoverFailedVideoProcessing(); + + expect(mockStart).not.toHaveBeenCalled(); + expect(mockStartLoomImportWorkflow).not.toHaveBeenCalled(); + expect(result.statuses).toEqual({ "already-claimed": 1 }); + expect(mockDb).toHaveBeenCalledTimes(3); + expect(mockIsLoomImportRunning).toHaveBeenCalledWith(receipt); + }, + ); + + it("rechecks the selected upload timestamp before a terminal run is recovered", async () => { + const claim = makeUpdateChain(0); + const receipt = { runId: "terminal-run", dispatch: "accepted" as const }; + mockDb + .mockReturnValueOnce( + makeSelectChain([ + { + ...candidate, + rawFileKey: null, + loomVideoId: "loom-video-1", + metadata: { loomImportRun: receipt }, + }, + ]), + ) + .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValueOnce(claim); + const { recoverFailedVideoProcessing } = await import( + "@/lib/video-processing-recovery" + ); + + const result = await recoverFailedVideoProcessing(); + + expect(claim.where).toHaveBeenCalledWith( + expect.arrayContaining([ + { left: "videoUploads.updatedAt", right: candidate.uploadUpdatedAt }, + ]), + ); + expect(mockStartLoomImportWorkflow).not.toHaveBeenCalled(); + expect(result.statuses).toEqual({ "already-claimed": 1 }); + }); + + it("retains the recovery claim when dispatch acknowledgement is uncertain", async () => { + mockDb + .mockReturnValueOnce( + makeSelectChain([ + { ...candidate, rawFileKey: null, loomVideoId: "loom-video-1" }, + ]), + ) + .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValueOnce(makeUpdateChain(1)); + const { LoomImportStartError } = await import("@/lib/loom-import-start"); + mockStartLoomImportWorkflow.mockRejectedValueOnce( + new LoomImportStartError(false, new Error("acknowledgement lost")), + ); + const { recoverFailedVideoProcessing } = await import( + "@/lib/video-processing-recovery" + ); + + const result = await recoverFailedVideoProcessing(); + + expect(mockDb).toHaveBeenCalledTimes(4); + expect(mockStart).not.toHaveBeenCalled(); + expect(result.statuses).toEqual({ "already-claimed": 1 }); + }); + + it("restores only the claimed pending upload after a confirmed queue rejection", async () => { + const claim = makeUpdateChain(1); + const restore = makeUpdateChain(1); + mockDb + .mockReturnValueOnce( + makeSelectChain([ + { ...candidate, rawFileKey: null, loomVideoId: "loom-video-1" }, + ]), + ) + .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValueOnce(claim) + .mockReturnValueOnce(restore); + const { LoomImportStartError } = await import("@/lib/loom-import-start"); + mockStartLoomImportWorkflow.mockRejectedValueOnce( + new LoomImportStartError(true, new Error("queue rejected")), + ); + const { recoverFailedVideoProcessing } = await import( + "@/lib/video-processing-recovery" + ); + + const result = await recoverFailedVideoProcessing(); + + expect(restore.set).toHaveBeenCalledWith( + expect.objectContaining({ phase: "uploading", rawFileKey: null }), + ); + expect(restore.where).toHaveBeenCalledWith( + expect.arrayContaining([ + { + left: "videoUploads.updatedAt", + right: claim.set.mock.calls[0]?.[0].updatedAt, + }, + ]), + ); + expect(result.statuses).toEqual({ "retry-scheduled": 1 }); + }); }); diff --git a/apps/web/lib/video-processing-recovery.ts b/apps/web/lib/video-processing-recovery.ts index c7fce3e6685..dee4b1d3732 100644 --- a/apps/web/lib/video-processing-recovery.ts +++ b/apps/web/lib/video-processing-recovery.ts @@ -1,5 +1,6 @@ import { db } from "@cap/database"; import { importedVideos, videos, videoUploads } from "@cap/database/schema"; +import type { VideoMetadata } from "@cap/database/types"; import type { S3Bucket, Video } from "@cap/web-domain"; import { and, @@ -14,9 +15,13 @@ import { sql, } from "drizzle-orm"; import { start } from "workflow/api"; +import { + isLoomImportRunning, + LoomImportStartError, + startLoomImportWorkflow, +} from "@/lib/loom-import-start"; import { setVideoProcessingError } from "@/lib/video-processing"; import { WORKFLOW_UPGRADE_ERROR_FRAGMENT } from "@/lib/workflow-recovery"; -import { importLoomVideoWorkflow } from "@/workflows/import-loom-video"; import { processVideoWorkflow } from "@/workflows/process-video"; export const VIDEO_PROCESSING_RECOVERY_BATCH_SIZE = 20; @@ -129,6 +134,8 @@ async function recoverLoomCandidate({ loomVideoId, rawFileKey: existingRawFileKey, phase, + uploadUpdatedAt, + receipt, }: { videoId: Video.VideoId; userId: string; @@ -136,9 +143,13 @@ async function recoverLoomCandidate({ loomVideoId: string; rawFileKey: string | null; phase: "uploading" | "error"; + uploadUpdatedAt: Date; + receipt: VideoMetadata["loomImportRun"]; }): Promise { + if (await isLoomImportRunning(receipt)) return "already-claimed"; const rawFileKey = existingRawFileKey ?? `${userId}/${videoId}/raw-upload.mp4`; + const claimedAt = new Date(Math.floor(Date.now() / 1000) * 1000); const claimResult = await db() .update(videoUploads) .set({ @@ -147,12 +158,13 @@ async function recoverLoomCandidate({ processingMessage: LOOM_IMPORT_RECOVERY_MESSAGE, processingError: null, rawFileKey, - updatedAt: new Date(), + updatedAt: claimedAt, }) .where( and( eq(videoUploads.videoId, videoId), eq(videoUploads.phase, phase), + eq(videoUploads.updatedAt, uploadUpdatedAt), ...(phase === "error" ? [ eq(videoUploads.rawFileKey, rawFileKey), @@ -170,19 +182,20 @@ async function recoverLoomCandidate({ } try { - await start(importLoomVideoWorkflow, [ - { - videoId, - userId, - rawFileKey, - bucketId, - loomDownloadUrl: "", - loomVideoId, - reuseExistingRawUpload: phase === "error", - }, - ]); + await startLoomImportWorkflow({ + videoId, + userId, + rawFileKey, + bucketId, + loomDownloadUrl: "", + loomVideoId, + reuseExistingRawUpload: phase === "error", + }); return "started"; } catch (error) { + if (!(error instanceof LoomImportStartError) || !error.canRetry) { + return "already-claimed"; + } const message = error instanceof Error ? error.message : String(error); if (phase === "error") { await db() @@ -199,6 +212,7 @@ async function recoverLoomCandidate({ and( eq(videoUploads.videoId, videoId), eq(videoUploads.phase, "processing"), + eq(videoUploads.updatedAt, claimedAt), eq(videoUploads.rawFileKey, rawFileKey), eq(videoUploads.processingMessage, LOOM_IMPORT_RECOVERY_MESSAGE), ), @@ -220,6 +234,7 @@ async function recoverLoomCandidate({ and( eq(videoUploads.videoId, videoId), eq(videoUploads.phase, "processing"), + eq(videoUploads.updatedAt, claimedAt), eq(videoUploads.rawFileKey, rawFileKey), eq(videoUploads.processingMessage, LOOM_IMPORT_RECOVERY_MESSAGE), ), @@ -243,6 +258,8 @@ export async function recoverFailedVideoProcessing({ bucketId: videos.bucket, loomVideoId: importedVideos.sourceId, rawFileKey: videoUploads.rawFileKey, + uploadUpdatedAt: videoUploads.updatedAt, + metadata: videos.metadata, }) .from(videos) .innerJoin(videoUploads, eq(videos.id, videoUploads.videoId)) @@ -273,6 +290,8 @@ export async function recoverFailedVideoProcessing({ orgId: videos.orgId, bucketId: videos.bucket, rawFileKey: videoUploads.rawFileKey, + uploadUpdatedAt: videoUploads.updatedAt, + metadata: videos.metadata, }) .from(videos) .innerJoin(videoUploads, eq(videos.id, videoUploads.videoId)) @@ -374,6 +393,8 @@ export async function recoverFailedVideoProcessing({ loomVideoId: candidate.loomVideoId, rawFileKey: candidate.rawFileKey, phase: candidate.phase, + uploadUpdatedAt: candidate.uploadUpdatedAt, + receipt: candidate.metadata?.loomImportRun, }); } catch (error) { status = "failed"; From bd36b8f68f4c7d3c34039cab8a35154d14b6e0f1 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 22 Oct 2026 02:24:15 +0800 Subject: [PATCH 08/21] improve: reuse fresh Loom sources while worker capacity is full --- .../unit/media-processing-workflows.test.ts | 47 +++++++++++++++++++ apps/web/workflows/import-loom-video.ts | 7 +-- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/apps/web/__tests__/unit/media-processing-workflows.test.ts b/apps/web/__tests__/unit/media-processing-workflows.test.ts index c41116c02a9..07b8c3ff1e2 100644 --- a/apps/web/__tests__/unit/media-processing-workflows.test.ts +++ b/apps/web/__tests__/unit/media-processing-workflows.test.ts @@ -62,6 +62,7 @@ vi.mock("@/lib/ai-generation-entitlement", () => ({ vi.mock("workflow", () => ({ sleep: mocks.sleep, FatalError: class FatalError extends Error {}, + RetryableError: class RetryableError extends Error {}, })); import { importLoomVideoWorkflow } from "@/workflows/import-loom-video"; @@ -236,6 +237,52 @@ describe("media processing workflows", () => { ); }); + it.each([false, true])( + "refreshes a source waiting for worker capacity only when it expires: %s", + async (expiresDuringWait) => { + const now = Date.now(); + const clock = vi.spyOn(Date, "now").mockReturnValue(now); + const expires = Math.floor(now / 1000) + 7200; + const loomDownloadUrl = `https://cdn.loom.com/original.mp4?Expires=${expires}`; + let workerAttempts = 0; + mocks.rows = [[video], [video], [], [metadata]]; + mocks.fetch.mockImplementation(async (url: string) => { + if (url.startsWith("https://www.loom.com/")) { + return Response.json({ url: "https://cdn.loom.com/refreshed.mp4" }); + } + workerAttempts++; + return workerAttempts <= 2 + ? Response.json( + { code: "SERVER_BUSY", error: "Server is busy" }, + { status: 503 }, + ) + : Response.json({ jobId: "job-1" }); + }); + mocks.sleep.mockImplementationOnce(async () => { + if (expiresDuringWait) clock.mockReturnValue(now + 2 * 60 * 60 * 1000); + }); + try { + await importLoomVideoWorkflow({ ...payload, loomDownloadUrl }); + expect(workerAttempts).toBe(3); + expect( + mocks.fetch.mock.calls.filter(([url]) => + String(url).startsWith("https://www.loom.com/"), + ), + ).toHaveLength(expiresDuringWait ? 1 : 0); + expect( + JSON.parse(mocks.fetch.mock.calls.at(-1)?.[1].body), + ).toMatchObject({ + videoUrl: expiresDuringWait + ? "https://cdn.loom.com/refreshed.mp4" + : loomDownloadUrl, + priority: "bulk", + }); + } finally { + clock.mockRestore(); + } + }, + ); + it("retries a failed Loom worker from its preserved original", async () => { mocks.rows = [ [video], diff --git a/apps/web/workflows/import-loom-video.ts b/apps/web/workflows/import-loom-video.ts index e970f73262e..84119b9ab68 100644 --- a/apps/web/workflows/import-loom-video.ts +++ b/apps/web/workflows/import-loom-video.ts @@ -172,16 +172,17 @@ export async function importLoomVideoWorkflow( let metadata: ProcessedVideoMetadata; for (let processingAttempt = 0; ; processingAttempt++) { let capacityRetryCount = 0; + let previousSourceUrl = + processingAttempt === 0 ? payload.loomDownloadUrl : undefined; while (true) { try { const loomSourceUrl = processingInput.importFromLoom ? await fetchFreshLoomDownloadUrl( payload.loomVideoId, - processingAttempt === 0 && capacityRetryCount === 0 - ? payload.loomDownloadUrl - : undefined, + previousSourceUrl, ) : undefined; + previousSourceUrl = loomSourceUrl; await processVideoOnMediaServer(payload, loomSourceUrl); break; } catch (error) { From 6c15234ee4f864e02d27399ee315dde4275900a2 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 22 Oct 2026 03:06:46 +0800 Subject: [PATCH 09/21] fix: expose unconfirmed Loom startup for reconciliation --- .../__tests__/unit/loom-import-start.test.ts | 22 +- apps/web/__tests__/unit/loom-import.test.ts | 37 +++- .../unit/loom-retry-processing.test.ts | 20 +- .../unit/video-processing-recovery.test.ts | 197 +++++++++++++++++- apps/web/actions/loom.ts | 11 +- apps/web/actions/video/retry-processing.ts | 9 +- apps/web/lib/loom-import-start.ts | 22 +- apps/web/lib/video-processing-recovery.ts | 156 ++++++++++---- 8 files changed, 404 insertions(+), 70 deletions(-) diff --git a/apps/web/__tests__/unit/loom-import-start.test.ts b/apps/web/__tests__/unit/loom-import-start.test.ts index e5a644cdd53..e7b17ae9740 100644 --- a/apps/web/__tests__/unit/loom-import-start.test.ts +++ b/apps/web/__tests__/unit/loom-import-start.test.ts @@ -39,7 +39,7 @@ vi.mock("@/workflows/import-loom-video", () => ({ })); import { - isLoomImportRunning, + getLoomImportRunState, LoomImportStartError, restoreLoomImportStartError, startLoomImportWorkflow, @@ -190,8 +190,14 @@ describe("Loom workflow startup receipts", () => { async (status) => { mocks.getRun.mockReturnValue({ status: Promise.resolve(status) }); expect( - await isLoomImportRunning({ runId: "run-1", dispatch: "uncertain" }), - ).toBe(status === "pending" || status === "running"); + await getLoomImportRunState({ runId: "run-1", dispatch: "uncertain" }), + ).toBe( + status === "pending" + ? "pending" + : status === "running" + ? "active" + : "inactive", + ); }, ); @@ -200,15 +206,15 @@ describe("Loom workflow startup receipts", () => { status: Promise.reject(new Error("Not found yet")), }); expect( - await isLoomImportRunning({ runId: "run-1", dispatch: "uncertain" }), - ).toBe(true); + await getLoomImportRunState({ runId: "run-1", dispatch: "uncertain" }), + ).toBe("unconfirmed"); }); it("allows rejected and legacy imports without an active run check", async () => { expect( - await isLoomImportRunning({ runId: "run-1", dispatch: "rejected" }), - ).toBe(false); - expect(await isLoomImportRunning()).toBe(false); + await getLoomImportRunState({ runId: "run-1", dispatch: "rejected" }), + ).toBe("inactive"); + expect(await getLoomImportRunState()).toBe("inactive"); expect(mocks.getRun).not.toHaveBeenCalled(); }); diff --git a/apps/web/__tests__/unit/loom-import.test.ts b/apps/web/__tests__/unit/loom-import.test.ts index 50f4e1e44b1..274457c1a0a 100644 --- a/apps/web/__tests__/unit/loom-import.test.ts +++ b/apps/web/__tests__/unit/loom-import.test.ts @@ -12,7 +12,7 @@ const headersMock = vi.hoisted(() => vi.fn()); const getOrganizationAccessMock = vi.hoisted(() => vi.fn()); const requireSpaceManagerMock = vi.hoisted(() => vi.fn()); const requireOrganizationSettingsManagerMock = vi.hoisted(() => vi.fn()); -const isLoomImportRunningMock = vi.hoisted(() => vi.fn()); +const getLoomImportRunStateMock = vi.hoisted(() => vi.fn()); const restoreLoomImportStartErrorMock = vi.hoisted(() => vi.fn()); const defaultSharing = vi.hoisted(() => ({ getNewVideoPublic: vi.fn(), @@ -255,7 +255,8 @@ vi.mock("@/lib/org-pro", () => ({ })); vi.mock("@/lib/loom-import-start", () => ({ - isLoomImportRunning: isLoomImportRunningMock, + getLoomImportRunState: getLoomImportRunStateMock, + LOOM_IMPORT_CONFIRMATION_MESSAGE: "Confirmation required", restoreLoomImportStartError: restoreLoomImportStartErrorMock, startLoomImportWorkflow: (payload: unknown) => startMock(Symbol("workflow"), [payload]), @@ -297,7 +298,7 @@ describe("importFromLoom", () => { valuesMock.mockResolvedValue(undefined); whereMock.mockResolvedValue([]); startMock.mockResolvedValue(undefined); - isLoomImportRunningMock.mockResolvedValue(false); + getLoomImportRunStateMock.mockResolvedValue("inactive"); checkRateLimitMock.mockResolvedValue({ rateLimited: false }); getOrganizationAccessMock.mockResolvedValue({ id: "org-1", @@ -404,7 +405,7 @@ describe("importFromLoom", () => { metadata: { loomImportRun: receipt }, }, ]); - isLoomImportRunningMock.mockResolvedValueOnce(true); + getLoomImportRunStateMock.mockResolvedValueOnce("active"); const { importFromLoom } = await import("@/actions/loom"); expect( await importFromLoom({ @@ -412,7 +413,31 @@ describe("importFromLoom", () => { orgId: "org-1" as never, }), ).toMatchObject({ success: false }); - expect(isLoomImportRunningMock).toHaveBeenCalledWith(receipt); + expect(getLoomImportRunStateMock).toHaveBeenCalledWith(receipt); + expect(mockDb.update).not.toHaveBeenCalled(); + expect(startMock).not.toHaveBeenCalled(); + }); + + it("reports an unconfirmed dispatch without claiming it again", async () => { + whereMock.mockResolvedValueOnce([ + { + videoId: "existing-video", + ownerId: "user-123", + phase: "error", + uploadUpdatedAt: new Date(), + metadata: { + loomImportRun: { runId: "missing-run", dispatch: "uncertain" }, + }, + }, + ]); + getLoomImportRunStateMock.mockResolvedValueOnce("unconfirmed"); + const { importFromLoom } = await import("@/actions/loom"); + expect( + await importFromLoom({ + loomUrl: "https://www.loom.com/share/loom-abc1234567", + orgId: "org-1" as never, + }), + ).toMatchObject({ success: false, error: "Confirmation required" }); expect(mockDb.update).not.toHaveBeenCalled(); expect(startMock).not.toHaveBeenCalled(); }); @@ -1406,7 +1431,7 @@ describe("importFromLoomCsvForConcierge", () => { valuesMock.mockResolvedValue(undefined); whereMock.mockResolvedValue([]); startMock.mockResolvedValue(undefined); - isLoomImportRunningMock.mockResolvedValue(false); + getLoomImportRunStateMock.mockResolvedValue("inactive"); conciergeMocks.requireMigrationOperator.mockResolvedValue({ id: "operator-1", }); diff --git a/apps/web/__tests__/unit/loom-retry-processing.test.ts b/apps/web/__tests__/unit/loom-retry-processing.test.ts index 32f16f32cc4..57d8147e422 100644 --- a/apps/web/__tests__/unit/loom-retry-processing.test.ts +++ b/apps/web/__tests__/unit/loom-retry-processing.test.ts @@ -44,7 +44,8 @@ vi.mock("@/workflows/import-loom-video", () => ({ importLoomVideoWorkflow: vi.fn(), })); vi.mock("@/lib/loom-import-start", () => ({ - isLoomImportRunning: mocks.isRunning, + getLoomImportRunState: mocks.isRunning, + LOOM_IMPORT_CONFIRMATION_MESSAGE: "Confirmation required", restoreLoomImportStartError: mocks.restoreStartError, startLoomImportWorkflow: (payload: unknown) => mocks.start(Symbol("workflow"), [payload]), @@ -77,7 +78,7 @@ describe("Loom processing retries", () => { mocks.currentUser.mockResolvedValue({ id: "owner-1" }); mocks.set.mockReturnValue({ where: mocks.where }); mocks.start.mockResolvedValue({}); - mocks.isRunning.mockResolvedValue(false); + mocks.isRunning.mockResolvedValue("inactive"); }); it.each([true, false])( @@ -114,7 +115,7 @@ describe("Loom processing retries", () => { { ...upload, phase: "processing", updatedAt: new Date(0) }, ]) .mockResolvedValueOnce([{ source: "loom", sourceId: "loom-1" }]); - mocks.isRunning.mockResolvedValueOnce(true); + mocks.isRunning.mockResolvedValueOnce("active"); await expect(retryVideoProcessing({ videoId })).resolves.toEqual({ success: true, status: "already-processing", @@ -123,6 +124,19 @@ describe("Loom processing retries", () => { expect(mocks.start).not.toHaveBeenCalled(); }); + it("reports missing provider confirmation without another retry", async () => { + mocks.where + .mockResolvedValueOnce([video]) + .mockResolvedValueOnce([upload]) + .mockResolvedValueOnce([{ source: "loom", sourceId: "loom-1" }]); + mocks.isRunning.mockResolvedValueOnce("unconfirmed"); + await expect(retryVideoProcessing({ videoId })).rejects.toThrow( + "Confirmation required", + ); + expect(mocks.set).not.toHaveBeenCalled(); + expect(mocks.start).not.toHaveBeenCalled(); + }); + it("recovers a source lookup failure that has no raw file key", async () => { mocks.where .mockResolvedValueOnce([video]) diff --git a/apps/web/__tests__/unit/video-processing-recovery.test.ts b/apps/web/__tests__/unit/video-processing-recovery.test.ts index d61d53b28e5..c15b5266367 100644 --- a/apps/web/__tests__/unit/video-processing-recovery.test.ts +++ b/apps/web/__tests__/unit/video-processing-recovery.test.ts @@ -4,7 +4,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const mockDb = vi.fn(); const mockStart = vi.fn(); const mockSetVideoProcessingError = vi.fn(); -const mockIsLoomImportRunning = vi.fn(); +const mockGetLoomImportRunState = vi.fn(); const mockStartLoomImportWorkflow = vi.fn(); vi.mock("@cap/database", () => ({ @@ -46,7 +46,10 @@ vi.mock("drizzle-orm", () => ({ lte: vi.fn((left: unknown, right: unknown) => ({ left, right })), notLike: vi.fn((left: unknown, right: unknown) => ({ left, right })), or: vi.fn((...conditions: unknown[]) => conditions), - sql: vi.fn(), + sql: vi.fn((strings: TemplateStringsArray, ...values: unknown[]) => ({ + strings, + values, + })), })); vi.mock("workflow/api", () => ({ @@ -58,7 +61,8 @@ vi.mock("@/lib/video-processing", () => ({ })); vi.mock("@/lib/loom-import-start", () => ({ - isLoomImportRunning: mockIsLoomImportRunning, + getLoomImportRunState: mockGetLoomImportRunState, + LOOM_IMPORT_CONFIRMATION_MESSAGE: "Confirmation required", startLoomImportWorkflow: mockStartLoomImportWorkflow, LoomImportStartError: class extends Error { constructor( @@ -131,13 +135,52 @@ beforeEach(() => { vi.resetAllMocks(); mockStart.mockResolvedValue({ runId: "run-1" }); mockSetVideoProcessingError.mockResolvedValue(undefined); - mockIsLoomImportRunning.mockResolvedValue(false); + mockGetLoomImportRunState.mockResolvedValue("inactive"); mockStartLoomImportWorkflow.mockResolvedValue(undefined); }); describe("recoverFailedVideoProcessing", () => { + it("keeps one recovery budget across indexed startup phases", async () => { + const createCandidates = ( + phase: "uploading" | "processing", + offset: number, + ) => + Array.from({ length: 8 }, (_, index) => ({ + ...candidate, + phase, + videoId: `video-${index * 2 + offset}`, + loomVideoId: `loom-${index * 2 + offset}`, + uploadUpdatedAt: new Date(index * 2000 + offset * 1000), + metadata: { + loomImportRun: { + runId: `run-${index * 2 + offset}`, + dispatch: "accepted" as const, + }, + }, + })); + mockDb + .mockReturnValueOnce(makeSelectChain(createCandidates("uploading", 0))) + .mockReturnValueOnce(makeSelectChain(createCandidates("processing", 1))) + .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValueOnce(makeSelectChain([])); + mockGetLoomImportRunState.mockResolvedValue("active"); + const { recoverFailedVideoProcessing } = await import( + "@/lib/video-processing-recovery" + ); + + const result = await recoverFailedVideoProcessing(); + + expect(result.checked).toBe(8); + expect(result.results.map((row) => row.videoId)).toEqual( + Array.from({ length: 8 }, (_, index) => `video-${index}`), + ); + expect(mockGetLoomImportRunState).toHaveBeenCalledTimes(8); + expect(mockStartLoomImportWorkflow).not.toHaveBeenCalled(); + }); + it("atomically claims and restarts an affected upload", async () => { mockDb + .mockReturnValueOnce(makeSelectChain([])) .mockReturnValueOnce(makeSelectChain([])) .mockReturnValueOnce(makeSelectChain([])) .mockReturnValueOnce(makeSelectChain([candidate])) @@ -154,6 +197,7 @@ describe("recoverFailedVideoProcessing", () => { it("does not start a duplicate workflow when another run owns the claim", async () => { mockDb + .mockReturnValueOnce(makeSelectChain([])) .mockReturnValueOnce(makeSelectChain([])) .mockReturnValueOnce(makeSelectChain([])) .mockReturnValueOnce(makeSelectChain([candidate])) @@ -170,6 +214,7 @@ describe("recoverFailedVideoProcessing", () => { it("keeps a transient start failure eligible for the next recovery run", async () => { mockDb + .mockReturnValueOnce(makeSelectChain([])) .mockReturnValueOnce(makeSelectChain([])) .mockReturnValueOnce(makeSelectChain([])) .mockReturnValueOnce(makeSelectChain([candidate])) @@ -193,6 +238,7 @@ describe("recoverFailedVideoProcessing", () => { it("restarts a Loom import that exhausted its capacity retries", async () => { mockDb + .mockReturnValueOnce(makeSelectChain([])) .mockReturnValueOnce(makeSelectChain([])) .mockReturnValueOnce( makeSelectChain([ @@ -225,6 +271,7 @@ describe("recoverFailedVideoProcessing", () => { it("keeps a failed Loom workflow start eligible for Loom-specific recovery", async () => { const restore = makeUpdateChain(1); mockDb + .mockReturnValueOnce(makeSelectChain([])) .mockReturnValueOnce(makeSelectChain([])) .mockReturnValueOnce( makeSelectChain([ @@ -262,6 +309,7 @@ describe("recoverFailedVideoProcessing", () => { async (dispatch) => { const receipt = { runId: "slow-source-run", dispatch }; mockDb + .mockReturnValueOnce(makeSelectChain([])) .mockReturnValueOnce( makeSelectChain([ { @@ -276,7 +324,7 @@ describe("recoverFailedVideoProcessing", () => { .mockReturnValueOnce(makeSelectChain([])) .mockReturnValueOnce(makeSelectChain([])) .mockReturnValueOnce(makeUpdateChain(1)); - mockIsLoomImportRunning.mockResolvedValueOnce(true); + mockGetLoomImportRunState.mockResolvedValueOnce("active"); const { recoverFailedVideoProcessing } = await import( "@/lib/video-processing-recovery" ); @@ -286,8 +334,8 @@ describe("recoverFailedVideoProcessing", () => { expect(mockStart).not.toHaveBeenCalled(); expect(mockStartLoomImportWorkflow).not.toHaveBeenCalled(); expect(result.statuses).toEqual({ "already-claimed": 1 }); - expect(mockDb).toHaveBeenCalledTimes(3); - expect(mockIsLoomImportRunning).toHaveBeenCalledWith(receipt); + expect(mockDb).toHaveBeenCalledTimes(4); + expect(mockGetLoomImportRunState).toHaveBeenCalledWith(receipt); }, ); @@ -307,6 +355,7 @@ describe("recoverFailedVideoProcessing", () => { ) .mockReturnValueOnce(makeSelectChain([])) .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValueOnce(makeSelectChain([])) .mockReturnValueOnce(claim); const { recoverFailedVideoProcessing } = await import( "@/lib/video-processing-recovery" @@ -323,6 +372,136 @@ describe("recoverFailedVideoProcessing", () => { expect(result.statuses).toEqual({ "already-claimed": 1 }); }); + it.each([ + { phase: "uploading", state: "pending" }, + { phase: "uploading", state: "unconfirmed" }, + { phase: "processing", state: "pending" }, + { phase: "processing", state: "unconfirmed" }, + { phase: "processing", state: "inactive" }, + ])( + "exposes startup reconciliation without redispatching: %j", + async (row) => { + const receipt = { + runId: "unresolved-run", + dispatch: "uncertain" as const, + }; + const selected = makeSelectChain([ + { + ...candidate, + phase: row.phase, + loomVideoId: "loom-video-1", + metadata: { loomImportRun: receipt }, + }, + ]); + const update = makeUpdateChain(1); + mockDb + .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValueOnce(selected) + .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValueOnce(update); + mockGetLoomImportRunState.mockResolvedValueOnce(row.state); + const { recoverFailedVideoProcessing } = await import( + "@/lib/video-processing-recovery" + ); + + const result = await recoverFailedVideoProcessing(); + + expect(result.statuses).toEqual({ "reconciliation-required": 1 }); + expect(update.set).toHaveBeenCalledWith({ + processingMessage: "Confirmation required", + updatedAt: expect.any(Date), + }); + expect(update.where).toHaveBeenCalledWith( + expect.arrayContaining([ + { left: "videoUploads.updatedAt", right: candidate.uploadUpdatedAt }, + expect.objectContaining({ + values: expect.arrayContaining(["unresolved-run"]), + }), + ]), + ); + expect(selected.select).toHaveBeenCalledWith( + expect.objectContaining({ phase: "videoUploads.phase" }), + ); + expect(mockStartLoomImportWorkflow).not.toHaveBeenCalled(); + expect(mockStart).not.toHaveBeenCalled(); + }, + ); + + it("does not change reconciliation state after the selected claim changed", async () => { + const update = makeUpdateChain(0); + mockDb + .mockReturnValueOnce( + makeSelectChain([ + { + ...candidate, + phase: "processing", + loomVideoId: "loom-video-1", + metadata: { + loomImportRun: { runId: "old-run", dispatch: "pending" }, + }, + }, + ]), + ) + .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValueOnce(update); + mockGetLoomImportRunState.mockResolvedValueOnce("unconfirmed"); + const { recoverFailedVideoProcessing } = await import( + "@/lib/video-processing-recovery" + ); + + expect((await recoverFailedVideoProcessing()).statuses).toEqual({ + "already-claimed": 1, + }); + expect(mockStartLoomImportWorkflow).not.toHaveBeenCalled(); + }); + + it("recovers an uploading confirmation hold after its run becomes terminal", async () => { + const update = makeUpdateChain(1); + mockDb + .mockReturnValueOnce( + makeSelectChain([ + { + ...candidate, + phase: "uploading", + rawFileKey: null, + loomVideoId: "loom-video-1", + metadata: { + loomImportRun: { runId: "terminal-run", dispatch: "uncertain" }, + }, + }, + ]), + ) + .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValueOnce(update); + mockGetLoomImportRunState.mockResolvedValueOnce("inactive"); + const { recoverFailedVideoProcessing } = await import( + "@/lib/video-processing-recovery" + ); + + expect((await recoverFailedVideoProcessing()).statuses).toEqual({ + started: 1, + }); + expect(update.where).toHaveBeenCalledWith( + expect.arrayContaining([ + expect.arrayContaining([ + { + left: "videoUploads.processingMessage", + right: "Confirmation required", + }, + ]), + expect.objectContaining({ + values: expect.arrayContaining(["terminal-run"]), + }), + ]), + ); + expect(mockStartLoomImportWorkflow).toHaveBeenCalledTimes(1); + }); + it("retains the recovery claim when dispatch acknowledgement is uncertain", async () => { mockDb .mockReturnValueOnce( @@ -332,6 +511,7 @@ describe("recoverFailedVideoProcessing", () => { ) .mockReturnValueOnce(makeSelectChain([])) .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValueOnce(makeSelectChain([])) .mockReturnValueOnce(makeUpdateChain(1)); const { LoomImportStartError } = await import("@/lib/loom-import-start"); mockStartLoomImportWorkflow.mockRejectedValueOnce( @@ -343,7 +523,7 @@ describe("recoverFailedVideoProcessing", () => { const result = await recoverFailedVideoProcessing(); - expect(mockDb).toHaveBeenCalledTimes(4); + expect(mockDb).toHaveBeenCalledTimes(5); expect(mockStart).not.toHaveBeenCalled(); expect(result.statuses).toEqual({ "already-claimed": 1 }); }); @@ -359,6 +539,7 @@ describe("recoverFailedVideoProcessing", () => { ) .mockReturnValueOnce(makeSelectChain([])) .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValueOnce(makeSelectChain([])) .mockReturnValueOnce(claim) .mockReturnValueOnce(restore); const { LoomImportStartError } = await import("@/lib/loom-import-start"); diff --git a/apps/web/actions/loom.ts b/apps/web/actions/loom.ts index b140e85c3de..09120edfef1 100644 --- a/apps/web/actions/loom.ts +++ b/apps/web/actions/loom.ts @@ -52,7 +52,8 @@ import { } from "@/lib/loom-download-url"; import type { LoomImportDestination } from "@/lib/loom-import-destination"; import { - isLoomImportRunning, + getLoomImportRunState, + LOOM_IMPORT_CONFIRMATION_MESSAGE, LoomImportStartError, restoreLoomImportStartError, startLoomImportWorkflow, @@ -360,7 +361,13 @@ async function importLoomVideoForOwner({ existingVideo.ownerId === ownerId && existingVideo.uploadUpdatedAt ) { - if (await isLoomImportRunning(existingVideo.metadata?.loomImportRun)) { + const runState = await getLoomImportRunState( + existingVideo.metadata?.loomImportRun, + ); + if (runState === "unconfirmed") { + return { success: false, error: LOOM_IMPORT_CONFIRMATION_MESSAGE }; + } + if (runState !== "inactive") { return { success: false, error: "This Loom import is already restarting.", diff --git a/apps/web/actions/video/retry-processing.ts b/apps/web/actions/video/retry-processing.ts index 2081ed8f81b..becb379b5c5 100644 --- a/apps/web/actions/video/retry-processing.ts +++ b/apps/web/actions/video/retry-processing.ts @@ -6,7 +6,8 @@ import { importedVideos, videos, videoUploads } from "@cap/database/schema"; import type { Video } from "@cap/web-domain"; import { and, eq } from "drizzle-orm"; import { - isLoomImportRunning, + getLoomImportRunState, + LOOM_IMPORT_CONFIRMATION_MESSAGE, LoomImportStartError, restoreLoomImportStartError, startLoomImportWorkflow, @@ -81,8 +82,12 @@ export async function retryVideoProcessing({ ); if (importedVideo?.source === "loom") { + const runState = await getLoomImportRunState(video.metadata?.loomImportRun); + if (runState === "unconfirmed") { + throw new Error(LOOM_IMPORT_CONFIRMATION_MESSAGE); + } if ( - (await isLoomImportRunning(video.metadata?.loomImportRun)) || + runState !== "inactive" || ((upload.phase === "processing" || upload.phase === "generating_thumbnail") && !shouldForceRetryProcessing(upload)) diff --git a/apps/web/lib/loom-import-start.ts b/apps/web/lib/loom-import-start.ts index 3b3725288ef..b5ef8ca2d4e 100644 --- a/apps/web/lib/loom-import-start.ts +++ b/apps/web/lib/loom-import-start.ts @@ -49,15 +49,25 @@ export class LoomImportStartError extends Error { } } -export async function isLoomImportRunning(receipt?: LoomImportRun) { - if (!receipt || receipt.dispatch === "rejected") return false; +export const LOOM_IMPORT_CONFIRMATION_MESSAGE = + "We're confirming whether your Loom import started. Please contact support before retrying."; + +export async function getLoomImportRunState( + receipt?: LoomImportRun, +): Promise<"inactive" | "pending" | "active" | "unconfirmed"> { + if (!receipt || receipt.dispatch === "rejected") return "inactive"; try { const status = await getRun(receipt.runId).status; - return ( - status !== "completed" && status !== "failed" && status !== "cancelled" - ); + if ( + status === "completed" || + status === "failed" || + status === "cancelled" + ) { + return "inactive"; + } + return status === "pending" ? "pending" : "active"; } catch { - return true; + return "unconfirmed"; } } diff --git a/apps/web/lib/video-processing-recovery.ts b/apps/web/lib/video-processing-recovery.ts index dee4b1d3732..09c14dccfc9 100644 --- a/apps/web/lib/video-processing-recovery.ts +++ b/apps/web/lib/video-processing-recovery.ts @@ -16,7 +16,8 @@ import { } from "drizzle-orm"; import { start } from "workflow/api"; import { - isLoomImportRunning, + getLoomImportRunState, + LOOM_IMPORT_CONFIRMATION_MESSAGE, LoomImportStartError, startLoomImportWorkflow, } from "@/lib/loom-import-start"; @@ -49,6 +50,7 @@ export type FailedVideoProcessingRecoveryStatus = | "already-claimed" | "retry-scheduled" | "missing-source" + | "reconciliation-required" | "failed"; export type FailedVideoProcessingRecoverySummary = { @@ -142,11 +144,38 @@ async function recoverLoomCandidate({ bucketId: S3Bucket.S3BucketId | null; loomVideoId: string; rawFileKey: string | null; - phase: "uploading" | "error"; + phase: "uploading" | "processing" | "error"; uploadUpdatedAt: Date; receipt: VideoMetadata["loomImportRun"]; }): Promise { - if (await isLoomImportRunning(receipt)) return "already-claimed"; + const runState = await getLoomImportRunState(receipt); + const receiptCondition = receipt + ? sql`EXISTS (SELECT 1 FROM ${videos} WHERE ${videos.id} = ${videoId} AND JSON_UNQUOTE(JSON_EXTRACT(${videos.metadata}, '$.loomImportRun.runId')) = ${receipt.runId})` + : undefined; + if (runState === "active") return "already-claimed"; + if ( + runState === "pending" || + runState === "unconfirmed" || + phase === "processing" + ) { + const result = await db() + .update(videoUploads) + .set({ + processingMessage: LOOM_IMPORT_CONFIRMATION_MESSAGE, + updatedAt: new Date(), + }) + .where( + and( + eq(videoUploads.videoId, videoId), + eq(videoUploads.phase, phase), + eq(videoUploads.updatedAt, uploadUpdatedAt), + receiptCondition, + ), + ); + return getAffectedRows(result) === 1 + ? "reconciliation-required" + : "already-claimed"; + } const rawFileKey = existingRawFileKey ?? `${userId}/${videoId}/raw-upload.mp4`; const claimedAt = new Date(Math.floor(Date.now() / 1000) * 1000); @@ -165,6 +194,7 @@ async function recoverLoomCandidate({ eq(videoUploads.videoId, videoId), eq(videoUploads.phase, phase), eq(videoUploads.updatedAt, uploadUpdatedAt), + receiptCondition, ...(phase === "error" ? [ eq(videoUploads.rawFileKey, rawFileKey), @@ -172,7 +202,13 @@ async function recoverLoomCandidate({ ] : [ isNull(videoUploads.rawFileKey), - eq(videoUploads.processingMessage, LOOM_IMPORT_PENDING_MESSAGE), + or( + eq(videoUploads.processingMessage, LOOM_IMPORT_PENDING_MESSAGE), + eq( + videoUploads.processingMessage, + LOOM_IMPORT_CONFIRMATION_MESSAGE, + ), + ), ]), ), ); @@ -251,37 +287,84 @@ export async function recoverFailedVideoProcessing({ const staleLoomBefore = new Date( Date.now() - LOOM_IMPORT_RECOVERY_MIN_AGE_MS, ); - const pendingLoomCandidates = await db() - .select({ - videoId: videos.id, - userId: videos.ownerId, - bucketId: videos.bucket, - loomVideoId: importedVideos.sourceId, - rawFileKey: videoUploads.rawFileKey, - uploadUpdatedAt: videoUploads.updatedAt, - metadata: videos.metadata, - }) - .from(videos) - .innerJoin(videoUploads, eq(videos.id, videoUploads.videoId)) - .innerJoin( - importedVideos, - and( - eq(videos.id, importedVideos.id), - eq(videos.orgId, importedVideos.orgId), - eq(importedVideos.source, "loom"), - ), - ) - .where( - and( - eq(importedVideos.source, "loom"), - eq(videoUploads.phase, "uploading"), - isNull(videoUploads.rawFileKey), - eq(videoUploads.processingMessage, LOOM_IMPORT_PENDING_MESSAGE), - lte(videoUploads.updatedAt, staleLoomBefore), - ), + const selectStaleLoomCandidates = (phase: "uploading" | "processing") => + db() + .select({ + videoId: videos.id, + userId: videos.ownerId, + bucketId: videos.bucket, + loomVideoId: importedVideos.sourceId, + rawFileKey: videoUploads.rawFileKey, + uploadUpdatedAt: videoUploads.updatedAt, + metadata: videos.metadata, + phase: videoUploads.phase, + }) + .from(videos) + .innerJoin(videoUploads, eq(videos.id, videoUploads.videoId)) + .innerJoin( + importedVideos, + and( + eq(videos.id, importedVideos.id), + eq(videos.orgId, importedVideos.orgId), + eq(importedVideos.source, "loom"), + ), + ) + .where( + and( + eq(importedVideos.source, "loom"), + eq(videoUploads.phase, phase), + phase === "uploading" + ? or( + and( + isNull(videoUploads.rawFileKey), + eq( + videoUploads.processingMessage, + LOOM_IMPORT_PENDING_MESSAGE, + ), + ), + and( + sql`JSON_EXTRACT(${videos.metadata}, '$.loomImportRun.runId') IS NOT NULL`, + eq( + videoUploads.processingMessage, + LOOM_IMPORT_CONFIRMATION_MESSAGE, + ), + ), + ) + : and( + sql`JSON_EXTRACT(${videos.metadata}, '$.loomImportRun.runId') IS NOT NULL`, + or( + eq( + videoUploads.processingMessage, + LOOM_IMPORT_RECOVERY_MESSAGE, + ), + eq(videoUploads.processingMessage, "Retrying Loom import..."), + eq( + videoUploads.processingMessage, + "Processing Loom video...", + ), + eq( + videoUploads.processingMessage, + LOOM_IMPORT_CONFIRMATION_MESSAGE, + ), + ), + ), + lte(videoUploads.updatedAt, staleLoomBefore), + ), + ) + .orderBy(asc(videoUploads.updatedAt)) + .limit(Math.min(limit, LOOM_IMPORT_RECOVERY_BATCH_SIZE)); + const pendingLoomCandidates = ( + await Promise.all([ + selectStaleLoomCandidates("uploading"), + selectStaleLoomCandidates("processing"), + ]) + ) + .flat() + .sort( + (left, right) => + left.uploadUpdatedAt.getTime() - right.uploadUpdatedAt.getTime(), ) - .orderBy(asc(videoUploads.updatedAt)) - .limit(Math.min(limit, LOOM_IMPORT_RECOVERY_BATCH_SIZE)); + .slice(0, Math.min(limit, LOOM_IMPORT_RECOVERY_BATCH_SIZE)); const failedLoomUploads = await db() .select({ @@ -336,7 +419,10 @@ export async function recoverFailedVideoProcessing({ const loomCandidates = [ ...pendingLoomCandidates.map((candidate) => ({ ...candidate, - phase: "uploading" as const, + phase: + candidate.phase === "processing" + ? ("processing" as const) + : ("uploading" as const), })), ...failedLoomCandidates.map((candidate) => ({ ...candidate, From 3d2d3b763d6c5c38d4dbf1a2a24bb2b941e50582 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 22 Oct 2026 19:02:05 +0800 Subject: [PATCH 10/21] fix: preserve original Loom import during manual retry --- .../unit/loom-retry-processing.test.ts | 25 +++++++++++++++++++ apps/web/actions/video/retry-processing.ts | 1 + 2 files changed, 26 insertions(+) diff --git a/apps/web/__tests__/unit/loom-retry-processing.test.ts b/apps/web/__tests__/unit/loom-retry-processing.test.ts index 57d8147e422..eb18ca41a6a 100644 --- a/apps/web/__tests__/unit/loom-retry-processing.test.ts +++ b/apps/web/__tests__/unit/loom-retry-processing.test.ts @@ -137,6 +137,31 @@ describe("Loom processing retries", () => { expect(mocks.start).not.toHaveBeenCalled(); }); + it.each([ + { rawFileKey: null, updatedAt: new Date() }, + { rawFileKey: null, updatedAt: new Date(0) }, + { rawFileKey: "owner-1/video-1/raw-upload.mp4", updatedAt: new Date() }, + { rawFileKey: "owner-1/video-1/raw-upload.mp4", updatedAt: new Date(0) }, + ])( + "does not replace an uploading import without a startup receipt: %j", + async (uploading) => { + mocks.where + .mockResolvedValueOnce([video]) + .mockResolvedValueOnce([ + { ...upload, ...uploading, phase: "uploading" }, + ]) + .mockResolvedValueOnce([{ source: "loom", sourceId: "loom-1" }]) + .mockResolvedValueOnce([{ affectedRows: 1 }]); + await expect(retryVideoProcessing({ videoId })).resolves.toEqual({ + success: true, + status: "already-processing", + }); + expect(mocks.set).not.toHaveBeenCalled(); + expect(mocks.start).not.toHaveBeenCalled(); + expect(mocks.startProcessing).not.toHaveBeenCalled(); + }, + ); + it("recovers a source lookup failure that has no raw file key", async () => { mocks.where .mockResolvedValueOnce([video]) diff --git a/apps/web/actions/video/retry-processing.ts b/apps/web/actions/video/retry-processing.ts index becb379b5c5..c2557ee2aac 100644 --- a/apps/web/actions/video/retry-processing.ts +++ b/apps/web/actions/video/retry-processing.ts @@ -88,6 +88,7 @@ export async function retryVideoProcessing({ } if ( runState !== "inactive" || + upload.phase === "uploading" || ((upload.phase === "processing" || upload.phase === "generating_thumbnail") && !shouldForceRetryProcessing(upload)) From 914516bdc6f6dbabd622f6e435cdc99a68ea623c Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 22 Oct 2026 23:12:02 +0800 Subject: [PATCH 11/21] fix: block Loom completion when destination imports disappear --- .../loom-concierge-database.test.ts | 44 +++++++++++++++ .../unit/loom-migration-state.test.ts | 53 ++++++++++++++++++- apps/web/actions/loom-concierge.tsx | 45 ++++++++-------- apps/web/lib/loom-migration-state.ts | 35 ++++++++++++ 4 files changed, 155 insertions(+), 22 deletions(-) diff --git a/apps/web/__tests__/integration/loom-concierge-database.test.ts b/apps/web/__tests__/integration/loom-concierge-database.test.ts index f296f046cfa..f6d347fcdc8 100644 --- a/apps/web/__tests__/integration/loom-concierge-database.test.ts +++ b/apps/web/__tests__/integration/loom-concierge-database.test.ts @@ -729,6 +729,50 @@ describe.skipIf(!databaseUrl)("Loom concierge migration database", () => { expect(reopened.id).not.toBe(requestId); }); + it.each(["recording", "source mapping"])( + "does not complete or notify when a migrated %s disappears", + async (missing) => { + const { ownerId, organizationId } = await makeOrganization(); + const requestId = await requestMigration(organizationId, ownerId); + const videoId = Video.VideoId.make(id()); + await db().insert(videos).values({ + id: videoId, + name: "Finished import", + ownerId, + orgId: organizationId, + }); + await db().insert(importedVideos).values({ + id: videoId, + orgId: organizationId, + source: "loom", + sourceId: loomId(), + }); + await db().insert(loomMigrationImports).values({ videoId, requestId }); + if (missing === "recording") { + await db().delete(videos).where(eq(videos.id, videoId)); + } else { + await db().delete(importedVideos).where(eq(importedVideos.id, videoId)); + } + signInAs(operatorId); + fixture.sendEmail.mockClear(); + await expect( + actions.updateLoomMigrationStatus({ + requestId, + nextStatus: "completed", + message: "All done", + verified: true, + expectedVideoCount: 1, + importedVideoCount: 1, + }), + ).rejects.toThrow("missing or changed"); + const request = await getRequest(requestId); + expect(request.status).not.toBe("completed"); + expect(request.activeOrganizationId).toBe(organizationId); + expect(request.completedAt).toBeNull(); + expect(fixture.sendEmail).not.toHaveBeenCalled(); + }, + ); + it("uploads restricted MP4 files to storage and starts processing", async () => { const { ownerId, organizationId } = await makeOrganization({ privateByDefault: true, diff --git a/apps/web/__tests__/unit/loom-migration-state.test.ts b/apps/web/__tests__/unit/loom-migration-state.test.ts index 87b1f90a544..74a30112c88 100644 --- a/apps/web/__tests__/unit/loom-migration-state.test.ts +++ b/apps/web/__tests__/unit/loom-migration-state.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from "vitest"; -import { validateOperatorMigrationUpdate } from "@/lib/loom-migration-state"; +import { + validateLoomMigrationInventory, + validateOperatorMigrationUpdate, +} from "@/lib/loom-migration-state"; const base = { currentStatus: "in_progress" as const, @@ -37,3 +40,51 @@ describe("Loom migration completion", () => { expect(validateOperatorMigrationUpdate(base)).toBe("Migration finished"); }); }); + +describe("Loom migration destination inventory", () => { + const imported = { + videoId: "video", + videoOrganizationId: "destination", + mappingId: "video", + mappingOrganizationId: "destination", + source: "loom", + phase: null, + }; + + it("accepts completed recordings with and without retained upload rows", () => { + expect(() => + validateLoomMigrationInventory({ + organizationId: "destination", + imports: [imported, { ...imported, phase: "complete" }], + }), + ).not.toThrow(); + }); + + it.each([ + { videoId: null, videoOrganizationId: null }, + { videoOrganizationId: "another-organization" }, + { mappingId: null, mappingOrganizationId: null, source: null }, + { mappingId: "another-recording" }, + { mappingOrganizationId: "another-organization" }, + { source: "another-source" }, + ])("blocks a missing or changed destination: %j", (changes) => { + expect(() => + validateLoomMigrationInventory({ + organizationId: "destination", + imports: [imported, { ...imported, ...changes }], + }), + ).toThrow("missing or changed"); + }); + + it.each(["uploading", "processing", "generating_thumbnail", "error"])( + "blocks an import that is %s", + (phase) => { + expect(() => + validateLoomMigrationInventory({ + organizationId: "destination", + imports: [imported, { ...imported, phase }], + }), + ).toThrow("still processing"); + }, + ); +}); diff --git a/apps/web/actions/loom-concierge.tsx b/apps/web/actions/loom-concierge.tsx index 5b9452dd102..2a51b906579 100644 --- a/apps/web/actions/loom-concierge.tsx +++ b/apps/web/actions/loom-concierge.tsx @@ -8,15 +8,17 @@ import { } from "@cap/database/emails/loom-migration"; import { nanoId } from "@cap/database/helpers"; import { + importedVideos, loomMigrationImports, loomMigrationRequests, organizations, users, + videos, videoUploads, } from "@cap/database/schema"; import { serverEnv } from "@cap/env"; import type { Organisation } from "@cap/web-domain"; -import { and, eq, inArray, isNotNull } from "drizzle-orm"; +import { and, eq, isNotNull } from "drizzle-orm"; import { revalidatePath } from "next/cache"; import { getCustomerMigrationRequests, @@ -31,6 +33,7 @@ import { LoomMigrationError, type LoomMigrationStatus, normalizeMigrationText, + validateLoomMigrationInventory, validateOperatorMigrationUpdate, } from "@/lib/loom-migration-state"; import { isOrganizationOwnerPro } from "@/lib/org-pro"; @@ -304,31 +307,31 @@ async function saveMigrationStatus({ ); } if (nextStatus === "completed") { - const [unfinishedImport] = await tx - .select({ videoId: videoUploads.videoId }) + const imports = await tx + .select({ + videoId: videos.id, + videoOrganizationId: videos.orgId, + mappingId: importedVideos.id, + mappingOrganizationId: importedVideos.orgId, + source: importedVideos.source, + phase: videoUploads.phase, + }) .from(loomMigrationImports) - .innerJoin( + .leftJoin(videos, eq(videos.id, loomMigrationImports.videoId)) + .leftJoin( + importedVideos, + eq(importedVideos.id, loomMigrationImports.videoId), + ) + .leftJoin( videoUploads, eq(videoUploads.videoId, loomMigrationImports.videoId), ) - .where( - and( - eq(loomMigrationImports.requestId, requestId), - inArray(videoUploads.phase, [ - "uploading", - "processing", - "generating_thumbnail", - "error", - ]), - ), - ) - .limit(1) + .where(eq(loomMigrationImports.requestId, requestId)) .for("share"); - if (unfinishedImport) { - throw new LoomMigrationError( - "Loom imports are still processing or have errors. Resolve them before completion.", - ); - } + validateLoomMigrationInventory({ + organizationId: request.organizationId, + imports, + }); } await tx .update(loomMigrationRequests) diff --git a/apps/web/lib/loom-migration-state.ts b/apps/web/lib/loom-migration-state.ts index 79ab2a6c476..c92ac7630df 100644 --- a/apps/web/lib/loom-migration-state.ts +++ b/apps/web/lib/loom-migration-state.ts @@ -44,6 +44,41 @@ export function normalizeMigrationText( return normalized; } +export function validateLoomMigrationInventory({ + organizationId, + imports, +}: { + organizationId: string; + imports: ReadonlyArray<{ + videoId: string | null; + videoOrganizationId: string | null; + mappingId: string | null; + mappingOrganizationId: string | null; + source: string | null; + phase: string | null; + }>; +}) { + if ( + imports.some( + (row) => + !row.videoId || + row.videoOrganizationId !== organizationId || + row.mappingId !== row.videoId || + row.mappingOrganizationId !== organizationId || + row.source !== "loom", + ) + ) { + throw new LoomMigrationError( + "Loom import recordings or source mappings are missing or changed. Reconcile the destination library before completion.", + ); + } + if (imports.some((row) => row.phase !== null && row.phase !== "complete")) { + throw new LoomMigrationError( + "Loom imports are still processing or have errors. Resolve them before completion.", + ); + } +} + export function validateOperatorMigrationUpdate({ currentStatus, nextStatus, From 949db7e2a9683f32600dc46db28a1073fa735e0d Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 22 Oct 2026 23:21:46 +0800 Subject: [PATCH 12/21] fix: reserve recovery capacity for failed Loom imports --- .../unit/video-processing-recovery.test.ts | 41 +++++++++++++++++++ apps/web/lib/video-processing-recovery.ts | 29 ++++++------- 2 files changed, 56 insertions(+), 14 deletions(-) diff --git a/apps/web/__tests__/unit/video-processing-recovery.test.ts b/apps/web/__tests__/unit/video-processing-recovery.test.ts index c15b5266367..ac74ddedd99 100644 --- a/apps/web/__tests__/unit/video-processing-recovery.test.ts +++ b/apps/web/__tests__/unit/video-processing-recovery.test.ts @@ -178,6 +178,47 @@ describe("recoverFailedVideoProcessing", () => { expect(mockStartLoomImportWorkflow).not.toHaveBeenCalled(); }); + it("reserves a failed-import slot while uncertain startups remain protected", async () => { + const pending = Array.from({ length: 8 }, (_, index) => ({ + ...candidate, + videoId: `pending-${index}`, + loomVideoId: `pending-loom-${index}`, + phase: "uploading", + metadata: { + loomImportRun: { runId: `run-${index}`, dispatch: "uncertain" }, + }, + })); + const failedQuery = makeSelectChain([ + { ...candidate, videoId: "failed-video", orgId: "organization-1" }, + ]); + mockDb + .mockReturnValueOnce(makeSelectChain(pending)) + .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValueOnce(failedQuery) + .mockReturnValueOnce(makeSelectChain([{ loomVideoId: "failed-loom" }])) + .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValue(makeUpdateChain(1)); + mockGetLoomImportRunState.mockImplementation(async (receipt) => + receipt ? "unconfirmed" : "inactive", + ); + const { recoverFailedVideoProcessing } = await import( + "@/lib/video-processing-recovery" + ); + + const result = await recoverFailedVideoProcessing(); + + expect(failedQuery.limit).toHaveBeenCalledWith(1); + expect(result.checked).toBe(8); + expect(result.statuses).toEqual({ + "reconciliation-required": 7, + started: 1, + }); + expect(mockStartLoomImportWorkflow).toHaveBeenCalledTimes(1); + expect(mockStartLoomImportWorkflow).toHaveBeenCalledWith( + expect.objectContaining({ videoId: "failed-video" }), + ); + }); + it("atomically claims and restarts an affected upload", async () => { mockDb .mockReturnValueOnce(makeSelectChain([])) diff --git a/apps/web/lib/video-processing-recovery.ts b/apps/web/lib/video-processing-recovery.ts index 09c14dccfc9..db0b41e2fd0 100644 --- a/apps/web/lib/video-processing-recovery.ts +++ b/apps/web/lib/video-processing-recovery.ts @@ -366,6 +366,11 @@ export async function recoverFailedVideoProcessing({ ) .slice(0, Math.min(limit, LOOM_IMPORT_RECOVERY_BATCH_SIZE)); + const loomRecoveryBudget = Math.min(limit, LOOM_IMPORT_RECOVERY_BATCH_SIZE); + const startupCountWithFailedImport = Math.min( + pendingLoomCandidates.length, + Math.max(0, loomRecoveryBudget - 1), + ); const failedLoomUploads = await db() .select({ videoId: videos.id, @@ -387,13 +392,7 @@ export async function recoverFailedVideoProcessing({ ), ) .orderBy(asc(videoUploads.updatedAt)) - .limit( - Math.max( - 0, - Math.min(limit, LOOM_IMPORT_RECOVERY_BATCH_SIZE) - - pendingLoomCandidates.length, - ), - ); + .limit(Math.max(0, loomRecoveryBudget - startupCountWithFailedImport)); const failedLoomCandidates = ( await Promise.all( @@ -417,13 +416,15 @@ export async function recoverFailedVideoProcessing({ ).filter((candidate) => candidate !== null); const loomCandidates = [ - ...pendingLoomCandidates.map((candidate) => ({ - ...candidate, - phase: - candidate.phase === "processing" - ? ("processing" as const) - : ("uploading" as const), - })), + ...pendingLoomCandidates + .slice(0, Math.max(0, loomRecoveryBudget - failedLoomCandidates.length)) + .map((candidate) => ({ + ...candidate, + phase: + candidate.phase === "processing" + ? ("processing" as const) + : ("uploading" as const), + })), ...failedLoomCandidates.map((candidate) => ({ ...candidate, phase: "error" as const, From cac46d9834087b48e9feba2df0a4d5b88f993034 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:31:02 +0100 Subject: [PATCH 13/21] fix: support 10 GB Loom imports with durable multipart uploads --- apps/media-server/Dockerfile | 1 + .../__tests__/lib/loom-import-limit.test.ts | 149 +++++++++++++ .../src/__tests__/lib/storage-upload.test.ts | 67 +++++- .../src/__tests__/routes/video.test.ts | 40 ++++ apps/media-server/src/lib/media-video.ts | 51 ++++- apps/media-server/src/routes/video.ts | 100 ++++++--- .../__tests__/unit/loom-import-start.test.ts | 49 +++- .../__tests__/unit/loom-import-upload.test.ts | 146 ++++++++++++ .../unit/loom-media-dispatch.test.ts | 51 +++++ .../unit/media-processing-workflows.test.ts | 74 +++++- .../unit/media-server-multipart.test.ts | 92 ++++++++ .../media-server/multipart/[action]/route.ts | 84 ++++++- apps/web/lib/loom-import-start.ts | 21 +- apps/web/lib/loom-import-upload.ts | 151 +++++++++++++ apps/web/lib/loom-media-dispatch.ts | 76 +++++++ apps/web/workflows/import-loom-video.ts | 211 +++++++++++++++--- packages/database/types/metadata.ts | 13 ++ 17 files changed, 1294 insertions(+), 82 deletions(-) create mode 100644 apps/media-server/src/__tests__/lib/loom-import-limit.test.ts create mode 100644 apps/web/__tests__/unit/loom-import-upload.test.ts create mode 100644 apps/web/__tests__/unit/loom-media-dispatch.test.ts create mode 100644 apps/web/lib/loom-import-upload.ts create mode 100644 apps/web/lib/loom-media-dispatch.ts diff --git a/apps/media-server/Dockerfile b/apps/media-server/Dockerfile index c09234479ff..956d6d53969 100644 --- a/apps/media-server/Dockerfile +++ b/apps/media-server/Dockerfile @@ -16,6 +16,7 @@ RUN bun test src/__tests__/lib/audio-quality-policy.test.ts src/__tests__/lib/au && bun test src/__tests__/lib/audio-quality-formats.integration.test.ts src/__tests__/lib/audio-quality-benchmark.test.ts \ && bun test src/__tests__/lib/audio-levels.integration.test.ts \ && bun test src/__tests__/lib/drive-resumable-upload.test.ts src/__tests__/lib/storage-upload.test.ts src/__tests__/lib/container-memory.test.ts \ + && bun test src/__tests__/lib/loom-import-limit.test.ts \ && bun test src/__tests__/lib/media-size.test.ts src/__tests__/lib/media-probe.integration.test.ts \ && bun test src/__tests__/lib/media-audio.integration.test.ts \ && bun test src/__tests__/lib/recording-verification.integration.test.ts src/__tests__/lib/job-manager.test.ts \ diff --git a/apps/media-server/src/__tests__/lib/loom-import-limit.test.ts b/apps/media-server/src/__tests__/lib/loom-import-limit.test.ts new file mode 100644 index 00000000000..64308146f5c --- /dev/null +++ b/apps/media-server/src/__tests__/lib/loom-import-limit.test.ts @@ -0,0 +1,149 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { execFileSync } from "node:child_process"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + downloadVideoToTemp, + LOOM_IMPORT_MAX_BYTES, +} from "../../lib/media-video"; + +const originalFetch = globalThis.fetch; + +function stubFetch(response: () => Promise) { + globalThis.fetch = Object.assign(response, { + preconnect: originalFetch.preconnect, + }); +} + +describe("Loom import file size", () => { + afterEach(() => { + globalThis.fetch = originalFetch; + }); + + test("permits exactly the configured boundary", async () => { + expect(LOOM_IMPORT_MAX_BYTES).toBe(10 * 1024 ** 3); + stubFetch(async () => new Response(new Uint8Array(16))); + const result = await downloadVideoToTemp( + "https://source.test/video.mp4", + ".mp4", + undefined, + 16, + ); + try { + expect(Bun.file(result.path).size).toBe(16); + } finally { + await result.cleanup(); + } + }); + + test("rejects an oversized declared body before reading it", async () => { + let cancelled = false; + stubFetch( + async () => + new Response( + new ReadableStream({ + cancel() { + cancelled = true; + }, + }), + { headers: { "Content-Length": "17" } }, + ), + ); + await expect( + downloadVideoToTemp( + "https://source.test/video.mp4", + ".mp4", + undefined, + 16, + ), + ).rejects.toThrow("10 GB"); + expect(cancelled).toBe(true); + }); + + test("cancels a chunked transfer when it exceeds the limit", async () => { + let cancelled = false; + stubFetch( + async () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(8)); + controller.enqueue(new Uint8Array(9)); + }, + cancel() { + cancelled = true; + }, + }), + ), + ); + await expect( + downloadVideoToTemp( + "https://source.test/video.mp4", + ".mp4", + undefined, + 16, + ), + ).rejects.toThrow("10 GB"); + expect(cancelled).toBe(true); + }); + + test("rejects an oversized real HLS recording without returning truncated media", async () => { + const directory = await mkdtemp(join(tmpdir(), "cap-loom-limit-")); + let server: ReturnType | undefined; + try { + execFileSync("ffmpeg", [ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-f", + "lavfi", + "-i", + "testsrc2=size=64x64:rate=10", + "-t", + "2", + "-c:v", + "libx264", + "-g", + "10", + "-f", + "hls", + "-hls_time", + "1", + "-hls_segment_filename", + join(directory, "segment%d.ts"), + join(directory, "index.m3u8"), + ]); + globalThis.fetch = originalFetch; + server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request) { + const path = new URL(request.url).pathname; + return /^\/(?:index\.m3u8|segment\d+\.ts)$/.test(path) + ? new Response(Bun.file(join(directory, path.slice(1)))) + : new Response(null, { status: 404 }); + }, + }); + const source = `${server.url}index.m3u8`; + const complete = await downloadVideoToTemp( + source, + ".m3u8", + undefined, + 1024 ** 2, + ); + try { + expect(Bun.file(complete.path).size).toBeGreaterThan(1); + } finally { + await complete.cleanup(); + } + await expect( + downloadVideoToTemp(source, ".m3u8", undefined, 1), + ).rejects.toThrow("10 GB"); + } finally { + server?.stop(true); + await rm(directory, { recursive: true, force: true }); + } + }); +}); diff --git a/apps/media-server/src/__tests__/lib/storage-upload.test.ts b/apps/media-server/src/__tests__/lib/storage-upload.test.ts index f260ea8ae61..8b887c908fa 100644 --- a/apps/media-server/src/__tests__/lib/storage-upload.test.ts +++ b/apps/media-server/src/__tests__/lib/storage-upload.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, test } from "bun:test"; import { createHash } from "node:crypto"; -import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, open, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { uploadFileToStorage } from "../../lib/media-video"; @@ -29,6 +29,71 @@ describe("uploadFileToStorage", () => { globalThis.fetch = originalFetch; }); + test("uploads a 10 GB sparse file as 160 bounded parts with its Loom attempt", async () => { + const uploadFile = await createTempUploadFile(1); + const handle = await open(uploadFile.path, "r+"); + try { + await handle.truncate(10 * 1024 ** 3); + } finally { + await handle.close(); + } + let uploadedParts = 0; + let uploadedBytes = 0; + let completed = false; + globalThis.fetch = (async (input, init) => { + const url = String(input); + if (url.endsWith("/sign-part")) { + const request = JSON.parse(String(init?.body)); + expect(request.loomImportAttemptId).toBe("loom-run:0"); + expect(request.contentLength).toBe(64 * 1024 ** 2); + return Response.json({ + url: `https://storage.test/part/${request.partNumber}`, + }); + } + if (init?.method === "PUT") { + const body = init.body; + if (!(body instanceof Blob)) + throw new Error("Expected a bounded file slice"); + uploadedParts++; + uploadedBytes += body.size; + expect(body.size).toBe(64 * 1024 ** 2); + return new Response(null, { + headers: { ETag: `"part-${uploadedParts}"` }, + }); + } + if (url.endsWith("/complete")) { + const request = JSON.parse(String(init?.body)); + expect(request.loomImportAttemptId).toBe("loom-run:0"); + expect(request.parts).toHaveLength(160); + completed = true; + return Response.json({ objectIdentity: '"multipart-result"' }); + } + throw new Error("Unexpected multipart request"); + }) as typeof fetch; + try { + await uploadFileToStorage( + uploadFile.path, + { + type: "multipart", + videoId: "video", + key: "owner/video/raw-upload.mp4", + uploadId: "upload", + partSize: 64 * 1024 ** 2, + signPartUrl: "https://cap.test/sign-part", + completeUrl: "https://cap.test/complete", + abortUrl: "https://cap.test/abort", + loomImportAttemptId: "loom-run:0", + }, + "video/mp4", + ); + expect(uploadedParts).toBe(160); + expect(uploadedBytes).toBe(10 * 1024 ** 3); + expect(completed).toBe(true); + } finally { + await uploadFile.cleanup(); + } + }); + test.each([false, true])( "checks full remote bytes after Drive metadata version drift (corrupt=%s)", async (corrupt) => { diff --git a/apps/media-server/src/__tests__/routes/video.test.ts b/apps/media-server/src/__tests__/routes/video.test.ts index d2643b3bf7f..51c8f19e450 100644 --- a/apps/media-server/src/__tests__/routes/video.test.ts +++ b/apps/media-server/src/__tests__/routes/video.test.ts @@ -51,6 +51,8 @@ describe("GET /video/status", () => { expect(data).toHaveProperty("canAcceptNewProbeProcess"); expect(data).toHaveProperty("jobCount"); expect(data).toHaveProperty("jobs"); + expect(data.loomMultipartUploads).toBe(true); + expect(data.loomImportMaxBytes).toBe(10 * 1024 ** 3); expect(Array.isArray(data.jobs)).toBe(true); }); }); @@ -404,6 +406,44 @@ describe("POST /video/process", () => { mock.restore(); }); + test("accepts a Loom multipart original without a single-PUT original URL", async () => { + spyOn(jobManager, "canAcceptNewVideoProcess").mockReturnValue(false); + const response = await app.fetch( + videoPostRequest("/video/import", { + videoId: "test-id", + userId: "user-id", + videoUrl: "https://example.com/video.mp4", + outputPresignedUrl: "https://s3.example.com/output", + maxImportBytes: 10 * 1024 ** 3, + sourceUpload: { + type: "multipart", + videoId: "test-id", + loomImportAttemptId: "run:0", + key: "user-id/test-id/raw-upload.mp4", + uploadId: "upload-id", + partSize: 64 * 1024 ** 2, + signPartUrl: "https://cap.example.com/sign", + completeUrl: "https://cap.example.com/complete", + abortUrl: "https://cap.example.com/abort", + }, + }), + ); + expect(response.status).toBe(503); + expect((await response.json()).code).toBe("SERVER_BUSY"); + }); + + test("rejects an import without an original upload target", async () => { + const response = await app.fetch( + videoPostRequest("/video/import", { + videoId: "test-id", + userId: "user-id", + videoUrl: "https://example.com/video.mp4", + outputPresignedUrl: "https://s3.example.com/output", + }), + ); + expect(response.status).toBe(400); + }); + test("returns 400 for missing required fields", async () => { const response = await app.fetch(videoPostRequest("/video/process", {})); diff --git a/apps/media-server/src/lib/media-video.ts b/apps/media-server/src/lib/media-video.ts index 48158979327..8615f9c2b59 100644 --- a/apps/media-server/src/lib/media-video.ts +++ b/apps/media-server/src/lib/media-video.ts @@ -58,6 +58,7 @@ const MULTIPART_MIN_PART_SIZE_BYTES = 5 * 1024 * 1024; const MULTIPART_MAX_PARTS = 10_000; const MULTIPART_ABORT_TIMEOUT_MS = 30_000; const STORAGE_ERROR_BODY_LIMIT_BYTES = 2_048; +export const LOOM_IMPORT_MAX_BYTES = 10 * 1024 ** 3; export type StorageUploadTarget = | { @@ -77,6 +78,7 @@ export type StorageUploadTarget = completeUrl: string; abortUrl: string; webhookSecret?: string; + loomImportAttemptId?: string; }; export interface StorageUploadReceipt { @@ -1186,10 +1188,24 @@ function getStreamingDownloadTimeoutMs(durationSeconds: number | null): number { async function downloadStreamingVideoToTemp( videoUrl: string, abortSignal?: AbortSignal, + maxBytes?: number, ): Promise { await ensureTempDir(); const manifestDir = await mkdtemp(join(getTempDir(), "stream-")); const tempFile = await createTempFile(".mkv"); + const sizeController = new AbortController(); + const signal = AbortSignal.any([ + sizeController.signal, + ...(abortSignal ? [abortSignal] : []), + ]); + const sizeCheck = maxBytes + ? setInterval(() => { + if (file(tempFile.path).size > maxBytes) { + sizeController.abort(new Error("Loom imports are limited to 10 GB")); + } + }, 250) + : undefined; + sizeCheck?.unref(); const cleanup = async () => { await tempFile.cleanup(); await rm(manifestDir, { force: true, recursive: true }).catch(() => {}); @@ -1200,7 +1216,7 @@ async function downloadStreamingVideoToTemp( const inputPath = await materializeStreamingInput( videoUrl, manifestDir, - abortSignal, + signal, ); const durationSeconds = await estimateMaterializedStreamingDurationSeconds(manifestDir); @@ -1213,8 +1229,11 @@ async function downloadStreamingVideoToTemp( ffmpegHlsCapabilities, ), downloadTimeoutMs, - abortSignal, + signal, ); + if (maxBytes && file(tempFile.path).size > maxBytes) { + throw new Error("Loom imports are limited to 10 GB"); + } if (file(tempFile.path).size === 0) { throw new Error("Streaming download produced empty file"); @@ -1226,7 +1245,10 @@ async function downloadStreamingVideoToTemp( }; } catch (err) { await cleanup(); + if (sizeController.signal.aborted) throw sizeController.signal.reason; throw err; + } finally { + if (sizeCheck) clearInterval(sizeCheck); } } @@ -1234,13 +1256,19 @@ export async function downloadVideoToTemp( videoUrl: string, inputExtension?: string, abortSignal?: AbortSignal, + maxBytes?: number, ): Promise { if (isStreamingUrl(videoUrl)) { - return await downloadStreamingVideoToTemp(videoUrl, abortSignal); + return await downloadStreamingVideoToTemp(videoUrl, abortSignal, maxBytes); } const local = await materializeMedia(videoUrl, abortSignal); - if (local) return { path: local.path, cleanup: async () => {} }; + if (local) { + if (maxBytes && file(local.path).size > maxBytes) { + throw new Error("Loom imports are limited to 10 GB"); + } + return { path: local.path, cleanup: async () => {} }; + } const tempFile = await createTempFile( normalizeVideoInputExtension(inputExtension), @@ -1266,17 +1294,27 @@ export async function downloadVideoToTemp( if (!response.body) { throw new Error("No response body"); } + if (maxBytes && Number(response.headers.get("content-length")) > maxBytes) { + await response.body.cancel(); + throw new Error("Loom imports are limited to 10 GB"); + } const reader = response.body.getReader(); const writer = file(tempFile.path).writer(); + let downloadedBytes = 0; try { while (true) { const { done, value } = await reader.read(); if (done) break; + downloadedBytes += value.byteLength; + if (maxBytes && downloadedBytes > maxBytes) { + throw new Error("Loom imports are limited to 10 GB"); + } writer.write(value); } - await writer.end(); } finally { + await reader.cancel().catch(() => {}); + await writer.end(); reader.releaseLock(); } @@ -2389,6 +2427,7 @@ async function getMultipartPartUrl( videoId: target.videoId, generation: target.generation, attemptId: target.attemptId, + loomImportAttemptId: target.loomImportAttemptId, key: target.key, uploadId: target.uploadId, partNumber, @@ -2490,6 +2529,7 @@ async function completeMultipartUpload( videoId: target.videoId, generation: target.generation, attemptId: target.attemptId, + loomImportAttemptId: target.loomImportAttemptId, key: target.key, uploadId: target.uploadId, parts, @@ -2524,6 +2564,7 @@ async function abortMultipartUpload( videoId: target.videoId, generation: target.generation, attemptId: target.attemptId, + loomImportAttemptId: target.loomImportAttemptId, key: target.key, uploadId: target.uploadId, }, diff --git a/apps/media-server/src/routes/video.ts b/apps/media-server/src/routes/video.ts index cd2b44e68b5..e635ae91a06 100644 --- a/apps/media-server/src/routes/video.ts +++ b/apps/media-server/src/routes/video.ts @@ -54,6 +54,7 @@ import { downloadVideoToTemp, generatePreviewGif, generateThumbnail, + LOOM_IMPORT_MAX_BYTES, muxMediaTracksToMp4, processVideo, repairContainer, @@ -111,6 +112,31 @@ const convertSchema = z.object({ inputExtension: z.string().optional(), }); +const storageUploadSchema = z.discriminatedUnion("type", [ + z.object({ + type: z.literal("put"), + url: z.string().url(), + ifNoneMatch: z.literal("*").optional(), + }), + z.object({ + type: z.literal("multipart"), + videoId: z.string(), + generation: z.string().min(1).max(200).optional(), + attemptId: z.string().min(1).max(200).optional(), + key: z.string().min(1), + uploadId: z.string().min(1), + partSize: z + .number() + .int() + .min(5 * 1024 * 1024), + signPartUrl: z.string().url(), + completeUrl: z.string().url(), + abortUrl: z.string().url(), + webhookSecret: z.string().optional(), + loomImportAttemptId: z.string().min(1).max(200).optional(), + }), +]); + const processSchema = z.object({ audioLevels: z.boolean().optional(), videoId: z.string(), @@ -118,6 +144,9 @@ const processSchema = z.object({ videoUrl: z.string().url(), outputPresignedUrl: z.string().url(), sourcePresignedUrl: z.string().url().optional(), + sourceUpload: storageUploadSchema.optional(), + outputUpload: storageUploadSchema.optional(), + maxImportBytes: z.literal(LOOM_IMPORT_MAX_BYTES).optional(), thumbnailPresignedUrl: z.string().url().optional(), previewGifPresignedUrl: z.string().url().optional(), webhookUrl: z.string().url().optional(), @@ -380,6 +409,8 @@ video.get("/status", (c) => { return c.json({ instanceId: getInstanceId(), pid: process.pid, + loomImportMaxBytes: LOOM_IMPORT_MAX_BYTES, + loomMultipartUploads: true, activeVideoProcesses: getActiveVideoProcessCount(), maxConcurrentVideoProcesses: getMaxConcurrentVideoProcesses(), effectiveMaxVideoProcesses: resources.effectiveMax, @@ -676,7 +707,10 @@ video.on("POST", ["/process", "/import"], async (c) => { const body = await c.req.json(); const result = ( c.req.path.endsWith("/import") - ? processSchema.extend({ sourcePresignedUrl: z.string().url() }) + ? processSchema.refine( + (input) => input.sourcePresignedUrl || input.sourceUpload, + { message: "An original video upload target is required" }, + ) : processSchema ).safeParse(body); @@ -1292,6 +1326,21 @@ async function processVideoAsync( let repairedTempFile: TempFileHandle | null = null; let lastResortRepairFile: TempFileHandle | null = null; + let sourceUploaded = false; + let outputUploaded = false; + const abortPendingUploads = async () => { + const targets = [ + !sourceUploaded ? options.sourceUpload : undefined, + !outputUploaded ? options.outputUpload : undefined, + ].filter((target): target is StorageUploadTarget => target !== undefined); + await Promise.all( + targets.map((target) => + abortStorageUploadTarget(target).catch((error) => { + console.warn("Loom multipart cleanup failed", error); + }), + ), + ); + }; try { updateJob(jobId, { @@ -1306,22 +1355,27 @@ async function processVideoAsync( videoUrl, options.inputExtension, abortController.signal, + options.maxImportBytes, ), ); updateJob(jobId, { inputTempFile }); const sourcePresignedUrl = options.sourcePresignedUrl; - if (sourcePresignedUrl) { + if (sourcePresignedUrl || options.sourceUpload) { updateJob(jobId, { message: "Saving original video..." }); await sendWebhook(job); await withJobHeartbeat(jobId, () => - uploadFileToS3( + uploadFileToStorage( inputTempFile.path, - sourcePresignedUrl, + options.sourceUpload ?? { + type: "put", + url: sourcePresignedUrl as string, + }, "video/mp4", abortController.signal, ), ); + sourceUploaded = true; } const isWebm = isWebmInput(options.inputExtension); @@ -1372,6 +1426,12 @@ async function processVideoAsync( ); lastResortRepairFile = lrrf; updateJob(jobId, { outputTempFile }); + if ( + options.maxImportBytes && + file(outputTempFile.path).size > options.maxImportBytes + ) { + throw new Error("Loom imports are limited to 10 GB"); + } updateJob(jobId, { phase: "uploading", @@ -1380,12 +1440,13 @@ async function processVideoAsync( }); await sendWebhook(job); - const uploadReceipt = await uploadFileToS3( + const uploadReceipt = await uploadFileToStorage( outputTempFile.path, - outputPresignedUrl, + options.outputUpload ?? { type: "put", url: outputPresignedUrl }, "video/mp4", abortController.signal, ); + outputUploaded = true; if (thumbnailPresignedUrl || previewGifPresignedUrl) { updateJob(jobId, { @@ -1459,6 +1520,7 @@ async function processVideoAsync( setTimeout(() => deleteJob(jobId), 5 * 60 * 1000); } catch (err) { + await abortPendingUploads(); if (!abortController.signal.aborted) { console.error(`[video/process] Error processing job ${jobId}:`, err); } @@ -1648,30 +1710,6 @@ video.post("/force-cleanup", (c) => { }); }); -const muxSegmentsOutputUploadSchema = z.discriminatedUnion("type", [ - z.object({ - type: z.literal("put"), - url: z.string().url(), - ifNoneMatch: z.literal("*").optional(), - }), - z.object({ - type: z.literal("multipart"), - videoId: z.string(), - generation: z.string().min(1).max(200).optional(), - attemptId: z.string().min(1).max(200).optional(), - key: z.string().min(1), - uploadId: z.string().min(1), - partSize: z - .number() - .int() - .min(5 * 1024 * 1024), - signPartUrl: z.string().url(), - completeUrl: z.string().url(), - abortUrl: z.string().url(), - webhookSecret: z.string().optional(), - }), -]); - const strongObjectIdentitySchema = z .string() .max(1_024) @@ -2013,7 +2051,7 @@ const muxSegmentsSchema = z videoId: z.string(), userId: z.string(), outputPresignedUrl: z.string().url().optional(), - outputUpload: muxSegmentsOutputUploadSchema.optional(), + outputUpload: storageUploadSchema.optional(), thumbnailPresignedUrl: z.string().url().optional(), previewGifPresignedUrl: z.string().url().optional(), webhookUrl: z.string().url().optional(), diff --git a/apps/web/__tests__/unit/loom-import-start.test.ts b/apps/web/__tests__/unit/loom-import-start.test.ts index e7b17ae9740..bf4127d8e2d 100644 --- a/apps/web/__tests__/unit/loom-import-start.test.ts +++ b/apps/web/__tests__/unit/loom-import-start.test.ts @@ -7,6 +7,7 @@ const mocks = vi.hoisted(() => ({ start: vi.fn(), getRun: vi.fn(), getWorld: vi.fn(), + mediaJobFailed: vi.fn(), })); vi.mock("@cap/database", () => ({ @@ -34,6 +35,9 @@ vi.mock("drizzle-orm", () => ({ })); vi.mock("workflow/api", () => ({ start: mocks.start, getRun: mocks.getRun })); vi.mock("workflow/runtime", () => ({ getWorld: mocks.getWorld })); +vi.mock("@/lib/loom-media-dispatch", () => ({ + isLoomMediaJobFailed: mocks.mediaJobFailed, +})); vi.mock("@/workflows/import-loom-video", () => ({ importLoomVideoWorkflow: vi.fn(), })); @@ -60,6 +64,7 @@ describe("Loom workflow startup receipts", () => { mocks.where.mockResolvedValue([{ affectedRows: 1 }]); mocks.queue.mockResolvedValue({ messageId: "message-1" }); mocks.getWorld.mockReturnValue({ queue: mocks.queue }); + mocks.mediaJobFailed.mockResolvedValue(false); mocks.start.mockImplementation( async ( _workflow: unknown, @@ -78,11 +83,32 @@ describe("Loom workflow startup receipts", () => { ); }); + it.each([false, true])( + "requires a matching failed worker before releasing an accepted dispatch: %s", + async (failed) => { + mocks.getRun.mockReturnValue({ status: Promise.resolve("failed") }); + mocks.mediaJobFailed.mockResolvedValue(failed); + expect( + await getLoomImportRunState({ + runId: "run-1", + dispatch: "accepted", + mediaDispatch: { + attemptId: "run-1:0", + videoId: "video-1", + jobId: "worker-1", + state: "accepted", + }, + }), + ).toBe(failed ? "inactive" : "unconfirmed"); + expect(mocks.mediaJobFailed).toHaveBeenCalledWith("worker-1", "video-1"); + }, + ); + it("records the run before dispatch and stores queue acceptance", async () => { await startLoomImportWorkflow(payload); expect( mocks.set.mock.calls.map(([value]) => - JSON.parse(value.metadata.values[1]), + JSON.parse(value.metadata.values.at(-1)), ), ).toEqual([ { runId: "run-1", dispatch: "pending" }, @@ -116,7 +142,7 @@ describe("Loom workflow startup receipts", () => { canRetry: true, }); expect( - JSON.parse(mocks.set.mock.calls.at(-1)?.[0].metadata.values[1]), + JSON.parse(mocks.set.mock.calls.at(-1)?.[0].metadata.values.at(-1)), ).toEqual({ runId: "run-1", dispatch: "rejected" }); }); @@ -126,7 +152,7 @@ describe("Loom workflow startup receipts", () => { canRetry: false, }); expect( - JSON.parse(mocks.set.mock.calls.at(-1)?.[0].metadata.values[1]), + JSON.parse(mocks.set.mock.calls.at(-1)?.[0].metadata.values.at(-1)), ).toEqual({ runId: "run-1", dispatch: "uncertain" }); }); @@ -152,7 +178,8 @@ describe("Loom workflow startup receipts", () => { ); await expect(startLoomImportWorkflow(payload)).resolves.toBeUndefined(); expect( - JSON.parse(mocks.set.mock.calls.at(-1)?.[0].metadata.values[1]).dispatch, + JSON.parse(mocks.set.mock.calls.at(-1)?.[0].metadata.values.at(-1)) + .dispatch, ).toBe("accepted"); }); @@ -210,6 +237,20 @@ describe("Loom workflow startup receipts", () => { ).toBe("unconfirmed"); }); + it.each(["completed", "failed", "cancelled"])( + "holds an uncertain media dispatch after the workflow is %s", + async (status) => { + mocks.getRun.mockReturnValue({ status: Promise.resolve(status) }); + expect( + await getLoomImportRunState({ + runId: "run-1", + dispatch: "accepted", + mediaDispatch: { attemptId: "run-1:0", state: "uncertain" }, + }), + ).toBe("unconfirmed"); + }, + ); + it("allows rejected and legacy imports without an active run check", async () => { expect( await getLoomImportRunState({ runId: "run-1", dispatch: "rejected" }), diff --git a/apps/web/__tests__/unit/loom-import-upload.test.ts b/apps/web/__tests__/unit/loom-import-upload.test.ts new file mode 100644 index 00000000000..8d83f304fe4 --- /dev/null +++ b/apps/web/__tests__/unit/loom-import-upload.test.ts @@ -0,0 +1,146 @@ +import { Effect } from "effect"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + set: vi.fn(), + where: vi.fn(), + create: vi.fn(), + abort: vi.fn(), +})); + +vi.mock("@cap/database", () => ({ + db: () => ({ update: () => ({ set: mocks.set }) }), +})); +vi.mock("@cap/database/schema", () => ({ + videos: { + id: "id", + ownerId: "ownerId", + orgId: "orgId", + metadata: "metadata", + }, +})); +vi.mock("@/lib/workflow-runtime", () => ({ + runWorkflowPromise: (effect: Effect.Effect) => + Effect.runPromise(effect), +})); + +import { + LOOM_IMPORT_MAX_BYTES, + LOOM_IMPORT_PART_BYTES, + prepareLoomMultipartUploads, +} from "@/lib/loom-import-upload"; + +type Input = Parameters[0]; + +const video = { + id: "video", + ownerId: "owner", + orgId: "organization", + metadata: { loomImportRun: { runId: "run", dispatch: "accepted" } }, +} as Input["video"]; + +function input(): Input { + return { + video, + bucket: { + provider: "s3", + multipart: { create: mocks.create, abort: mocks.abort }, + } as unknown as Input["bucket"], + attemptId: "run:0", + sourceKey: "owner/video/raw-upload.mp4", + webhookBaseUrl: "https://cap.example.com", + webhookSecret: "fixture-secret", + }; +} + +describe("Loom multipart upload preparation", () => { + beforeEach(() => { + vi.resetAllMocks(); + mocks.set.mockReturnValue({ where: mocks.where }); + mocks.where.mockResolvedValue([{ affectedRows: 1 }]); + mocks.create.mockImplementation((key: string) => + Effect.succeed({ UploadId: `upload:${key}` }), + ); + mocks.abort.mockReturnValue(Effect.void); + }); + + it("prepares original and output parts for files up to 10 GB", async () => { + const result = await prepareLoomMultipartUploads(input()); + expect(LOOM_IMPORT_MAX_BYTES).toBe(10 * 1024 ** 3); + expect(Math.ceil(LOOM_IMPORT_MAX_BYTES / LOOM_IMPORT_PART_BYTES)).toBe(160); + expect(mocks.create.mock.calls.map(([key]) => key)).toEqual([ + "owner/video/raw-upload.mp4", + "owner/video/result.mp4", + ]); + expect(result.sourceUpload).toMatchObject({ + type: "multipart", + key: "owner/video/raw-upload.mp4", + loomImportAttemptId: "run:0", + partSize: 64 * 1024 ** 2, + }); + expect(result.outputUpload).toMatchObject({ + type: "multipart", + key: "owner/video/result.mp4", + loomImportAttemptId: "run:0", + }); + expect(mocks.where).toHaveBeenCalledOnce(); + }); + + it("reuses the same multipart reservations after worker backpressure", async () => { + const args = input(); + args.video = { + ...video, + metadata: { + ...video.metadata, + loomImportUpload: { + attemptId: "run:0", + ownerId: "owner", + organizationId: "organization", + source: { key: "owner/video/raw-upload.mp4", uploadId: "source-id" }, + output: { key: "owner/video/result.mp4", uploadId: "output-id" }, + }, + }, + }; + const result = await prepareLoomMultipartUploads(args); + expect(result.sourceUpload?.uploadId).toBe("source-id"); + expect(result.outputUpload?.uploadId).toBe("output-id"); + expect(mocks.create).not.toHaveBeenCalled(); + expect(mocks.where).not.toHaveBeenCalled(); + }); + + it("aborts only newly reserved uploads when recording ownership changes", async () => { + mocks.where.mockResolvedValue([{ affectedRows: 0 }]); + await expect(prepareLoomMultipartUploads(input())).rejects.toThrow( + "ownership changed", + ); + expect(mocks.abort.mock.calls).toEqual([ + ["owner/video/raw-upload.mp4", "upload:owner/video/raw-upload.mp4"], + ["owner/video/result.mp4", "upload:owner/video/result.mp4"], + ]); + }); + + it("cleans up the original reservation if the output cannot be reserved", async () => { + mocks.create + .mockReturnValueOnce(Effect.succeed({ UploadId: "source-id" })) + .mockReturnValueOnce(Effect.fail(new Error("Storage unavailable"))); + await expect(prepareLoomMultipartUploads(input())).rejects.toThrow(); + expect(mocks.abort).toHaveBeenCalledExactlyOnceWith( + "owner/video/raw-upload.mp4", + "source-id", + ); + expect(mocks.where).not.toHaveBeenCalled(); + }); + + it("refuses a stale workflow and another recording's source key", async () => { + await expect( + prepareLoomMultipartUploads({ ...input(), attemptId: "another-run:0" }), + ).rejects.toThrow("workflow ownership changed"); + await expect( + prepareLoomMultipartUploads({ + ...input(), + sourceKey: "other/video/raw.mp4", + }), + ).rejects.toThrow("does not match"); + expect(mocks.create).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/__tests__/unit/loom-media-dispatch.test.ts b/apps/web/__tests__/unit/loom-media-dispatch.test.ts new file mode 100644 index 00000000000..f0fbaf347ab --- /dev/null +++ b/apps/web/__tests__/unit/loom-media-dispatch.test.ts @@ -0,0 +1,51 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@cap/env", () => ({ + serverEnv: () => ({ + MEDIA_SERVER_URL: "https://worker.example.com", + MEDIA_SERVER_WEBHOOK_SECRET: "fixture-secret", + }), +})); + +import { isLoomMediaJobFailed } from "@/lib/loom-media-dispatch"; + +describe("Loom worker failure reconciliation", () => { + afterEach(() => vi.unstubAllGlobals()); + + it.each([ + [{ jobId: "job", videoId: "video", phase: "error" }, true], + [{ jobId: "other", videoId: "video", phase: "error" }, false], + [{ jobId: "job", videoId: "other", phase: "error" }, false], + [{ jobId: "job", videoId: "video", phase: "processing" }, false], + [{ jobId: "job", videoId: "video", phase: "complete" }, false], + [{ jobId: "job", videoId: "video", phase: "cancelled" }, false], + [{}, false], + ])( + "requires the exact job, recording and failed phase: %j", + async (body, failed) => { + const fetch = vi.fn().mockResolvedValue(Response.json(body)); + vi.stubGlobal("fetch", fetch); + expect(await isLoomMediaJobFailed("job", "video")).toBe(failed); + expect(fetch).toHaveBeenCalledWith( + "https://worker.example.com/video/process/job/status", + expect.objectContaining({ redirect: "error" }), + ); + }, + ); + + it.each([404, 500])( + "does not treat HTTP %s as a failed job", + async (status) => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue(new Response(null, { status })), + ); + expect(await isLoomMediaJobFailed("job", "video")).toBe(false); + }, + ); + + it("holds an unavailable worker instead of authorizing another dispatch", async () => { + vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("Timed out"))); + expect(await isLoomMediaJobFailed("job", "video")).toBe(false); + }); +}); diff --git a/apps/web/__tests__/unit/media-processing-workflows.test.ts b/apps/web/__tests__/unit/media-processing-workflows.test.ts index e3ce61f7a13..2766fc032a2 100644 --- a/apps/web/__tests__/unit/media-processing-workflows.test.ts +++ b/apps/web/__tests__/unit/media-processing-workflows.test.ts @@ -11,6 +11,9 @@ const mocks = vi.hoisted(() => ({ head: vi.fn(), remove: vi.fn(), fetch: vi.fn(), + statusFetch: vi.fn(), + dispatch: vi.fn(), + mediaJobFailed: vi.fn(), sleep: vi.fn(), })); @@ -62,9 +65,15 @@ vi.mock("@/lib/ai-generation-entitlement", () => ({ vi.mock("workflow", () => ({ sleep: mocks.sleep, FatalError: class FatalError extends Error {}, + getWorkflowMetadata: () => ({ workflowRunId: "run-import" }), RetryableError: class RetryableError extends Error {}, })); +vi.mock("@/lib/loom-media-dispatch", () => ({ + recordLoomMediaDispatch: mocks.dispatch, + isLoomMediaJobFailed: mocks.mediaJobFailed, +})); + import { importLoomVideoWorkflow } from "@/workflows/import-loom-video"; import { processVideoWorkflow } from "@/workflows/process-video"; @@ -77,6 +86,7 @@ const payload = { }; const video = { id: "video", + ownerId: "owner", source: { type: "webMP4" }, createdAt: new Date(), updatedAt: new Date(), @@ -122,8 +132,20 @@ describe("media processing workflows", () => { Effect.succeed({ ContentLength: 2048 }), ); mocks.remove.mockImplementation(() => Effect.void); + mocks.dispatch.mockReset().mockResolvedValue(undefined); + mocks.mediaJobFailed.mockReset().mockResolvedValue(true); mocks.sleep.mockResolvedValue(undefined); - vi.stubGlobal("fetch", mocks.fetch); + mocks.statusFetch.mockReset().mockImplementation(async () => + Response.json({ + loomMultipartUploads: true, + loomImportMaxBytes: 10 * 1024 ** 3, + }), + ); + vi.stubGlobal("fetch", (url: string, init?: RequestInit) => + url.endsWith("/video/status") + ? mocks.statusFetch(url, init) + : mocks.fetch(url, init), + ); mocks.fetch.mockReset().mockImplementation(async (url: string) => { if (url.startsWith("https://www.loom.com/")) { return Response.json({ url: "https://cdn.loom.com/original.mp4" }); @@ -365,6 +387,56 @@ describe("media processing workflows", () => { ); }); + it("does not retry a failed workflow while its accepted worker is unconfirmed", async () => { + mocks.mediaJobFailed.mockResolvedValue(false); + mocks.rows = [ + [video], + [{ ...pending, processingError: "Processing status interrupted" }], + ]; + await expect(importLoomVideoWorkflow(payload)).rejects.toThrow( + "must be reconciled before retrying", + ); + expect(mocks.mediaJobFailed).toHaveBeenCalledWith("job-1", "video"); + expect( + mocks.fetch.mock.calls.filter(([url]) => + url.startsWith("https://worker.example.com/"), + ), + ).toHaveLength(1); + }); + + it("blocks an old worker before creating upload targets or dispatching", async () => { + mocks.statusFetch.mockResolvedValue(Response.json({})); + await expect(importLoomVideoWorkflow(payload)).rejects.toThrow( + "must be updated for 10 GB", + ); + expect(mocks.fetch).not.toHaveBeenCalled(); + expect(mocks.put).not.toHaveBeenCalled(); + expect(mocks.dispatch).not.toHaveBeenCalled(); + }); + + it("leaves an uncertain dispatch receipt after a worker request times out", async () => { + mocks.rows = [[video]]; + mocks.fetch.mockImplementation(async (url: string) => { + if (url.startsWith("https://www.loom.com/")) + return Response.json({ url: "https://cdn.loom.com/original.mp4" }); + throw new Error("Response timed out after dispatch"); + }); + await expect(importLoomVideoWorkflow(payload)).rejects.toThrow("timed out"); + expect(mocks.dispatch).toHaveBeenCalledTimes(1); + expect(mocks.dispatch).toHaveBeenCalledWith( + expect.objectContaining({ + runId: "run-import", + attemptId: "run-import:0", + state: "uncertain", + }), + ); + expect( + mocks.fetch.mock.calls.filter(([url]) => + String(url).startsWith("https://worker.example.com/"), + ), + ).toHaveLength(1); + }); + it("supports Loom streaming fallbacks without reading a nonexistent raw object", async () => { mocks.fetch.mockImplementation(async (url: string) => Response.json( diff --git a/apps/web/__tests__/unit/media-server-multipart.test.ts b/apps/web/__tests__/unit/media-server-multipart.test.ts index 886171c3a01..9aa4cd46e5f 100644 --- a/apps/web/__tests__/unit/media-server-multipart.test.ts +++ b/apps/web/__tests__/unit/media-server-multipart.test.ts @@ -364,4 +364,96 @@ describe("media-server multipart fencing", () => { expect((await request("unexpected")).status).toBe(404); expect(mocks.storage).not.toHaveBeenCalled(); }); + + function loomUpload(source = true) { + const sourceKey = `${ownerId}/${videoId}/raw-upload.mp4`; + const resultKey = `${ownerId}/${videoId}/result.mp4`; + const row = { + id: videoId, + ownerId, + orgId: "organization", + metadata: { + loomImportUpload: { + attemptId: "loom-run:0", + ownerId, + organizationId: "organization", + source: { key: sourceKey, uploadId: "source-upload" }, + output: { key: resultKey, uploadId: "result-upload" }, + }, + }, + }; + mocks.rows = [row]; + return { + row, + body: { + videoId, + key: source ? sourceKey : resultKey, + uploadId: source ? "source-upload" : "result-upload", + loomImportAttemptId: "loom-run:0", + partNumber: 1, + contentLength: 64 * 1024 ** 2, + parts: [{ partNumber: 1, etag: '"part-1"', size: 64 * 1024 ** 2 }], + }, + }; + } + + it.each([true, false])( + "signs only the registered Loom upload and preserves existing objects (source=%s)", + async (source) => { + const { body } = loomUpload(source); + expect((await request("sign-part", body)).status).toBe(200); + expect(mocks.signPart).toHaveBeenCalledWith(body.key, body.uploadId, 1, { + ContentLength: body.contentLength, + }); + expect((await request("complete", body)).status).toBe(200); + expect(mocks.complete).toHaveBeenCalledWith( + body.key, + body.uploadId, + expect.objectContaining({ IfNoneMatch: "*" }), + ); + }, + ); + + it.each([ + { loomImportAttemptId: "old-run:0" }, + { loomImportAttemptId: undefined }, + { uploadId: "unregistered-upload" }, + { key: "other-owner/other-video/raw-upload.mp4" }, + ])("blocks a stale or substituted Loom upload %#", async (change) => { + const { body } = loomUpload(false); + expect((await request("sign-part", { ...body, ...change })).status).toBe( + 409, + ); + expect((await request("complete", { ...body, ...change })).status).toBe( + 409, + ); + expect(mocks.signPart).not.toHaveBeenCalled(); + expect(mocks.complete).not.toHaveBeenCalled(); + }); + + it("blocks Loom upload finalization after a destination move", async () => { + const { body, row } = loomUpload(); + row.orgId = "another-organization"; + expect((await request("complete", body)).status).toBe(409); + expect(mocks.complete).not.toHaveBeenCalled(); + }); + + it("accepts exactly 10 GB and rejects a further multipart part", async () => { + const { body } = loomUpload(); + const parts = Array.from({ length: 160 }, (_, index) => ({ + partNumber: index + 1, + etag: `"part-${index + 1}"`, + size: 64 * 1024 ** 2, + })); + expect((await request("complete", { ...body, parts })).status).toBe(200); + mocks.complete.mockClear(); + expect( + (await request("sign-part", { ...body, partNumber: 161 })).status, + ).toBe(400); + expect( + (await request("complete", { ...body, parts: [...parts, parts[0]] })) + .status, + ).toBe(400); + expect(mocks.complete).not.toHaveBeenCalled(); + }); }); diff --git a/apps/web/app/api/webhooks/media-server/multipart/[action]/route.ts b/apps/web/app/api/webhooks/media-server/multipart/[action]/route.ts index 1a956e0726a..91d4f0bf21a 100644 --- a/apps/web/app/api/webhooks/media-server/multipart/[action]/route.ts +++ b/apps/web/app/api/webhooks/media-server/multipart/[action]/route.ts @@ -9,6 +9,10 @@ import { type NextRequest, NextResponse } from "next/server"; import { z } from "zod"; import { getProcessingState } from "@/lib/desktop-recording-jobs"; import { getDesktopRecordingOutputKey } from "@/lib/desktop-recording-source"; +import { + LOOM_IMPORT_MAX_BYTES, + LOOM_IMPORT_PART_BYTES, +} from "@/lib/loom-import-upload"; import { runPromise } from "@/lib/server"; import { decodeStorageVideo } from "@/lib/video-storage"; @@ -16,6 +20,7 @@ const baseSchema = z.object({ videoId: z.string().min(1), key: z.string().min(1), uploadId: z.string().min(1), + loomImportAttemptId: z.string().min(1).max(200).optional(), generation: z .string() .regex(/^[a-zA-Z0-9_-]{1,64}$/) @@ -80,6 +85,32 @@ async function getMultipartContext( const fenced = payload.generation !== undefined || payload.attemptId !== undefined; + const loom = + payload.loomImportAttemptId !== undefined || + video.metadata?.loomImportUpload !== undefined; + if (loom) { + const receipt = video.metadata?.loomImportUpload; + const upload = [receipt?.source, receipt?.output].find( + (value) => + value?.key === payload.key && value.uploadId === payload.uploadId, + ); + if ( + fenced || + !receipt || + receipt.attemptId !== payload.loomImportAttemptId || + receipt.ownerId !== video.ownerId || + receipt.organizationId !== video.orgId || + !upload || + !payload.key.startsWith(`${video.ownerId}/${video.id}/`) + ) { + return { + response: NextResponse.json( + { error: "Loom import upload is no longer current" }, + { status: 409 }, + ), + }; + } + } const expectedKey = fenced && payload.generation && payload.attemptId ? getDesktopRecordingOutputKey( @@ -90,7 +121,7 @@ async function getMultipartContext( ) : `${video.ownerId}/${video.id}/result.mp4`; if ( - payload.key !== expectedKey || + (!loom && payload.key !== expectedKey) || (fenced && (!payload.generation || !payload.attemptId)) ) { return { @@ -134,7 +165,7 @@ async function getMultipartContext( }; } - return { bucket, fenced }; + return { bucket, fenced, loom }; } async function readJson(request: NextRequest) { @@ -168,14 +199,32 @@ export async function POST( const multipartContext = await getMultipartContext(payload.data); if ("response" in multipartContext) return multipartContext.response; + if ( + multipartContext.loom && + (payload.data.partNumber > + Math.ceil(LOOM_IMPORT_MAX_BYTES / LOOM_IMPORT_PART_BYTES) || + payload.data.contentLength > LOOM_IMPORT_PART_BYTES) + ) { + return NextResponse.json( + { error: "Loom imports are limited to 10 GB" }, + { status: 400 }, + ); + } - const url = await multipartContext.bucket.multipart - .getPresignedUploadPartUrl( - payload.data.key, - payload.data.uploadId, - payload.data.partNumber, - ) - .pipe(runPromise); + const partArguments = [ + payload.data.key, + payload.data.uploadId, + payload.data.partNumber, + ] as const; + const url = await (multipartContext.loom + ? multipartContext.bucket.multipart.getPresignedUploadPartUrl( + ...partArguments, + { ContentLength: payload.data.contentLength }, + ) + : multipartContext.bucket.multipart.getPresignedUploadPartUrl( + ...partArguments, + ) + ).pipe(runPromise); return NextResponse.json({ url }); } @@ -191,6 +240,19 @@ export async function POST( const multipartContext = await getMultipartContext(payload.data); if ("response" in multipartContext) return multipartContext.response; + if ( + multipartContext.loom && + (payload.data.parts.length > + Math.ceil(LOOM_IMPORT_MAX_BYTES / LOOM_IMPORT_PART_BYTES) || + payload.data.parts.reduce((size, part) => size + part.size, 0) > + LOOM_IMPORT_MAX_BYTES || + payload.data.parts.some((part) => part.size > LOOM_IMPORT_PART_BYTES)) + ) { + return NextResponse.json( + { error: "Loom imports are limited to 10 GB" }, + { status: 400 }, + ); + } const parts = [...payload.data.parts] .sort((a, b) => a.partNumber - b.partNumber) @@ -208,7 +270,9 @@ export async function POST( const result = await multipartContext.bucket.multipart .complete(payload.data.key, payload.data.uploadId, { MultipartUpload: { Parts: parts }, - ...(multipartContext.fenced ? { IfNoneMatch: "*" } : {}), + ...(multipartContext.fenced || multipartContext.loom + ? { IfNoneMatch: "*" } + : {}), }) .pipe(runPromise); diff --git a/apps/web/lib/loom-import-start.ts b/apps/web/lib/loom-import-start.ts index b5ef8ca2d4e..d850973791d 100644 --- a/apps/web/lib/loom-import-start.ts +++ b/apps/web/lib/loom-import-start.ts @@ -5,6 +5,7 @@ import { User, Video } from "@cap/web-domain"; import { and, eq, sql } from "drizzle-orm"; import { getRun, start } from "workflow/api"; import { getWorld } from "workflow/runtime"; +import { isLoomMediaJobFailed } from "@/lib/loom-media-dispatch"; import { importLoomVideoWorkflow } from "@/workflows/import-loom-video"; type LoomImportRun = NonNullable; @@ -63,7 +64,20 @@ export async function getLoomImportRunState( status === "failed" || status === "cancelled" ) { - return "inactive"; + const mediaDispatch = receipt.mediaDispatch; + if (mediaDispatch?.state === "accepted") { + return mediaDispatch.jobId && + mediaDispatch.videoId && + (await isLoomMediaJobFailed( + mediaDispatch.jobId, + mediaDispatch.videoId, + )) + ? "inactive" + : "unconfirmed"; + } + return receipt.mediaDispatch?.state === "uncertain" + ? "unconfirmed" + : "inactive"; } return status === "pending" ? "pending" : "active"; } catch { @@ -75,7 +89,10 @@ async function saveReceipt(payload: ImportPayload, receipt: LoomImportRun) { const result = await db() .update(videos) .set({ - metadata: sql`JSON_SET(COALESCE(${videos.metadata}, JSON_OBJECT()), '$.loomImportRun', JSON_EXTRACT(${JSON.stringify(receipt)}, '$'))`, + metadata: + receipt.dispatch === "pending" + ? sql`JSON_SET(COALESCE(${videos.metadata}, JSON_OBJECT()), '$.loomImportRun', JSON_EXTRACT(${JSON.stringify(receipt)}, '$'))` + : sql`JSON_SET(COALESCE(${videos.metadata}, JSON_OBJECT()), '$.loomImportRun', JSON_MERGE_PATCH(COALESCE(JSON_EXTRACT(${videos.metadata}, '$.loomImportRun'), JSON_OBJECT()), JSON_EXTRACT(${JSON.stringify(receipt)}, '$')))`, }) .where( and( diff --git a/apps/web/lib/loom-import-upload.ts b/apps/web/lib/loom-import-upload.ts new file mode 100644 index 00000000000..fe06888caf2 --- /dev/null +++ b/apps/web/lib/loom-import-upload.ts @@ -0,0 +1,151 @@ +import { db } from "@cap/database"; +import { videos } from "@cap/database/schema"; +import type { VideoMetadata } from "@cap/database/types"; +import type { Storage } from "@cap/web-backend/src/Storage/index"; +import { and, eq, isNull, sql } from "drizzle-orm"; +import { Effect } from "effect"; +import { runWorkflowPromise } from "@/lib/workflow-runtime"; + +export const LOOM_IMPORT_MAX_BYTES = 10 * 1024 ** 3; +export const LOOM_IMPORT_PART_BYTES = 64 * 1024 ** 2; + +type Bucket = Effect.Effect.Success< + ReturnType +>[0]; +type UploadReceipt = NonNullable; + +export type LoomMultipartUploadTarget = { + type: "multipart"; + videoId: string; + key: string; + uploadId: string; + partSize: number; + signPartUrl: string; + completeUrl: string; + abortUrl: string; + webhookSecret: string; + loomImportAttemptId: string; +}; + +export async function prepareLoomMultipartUploads({ + video, + bucket, + attemptId, + sourceKey, + webhookBaseUrl, + webhookSecret, +}: { + video: typeof videos.$inferSelect; + bucket: Bucket; + attemptId: string; + sourceKey?: string; + webhookBaseUrl: string; + webhookSecret?: string; +}): Promise<{ + sourceUpload?: LoomMultipartUploadTarget; + outputUpload?: LoomMultipartUploadTarget; +}> { + if (bucket.provider !== "s3") return {}; + if (!webhookSecret) + throw new Error( + "Loom multipart uploads require media server authentication", + ); + const run = video.metadata?.loomImportRun; + if (run && !attemptId.startsWith(`${run.runId}:`)) { + throw new Error("Loom import workflow ownership changed"); + } + const previous = video.metadata?.loomImportUpload; + const outputKey = `${video.ownerId}/${video.id}/result.mp4`; + if ( + sourceKey && + (!sourceKey.startsWith(`${video.ownerId}/${video.id}/`) || + sourceKey === outputKey) + ) { + throw new Error("Loom import source key does not match the recording"); + } + const reusable = + previous?.attemptId === attemptId && + previous.ownerId === video.ownerId && + previous.organizationId === video.orgId && + previous.output.key === outputKey && + (!sourceKey || previous.source?.key === sourceKey); + let receipt = reusable ? previous : undefined; + const created: Array<{ key: string; uploadId: string }> = []; + const create = async (key: string) => { + const { UploadId: uploadId } = await bucket.multipart + .create(key, { ContentType: "video/mp4" }) + .pipe(runWorkflowPromise); + if (!uploadId) + throw new Error("Storage did not return a multipart upload id"); + const upload = { key, uploadId }; + created.push(upload); + return upload; + }; + try { + if (!receipt) { + receipt = { + attemptId, + ownerId: video.ownerId, + organizationId: video.orgId, + source: sourceKey ? await create(sourceKey) : undefined, + output: await create(outputKey), + }; + const result = await db() + .update(videos) + .set({ + metadata: sql`JSON_SET(COALESCE(${videos.metadata}, JSON_OBJECT()), '$.loomImportUpload', JSON_EXTRACT(${JSON.stringify(receipt)}, '$'))`, + }) + .where( + and( + eq(videos.id, video.id), + eq(videos.ownerId, video.ownerId), + video.orgId ? eq(videos.orgId, video.orgId) : isNull(videos.orgId), + run + ? sql`JSON_UNQUOTE(JSON_EXTRACT(${videos.metadata}, '$.loomImportRun.runId')) = ${run.runId}` + : sql`JSON_EXTRACT(${videos.metadata}, '$.loomImportRun') IS NULL`, + previous + ? sql`JSON_UNQUOTE(JSON_EXTRACT(${videos.metadata}, '$.loomImportUpload.attemptId')) = ${previous.attemptId}` + : sql`JSON_EXTRACT(${videos.metadata}, '$.loomImportUpload') IS NULL`, + ), + ); + const affectedRows = Array.isArray(result) + ? (result[0] as { affectedRows?: number } | undefined)?.affectedRows + : (result as { affectedRows?: number }).affectedRows; + if (affectedRows !== 1) + throw new Error("Loom import upload ownership changed"); + } + } catch (error) { + await Promise.all( + created.map(({ key, uploadId }) => + bucket.multipart.abort(key, uploadId).pipe( + Effect.catchAll(() => Effect.void), + runWorkflowPromise, + ), + ), + ); + throw error; + } + const target = ( + upload: UploadReceipt["output"], + ): LoomMultipartUploadTarget => { + const callback = (action: string) => + new URL(`/api/webhooks/media-server/multipart/${action}`, webhookBaseUrl) + .href; + return { + type: "multipart", + videoId: video.id, + ...upload, + partSize: LOOM_IMPORT_PART_BYTES, + signPartUrl: callback("sign-part"), + completeUrl: callback("complete"), + abortUrl: callback("abort"), + webhookSecret, + loomImportAttemptId: attemptId, + }; + }; + return { + sourceUpload: + sourceKey && receipt.source ? target(receipt.source) : undefined, + outputUpload: target(receipt.output), + }; +} diff --git a/apps/web/lib/loom-media-dispatch.ts b/apps/web/lib/loom-media-dispatch.ts new file mode 100644 index 00000000000..a9b73e8a639 --- /dev/null +++ b/apps/web/lib/loom-media-dispatch.ts @@ -0,0 +1,76 @@ +import { db } from "@cap/database"; +import { videos } from "@cap/database/schema"; +import { serverEnv } from "@cap/env"; +import { User, Video } from "@cap/web-domain"; +import { and, eq, or, sql } from "drizzle-orm"; + +export async function recordLoomMediaDispatch({ + videoId, + userId, + runId, + attemptId, + state, + jobId, +}: { + videoId: string; + userId: string; + runId: string; + attemptId: string; + state: "accepted" | "rejected" | "uncertain"; + jobId?: string; +}) { + const receipt = { + runId, + dispatch: "accepted", + mediaDispatch: { attemptId, videoId, state, jobId: jobId ?? null }, + }; + const result = await db() + .update(videos) + .set({ + metadata: sql`JSON_SET(COALESCE(${videos.metadata}, JSON_OBJECT()), '$.loomImportRun', JSON_MERGE_PATCH(COALESCE(JSON_EXTRACT(${videos.metadata}, '$.loomImportRun'), JSON_OBJECT()), JSON_EXTRACT(${JSON.stringify(receipt)}, '$')))`, + }) + .where( + and( + eq(videos.id, Video.VideoId.make(videoId)), + eq(videos.ownerId, User.UserId.make(userId)), + or( + sql`JSON_EXTRACT(${videos.metadata}, '$.loomImportRun') IS NULL`, + sql`JSON_UNQUOTE(JSON_EXTRACT(${videos.metadata}, '$.loomImportRun.runId')) = ${runId}`, + ), + ), + ); + const affectedRows = Array.isArray(result) + ? (result[0] as { affectedRows?: number } | undefined)?.affectedRows + : (result as { affectedRows?: number }).affectedRows; + if (affectedRows !== 1) + throw new Error("Loom media dispatch ownership changed"); +} + +export async function isLoomMediaJobFailed(jobId: string, videoId: string) { + const env = serverEnv(); + if (!env.MEDIA_SERVER_URL || !env.MEDIA_SERVER_WEBHOOK_SECRET) return false; + try { + const response = await fetch( + `${env.MEDIA_SERVER_URL}/video/process/${encodeURIComponent(jobId)}/status`, + { + headers: { "x-media-server-secret": env.MEDIA_SERVER_WEBHOOK_SECRET }, + signal: AbortSignal.timeout(15_000), + redirect: "error", + }, + ); + if (!response.ok) return false; + const progress: unknown = await response.json(); + return ( + typeof progress === "object" && + progress !== null && + "jobId" in progress && + progress.jobId === jobId && + "videoId" in progress && + progress.videoId === videoId && + "phase" in progress && + progress.phase === "error" + ); + } catch { + return false; + } +} diff --git a/apps/web/workflows/import-loom-video.ts b/apps/web/workflows/import-loom-video.ts index a854f21ec5b..422b27ccbd7 100644 --- a/apps/web/workflows/import-loom-video.ts +++ b/apps/web/workflows/import-loom-video.ts @@ -5,12 +5,26 @@ import { Storage } from "@cap/web-backend/src/Storage/index"; import { Video } from "@cap/web-domain"; import { eq } from "drizzle-orm"; import { Effect } from "effect"; -import { FatalError, RetryableError, sleep } from "workflow"; +import { + FatalError, + getWorkflowMetadata, + RetryableError, + sleep, +} from "workflow"; import { getLoomDownloadUrl, getReusableLoomDownloadUrl, LoomDownloadTemporaryError, } from "@/lib/loom-download-url"; +import { + LOOM_IMPORT_MAX_BYTES, + type LoomMultipartUploadTarget, + prepareLoomMultipartUploads, +} from "@/lib/loom-import-upload"; +import { + isLoomMediaJobFailed, + recordLoomMediaDispatch, +} from "@/lib/loom-media-dispatch"; import { createMediaServerCapacityError, isMediaServerCapacityError, @@ -90,6 +104,41 @@ interface LoomProcessingInput { importFromLoom?: boolean; } +async function requireLoomMediaServerSupport() { + "use step"; + + const env = serverEnv(); + if (!env.MEDIA_SERVER_URL) + throw new FatalError("MEDIA_SERVER_URL is not configured"); + const response = await fetch(`${env.MEDIA_SERVER_URL}/video/status`, { + headers: env.MEDIA_SERVER_WEBHOOK_SECRET + ? { "x-media-server-secret": env.MEDIA_SERVER_WEBHOOK_SECRET } + : undefined, + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) + throw new RetryableError( + "Could not verify Loom import worker capabilities", + { retryAfter: 60_000 }, + ); + const capabilities: unknown = await response.json(); + if ( + typeof capabilities !== "object" || + capabilities === null || + !("loomMultipartUploads" in capabilities) || + capabilities.loomMultipartUploads !== true || + !("loomImportMaxBytes" in capabilities) || + capabilities.loomImportMaxBytes !== LOOM_IMPORT_MAX_BYTES + ) { + throw new RetryableError( + "The media server must be updated for 10 GB Loom imports", + { retryAfter: 60_000 }, + ); + } +} + +requireLoomMediaServerSupport.maxRetries = 60; + async function claimAgentImport(operationId: string | undefined) { "use step"; @@ -160,6 +209,7 @@ export async function importLoomVideoWorkflow( ): Promise { "use workflow"; + const { workflowRunId } = getWorkflowMetadata(); if (!(await claimAgentImport(payload.agentOperationId))) { return { success: true, message: "Loom import is already running" }; } @@ -169,9 +219,11 @@ export async function importLoomVideoWorkflow( let processingInput: LoomProcessingInput = reuseExistingRawUpload ? {} : { importFromLoom: true }; + await requireLoomMediaServerSupport(); let metadata: ProcessedVideoMetadata; for (let processingAttempt = 0; ; processingAttempt++) { + let mediaJobId: string; let capacityRetryCount = 0; let previousSourceUrl = processingAttempt === 0 ? payload.loomDownloadUrl : undefined; @@ -184,7 +236,12 @@ export async function importLoomVideoWorkflow( ) : undefined; previousSourceUrl = loomSourceUrl; - await processVideoOnMediaServer(payload, loomSourceUrl); + mediaJobId = await processVideoOnMediaServer( + payload, + loomSourceUrl, + `${workflowRunId}:${processingAttempt}`, + workflowRunId, + ); break; } catch (error) { if (!isMediaServerCapacityError(error)) throw error; @@ -203,6 +260,11 @@ export async function importLoomVideoWorkflow( ) { throw error; } + if (!(await confirmLoomMediaJobFailure(mediaJobId, payload.videoId))) { + throw new FatalError( + "The previous Loom worker must be reconciled before retrying", + ); + } await markLoomImportWaitingForCapacity(payload.videoId); if (await hasExistingLoomUpload(payload)) processingInput = {}; await sleep(15_000 * (processingAttempt + 1)); @@ -242,6 +304,12 @@ function getInputExtension(url: string): string | undefined { return undefined; } +async function confirmLoomMediaJobFailure(jobId: string, videoId: string) { + "use step"; + + return isLoomMediaJobFailed(jobId, videoId); +} + async function hasExistingLoomUpload( payload: ImportLoomPayload, ): Promise { @@ -275,6 +343,8 @@ async function waitForRetry(delayMs: number): Promise { await new Promise((resolve) => setTimeout(resolve, delayMs)); } +class LoomMediaRequestRejectedError extends Error {} + async function startMediaServerProcessJob( mediaServerUrl: string, body: { @@ -282,6 +352,9 @@ async function startMediaServerProcessJob( userId: string; videoUrl: string; sourcePresignedUrl?: string; + sourceUpload?: LoomMultipartUploadTarget; + outputUpload?: LoomMultipartUploadTarget; + maxImportBytes: number; outputPresignedUrl: string; thumbnailPresignedUrl: string; previewGifPresignedUrl: string; @@ -290,6 +363,10 @@ async function startMediaServerProcessJob( inputExtension?: string; priority: "bulk"; }, + recordDispatch: ( + state: "accepted" | "rejected" | "uncertain", + jobId?: string, + ) => Promise, ): Promise { for (let attempt = 0; attempt < MEDIA_SERVER_START_MAX_ATTEMPTS; attempt++) { const headers: Record = { @@ -299,7 +376,9 @@ async function startMediaServerProcessJob( headers["x-media-server-secret"] = body.webhookSecret; } - const endpoint = body.sourcePresignedUrl ? "import" : "process"; + const endpoint = + body.sourcePresignedUrl || body.sourceUpload ? "import" : "process"; + await recordDispatch("uncertain"); const response = await fetch(`${mediaServerUrl}/video/${endpoint}`, { method: "POST", headers, @@ -308,8 +387,16 @@ async function startMediaServerProcessJob( if (response.ok) { const { jobId } = (await response.json()) as { jobId: string }; + await recordDispatch("accepted", jobId).catch((error) => { + console.error("Could not record accepted Loom media dispatch", error); + }); return jobId; } + if ( + (response.status >= 400 && response.status < 500) || + response.status === 503 + ) + await recordDispatch("rejected"); const errorData = (await response.json().catch(() => ({}))) as { error?: string; @@ -339,6 +426,8 @@ async function startMediaServerProcessJob( }); } + if (response.status >= 400 && response.status < 500) + throw new LoomMediaRequestRejectedError(errorMessage); throw new Error(errorMessage); } @@ -348,7 +437,9 @@ async function startMediaServerProcessJob( async function processVideoOnMediaServer( payload: ImportLoomPayload, loomSourceUrl: string | undefined, -): Promise { + attemptId: string, + runId: string, +): Promise { "use step"; const { videoId, userId, rawFileKey } = payload; @@ -367,6 +458,9 @@ async function processVideoOnMediaServer( outputPresignedUrl, thumbnailPresignedUrl, previewGifPresignedUrl, + sourceUpload, + outputUpload, + abortUploads, } = await Effect.gen(function* () { const [video] = yield* Effect.promise(() => db() @@ -377,6 +471,9 @@ async function processVideoOnMediaServer( if (!video) { return yield* Effect.fail(new FatalError("Video does not exist")); } + if (video.ownerId !== userId) { + return yield* Effect.fail(new FatalError("Loom import owner changed")); + } const videoDomain = Video.Video.decodeSync({ ...video, bucketId: video.bucket, @@ -426,45 +523,103 @@ async function processVideoOnMediaServer( }, { expiresIn: MEDIA_SERVER_PRESIGNED_PUT_EXPIRES_SECONDS }, ); + const multipart = yield* Effect.promise(() => + prepareLoomMultipartUploads({ + video, + bucket, + attemptId, + sourceKey: + loomSourceUrl && !isStreamingUrl(loomSourceUrl) + ? rawFileKey + : undefined, + webhookBaseUrl, + webhookSecret: serverEnv().MEDIA_SERVER_WEBHOOK_SECRET || undefined, + }), + ); return { rawVideoUrl, - sourcePresignedUrl, + sourcePresignedUrl: multipart.sourceUpload + ? undefined + : sourcePresignedUrl, outputPresignedUrl, thumbnailPresignedUrl, previewGifPresignedUrl, + ...multipart, + abortUploads: async () => { + await Promise.all( + Object.values(multipart) + .filter( + (upload): upload is LoomMultipartUploadTarget => + upload !== undefined, + ) + .map((upload) => + bucket.multipart.abort(upload.key, upload.uploadId).pipe( + Effect.catchAll(() => Effect.void), + runWorkflowPromise, + ), + ), + ); + }, }; }).pipe(runWorkflowPromise); const webhookUrl = `${webhookBaseUrl}/api/webhooks/media-server/progress?retryable=true`; const webhookSecret = serverEnv().MEDIA_SERVER_WEBHOOK_SECRET; - await db() - .update(videoUploads) - .set({ - phase: "processing", - processingProgress: 0, - processingMessage: "Starting video processing...", - processingError: null, - updatedAt: new Date(), - }) - .where(eq(videoUploads.videoId, videoId as Video.VideoId)); + try { + await db() + .update(videoUploads) + .set({ + phase: "processing", + processingProgress: 0, + processingMessage: "Starting video processing...", + processingError: null, + updatedAt: new Date(), + }) + .where(eq(videoUploads.videoId, videoId as Video.VideoId)); + } catch (error) { + await abortUploads(); + throw error; + } - await startMediaServerProcessJob(mediaServerUrl, { - videoId, - userId, - videoUrl: rawVideoUrl, - sourcePresignedUrl, - outputPresignedUrl, - thumbnailPresignedUrl, - previewGifPresignedUrl, - webhookUrl, - webhookSecret: webhookSecret || undefined, - inputExtension: getInputExtension(rawVideoUrl), - priority: "bulk", - }); + try { + return await startMediaServerProcessJob( + mediaServerUrl, + { + videoId, + userId, + videoUrl: rawVideoUrl, + sourcePresignedUrl, + sourceUpload, + outputUpload, + maxImportBytes: LOOM_IMPORT_MAX_BYTES, + outputPresignedUrl, + thumbnailPresignedUrl, + previewGifPresignedUrl, + webhookUrl, + webhookSecret: webhookSecret || undefined, + inputExtension: getInputExtension(rawVideoUrl), + priority: "bulk", + }, + (state, jobId) => + recordLoomMediaDispatch({ + videoId, + userId, + runId, + attemptId, + state, + jobId, + }), + ); + } catch (error) { + if (error instanceof LoomMediaRequestRejectedError) await abortUploads(); + throw error; + } } +processVideoOnMediaServer.maxRetries = 0; + async function saveMetadataAndComplete( videoId: string, metadata: { duration: number; width: number; height: number; fps: number }, diff --git a/packages/database/types/metadata.ts b/packages/database/types/metadata.ts index 1e70525fca9..882832b5ebf 100644 --- a/packages/database/types/metadata.ts +++ b/packages/database/types/metadata.ts @@ -9,6 +9,19 @@ export interface VideoMetadata { loomImportRun?: { runId: string; dispatch: "pending" | "accepted" | "rejected" | "uncertain"; + mediaDispatch?: { + attemptId: string; + videoId?: string; + state: "accepted" | "rejected" | "uncertain"; + jobId?: string; + }; + }; + loomImportUpload?: { + attemptId: string; + ownerId: string; + organizationId: string | null; + source?: { key: string; uploadId: string }; + output: { key: string; uploadId: string }; }; editProcessing?: { token: string; From f35867b90c02865a2a945b589152b65a40ef13df Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:35:08 +0100 Subject: [PATCH 14/21] fix: reconcile deleted recordings in migration inventory --- .../loom-concierge-database.test.ts | 79 ++++++++++ .../unit/loom-migration-inventory.test.ts | 138 ++++++++++++++++++ apps/web/actions/loom-concierge.tsx | 12 ++ .../loom-migrations/LoomMigrationQueue.tsx | 58 ++++++++ apps/web/lib/loom-migration-inventory.ts | 88 +++++++++++ apps/web/lib/loom-migration-state.ts | 2 +- 6 files changed, 376 insertions(+), 1 deletion(-) create mode 100644 apps/web/__tests__/unit/loom-migration-inventory.test.ts create mode 100644 apps/web/lib/loom-migration-inventory.ts diff --git a/apps/web/__tests__/integration/loom-concierge-database.test.ts b/apps/web/__tests__/integration/loom-concierge-database.test.ts index f6d347fcdc8..b1c9e54c133 100644 --- a/apps/web/__tests__/integration/loom-concierge-database.test.ts +++ b/apps/web/__tests__/integration/loom-concierge-database.test.ts @@ -256,6 +256,11 @@ describe.skipIf(!databaseUrl)("Loom concierge migration database", () => { (...args: Parameters) => rawActions.updateLoomMigrationStatus(...args), ), + reconcileLoomMigrationInventory: unwrapped( + ( + ...args: Parameters + ) => rawActions.reconcileLoomMigrationInventory(...args), + ), }; const loom = { importFromLoomCsvForConcierge: unwrapped( @@ -773,6 +778,80 @@ describe.skipIf(!databaseUrl)("Loom concierge migration database", () => { }, ); + it("reconciles only deleted inventory entries in the selected request", async () => { + const { ownerId, organizationId } = await makeOrganization(); + const requestId = await requestMigration(organizationId, ownerId); + const other = await makeOrganization(); + const otherRequestId = await requestMigration( + other.organizationId, + other.ownerId, + ); + const retainedVideoId = Video.VideoId.make(id()); + const deletedVideoId = Video.VideoId.make(id()); + const otherDeletedVideoId = Video.VideoId.make(id()); + await db() + .insert(videos) + .values({ + id: retainedVideoId, + name: "Retained", + ownerId, + orgId: organizationId, + }); + await db() + .insert(importedVideos) + .values({ + id: retainedVideoId, + orgId: organizationId, + source: "loom", + sourceId: loomId(), + }); + await db() + .insert(loomMigrationImports) + .values([ + { videoId: retainedVideoId, requestId }, + { videoId: deletedVideoId, requestId }, + { videoId: otherDeletedVideoId, requestId: otherRequestId }, + ]); + signInAs(ownerId); + await expect( + actions.reconcileLoomMigrationInventory({ requestId, verified: true }), + ).rejects.toThrow(); + signInAs(operatorId); + await expect( + actions.reconcileLoomMigrationInventory({ requestId, verified: false }), + ).rejects.toThrow("source list first"); + fixture.sendEmail.mockClear(); + await expect( + actions.reconcileLoomMigrationInventory({ requestId, verified: true }), + ).resolves.toEqual({ removedCount: 1, hasMore: false }); + expect( + await db() + .select({ videoId: loomMigrationImports.videoId }) + .from(loomMigrationImports) + .where(eq(loomMigrationImports.requestId, requestId)), + ).toEqual([{ videoId: retainedVideoId }]); + expect( + await db() + .select({ videoId: loomMigrationImports.videoId }) + .from(loomMigrationImports) + .where(eq(loomMigrationImports.requestId, otherRequestId)), + ).toEqual([{ videoId: otherDeletedVideoId }]); + expect( + await db() + .select({ id: videos.id }) + .from(videos) + .where(eq(videos.id, retainedVideoId)), + ).toHaveLength(1); + expect( + await db() + .select({ id: importedVideos.id }) + .from(importedVideos) + .where(eq(importedVideos.id, retainedVideoId)), + ).toHaveLength(1); + expect((await getRequest(requestId)).status).not.toBe("completed"); + expect(fixture.sendEmail).not.toHaveBeenCalled(); + }); + it("uploads restricted MP4 files to storage and starts processing", async () => { const { ownerId, organizationId } = await makeOrganization({ privateByDefault: true, diff --git a/apps/web/__tests__/unit/loom-migration-inventory.test.ts b/apps/web/__tests__/unit/loom-migration-inventory.test.ts new file mode 100644 index 00000000000..8a7fcc76444 --- /dev/null +++ b/apps/web/__tests__/unit/loom-migration-inventory.test.ts @@ -0,0 +1,138 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + operator: vi.fn(), + transaction: vi.fn(), + rows: [] as unknown[][], + select: vi.fn(), + delete: vi.fn(), + deleteWhere: vi.fn(), + update: vi.fn(), + set: vi.fn(), +})); + +vi.mock("@cap/database", () => ({ + db: () => ({ transaction: mocks.transaction }), +})); +vi.mock("@cap/database/schema", () => ({ + loomMigrationRequests: { id: "request.id" }, + loomMigrationImports: { + requestId: "inventory.requestId", + videoId: "inventory.videoId", + }, + videos: { id: "video.id" }, +})); +vi.mock("drizzle-orm", () => ({ + and: (...conditions: unknown[]) => ({ and: conditions }), + eq: (left: unknown, right: unknown) => ({ eq: [left, right] }), + inArray: (column: unknown, values: unknown[]) => ({ + inArray: [column, values], + }), + isNull: (column: unknown) => ({ isNull: column }), + notExists: (query: unknown) => ({ notExists: query }), +})); +vi.mock("@/lib/loom-concierge", () => ({ + requireMigrationOperator: mocks.operator, +})); + +import { reconcileMissingLoomMigrationImports } from "@/lib/loom-migration-inventory"; + +const input = { requestId: "request12345678", verified: true }; +const request = { + status: "in_progress", + organizationId: "destination", + activeOrganizationId: "destination", + activeImportCount: 0, +}; + +describe("operator migration inventory reconciliation", () => { + beforeEach(() => { + vi.resetAllMocks(); + mocks.rows = [[request], []]; + mocks.operator.mockResolvedValue({ id: "operator" }); + mocks.select.mockImplementation(() => { + const builder = { + from: vi.fn().mockReturnThis(), + leftJoin: vi.fn().mockReturnThis(), + where: vi.fn().mockReturnThis(), + limit: vi.fn().mockReturnThis(), + for: vi.fn(async () => mocks.rows.shift() ?? []), + }; + return builder; + }); + mocks.delete.mockReturnValue({ where: mocks.deleteWhere }); + mocks.deleteWhere.mockResolvedValue([{ affectedRows: 1 }]); + mocks.set.mockReturnValue({ where: vi.fn().mockResolvedValue(undefined) }); + mocks.update.mockReturnValue({ set: mocks.set }); + mocks.transaction.mockImplementation(async (fn: (tx: unknown) => unknown) => + fn({ select: mocks.select, delete: mocks.delete, update: mocks.update }), + ); + }); + + it("requires operator access before reading migration state", async () => { + mocks.operator.mockRejectedValue(new Error("Forbidden")); + await expect(reconcileMissingLoomMigrationImports(input)).rejects.toThrow( + "Forbidden", + ); + expect(mocks.transaction).not.toHaveBeenCalled(); + }); + + it("requires an explicit source-list reconciliation confirmation", async () => { + await expect( + reconcileMissingLoomMigrationImports({ ...input, verified: false }), + ).rejects.toThrow("source list first"); + expect(mocks.transaction).not.toHaveBeenCalled(); + }); + + it.each([ + undefined, + { ...request, status: "completed" }, + { ...request, activeOrganizationId: null }, + { ...request, activeOrganizationId: "other" }, + { ...request, activeImportCount: 1 }, + ])("holds inactive requests and concurrent imports: %j", async (row) => { + mocks.rows = [row ? [row] : []]; + await expect(reconcileMissingLoomMigrationImports(input)).rejects.toThrow( + "inactive or imports", + ); + expect(mocks.delete).not.toHaveBeenCalled(); + }); + + it("does not mutate requests without missing recordings", async () => { + await expect(reconcileMissingLoomMigrationImports(input)).resolves.toEqual({ + removedCount: 0, + hasMore: false, + }); + expect(mocks.delete).not.toHaveBeenCalled(); + expect(mocks.update).not.toHaveBeenCalled(); + }); + + it("bounds cleanup, rechecks absent videos and never changes completion or counts", async () => { + mocks.rows = [ + [request], + Array.from({ length: 101 }, (_, i) => ({ videoId: `deleted-${i}` })), + ]; + mocks.deleteWhere.mockResolvedValue([{ affectedRows: 100 }]); + await expect(reconcileMissingLoomMigrationImports(input)).resolves.toEqual({ + removedCount: 100, + hasMore: true, + }); + expect(mocks.delete).toHaveBeenCalledWith({ + requestId: "inventory.requestId", + videoId: "inventory.videoId", + }); + const conditions = mocks.deleteWhere.mock.calls[0]?.[0].and; + expect(conditions[0]).toEqual({ + eq: ["inventory.requestId", input.requestId], + }); + expect(conditions[1].inArray[1]).toHaveLength(100); + expect(conditions[2].notExists.where).toHaveBeenCalledWith({ + eq: ["video.id", "inventory.videoId"], + }); + expect(mocks.set).toHaveBeenCalledWith({ + lastOperatorUserId: "operator", + lastOperatorAt: expect.any(Date), + updatedAt: expect.any(Date), + }); + }); +}); diff --git a/apps/web/actions/loom-concierge.tsx b/apps/web/actions/loom-concierge.tsx index 2a51b906579..6daa9d42628 100644 --- a/apps/web/actions/loom-concierge.tsx +++ b/apps/web/actions/loom-concierge.tsx @@ -28,6 +28,7 @@ import { requireProCustomerMigrationAccess, toLoomMigrationResult, } from "@/lib/loom-concierge"; +import { reconcileMissingLoomMigrationImports } from "@/lib/loom-migration-inventory"; import { LOOM_MIGRATION_STATUS_LABELS, LoomMigrationError, @@ -249,6 +250,17 @@ export async function updateLoomMigrationStatus( return toLoomMigrationResult(() => saveMigrationStatus(input)); } +export async function reconcileLoomMigrationInventory(input: { + requestId: string; + verified: boolean; +}) { + return toLoomMigrationResult(async () => { + const result = await reconcileMissingLoomMigrationImports(input); + refreshMigrationPages(); + return result; + }); +} + async function saveMigrationStatus({ requestId, nextStatus, diff --git a/apps/web/app/(org)/dashboard/admin/loom-migrations/LoomMigrationQueue.tsx b/apps/web/app/(org)/dashboard/admin/loom-migrations/LoomMigrationQueue.tsx index f9d860b04d6..7bd17c0c494 100644 --- a/apps/web/app/(org)/dashboard/admin/loom-migrations/LoomMigrationQueue.tsx +++ b/apps/web/app/(org)/dashboard/admin/loom-migrations/LoomMigrationQueue.tsx @@ -14,6 +14,7 @@ import { } from "@/actions/loom"; import { getLoomMigrationOperatorQueue, + reconcileLoomMigrationInventory, updateLoomMigrationStatus, } from "@/actions/loom-concierge"; import type { OperatorLoomMigrationView } from "@/lib/loom-concierge"; @@ -45,6 +46,8 @@ function QueueItem({ request }: { request: OperatorLoomMigrationView }) { request.importedVideoCount.toString(), ); const [verified, setVerified] = useState(false); + const [deletedRecordingsVerified, setDeletedRecordingsVerified] = + useState(false); const [csvRows, setCsvRows] = useState([]); const [processedRows, setProcessedRows] = useState(0); const [startedRows, setStartedRows] = useState(0); @@ -62,6 +65,7 @@ function QueueItem({ request }: { request: OperatorLoomMigrationView }) { setExpectedVideoCount(request.expectedVideoCount?.toString() ?? ""); setImportedVideoCount(request.importedVideoCount.toString()); setVerified(false); + setDeletedRecordingsVerified(false); }, [ request.status, request.capMessage, @@ -92,6 +96,24 @@ function QueueItem({ request }: { request: OperatorLoomMigrationView }) { }, onError: (error) => toast.error(error.message), }); + const reconcileMutation = useMutation({ + mutationFn: async () => + unwrapLoomMigrationResult( + await reconcileLoomMigrationInventory({ + requestId: request.id, + verified: deletedRecordingsVerified, + }), + ), + onSuccess: async ({ removedCount, hasMore }) => { + setVerified(false); + setDeletedRecordingsVerified(false); + await refresh(); + toast.success( + `Removed ${removedCount} deleted recording entries. ${hasMore ? "More entries remain; reconcile again." : "Check source counts before completing the migration."}`, + ); + }, + onError: (error) => toast.error(error.message), + }); const importMutation = useMutation({ mutationFn: async () => { if (csvRows.length === 0) throw new Error("Choose a Loom CSV first."); @@ -488,6 +510,7 @@ function QueueItem({ request }: { request: OperatorLoomMigrationView }) { +
+

+ Remove inventory entries for recordings that no longer exist. Existing + recordings and source mappings are kept. This does not complete the + migration. +

+ + +
); } diff --git a/apps/web/lib/loom-migration-inventory.ts b/apps/web/lib/loom-migration-inventory.ts new file mode 100644 index 00000000000..affbd620fb5 --- /dev/null +++ b/apps/web/lib/loom-migration-inventory.ts @@ -0,0 +1,88 @@ +import { db } from "@cap/database"; +import { + loomMigrationImports, + loomMigrationRequests, + videos, +} from "@cap/database/schema"; +import { and, eq, inArray, isNull, notExists } from "drizzle-orm"; +import { requireMigrationOperator } from "@/lib/loom-concierge"; +import { LoomMigrationError } from "@/lib/loom-migration-state"; + +const RECONCILE_BATCH_SIZE = 100; + +export async function reconcileMissingLoomMigrationImports({ + requestId, + verified, +}: { + requestId: string; + verified: boolean; +}) { + const operator = await requireMigrationOperator(); + if (typeof requestId !== "string" || !/^[A-Za-z0-9_-]{15}$/.test(requestId)) + throw new LoomMigrationError("Invalid migration request."); + if (verified !== true) + throw new LoomMigrationError( + "Reconcile deleted recordings against the source list first.", + ); + return db().transaction(async (tx) => { + const [request] = await tx + .select() + .from(loomMigrationRequests) + .where(eq(loomMigrationRequests.id, requestId)) + .limit(1) + .for("update"); + if ( + !request || + request.status === "completed" || + request.activeOrganizationId !== request.organizationId || + request.activeImportCount > 0 + ) { + throw new LoomMigrationError( + "The migration is inactive or imports are starting. Refresh and try again.", + ); + } + const missing = await tx + .select({ videoId: loomMigrationImports.videoId }) + .from(loomMigrationImports) + .leftJoin(videos, eq(videos.id, loomMigrationImports.videoId)) + .where( + and(eq(loomMigrationImports.requestId, requestId), isNull(videos.id)), + ) + .limit(RECONCILE_BATCH_SIZE + 1) + .for("update"); + if (missing.length === 0) return { removedCount: 0, hasMore: false }; + const result = await tx.delete(loomMigrationImports).where( + and( + eq(loomMigrationImports.requestId, requestId), + inArray( + loomMigrationImports.videoId, + missing.slice(0, RECONCILE_BATCH_SIZE).map((row) => row.videoId), + ), + notExists( + tx + .select({ id: videos.id }) + .from(videos) + .where(eq(videos.id, loomMigrationImports.videoId)), + ), + ), + ); + const removedCount = (Array.isArray(result) ? result[0] : result) as { + affectedRows?: number; + }; + if (typeof removedCount.affectedRows !== "number") + throw new Error("Could not verify inventory reconciliation"); + const now = new Date(); + await tx + .update(loomMigrationRequests) + .set({ + lastOperatorUserId: operator.id, + lastOperatorAt: now, + updatedAt: now, + }) + .where(eq(loomMigrationRequests.id, requestId)); + return { + removedCount: removedCount.affectedRows, + hasMore: missing.length > RECONCILE_BATCH_SIZE, + }; + }); +} diff --git a/apps/web/lib/loom-migration-state.ts b/apps/web/lib/loom-migration-state.ts index c92ac7630df..34c29adb1a7 100644 --- a/apps/web/lib/loom-migration-state.ts +++ b/apps/web/lib/loom-migration-state.ts @@ -69,7 +69,7 @@ export function validateLoomMigrationInventory({ ) ) { throw new LoomMigrationError( - "Loom import recordings or source mappings are missing or changed. Reconcile the destination library before completion.", + "Loom import recordings or source mappings are missing or changed. Re-import missing sources, then use Reconcile deleted recordings for deleted inventory entries. Restore changed mappings before completion.", ); } if (imports.some((row) => row.phase !== null && row.phase !== "complete")) { From e65db5eeb7322b88b59f4d11dcb6ac42111665af Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:37:05 +0100 Subject: [PATCH 15/21] fix: preserve cancellation in legacy processing uploads --- apps/media-server/src/routes/video.ts | 41 +++++++++++++++++---------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/apps/media-server/src/routes/video.ts b/apps/media-server/src/routes/video.ts index e635ae91a06..11bebdc6a52 100644 --- a/apps/media-server/src/routes/video.ts +++ b/apps/media-server/src/routes/video.ts @@ -1365,15 +1365,19 @@ async function processVideoAsync( updateJob(jobId, { message: "Saving original video..." }); await sendWebhook(job); await withJobHeartbeat(jobId, () => - uploadFileToStorage( - inputTempFile.path, - options.sourceUpload ?? { - type: "put", - url: sourcePresignedUrl as string, - }, - "video/mp4", - abortController.signal, - ), + options.sourceUpload + ? uploadFileToStorage( + inputTempFile.path, + options.sourceUpload, + "video/mp4", + abortController.signal, + ) + : uploadFileToS3( + inputTempFile.path, + sourcePresignedUrl as string, + "video/mp4", + abortController.signal, + ), ); sourceUploaded = true; } @@ -1440,12 +1444,19 @@ async function processVideoAsync( }); await sendWebhook(job); - const uploadReceipt = await uploadFileToStorage( - outputTempFile.path, - options.outputUpload ?? { type: "put", url: outputPresignedUrl }, - "video/mp4", - abortController.signal, - ); + const uploadReceipt = options.outputUpload + ? await uploadFileToStorage( + outputTempFile.path, + options.outputUpload, + "video/mp4", + abortController.signal, + ) + : await uploadFileToS3( + outputTempFile.path, + outputPresignedUrl, + "video/mp4", + abortController.signal, + ); outputUploaded = true; if (thumbnailPresignedUrl || previewGifPresignedUrl) { From aa3eab10b28ea6201b8c4ef5b43768cab99c2997 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:41:34 +0100 Subject: [PATCH 16/21] style: format migration inventory database fixture --- .../loom-concierge-database.test.ts | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/apps/web/__tests__/integration/loom-concierge-database.test.ts b/apps/web/__tests__/integration/loom-concierge-database.test.ts index b1c9e54c133..2604688e7e0 100644 --- a/apps/web/__tests__/integration/loom-concierge-database.test.ts +++ b/apps/web/__tests__/integration/loom-concierge-database.test.ts @@ -789,22 +789,18 @@ describe.skipIf(!databaseUrl)("Loom concierge migration database", () => { const retainedVideoId = Video.VideoId.make(id()); const deletedVideoId = Video.VideoId.make(id()); const otherDeletedVideoId = Video.VideoId.make(id()); - await db() - .insert(videos) - .values({ - id: retainedVideoId, - name: "Retained", - ownerId, - orgId: organizationId, - }); - await db() - .insert(importedVideos) - .values({ - id: retainedVideoId, - orgId: organizationId, - source: "loom", - sourceId: loomId(), - }); + await db().insert(videos).values({ + id: retainedVideoId, + name: "Retained", + ownerId, + orgId: organizationId, + }); + await db().insert(importedVideos).values({ + id: retainedVideoId, + orgId: organizationId, + source: "loom", + sourceId: loomId(), + }); await db() .insert(loomMigrationImports) .values([ From f3c645b90ec8ffde6cd047d77bc067a8d6cb66da Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:50:02 +0100 Subject: [PATCH 17/21] fix: honor expired leases during migration reconciliation --- .../unit/loom-migration-inventory.test.ts | 23 +++++++++++++++++++ apps/web/lib/loom-migration-inventory.ts | 4 +++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/apps/web/__tests__/unit/loom-migration-inventory.test.ts b/apps/web/__tests__/unit/loom-migration-inventory.test.ts index 8a7fcc76444..0208fdd6c99 100644 --- a/apps/web/__tests__/unit/loom-migration-inventory.test.ts +++ b/apps/web/__tests__/unit/loom-migration-inventory.test.ts @@ -90,6 +90,11 @@ describe("operator migration inventory reconciliation", () => { { ...request, activeOrganizationId: null }, { ...request, activeOrganizationId: "other" }, { ...request, activeImportCount: 1 }, + { + ...request, + activeImportCount: 1, + activeImportLeaseUntil: new Date(Date.now() + 60_000), + }, ])("holds inactive requests and concurrent imports: %j", async (row) => { mocks.rows = [row ? [row] : []]; await expect(reconcileMissingLoomMigrationImports(input)).rejects.toThrow( @@ -107,6 +112,24 @@ describe("operator migration inventory reconciliation", () => { expect(mocks.update).not.toHaveBeenCalled(); }); + it("permits inventory-only reconciliation after an import reservation expires", async () => { + mocks.rows = [ + [ + { + ...request, + activeImportCount: 1, + activeImportLeaseUntil: new Date(0), + }, + ], + [], + ]; + await expect(reconcileMissingLoomMigrationImports(input)).resolves.toEqual({ + removedCount: 0, + hasMore: false, + }); + expect(mocks.delete).not.toHaveBeenCalled(); + }); + it("bounds cleanup, rechecks absent videos and never changes completion or counts", async () => { mocks.rows = [ [request], diff --git a/apps/web/lib/loom-migration-inventory.ts b/apps/web/lib/loom-migration-inventory.ts index affbd620fb5..6c335a4c7ae 100644 --- a/apps/web/lib/loom-migration-inventory.ts +++ b/apps/web/lib/loom-migration-inventory.ts @@ -35,7 +35,9 @@ export async function reconcileMissingLoomMigrationImports({ !request || request.status === "completed" || request.activeOrganizationId !== request.organizationId || - request.activeImportCount > 0 + (request.activeImportCount > 0 && + (!request.activeImportLeaseUntil || + request.activeImportLeaseUntil > new Date())) ) { throw new LoomMigrationError( "The migration is inactive or imports are starting. Refresh and try again.", From ee021b3fa23f697f46fc36695dd9b605e85f5d1a Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:50:33 +0100 Subject: [PATCH 18/21] fix: verify and reuse saved Loom outputs during recovery --- ...edia-routes-real-world.integration.test.ts | 75 ++++++++++ apps/media-server/src/routes/video.ts | 132 +++++++++++++----- .../__tests__/unit/loom-import-upload.test.ts | 44 ++++++ .../unit/media-processing-workflows.test.ts | 24 ++++ .../unit/media-server-multipart.test.ts | 19 +++ .../media-server/multipart/[action]/route.ts | 4 + apps/web/lib/loom-import-upload.ts | 31 ++++ apps/web/workflows/import-loom-video.ts | 52 +++++-- 8 files changed, 330 insertions(+), 51 deletions(-) diff --git a/apps/media-server/src/__tests__/lib/media-routes-real-world.integration.test.ts b/apps/media-server/src/__tests__/lib/media-routes-real-world.integration.test.ts index 1c7a4cc54eb..3c5c3aa437e 100644 --- a/apps/media-server/src/__tests__/lib/media-routes-real-world.integration.test.ts +++ b/apps/media-server/src/__tests__/lib/media-routes-real-world.integration.test.ts @@ -1163,6 +1163,81 @@ describe("media routes real-world integration tests", () => { 30_000, ); + test("recovers a saved output after preview failure without overwriting it", async () => { + const thumbnail = spyOn(mediaVideo, "generateThumbnail").mockRejectedValue( + new Error("Thumbnail decode interrupted"), + ); + const jobs: string[] = []; + try { + const body = { + videoId: "saved-loom-output", + userId: "real-process-user", + videoUrl: fixtureUrl(), + outputPresignedUrl: uploadUrl("saved-output.mp4"), + thumbnailPresignedUrl: uploadUrl("saved-output.jpg"), + inputExtension: ".mp4", + maxImportBytes: 10 * 1024 ** 3, + }; + const first = await app.fetch(mediaPostRequest("/video/process", body)); + const firstJobId = ((await first.json()) as { jobId: string }).jobId; + jobs.push(firstJobId); + expect((await waitForTerminalJob(firstJobId)).phase).toBe("error"); + const original = Uint8Array.from( + uploadedBytes("/uploads/saved-output.mp4"), + ); + const second = await app.fetch( + mediaPostRequest("/video/process", { + ...body, + existingOutput: { + url: uploadUrl("saved-output.mp4"), + objectIdentity: objectIdentity(original), + fileSize: original.byteLength, + }, + }), + ); + expect(second.status).toBe(200); + const secondJobId = ((await second.json()) as { jobId: string }).jobId; + jobs.push(secondJobId); + expect((await waitForTerminalJob(secondJobId)).phase).toBe("complete"); + expect(uploadRequests.filter((path) => path.endsWith(".mp4"))).toEqual([ + "/uploads/saved-output.mp4", + ]); + expect(uploadedBytes("/uploads/saved-output.mp4")).toEqual(original); + } finally { + thumbnail.mockRestore(); + for (const jobId of jobs) deleteJob(jobId); + } + }, 30_000); + + test("holds a saved output whose storage identity changed without overwriting it", async () => { + const original = new Uint8Array( + await Bun.file(TEST_VIDEO_WITH_AUDIO).arrayBuffer(), + ); + uploadedArtifacts.set("/uploads/changed-output.mp4", original); + const response = await app.fetch( + mediaPostRequest("/video/process", { + videoId: "changed-output", + userId: "real-process-user", + videoUrl: fixtureUrl(), + outputPresignedUrl: uploadUrl("changed-output.mp4"), + maxImportBytes: 10 * 1024 ** 3, + existingOutput: { + url: uploadUrl("changed-output.mp4"), + objectIdentity: '"previous-identity"', + fileSize: original.byteLength, + }, + }), + ); + const { jobId } = (await response.json()) as { jobId: string }; + try { + expect((await waitForTerminalJob(jobId)).phase).toBe("error"); + expect(uploadRequests).toHaveLength(0); + expect(uploadedBytes("/uploads/changed-output.mp4")).toEqual(original); + } finally { + deleteJob(jobId); + } + }, 30_000); + test("still fails processing when the video output cannot be uploaded", async () => { uploadFailures.set("/uploads/failed-output.mp4", 403); const response = await app.fetch( diff --git a/apps/media-server/src/routes/video.ts b/apps/media-server/src/routes/video.ts index 11bebdc6a52..b56e124e308 100644 --- a/apps/media-server/src/routes/video.ts +++ b/apps/media-server/src/routes/video.ts @@ -66,6 +66,7 @@ import { RecordingTimingError } from "../lib/recording-timing"; import { hashRecordingFile, isRetryableRecordingVerificationError, + verifyRecording, verifyRemoteRecording, verifyRemoteRecordingBytes, verifyRemuxedRecording, @@ -146,6 +147,13 @@ const processSchema = z.object({ sourcePresignedUrl: z.string().url().optional(), sourceUpload: storageUploadSchema.optional(), outputUpload: storageUploadSchema.optional(), + existingOutput: z + .object({ + url: z.string().url(), + objectIdentity: z.string().regex(/^"[^"\r\n]+"$/), + fileSize: z.number().int().positive().max(LOOM_IMPORT_MAX_BYTES), + }) + .optional(), maxImportBytes: z.literal(LOOM_IMPORT_MAX_BYTES).optional(), thumbnailPresignedUrl: z.string().url().optional(), previewGifPresignedUrl: z.string().url().optional(), @@ -725,6 +733,21 @@ video.on("POST", ["/process", "/import"], async (c) => { ); } + if ( + result.data.existingOutput && + (result.data.sourceUpload || + result.data.sourcePresignedUrl || + result.data.outputUpload || + result.data.maxImportBytes !== LOOM_IMPORT_MAX_BYTES) + ) { + return c.json( + { + error: "Existing output cannot be combined with upload targets", + code: "INVALID_REQUEST", + }, + 400, + ); + } const capacity = getVideoCapacitySnapshot(); const bulkCapacityReached = result.data.priority === "bulk" && @@ -1352,13 +1375,32 @@ async function processVideoAsync( const inputTempFile = await withJobHeartbeat(jobId, () => downloadVideoToTemp( - videoUrl, + options.existingOutput?.url ?? videoUrl, options.inputExtension, abortController.signal, options.maxImportBytes, ), ); updateJob(jobId, { inputTempFile }); + if (options.existingOutput) { + const existing = options.existingOutput; + await withJobHeartbeat(jobId, async () => { + await verifyRecording(inputTempFile.path, { + requireAudio: false, + allowObservedDuration: true, + abortSignal: abortController.signal, + }); + await verifyRemoteRecordingBytes(existing.url, { + expectedObjectIdentity: existing.objectIdentity, + expectedFileSize: existing.fileSize, + expectedSha256: await hashRecordingFile( + inputTempFile.path, + abortController.signal, + ), + abortSignal: abortController.signal, + }); + }); + } const sourcePresignedUrl = options.sourcePresignedUrl; if (sourcePresignedUrl || options.sourceUpload) { @@ -1397,11 +1439,16 @@ async function processVideoAsync( }); await sendWebhook(job); - const { metadata, repairedFile } = await probeWithRepairFallback( - inputTempFile.path, - isWebm, - abortController.signal, - ); + const { metadata, repairedFile } = options.existingOutput + ? { + metadata: await probeVideoFile(inputTempFile.path), + repairedFile: null, + } + : await probeWithRepairFallback( + inputTempFile.path, + isWebm, + abortController.signal, + ); repairedTempFile = repairedFile; updateJob(jobId, { metadata }); @@ -1419,15 +1466,17 @@ async function processVideoAsync( await sendWebhook(job); const { outputFile: outputTempFile, lastResortRepairFile: lrrf } = - await processWithResilientRetry( - processingInputPath, - inputTempFile.path, - metadata, - options, - isWebm, - jobId, - abortController.signal, - ); + options.existingOutput + ? { outputFile: inputTempFile, lastResortRepairFile: null } + : await processWithResilientRetry( + processingInputPath, + inputTempFile.path, + metadata, + options, + isWebm, + jobId, + abortController.signal, + ); lastResortRepairFile = lrrf; updateJob(jobId, { outputTempFile }); if ( @@ -1444,19 +1493,21 @@ async function processVideoAsync( }); await sendWebhook(job); - const uploadReceipt = options.outputUpload - ? await uploadFileToStorage( - outputTempFile.path, - options.outputUpload, - "video/mp4", - abortController.signal, - ) - : await uploadFileToS3( - outputTempFile.path, - outputPresignedUrl, - "video/mp4", - abortController.signal, - ); + const uploadReceipt = options.existingOutput + ? { objectIdentity: options.existingOutput.objectIdentity } + : options.outputUpload + ? await uploadFileToStorage( + outputTempFile.path, + options.outputUpload, + "video/mp4", + abortController.signal, + ) + : await uploadFileToS3( + outputTempFile.path, + outputPresignedUrl, + "video/mp4", + abortController.signal, + ); outputUploaded = true; if (thumbnailPresignedUrl || previewGifPresignedUrl) { @@ -1469,20 +1520,29 @@ async function processVideoAsync( } if (thumbnailPresignedUrl) { - // The uploaded MP4 has not been independently verified, so decode failures must still fail the job. + // Only the fully verified reuse path can finish without a decoded preview. const thumbnailData = await generateThumbnail( outputTempFile.path, metadata.duration, {}, abortController.signal, - ); - try { - await uploadToS3( - thumbnailData, - thumbnailPresignedUrl, - "image/jpeg", - abortController.signal, + ).catch((error) => { + abortController.signal.throwIfAborted(); + if (!options.existingOutput) throw error; + console.warn( + "Verified Loom output preview could not be generated", + error, ); + return undefined; + }); + try { + if (thumbnailData) + await uploadToS3( + thumbnailData, + thumbnailPresignedUrl, + "image/jpeg", + abortController.signal, + ); } catch (error) { abortController.signal.throwIfAborted(); console.warn( diff --git a/apps/web/__tests__/unit/loom-import-upload.test.ts b/apps/web/__tests__/unit/loom-import-upload.test.ts index 8d83f304fe4..40539377759 100644 --- a/apps/web/__tests__/unit/loom-import-upload.test.ts +++ b/apps/web/__tests__/unit/loom-import-upload.test.ts @@ -6,6 +6,8 @@ const mocks = vi.hoisted(() => ({ where: vi.fn(), create: vi.fn(), abort: vi.fn(), + head: vi.fn(), + get: vi.fn(), })); vi.mock("@cap/database", () => ({ @@ -44,6 +46,8 @@ function input(): Input { video, bucket: { provider: "s3", + headObject: mocks.head, + getInternalSignedObjectUrl: mocks.get, multipart: { create: mocks.create, abort: mocks.abort }, } as unknown as Input["bucket"], attemptId: "run:0", @@ -62,6 +66,10 @@ describe("Loom multipart upload preparation", () => { Effect.succeed({ UploadId: `upload:${key}` }), ); mocks.abort.mockReturnValue(Effect.void); + mocks.head.mockReturnValue(Effect.fail({ name: "NotFound" })); + mocks.get.mockReturnValue( + Effect.succeed("https://storage.example.com/result.mp4"), + ); }); it("prepares original and output parts for files up to 10 GB", async () => { @@ -108,6 +116,42 @@ describe("Loom multipart upload preparation", () => { expect(mocks.where).not.toHaveBeenCalled(); }); + it("reuses an identity-bound saved output without reserving or overwriting objects", async () => { + const args = input(); + args.video = { + ...video, + metadata: { + ...video.metadata, + loomImportUpload: { + attemptId: "run:0", + ownerId: "owner", + organizationId: "organization", + output: { + key: "owner/video/result.mp4", + uploadId: "completed-upload", + }, + }, + }, + }; + mocks.head.mockReturnValue( + Effect.succeed({ ContentLength: 4096, ETag: '"saved-output"' }), + ); + const result = await prepareLoomMultipartUploads({ + ...args, + attemptId: "run:1", + }); + expect(result).toEqual({ + existingOutput: { + url: "https://storage.example.com/result.mp4", + objectIdentity: '"saved-output"', + fileSize: 4096, + }, + }); + expect(mocks.create).not.toHaveBeenCalled(); + expect(mocks.abort).not.toHaveBeenCalled(); + expect(mocks.where).not.toHaveBeenCalled(); + }); + it("aborts only newly reserved uploads when recording ownership changes", async () => { mocks.where.mockResolvedValue([{ affectedRows: 0 }]); await expect(prepareLoomMultipartUploads(input())).rejects.toThrow( diff --git a/apps/web/__tests__/unit/media-processing-workflows.test.ts b/apps/web/__tests__/unit/media-processing-workflows.test.ts index 2766fc032a2..5e8fb252bb5 100644 --- a/apps/web/__tests__/unit/media-processing-workflows.test.ts +++ b/apps/web/__tests__/unit/media-processing-workflows.test.ts @@ -437,6 +437,30 @@ describe("media processing workflows", () => { ).toHaveLength(1); }); + it.each([ + [503, { error: "Upstream response unavailable" }], + [200, {}], + ])( + "holds an ambiguous HTTP %s worker response without redispatching", + async (status, body) => { + mocks.rows = [[video]]; + mocks.fetch.mockImplementation(async (url: string) => + url.startsWith("https://www.loom.com/") + ? Response.json({ url: "https://cdn.loom.com/original.mp4" }) + : Response.json(body, { status }), + ); + await expect(importLoomVideoWorkflow(payload)).rejects.toThrow(); + expect(mocks.dispatch).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ state: "uncertain" }), + ); + expect( + mocks.fetch.mock.calls.filter(([url]) => + url.startsWith("https://worker.example.com/"), + ), + ).toHaveLength(1); + }, + ); + it("supports Loom streaming fallbacks without reading a nonexistent raw object", async () => { mocks.fetch.mockImplementation(async (url: string) => Response.json( diff --git a/apps/web/__tests__/unit/media-server-multipart.test.ts b/apps/web/__tests__/unit/media-server-multipart.test.ts index 9aa4cd46e5f..add20925a5c 100644 --- a/apps/web/__tests__/unit/media-server-multipart.test.ts +++ b/apps/web/__tests__/unit/media-server-multipart.test.ts @@ -438,6 +438,25 @@ describe("media-server multipart fencing", () => { expect(mocks.complete).not.toHaveBeenCalled(); }); + it("blocks callbacks after a replacement workflow takes ownership", async () => { + const { body, row } = loomUpload(); + mocks.rows = [ + { + ...row, + metadata: { + ...row.metadata, + loomImportRun: { runId: "replacement-run", dispatch: "accepted" }, + }, + }, + ]; + expect((await request("sign-part", body)).status).toBe(409); + expect((await request("complete", body)).status).toBe(409); + expect((await request("abort", body)).status).toBe(409); + expect(mocks.signPart).not.toHaveBeenCalled(); + expect(mocks.complete).not.toHaveBeenCalled(); + expect(mocks.abort).not.toHaveBeenCalled(); + }); + it("accepts exactly 10 GB and rejects a further multipart part", async () => { const { body } = loomUpload(); const parts = Array.from({ length: 160 }, (_, index) => ({ diff --git a/apps/web/app/api/webhooks/media-server/multipart/[action]/route.ts b/apps/web/app/api/webhooks/media-server/multipart/[action]/route.ts index 91d4f0bf21a..cf5de3f0d46 100644 --- a/apps/web/app/api/webhooks/media-server/multipart/[action]/route.ts +++ b/apps/web/app/api/webhooks/media-server/multipart/[action]/route.ts @@ -98,6 +98,10 @@ async function getMultipartContext( fenced || !receipt || receipt.attemptId !== payload.loomImportAttemptId || + (video.metadata?.loomImportRun && + !receipt.attemptId.startsWith( + `${video.metadata.loomImportRun.runId}:`, + )) || receipt.ownerId !== video.ownerId || receipt.organizationId !== video.orgId || !upload || diff --git a/apps/web/lib/loom-import-upload.ts b/apps/web/lib/loom-import-upload.ts index fe06888caf2..a1330e0ee72 100644 --- a/apps/web/lib/loom-import-upload.ts +++ b/apps/web/lib/loom-import-upload.ts @@ -4,6 +4,7 @@ import type { VideoMetadata } from "@cap/database/types"; import type { Storage } from "@cap/web-backend/src/Storage/index"; import { and, eq, isNull, sql } from "drizzle-orm"; import { Effect } from "effect"; +import { isStorageObjectNotFound } from "@/lib/storage-not-found"; import { runWorkflowPromise } from "@/lib/workflow-runtime"; export const LOOM_IMPORT_MAX_BYTES = 10 * 1024 ** 3; @@ -44,6 +45,7 @@ export async function prepareLoomMultipartUploads({ }): Promise<{ sourceUpload?: LoomMultipartUploadTarget; outputUpload?: LoomMultipartUploadTarget; + existingOutput?: { url: string; objectIdentity: string; fileSize: number }; }> { if (bucket.provider !== "s3") return {}; if (!webhookSecret) @@ -56,6 +58,35 @@ export async function prepareLoomMultipartUploads({ } const previous = video.metadata?.loomImportUpload; const outputKey = `${video.ownerId}/${video.id}/result.mp4`; + if ( + previous?.ownerId === video.ownerId && + previous.organizationId === video.orgId && + previous.output.key === outputKey + ) { + const output = await bucket.headObject(outputKey).pipe( + Effect.catchIf(isStorageObjectNotFound, () => Effect.succeed(null)), + runWorkflowPromise, + ); + if (output) { + if ( + !output.ETag || + !/^"[^"\r\n]+"$/.test(output.ETag) || + !output.ContentLength || + output.ContentLength > LOOM_IMPORT_MAX_BYTES + ) { + throw new Error("Existing Loom output requires storage reconciliation"); + } + return { + existingOutput: { + url: await bucket + .getInternalSignedObjectUrl(outputKey, { expiresIn: 3 * 60 * 60 }) + .pipe(runWorkflowPromise), + objectIdentity: output.ETag, + fileSize: output.ContentLength, + }, + }; + } + } if ( sourceKey && (!sourceKey.startsWith(`${video.ownerId}/${video.id}/`) || diff --git a/apps/web/workflows/import-loom-video.ts b/apps/web/workflows/import-loom-video.ts index 422b27ccbd7..44216839fa4 100644 --- a/apps/web/workflows/import-loom-video.ts +++ b/apps/web/workflows/import-loom-video.ts @@ -332,6 +332,20 @@ async function hasExistingLoomUpload( metadata: video.metadata, }); const [bucket] = yield* Storage.getAccessForVideo(videoDomain); + const previous = video.metadata?.loomImportUpload; + if ( + previous?.ownerId === video.ownerId && + previous.organizationId === video.orgId && + previous.output.key === `${video.ownerId}/${video.id}/result.mp4` + ) { + const savedOutput = yield* bucket.headObject(previous.output.key).pipe( + Effect.map( + (object) => (object.ContentLength ?? 0) >= MINIMUM_VIDEO_SIZE, + ), + Effect.catchIf(isStorageObjectNotFound, () => Effect.succeed(false)), + ); + if (savedOutput) return true; + } return yield* bucket.headObject(payload.rawFileKey).pipe( Effect.map((object) => (object.ContentLength ?? 0) >= MINIMUM_VIDEO_SIZE), Effect.catchIf(isStorageObjectNotFound, () => Effect.succeed(false)), @@ -354,6 +368,7 @@ async function startMediaServerProcessJob( sourcePresignedUrl?: string; sourceUpload?: LoomMultipartUploadTarget; outputUpload?: LoomMultipartUploadTarget; + existingOutput?: { url: string; objectIdentity: string; fileSize: number }; maxImportBytes: number; outputPresignedUrl: string; thumbnailPresignedUrl: string; @@ -386,18 +401,22 @@ async function startMediaServerProcessJob( }); if (response.ok) { - const { jobId } = (await response.json()) as { jobId: string }; + const accepted: unknown = await response.json(); + if ( + typeof accepted !== "object" || + accepted === null || + !("jobId" in accepted) || + typeof accepted.jobId !== "string" || + !accepted.jobId + ) { + throw new Error("Loom worker acceptance requires job reconciliation"); + } + const jobId = accepted.jobId; await recordDispatch("accepted", jobId).catch((error) => { console.error("Could not record accepted Loom media dispatch", error); }); return jobId; } - if ( - (response.status >= 400 && response.status < 500) || - response.status === 503 - ) - await recordDispatch("rejected"); - const errorData = (await response.json().catch(() => ({}))) as { error?: string; code?: string; @@ -408,9 +427,9 @@ async function startMediaServerProcessJob( errorData.details || "Video processing failed to start"; const shouldRetry = - response.status === 503 && - (errorData.code === "SERVER_BUSY" || - errorMessage.includes("Server is busy")); + response.status === 503 && errorData.code === "SERVER_BUSY"; + if ((response.status >= 400 && response.status < 500) || shouldRetry) + await recordDispatch("rejected"); if (shouldRetry && attempt < MEDIA_SERVER_START_MAX_ATTEMPTS - 1) { await waitForRetry(MEDIA_SERVER_START_RETRY_BASE_MS * 2 ** attempt); @@ -460,6 +479,7 @@ async function processVideoOnMediaServer( previewGifPresignedUrl, sourceUpload, outputUpload, + existingOutput, abortUploads, } = await Effect.gen(function* () { const [video] = yield* Effect.promise(() => @@ -538,17 +558,18 @@ async function processVideoOnMediaServer( ); return { - rawVideoUrl, - sourcePresignedUrl: multipart.sourceUpload - ? undefined - : sourcePresignedUrl, + rawVideoUrl: multipart.existingOutput?.url ?? rawVideoUrl, + sourcePresignedUrl: + multipart.sourceUpload || multipart.existingOutput + ? undefined + : sourcePresignedUrl, outputPresignedUrl, thumbnailPresignedUrl, previewGifPresignedUrl, ...multipart, abortUploads: async () => { await Promise.all( - Object.values(multipart) + [multipart.sourceUpload, multipart.outputUpload] .filter( (upload): upload is LoomMultipartUploadTarget => upload !== undefined, @@ -593,6 +614,7 @@ async function processVideoOnMediaServer( sourcePresignedUrl, sourceUpload, outputUpload, + existingOutput, maxImportBytes: LOOM_IMPORT_MAX_BYTES, outputPresignedUrl, thumbnailPresignedUrl, From 145ef14f4bcf572cc81b8cf31070c6648b582919 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 23 Oct 2026 15:55:58 +0100 Subject: [PATCH 19/21] fix: preserve Loom ownership during stalled pipeline recovery --- .../unit/video-pipeline-recovery.test.ts | 103 +++++++++++++++++- apps/web/lib/video-pipeline-recovery.ts | 74 ++++++++++--- 2 files changed, 160 insertions(+), 17 deletions(-) diff --git a/apps/web/__tests__/unit/video-pipeline-recovery.test.ts b/apps/web/__tests__/unit/video-pipeline-recovery.test.ts index dd1de83dfdd..7d048a6f621 100644 --- a/apps/web/__tests__/unit/video-pipeline-recovery.test.ts +++ b/apps/web/__tests__/unit/video-pipeline-recovery.test.ts @@ -2,6 +2,8 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const mockDb = vi.fn(); const mockStart = vi.fn(); +const mockStartLoomImport = vi.fn(); +const mockLoomImportRunState = vi.fn(); const mockTranscribeVideo = vi.fn(); const mockStartAiGeneration = vi.fn(); @@ -69,6 +71,17 @@ vi.mock("workflow/api", () => ({ start: mockStart, })); +vi.mock("@/lib/loom-import-start", () => ({ + startLoomImportWorkflow: mockStartLoomImport, + getLoomImportRunState: mockLoomImportRunState, + LOOM_IMPORT_CONFIRMATION_MESSAGE: "Confirming Loom import", + LoomImportStartError: class extends Error { + constructor(readonly canRetry: boolean) { + super("Loom startup failed"); + } + }, +})); + vi.mock("@/lib/transcribe", () => ({ transcribeVideo: mockTranscribeVideo, })); @@ -127,6 +140,8 @@ const staleAt = new Date("2026-07-20T14:00:00.000Z"); beforeEach(() => { vi.clearAllMocks(); mockStart.mockResolvedValue({ runId: "run-1" }); + mockStartLoomImport.mockReset().mockResolvedValue(undefined); + mockLoomImportRunState.mockReset().mockResolvedValue("inactive"); mockTranscribeVideo.mockResolvedValue({ success: true, message: "Transcription workflow started", @@ -226,6 +241,7 @@ describe("recoverStalledVideoPipeline", () => { bucketId: null, rawFileKey: "user-1/video-loom/raw.mp4", loomVideoId: "loom-video-1", + metadata: { loomImportRun: { runId: "old-run", dispatch: "accepted" } }, processingMessage, updatedAt: staleAt, }; @@ -248,17 +264,100 @@ describe("recoverStalledVideoPipeline", () => { "videoUploads.processingMessage", expect.arrayContaining([processingMessage]), ); - expect(mockStart).toHaveBeenCalledWith(expect.any(Function), [ + expect(mockStartLoomImport).toHaveBeenCalledWith( expect.objectContaining({ videoId: "video-loom", loomVideoId: "loom-video-1", reuseExistingRawUpload: true, }), - ]); + ); + expect(mockStart).not.toHaveBeenCalled(); expect(result.media.statuses).toEqual({ started: 1 }); }, ); + it.each(["active", "pending", "unconfirmed", "legacy"])( + "preserves a %s Loom attempt without starting a replacement", + async (state) => { + mockLoomImportRunState.mockResolvedValue( + state === "legacy" ? "inactive" : state, + ); + const mediaCandidate = { + videoId: "video-loom", + userId: "user-1", + bucketId: null, + rawFileKey: "user-1/video-loom/raw.mp4", + loomVideoId: "loom-video-1", + metadata: + state === "legacy" + ? null + : { loomImportRun: { runId: "old-run", dispatch: "accepted" } }, + processingMessage: "Downloading video...", + updatedAt: staleAt, + }; + const update = makeUpdateChain(1); + let dbCall = 0; + mockDb.mockImplementation(() => { + dbCall++; + if (dbCall === 1) return makeSelectChain([mediaCandidate]); + if (dbCall <= 3) return makeSelectChain([]); + return update; + }); + const { recoverStalledVideoPipeline } = await import( + "@/lib/video-pipeline-recovery" + ); + const result = await recoverStalledVideoPipeline({ now, concurrency: 1 }); + expect(mockStartLoomImport).not.toHaveBeenCalled(); + expect(mockStart).not.toHaveBeenCalled(); + expect(result.media.statuses).toEqual({ + [state === "active" ? "already-claimed" : "reconciliation-required"]: 1, + }); + if (state === "active") expect(update.set).not.toHaveBeenCalled(); + }, + ); + + it.each([true, false])( + "keeps the claimed Loom attempt safe after retryable=%s startup failure", + async (canRetry) => { + const { LoomImportStartError } = await import("@/lib/loom-import-start"); + mockStartLoomImport.mockRejectedValue( + new LoomImportStartError(canRetry, new Error("startup")), + ); + const mediaCandidate = { + videoId: "video-loom", + userId: "user-1", + bucketId: null, + rawFileKey: "user-1/video-loom/raw.mp4", + loomVideoId: "loom-video-1", + metadata: { loomImportRun: { runId: "old-run", dispatch: "accepted" } }, + processingMessage: "Downloading video...", + updatedAt: staleAt, + }; + const update = makeUpdateChain(1); + let dbCall = 0; + mockDb.mockImplementation(() => { + dbCall++; + if (dbCall === 1) return makeSelectChain([mediaCandidate]); + if (dbCall <= 3) return makeSelectChain([]); + return update; + }); + const { recoverStalledVideoPipeline } = await import( + "@/lib/video-pipeline-recovery" + ); + const result = await recoverStalledVideoPipeline({ now, concurrency: 1 }); + expect(result.media.statuses).toEqual({ + [canRetry ? "retry-scheduled" : "reconciliation-required"]: 1, + }); + expect(update.set).toHaveBeenLastCalledWith( + expect.objectContaining({ + processingMessage: canRetry + ? "Processing recovery will retry automatically" + : "Confirming Loom import", + }), + ); + }, + ); + it("starts eligible AI generation when legacy metadata is null", async () => { const aiCandidate = { videoId: "video-ai", diff --git a/apps/web/lib/video-pipeline-recovery.ts b/apps/web/lib/video-pipeline-recovery.ts index 060df34392d..e8528bc016a 100644 --- a/apps/web/lib/video-pipeline-recovery.ts +++ b/apps/web/lib/video-pipeline-recovery.ts @@ -23,8 +23,13 @@ import { import { start } from "workflow/api"; import { isAiGenerationEnabledForUser } from "@/lib/ai-generation-entitlement"; import { startAiGeneration } from "@/lib/generate-ai"; +import { + getLoomImportRunState, + LOOM_IMPORT_CONFIRMATION_MESSAGE, + LoomImportStartError, + startLoomImportWorkflow, +} from "@/lib/loom-import-start"; import { transcribeVideo } from "@/lib/transcribe"; -import { importLoomVideoWorkflow } from "@/workflows/import-loom-video"; import { processVideoWorkflow } from "@/workflows/process-video"; export const STALLED_PIPELINE_MIN_AGE_MS = 60 * 60 * 1000; @@ -45,6 +50,7 @@ const STALLED_MEDIA_PROCESSING_MESSAGES = [ type PipelineRecoveryStatus = | "started" | "already-claimed" + | "reconciliation-required" | "retry-scheduled" | "failed"; @@ -128,6 +134,7 @@ type MediaCandidate = { bucketId: string | null; rawFileKey: string; loomVideoId: string | null; + metadata: VideoMetadata | null; processingMessage: string; updatedAt: Date; }; @@ -135,13 +142,43 @@ type MediaCandidate = { async function recoverMediaCandidate( candidate: MediaCandidate, ): Promise { + const receipt = candidate.metadata?.loomImportRun; + const receiptCondition = candidate.loomVideoId + ? receipt + ? sql`EXISTS (SELECT 1 FROM ${videos} WHERE ${videos.id} = ${candidate.videoId} AND JSON_UNQUOTE(JSON_EXTRACT(${videos.metadata}, '$.loomImportRun.runId')) = ${receipt.runId})` + : sql`EXISTS (SELECT 1 FROM ${videos} WHERE ${videos.id} = ${candidate.videoId} AND JSON_EXTRACT(${videos.metadata}, '$.loomImportRun') IS NULL)` + : undefined; + if (candidate.loomVideoId) { + const state = await getLoomImportRunState(receipt); + if (state === "active") return "already-claimed"; + if (!receipt || state !== "inactive") { + const result = await db() + .update(videoUploads) + .set({ + processingMessage: LOOM_IMPORT_CONFIRMATION_MESSAGE, + updatedAt: new Date(), + }) + .where( + and( + eq(videoUploads.videoId, candidate.videoId), + eq(videoUploads.phase, "processing"), + eq(videoUploads.updatedAt, candidate.updatedAt), + receiptCondition, + ), + ); + return getAffectedRows(result) === 1 + ? "reconciliation-required" + : "already-claimed"; + } + } + const claimedAt = new Date(Math.floor(Date.now() / 1000) * 1000); const claimResult = await db() .update(videoUploads) .set({ processingProgress: 0, processingMessage: "Restarting video processing", processingError: null, - updatedAt: new Date(), + updatedAt: claimedAt, }) .where( and( @@ -151,6 +188,7 @@ async function recoverMediaCandidate( eq(videoUploads.processingMessage, candidate.processingMessage), eq(videoUploads.rawFileKey, candidate.rawFileKey), eq(videoUploads.updatedAt, candidate.updatedAt), + receiptCondition, ), ); @@ -158,16 +196,14 @@ async function recoverMediaCandidate( try { if (candidate.loomVideoId) { - await start(importLoomVideoWorkflow, [ - { - videoId: candidate.videoId, - userId: candidate.userId, - rawFileKey: candidate.rawFileKey, - bucketId: candidate.bucketId, - loomVideoId: candidate.loomVideoId, - reuseExistingRawUpload: true, - }, - ]); + await startLoomImportWorkflow({ + videoId: candidate.videoId, + userId: candidate.userId, + rawFileKey: candidate.rawFileKey, + bucketId: candidate.bucketId, + loomVideoId: candidate.loomVideoId, + reuseExistingRawUpload: true, + }); } else { await start(processVideoWorkflow, [ { @@ -180,20 +216,27 @@ async function recoverMediaCandidate( } return "started"; - } catch { + } catch (error) { + const needsReconciliation = + Boolean(candidate.loomVideoId) && + (!(error instanceof LoomImportStartError) || !error.canRetry); await db() .update(videoUploads) .set({ - processingMessage: "Processing recovery will retry automatically", + processingMessage: needsReconciliation + ? LOOM_IMPORT_CONFIRMATION_MESSAGE + : "Processing recovery will retry automatically", updatedAt: new Date(), }) .where( and( eq(videoUploads.videoId, candidate.videoId), eq(videoUploads.phase, "processing"), + eq(videoUploads.updatedAt, claimedAt), + eq(videoUploads.processingMessage, "Restarting video processing"), ), ); - return "retry-scheduled"; + return needsReconciliation ? "reconciliation-required" : "retry-scheduled"; } } @@ -300,6 +343,7 @@ export async function recoverStalledVideoPipeline({ bucketId: videos.bucket, rawFileKey: videoUploads.rawFileKey, loomVideoId: importedVideos.sourceId, + metadata: videos.metadata, processingMessage: videoUploads.processingMessage, updatedAt: videoUploads.updatedAt, }) From 8d0b87d5a07ac24f95aeab30aa418924734c4210 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 23 Oct 2026 16:02:37 +0100 Subject: [PATCH 20/21] fix: keep held Loom imports eligible for safe recovery --- apps/web/__tests__/unit/video-pipeline-recovery.test.ts | 6 +++++- apps/web/__tests__/unit/video-processing-recovery.test.ts | 2 +- apps/web/lib/video-pipeline-recovery.ts | 1 + apps/web/lib/video-processing-recovery.ts | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/apps/web/__tests__/unit/video-pipeline-recovery.test.ts b/apps/web/__tests__/unit/video-pipeline-recovery.test.ts index 7d048a6f621..561debc488b 100644 --- a/apps/web/__tests__/unit/video-pipeline-recovery.test.ts +++ b/apps/web/__tests__/unit/video-pipeline-recovery.test.ts @@ -232,7 +232,11 @@ describe("recoverStalledVideoPipeline", () => { expect(result.media.statuses).toEqual({ "already-claimed": 1 }); }); - it.each(["Downloading video...", "Queued for Loom import processing..."])( + it.each([ + "Downloading video...", + "Queued for Loom import processing...", + "Confirming Loom import", + ])( "restarts orphaned Loom jobs in the %s state through the bulk import workflow", async (processingMessage) => { const mediaCandidate = { diff --git a/apps/web/__tests__/unit/video-processing-recovery.test.ts b/apps/web/__tests__/unit/video-processing-recovery.test.ts index ac74ddedd99..d725b3e79f8 100644 --- a/apps/web/__tests__/unit/video-processing-recovery.test.ts +++ b/apps/web/__tests__/unit/video-processing-recovery.test.ts @@ -451,7 +451,7 @@ describe("recoverFailedVideoProcessing", () => { expect(result.statuses).toEqual({ "reconciliation-required": 1 }); expect(update.set).toHaveBeenCalledWith({ processingMessage: "Confirmation required", - updatedAt: expect.any(Date), + ...(row.phase === "processing" ? {} : { updatedAt: expect.any(Date) }), }); expect(update.where).toHaveBeenCalledWith( expect.arrayContaining([ diff --git a/apps/web/lib/video-pipeline-recovery.ts b/apps/web/lib/video-pipeline-recovery.ts index e8528bc016a..67b2fda0835 100644 --- a/apps/web/lib/video-pipeline-recovery.ts +++ b/apps/web/lib/video-pipeline-recovery.ts @@ -45,6 +45,7 @@ const STALLED_MEDIA_PROCESSING_MESSAGES = [ "Queued for video processing...", "Queued for Loom import processing...", "Processing recovery will retry automatically", + LOOM_IMPORT_CONFIRMATION_MESSAGE, ] as const; type PipelineRecoveryStatus = diff --git a/apps/web/lib/video-processing-recovery.ts b/apps/web/lib/video-processing-recovery.ts index db0b41e2fd0..4340693f166 100644 --- a/apps/web/lib/video-processing-recovery.ts +++ b/apps/web/lib/video-processing-recovery.ts @@ -162,7 +162,7 @@ async function recoverLoomCandidate({ .update(videoUploads) .set({ processingMessage: LOOM_IMPORT_CONFIRMATION_MESSAGE, - updatedAt: new Date(), + ...(phase === "processing" ? {} : { updatedAt: new Date() }), }) .where( and( From da3376247e84ed7007cbc65fac59ad406e623965 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 23 Oct 2026 16:10:44 +0100 Subject: [PATCH 21/21] fix: rotate Loom recovery checks without starving new imports --- .../unit/video-processing-recovery.test.ts | 136 +++++++++++++++++- apps/web/lib/video-processing-recovery.ts | 53 +++++-- 2 files changed, 170 insertions(+), 19 deletions(-) diff --git a/apps/web/__tests__/unit/video-processing-recovery.test.ts b/apps/web/__tests__/unit/video-processing-recovery.test.ts index d725b3e79f8..b2d59f6fa19 100644 --- a/apps/web/__tests__/unit/video-processing-recovery.test.ts +++ b/apps/web/__tests__/unit/video-processing-recovery.test.ts @@ -162,7 +162,8 @@ describe("recoverFailedVideoProcessing", () => { .mockReturnValueOnce(makeSelectChain(createCandidates("uploading", 0))) .mockReturnValueOnce(makeSelectChain(createCandidates("processing", 1))) .mockReturnValueOnce(makeSelectChain([])) - .mockReturnValueOnce(makeSelectChain([])); + .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValue(makeUpdateChain(1)); mockGetLoomImportRunState.mockResolvedValue("active"); const { recoverFailedVideoProcessing } = await import( "@/lib/video-processing-recovery" @@ -178,6 +179,75 @@ describe("recoverFailedVideoProcessing", () => { expect(mockStartLoomImportWorkflow).not.toHaveBeenCalled(); }); + it("rotates eight unresolved holds so a newer import is checked on the next scan", async () => { + const holds = Array.from({ length: 8 }, (_, i) => ({ + ...candidate, + videoId: `hold-${i}`, + phase: "processing", + loomVideoId: `loom-${i}`, + uploadUpdatedAt: new Date(1000 + i), + metadata: { + loomImportRun: { runId: `hold-${i}`, dispatch: "uncertain" as const }, + }, + })); + const pending = { + ...candidate, + videoId: "newer", + phase: "uploading", + rawFileKey: null, + loomVideoId: "newer-loom", + uploadUpdatedAt: new Date(9000), + metadata: { + loomImportRun: { runId: "newer", dispatch: "rejected" as const }, + }, + }; + let selection = 0; + mockDb.mockImplementation(() => ({ + select: (...args: unknown[]) => { + const slot = selection++ % 4; + const rows = + slot === 0 + ? [pending] + : slot === 1 + ? [...holds].sort( + (a, b) => + a.uploadUpdatedAt.getTime() - b.uploadUpdatedAt.getTime(), + ) + : []; + return makeSelectChain(rows).select(...args); + }, + update: () => ({ + set: (values: { updatedAt?: Date }) => ({ + where: async ( + conditions: Array<{ left?: string; right?: unknown }>, + ) => { + const id = conditions.find( + (c) => c?.left === "videoUploads.videoId", + )?.right; + const row = [...holds, pending].find((row) => row.videoId === id); + if (row && values.updatedAt) row.uploadUpdatedAt = values.updatedAt; + return [{ affectedRows: 1 }]; + }, + }), + }), + })); + mockGetLoomImportRunState.mockImplementation(async (receipt) => + receipt.runId === "newer" ? "inactive" : "unconfirmed", + ); + const { recoverFailedVideoProcessing } = await import( + "@/lib/video-processing-recovery" + ); + const first = await recoverFailedVideoProcessing(); + expect(first.results.map((row) => row.videoId)).not.toContain("newer"); + const second = await recoverFailedVideoProcessing(); + expect(second.results).toContainEqual({ + videoId: "newer", + kind: "loom", + status: "started", + }); + expect(mockStartLoomImportWorkflow).toHaveBeenCalledTimes(1); + }); + it("reserves a failed-import slot while uncertain startups remain protected", async () => { const pending = Array.from({ length: 8 }, (_, index) => ({ ...candidate, @@ -375,7 +445,7 @@ describe("recoverFailedVideoProcessing", () => { expect(mockStart).not.toHaveBeenCalled(); expect(mockStartLoomImportWorkflow).not.toHaveBeenCalled(); expect(result.statuses).toEqual({ "already-claimed": 1 }); - expect(mockDb).toHaveBeenCalledTimes(4); + expect(mockDb).toHaveBeenCalledTimes(5); expect(mockGetLoomImportRunState).toHaveBeenCalledWith(receipt); }, ); @@ -418,7 +488,6 @@ describe("recoverFailedVideoProcessing", () => { { phase: "uploading", state: "unconfirmed" }, { phase: "processing", state: "pending" }, { phase: "processing", state: "unconfirmed" }, - { phase: "processing", state: "inactive" }, ])( "exposes startup reconciliation without redispatching: %j", async (row) => { @@ -451,7 +520,7 @@ describe("recoverFailedVideoProcessing", () => { expect(result.statuses).toEqual({ "reconciliation-required": 1 }); expect(update.set).toHaveBeenCalledWith({ processingMessage: "Confirmation required", - ...(row.phase === "processing" ? {} : { updatedAt: expect.any(Date) }), + updatedAt: expect.any(Date), }); expect(update.where).toHaveBeenCalledWith( expect.arrayContaining([ @@ -469,6 +538,65 @@ describe("recoverFailedVideoProcessing", () => { }, ); + it.each([false, true])( + "reclaims a confirmed inactive processing attempt without losing its source, rejection=%s", + async (reject) => { + const { LoomImportStartError } = await import("@/lib/loom-import-start"); + const receipt = { runId: "failed-run", dispatch: "accepted" as const }; + const claim = makeUpdateChain(1); + const restore = makeUpdateChain(1); + mockDb + .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValueOnce( + makeSelectChain([ + { + ...candidate, + phase: "processing", + loomVideoId: "loom-1", + metadata: { loomImportRun: receipt }, + }, + ]), + ) + .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValueOnce(makeSelectChain([])) + .mockReturnValueOnce(claim) + .mockReturnValueOnce(restore); + if (reject) + mockStartLoomImportWorkflow.mockRejectedValueOnce( + new LoomImportStartError(true, new Error("busy")), + ); + const { recoverFailedVideoProcessing } = await import( + "@/lib/video-processing-recovery" + ); + const result = await recoverFailedVideoProcessing(); + expect(result.statuses).toEqual({ + [reject ? "retry-scheduled" : "started"]: 1, + }); + expect(mockStartLoomImportWorkflow).toHaveBeenCalledWith( + expect.objectContaining({ + rawFileKey: candidate.rawFileKey, + reuseExistingRawUpload: true, + }), + ); + expect(claim.where).toHaveBeenCalledWith( + expect.arrayContaining([ + { left: "videoUploads.rawFileKey", right: candidate.rawFileKey }, + { left: "videoUploads.updatedAt", right: candidate.uploadUpdatedAt }, + expect.objectContaining({ + values: expect.arrayContaining(["failed-run"]), + }), + ]), + ); + if (reject) + expect(restore.set).toHaveBeenCalledWith( + expect.objectContaining({ + phase: "error", + rawFileKey: candidate.rawFileKey, + }), + ); + }, + ); + it("does not change reconciliation state after the selected claim changed", async () => { const update = makeUpdateChain(0); mockDb diff --git a/apps/web/lib/video-processing-recovery.ts b/apps/web/lib/video-processing-recovery.ts index 4340693f166..4bdd3d8a753 100644 --- a/apps/web/lib/video-processing-recovery.ts +++ b/apps/web/lib/video-processing-recovery.ts @@ -151,18 +151,31 @@ async function recoverLoomCandidate({ const runState = await getLoomImportRunState(receipt); const receiptCondition = receipt ? sql`EXISTS (SELECT 1 FROM ${videos} WHERE ${videos.id} = ${videoId} AND JSON_UNQUOTE(JSON_EXTRACT(${videos.metadata}, '$.loomImportRun.runId')) = ${receipt.runId})` - : undefined; - if (runState === "active") return "already-claimed"; + : sql`EXISTS (SELECT 1 FROM ${videos} WHERE ${videos.id} = ${videoId} AND JSON_EXTRACT(${videos.metadata}, '$.loomImportRun') IS NULL)`; + if (runState === "active") { + await db() + .update(videoUploads) + .set({ updatedAt: new Date() }) + .where( + and( + eq(videoUploads.videoId, videoId), + eq(videoUploads.phase, phase), + eq(videoUploads.updatedAt, uploadUpdatedAt), + receiptCondition, + ), + ); + return "already-claimed"; + } if ( runState === "pending" || runState === "unconfirmed" || - phase === "processing" + (phase === "processing" && !receipt) ) { const result = await db() .update(videoUploads) .set({ processingMessage: LOOM_IMPORT_CONFIRMATION_MESSAGE, - ...(phase === "processing" ? {} : { updatedAt: new Date() }), + updatedAt: new Date(), }) .where( and( @@ -200,16 +213,25 @@ async function recoverLoomCandidate({ eq(videoUploads.rawFileKey, rawFileKey), recoverableProcessingError(), ] - : [ - isNull(videoUploads.rawFileKey), - or( - eq(videoUploads.processingMessage, LOOM_IMPORT_PENDING_MESSAGE), - eq( - videoUploads.processingMessage, - LOOM_IMPORT_CONFIRMATION_MESSAGE, + : phase === "processing" + ? [ + existingRawFileKey + ? eq(videoUploads.rawFileKey, existingRawFileKey) + : isNull(videoUploads.rawFileKey), + ] + : [ + isNull(videoUploads.rawFileKey), + or( + eq( + videoUploads.processingMessage, + LOOM_IMPORT_PENDING_MESSAGE, + ), + eq( + videoUploads.processingMessage, + LOOM_IMPORT_CONFIRMATION_MESSAGE, + ), ), - ), - ]), + ]), ), ); @@ -225,7 +247,8 @@ async function recoverLoomCandidate({ bucketId, loomDownloadUrl: "", loomVideoId, - reuseExistingRawUpload: phase === "error", + reuseExistingRawUpload: + phase !== "uploading" && existingRawFileKey !== null, }); return "started"; } catch (error) { @@ -233,7 +256,7 @@ async function recoverLoomCandidate({ return "already-claimed"; } const message = error instanceof Error ? error.message : String(error); - if (phase === "error") { + if (phase !== "uploading") { await db() .update(videoUploads) .set({