From 649c06c3aefd727c437ccebca9c42bf048d9242c Mon Sep 17 00:00:00 2001 From: 07rjain <07rjain@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:53:26 +0000 Subject: [PATCH 01/20] fix(email): never send from auth@undefined, fall back to logged codes and surface send failures --- apps/web/__tests__/unit/email-config.test.ts | 81 +++++++++++++++++++ apps/web/actions/organization/send-invites.ts | 9 ++- apps/web/app/(org)/verify-otp/form.tsx | 16 +++- apps/web/app/(org)/verify-otp/page.tsx | 2 + apps/web/app/api/mobile/[...route]/route.ts | 18 +++-- apps/web/lib/account-deletion-request.ts | 4 +- packages/database/auth/auth-options.ts | 15 +++- packages/database/emails/config.ts | 69 +++++++++++++++- 8 files changed, 191 insertions(+), 23 deletions(-) create mode 100644 apps/web/__tests__/unit/email-config.test.ts diff --git a/apps/web/__tests__/unit/email-config.test.ts b/apps/web/__tests__/unit/email-config.test.ts new file mode 100644 index 00000000000..f0ef22ef6ed --- /dev/null +++ b/apps/web/__tests__/unit/email-config.test.ts @@ -0,0 +1,81 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + env: {} as Record, + build: {} as Record, + send: vi.fn(), +})); + +vi.mock("@cap/env", () => ({ + serverEnv: () => mocks.env, + buildEnv: mocks.build, +})); + +vi.mock("resend", () => ({ + Resend: class { + emails = { send: mocks.send }; + }, +})); + +import { getEmailDeliveryStatus, sendEmail } from "@cap/database/emails/config"; + +const message = { + email: "user@example.com", + subject: "Hello", + react: null as unknown as Parameters[0]["react"], +}; + +describe("email config", () => { + beforeEach(() => { + mocks.send.mockReset(); + for (const key of Object.keys(mocks.env)) delete mocks.env[key]; + for (const key of Object.keys(mocks.build)) delete mocks.build[key]; + }); + + it("reports missing RESEND_API_KEY", () => { + const status = getEmailDeliveryStatus(); + expect(status.configured).toBe(false); + expect(status.configured ? "" : status.reason).toContain( + "Email (Resend) is not configured", + ); + }); + + it("treats a missing RESEND_FROM_DOMAIN as not configured on self-hosted builds", async () => { + mocks.env.RESEND_API_KEY = "re_test"; + expect(getEmailDeliveryStatus().configured).toBe(false); + + const result = await sendEmail(message); + expect(mocks.send).not.toHaveBeenCalled(); + expect(result?.error?.name).toBe("email_not_configured"); + }); + + it("never sends from auth@undefined", async () => { + mocks.env.RESEND_API_KEY = "re_test"; + await sendEmail(message); + for (const call of mocks.send.mock.calls) { + expect(call[0].from).not.toContain("undefined"); + } + }); + + it("sends from the configured domain and returns the provider id", async () => { + mocks.env.RESEND_API_KEY = "re_test"; + mocks.env.RESEND_FROM_DOMAIN = "example.com"; + mocks.send.mockResolvedValue({ data: { id: "email_1" }, error: null }); + + const result = await sendEmail(message); + expect(mocks.send.mock.calls[0]?.[0].from).toBe("auth@example.com"); + expect(result).toEqual({ data: { id: "email_1" }, error: null }); + }); + + it("surfaces provider rejections instead of claiming success", async () => { + mocks.env.RESEND_API_KEY = "re_test"; + mocks.env.RESEND_FROM_DOMAIN = "example.com"; + mocks.send.mockResolvedValue({ + data: null, + error: { name: "validation_error", message: "domain not verified" }, + }); + + const result = await sendEmail(message); + expect(result?.error?.message).toBe("domain not verified"); + }); +}); diff --git a/apps/web/actions/organization/send-invites.ts b/apps/web/actions/organization/send-invites.ts index ad3642d4e3b..94c25f97ebf 100644 --- a/apps/web/actions/organization/send-invites.ts +++ b/apps/web/actions/organization/send-invites.ts @@ -184,9 +184,12 @@ export async function sendOrganizationInvites( }), ); - const failedInvites = inviteRecords.filter( - (_, i) => emailResults[i]?.status === "rejected", - ); + const failedInvites = inviteRecords.filter((_, i) => { + const result = emailResults[i]; + if (!result || result.status === "rejected") return true; + const error = result.value?.error; + return Boolean(error && error.name !== "email_not_configured"); + }); const failedEmails = failedInvites.map((r) => r.email); if (failedInvites.length > 0) { diff --git a/apps/web/app/(org)/verify-otp/form.tsx b/apps/web/app/(org)/verify-otp/form.tsx index ca8a88d9b68..8a9da42966d 100644 --- a/apps/web/app/(org)/verify-otp/form.tsx +++ b/apps/web/app/(org)/verify-otp/form.tsx @@ -16,10 +16,12 @@ export function VerifyOTPForm({ email, next, lastSent, + emailDeliveryConfigured = true, }: { email: string; next?: string; lastSent?: string; + emailDeliveryConfigured?: boolean; }) { const [code, setCode] = useState(["", "", "", "", "", ""]); const [lastResendTime, setLastResendTime] = useState( @@ -171,9 +173,17 @@ export function VerifyOTPForm({

Enter verification code

-

- We sent a 6-digit code to {normalizedEmail} -

+ {emailDeliveryConfigured ? ( +

+ We sent a 6-digit code to {normalizedEmail} +

+ ) : ( +

+ Email (Resend) is not configured on this server. The 6-digit code + for {normalizedEmail} was printed to the server logs. Ask your + administrator, or set RESEND_API_KEY and RESEND_FROM_DOMAIN. +

+ )}
diff --git a/apps/web/app/(org)/verify-otp/page.tsx b/apps/web/app/(org)/verify-otp/page.tsx index 37abfc47456..9c30b0d1ec9 100644 --- a/apps/web/app/(org)/verify-otp/page.tsx +++ b/apps/web/app/(org)/verify-otp/page.tsx @@ -1,4 +1,5 @@ import { getCurrentUser } from "@cap/database/auth/session"; +import { isEmailDeliveryConfigured } from "@cap/database/emails/config"; import { serverEnv } from "@cap/env"; import { redirect } from "next/navigation"; import { Suspense } from "react"; @@ -30,6 +31,7 @@ export default async function VerifyOTPPage(props: { email={searchParams.email?.toLowerCase() ?? ""} next={searchParams.next} lastSent={searchParams.lastSent} + emailDeliveryConfigured={isEmailDeliveryConfigured()} />
diff --git a/apps/web/app/api/mobile/[...route]/route.ts b/apps/web/app/api/mobile/[...route]/route.ts index 3298f890c5e..f844ffefa78 100644 --- a/apps/web/app/api/mobile/[...route]/route.ts +++ b/apps/web/app/api/mobile/[...route]/route.ts @@ -2,7 +2,7 @@ import crypto from "node:crypto"; import { authOptions } from "@cap/database/auth/auth-options"; import { isEmailAllowedForSignup } from "@cap/database/auth/domain-utils"; import { hashPassword } from "@cap/database/crypto"; -import { sendEmail } from "@cap/database/emails/config"; +import { getEmailDeliveryStatus, sendEmail } from "@cap/database/emails/config"; import { OTPEmail } from "@cap/database/emails/otp-email"; import { nanoId } from "@cap/database/helpers"; import * as Db from "@cap/database/schema"; @@ -154,12 +154,11 @@ const hashEmailCode = (code: string) => .digest("hex"); const sendMobileEmailCode = async (email: string, code: string) => { - if (!serverEnv().RESEND_API_KEY) { - if (process.env.NODE_ENV === "production") { - throw new Error("RESEND_API_KEY is required to send mobile email codes"); - } + const emailStatus = getEmailDeliveryStatus(); + if (!emailStatus.configured) { console.log(""); - console.log("Cap mobile verification code"); + console.log("Cap mobile verification code (email delivery disabled)"); + console.log(emailStatus.reason); console.log(`Email: ${email}`); console.log(`Code: ${code}`); console.log("Expires in: 10 minutes"); @@ -167,11 +166,16 @@ const sendMobileEmailCode = async (email: string, code: string) => { return; } - await sendEmail({ + const delivery = await sendEmail({ email, subject: "Your Cap Verification Code", react: OTPEmail({ code, email }), }); + if (delivery?.error) { + throw new Error( + `Failed to send verification email: ${delivery.error.message}`, + ); + } }; const getMobileRedirectUrl = (redirectUri: string) => { diff --git a/apps/web/lib/account-deletion-request.ts b/apps/web/lib/account-deletion-request.ts index 8a21645b71f..3468a026319 100644 --- a/apps/web/lib/account-deletion-request.ts +++ b/apps/web/lib/account-deletion-request.ts @@ -164,7 +164,7 @@ export const createAccountDeletionRequest = async ({ fromOverride: ACCOUNT_DELETION_EMAIL_FROM, idempotencyKey: `account-deletion-${request.id}`, }) - .then(() => true) + .then((delivery) => !delivery?.error) .catch(() => false); return { @@ -264,7 +264,7 @@ export const createMobileContentReport = async ({ fromOverride: ACCOUNT_DELETION_EMAIL_FROM, idempotencyKey: `mobile-content-report-${request.id}`, }) - .then(() => true) + .then((delivery) => !delivery?.error) .catch(() => false); return { diff --git a/packages/database/auth/auth-options.ts b/packages/database/auth/auth-options.ts index b8641c378aa..5a6aa7af670 100644 --- a/packages/database/auth/auth-options.ts +++ b/packages/database/auth/auth-options.ts @@ -11,7 +11,7 @@ import EmailProvider from "next-auth/providers/email"; import GoogleProvider from "next-auth/providers/google"; import type { Provider } from "next-auth/providers/index"; import WorkOSProvider from "next-auth/providers/workos"; -import { sendEmail } from "../emails/config.ts"; +import { getEmailDeliveryStatus, sendEmail } from "../emails/config.ts"; import { db } from "../index.ts"; import { users } from "../schema.ts"; import { @@ -145,12 +145,14 @@ export const authOptions = (ssoContext?: SsoAuthContext): NextAuthOptions => { return crypto.randomInt(100000, 1000000).toString(); }, async sendVerificationRequest({ identifier, token }) { - if (!serverEnv().RESEND_API_KEY) { + const emailStatus = getEmailDeliveryStatus(); + if (!emailStatus.configured) { console.log("\n"); console.log( "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━", ); - console.log("🔐 VERIFICATION CODE (Development Mode)"); + console.log("🔐 VERIFICATION CODE (email delivery disabled)"); + console.log(`ℹ️ ${emailStatus.reason}`); console.log( "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━", ); @@ -166,11 +168,16 @@ export const authOptions = (ssoContext?: SsoAuthContext): NextAuthOptions => { } else { const { OTPEmail } = await import("../emails/otp-email"); const email = OTPEmail({ code: token, email: identifier }); - await sendEmail({ + const delivery = await sendEmail({ email: identifier, subject: `Your Cap Verification Code`, react: email, }); + if (delivery?.error) { + throw new Error( + `Failed to send verification email: ${delivery.error.message}`, + ); + } } }, }), diff --git a/packages/database/emails/config.ts b/packages/database/emails/config.ts index a5145c0d7d9..5f049c77f2f 100644 --- a/packages/database/emails/config.ts +++ b/packages/database/emails/config.ts @@ -5,6 +5,46 @@ import { Resend } from "resend"; export const resend = () => serverEnv().RESEND_API_KEY ? new Resend(serverEnv().RESEND_API_KEY) : null; +export const EMAIL_NOT_CONFIGURED_MESSAGE = "Email (Resend) is not configured"; + +export type EmailDeliveryStatus = + | { configured: true } + | { configured: false; reason: string }; + +export function getEmailDeliveryStatus(): EmailDeliveryStatus { + const env = serverEnv(); + if (!env.RESEND_API_KEY) { + return { + configured: false, + reason: `${EMAIL_NOT_CONFIGURED_MESSAGE}: RESEND_API_KEY is not set`, + }; + } + if (!buildEnv.NEXT_PUBLIC_IS_CAP && !env.RESEND_FROM_DOMAIN?.trim()) { + return { + configured: false, + reason: `${EMAIL_NOT_CONFIGURED_MESSAGE}: RESEND_FROM_DOMAIN is not set`, + }; + } + return { configured: true }; +} + +export const isEmailDeliveryConfigured = () => + getEmailDeliveryStatus().configured; + +let warnedEmailNotConfigured = false; + +function warnEmailNotConfigured(reason: string) { + if (warnedEmailNotConfigured) return; + warnedEmailNotConfigured = true; + console.warn( + `[email] ${reason}. Emails will not be sent; login codes are printed to the server log instead.`, + ); +} + +export type SendEmailResult = + | { data: { id: string } | null; error: null } + | { data: null; error: { name: string; message: string } }; + export const sendEmail = async ({ email, subject, @@ -33,10 +73,18 @@ export const sendEmail = async ({ content: Buffer | string; contentType?: string; }[]; -}) => { +}): Promise => { + const status = getEmailDeliveryStatus(); const r = resend(); - if (!r) { - return Promise.resolve(); + if (!status.configured || !r) { + const reason = status.configured + ? EMAIL_NOT_CONFIGURED_MESSAGE + : status.reason; + warnEmailNotConfigured(reason); + return { + data: null, + error: { name: "email_not_configured", message: reason }, + }; } if (marketing && !buildEnv.NEXT_PUBLIC_IS_CAP) return; @@ -48,7 +96,7 @@ export const sendEmail = async ({ from = "Cap Auth "; else from = `auth@${serverEnv().RESEND_FROM_DOMAIN}`; - return r.emails.send( + const result = await r.emails.send( { from, to: test ? "delivered@resend.dev" : email, @@ -61,4 +109,17 @@ export const sendEmail = async ({ }, idempotencyKey ? { idempotencyKey } : undefined, ); + + if (result.error) { + console.error( + `[email] Resend rejected email "${subject}" from ${from}:`, + result.error.message, + ); + return { + data: null, + error: { name: result.error.name, message: result.error.message }, + }; + } + + return { data: result.data, error: null }; }; From cec843fc5d2e77f9a33add9f43377cc76ae480c2 Mon Sep 17 00:00:00 2001 From: 07rjain <07rjain@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:57:26 +0000 Subject: [PATCH 02/20] fix(self-host): remove public default secrets and refuse to start with missing or placeholder secrets --- .env.self-hosting.example | 20 +++++++ .../src/__tests__/lib/startup-checks.test.ts | 34 ++++++++++++ apps/media-server/src/index.ts | 3 ++ apps/media-server/src/lib/startup-checks.ts | 39 ++++++++++++++ .../__tests__/unit/insecure-secrets.test.ts | 40 ++++++++++++++ apps/web/content/docs/self-hosting.mdx | 22 +++++--- apps/web/instrumentation.node.ts | 21 +++++++- docker-compose.template.yml | 8 +-- docker-compose.yml | 8 +-- packages/env/index.ts | 6 +++ packages/env/secrets.ts | 52 +++++++++++++++++++ packages/local-docker/docker-compose.yml | 1 + 12 files changed, 238 insertions(+), 16 deletions(-) create mode 100644 .env.self-hosting.example create mode 100644 apps/media-server/src/__tests__/lib/startup-checks.test.ts create mode 100644 apps/media-server/src/lib/startup-checks.ts create mode 100644 apps/web/__tests__/unit/insecure-secrets.test.ts create mode 100644 packages/env/secrets.ts diff --git a/.env.self-hosting.example b/.env.self-hosting.example new file mode 100644 index 00000000000..cd24c6bece9 --- /dev/null +++ b/.env.self-hosting.example @@ -0,0 +1,20 @@ +# Copy to .env next to docker-compose.yml and fill in every value. +# Generate each secret with: openssl rand -hex 32 +# cap-web and the media server refuse to start when these are missing or left at a known placeholder. +NEXTAUTH_SECRET= +DATABASE_ENCRYPTION_KEY= +MEDIA_SERVER_WEBHOOK_SECRET= + +# Optional: database and storage passwords (internal-only defaults are used when unset) +MYSQL_PASSWORD= +MYSQL_ROOT_PASSWORD= +MINIO_ROOT_USER=cap-admin +MINIO_ROOT_PASSWORD= + +# Public URLs +CAP_URL=http://localhost:3000 +S3_PUBLIC_URL=http://localhost:9000 + +# Optional: email. Without both values, login codes are printed to the cap-web logs. +RESEND_API_KEY= +RESEND_FROM_DOMAIN= diff --git a/apps/media-server/src/__tests__/lib/startup-checks.test.ts b/apps/media-server/src/__tests__/lib/startup-checks.test.ts new file mode 100644 index 00000000000..209f10adfc5 --- /dev/null +++ b/apps/media-server/src/__tests__/lib/startup-checks.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, test } from "bun:test"; +import { getWebhookSecretProblem } from "../../lib/startup-checks"; + +describe("getWebhookSecretProblem", () => { + test("rejects a missing secret", () => { + expect(getWebhookSecretProblem({})).toContain("not set"); + }); + + test("rejects the old docker-compose placeholder", () => { + expect( + getWebhookSecretProblem({ + MEDIA_SERVER_WEBHOOK_SECRET: + "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210", + }), + ).toContain("placeholder"); + }); + + test("allows a placeholder only with the explicit local override", () => { + expect( + getWebhookSecretProblem({ + MEDIA_SERVER_WEBHOOK_SECRET: "local-media-server-secret", + MEDIA_SERVER_ALLOW_INSECURE_SECRET: "true", + }), + ).toBeNull(); + }); + + test("accepts a unique secret", () => { + expect( + getWebhookSecretProblem({ + MEDIA_SERVER_WEBHOOK_SECRET: "4b1f0c9e7a2d43e8b6c5f1a09d8e7c6b", + }), + ).toBeNull(); + }); +}); diff --git a/apps/media-server/src/index.ts b/apps/media-server/src/index.ts index 7a5b2151dfd..81f5fca3ac0 100644 --- a/apps/media-server/src/index.ts +++ b/apps/media-server/src/index.ts @@ -2,6 +2,9 @@ import app from "./app"; import { abortAllJobs } from "./lib/job-manager"; import { cancelAllMediaOperations } from "./lib/media-operations"; import { cleanupMediaTransferCache } from "./lib/media-transfer"; +import { assertSecureStartup } from "./lib/startup-checks"; + +assertSecureStartup(); const port = Number(process.env.PORT) || 3456; diff --git a/apps/media-server/src/lib/startup-checks.ts b/apps/media-server/src/lib/startup-checks.ts new file mode 100644 index 00000000000..718b705f759 --- /dev/null +++ b/apps/media-server/src/lib/startup-checks.ts @@ -0,0 +1,39 @@ +const KNOWN_PLACEHOLDER_SECRETS = new Set([ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789", + "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210", + "c7a1e98e1e5e4cec5a3fbf5eaf502d262fb99807ae2be8ee70537409e29cb6f9", + "fe337b52749070bb7b5d2c78cff9948439ea73cbc1869ba39d350e6c24db53b1", + "local-media-server-secret", + "changeme", + "change-me", + "secret", +]); + +export function getWebhookSecretProblem( + env: Record = process.env, +): string | null { + const value = env.MEDIA_SERVER_WEBHOOK_SECRET?.trim(); + if (!value) return "MEDIA_SERVER_WEBHOOK_SECRET is not set"; + if ( + KNOWN_PLACEHOLDER_SECRETS.has(value.toLowerCase()) && + env.MEDIA_SERVER_ALLOW_INSECURE_SECRET !== "true" + ) { + return "MEDIA_SERVER_WEBHOOK_SECRET is set to a publicly known placeholder value"; + } + return null; +} + +export function assertSecureStartup( + env: Record = process.env, +) { + const problem = getWebhookSecretProblem(env); + if (!problem) return; + console.error( + [ + `[media-server] Refusing to start: ${problem}.`, + "Generate one with `openssl rand -hex 32` and set the same value on cap-web and the media server.", + ].join("\n"), + ); + process.exit(1); +} diff --git a/apps/web/__tests__/unit/insecure-secrets.test.ts b/apps/web/__tests__/unit/insecure-secrets.test.ts new file mode 100644 index 00000000000..dd04d14d0ae --- /dev/null +++ b/apps/web/__tests__/unit/insecure-secrets.test.ts @@ -0,0 +1,40 @@ +import { findInsecureSecrets, formatInsecureSecrets } from "@cap/env"; +import { describe, expect, it } from "vitest"; + +const strong = + "9f1c2b7e4d5a6f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e6f708"; + +describe("findInsecureSecrets", () => { + it("flags missing and placeholder secrets", () => { + const problems = findInsecureSecrets({ + NEXTAUTH_SECRET: + "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789", + DATABASE_ENCRYPTION_KEY: "", + MEDIA_SERVER_WEBHOOK_SECRET: strong, + }); + expect(problems).toEqual([ + { name: "NEXTAUTH_SECRET", problem: "placeholder" }, + { name: "DATABASE_ENCRYPTION_KEY", problem: "missing" }, + ]); + expect(formatInsecureSecrets(problems)).toContain("openssl rand -hex 32"); + }); + + it("accepts unique secrets", () => { + expect( + findInsecureSecrets({ + NEXTAUTH_SECRET: strong, + DATABASE_ENCRYPTION_KEY: `${strong.slice(1)}0`, + MEDIA_SERVER_WEBHOOK_SECRET: `${strong.slice(2)}00`, + }), + ).toEqual([]); + }); + + it("only checks the requested names", () => { + expect( + findInsecureSecrets( + { NEXTAUTH_SECRET: strong, DATABASE_ENCRYPTION_KEY: strong }, + ["NEXTAUTH_SECRET", "DATABASE_ENCRYPTION_KEY"], + ), + ).toEqual([]); + }); +}); diff --git a/apps/web/content/docs/self-hosting.mdx b/apps/web/content/docs/self-hosting.mdx index 45219a81bd0..eda684972de 100644 --- a/apps/web/content/docs/self-hosting.mdx +++ b/apps/web/content/docs/self-hosting.mdx @@ -22,11 +22,18 @@ docker compose up -d That's it! Cap will be available at `http://localhost:3000`. - -The default `docker-compose.yml` includes placeholder secrets for quick local testing. **If you're deploying to a public server or production environment**, you must change these values. See the [Production Checklist](#production-checklist) below. + +`docker compose up` stops with an error until `NEXTAUTH_SECRET`, `DATABASE_ENCRYPTION_KEY` and `MEDIA_SERVER_WEBHOOK_SECRET` are set. cap-web and the media server also refuse to start if any of them is left at a publicly known placeholder. Create your `.env` first: + +```bash +cp .env.self-hosting.example .env +for name in NEXTAUTH_SECRET DATABASE_ENCRYPTION_KEY MEDIA_SERVER_WEBHOOK_SECRET; do + sed -i.bak "s/^$name=.*/$name=$(openssl rand -hex 32)/" .env +done +``` -Login links will appear in the logs since email isn't configured: +Login codes will appear in the logs until email is configured, and the verify screen says "Email (Resend) is not configured": ```bash docker compose logs cap-web ``` @@ -97,7 +104,7 @@ For Coolify users, use `docker-compose.coolify.yml` which includes environment v ## Email Configuration -By default, login links are printed to the server logs. To send emails: +Email is optional. Without it, login codes are printed to the cap-web logs, the verify screen shows "Email (Resend) is not configured", and invite or notification emails are skipped (invite links still work). Cap only sends email when **both** variables below are set; if Resend rejects a message, the error is logged and the sign-in fails instead of claiming a code was sent. To send emails: 1. Create an account at [Resend](https://resend.com) 2. Add these environment variables: @@ -105,6 +112,7 @@ By default, login links are printed to the server logs. To send emails: RESEND_API_KEY=re_xxxxx RESEND_FROM_DOMAIN=yourdomain.com ``` +3. Verify the domain in Resend. Login emails are sent from `auth@RESEND_FROM_DOMAIN`. ## Optional Features @@ -188,14 +196,14 @@ expiration date. ## Production Checklist - -The default `docker-compose.yml` contains **hardcoded placeholder secrets** that are visible in the public repository. Anyone who knows you're using Cap with defaults could potentially: + +`docker-compose.yml` no longer ships default secrets, and Cap refuses to start with the old public placeholders. Anyone who knows a deployment's secrets could: - **Forge authentication sessions** (via `NEXTAUTH_SECRET`) - **Decrypt sensitive database fields** (via `DATABASE_ENCRYPTION_KEY`) - **Spoof webhook requests** (via `MEDIA_SERVER_WEBHOOK_SECRET`) -This is fine for local development or testing on a private network, but **you must generate unique secrets before exposing Cap to the internet**. +Generate unique values even for testing. **Generate secure secrets:** diff --git a/apps/web/instrumentation.node.ts b/apps/web/instrumentation.node.ts index ce13ac04e3a..22c375768c3 100644 --- a/apps/web/instrumentation.node.ts +++ b/apps/web/instrumentation.node.ts @@ -9,11 +9,30 @@ import { S3Client, } from "@aws-sdk/client-s3"; import { migrateDb } from "@cap/database/migrate"; -import { buildEnv, serverEnv } from "@cap/env"; +import { + buildEnv, + findInsecureSecrets, + formatInsecureSecrets, + REQUIRED_SELF_HOST_SECRETS, + serverEnv, +} from "@cap/env"; export async function register() { if (process.env.NEXT_PUBLIC_IS_CAP) return; + if (process.env.NODE_ENV === "production") { + const insecureSecrets = findInsecureSecrets( + process.env, + process.env.MEDIA_SERVER_URL || process.env.MEDIA_SERVER_WEBHOOK_SECRET + ? REQUIRED_SELF_HOST_SECRETS + : ["NEXTAUTH_SECRET", "DATABASE_ENCRYPTION_KEY"], + ); + if (insecureSecrets.length > 0) { + console.error(`🚨 ${formatInsecureSecrets(insecureSecrets)}`); + process.exit(1); + } + } + console.log("Waiting 5 seconds to run migrations"); // Function to trigger migrations with retry logic const triggerMigrations = async (retryCount = 0, maxRetries = 3) => { diff --git a/docker-compose.template.yml b/docker-compose.template.yml index 1360b3d8715..8a5d51b93a9 100644 --- a/docker-compose.template.yml +++ b/docker-compose.template.yml @@ -14,8 +14,8 @@ services: WEB_URL: http://localhost:3000 NEXTAUTH_URL: http://localhost:3000 # CHANGE THESE TO YOUR OWN VALUES - DATABASE_ENCRYPTION_KEY: c7a1e98e1e5e4cec5a3fbf5eaf502d262fb99807ae2be8ee70537409e29cb6f9 - NEXTAUTH_SECRET: c7a1e98e1e5e4cec5a3fbf5eaf502d262fb99807ae2be8ee70537409e29cb6f9 + DATABASE_ENCRYPTION_KEY: ${DATABASE_ENCRYPTION_KEY:?Set DATABASE_ENCRYPTION_KEY (run: openssl rand -hex 32)} + NEXTAUTH_SECRET: ${NEXTAUTH_SECRET:?Set NEXTAUTH_SECRET (run: openssl rand -hex 32)} # CHANGE THESE TO MATCH YOUR MINIO or S3 STORAGE SETUP CAP_AWS_ACCESS_KEY: capS3root CAP_AWS_SECRET_KEY: capS3root @@ -25,7 +25,7 @@ services: S3_INTERNAL_ENDPOINT: http://minio:3902 MEDIA_SERVER_URL: http://cap-media-server:3456 # CHANGE THESE TO YOUR OWN VALUES - MEDIA_SERVER_WEBHOOK_SECRET: fe337b52749070bb7b5d2c78cff9948439ea73cbc1869ba39d350e6c24db53b1 + MEDIA_SERVER_WEBHOOK_SECRET: ${MEDIA_SERVER_WEBHOOK_SECRET:?Set MEDIA_SERVER_WEBHOOK_SECRET (run: openssl rand -hex 32)} ports: - 3000:3000 @@ -57,7 +57,7 @@ services: restart: unless-stopped environment: PORT: 3456 - MEDIA_SERVER_WEBHOOK_SECRET: fe337b52749070bb7b5d2c78cff9948439ea73cbc1869ba39d350e6c24db53b1 + MEDIA_SERVER_WEBHOOK_SECRET: ${MEDIA_SERVER_WEBHOOK_SECRET:?Set MEDIA_SERVER_WEBHOOK_SECRET (run: openssl rand -hex 32)} ports: - "3456:3456" healthcheck: diff --git a/docker-compose.yml b/docker-compose.yml index 7759c6b638b..93ea56fccc7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,9 +14,9 @@ services: DATABASE_URL: mysql://cap:${MYSQL_PASSWORD:-cap-local-pwd-123}@mysql:3306/cap WEB_URL: ${CAP_URL:-http://localhost:3000} NEXTAUTH_URL: ${CAP_URL:-http://localhost:3000} - DATABASE_ENCRYPTION_KEY: ${DATABASE_ENCRYPTION_KEY:-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef} - NEXTAUTH_SECRET: ${NEXTAUTH_SECRET:-abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789} - MEDIA_SERVER_WEBHOOK_SECRET: ${MEDIA_SERVER_WEBHOOK_SECRET:-fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210} + DATABASE_ENCRYPTION_KEY: ${DATABASE_ENCRYPTION_KEY:?Set DATABASE_ENCRYPTION_KEY in .env (run: openssl rand -hex 32)} + NEXTAUTH_SECRET: ${NEXTAUTH_SECRET:?Set NEXTAUTH_SECRET in .env (run: openssl rand -hex 32)} + MEDIA_SERVER_WEBHOOK_SECRET: ${MEDIA_SERVER_WEBHOOK_SECRET:?Set MEDIA_SERVER_WEBHOOK_SECRET in .env (run: openssl rand -hex 32)} CAP_AWS_ACCESS_KEY: ${MINIO_ROOT_USER:-cap-admin} CAP_AWS_SECRET_KEY: ${MINIO_ROOT_PASSWORD:-cap-minio-pwd-456} CAP_AWS_BUCKET: cap @@ -51,7 +51,7 @@ services: restart: unless-stopped environment: PORT: 3456 - MEDIA_SERVER_WEBHOOK_SECRET: ${MEDIA_SERVER_WEBHOOK_SECRET:-fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210} + MEDIA_SERVER_WEBHOOK_SECRET: ${MEDIA_SERVER_WEBHOOK_SECRET:?Set MEDIA_SERVER_WEBHOOK_SECRET in .env (run: openssl rand -hex 32)} healthcheck: test: ["CMD", "bun", "-e", "fetch('http://127.0.0.1:3456/health').then((response) => process.exit(response.ok ? 0 : 1)).catch(() => process.exit(1))"] interval: 30s diff --git a/packages/env/index.ts b/packages/env/index.ts index 815f62a2ee8..e10c3b82b1b 100644 --- a/packages/env/index.ts +++ b/packages/env/index.ts @@ -1,2 +1,8 @@ export { buildEnv, NODE_ENV } from "./build.ts"; +export { + findInsecureSecrets, + formatInsecureSecrets, + KNOWN_PLACEHOLDER_SECRETS, + REQUIRED_SELF_HOST_SECRETS, +} from "./secrets.ts"; export { serverEnv } from "./server.ts"; diff --git a/packages/env/secrets.ts b/packages/env/secrets.ts new file mode 100644 index 00000000000..ab7ce8917cc --- /dev/null +++ b/packages/env/secrets.ts @@ -0,0 +1,52 @@ +export const REQUIRED_SELF_HOST_SECRETS = [ + "NEXTAUTH_SECRET", + "DATABASE_ENCRYPTION_KEY", + "MEDIA_SERVER_WEBHOOK_SECRET", +] as const; + +export type RequiredSelfHostSecret = + (typeof REQUIRED_SELF_HOST_SECRETS)[number]; + +export const KNOWN_PLACEHOLDER_SECRETS: ReadonlySet = new Set([ + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789", + "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210", + "c7a1e98e1e5e4cec5a3fbf5eaf502d262fb99807ae2be8ee70537409e29cb6f9", + "fe337b52749070bb7b5d2c78cff9948439ea73cbc1869ba39d350e6c24db53b1", + "local-media-server-secret", + "changeme", + "change-me", + "secret", +]); + +export type InsecureSecret = { + name: RequiredSelfHostSecret; + problem: "missing" | "placeholder"; +}; + +export function findInsecureSecrets( + env: Record, + names: readonly RequiredSelfHostSecret[] = REQUIRED_SELF_HOST_SECRETS, +): InsecureSecret[] { + const problems: InsecureSecret[] = []; + for (const name of names) { + const value = env[name]?.trim(); + if (!value) problems.push({ name, problem: "missing" }); + else if (KNOWN_PLACEHOLDER_SECRETS.has(value.toLowerCase())) + problems.push({ name, problem: "placeholder" }); + } + return problems; +} + +export function formatInsecureSecrets(problems: InsecureSecret[]): string { + const lines = problems.map( + ({ name, problem }) => + ` - ${name} is ${problem === "missing" ? "not set" : "set to a publicly known placeholder value"}`, + ); + return [ + "Refusing to start: insecure secrets.", + ...lines, + "Generate a unique value for each with `openssl rand -hex 32` and set it in your environment (.env / platform variables).", + "MEDIA_SERVER_WEBHOOK_SECRET must be identical on cap-web and the media server.", + ].join("\n"); +} diff --git a/packages/local-docker/docker-compose.yml b/packages/local-docker/docker-compose.yml index 5f250568836..a3239e05556 100644 --- a/packages/local-docker/docker-compose.yml +++ b/packages/local-docker/docker-compose.yml @@ -30,6 +30,7 @@ services: environment: PORT: 3456 MEDIA_SERVER_WEBHOOK_SECRET: ${MEDIA_SERVER_WEBHOOK_SECRET:-local-media-server-secret} + MEDIA_SERVER_ALLOW_INSECURE_SECRET: "true" # Local S3 Strorage minio: From 4db89f2541d9a198e9d7e39b69e41d4c826a76b1 Mon Sep 17 00:00:00 2001 From: 07rjain <07rjain@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:57:45 +0000 Subject: [PATCH 03/20] feat(self-host): check MEDIA_SERVER_URL reachability at startup and healthcheck cap-web on /api/status --- .../unit/media-server-startup-check.test.ts | 57 ++++++++++++ apps/web/instrumentation.node.ts | 6 ++ apps/web/lib/media-server-startup-check.ts | 91 +++++++++++++++++++ docker-compose.yml | 2 +- 4 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 apps/web/__tests__/unit/media-server-startup-check.test.ts create mode 100644 apps/web/lib/media-server-startup-check.ts diff --git a/apps/web/__tests__/unit/media-server-startup-check.test.ts b/apps/web/__tests__/unit/media-server-startup-check.test.ts new file mode 100644 index 00000000000..752c48d8f0f --- /dev/null +++ b/apps/web/__tests__/unit/media-server-startup-check.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it, vi } from "vitest"; +import { + checkMediaServerUrl, + parseMediaServerUrl, +} from "@/lib/media-server-startup-check"; + +describe("media server startup check", () => { + it("reports an unset URL without fetching", async () => { + const fetchImpl = vi.fn(); + expect(await checkMediaServerUrl(undefined, { fetchImpl })).toEqual({ + status: "unset", + }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it("rejects malformed URLs and non-http schemes", () => { + expect(parseMediaServerUrl("media-server:3456").ok).toBe(false); + expect(parseMediaServerUrl("not a url").ok).toBe(false); + expect(parseMediaServerUrl("http://media-server:3456").ok).toBe(true); + }); + + it("calls /health on the configured host", async () => { + const fetchImpl = vi.fn().mockResolvedValue(new Response("{}")); + const result = await checkMediaServerUrl("http://media-server:3456/", { + fetchImpl, + }); + expect(fetchImpl.mock.calls[0]?.[0]).toBe( + "http://media-server:3456/health", + ); + expect(result.status).toBe("ok"); + }); + + it("reports unreachable hosts instead of throwing", async () => { + const fetchImpl = vi.fn().mockRejectedValue( + new TypeError("fetch failed", { + cause: new Error("ENOTFOUND cap-media"), + }), + ); + const result = await checkMediaServerUrl("http://cap-media:3456", { + fetchImpl, + }); + expect(result).toMatchObject({ status: "unreachable" }); + expect(result.status === "unreachable" && result.message).toContain( + "ENOTFOUND", + ); + }); + + it("reports non-2xx health responses", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValue(new Response("down", { status: 503 })); + const result = await checkMediaServerUrl("http://media:3456", { + fetchImpl, + }); + expect(result.status).toBe("unreachable"); + }); +}); diff --git a/apps/web/instrumentation.node.ts b/apps/web/instrumentation.node.ts index 22c375768c3..14b68554fff 100644 --- a/apps/web/instrumentation.node.ts +++ b/apps/web/instrumentation.node.ts @@ -16,6 +16,7 @@ import { REQUIRED_SELF_HOST_SECRETS, serverEnv, } from "@cap/env"; +import { logMediaServerUrlCheck } from "./lib/media-server-startup-check"; export async function register() { if (process.env.NEXT_PUBLIC_IS_CAP) return; @@ -57,6 +58,11 @@ export async function register() { // Add a timeout to trigger migrations after 5 seconds on server start setTimeout(() => triggerMigrations(), 5000); setTimeout(() => createS3Bucket(), 5000); + setTimeout(() => { + void logMediaServerUrlCheck().catch((error) => { + console.warn("⚠️ Media server check failed:", error); + }); + }, 5000); } async function createS3Bucket() { diff --git a/apps/web/lib/media-server-startup-check.ts b/apps/web/lib/media-server-startup-check.ts new file mode 100644 index 00000000000..7350da36666 --- /dev/null +++ b/apps/web/lib/media-server-startup-check.ts @@ -0,0 +1,91 @@ +export type MediaServerUrlCheck = + | { status: "unset" } + | { status: "invalid"; message: string } + | { status: "unreachable"; url: string; message: string } + | { status: "ok"; url: string }; + +export function parseMediaServerUrl( + raw: string, +): { ok: true; url: URL } | { ok: false; message: string } { + let url: URL; + try { + url = new URL(raw); + } catch { + return { + ok: false, + message: `MEDIA_SERVER_URL "${raw}" is not a valid URL (expected e.g. http://media-server:3456)`, + }; + } + if (url.protocol !== "http:" && url.protocol !== "https:") { + return { + ok: false, + message: `MEDIA_SERVER_URL must use http or https, got "${url.protocol}"`, + }; + } + if (!url.hostname) { + return { ok: false, message: "MEDIA_SERVER_URL has no host" }; + } + return { ok: true, url }; +} + +export async function checkMediaServerUrl( + raw: string | undefined, + options: { timeoutMs?: number; fetchImpl?: typeof fetch } = {}, +): Promise { + if (!raw?.trim()) return { status: "unset" }; + const parsed = parseMediaServerUrl(raw.trim()); + if (!parsed.ok) return { status: "invalid", message: parsed.message }; + + const healthUrl = new URL("/health", parsed.url).toString(); + const fetchImpl = options.fetchImpl ?? fetch; + try { + const response = await fetchImpl(healthUrl, { + signal: AbortSignal.timeout(options.timeoutMs ?? 3000), + }); + if (!response.ok) { + return { + status: "unreachable", + url: healthUrl, + message: `${healthUrl} returned HTTP ${response.status}`, + }; + } + return { status: "ok", url: healthUrl }; + } catch (error) { + const reason = + error instanceof Error + ? (error.cause instanceof Error ? error.cause.message : null) || + error.message + : String(error); + return { + status: "unreachable", + url: healthUrl, + message: `${healthUrl} is not reachable (${reason})`, + }; + } +} + +export async function logMediaServerUrlCheck( + raw: string | undefined = process.env.MEDIA_SERVER_URL, +) { + const result = await checkMediaServerUrl(raw); + switch (result.status) { + case "unset": + console.warn( + "⚠️ MEDIA_SERVER_URL is not set: video conversion, thumbnails and audio processing are disabled.", + ); + return result; + case "invalid": + console.warn( + `⚠️ ${result.message}. Video processing will fail until it is fixed.`, + ); + return result; + case "unreachable": + console.warn( + `⚠️ Media server check failed: ${result.message}. Check that the media server is running and that MEDIA_SERVER_URL points at its service name/private host.`, + ); + return result; + case "ok": + console.log(`✅ Media server reachable at ${result.url}`); + return result; + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 93ea56fccc7..10010f5c8ae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,7 +35,7 @@ services: ports: - "${CAP_PORT:-3000}:3000" healthcheck: - test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:3000/"] + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:3000/api/status"] interval: 30s timeout: 10s retries: 3 From 0e1a7d1efa8263a3d87db2be95df113d14128f2c Mon Sep 17 00:00:00 2001 From: 07rjain <07rjain@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:58:07 +0000 Subject: [PATCH 04/20] perf(media-server): size direct conversion concurrency from available CPUs instead of 1 --- .../src/__tests__/lib/video-capacity.test.ts | 16 ++++++++++++++++ apps/media-server/src/lib/video-capacity.ts | 19 +++++++++++++++++-- apps/web/content/docs/self-hosting.mdx | 9 +++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 apps/media-server/src/__tests__/lib/video-capacity.test.ts diff --git a/apps/media-server/src/__tests__/lib/video-capacity.test.ts b/apps/media-server/src/__tests__/lib/video-capacity.test.ts new file mode 100644 index 00000000000..d73be931803 --- /dev/null +++ b/apps/media-server/src/__tests__/lib/video-capacity.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, test } from "bun:test"; +import { getDefaultMaxConcurrentDirectVideoProcesses } from "../../lib/video-capacity"; + +describe("getDefaultMaxConcurrentDirectVideoProcesses", () => { + test("keeps one conversion on small machines", () => { + expect(getDefaultMaxConcurrentDirectVideoProcesses(1)).toBe(1); + expect(getDefaultMaxConcurrentDirectVideoProcesses(2)).toBe(1); + expect(getDefaultMaxConcurrentDirectVideoProcesses(0.5)).toBe(1); + }); + + test("uses half the CPUs, capped at four", () => { + expect(getDefaultMaxConcurrentDirectVideoProcesses(4)).toBe(2); + expect(getDefaultMaxConcurrentDirectVideoProcesses(8)).toBe(4); + expect(getDefaultMaxConcurrentDirectVideoProcesses(32)).toBe(4); + }); +}); diff --git a/apps/media-server/src/lib/video-capacity.ts b/apps/media-server/src/lib/video-capacity.ts index da08f193524..97fbebeb365 100644 --- a/apps/media-server/src/lib/video-capacity.ts +++ b/apps/media-server/src/lib/video-capacity.ts @@ -1,3 +1,6 @@ +import os from "node:os"; +import { getContainerCpuLimit } from "./container-cpu"; + export interface VideoProcessSlot { release: () => void; } @@ -7,7 +10,19 @@ const configuredMaxDirectVideoProcesses = process.env.MEDIA_SERVER_MAX_CONCURRENT_DIRECT_CONVERSIONS ?? "0", 10, ) || 0; -const DEFAULT_MAX_CONCURRENT_DIRECT_VIDEO_PROCESSES = 1; +const MAX_DEFAULT_DIRECT_VIDEO_PROCESSES = 4; + +export function getDefaultMaxConcurrentDirectVideoProcesses( + cpuCapacity: number = getContainerCpuLimit() || os.availableParallelism(), +): number { + return Math.max( + 1, + Math.min(MAX_DEFAULT_DIRECT_VIDEO_PROCESSES, Math.floor(cpuCapacity / 2)), + ); +} + +const defaultMaxDirectVideoProcesses = + getDefaultMaxConcurrentDirectVideoProcesses(); let activeDirectVideoProcessCount = 0; @@ -18,7 +33,7 @@ export function getActiveDirectVideoProcessCount(): number { export function getMaxConcurrentDirectVideoProcesses(): number { return configuredMaxDirectVideoProcesses > 0 ? configuredMaxDirectVideoProcesses - : DEFAULT_MAX_CONCURRENT_DIRECT_VIDEO_PROCESSES; + : defaultMaxDirectVideoProcesses; } export function tryAcquireDirectVideoProcessSlot( diff --git a/apps/web/content/docs/self-hosting.mdx b/apps/web/content/docs/self-hosting.mdx index eda684972de..314ec3db75e 100644 --- a/apps/web/content/docs/self-hosting.mdx +++ b/apps/web/content/docs/self-hosting.mdx @@ -114,6 +114,15 @@ Email is optional. Without it, login codes are printed to the cap-web logs, the ``` 3. Verify the domain in Resend. Login emails are sent from `auth@RESEND_FROM_DOMAIN`. +## Media Server Tuning + +The media server converts uploads in parallel based on the CPUs it can use: half the available CPUs, at least 1 and at most 4. It still turns work away when CPU or memory pressure is high. Override this with: + +```bash +MEDIA_SERVER_MAX_CONCURRENT_DIRECT_CONVERSIONS=2 # direct /video/convert requests +MEDIA_SERVER_MAX_CONCURRENT_VIDEO_PROCESSES=4 # all video jobs +``` + ## Optional Features ### AI Features From 7ca66e69e9480a282641e8e78c3d77d41989582a Mon Sep 17 00:00:00 2001 From: 07rjain <07rjain@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:58:50 +0000 Subject: [PATCH 05/20] feat(media-server): drain active jobs on SIGTERM before aborting and report draining on /health --- apps/media-server/src/index.ts | 25 ++++++++++++- apps/media-server/src/lib/job-manager.ts | 22 ++++++++++++ apps/media-server/src/routes/health.ts | 46 ++++++++++++++---------- docker-compose.yml | 1 + 4 files changed, 74 insertions(+), 20 deletions(-) diff --git a/apps/media-server/src/index.ts b/apps/media-server/src/index.ts index 81f5fca3ac0..487b4331743 100644 --- a/apps/media-server/src/index.ts +++ b/apps/media-server/src/index.ts @@ -1,5 +1,10 @@ import app from "./app"; -import { abortAllJobs } from "./lib/job-manager"; +import { + abortAllJobs, + beginDraining, + getActiveVideoProcessCount, + waitForActiveVideoProcesses, +} from "./lib/job-manager"; import { cancelAllMediaOperations } from "./lib/media-operations"; import { cleanupMediaTransferCache } from "./lib/media-transfer"; import { assertSecureStartup } from "./lib/startup-checks"; @@ -7,6 +12,11 @@ import { assertSecureStartup } from "./lib/startup-checks"; assertSecureStartup(); const port = Number(process.env.PORT) || 3456; +const shutdownGraceMs = Math.max( + 0, + Number.parseInt(process.env.MEDIA_SERVER_SHUTDOWN_GRACE_MS ?? "25000", 10) || + 0, +); console.log(`[media-server] Starting on port ${port}`); @@ -15,6 +25,19 @@ let shuttingDown = false; const shutdown = async () => { if (shuttingDown) return; shuttingDown = true; + beginDraining(); + const active = getActiveVideoProcessCount(); + if (active > 0 && shutdownGraceMs > 0) { + console.log( + `[media-server] Draining: waiting up to ${Math.round(shutdownGraceMs / 1000)}s for ${active} active video job(s); new work is rejected with SERVER_BUSY`, + ); + const remaining = await waitForActiveVideoProcesses(shutdownGraceMs); + console.log( + remaining === 0 + ? "[media-server] All active jobs finished" + : `[media-server] Grace period over with ${remaining} job(s) still running`, + ); + } console.log("[media-server] Shutting down..."); const abortedJobs = await abortAllJobs(); if (abortedJobs > 0) { diff --git a/apps/media-server/src/lib/job-manager.ts b/apps/media-server/src/lib/job-manager.ts index 74caa0deaa7..7a87a6e0cc0 100644 --- a/apps/media-server/src/lib/job-manager.ts +++ b/apps/media-server/src/lib/job-manager.ts @@ -329,9 +329,31 @@ export function hasCriticalMemoryPressure(): boolean { return getSystemResources().memoryPressure >= MEMORY_REJECT_THRESHOLD; } +let draining = false; + +export function beginDraining() { + draining = true; +} + +export function isDraining(): boolean { + return draining; +} + +export async function waitForActiveVideoProcesses( + timeoutMs: number, + pollMs = 250, +): Promise { + const deadline = Date.now() + timeoutMs; + while (getActiveVideoProcessCount() > 0 && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, pollMs)); + } + return getActiveVideoProcessCount(); +} + export function canAcceptNewVideoProcess( priority: "interactive" | "recovery" = "interactive", ): boolean { + if (draining) return false; const active = getActiveVideoProcessCount(); const resources = getSystemResources(); return ( diff --git a/apps/media-server/src/routes/health.ts b/apps/media-server/src/routes/health.ts index 07b2731ab43..fbc5e82123c 100644 --- a/apps/media-server/src/routes/health.ts +++ b/apps/media-server/src/routes/health.ts @@ -1,6 +1,6 @@ import os from "node:os"; import { Hono } from "hono"; -import { getSystemResources } from "../lib/job-manager"; +import { getSystemResources, isDraining } from "../lib/job-manager"; import { getMediaEngineStatus } from "../lib/media-engine"; const health = new Hono(); @@ -17,25 +17,33 @@ health.get("/", (c) => { ); const resources = getSystemResources(); - return c.json({ - status: - mediaEngine.available && resources.effectiveMax > 0 ? "ok" : "degraded", - mediaEngine, - ["ff" + "mpeg"]: { - available: mediaEngine.available, - version: mediaEngine.version, - }, - system: { - ...resources, - loadAvg5m, - loadAvg15m, - totalMemoryMB, - freeMemoryMB, - memoryUsagePercent, - uptimeSeconds: Math.round(os.uptime()), - processUptimeSeconds: Math.round(process.uptime()), + const draining = isDraining(); + + return c.json( + { + status: draining + ? "draining" + : mediaEngine.available && resources.effectiveMax > 0 + ? "ok" + : "degraded", + mediaEngine, + ["ff" + "mpeg"]: { + available: mediaEngine.available, + version: mediaEngine.version, + }, + system: { + ...resources, + loadAvg5m, + loadAvg15m, + totalMemoryMB, + freeMemoryMB, + memoryUsagePercent, + uptimeSeconds: Math.round(os.uptime()), + processUptimeSeconds: Math.round(process.uptime()), + }, }, - }); + draining ? 503 : 200, + ); }); export default health; diff --git a/docker-compose.yml b/docker-compose.yml index 10010f5c8ae..fb2a252944f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,6 +49,7 @@ services: context: ./apps/media-server dockerfile: Dockerfile.standalone restart: unless-stopped + stop_grace_period: 30s environment: PORT: 3456 MEDIA_SERVER_WEBHOOK_SECRET: ${MEDIA_SERVER_WEBHOOK_SECRET:?Set MEDIA_SERVER_WEBHOOK_SECRET in .env (run: openssl rand -hex 32)} From 51770779c84583c991bdceffd5849caff77ac10c Mon Sep 17 00:00:00 2001 From: 07rjain <07rjain@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:13:27 +0000 Subject: [PATCH 06/20] fix(media-server): block private, metadata and non-http URLs with an optional host allow-list and compare the secret in constant time --- apps/media-server/bunfig.toml | 2 + .../src/__tests__/lib/url-policy.test.ts | 117 ++++++++++ apps/media-server/src/__tests__/setup.ts | 1 + apps/media-server/src/app.ts | 3 + apps/media-server/src/lib/auth.ts | 8 +- apps/media-server/src/lib/url-policy.ts | 216 ++++++++++++++++++ apps/web/content/docs/self-hosting.mdx | 11 + docker-compose.coolify.yml | 3 + docker-compose.template.yml | 2 + docker-compose.yml | 3 + packages/local-docker/docker-compose.yml | 1 + 11 files changed, 366 insertions(+), 1 deletion(-) create mode 100644 apps/media-server/bunfig.toml create mode 100644 apps/media-server/src/__tests__/lib/url-policy.test.ts create mode 100644 apps/media-server/src/__tests__/setup.ts create mode 100644 apps/media-server/src/lib/url-policy.ts diff --git a/apps/media-server/bunfig.toml b/apps/media-server/bunfig.toml new file mode 100644 index 00000000000..37ff1f94820 --- /dev/null +++ b/apps/media-server/bunfig.toml @@ -0,0 +1,2 @@ +[test] +preload = ["./src/__tests__/setup.ts"] diff --git a/apps/media-server/src/__tests__/lib/url-policy.test.ts b/apps/media-server/src/__tests__/lib/url-policy.test.ts new file mode 100644 index 00000000000..cc94510d735 --- /dev/null +++ b/apps/media-server/src/__tests__/lib/url-policy.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, test } from "bun:test"; +import { + assertUrlAllowed, + collectUrls, + getUrlPolicy, + isPrivateAddress, +} from "../../lib/url-policy"; + +const resolveTo = + (...addresses: string[]) => + async () => + addresses; + +describe("isPrivateAddress", () => { + test("flags loopback, private, link-local, CGNAT and ULA addresses", () => { + for (const address of [ + "127.0.0.1", + "10.1.2.3", + "172.20.0.5", + "192.168.1.10", + "169.254.169.254", + "100.100.1.1", + "::1", + "fd12:3456::1", + "fe80::1", + "::ffff:127.0.0.1", + ]) { + expect(isPrivateAddress(address)).toBe(true); + } + }); + + test("allows public addresses", () => { + expect(isPrivateAddress("93.184.216.34")).toBe(false); + expect(isPrivateAddress("2606:4700::6810:85e5")).toBe(false); + }); +}); + +describe("assertUrlAllowed", () => { + const open = getUrlPolicy({}); + + test("blocks cloud metadata and private hosts by default", async () => { + await expect( + assertUrlAllowed("http://169.254.169.254/latest/meta-data/", open), + ).rejects.toThrow("private address"); + await expect( + assertUrlAllowed( + "http://internal.example/x", + open, + resolveTo("10.0.0.7"), + ), + ).rejects.toThrow("private address"); + }); + + test("blocks non-http schemes", async () => { + await expect(assertUrlAllowed("file:///etc/passwd", open)).rejects.toThrow( + "scheme", + ); + }); + + test("allows public hosts when no allow-list is configured", async () => { + await expect( + assertUrlAllowed( + "https://bucket.s3.amazonaws.com/v.mp4", + open, + resolveTo("52.216.1.1"), + ), + ).resolves.toBeUndefined(); + }); + + test("allows the configured S3 endpoint on a private network", async () => { + const policy = getUrlPolicy({ S3_INTERNAL_ENDPOINT: "http://minio:9000" }); + await expect( + assertUrlAllowed( + "http://minio:9000/cap/v.mp4", + policy, + resolveTo("172.18.0.3"), + ), + ).resolves.toBeUndefined(); + }); + + test("enforces MEDIA_SERVER_ALLOWED_HOSTS when set", async () => { + const policy = getUrlPolicy({ + MEDIA_SERVER_ALLOWED_HOSTS: "*.r2.cloudflarestorage.com", + MEDIA_SERVER_WEBHOOK_URL: "http://cap-web:3000", + }); + await expect( + assertUrlAllowed( + "https://acct.r2.cloudflarestorage.com/v.mp4", + policy, + resolveTo("104.18.1.1"), + ), + ).resolves.toBeUndefined(); + await expect( + assertUrlAllowed("http://cap-web:3000/api/webhooks", policy), + ).resolves.toBeUndefined(); + await expect( + assertUrlAllowed( + "https://evil.example/v.mp4", + policy, + resolveTo("1.1.1.1"), + ), + ).rejects.toThrow("MEDIA_SERVER_ALLOWED_HOSTS"); + }); +}); + +describe("collectUrls", () => { + test("collects nested url fields only", () => { + expect( + collectUrls({ + videoUrl: "http://a/1", + name: "http://not-a-url-field", + segments: [{ url: "http://b/2" }], + videoSegmentUrls: ["http://c/3"], + }), + ).toEqual(["http://a/1", "http://b/2", "http://c/3"]); + }); +}); diff --git a/apps/media-server/src/__tests__/setup.ts b/apps/media-server/src/__tests__/setup.ts new file mode 100644 index 00000000000..8f6bb606ef7 --- /dev/null +++ b/apps/media-server/src/__tests__/setup.ts @@ -0,0 +1 @@ +process.env.MEDIA_SERVER_ALLOW_PRIVATE_URLS ??= "true"; diff --git a/apps/media-server/src/app.ts b/apps/media-server/src/app.ts index e50979955ad..a78d57fee22 100644 --- a/apps/media-server/src/app.ts +++ b/apps/media-server/src/app.ts @@ -1,5 +1,6 @@ import { Hono } from "hono"; import { logger } from "hono/logger"; +import { urlPolicyMiddleware } from "./lib/url-policy"; import audio from "./routes/audio"; import health from "./routes/health"; import video from "./routes/video"; @@ -7,6 +8,8 @@ import video from "./routes/video"; const app = new Hono(); app.use("*", logger()); +app.use("/audio/*", urlPolicyMiddleware); +app.use("/video/*", urlPolicyMiddleware); app.route("/health", health); app.route("/audio", audio); diff --git a/apps/media-server/src/lib/auth.ts b/apps/media-server/src/lib/auth.ts index a8e322d2861..aaeae02a425 100644 --- a/apps/media-server/src/lib/auth.ts +++ b/apps/media-server/src/lib/auth.ts @@ -1,3 +1,5 @@ +import { createHash, timingSafeEqual } from "node:crypto"; + type MediaServerContext = { req: { header: (name: string) => string | undefined }; }; @@ -10,5 +12,9 @@ export function validateMediaServerSecret(c: MediaServerContext): boolean { ); return false; } - return c.req.header("x-media-server-secret") === secret; + const provided = c.req.header("x-media-server-secret"); + if (!provided) return false; + const expectedDigest = createHash("sha256").update(secret).digest(); + const providedDigest = createHash("sha256").update(provided).digest(); + return timingSafeEqual(expectedDigest, providedDigest); } diff --git a/apps/media-server/src/lib/url-policy.ts b/apps/media-server/src/lib/url-policy.ts new file mode 100644 index 00000000000..75ff4778b09 --- /dev/null +++ b/apps/media-server/src/lib/url-policy.ts @@ -0,0 +1,216 @@ +import { lookup } from "node:dns/promises"; +import { BlockList, isIP } from "node:net"; +import type { MiddlewareHandler } from "hono"; +import { validateMediaServerSecret } from "./auth"; + +const blockedRanges = new BlockList(); +for (const [network, prefix] of [ + ["0.0.0.0", 8], + ["10.0.0.0", 8], + ["100.64.0.0", 10], + ["127.0.0.0", 8], + ["169.254.0.0", 16], + ["172.16.0.0", 12], + ["192.0.0.0", 24], + ["192.168.0.0", 16], + ["198.18.0.0", 15], + ["224.0.0.0", 4], + ["240.0.0.0", 4], +] as const) { + blockedRanges.addSubnet(network, prefix, "ipv4"); +} +for (const [network, prefix] of [ + ["::", 128], + ["::1", 128], + ["fc00::", 7], + ["fe80::", 10], + ["ff00::", 8], +] as const) { + blockedRanges.addSubnet(network, prefix, "ipv6"); +} + +export function isPrivateAddress(address: string): boolean { + const family = isIP(address); + if (family === 4) return blockedRanges.check(address, "ipv4"); + if (family === 6) { + const mapped = address.toLowerCase().match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/); + if (mapped?.[1]) return blockedRanges.check(mapped[1], "ipv4"); + return blockedRanges.check(address, "ipv6"); + } + return true; +} + +function hostOf(value: string | undefined): string | null { + if (!value?.trim()) return null; + try { + return new URL(value.trim()).hostname.toLowerCase(); + } catch { + return null; + } +} + +function normalizeHost(host: string): string { + return host + .trim() + .toLowerCase() + .replace(/^\[|\]$/g, ""); +} + +export type UrlPolicy = { + strictHosts: boolean; + allowedHosts: string[]; + allowPrivate: boolean; +}; + +export function getUrlPolicy( + env: Record = process.env, +): UrlPolicy { + const explicit = (env.MEDIA_SERVER_ALLOWED_HOSTS ?? "") + .split(",") + .map(normalizeHost) + .filter(Boolean); + const implicit = [ + env.S3_INTERNAL_ENDPOINT, + env.S3_PUBLIC_ENDPOINT, + env.CAP_AWS_ENDPOINT, + env.MEDIA_SERVER_WEBHOOK_URL, + env.WEB_URL, + ] + .map(hostOf) + .filter((host): host is string => Boolean(host)); + return { + strictHosts: explicit.length > 0, + allowedHosts: [...new Set([...explicit, ...implicit])], + allowPrivate: env.MEDIA_SERVER_ALLOW_PRIVATE_URLS === "true", + }; +} + +function hostMatches(host: string, pattern: string): boolean { + if (pattern.startsWith("*.")) { + const suffix = pattern.slice(1); + return host.endsWith(suffix) && host.length > suffix.length; + } + return host === pattern; +} + +export class BlockedUrlError extends Error { + constructor( + readonly url: string, + readonly reason: string, + ) { + super(`URL not allowed (${reason}): ${redactUrl(url)}`); + this.name = "BlockedUrlError"; + } +} + +function redactUrl(value: string): string { + try { + const url = new URL(value); + return `${url.protocol}//${url.host}${url.pathname}`; + } catch { + return ""; + } +} + +type Resolver = (host: string) => Promise; + +const defaultResolver: Resolver = async (host) => + (await lookup(host, { all: true, verbatim: true })).map( + (entry) => entry.address, + ); + +export async function assertUrlAllowed( + value: string, + policy: UrlPolicy = getUrlPolicy(), + resolve: Resolver = defaultResolver, +): Promise { + let url: URL; + try { + url = new URL(value); + } catch { + throw new BlockedUrlError(value, "invalid URL"); + } + if (url.protocol === "file:" && policy.allowPrivate) return; + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new BlockedUrlError(value, `scheme ${url.protocol} is not allowed`); + } + const host = normalizeHost(url.hostname); + const explicitlyAllowed = policy.allowedHosts.some((pattern) => + hostMatches(host, pattern), + ); + if (policy.strictHosts && !explicitlyAllowed) { + throw new BlockedUrlError( + value, + `host ${host} is not in MEDIA_SERVER_ALLOWED_HOSTS`, + ); + } + if (explicitlyAllowed || policy.allowPrivate) return; + + let addresses: string[]; + try { + addresses = isIP(host) ? [host] : await resolve(host); + } catch { + throw new BlockedUrlError(value, `host ${host} does not resolve`); + } + const privateAddress = addresses.find(isPrivateAddress); + if (addresses.length === 0 || privateAddress) { + throw new BlockedUrlError( + value, + `host ${host} resolves to private address ${privateAddress ?? "none"}; add it to MEDIA_SERVER_ALLOWED_HOSTS if this is your storage or Cap web host`, + ); + } +} + +const URL_KEY = /urls?$/i; + +export function collectUrls(input: unknown, key = ""): string[] { + if (typeof input === "string") { + return URL_KEY.test(key) && /^[a-z][a-z0-9+.-]*:/i.test(input) + ? [input] + : []; + } + if (Array.isArray(input)) { + return input.flatMap((item) => collectUrls(item, key)); + } + if (input && typeof input === "object") { + return Object.entries(input).flatMap(([childKey, value]) => + collectUrls(value, childKey), + ); + } + return []; +} + +export const urlPolicyMiddleware: MiddlewareHandler = async (c, next) => { + if ( + c.req.method !== "POST" || + !c.req.header("content-type")?.includes("application/json") || + !validateMediaServerSecret(c) + ) { + return next(); + } + let body: unknown; + try { + body = await c.req.json(); + } catch { + return next(); + } + const policy = getUrlPolicy(); + const urls = [...new Set(collectUrls(body))]; + try { + await Promise.all(urls.map((url) => assertUrlAllowed(url, policy))); + } catch (error) { + if (error instanceof BlockedUrlError) { + console.warn(`[media-server] ${error.message}`); + return c.json( + { + error: "URL not allowed", + code: "URL_NOT_ALLOWED", + details: error.message, + }, + 400, + ); + } + throw error; + } + return next(); +}; diff --git a/apps/web/content/docs/self-hosting.mdx b/apps/web/content/docs/self-hosting.mdx index 314ec3db75e..8eb2ff31344 100644 --- a/apps/web/content/docs/self-hosting.mdx +++ b/apps/web/content/docs/self-hosting.mdx @@ -123,6 +123,17 @@ MEDIA_SERVER_MAX_CONCURRENT_DIRECT_CONVERSIONS=2 # direct /video/convert reque MEDIA_SERVER_MAX_CONCURRENT_VIDEO_PROCESSES=4 # all video jobs ``` +### Media server URL policy + +The media server only fetches the URLs cap-web sends it, and it checks each one first: + +- Only `http` and `https` URLs are accepted. +- Hosts that resolve to loopback, private, link-local (including cloud metadata at `169.254.169.254`), CGNAT or IPv6 ULA addresses are rejected, unless the host is trusted. +- Trusted hosts are those in `S3_INTERNAL_ENDPOINT`, `S3_PUBLIC_ENDPOINT`, `CAP_AWS_ENDPOINT`, `MEDIA_SERVER_WEBHOOK_URL` and `WEB_URL` (set them on the media server too), plus anything in `MEDIA_SERVER_ALLOWED_HOSTS`. +- When `MEDIA_SERVER_ALLOWED_HOSTS` is set, it becomes a strict allow-list: every URL must use one of those hosts or the trusted hosts above. Entries are comma-separated and may use a `*.` prefix, e.g. `minio.railway.internal,*.r2.cloudflarestorage.com`. + +If your storage lives on a private network (MinIO in Docker or on Railway's private network), set `S3_INTERNAL_ENDPOINT` on the media server or add its host to `MEDIA_SERVER_ALLOWED_HOSTS`. Otherwise processing fails with `URL_NOT_ALLOWED`. `MEDIA_SERVER_ALLOW_PRIVATE_URLS=true` turns the private-address check off and also allows `file:` URLs; it's meant for local development and tests only. + ## Optional Features ### AI Features diff --git a/docker-compose.coolify.yml b/docker-compose.coolify.yml index 539d245f2ec..30fbe4eaa84 100644 --- a/docker-compose.coolify.yml +++ b/docker-compose.coolify.yml @@ -48,6 +48,9 @@ services: restart: unless-stopped environment: PORT: 3456 + S3_INTERNAL_ENDPOINT: 'http://minio:9000' + S3_PUBLIC_ENDPOINT: '${SERVICE_URL_MINIO}' + MEDIA_SERVER_WEBHOOK_URL: 'http://cap-web:3000' MEDIA_SERVER_WEBHOOK_SECRET: '${SERVICE_HEX_32_MEDIASERVER}' healthcheck: test: diff --git a/docker-compose.template.yml b/docker-compose.template.yml index 8a5d51b93a9..f4046a1690c 100644 --- a/docker-compose.template.yml +++ b/docker-compose.template.yml @@ -56,6 +56,8 @@ services: dockerfile: Dockerfile.standalone restart: unless-stopped environment: + S3_INTERNAL_ENDPOINT: http://minio:3902 + MEDIA_SERVER_WEBHOOK_URL: http://cap-web:3000 PORT: 3456 MEDIA_SERVER_WEBHOOK_SECRET: ${MEDIA_SERVER_WEBHOOK_SECRET:?Set MEDIA_SERVER_WEBHOOK_SECRET (run: openssl rand -hex 32)} ports: diff --git a/docker-compose.yml b/docker-compose.yml index fb2a252944f..a2e063eb886 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,6 +52,9 @@ services: stop_grace_period: 30s environment: PORT: 3456 + S3_INTERNAL_ENDPOINT: http://minio:9000 + S3_PUBLIC_ENDPOINT: ${S3_PUBLIC_URL:-http://localhost:9000} + MEDIA_SERVER_WEBHOOK_URL: http://cap-web:3000 MEDIA_SERVER_WEBHOOK_SECRET: ${MEDIA_SERVER_WEBHOOK_SECRET:?Set MEDIA_SERVER_WEBHOOK_SECRET in .env (run: openssl rand -hex 32)} healthcheck: test: ["CMD", "bun", "-e", "fetch('http://127.0.0.1:3456/health').then((response) => process.exit(response.ok ? 0 : 1)).catch(() => process.exit(1))"] diff --git a/packages/local-docker/docker-compose.yml b/packages/local-docker/docker-compose.yml index a3239e05556..a8987a6ba95 100644 --- a/packages/local-docker/docker-compose.yml +++ b/packages/local-docker/docker-compose.yml @@ -31,6 +31,7 @@ services: PORT: 3456 MEDIA_SERVER_WEBHOOK_SECRET: ${MEDIA_SERVER_WEBHOOK_SECRET:-local-media-server-secret} MEDIA_SERVER_ALLOW_INSECURE_SECRET: "true" + MEDIA_SERVER_ALLOW_PRIVATE_URLS: "true" # Local S3 Strorage minio: From ea9f597da88de7db3ee66719afdb168b22f3e254 Mon Sep 17 00:00:00 2001 From: 07rjain <07rjain@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:16:19 +0000 Subject: [PATCH 07/20] perf(embed): code-split the players and upload tracker and drop the duplicate framer-motion runtime --- .../unit/embed-video-playback-chrome.test.ts | 28 +++++++++++- .../[videoId]/_components/EmbedVideo.tsx | 43 +++++++++++++++---- 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/apps/web/__tests__/unit/embed-video-playback-chrome.test.ts b/apps/web/__tests__/unit/embed-video-playback-chrome.test.ts index fac9bc0aa5c..8fc144009bd 100644 --- a/apps/web/__tests__/unit/embed-video-playback-chrome.test.ts +++ b/apps/web/__tests__/unit/embed-video-playback-chrome.test.ts @@ -3,6 +3,7 @@ import { act, type ComponentProps, + type ComponentType, createElement, type MouseEventHandler, type ReactNode, @@ -24,6 +25,27 @@ import { type SharePageBrandingInput, } from "@/lib/share-branding"; +const dynamicImports = vi.hoisted(() => [] as Promise[]); + +vi.mock("next/dynamic", async () => { + const { createElement } = await import("react"); + return { + default: ( + loader: () => Promise< + ComponentType | { default: ComponentType } + >, + ) => { + let Loaded: ComponentType | null = null; + dynamicImports.push( + loader().then((module) => { + Loaded = "default" in module ? module.default : module; + }), + ); + return (props: object) => (Loaded ? createElement(Loaded, props) : null); + }, + }; +}); + vi.mock("@cap/env", () => ({ NODE_ENV: "test" })); vi.mock("@cap/ui", async () => { @@ -34,7 +56,7 @@ vi.mock("@cap/ui", async () => { }; }); -vi.mock("framer-motion", async () => { +vi.mock("motion/react", async () => { const { createElement } = await import("react"); type MotionProps = { children?: ReactNode; @@ -176,6 +198,10 @@ const expectChromeHidden = (container: HTMLElement) => { expect(container.querySelector('[aria-label="Powered by Cap"]')).toBeNull(); }; +beforeAll(async () => { + await Promise.all(dynamicImports); +}); + describe("EmbedVideo playback chrome", () => { beforeAll(() => { actEnvironment.IS_REACT_ACT_ENVIRONMENT = true; diff --git a/apps/web/app/embed/[videoId]/_components/EmbedVideo.tsx b/apps/web/app/embed/[videoId]/_components/EmbedVideo.tsx index 4339c337f2a..f31542952e8 100644 --- a/apps/web/app/embed/[videoId]/_components/EmbedVideo.tsx +++ b/apps/web/app/embed/[videoId]/_components/EmbedVideo.tsx @@ -5,8 +5,9 @@ import type { comments as commentsSchema, videos } from "@cap/database/schema"; import { NODE_ENV } from "@cap/env"; import { Avatar, Logo } from "@cap/ui"; import type { ViewerSettings } from "@cap/web-backend"; -import { AnimatePresence, motion } from "framer-motion"; import { useTranscript } from "hooks/use-transcript"; +import { AnimatePresence, motion } from "motion/react"; +import dynamic from "next/dynamic"; import Image from "next/image"; import { forwardRef, @@ -15,13 +16,11 @@ import { useRef, useState, } from "react"; -import { CapVideoPlayer } from "@/app/s/[videoId]/_components/CapVideoPlayer"; -import { HLSVideoPlayer } from "@/app/s/[videoId]/_components/HLSVideoPlayer"; -import { useUploadProgress } from "@/app/s/[videoId]/_components/ProgressCircle"; import { PreparingVideoOverlay, RecordingInProgressOverlay, } from "@/app/s/[videoId]/_components/RecordingInProgress"; +import type { UploadProgress } from "@/app/s/[videoId]/_components/upload-progress"; import { formatChaptersAsVTT, formatTranscriptAsVTT, @@ -32,6 +31,21 @@ import type { SharePageBranding } from "@/lib/share-branding"; import type { ShareCallToAction } from "@/lib/share-call-to-action"; import { usePlayerJsReceiver } from "./use-player-js-receiver"; +const CapVideoPlayer = dynamic(() => + import("@/app/s/[videoId]/_components/CapVideoPlayer").then( + (m) => m.CapVideoPlayer, + ), +); +const UploadProgressTracker = dynamic( + () => import("@/app/s/[videoId]/_components/UploadProgressTracker"), + { ssr: false }, +); +const HLSVideoPlayer = dynamic(() => + import("@/app/s/[videoId]/_components/HLSVideoPlayer").then( + (m) => m.HLSVideoPlayer, + ), +); + declare global { interface Window { MSStream: unknown; @@ -103,10 +117,17 @@ export const EmbedVideo = forwardRef< ); const [isPlaying, setIsPlaying] = useState(false); const [userConfirmedStopped, setUserConfirmedStopped] = useState(false); - const segmentUploadProgress = useUploadProgress( - data.id, - data.source.type === "desktopSegments" && (data.hasActiveUpload ?? false), - ); + const trackUploadProgress = + data.source.type === "desktopSegments" && (data.hasActiveUpload ?? false); + const [segmentUploadProgress, setSegmentUploadProgress] = + useState( + trackUploadProgress ? { status: "fetching" } : null, + ); + useEffect(() => { + setSegmentUploadProgress( + trackUploadProgress ? { status: "fetching" } : null, + ); + }, [trackUploadProgress]); const [subtitleUrl, setSubtitleUrl] = useState(null); const [chaptersUrl, setChaptersUrl] = useState(null); const captionsDisabled = viewerSettings?.disableCaptions ?? false; @@ -395,6 +416,12 @@ export const EmbedVideo = forwardRef< )} )} + {trackUploadProgress && ( + + )} ); }, From 7d387c68384df03b8f24d647984a150d839586e1 Mon Sep 17 00:00:00 2001 From: 07rjain <07rjain@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:16:38 +0000 Subject: [PATCH 08/20] docs(self-host): explain why to pin cap-web and media-server images --- apps/web/content/docs/self-hosting.mdx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/web/content/docs/self-hosting.mdx b/apps/web/content/docs/self-hosting.mdx index 8eb2ff31344..de104cdfc5c 100644 --- a/apps/web/content/docs/self-hosting.mdx +++ b/apps/web/content/docs/self-hosting.mdx @@ -214,6 +214,10 @@ APPLE_CLIENT_SECRET=your-client-secret-jwt Apple client-secret JWTs expire, so rotate the secret before its configured expiration date. +## Pinning Image Versions + +`cap-web:latest` and `cap-media-server:latest` follow `main`. cap-web runs database migrations automatically on start, and they only go forward, so after a newer image has migrated your database you can't safely roll back to an older one. New required settings also arrive with updates (for example, `MEDIA_SERVER_WEBHOOK_SECRET` became mandatory for media processing). For production, pin both images to a specific tag or digest, back up MySQL before upgrading, and read the self-hosting changes before moving the pin. + ## Production Checklist From 855c51f912373d2e44ad46d6e931fa45c9c2df70 Mon Sep 17 00:00:00 2001 From: 07rjain <07rjain@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:21:41 +0000 Subject: [PATCH 09/20] fix(media-server): keep thumbnail defaults when a request omits width, height, quality or timestamp --- .../__tests__/lib/thumbnail-options.test.ts | 23 +++++++++++++++++++ apps/media-server/src/lib/media-video.ts | 11 ++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 apps/media-server/src/__tests__/lib/thumbnail-options.test.ts diff --git a/apps/media-server/src/__tests__/lib/thumbnail-options.test.ts b/apps/media-server/src/__tests__/lib/thumbnail-options.test.ts new file mode 100644 index 00000000000..230a32314a6 --- /dev/null +++ b/apps/media-server/src/__tests__/lib/thumbnail-options.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, test } from "bun:test"; +import { resolveThumbnailOptions } from "../../lib/media-video"; + +describe("resolveThumbnailOptions", () => { + test("keeps defaults for fields a request leaves undefined", () => { + expect( + resolveThumbnailOptions({ + timestamp: undefined, + width: undefined, + height: undefined, + quality: undefined, + }), + ).toEqual({ timestamp: 1, width: 1280, height: 720, quality: 85 }); + }); + + test("applies provided values", () => { + expect(resolveThumbnailOptions({ width: 640, quality: 50 })).toMatchObject({ + width: 640, + height: 720, + quality: 50, + }); + }); +}); diff --git a/apps/media-server/src/lib/media-video.ts b/apps/media-server/src/lib/media-video.ts index 48158979327..d1d9efac0b6 100644 --- a/apps/media-server/src/lib/media-video.ts +++ b/apps/media-server/src/lib/media-video.ts @@ -1776,6 +1776,15 @@ async function processVideoWithCapcodec( } } +export function resolveThumbnailOptions( + options: ThumbnailOptions = {}, +): Required { + const defined = Object.fromEntries( + Object.entries(options).filter(([, value]) => value !== undefined), + ) as ThumbnailOptions; + return { ...DEFAULT_THUMBNAIL_OPTIONS, ...defined }; +} + function getThumbnailTimestamp( duration: number, requestedTimestamp: number, @@ -1798,7 +1807,7 @@ export async function generateThumbnail( abortSignal?: AbortSignal, ): Promise { abortSignal?.throwIfAborted(); - const opts = { ...DEFAULT_THUMBNAIL_OPTIONS, ...options }; + const opts = resolveThumbnailOptions(options); const timestamp = getThumbnailTimestamp(duration, opts.timestamp); const deadline = performance.now() + THUMBNAIL_TIMEOUT_MS; try { From f19424791469a8bb7d1425541c3d2ef891aea424 Mon Sep 17 00:00:00 2001 From: 07rjain <07rishabhjain@gmail.com> Date: Wed, 14 Oct 2026 19:59:04 +0530 Subject: [PATCH 10/20] fix(media-server): close the URL and drain gaps from review Trusted endpoints match host and port, and every redirect hop is checked before fetch or ffmpeg sees it. Drain keeps /health at 200 and waits for probe and audio work, and a missing mail setup reports not_configured instead of failing the invite. Co-authored-by: Cursor --- .../src/__tests__/lib/url-policy.test.ts | 65 +++++++++ .../src/__tests__/routes/health.test.ts | 13 ++ apps/media-server/src/index.ts | 23 +++- apps/media-server/src/lib/job-manager.ts | 4 + apps/media-server/src/lib/media-audio.ts | 6 +- apps/media-server/src/lib/media-operations.ts | 4 +- apps/media-server/src/lib/media-probe.ts | 9 +- apps/media-server/src/lib/media-transfer.ts | 3 +- apps/media-server/src/lib/media-video.ts | 29 +++- apps/media-server/src/lib/url-policy.ts | 124 ++++++++++++++++-- apps/media-server/src/routes/audio.ts | 10 ++ apps/media-server/src/routes/health.ts | 2 +- apps/media-server/src/routes/video.ts | 10 ++ apps/web/app/api/v1/[...route]/route.ts | 9 +- apps/web/lib/media-server-startup-check.ts | 15 ++- docker-compose.coolify.yml | 3 +- docker-compose.template.yml | 1 + packages/web-domain/src/Agent.ts | 2 +- 18 files changed, 294 insertions(+), 38 deletions(-) diff --git a/apps/media-server/src/__tests__/lib/url-policy.test.ts b/apps/media-server/src/__tests__/lib/url-policy.test.ts index cc94510d735..9e3129fee9a 100644 --- a/apps/media-server/src/__tests__/lib/url-policy.test.ts +++ b/apps/media-server/src/__tests__/lib/url-policy.test.ts @@ -2,8 +2,10 @@ import { describe, expect, test } from "bun:test"; import { assertUrlAllowed, collectUrls, + fetchAllowed, getUrlPolicy, isPrivateAddress, + resolveAllowedUrl, } from "../../lib/url-policy"; const resolveTo = @@ -78,6 +80,37 @@ describe("assertUrlAllowed", () => { ).resolves.toBeUndefined(); }); + test("does not treat a trusted host as allowed on every port", async () => { + const policy = getUrlPolicy({ + S3_PUBLIC_ENDPOINT: "http://localhost:9000", + MEDIA_SERVER_WEBHOOK_URL: "http://cap-web:3000", + }); + await expect( + assertUrlAllowed( + "http://minio:445/cap/v.mp4", + policy, + resolveTo("10.0.0.2"), + ), + ).rejects.toThrow("private address"); + await expect( + assertUrlAllowed( + "http://localhost:6379/", + policy, + resolveTo("127.0.0.1"), + ), + ).rejects.toThrow("private address"); + await expect( + assertUrlAllowed("http://localhost:9000/bucket/v.mp4", policy), + ).resolves.toBeUndefined(); + }); + + test("blocks file URLs even when private URLs are allowed", async () => { + const policy = getUrlPolicy({ MEDIA_SERVER_ALLOW_PRIVATE_URLS: "true" }); + await expect( + assertUrlAllowed("file:///etc/passwd", policy), + ).rejects.toThrow("scheme"); + }); + test("enforces MEDIA_SERVER_ALLOWED_HOSTS when set", async () => { const policy = getUrlPolicy({ MEDIA_SERVER_ALLOWED_HOSTS: "*.r2.cloudflarestorage.com", @@ -103,6 +136,38 @@ describe("assertUrlAllowed", () => { }); }); +describe("redirect checks", () => { + test("does not follow a redirect to a private address", async () => { + const policy = getUrlPolicy({}); + const fetcher = async (input: string) => { + if (input === "https://cdn.example/v.mp4") { + return new Response(null, { + status: 302, + headers: { location: "http://169.254.169.254/latest/meta-data/" }, + }); + } + return new Response("ok"); + }; + await expect( + fetchAllowed( + "https://cdn.example/v.mp4", + {}, + policy, + resolveTo("93.184.216.34"), + fetcher, + ), + ).rejects.toThrow("private address"); + await expect( + resolveAllowedUrl( + "https://cdn.example/v.mp4", + policy, + resolveTo("93.184.216.34"), + fetcher, + ), + ).rejects.toThrow("private address"); + }); +}); + describe("collectUrls", () => { test("collects nested url fields only", () => { expect( diff --git a/apps/media-server/src/__tests__/routes/health.test.ts b/apps/media-server/src/__tests__/routes/health.test.ts index cd81c6bd266..546e02ef2a6 100644 --- a/apps/media-server/src/__tests__/routes/health.test.ts +++ b/apps/media-server/src/__tests__/routes/health.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test"; import app from "../../app"; +import { beginDraining, endDraining } from "../../lib/job-manager"; describe("GET /health", () => { test("returns status ok with media engine info", async () => { @@ -22,4 +23,16 @@ describe("GET /health", () => { expect(typeof data.system.uptimeSeconds).toBe("number"); expect(typeof data.system.processUptimeSeconds).toBe("number"); }); + + test("stays HTTP 200 while draining so the healthcheck can finish the drain", async () => { + beginDraining(); + try { + const response = await app.fetch(new Request("http://localhost/health")); + expect(response.status).toBe(200); + const data = await response.json(); + expect(data.status).toBe("draining"); + } finally { + endDraining(); + } + }); }); diff --git a/apps/media-server/src/index.ts b/apps/media-server/src/index.ts index 487b4331743..810191340a9 100644 --- a/apps/media-server/src/index.ts +++ b/apps/media-server/src/index.ts @@ -3,9 +3,12 @@ import { abortAllJobs, beginDraining, getActiveVideoProcessCount, - waitForActiveVideoProcesses, } from "./lib/job-manager"; -import { cancelAllMediaOperations } from "./lib/media-operations"; +import { + cancelAllMediaOperations, + getActiveAudioOperationCount, + getActiveProbeOperationCount, +} from "./lib/media-operations"; import { cleanupMediaTransferCache } from "./lib/media-transfer"; import { assertSecureStartup } from "./lib/startup-checks"; @@ -26,12 +29,20 @@ const shutdown = async () => { if (shuttingDown) return; shuttingDown = true; beginDraining(); - const active = getActiveVideoProcessCount(); - if (active > 0 && shutdownGraceMs > 0) { + const active = () => + getActiveVideoProcessCount() + + getActiveAudioOperationCount() + + getActiveProbeOperationCount(); + const inflight = active(); + if (inflight > 0 && shutdownGraceMs > 0) { console.log( - `[media-server] Draining: waiting up to ${Math.round(shutdownGraceMs / 1000)}s for ${active} active video job(s); new work is rejected with SERVER_BUSY`, + `[media-server] Draining: waiting up to ${Math.round(shutdownGraceMs / 1000)}s for ${inflight} active media operation(s); new work is rejected with SERVER_BUSY`, ); - const remaining = await waitForActiveVideoProcesses(shutdownGraceMs); + const deadline = Date.now() + shutdownGraceMs; + while (active() > 0 && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 250)); + } + const remaining = active(); console.log( remaining === 0 ? "[media-server] All active jobs finished" diff --git a/apps/media-server/src/lib/job-manager.ts b/apps/media-server/src/lib/job-manager.ts index 7a87a6e0cc0..c34d0a18dd2 100644 --- a/apps/media-server/src/lib/job-manager.ts +++ b/apps/media-server/src/lib/job-manager.ts @@ -335,6 +335,10 @@ export function beginDraining() { draining = true; } +export function endDraining() { + draining = false; +} + export function isDraining(): boolean { return draining; } diff --git a/apps/media-server/src/lib/media-audio.ts b/apps/media-server/src/lib/media-audio.ts index 5e438acbd2b..4248c106643 100644 --- a/apps/media-server/src/lib/media-audio.ts +++ b/apps/media-server/src/lib/media-audio.ts @@ -13,6 +13,7 @@ import { import { materializeStreamingInput } from "./media-video"; import { registerSubprocess, terminateProcess } from "./subprocess"; import { ensureTempDir, getTempDir } from "./temp-files"; +import { resolveAllowedUrl } from "./url-policy"; export interface AudioExtractionOptions { format?: "mp3"; @@ -168,7 +169,10 @@ async function prepareAudioProbeInput( abortSignal: AbortSignal, ): Promise<{ inputPath: string; cleanup: () => Promise }> { if (!isRemoteStreamingUrl(videoUrl)) { - return { inputPath: videoUrl, cleanup: async () => {} }; + const inputPath = /^https?:/i.test(videoUrl) + ? await resolveAllowedUrl(videoUrl) + : videoUrl; + return { inputPath, cleanup: async () => {} }; } await ensureTempDir(); diff --git a/apps/media-server/src/lib/media-operations.ts b/apps/media-server/src/lib/media-operations.ts index 1c9901dd89e..ad44db4f086 100644 --- a/apps/media-server/src/lib/media-operations.ts +++ b/apps/media-server/src/lib/media-operations.ts @@ -1,4 +1,4 @@ -import { hasCriticalMemoryPressure } from "./job-manager"; +import { hasCriticalMemoryPressure, isDraining } from "./job-manager"; import { terminateAllSubprocesses } from "./subprocess"; export interface MediaOperationHandle { @@ -32,6 +32,7 @@ export function getActiveAudioOperationCount(): number { export function canAcceptNewAudioOperation(): boolean { return ( + !isDraining() && !hasCriticalMemoryPressure() && getActiveAudioOperationCount() < MAX_CONCURRENT_AUDIO_OPERATIONS ); @@ -47,6 +48,7 @@ export function getActiveProbeOperationCount(): number { export function canAcceptNewProbeOperation(): boolean { return ( + !isDraining() && !hasCriticalMemoryPressure() && getActiveProbeOperationCount() < MAX_CONCURRENT_PROBE_OPERATIONS ); diff --git a/apps/media-server/src/lib/media-probe.ts b/apps/media-server/src/lib/media-probe.ts index ef6ee8ace8f..ed56443e1f9 100644 --- a/apps/media-server/src/lib/media-probe.ts +++ b/apps/media-server/src/lib/media-probe.ts @@ -12,9 +12,9 @@ import { withMediaOperation, } from "./media-operations"; import { materializeMedia } from "./media-transfer"; +import { fetchAllowed, resolveAllowedUrl } from "./url-policy"; const PROBE_TIMEOUT_MS = 30_000; -const probeFetch: typeof fetch = globalThis.fetch.bind(globalThis); export { canAcceptNewProbeOperation, getActiveProbeOperationCount }; @@ -48,7 +48,7 @@ function isHttpUrl(path: string): boolean { async function hasHttpNetworkFailure(path: string): Promise { try { - const response = await probeFetch(path, { + const response = await fetchAllowed(path, { headers: { Range: "bytes=0-0" }, signal: AbortSignal.timeout(10_000), }); @@ -126,8 +126,9 @@ export async function probeVideo(videoUrl: string): Promise { throw new Error("Server is busy, please try again later"); } + const allowedUrl = await resolveAllowedUrl(videoUrl); return await withMediaOperation("probe", () => - withTimeout(probeMedia(videoUrl), PROBE_TIMEOUT_MS), + withTimeout(probeMedia(allowedUrl), PROBE_TIMEOUT_MS), ); } @@ -143,7 +144,7 @@ export async function probeVideoFile(filePath: string): Promise { export async function checkVideoAccessible(videoUrl: string): Promise { try { - const response = await probeFetch(videoUrl, { + const response = await fetchAllowed(videoUrl, { method: "HEAD", signal: AbortSignal.timeout(10_000), }); diff --git a/apps/media-server/src/lib/media-transfer.ts b/apps/media-server/src/lib/media-transfer.ts index 62720779397..fd81e7043e5 100644 --- a/apps/media-server/src/lib/media-transfer.ts +++ b/apps/media-server/src/lib/media-transfer.ts @@ -4,6 +4,7 @@ import { mkdir, open, readdir, rm, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { z } from "zod"; +import { fetchAllowed } from "./url-policy"; const targetSchema = z.object({ version: z.literal(1), @@ -470,7 +471,7 @@ export async function fetchMedia( init.signal ?? undefined, headers.get("if-match") ?? undefined, ); - if (!local) return fetch(input, init); + if (!local) return fetchAllowed(input, init); const range = headers.get("range"); const match = range?.match(/^bytes=(\d+)-(\d*)$/); const start = match ? Number(match[1]) : 0; diff --git a/apps/media-server/src/lib/media-video.ts b/apps/media-server/src/lib/media-video.ts index d1d9efac0b6..3b3d46afb69 100644 --- a/apps/media-server/src/lib/media-video.ts +++ b/apps/media-server/src/lib/media-video.ts @@ -33,6 +33,7 @@ import { getTempDir, type TempFileHandle, } from "./temp-files"; +import { assertUrlAllowed, fetchAllowed } from "./url-policy"; const PROCESS_TIMEOUT_PER_SECOND_MS = 20_000; const MAX_PROCESS_TIMEOUT_MS = 2 * 60 * 60 * 1000; @@ -286,7 +287,7 @@ export async function materializeHlsPlaylist( const cached = cache.get(playlistUrl); if (cached) return cached; - const response = await fetch(playlistUrl, { + const response = await fetchAllowed(playlistUrl, { signal: getFetchSignal(DOWNLOAD_TIMEOUT_MS, abortSignal), }); @@ -312,9 +313,16 @@ export async function materializeHlsPlaylist( if (!trimmed.startsWith("#")) { const resolved = resolveResourceUrl(trimmed, baseUrl, query); - return isHlsUrl(resolved) - ? await materializeHlsPlaylist(resolved, dirPath, cache, abortSignal) - : resolved; + if (isHlsUrl(resolved)) { + return await materializeHlsPlaylist( + resolved, + dirPath, + cache, + abortSignal, + ); + } + await assertUrlAllowed(resolved); + return resolved; } if (!line.includes('URI="')) return line; @@ -330,6 +338,7 @@ export async function materializeHlsPlaylist( const replacement = isHlsUrl(resolved) ? await materializeHlsPlaylist(resolved, dirPath, cache, abortSignal) : resolved; + if (!isHlsUrl(resolved)) await assertUrlAllowed(resolved); rewritten = rewritten.replace( `URI="${original}"`, @@ -350,7 +359,7 @@ export async function materializeMpdManifest( dirPath: string, abortSignal?: AbortSignal, ): Promise { - const response = await fetch(manifestUrl, { + const response = await fetchAllowed(manifestUrl, { signal: getFetchSignal(DOWNLOAD_TIMEOUT_MS, abortSignal), }); @@ -365,6 +374,7 @@ export async function materializeMpdManifest( const baseUrl = new URL(".", parsedUrl).toString(); const query = parsedUrl.search; const filePath = join(dirPath, `${randomUUID()}.mpd`); + const resolvedUrls: string[] = []; const rewrittenElements = content.replace( /<(BaseURL|Location)(\b[^>]*)>([\s\S]*?)<\/\1>/gi, (_, tag: string, attributes: string, resource: string) => { @@ -373,6 +383,7 @@ export async function materializeMpdManifest( baseUrl, query, ); + resolvedUrls.push(resolved); return `<${tag}${attributes}>${escapeXmlAttribute(resolved)}`; }, ); @@ -385,9 +396,11 @@ export async function materializeMpdManifest( baseUrl, query, ); + resolvedUrls.push(resolved); return `${attribute}="${escapeXmlAttribute(resolved)}"`; }, ); + await Promise.all(resolvedUrls.map((url) => assertUrlAllowed(url))); await writeFile(filePath, rewritten); return filePath; @@ -729,7 +742,7 @@ export async function materializeMpdAsHlsPlaylist( dirPath: string, abortSignal?: AbortSignal, ): Promise { - const response = await fetch(manifestUrl, { + const response = await fetchAllowed(manifestUrl, { signal: getFetchSignal(DOWNLOAD_TIMEOUT_MS, abortSignal), }); @@ -822,6 +835,10 @@ export async function materializeMpdAsHlsPlaylist( query, presentationDuration, ); + await assertUrlAllowed(initUrl); + await Promise.all( + segments.map((segment) => assertUrlAllowed(segment.url)), + ); const path = await writeDashRepresentationPlaylist( dirPath, initUrl, diff --git a/apps/media-server/src/lib/url-policy.ts b/apps/media-server/src/lib/url-policy.ts index 75ff4778b09..1de9e6a9c4b 100644 --- a/apps/media-server/src/lib/url-policy.ts +++ b/apps/media-server/src/lib/url-policy.ts @@ -40,10 +40,21 @@ export function isPrivateAddress(address: string): boolean { return true; } -function hostOf(value: string | undefined): string | null { +function defaultPort(protocol: string): string { + if (protocol === "http:") return "80"; + if (protocol === "https:") return "443"; + return ""; +} + +function originOf(value: string | undefined): TrustedOrigin | null { if (!value?.trim()) return null; try { - return new URL(value.trim()).hostname.toLowerCase(); + const url = new URL(value.trim()); + if (url.protocol !== "http:" && url.protocol !== "https:") return null; + return { + host: normalizeHost(url.hostname), + port: url.port || defaultPort(url.protocol), + }; } catch { return null; } @@ -56,9 +67,12 @@ function normalizeHost(host: string): string { .replace(/^\[|\]$/g, ""); } +export type TrustedOrigin = { host: string; port: string }; + export type UrlPolicy = { strictHosts: boolean; allowedHosts: string[]; + trustedOrigins: TrustedOrigin[]; allowPrivate: boolean; }; @@ -69,18 +83,25 @@ export function getUrlPolicy( .split(",") .map(normalizeHost) .filter(Boolean); - const implicit = [ + const trustedOrigins = [ env.S3_INTERNAL_ENDPOINT, env.S3_PUBLIC_ENDPOINT, env.CAP_AWS_ENDPOINT, env.MEDIA_SERVER_WEBHOOK_URL, env.WEB_URL, ] - .map(hostOf) - .filter((host): host is string => Boolean(host)); + .map(originOf) + .filter((origin): origin is TrustedOrigin => Boolean(origin)); + const seen = new Set(); return { strictHosts: explicit.length > 0, - allowedHosts: [...new Set([...explicit, ...implicit])], + allowedHosts: [...new Set(explicit)], + trustedOrigins: trustedOrigins.filter((origin) => { + const key = `${origin.host}\0${origin.port}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }), allowPrivate: env.MEDIA_SERVER_ALLOW_PRIVATE_URLS === "true", }; } @@ -103,6 +124,17 @@ export class BlockedUrlError extends Error { } } +export function blockedUrlPayload( + error: unknown, +): { error: string; code: "URL_NOT_ALLOWED"; details: string } | null { + if (!(error instanceof BlockedUrlError)) return null; + return { + error: "URL not allowed", + code: "URL_NOT_ALLOWED", + details: error.message, + }; +} + function redactUrl(value: string): string { try { const url = new URL(value); @@ -130,21 +162,24 @@ export async function assertUrlAllowed( } catch { throw new BlockedUrlError(value, "invalid URL"); } - if (url.protocol === "file:" && policy.allowPrivate) return; if (url.protocol !== "http:" && url.protocol !== "https:") { throw new BlockedUrlError(value, `scheme ${url.protocol} is not allowed`); } const host = normalizeHost(url.hostname); + const port = url.port || defaultPort(url.protocol); const explicitlyAllowed = policy.allowedHosts.some((pattern) => hostMatches(host, pattern), ); - if (policy.strictHosts && !explicitlyAllowed) { + const trustedOrigin = policy.trustedOrigins.some( + (origin) => origin.host === host && origin.port === port, + ); + if (explicitlyAllowed || trustedOrigin || policy.allowPrivate) return; + if (policy.strictHosts) { throw new BlockedUrlError( value, `host ${host} is not in MEDIA_SERVER_ALLOWED_HOSTS`, ); } - if (explicitlyAllowed || policy.allowPrivate) return; let addresses: string[]; try { @@ -180,12 +215,73 @@ export function collectUrls(input: unknown, key = ""): string[] { return []; } +const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]); +const MAX_REDIRECTS = 3; + +type Fetcher = (input: string, init?: RequestInit) => Promise; + +async function redirectTarget( + response: Response, + current: string, +): Promise { + if (!REDIRECT_STATUSES.has(response.status)) return null; + const location = response.headers.get("location"); + if (!location) return null; + return new URL(location, current).toString(); +} + +export async function resolveAllowedUrl( + input: string, + policy: UrlPolicy = getUrlPolicy(), + resolve: Resolver = defaultResolver, + fetcher: Fetcher = fetch, +): Promise { + let current = input; + for (let hop = 0; hop <= MAX_REDIRECTS; hop++) { + await assertUrlAllowed(current, policy, resolve); + let response = await fetcher(current, { + method: "HEAD", + redirect: "manual", + }); + if (response.status === 405 || response.status === 501) { + await response.body?.cancel(); + response = await fetcher(current, { method: "GET", redirect: "manual" }); + } + const next = await redirectTarget(response, current); + await response.body?.cancel(); + if (!next) return current; + if (hop === MAX_REDIRECTS) { + throw new BlockedUrlError(current, "too many redirects"); + } + current = next; + } + throw new BlockedUrlError(current, "too many redirects"); +} + +export async function fetchAllowed( + input: string, + init: RequestInit = {}, + policy: UrlPolicy = getUrlPolicy(), + resolve: Resolver = defaultResolver, + fetcher: Fetcher = fetch, +): Promise { + let current = input; + for (let hop = 0; hop <= MAX_REDIRECTS; hop++) { + await assertUrlAllowed(current, policy, resolve); + const response = await fetcher(current, { ...init, redirect: "manual" }); + const next = await redirectTarget(response, current); + if (!next) return response; + await response.body?.cancel(); + if (hop === MAX_REDIRECTS) { + throw new BlockedUrlError(current, "too many redirects"); + } + current = next; + } + throw new BlockedUrlError(current, "too many redirects"); +} + export const urlPolicyMiddleware: MiddlewareHandler = async (c, next) => { - if ( - c.req.method !== "POST" || - !c.req.header("content-type")?.includes("application/json") || - !validateMediaServerSecret(c) - ) { + if (c.req.method !== "POST" || !validateMediaServerSecret(c)) { return next(); } let body: unknown; diff --git a/apps/media-server/src/routes/audio.ts b/apps/media-server/src/routes/audio.ts index d2f8cc71e2b..d36306ecb33 100644 --- a/apps/media-server/src/routes/audio.ts +++ b/apps/media-server/src/routes/audio.ts @@ -8,6 +8,7 @@ import { extractAudioStream, getActiveAudioOperationCount as getActiveProcessCount, } from "../lib/media-audio"; +import { blockedUrlPayload } from "../lib/url-policy"; const audio = new Hono(); const MEDIA_ENGINE_ERROR_CODE = ["FF", "MPEG_ERROR"].join(""); @@ -67,6 +68,9 @@ audio.post("/check", async (c) => { } catch (err) { console.error("[audio/check] Error:", err); + const blocked = blockedUrlPayload(err); + if (blocked) return c.json(blocked, 400); + if (isBusyError(err)) { return c.json( { @@ -156,6 +160,9 @@ audio.post("/extract", async (c) => { } catch (err) { console.error("[audio/extract] Error:", err); + const blocked = blockedUrlPayload(err); + if (blocked) return c.json(blocked, 400); + if (isBusyError(err)) { return c.json( { @@ -229,6 +236,9 @@ audio.post("/convert", async (c) => { } catch (err) { console.error("[audio/convert] Error:", err); + const blocked = blockedUrlPayload(err); + if (blocked) return c.json(blocked, 400); + if (isBusyError(err)) { return c.json( { diff --git a/apps/media-server/src/routes/health.ts b/apps/media-server/src/routes/health.ts index fbc5e82123c..08ff3e071f0 100644 --- a/apps/media-server/src/routes/health.ts +++ b/apps/media-server/src/routes/health.ts @@ -42,7 +42,7 @@ health.get("/", (c) => { processUptimeSeconds: Math.round(process.uptime()), }, }, - draining ? 503 : 200, + 200, ); }); diff --git a/apps/media-server/src/routes/video.ts b/apps/media-server/src/routes/video.ts index cd2b44e68b5..a734a5f46f6 100644 --- a/apps/media-server/src/routes/video.ts +++ b/apps/media-server/src/routes/video.ts @@ -71,6 +71,7 @@ import { } from "../lib/recording-verification"; import type { TempFileHandle } from "../lib/temp-files"; import { cleanupStaleTempFiles } from "../lib/temp-files"; +import { blockedUrlPayload } from "../lib/url-policy"; import { getActiveDirectVideoProcessCount, getMaxConcurrentDirectVideoProcesses, @@ -429,6 +430,9 @@ video.post("/probe", async (c) => { } catch (err) { console.error("[video/probe] Error:", err); + const blocked = blockedUrlPayload(err); + if (blocked) return c.json(blocked, 400); + if (isBusyError(err)) { return c.json( { @@ -504,6 +508,9 @@ video.post("/thumbnail", async (c) => { } catch (err) { console.error("[video/thumbnail] Error:", err); + const blocked = blockedUrlPayload(err); + if (blocked) return c.json(blocked, 400); + if (isBusyError(err)) { return c.json( { @@ -634,6 +641,9 @@ video.post("/convert", async (c) => { }); console.error("[video/convert] Error:", err); + const blocked = blockedUrlPayload(err); + if (blocked) return c.json(blocked, 400); + if (isBusyError(err)) { c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString()); return c.json( diff --git a/apps/web/app/api/v1/[...route]/route.ts b/apps/web/app/api/v1/[...route]/route.ts index 5e889e3c47c..0cfb4b5503a 100644 --- a/apps/web/app/api/v1/[...route]/route.ts +++ b/apps/web/app/api/v1/[...route]/route.ts @@ -4912,8 +4912,15 @@ const AgentManagementHandlersLive = HttpApiBuilder.group( }), idempotencyKey: providerIdempotencyKey, }); - if (delivery?.error) + if (delivery?.error) { + if (delivery.error.name === "email_not_configured") { + return { + ...invitation, + emailDelivery: "not_configured" as const, + }; + } throw new Error(delivery.error.message); + } return { ...invitation, emailDelivery: "accepted" as const, diff --git a/apps/web/lib/media-server-startup-check.ts b/apps/web/lib/media-server-startup-check.ts index 7350da36666..65af01b1083 100644 --- a/apps/web/lib/media-server-startup-check.ts +++ b/apps/web/lib/media-server-startup-check.ts @@ -64,10 +64,23 @@ export async function checkMediaServerUrl( } } +const MEDIA_SERVER_CHECK_ATTEMPTS = 5; +const MEDIA_SERVER_CHECK_RETRY_MS = 2_000; + export async function logMediaServerUrlCheck( raw: string | undefined = process.env.MEDIA_SERVER_URL, ) { - const result = await checkMediaServerUrl(raw); + let result = await checkMediaServerUrl(raw); + for ( + let attempt = 1; + result.status === "unreachable" && attempt < MEDIA_SERVER_CHECK_ATTEMPTS; + attempt++ + ) { + await new Promise((resolve) => + setTimeout(resolve, MEDIA_SERVER_CHECK_RETRY_MS), + ); + result = await checkMediaServerUrl(raw); + } switch (result.status) { case "unset": console.warn( diff --git a/docker-compose.coolify.yml b/docker-compose.coolify.yml index 30fbe4eaa84..329fbc617d6 100644 --- a/docker-compose.coolify.yml +++ b/docker-compose.coolify.yml @@ -38,7 +38,7 @@ services: - '-q' - '-O' - /dev/null - - 'http://127.0.0.1:3000/' + - 'http://127.0.0.1:3000/api/status' interval: 30s timeout: 10s retries: 3 @@ -46,6 +46,7 @@ services: media-server: image: 'ghcr.io/capsoftware/cap-media-server:latest' restart: unless-stopped + stop_grace_period: 30s environment: PORT: 3456 S3_INTERNAL_ENDPOINT: 'http://minio:9000' diff --git a/docker-compose.template.yml b/docker-compose.template.yml index f4046a1690c..0b9bc5f7978 100644 --- a/docker-compose.template.yml +++ b/docker-compose.template.yml @@ -55,6 +55,7 @@ services: context: ./apps/media-server dockerfile: Dockerfile.standalone restart: unless-stopped + stop_grace_period: 30s environment: S3_INTERNAL_ENDPOINT: http://minio:3902 MEDIA_SERVER_WEBHOOK_URL: http://cap-web:3000 diff --git a/packages/web-domain/src/Agent.ts b/packages/web-domain/src/Agent.ts index b09795d80ad..9a4c1f8170f 100644 --- a/packages/web-domain/src/Agent.ts +++ b/packages/web-domain/src/Agent.ts @@ -695,7 +695,7 @@ export const AgentInvite = Schema.Struct({ export const AgentOrganizationInviteResponse = Schema.Struct({ invite: AgentInvite, inviteUrl: Schema.String, - emailDelivery: Schema.Literal("not_requested", "accepted"), + emailDelivery: Schema.Literal("not_requested", "accepted", "not_configured"), requestId: Schema.String, }); From b7b4d597880ac308622185ce270dfbb2bfc25500 Mon Sep 17 00:00:00 2001 From: 07rjain <07rishabhjain@gmail.com> Date: Wed, 14 Oct 2026 20:35:32 +0530 Subject: [PATCH 11/20] fix(media-server): keep redirects and probe waits inside the URL policy Playlist and audio inputs are downloaded before ffmpeg runs, and uploads and webhooks check each redirect hop. Probes reserve a slot before that check, and an explicit email sender no longer depends on an unused default domain. Co-authored-by: Cursor --- .../lib/media-audio.integration.test.ts | 2 +- .../lib/media-video.integration.test.ts | 86 ++++---- apps/media-server/src/lib/audio-levels.ts | 4 +- apps/media-server/src/lib/job-manager.ts | 3 +- apps/media-server/src/lib/media-audio.ts | 200 ++++++++++++++---- apps/media-server/src/lib/media-common.ts | 11 +- apps/media-server/src/lib/media-probe.ts | 40 +++- apps/media-server/src/lib/media-video.ts | 136 ++++++++++-- apps/media-server/src/lib/url-policy.ts | 19 +- apps/web/__tests__/unit/email-config.test.ts | 15 ++ apps/web/content/docs/self-hosting.mdx | 2 +- packages/database/emails/config.ts | 32 +-- 12 files changed, 424 insertions(+), 126 deletions(-) diff --git a/apps/media-server/src/__tests__/lib/media-audio.integration.test.ts b/apps/media-server/src/__tests__/lib/media-audio.integration.test.ts index 826d49234f2..402b47be763 100644 --- a/apps/media-server/src/__tests__/lib/media-audio.integration.test.ts +++ b/apps/media-server/src/__tests__/lib/media-audio.integration.test.ts @@ -201,7 +201,7 @@ describe("mediaAudio integration tests", () => { try { await expect( checkHasAudioTrack(`${server.url}video.mp4`), - ).rejects.toThrow("FFprobe exited"); + ).rejects.toThrow("Failed to fetch media: 503"); await waitForAudioOperations(beforeCount); } finally { await server.stop(true); diff --git a/apps/media-server/src/__tests__/lib/media-video.integration.test.ts b/apps/media-server/src/__tests__/lib/media-video.integration.test.ts index dd9844d100c..1a2c9da0b5b 100644 --- a/apps/media-server/src/__tests__/lib/media-video.integration.test.ts +++ b/apps/media-server/src/__tests__/lib/media-video.integration.test.ts @@ -645,11 +645,16 @@ describe("processVideo integration tests", () => { const originalFetch = globalThis.fetch; const manifestDir = mkdtempSync(join(tmpdir(), "cap-mpd-")); - globalThis.fetch = (async () => - new Response( - '', + const requestedUrls: string[] = []; + globalThis.fetch = (async (input: RequestInfo | URL) => { + requestedUrls.push( + input instanceof Request ? input.url : input.toString(), + ); + return new Response( + '', { status: 200, statusText: "OK" }, - )) as unknown as typeof fetch; + ); + }) as unknown as typeof fetch; try { const path = await materializeMpdManifest( @@ -658,13 +663,13 @@ describe("processVideo integration tests", () => { ); const content = readFileSync(path, "utf8"); - expect(content).toContain( - "init.mp4?Policy=a&Signature=b&Key-Pair-Id=c", + expect(requestedUrls).toContain( + "https://cdn.example/video/init.mp4?Policy=a&Signature=b&Key-Pair-Id=c", ); - expect(content).toContain( - "chunk-$Number$.m4s?Policy=a&Signature=b&Key-Pair-Id=c", + expect(requestedUrls).toContain( + "https://cdn.example/video/escaped.mp4?part=1&token=x", ); - expect(content).toContain("escaped.mp4?part=1&token=x"); + expect(content).not.toContain("https://cdn.example"); expect(content).not.toContain("&amp;"); } finally { globalThis.fetch = originalFetch; @@ -717,21 +722,10 @@ describe("processVideo integration tests", () => { const videoPlaylist = readFileSync(videoPlaylistPath ?? "", "utf8"); const audioPlaylist = readFileSync(mediaMatch?.[1] ?? "", "utf8"); - expect(videoPlaylist).toContain( - "https://cdn.example/video/video/init-v1.mp4?Policy=a&Signature=b", - ); - expect(videoPlaylist).toContain( - "https://cdn.example/video/video/chunk-00003.m4s?Policy=a&Signature=b", - ); - expect(videoPlaylist).toContain( - "https://cdn.example/video/video/chunk-00004.m4s?Policy=a&Signature=b", - ); - expect(audioPlaylist).toContain( - "https://cdn.example/video/audio/chunk-0.m4s?Policy=a&Signature=b", - ); - expect(audioPlaylist).toContain( - "https://cdn.example/video/audio/chunk-48000.m4s?Policy=a&Signature=b", - ); + expect(videoPlaylist).not.toContain("https://cdn.example"); + expect(audioPlaylist).not.toContain("https://cdn.example"); + expect(videoPlaylist).toContain(".m4s"); + expect(audioPlaylist).toContain(".m4s"); } finally { globalThis.fetch = originalFetch; rmSync(manifestDir, { recursive: true, force: true }); @@ -741,10 +735,12 @@ describe("processVideo integration tests", () => { test("falls back to generic DASH manifest materialization for unsupported MPD shapes", async () => { const originalFetch = globalThis.fetch; const manifestDir = mkdtempSync(join(tmpdir(), "cap-mpd-fallback-")); - let requests = 0; + const requestedUrls: string[] = []; - globalThis.fetch = (async () => { - requests++; + globalThis.fetch = (async (input: RequestInfo | URL) => { + requestedUrls.push( + input instanceof Request ? input.url : input.toString(), + ); return new Response( ` @@ -771,11 +767,13 @@ describe("processVideo integration tests", () => { const content = readFileSync(path, "utf8"); expect(path.endsWith(".mpd")).toBe(true); - expect(requests).toBe(2); - expect(content).toContain( - "https://cdn.example/video/media/?Policy=a&Signature=b", + expect(requestedUrls).toContain( + "https://cdn.example/video/init.mp4?Policy=a&Signature=b", ); - expect(content).toContain("seg-1.m4s?Policy=a&Signature=b"); + expect(requestedUrls).toContain( + "https://cdn.example/video/seg-1.m4s?Policy=a&Signature=b", + ); + expect(content).not.toContain("https://cdn.example"); } finally { globalThis.fetch = originalFetch; rmSync(manifestDir, { recursive: true, force: true }); @@ -877,10 +875,7 @@ describe("processVideo integration tests", () => { ); } - return new Response("not found", { - status: 404, - statusText: "Not Found", - }); + return new Response("segment", { status: 200, statusText: "OK" }); }) as unknown as typeof fetch; try { @@ -896,21 +891,24 @@ describe("processVideo integration tests", () => { expect(requestedUrls).toContain( "https://cdn.example/video/variant.m3u8?Policy=a&Signature=b", ); - expect(variantPath).toBeTruthy(); - - const variant = readFileSync(variantPath ?? "", "utf8"); - expect(variant).toContain( - 'URI="https://cdn.example/video/init.mp4?Policy=a&Signature=b"', + expect(requestedUrls).toContain( + "https://cdn.example/video/init.mp4?Policy=a&Signature=b", ); - expect(variant).toContain( - 'URI="https://cdn.example/video/key.bin?Policy=a&Signature=b"', + expect(requestedUrls).toContain( + "https://cdn.example/video/key.bin?Policy=a&Signature=b", ); - expect(variant).toContain( + expect(requestedUrls).toContain( "https://cdn.example/video/segment-1.ts?Policy=a&Signature=b", ); - expect(variant).toContain( + expect(requestedUrls).toContain( "https://cdn.example/video/segment-2.ts?Policy=a&Signature=b", ); + expect(variantPath).toBeTruthy(); + + const variant = readFileSync(variantPath ?? "", "utf8"); + expect(variant).not.toContain("https://cdn.example"); + expect(variant).toContain(".ts"); + expect(variant).toContain(".bin"); } finally { globalThis.fetch = originalFetch; rmSync(manifestDir, { recursive: true, force: true }); diff --git a/apps/media-server/src/lib/audio-levels.ts b/apps/media-server/src/lib/audio-levels.ts index 90b639adf46..273b15c393f 100644 --- a/apps/media-server/src/lib/audio-levels.ts +++ b/apps/media-server/src/lib/audio-levels.ts @@ -9,6 +9,7 @@ import { import { withMediaTransfers } from "./media-transfer"; import { uploadFileToStorage } from "./media-video"; import { verifyRemoteRecordingBytes } from "./recording-verification"; +import { fetchAllowed } from "./url-policy"; import { tryAcquireDirectVideoProcessSlot } from "./video-capacity"; const preparedSchema = z.object({ @@ -74,14 +75,13 @@ export async function enhanceLocalRecording(input: { AbortSignal.timeout(120_000), ]); const callback = async (body: Record) => { - const response = await fetch(webhookUrl, { + const response = await fetchAllowed(webhookUrl, { method: "POST", headers: { "Content-Type": "application/json", "x-media-server-secret": webhookSecret, }, body: JSON.stringify({ kind: "audio-levels", ...body }), - redirect: "error", signal, }); if (!response.ok) throw new Error("Audio publication unavailable"); diff --git a/apps/media-server/src/lib/job-manager.ts b/apps/media-server/src/lib/job-manager.ts index c34d0a18dd2..3ad843e5bbe 100644 --- a/apps/media-server/src/lib/job-manager.ts +++ b/apps/media-server/src/lib/job-manager.ts @@ -6,6 +6,7 @@ import { import { getContainerMemoryMetrics } from "./container-memory"; import type { MediaOperationHandle } from "./media-operations"; import type { TempFileHandle } from "./temp-files"; +import { fetchAllowed } from "./url-policy"; import { getActiveDirectVideoProcessCount } from "./video-capacity"; export type JobPhase = @@ -753,7 +754,7 @@ async function deliverWebhook( const startedAt = Date.now(); job.webhookLastAttemptAt = startedAt; try { - const resp = await fetch(job.webhookUrl, { + const resp = await fetchAllowed(job.webhookUrl, { method: "POST", headers, body, diff --git a/apps/media-server/src/lib/media-audio.ts b/apps/media-server/src/lib/media-audio.ts index 4248c106643..e7497d34081 100644 --- a/apps/media-server/src/lib/media-audio.ts +++ b/apps/media-server/src/lib/media-audio.ts @@ -13,7 +13,7 @@ import { import { materializeStreamingInput } from "./media-video"; import { registerSubprocess, terminateProcess } from "./subprocess"; import { ensureTempDir, getTempDir } from "./temp-files"; -import { resolveAllowedUrl } from "./url-policy"; +import { fetchAllowed } from "./url-policy"; export interface AudioExtractionOptions { format?: "mp3"; @@ -120,6 +120,8 @@ function getAudioExtractArgs( ): string[] { return [ "ffmpeg", + "-protocol_whitelist", + "file,pipe", "-i", videoUrl, "-vn", @@ -135,10 +137,13 @@ function getAudioExtractArgs( function getAudioProbeArgs(inputPath: string): string[] { const normalizedPath = (inputPath.split("?")[0] ?? "").toLowerCase(); - const args = ["ffprobe", "-v", "error"]; - if (normalizedPath.endsWith(".m3u8") || normalizedPath.endsWith(".mpd")) { - args.push("-protocol_whitelist", "file,http,https,tcp,tls,crypto,data"); - } + const args = [ + "ffprobe", + "-v", + "error", + "-protocol_whitelist", + "file,pipe,crypto,data", + ]; if (normalizedPath.endsWith(".m3u8")) { args.push( "-allowed_extensions", @@ -164,33 +169,41 @@ function isRemoteStreamingUrl(value: string): boolean { } } -async function prepareAudioProbeInput( +async function stageRemoteInput( videoUrl: string, - abortSignal: AbortSignal, + abortSignal?: AbortSignal, ): Promise<{ inputPath: string; cleanup: () => Promise }> { - if (!isRemoteStreamingUrl(videoUrl)) { - const inputPath = /^https?:/i.test(videoUrl) - ? await resolveAllowedUrl(videoUrl) - : videoUrl; - return { inputPath, cleanup: async () => {} }; + if (!/^https?:/i.test(videoUrl)) { + return { inputPath: videoUrl, cleanup: async () => {} }; } await ensureTempDir(); - const dirPath = await mkdtemp(join(getTempDir(), "audio-probe-")); + const dirPath = await mkdtemp(join(getTempDir(), "audio-input-")); + const cleanup = async () => { + await rm(dirPath, { recursive: true, force: true }).catch(() => {}); + }; try { - const inputPath = await materializeStreamingInput( - videoUrl, - dirPath, - abortSignal, - ); - return { - inputPath, - cleanup: async () => { - await rm(dirPath, { recursive: true, force: true }); - }, - }; + if (isRemoteStreamingUrl(videoUrl)) { + return { + inputPath: await materializeStreamingInput( + videoUrl, + dirPath, + abortSignal, + ), + cleanup, + }; + } + const response = await fetchAllowed(videoUrl, { signal: abortSignal }); + if (!response.ok) { + throw new Error( + `Failed to fetch media: ${response.status} ${response.statusText}`, + ); + } + const inputPath = join(dirPath, "input"); + await Bun.write(inputPath, response); + return { inputPath, cleanup }; } catch (error) { - await rm(dirPath, { recursive: true, force: true }).catch(() => {}); + await cleanup(); throw error; } } @@ -274,7 +287,7 @@ export async function checkHasAudioTrack(videoUrl: string): Promise { setCancel(cancel); return await withTimeout( (async () => { - const preparedInput = await prepareAudioProbeInput( + const preparedInput = await stageRemoteInput( videoUrl, abortController.signal, ); @@ -309,18 +322,31 @@ export async function extractAudio( const opts = { ...DEFAULT_OPTIONS, ...options }; return await withMediaOperation("audio", async (setCancel) => { - const proc = registerSubprocess( - spawn({ - cmd: getAudioExtractArgs(videoUrl, opts), - stdout: "pipe", - stderr: "pipe", - }), - ); - setCancel(() => terminateProcess(proc)); + const abortController = new AbortController(); + let proc: Subprocess | undefined; + let cleanupStaged = async () => {}; + const cancel = async () => { + abortController.abort(); + if (proc) await terminateProcess(proc); + await cleanupStaged(); + }; + setCancel(cancel); try { return await withTimeout( (async () => { + const staged = await stageRemoteInput( + videoUrl, + abortController.signal, + ); + cleanupStaged = staged.cleanup; + proc = registerSubprocess( + spawn({ + cmd: getAudioExtractArgs(staged.inputPath, opts), + stdout: "pipe", + stderr: "pipe", + }), + ); const stderrPromise = readStreamWithLimit( proc.stderr as ReadableStream, MAX_STDERR_BYTES, @@ -372,10 +398,11 @@ export async function extractAudio( return output; })(), opts.timeoutMs, - () => terminateProcess(proc), + cancel, ); } finally { - await terminateProcess(proc); + if (proc) await terminateProcess(proc); + await cleanupStaged(); } }); } @@ -389,6 +416,16 @@ export function extractAudioStream( } const opts = { ...DEFAULT_OPTIONS, ...options }; + if (/^https?:/i.test(videoUrl)) { + return openRemoteAudioExtractStream(videoUrl, opts); + } + return openAudioExtractStream(videoUrl, opts); +} + +function openAudioExtractStream( + videoUrl: string, + opts: Required, +): StreamingExtractResult { const proc: Subprocess = registerSubprocess( spawn({ cmd: getAudioExtractArgs(videoUrl, opts), @@ -464,3 +501,94 @@ export function extractAudioStream( return { stream, cleanup }; } + +function openRemoteAudioExtractStream( + videoUrl: string, + opts: Required, +): StreamingExtractResult { + const abortController = new AbortController(); + let cleaned = false; + let proc: Subprocess | undefined; + let cleanupStaged = async () => {}; + let reader: ReadableStreamDefaultReader | null = null; + let timeoutId: ReturnType | undefined; + let resolveReady: () => void = () => {}; + const ready = new Promise((resolve) => { + resolveReady = resolve; + }); + let operation: ReturnType | undefined; + + const cleanup = () => { + if (cleaned) return; + cleaned = true; + abortController.abort(); + if (timeoutId) clearTimeout(timeoutId); + if (reader) { + try { + reader.cancel().catch(() => {}); + reader.releaseLock(); + } catch {} + reader = null; + } + if (operation) unregisterMediaOperation(operation); + if (proc) void terminateProcess(proc); + void cleanupStaged(); + }; + operation = registerMediaOperation("audio", cleanup); + + timeoutId = setTimeout(cleanup, opts.timeoutMs); + + const stream = new ReadableStream( + { + async start() { + try { + const staged = await stageRemoteInput( + videoUrl, + abortController.signal, + ); + cleanupStaged = staged.cleanup; + if (cleaned) return; + proc = registerSubprocess( + spawn({ + cmd: getAudioExtractArgs(staged.inputPath, opts), + stdout: "pipe", + stderr: "pipe", + }), + ); + void drainStream(proc.stderr as ReadableStream); + proc.exited.then(() => { + cleanup(); + }); + reader = (proc.stdout as ReadableStream).getReader(); + } finally { + resolveReady(); + } + }, + async pull(controller) { + await ready; + if (!reader || cleaned) { + controller.close(); + return; + } + try { + const { done, value } = await reader.read(); + if (done) { + controller.close(); + cleanup(); + } else { + controller.enqueue(value); + } + } catch (err) { + controller.error(err); + cleanup(); + } + }, + cancel() { + cleanup(); + }, + }, + new CountQueuingStrategy({ highWaterMark: 4 }), + ); + + return { stream, cleanup }; +} diff --git a/apps/media-server/src/lib/media-common.ts b/apps/media-server/src/lib/media-common.ts index ad3c692242e..0d9d74e26f2 100644 --- a/apps/media-server/src/lib/media-common.ts +++ b/apps/media-server/src/lib/media-common.ts @@ -24,13 +24,22 @@ import { UrlSource, } from "mediabunny"; import { registerMediaEngine } from "./media-engine"; +import { fetchAllowed } from "./url-policy"; export type ProgressCallback = (progress: number, message: string) => void; export const PROCESS_TIMEOUT_MS = 45 * 60 * 1000; export const DOWNLOAD_TIMEOUT_MS = 10 * 60 * 1000; export const UPLOAD_TIMEOUT_MS = 10 * 60 * 1000; -const mediaFetch: typeof fetch = globalThis.fetch.bind(globalThis); +const mediaFetch: typeof fetch = (input, init) => { + const url = + typeof input === "string" + ? input + : input instanceof URL + ? input.toString() + : input.url; + return fetchAllowed(url, init); +}; export async function withTimeout( promise: Promise, diff --git a/apps/media-server/src/lib/media-probe.ts b/apps/media-server/src/lib/media-probe.ts index ed56443e1f9..185397d367b 100644 --- a/apps/media-server/src/lib/media-probe.ts +++ b/apps/media-server/src/lib/media-probe.ts @@ -12,7 +12,7 @@ import { withMediaOperation, } from "./media-operations"; import { materializeMedia } from "./media-transfer"; -import { fetchAllowed, resolveAllowedUrl } from "./url-policy"; +import { fetchAllowed, getUrlPolicy, resolveAllowedUrl } from "./url-policy"; const PROBE_TIMEOUT_MS = 30_000; @@ -46,11 +46,15 @@ function isHttpUrl(path: string): boolean { } } -async function hasHttpNetworkFailure(path: string): Promise { +async function hasHttpNetworkFailure( + path: string, + signal?: AbortSignal, +): Promise { try { + const timeout = AbortSignal.timeout(10_000); const response = await fetchAllowed(path, { headers: { Range: "bytes=0-0" }, - signal: AbortSignal.timeout(10_000), + signal: signal ? AbortSignal.any([signal, timeout]) : timeout, }); await response.body?.cancel(); return !response.ok; @@ -59,8 +63,11 @@ async function hasHttpNetworkFailure(path: string): Promise { } } -async function probeMedia(path: string): Promise { - if (isHttpUrl(path) && (await hasHttpNetworkFailure(path))) { +async function probeMedia( + path: string, + signal?: AbortSignal, +): Promise { + if (isHttpUrl(path) && (await hasHttpNetworkFailure(path, signal))) { throw new Error("Media input is not accessible"); } @@ -126,10 +133,25 @@ export async function probeVideo(videoUrl: string): Promise { throw new Error("Server is busy, please try again later"); } - const allowedUrl = await resolveAllowedUrl(videoUrl); - return await withMediaOperation("probe", () => - withTimeout(probeMedia(allowedUrl), PROBE_TIMEOUT_MS), - ); + return await withMediaOperation("probe", () => { + const controller = new AbortController(); + return withTimeout( + (async () => { + const allowedUrl = /^https?:/i.test(videoUrl) + ? await resolveAllowedUrl( + videoUrl, + getUrlPolicy(), + undefined, + undefined, + controller.signal, + ) + : videoUrl; + return probeMedia(allowedUrl, controller.signal); + })(), + PROBE_TIMEOUT_MS, + () => controller.abort(), + ); + }); } export async function probeVideoFile(filePath: string): Promise { diff --git a/apps/media-server/src/lib/media-video.ts b/apps/media-server/src/lib/media-video.ts index 3b3d46afb69..0315802dc94 100644 --- a/apps/media-server/src/lib/media-video.ts +++ b/apps/media-server/src/lib/media-video.ts @@ -33,7 +33,7 @@ import { getTempDir, type TempFileHandle, } from "./temp-files"; -import { assertUrlAllowed, fetchAllowed } from "./url-policy"; +import { fetchAllowed } from "./url-policy"; const PROCESS_TIMEOUT_PER_SECOND_MS = 20_000; const MAX_PROCESS_TIMEOUT_MS = 2 * 60 * 60 * 1000; @@ -238,6 +238,56 @@ function resolveResourceUrl( return withQuery(resolved.toString(), query); } +type Limit = (task: () => Promise) => Promise; + +function createLimiter(limit: number): Limit { + let active = 0; + const waiting: Array<() => void> = []; + return async function run(task: () => Promise): Promise { + active += 1; + if (active > limit) { + await new Promise((resolve) => waiting.push(resolve)); + } + try { + return await task(); + } finally { + active -= 1; + waiting.shift()?.(); + } + }; +} + +function localResourceName(url: string): string { + let extension = ""; + try { + const base = new URL(url).pathname.split("/").pop() ?? ""; + const match = base.match(/(\.[a-z0-9]{1,8})$/i); + if (match?.[1]) extension = match[1].toLowerCase(); + } catch {} + return `${randomUUID()}${extension}`; +} + +async function downloadAllowedResource( + url: string, + dirPath: string, + abortSignal: AbortSignal | undefined, + limit: Limit, +): Promise { + return limit(async () => { + const response = await fetchAllowed(url, { + signal: getFetchSignal(DOWNLOAD_TIMEOUT_MS, abortSignal), + }); + if (!response.ok) { + throw new Error( + `Failed to fetch media resource: ${response.status} ${response.statusText}`, + ); + } + const name = localResourceName(url); + await Bun.write(join(dirPath, name), response); + return name; + }); +} + function getFetchSignal(timeoutMs: number, abortSignal?: AbortSignal) { const timeoutSignal = AbortSignal.timeout(timeoutMs); return abortSignal @@ -283,6 +333,7 @@ export async function materializeHlsPlaylist( dirPath: string, cache = new Map(), abortSignal?: AbortSignal, + limit: Limit = createLimiter(4), ): Promise { const cached = cache.get(playlistUrl); if (cached) return cached; @@ -319,10 +370,15 @@ export async function materializeHlsPlaylist( dirPath, cache, abortSignal, + limit, ); } - await assertUrlAllowed(resolved); - return resolved; + return await downloadAllowedResource( + resolved, + dirPath, + abortSignal, + limit, + ); } if (!line.includes('URI="')) return line; @@ -336,9 +392,19 @@ export async function materializeHlsPlaylist( const resolved = resolveResourceUrl(original, baseUrl, query); const replacement = isHlsUrl(resolved) - ? await materializeHlsPlaylist(resolved, dirPath, cache, abortSignal) - : resolved; - if (!isHlsUrl(resolved)) await assertUrlAllowed(resolved); + ? await materializeHlsPlaylist( + resolved, + dirPath, + cache, + abortSignal, + limit, + ) + : await downloadAllowedResource( + resolved, + dirPath, + abortSignal, + limit, + ); rewritten = rewritten.replace( `URI="${original}"`, @@ -400,9 +466,31 @@ export async function materializeMpdManifest( return `${attribute}="${escapeXmlAttribute(resolved)}"`; }, ); - await Promise.all(resolvedUrls.map((url) => assertUrlAllowed(url))); + for (const url of resolvedUrls) { + if (url.includes("$")) { + throw new Error("DASH template URL cannot be fetched locally"); + } + } + const limit = createLimiter(4); + const localByUrl = new Map(); + await Promise.all( + [...new Set(resolvedUrls)].map(async (url) => { + localByUrl.set( + url, + await downloadAllowedResource(url, dirPath, abortSignal, limit), + ); + }), + ); + let localized = rewritten; + for (const [url, name] of [...localByUrl.entries()].sort( + (left, right) => right[0].length - left[0].length, + )) { + localized = localized + .split(escapeXmlAttribute(url)) + .join(escapeXmlAttribute(name)); + } - await writeFile(filePath, rewritten); + await writeFile(filePath, localized); return filePath; } @@ -835,14 +923,28 @@ export async function materializeMpdAsHlsPlaylist( query, presentationDuration, ); - await assertUrlAllowed(initUrl); - await Promise.all( - segments.map((segment) => assertUrlAllowed(segment.url)), + const limit = createLimiter(4); + const localInit = await downloadAllowedResource( + initUrl, + dirPath, + abortSignal, + limit, + ); + const localSegments = await Promise.all( + segments.map(async (segment) => ({ + ...segment, + url: await downloadAllowedResource( + segment.url, + dirPath, + abortSignal, + limit, + ), + })), ); const path = await writeDashRepresentationPlaylist( dirPath, - initUrl, - segments, + localInit, + localSegments, ); playlists.push({ @@ -1139,7 +1241,7 @@ export function buildStreamingDownloadFfmpegArgs( "-threads", "2", "-protocol_whitelist", - "file,http,https,tcp,tls,crypto,data", + "file,crypto,data", "-allowed_extensions", "ALL", ...(capabilities.allowedSegmentExtensions @@ -2288,7 +2390,7 @@ async function uploadWithRetry( }; if (ifNoneMatch) headers["If-None-Match"] = ifNoneMatch; - response = await fetch(presignedUrl, { + response = await fetchAllowed(presignedUrl, { method: "PUT", headers, body: bodyFactory(), @@ -2395,7 +2497,7 @@ async function postMultipartJson>( headers["x-media-server-secret"] = webhookSecret; } - return await fetch(url, { + return await fetchAllowed(url, { method: "POST", headers, body: JSON.stringify(body), @@ -2451,7 +2553,7 @@ async function uploadMultipartPart( abortSignal?.throwIfAborted(); let response: Response; try { - response = await fetch(url, { + response = await fetchAllowed(url, { method: "PUT", headers: { "Content-Length": contentLength.toString(), diff --git a/apps/media-server/src/lib/url-policy.ts b/apps/media-server/src/lib/url-policy.ts index 1de9e6a9c4b..7bcac240547 100644 --- a/apps/media-server/src/lib/url-policy.ts +++ b/apps/media-server/src/lib/url-policy.ts @@ -230,11 +230,21 @@ async function redirectTarget( return new URL(location, current).toString(); } +function withManualRedirect(init: RequestInit): RequestInit { + const body = init.body; + return { + ...init, + body: body instanceof Blob ? body.slice() : body, + redirect: "manual", + }; +} + export async function resolveAllowedUrl( input: string, policy: UrlPolicy = getUrlPolicy(), resolve: Resolver = defaultResolver, fetcher: Fetcher = fetch, + signal?: AbortSignal, ): Promise { let current = input; for (let hop = 0; hop <= MAX_REDIRECTS; hop++) { @@ -242,10 +252,15 @@ export async function resolveAllowedUrl( let response = await fetcher(current, { method: "HEAD", redirect: "manual", + signal, }); if (response.status === 405 || response.status === 501) { await response.body?.cancel(); - response = await fetcher(current, { method: "GET", redirect: "manual" }); + response = await fetcher(current, { + method: "GET", + redirect: "manual", + signal, + }); } const next = await redirectTarget(response, current); await response.body?.cancel(); @@ -268,7 +283,7 @@ export async function fetchAllowed( let current = input; for (let hop = 0; hop <= MAX_REDIRECTS; hop++) { await assertUrlAllowed(current, policy, resolve); - const response = await fetcher(current, { ...init, redirect: "manual" }); + const response = await fetcher(current, withManualRedirect(init)); const next = await redirectTarget(response, current); if (!next) return response; await response.body?.cancel(); diff --git a/apps/web/__tests__/unit/email-config.test.ts b/apps/web/__tests__/unit/email-config.test.ts index f0ef22ef6ed..56aeb14796b 100644 --- a/apps/web/__tests__/unit/email-config.test.ts +++ b/apps/web/__tests__/unit/email-config.test.ts @@ -57,6 +57,21 @@ describe("email config", () => { } }); + it("sends an explicit sender without a default from-domain", async () => { + mocks.env.RESEND_API_KEY = "re_test"; + mocks.send.mockResolvedValue({ data: { id: "email_2" }, error: null }); + + const result = await sendEmail({ + ...message, + fromOverride: "Cap Support ", + }); + + expect(mocks.send.mock.calls[0]?.[0].from).toBe( + "Cap Support ", + ); + expect(result).toEqual({ data: { id: "email_2" }, error: null }); + }); + it("sends from the configured domain and returns the provider id", async () => { mocks.env.RESEND_API_KEY = "re_test"; mocks.env.RESEND_FROM_DOMAIN = "example.com"; diff --git a/apps/web/content/docs/self-hosting.mdx b/apps/web/content/docs/self-hosting.mdx index de104cdfc5c..33fb1af5690 100644 --- a/apps/web/content/docs/self-hosting.mdx +++ b/apps/web/content/docs/self-hosting.mdx @@ -132,7 +132,7 @@ The media server only fetches the URLs cap-web sends it, and it checks each one - Trusted hosts are those in `S3_INTERNAL_ENDPOINT`, `S3_PUBLIC_ENDPOINT`, `CAP_AWS_ENDPOINT`, `MEDIA_SERVER_WEBHOOK_URL` and `WEB_URL` (set them on the media server too), plus anything in `MEDIA_SERVER_ALLOWED_HOSTS`. - When `MEDIA_SERVER_ALLOWED_HOSTS` is set, it becomes a strict allow-list: every URL must use one of those hosts or the trusted hosts above. Entries are comma-separated and may use a `*.` prefix, e.g. `minio.railway.internal,*.r2.cloudflarestorage.com`. -If your storage lives on a private network (MinIO in Docker or on Railway's private network), set `S3_INTERNAL_ENDPOINT` on the media server or add its host to `MEDIA_SERVER_ALLOWED_HOSTS`. Otherwise processing fails with `URL_NOT_ALLOWED`. `MEDIA_SERVER_ALLOW_PRIVATE_URLS=true` turns the private-address check off and also allows `file:` URLs; it's meant for local development and tests only. +If your storage lives on a private network (MinIO in Docker or on Railway's private network), set `S3_INTERNAL_ENDPOINT` on the media server or add its host to `MEDIA_SERVER_ALLOWED_HOSTS`. Otherwise processing fails with `URL_NOT_ALLOWED`. `MEDIA_SERVER_ALLOW_PRIVATE_URLS=true` turns the private-address check off for `http` and `https` URLs. It is meant for local development and tests only. ## Optional Features diff --git a/packages/database/emails/config.ts b/packages/database/emails/config.ts index 5f049c77f2f..87ce2ea3572 100644 --- a/packages/database/emails/config.ts +++ b/packages/database/emails/config.ts @@ -74,12 +74,10 @@ export const sendEmail = async ({ contentType?: string; }[]; }): Promise => { - const status = getEmailDeliveryStatus(); + const env = serverEnv(); const r = resend(); - if (!status.configured || !r) { - const reason = status.configured - ? EMAIL_NOT_CONFIGURED_MESSAGE - : status.reason; + if (!env.RESEND_API_KEY || !r) { + const reason = `${EMAIL_NOT_CONFIGURED_MESSAGE}: RESEND_API_KEY is not set`; warnEmailNotConfigured(reason); return { data: null, @@ -88,13 +86,23 @@ export const sendEmail = async ({ } if (marketing && !buildEnv.NEXT_PUBLIC_IS_CAP) return; - let from: string; - - if (fromOverride) from = fromOverride; - else if (marketing) from = "Richie from Cap "; - else if (buildEnv.NEXT_PUBLIC_IS_CAP) - from = "Cap Auth "; - else from = `auth@${serverEnv().RESEND_FROM_DOMAIN}`; + const from = fromOverride + ? fromOverride + : marketing + ? "Richie from Cap " + : buildEnv.NEXT_PUBLIC_IS_CAP + ? "Cap Auth " + : env.RESEND_FROM_DOMAIN?.trim() + ? `auth@${env.RESEND_FROM_DOMAIN.trim()}` + : undefined; + if (!from?.includes("@")) { + const reason = `${EMAIL_NOT_CONFIGURED_MESSAGE}: RESEND_FROM_DOMAIN is not set`; + warnEmailNotConfigured(reason); + return { + data: null, + error: { name: "email_not_configured", message: reason }, + }; + } const result = await r.emails.send( { From 7d40c0a38396196ac319b6a25dfff8c81f1f0951 Mon Sep 17 00:00:00 2001 From: 07rjain <07rishabhjain@gmail.com> Date: Wed, 14 Oct 2026 20:53:28 +0530 Subject: [PATCH 12/20] fix(media-server): download thumbnails before ffmpeg sees the URL A remote thumbnail input was still opened by ffmpeg, so a redirect to a private address was fetched. The file is downloaded through the URL check first, and cancellation during that download still aborts the request. Co-authored-by: Cursor --- .../lib/media-video.integration.test.ts | 30 +------ .../src/__tests__/lib/redirect-follow.test.ts | 84 +++++++++++++++++++ apps/media-server/src/lib/media-video.ts | 60 ++++++++----- 3 files changed, 122 insertions(+), 52 deletions(-) create mode 100644 apps/media-server/src/__tests__/lib/redirect-follow.test.ts diff --git a/apps/media-server/src/__tests__/lib/media-video.integration.test.ts b/apps/media-server/src/__tests__/lib/media-video.integration.test.ts index 1a2c9da0b5b..e229a640c62 100644 --- a/apps/media-server/src/__tests__/lib/media-video.integration.test.ts +++ b/apps/media-server/src/__tests__/lib/media-video.integration.test.ts @@ -3,7 +3,6 @@ import { execFileSync } from "node:child_process"; import { existsSync, mkdtempSync, - readdirSync, readFileSync, rmSync, statSync, @@ -329,7 +328,7 @@ describe("generateThumbnail integration tests", () => { } }); - test("joins an in-flight thumbnail decoder when its worker is cancelled", async () => { + test("cancels an in-flight remote thumbnail download", async () => { let ready: (() => void) | undefined; const started = new Promise((resolve) => { ready = resolve; @@ -358,35 +357,8 @@ describe("generateThumbnail integration tests", () => { ); try { await withTimeout(started, 5_000); - const commands = - process.platform === "linux" - ? readdirSync("/proc") - .filter((entry) => /^\d+$/.test(entry)) - .map((pid) => { - try { - return `${pid} ${readFileSync(`/proc/${pid}/cmdline`, "utf8")}`; - } catch (error) { - if ( - error instanceof Error && - "code" in error && - (error.code === "ENOENT" || error.code === "ESRCH") - ) - return ""; - throw error; - } - }) - : execFileSync("ps", ["-axo", "pid=,command="]) - .toString() - .split("\n"); - const pids = commands - .filter( - (command) => command.includes("ffmpeg") && command.includes(input), - ) - .map((command) => Number.parseInt(command.trim(), 10)); - expect(pids).toHaveLength(1); controller.abort(reason); expect(await withTimeout(outcome, 7_000)).toBe(reason); - for (const pid of pids) expect(() => process.kill(pid, 0)).toThrow(); } finally { controller.abort(reason); await server.stop(true); diff --git a/apps/media-server/src/__tests__/lib/redirect-follow.test.ts b/apps/media-server/src/__tests__/lib/redirect-follow.test.ts new file mode 100644 index 00000000000..6be48a73e9f --- /dev/null +++ b/apps/media-server/src/__tests__/lib/redirect-follow.test.ts @@ -0,0 +1,84 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { extractAudio } from "../../lib/media-audio"; +import { probeVideo } from "../../lib/media-probe"; +import { generateThumbnail } from "../../lib/media-video"; + +const video = readFileSync( + join(import.meta.dir, "..", "fixtures", "test-with-audio.mp4"), +); + +describe("redirects to a private address", () => { + const previous = process.env.MEDIA_SERVER_ALLOW_PRIVATE_URLS; + + afterEach(() => { + if (previous === undefined) { + delete process.env.MEDIA_SERVER_ALLOW_PRIVATE_URLS; + } else { + process.env.MEDIA_SERVER_ALLOW_PRIVATE_URLS = previous; + } + }); + + function privateRedirectServer() { + let secretHits = 0; + const server = Bun.serve({ + port: 0, + fetch(request) { + const url = new URL(request.url); + if (url.pathname === "/secret") { + secretHits += 1; + return new Response(video, { + headers: { "content-type": "video/mp4" }, + }); + } + return new Response(null, { + status: 302, + headers: { + location: new URL("/secret", server.url).toString(), + }, + }); + }, + }); + return { + server, + url: `${server.url}clip.mp4`, + secretHits: () => secretHits, + }; + } + + test("probe does not fetch the redirect target", async () => { + process.env.MEDIA_SERVER_ALLOW_PRIVATE_URLS = "false"; + const target = privateRedirectServer(); + try { + await expect(probeVideo(target.url)).rejects.toThrow("private address"); + expect(target.secretHits()).toBe(0); + } finally { + target.server.stop(true); + } + }); + + test("audio extraction does not fetch the redirect target", async () => { + process.env.MEDIA_SERVER_ALLOW_PRIVATE_URLS = "false"; + const target = privateRedirectServer(); + try { + await expect(extractAudio(target.url)).rejects.toThrow("private address"); + expect(target.secretHits()).toBe(0); + } finally { + target.server.stop(true); + } + }); + + test("thumbnail generation does not fetch the redirect target", async () => { + process.env.MEDIA_SERVER_ALLOW_PRIVATE_URLS = "false"; + const target = privateRedirectServer(); + try { + await expect(generateThumbnail(target.url, 1)).rejects.toThrow( + "private address", + ); + expect(target.secretHits()).toBe(0); + } finally { + target.server.stop(true); + } + }); +}); diff --git a/apps/media-server/src/lib/media-video.ts b/apps/media-server/src/lib/media-video.ts index 0315802dc94..a0488b007b0 100644 --- a/apps/media-server/src/lib/media-video.ts +++ b/apps/media-server/src/lib/media-video.ts @@ -1929,33 +1929,47 @@ export async function generateThumbnail( const opts = resolveThumbnailOptions(options); const timestamp = getThumbnailTimestamp(duration, opts.timestamp); const deadline = performance.now() + THUMBNAIL_TIMEOUT_MS; + let localInput: { path: string; cleanup: () => Promise }; try { - return await generateThumbnailFrame( - inputPath, - opts, - timestamp, - THUMBNAIL_TIMEOUT_MS, - abortSignal, - ); + localInput = /^https?:/i.test(inputPath) + ? await downloadVideoToTemp(inputPath, undefined, abortSignal) + : { path: inputPath, cleanup: async () => {} }; } catch (error) { abortSignal?.throwIfAborted(); - const remainingMs = deadline - performance.now(); - if ( - !(error instanceof EmptyThumbnailError) || - !isAbsolute(normalizeLocalPath(inputPath)) || - timestamp <= 0 || - remainingMs <= 0 - ) { - throw error; + throw error; + } + try { + try { + return await generateThumbnailFrame( + localInput.path, + opts, + timestamp, + THUMBNAIL_TIMEOUT_MS, + abortSignal, + true, + ); + } catch (error) { + abortSignal?.throwIfAborted(); + const remainingMs = deadline - performance.now(); + if ( + !(error instanceof EmptyThumbnailError) || + !isAbsolute(normalizeLocalPath(localInput.path)) || + timestamp <= 0 || + remainingMs <= 0 + ) { + throw error; + } + return await generateThumbnailFrame( + localInput.path, + opts, + 0, + remainingMs, + abortSignal, + true, + ); } - return await generateThumbnailFrame( - inputPath, - opts, - 0, - remainingMs, - abortSignal, - true, - ); + } finally { + await localInput.cleanup(); } } From 49bc7349a25a23f9c5e87bda4a5cc26caf56e606 Mon Sep 17 00:00:00 2001 From: 07rjain <07rishabhjain@gmail.com> Date: Wed, 14 Oct 2026 21:00:30 +0530 Subject: [PATCH 13/20] fix(media-server): decode remote recordings from a checked local file ffmpeg was still opening the recording URL, so a redirect to a private address was fetched. The bytes are downloaded through the URL check first, and cancellation or the verification deadline still stops that download. Co-authored-by: Cursor --- ...recording-verification.integration.test.ts | 12 +- .../src/__tests__/lib/redirect-follow.test.ts | 17 +++ .../src/lib/recording-verification.ts | 119 ++++++++++++++---- 3 files changed, 116 insertions(+), 32 deletions(-) diff --git a/apps/media-server/src/__tests__/lib/recording-verification.integration.test.ts b/apps/media-server/src/__tests__/lib/recording-verification.integration.test.ts index 0d11a2fb7f9..f58754fed04 100644 --- a/apps/media-server/src/__tests__/lib/recording-verification.integration.test.ts +++ b/apps/media-server/src/__tests__/lib/recording-verification.integration.test.ts @@ -1940,10 +1940,6 @@ async function decoderPids(input: string): Promise { .map((line) => Number.parseInt(line.trim(), 10)); } -function expectExited(pid: number) { - expect(() => process.kill(pid, 0)).toThrow(); -} - describe("recording timing metadata lifetime", () => { test.each(["cancel", "timeout"])( "joins a stalled raw terminal-duration read after %s", @@ -2353,15 +2349,13 @@ describe("recording verification lifetime", () => { (error: unknown) => error, ); await request; - const pids = await decoderPids(input); - expect(pids.length).toBe(1); if (cause === "cancel") controller.abort(); const error = await outcome; expect(error).toBeInstanceOf(Error); expect((error as Error).message).toContain( cause === "cancel" ? "cancelled" : "timed out", ); - for (const pid of pids) expectExited(pid); + expect(await decoderPids(input)).toEqual([]); } finally { controller.abort(); await server.stop(true); @@ -2923,8 +2917,6 @@ describe("remote recording object identity", () => { (error: unknown) => error, ); await decoding; - const pids = await decoderPids(input); - expect(pids.length).toBe(1); if (cause === "cancel") controller.abort(); const error = await outcome; expect(error).toBeInstanceOf(Error); @@ -2935,7 +2927,7 @@ describe("remote recording object identity", () => { expect(isRetryableRecordingVerificationError(error)).toBe( cause === "timeout", ); - for (const pid of pids) expectExited(pid); + expect(await decoderPids(input)).toEqual([]); } finally { controller.abort(); await server.stop(true); diff --git a/apps/media-server/src/__tests__/lib/redirect-follow.test.ts b/apps/media-server/src/__tests__/lib/redirect-follow.test.ts index 6be48a73e9f..fc5ebd88051 100644 --- a/apps/media-server/src/__tests__/lib/redirect-follow.test.ts +++ b/apps/media-server/src/__tests__/lib/redirect-follow.test.ts @@ -4,6 +4,7 @@ import { join } from "node:path"; import { extractAudio } from "../../lib/media-audio"; import { probeVideo } from "../../lib/media-probe"; import { generateThumbnail } from "../../lib/media-video"; +import { verifyRecording } from "../../lib/recording-verification"; const video = readFileSync( join(import.meta.dir, "..", "fixtures", "test-with-audio.mp4"), @@ -81,4 +82,20 @@ describe("redirects to a private address", () => { target.server.stop(true); } }); + + test("recording verification does not fetch the redirect target", async () => { + process.env.MEDIA_SERVER_ALLOW_PRIVATE_URLS = "false"; + const target = privateRedirectServer(); + try { + await expect( + verifyRecording(target.url, { + allowObservedDuration: true, + requireAudio: false, + }), + ).rejects.toThrow("private address"); + expect(target.secretHits()).toBe(0); + } finally { + target.server.stop(true); + } + }); }); diff --git a/apps/media-server/src/lib/recording-verification.ts b/apps/media-server/src/lib/recording-verification.ts index 9a599fbf010..9d64396e6f0 100644 --- a/apps/media-server/src/lib/recording-verification.ts +++ b/apps/media-server/src/lib/recording-verification.ts @@ -1,8 +1,9 @@ import { spawn } from "node:child_process"; import { createHash, type Hash } from "node:crypto"; import { createReadStream } from "node:fs"; -import { lstat } from "node:fs/promises"; -import { isAbsolute } from "node:path"; +import { lstat, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { isAbsolute, join } from "node:path"; import { Readable } from "node:stream"; import { setTimeout as sleep } from "node:timers/promises"; import { PROCESS_TIMEOUT_MS } from "./media-common"; @@ -730,6 +731,62 @@ export async function verifyRemuxedRecording( }; } +async function stageRemoteRecording( + input: string, + abortSignal?: AbortSignal, + objectIdentity?: string, +): Promise<{ path: string; cleanup: () => Promise }> { + const directory = await mkdtemp(join(tmpdir(), "cap-recording-")); + const path = join(directory, "recording.mp4"); + const cleanup = () => rm(directory, { force: true, recursive: true }); + try { + abortSignal?.throwIfAborted(); + const response = await fetchMedia(input, { + headers: { + Range: "bytes=0-", + "X-Cap-Recording-Verification": "1", + ...(objectIdentity ? { "If-Match": objectIdentity } : {}), + }, + signal: abortSignal, + }); + try { + if ( + response.status === 408 || + response.status === 429 || + response.status >= 500 + ) { + throw new RecordingVerificationError( + `Recording object read failed: ${response.status}`, + true, + ); + } + if (!response.body) { + throw new Error(`Failed to download recording: ${response.status}`); + } + const reader = response.body.getReader(); + const writer = Bun.file(path).writer(); + try { + while (true) { + abortSignal?.throwIfAborted(); + const { done, value } = await reader.read(); + if (done) break; + writer.write(value); + } + await writer.end(); + } finally { + reader.releaseLock(); + } + } finally { + if (!response.body?.locked) await response.body?.cancel().catch(() => {}); + } + return { path, cleanup }; + } catch (error) { + await cleanup(); + abortSignal?.throwIfAborted(); + throw error; + } +} + export async function verifyRecording( input: string, options: RecordingVerificationOptions, @@ -762,13 +819,7 @@ async function decodeRecording( if (options.abortSignal?.aborted) { throw new Error("Recording verification was cancelled"); } - let remote = false; - if (isAbsolute(input)) { - const metadata = await lstat(input); - if (!metadata.isFile()) { - throw new Error("Recording verification requires a regular MP4 file"); - } - } else { + if (!isAbsolute(input)) { const url = new URL(input); if ( (url.protocol !== "http:" && url.protocol !== "https:") || @@ -777,7 +828,42 @@ async function decodeRecording( ) { throw new Error("Recording verification requires an HTTP(S) MP4 URL"); } - remote = true; + const deadline = new AbortController(); + const timeout = setTimeout(() => deadline.abort(), timeoutMs); + const signal = options.abortSignal + ? AbortSignal.any([deadline.signal, options.abortSignal]) + : deadline.signal; + try { + const staged = await stageRemoteRecording(input, signal, objectIdentity); + try { + return await decodeRecording( + staged.path, + options, + undefined, + sourceAudioInput, + packetProof, + ); + } finally { + await staged.cleanup(); + } + } catch (error) { + if (options.abortSignal?.aborted) { + throw new Error("Recording verification was cancelled"); + } + if (deadline.signal.aborted) { + throw new RecordingVerificationError( + "Recording verification timed out", + true, + ); + } + throw error; + } finally { + clearTimeout(timeout); + } + } + const metadata = await lstat(input); + if (!metadata.isFile()) { + throw new Error("Recording verification requires a regular MP4 file"); } if (sourceAudioInput && !(await lstat(sourceAudioInput)).isFile()) { throw new Error("Recording verification requires a regular MP4 file"); @@ -800,10 +886,6 @@ async function decodeRecording( ? AbortSignal.any([deadline.signal, options.abortSignal]) : deadline.signal; try { - const remoteObject = remote - ? await readRemoteIdentity(input, signal, objectIdentity) - : undefined; - objectIdentity = remoteObject?.objectIdentity ?? objectIdentity; const timingBudgetMs = timeoutMs - (performance.now() - startedAt); if (timingBudgetMs <= 0) { throw new RecordingVerificationError( @@ -812,7 +894,6 @@ async function decodeRecording( ); } videoTiming = await readRecordingVideoTiming(input, { - remoteObject, abortSignal: signal, timeoutMs: Math.ceil(timingBudgetMs), }); @@ -893,15 +974,9 @@ async function decodeRecording( "-threads", "1", "-protocol_whitelist", - remote ? "http,https,tcp,tls" : "file", + "file", "-f", "mov", - ...(objectIdentity - ? [ - "-headers", - `If-Match: ${objectIdentity}\r\nX-Cap-Recording-Verification: 1\r\n`, - ] - : []), "-i", input, ...(sourceAudioInput From a1aacbe599435259befd5c894675b43fd70626bb Mon Sep 17 00:00:00 2001 From: 07rjain <07rishabhjain@gmail.com> Date: Sun, 18 Oct 2026 16:13:28 +0530 Subject: [PATCH 14/20] fix(media-server): keep DASH bases and failed audio downloads from sticking A directory BaseURL was downloaded as a media file, so a 404 on that directory failed conversion. Segment URLs are resolved against the base and only the files are fetched. A failed remote audio download now releases its operation immediately. Co-authored-by: Cursor --- .../lib/media-audio.integration.test.ts | 18 +++ .../lib/media-video.integration.test.ts | 27 ++-- apps/media-server/src/lib/media-audio.ts | 3 + apps/media-server/src/lib/media-video.ts | 132 +++++++++++++----- 4 files changed, 139 insertions(+), 41 deletions(-) diff --git a/apps/media-server/src/__tests__/lib/media-audio.integration.test.ts b/apps/media-server/src/__tests__/lib/media-audio.integration.test.ts index 402b47be763..ae1191baf60 100644 --- a/apps/media-server/src/__tests__/lib/media-audio.integration.test.ts +++ b/apps/media-server/src/__tests__/lib/media-audio.integration.test.ts @@ -233,6 +233,24 @@ describe("mediaAudio integration tests", () => { await waitForAudioOperations(beforeCount); }); + test("releases the operation when a remote download fails", async () => { + const beforeCount = getActiveProcessCount(); + const server = Bun.serve({ + port: 0, + fetch: () => new Response("Missing", { status: 404 }), + }); + + try { + const { stream } = extractAudioStream(`${server.url}clip.mp4`); + await expect(readStream(stream)).rejects.toThrow( + "Failed to fetch media: 404", + ); + await waitForAudioOperations(beforeCount); + } finally { + await server.stop(true); + } + }); + test("cleans up when a real stream is cancelled", async () => { const beforeCount = getActiveProcessCount(); const { stream } = extractAudioStream(TEST_VIDEO_WITH_AUDIO); diff --git a/apps/media-server/src/__tests__/lib/media-video.integration.test.ts b/apps/media-server/src/__tests__/lib/media-video.integration.test.ts index e229a640c62..4438fa500ff 100644 --- a/apps/media-server/src/__tests__/lib/media-video.integration.test.ts +++ b/apps/media-server/src/__tests__/lib/media-video.integration.test.ts @@ -710,11 +710,14 @@ describe("processVideo integration tests", () => { const requestedUrls: string[] = []; globalThis.fetch = (async (input: RequestInfo | URL) => { - requestedUrls.push( - input instanceof Request ? input.url : input.toString(), - ); - return new Response( - ` + const url = input instanceof Request ? input.url : input.toString(); + requestedUrls.push(url); + if (new URL(url).pathname.endsWith("/")) { + return new Response("Missing directory", { status: 404 }); + } + if (url.includes("manifest.mpd")) { + return new Response( + ` @@ -727,8 +730,10 @@ describe("processVideo integration tests", () => { `, - { status: 200, statusText: "OK" }, - ); + { status: 200, statusText: "OK" }, + ); + } + return new Response("segment", { status: 200, statusText: "OK" }); }) as unknown as typeof fetch; try { @@ -740,12 +745,16 @@ describe("processVideo integration tests", () => { expect(path.endsWith(".mpd")).toBe(true); expect(requestedUrls).toContain( - "https://cdn.example/video/init.mp4?Policy=a&Signature=b", + "https://cdn.example/video/media/init.mp4?Policy=a&Signature=b", ); expect(requestedUrls).toContain( - "https://cdn.example/video/seg-1.m4s?Policy=a&Signature=b", + "https://cdn.example/video/media/seg-1.m4s?Policy=a&Signature=b", ); + expect( + requestedUrls.some((url) => new URL(url).pathname.endsWith("/")), + ).toBe(false); expect(content).not.toContain("https://cdn.example"); + expect(content).not.toContain(""); } finally { globalThis.fetch = originalFetch; rmSync(manifestDir, { recursive: true, force: true }); diff --git a/apps/media-server/src/lib/media-audio.ts b/apps/media-server/src/lib/media-audio.ts index e7497d34081..a41d5140d39 100644 --- a/apps/media-server/src/lib/media-audio.ts +++ b/apps/media-server/src/lib/media-audio.ts @@ -560,6 +560,9 @@ function openRemoteAudioExtractStream( cleanup(); }); reader = (proc.stdout as ReadableStream).getReader(); + } catch (error) { + cleanup(); + throw error; } finally { resolveReady(); } diff --git a/apps/media-server/src/lib/media-video.ts b/apps/media-server/src/lib/media-video.ts index a0488b007b0..e8f3de422ee 100644 --- a/apps/media-server/src/lib/media-video.ts +++ b/apps/media-server/src/lib/media-video.ts @@ -420,6 +420,102 @@ export async function materializeHlsPlaylist( return filePath; } +function currentDashBase(bases: string[], fallback: string): string { + return bases[bases.length - 1] ?? fallback; +} + +function rememberDashResource( + resource: string, + base: string, + query: string, + urls: string[], +): string { + const resolved = resolveResourceUrl(resource, base, query); + if (resolved.includes("$")) { + throw new Error("DASH template URL cannot be fetched locally"); + } + if (!new URL(resolved).pathname.endsWith("/")) urls.push(resolved); + return resolved; +} + +function rewriteDashAttributes( + attributes: string, + base: string, + query: string, + urls: string[], +): string { + return attributes.replace( + /(initialization|media|sourceURL|xlink:href|href)="([^"]+)"/gi, + (_, attribute: string, resource: string) => { + const resolved = rememberDashResource( + decodeXmlAttribute(resource), + base, + query, + urls, + ); + return `${attribute}="${escapeXmlAttribute(resolved)}"`; + }, + ); +} + +function rewriteDashManifest( + content: string, + manifestBase: string, + query: string, +): { urls: string[]; xml: string } { + const urls: string[] = []; + const bases = [manifestBase]; + const pattern = + /||<(BaseURL|Location)\b([^>]*)>([\s\S]*?)<\/\1>|<\/([A-Za-z0-9:_-]+)\s*>|<([A-Za-z0-9:_-]+)\b([^>]*?)(\/)?\s*>/gi; + let xml = ""; + let cursor = 0; + + for (const match of content.matchAll(pattern)) { + const index = match.index ?? 0; + xml += content.slice(cursor, index); + cursor = index + match[0].length; + if (match[0].startsWith("||<(BaseURL|Location)\b([^>]*)>([\s\S]*?)<\/\1>|<\/([A-Za-z0-9:_-]+)\s*>|<([A-Za-z0-9:_-]+)\b([^>]*?)(\/)?\s*>/gi; let xml = ""; @@ -481,39 +495,58 @@ function rewriteDashManifest( const baseElement = match[1]; if (baseElement) { + // MPD@Location points at a refreshed manifest, not at media. The + // materialized copy is static, so drop it instead of fetching it or + // letting it change how segments resolve. + if (baseElement.toLowerCase() === "location") continue; + + const frame = frames[frames.length - 1] ?? { + inherited: manifestBase, + effective: manifestBase, + hasBaseUrl: false, + }; + // Sibling BaseURLs are alternatives, so each one resolves against the + // enclosing element's base rather than against the previous sibling. const resolved = resolveResourceUrl( decodeXmlAttribute((match[3] ?? "").trim()), - currentDashBase(bases, manifestBase), + frame.inherited, query, ); const directory = new URL(resolved).pathname.endsWith("/"); - if (directory || baseElement.toLowerCase() === "location") { - bases[bases.length - 1] = new URL(".", resolved).toString(); - continue; + const primary = !frame.hasBaseUrl; + if (primary) { + frame.hasBaseUrl = true; + frame.effective = directory + ? new URL(".", resolved).toString() + : resolved; } + if (directory) continue; if (resolved.includes("$")) { throw new Error("DASH template URL cannot be fetched locally"); } - urls.push(resolved); - bases[bases.length - 1] = resolved; - xml += `<${baseElement}${rewriteDashAttributes(match[2] ?? "", resolved, query, urls)}>${escapeXmlAttribute(resolved)}`; + const element = `<${baseElement}${rewriteDashAttributes(match[2] ?? "", frame.effective, query, urls)}>${escapeXmlAttribute(resolved)}`; + if (primary) urls.push(resolved); + else alternatives.push({ url: resolved, element }); + xml += element; continue; } if (match[4]) { - if (bases.length > 1) bases.pop(); + if (frames.length > 1) frames.pop(); xml += match[0]; continue; } - const base = currentDashBase(bases, manifestBase); + const base = currentDashBase(frames, manifestBase); const attributes = rewriteDashAttributes(match[6] ?? "", base, query, urls); const selfClosing = Boolean(match[7]); xml += `<${match[5]}${attributes}${selfClosing ? " />" : ">"}`; - if (!selfClosing) bases.push(base); + if (!selfClosing) { + frames.push({ inherited: base, effective: base, hasBaseUrl: false }); + } } - return { urls, xml: xml + content.slice(cursor) }; + return { urls, alternatives, xml: xml + content.slice(cursor) }; } export async function materializeMpdManifest( @@ -534,15 +567,20 @@ export async function materializeMpdManifest( const content = await response.text(); const parsedUrl = new URL(manifestUrl); const filePath = join(dirPath, `${randomUUID()}.mpd`); - const { urls: resolvedUrls, xml: rewritten } = rewriteDashManifest( + const { + urls: resolvedUrls, + alternatives, + xml: rewritten, + } = rewriteDashManifest( content, new URL(".", parsedUrl).toString(), parsedUrl.search, ); const limit = createLimiter(4); const localByUrl = new Map(); + const required = new Set(resolvedUrls); await Promise.all( - [...new Set(resolvedUrls)].map(async (url) => { + [...required].map(async (url) => { localByUrl.set( url, await downloadAllowedResource(url, dirPath, abortSignal, limit), @@ -550,6 +588,32 @@ export async function materializeMpdManifest( }), ); let localized = rewritten; + // Alternative BaseURLs are only fallbacks, so one that is blocked or + // missing is removed from the local copy instead of failing the job. + const optional = [ + ...new Set( + alternatives.map(({ url }) => url).filter((url) => !required.has(url)), + ), + ]; + const failedAlternatives = new Set(); + await Promise.all( + optional.map(async (url) => { + try { + localByUrl.set( + url, + await downloadAllowedResource(url, dirPath, abortSignal, limit), + ); + } catch (error) { + if (abortSignal?.aborted) throw error; + failedAlternatives.add(url); + } + }), + ); + for (const { url, element } of alternatives) { + if (failedAlternatives.has(url)) { + localized = localized.split(element).join(""); + } + } for (const [url, name] of [...localByUrl.entries()].sort( (left, right) => right[0].length - left[0].length, )) { From dfab42214dbea28ea72bdf53a18bb50b45ab79f4 Mon Sep 17 00:00:00 2001 From: 07rjain <07rjain@users.noreply.github.com> Date: Fri, 23 Oct 2026 14:08:24 +0000 Subject: [PATCH 16/20] fix(media-server): send parallel ranged downloads through the URL policy Main's opt-in parallel download path (MEDIA_SERVER_DOWNLOAD_CONCURRENCY > 1) called fetch directly, so it skipped the private-address and redirect checks that every other remote read goes through. --- .../src/__tests__/lib/redirect-follow.test.ts | 21 ++++++++++++++++++- apps/media-server/src/lib/media-video.ts | 6 ++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/apps/media-server/src/__tests__/lib/redirect-follow.test.ts b/apps/media-server/src/__tests__/lib/redirect-follow.test.ts index fc5ebd88051..98e309442bb 100644 --- a/apps/media-server/src/__tests__/lib/redirect-follow.test.ts +++ b/apps/media-server/src/__tests__/lib/redirect-follow.test.ts @@ -3,7 +3,7 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { extractAudio } from "../../lib/media-audio"; import { probeVideo } from "../../lib/media-probe"; -import { generateThumbnail } from "../../lib/media-video"; +import { downloadVideoToTemp, generateThumbnail } from "../../lib/media-video"; import { verifyRecording } from "../../lib/recording-verification"; const video = readFileSync( @@ -98,4 +98,23 @@ describe("redirects to a private address", () => { target.server.stop(true); } }); + test("parallel ranged downloads do not fetch the redirect target", async () => { + process.env.MEDIA_SERVER_ALLOW_PRIVATE_URLS = "false"; + const previousConcurrency = process.env.MEDIA_SERVER_DOWNLOAD_CONCURRENCY; + process.env.MEDIA_SERVER_DOWNLOAD_CONCURRENCY = "4"; + const target = privateRedirectServer(); + try { + await expect(downloadVideoToTemp(target.url, "mp4")).rejects.toThrow( + "private address", + ); + expect(target.secretHits()).toBe(0); + } finally { + if (previousConcurrency === undefined) { + delete process.env.MEDIA_SERVER_DOWNLOAD_CONCURRENCY; + } else { + process.env.MEDIA_SERVER_DOWNLOAD_CONCURRENCY = previousConcurrency; + } + target.server.stop(true); + } + }); }); diff --git a/apps/media-server/src/lib/media-video.ts b/apps/media-server/src/lib/media-video.ts index b5878ea5a3c..86e2369f415 100644 --- a/apps/media-server/src/lib/media-video.ts +++ b/apps/media-server/src/lib/media-video.ts @@ -1525,6 +1525,12 @@ export async function downloadVideoToTemp( if (parallel.concurrency > 1) { await downloadUrlToPathParallel(videoUrl, tempFile.path, { ...parallel, + // Ranged parts go through the same URL policy as a single download. + fetchImpl: ((input, init) => + fetchAllowed( + input instanceof Request ? input.url : input.toString(), + init, + )) as typeof fetch, signal: combinedSignal, }); if (file(tempFile.path).size === 0) { From bad4a43b6e2f5e077900bf6e91e0d5c82f34d5d3 Mon Sep 17 00:00:00 2001 From: 07rjain <07rjain@users.noreply.github.com> Date: Fri, 23 Oct 2026 14:08:24 +0000 Subject: [PATCH 17/20] chore(self-host): turn on main's shutdown drain in the compose files Main's drain is opt-in through MEDIA_SERVER_SHUTDOWN_GRACE_MS. Set it to 25s, under the 30s stop_grace_period, so docker compose stop lets active conversions finish. --- docker-compose.coolify.yml | 2 ++ docker-compose.template.yml | 2 ++ docker-compose.yml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/docker-compose.coolify.yml b/docker-compose.coolify.yml index 329fbc617d6..e6789a4bd9f 100644 --- a/docker-compose.coolify.yml +++ b/docker-compose.coolify.yml @@ -48,6 +48,8 @@ services: restart: unless-stopped stop_grace_period: 30s environment: + # Finish active work for up to 25s on stop (below stop_grace_period). + MEDIA_SERVER_SHUTDOWN_GRACE_MS: '25000' PORT: 3456 S3_INTERNAL_ENDPOINT: 'http://minio:9000' S3_PUBLIC_ENDPOINT: '${SERVICE_URL_MINIO}' diff --git a/docker-compose.template.yml b/docker-compose.template.yml index 0b9bc5f7978..77720e830de 100644 --- a/docker-compose.template.yml +++ b/docker-compose.template.yml @@ -57,6 +57,8 @@ services: restart: unless-stopped stop_grace_period: 30s environment: + # Finish active work for up to 25s on stop (below stop_grace_period). + MEDIA_SERVER_SHUTDOWN_GRACE_MS: "25000" S3_INTERNAL_ENDPOINT: http://minio:3902 MEDIA_SERVER_WEBHOOK_URL: http://cap-web:3000 PORT: 3456 diff --git a/docker-compose.yml b/docker-compose.yml index a2e063eb886..b53dc21d584 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,6 +51,8 @@ services: restart: unless-stopped stop_grace_period: 30s environment: + # Finish active work for up to 25s on stop (below stop_grace_period). + MEDIA_SERVER_SHUTDOWN_GRACE_MS: "25000" PORT: 3456 S3_INTERNAL_ENDPOINT: http://minio:9000 S3_PUBLIC_ENDPOINT: ${S3_PUBLIC_URL:-http://localhost:9000} From 177ed4560a381883000dc476e65daa8eaed37827 Mon Sep 17 00:00:00 2001 From: 07rjain <07rjain@users.noreply.github.com> Date: Fri, 23 Oct 2026 14:08:24 +0000 Subject: [PATCH 18/20] docs(self-host): explain that upgrades stop on the old example secrets and how to replace them --- apps/web/content/docs/self-hosting.mdx | 28 +++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/apps/web/content/docs/self-hosting.mdx b/apps/web/content/docs/self-hosting.mdx index 33fb1af5690..9b7595f7b57 100644 --- a/apps/web/content/docs/self-hosting.mdx +++ b/apps/web/content/docs/self-hosting.mdx @@ -23,7 +23,7 @@ docker compose up -d That's it! Cap will be available at `http://localhost:3000`. -`docker compose up` stops with an error until `NEXTAUTH_SECRET`, `DATABASE_ENCRYPTION_KEY` and `MEDIA_SERVER_WEBHOOK_SECRET` are set. cap-web and the media server also refuse to start if any of them is left at a publicly known placeholder. Create your `.env` first: +`docker compose up` stops with an error until `NEXTAUTH_SECRET`, `DATABASE_ENCRYPTION_KEY` and `MEDIA_SERVER_WEBHOOK_SECRET` are set. cap-web and the media server also refuse to start if any of them is left at a publicly known placeholder, including the example values from older compose files. Upgrading an existing install? Read [Upgrading From an Older Release](#upgrading-from-an-older-release) first. Create your `.env` first: ```bash cp .env.self-hosting.example .env @@ -214,6 +214,32 @@ APPLE_CLIENT_SECRET=your-client-secret-jwt Apple client-secret JWTs expire, so rotate the secret before its configured expiration date. +## Upgrading From an Older Release + + +Older `docker-compose.yml`, `docker-compose.template.yml` and Coolify files shipped fixed example secrets. Starting with this release, **cap-web and the media server refuse to start** if any of these is missing or still set to one of those public example values: + +- `NEXTAUTH_SECRET` (cap-web) +- `DATABASE_ENCRYPTION_KEY` (cap-web) +- `MEDIA_SERVER_WEBHOOK_SECRET` (cap-web and the media server; it must be the same on both) + +With Docker Compose, `docker compose up` stops before starting any container and names the missing variable. Otherwise the container exits on start with `Refusing to start: insecure secrets` (cap-web) or `[media-server] Refusing to start` in its logs. + + +If you never changed the example secrets, do this before you pull the new images: + +1. Back up MySQL. +2. Generate a new value for each secret with `openssl rand -hex 32` and put them in the `.env` next to your compose file (or in your platform's variables). Use the same `MEDIA_SERVER_WEBHOOK_SECRET` on cap-web and the media server. +3. Pull and restart: `docker compose pull && docker compose up -d`. + +What changes when you replace each secret: + +- `NEXTAUTH_SECRET`: everyone is signed out and signs in again. +- `MEDIA_SERVER_WEBHOOK_SECRET`: nothing, as long as cap-web and the media server get the same new value. +- `DATABASE_ENCRYPTION_KEY`: values Cap encrypted with the old key can no longer be read. This covers custom S3 storage settings (organization and desktop), developer app API keys, and connected integrations. After switching keys, re-enter your storage settings and regenerate developer app keys. + +If you already set your own values, nothing changes for you. + ## Pinning Image Versions `cap-web:latest` and `cap-media-server:latest` follow `main`. cap-web runs database migrations automatically on start, and they only go forward, so after a newer image has migrated your database you can't safely roll back to an older one. New required settings also arrive with updates (for example, `MEDIA_SERVER_WEBHOOK_SECRET` became mandatory for media processing). For production, pin both images to a specific tag or digest, back up MySQL before upgrading, and read the self-hosting changes before moving the pin. From 8284b71193e7ef1065c307380e8ecb6471d45b19 Mon Sep 17 00:00:00 2001 From: 07rjain <07rjain@users.noreply.github.com> Date: Fri, 23 Oct 2026 14:16:51 +0000 Subject: [PATCH 19/20] test(media-server): serve route fixtures over http and wait for policy-checked webhooks The routes reject file: URLs, so the audio memory and real-world route tests now fetch fixtures from a local server. The webhook serialization test waits for the request to start, since the URL policy check runs before fetch. --- .../src/__tests__/lib/job-manager.test.ts | 4 +++- ...edia-routes-real-world.integration.test.ts | 15 +++++-------- .../src/__tests__/routes/audio-memory.test.ts | 22 +++++++++++++++++-- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/apps/media-server/src/__tests__/lib/job-manager.test.ts b/apps/media-server/src/__tests__/lib/job-manager.test.ts index 5bada4e694c..bc1e06a4cc9 100644 --- a/apps/media-server/src/__tests__/lib/job-manager.test.ts +++ b/apps/media-server/src/__tests__/lib/job-manager.test.ts @@ -156,8 +156,10 @@ describe("durable recording workers", () => { const sending = sendWebhook(job); updateJob(job.jobId, { phase: "complete", progress: 100 }); expect(sendWebhook(job)).toBe(sending); + // The URL policy check runs before fetch, so wait for the request to start. + while (!release) await Bun.sleep(1); const started = performance.now(); - release?.(new Response(null, { status: 503 })); + release(new Response(null, { status: 503 })); await sending; expect(performance.now() - started).toBeLessThan(400); expect(maximumInFlight).toBe(1); diff --git a/apps/media-server/src/__tests__/lib/media-routes-real-world.integration.test.ts b/apps/media-server/src/__tests__/lib/media-routes-real-world.integration.test.ts index 1c7a4cc54eb..a91a2d9a650 100644 --- a/apps/media-server/src/__tests__/lib/media-routes-real-world.integration.test.ts +++ b/apps/media-server/src/__tests__/lib/media-routes-real-world.integration.test.ts @@ -14,7 +14,6 @@ import { mkdtempSync, rmSync } from "node:fs"; import { readFile, writeFile } from "node:fs/promises"; import os, { tmpdir } from "node:os"; import { join } from "node:path"; -import { pathToFileURL } from "node:url"; import type appType from "../../app"; import * as audioLevels from "../../lib/audio-levels"; import * as containerCpu from "../../lib/container-cpu"; @@ -66,10 +65,6 @@ let transientFixtureFailures = 0; let permanentFixtureFailures = 0; let slowFixtureCancellations = 0; -function fileUrl(path: string) { - return pathToFileURL(path).toString(); -} - function fixtureUrl(name = "test-with-audio.mp4") { return `${baseUrl}/fixtures/${name}`; } @@ -1000,12 +995,12 @@ describe("media routes real-world integration tests", () => { test("checks real audio tracks through the route stack", async () => { const withAudioResponse = await app.fetch( mediaPostRequest("/audio/check", { - videoUrl: fileUrl(TEST_VIDEO_WITH_AUDIO), + videoUrl: fixtureUrl("test-with-audio.mp4"), }), ); const noAudioResponse = await app.fetch( mediaPostRequest("/audio/check", { - videoUrl: fileUrl(TEST_VIDEO_NO_AUDIO), + videoUrl: fixtureUrl("test-no-audio.mp4"), }), ); @@ -1021,7 +1016,7 @@ describe("media routes real-world integration tests", () => { test("extracts real audio through the non-streaming route", async () => { const response = await app.fetch( mediaPostRequest("/audio/extract", { - videoUrl: fileUrl(TEST_VIDEO_WITH_AUDIO), + videoUrl: fixtureUrl("test-with-audio.mp4"), stream: false, }), ); @@ -1034,7 +1029,7 @@ describe("media routes real-world integration tests", () => { test("probes and thumbnails a real video through the route stack", async () => { const probeResponse = await app.fetch( mediaPostRequest("/video/probe", { - videoUrl: fileUrl(TEST_VIDEO_WITH_AUDIO), + videoUrl: fixtureUrl("test-with-audio.mp4"), }), ); expect(probeResponse.status).toBe(200); @@ -1046,7 +1041,7 @@ describe("media routes real-world integration tests", () => { const thumbnailResponse = await app.fetch( mediaPostRequest("/video/thumbnail", { - videoUrl: fileUrl(TEST_VIDEO_WITH_AUDIO), + videoUrl: fixtureUrl("test-with-audio.mp4"), timestamp: 0.2, width: 160, height: 120, diff --git a/apps/media-server/src/__tests__/routes/audio-memory.test.ts b/apps/media-server/src/__tests__/routes/audio-memory.test.ts index 82241f830ab..8f5cd532138 100644 --- a/apps/media-server/src/__tests__/routes/audio-memory.test.ts +++ b/apps/media-server/src/__tests__/routes/audio-memory.test.ts @@ -1,9 +1,27 @@ -import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test"; +import { + afterAll, + afterEach, + beforeEach, + describe, + expect, + mock, + test, +} from "bun:test"; import { join } from "node:path"; import { getActiveProcessCount } from "../../lib/media-audio"; const FIXTURES_DIR = join(import.meta.dir, "..", "fixtures"); -const TEST_VIDEO_WITH_AUDIO = `file://${join(FIXTURES_DIR, "test-with-audio.mp4")}`; +// The routes only accept http(s) URLs, so serve the fixture locally. +const fixtureServer = Bun.serve({ + port: 0, + fetch: () => + new Response(Bun.file(join(FIXTURES_DIR, "test-with-audio.mp4"))), +}); +const TEST_VIDEO_WITH_AUDIO = `${fixtureServer.url}test-with-audio.mp4`; + +afterAll(() => { + fixtureServer.stop(true); +}); const MEDIA_SERVER_SECRET = "test-secret"; const AUTH_HEADERS = { "Content-Type": "application/json", From a55034030fba2804bbdd854da7e643c9c55e48df Mon Sep 17 00:00:00 2001 From: 07rjain <07rjain@users.noreply.github.com> Date: Fri, 23 Oct 2026 14:17:36 +0000 Subject: [PATCH 20/20] fix(media-server): fix the type errors the URL policy changes added --- apps/media-server/src/__tests__/lib/redirect-follow.test.ts | 4 ++-- apps/media-server/src/lib/media-common.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/media-server/src/__tests__/lib/redirect-follow.test.ts b/apps/media-server/src/__tests__/lib/redirect-follow.test.ts index 98e309442bb..83b6a72b2a9 100644 --- a/apps/media-server/src/__tests__/lib/redirect-follow.test.ts +++ b/apps/media-server/src/__tests__/lib/redirect-follow.test.ts @@ -23,9 +23,9 @@ describe("redirects to a private address", () => { function privateRedirectServer() { let secretHits = 0; - const server = Bun.serve({ + const server: ReturnType = Bun.serve({ port: 0, - fetch(request) { + fetch(request): Response { const url = new URL(request.url); if (url.pathname === "/secret") { secretHits += 1; diff --git a/apps/media-server/src/lib/media-common.ts b/apps/media-server/src/lib/media-common.ts index cac630f69fe..739ae833a3b 100644 --- a/apps/media-server/src/lib/media-common.ts +++ b/apps/media-server/src/lib/media-common.ts @@ -31,7 +31,7 @@ export type ProgressCallback = (progress: number, message: string) => void; export const PROCESS_TIMEOUT_MS = 45 * 60 * 1000; export const DOWNLOAD_TIMEOUT_MS = 10 * 60 * 1000; export const UPLOAD_TIMEOUT_MS = 10 * 60 * 1000; -const mediaFetch: typeof fetch = (input, init) => { +const mediaFetch = ((input: string | URL | Request, init?: RequestInit) => { const url = typeof input === "string" ? input @@ -39,7 +39,7 @@ const mediaFetch: typeof fetch = (input, init) => { ? input.toString() : input.url; return fetchAllowed(url, init); -}; +}) as typeof fetch; export function decodeThreadsFromEnv( name: string,