feat(webapp,webhook-engine): keep the old webhook signing secret valid during rotation

Mono-RevId: 9d6143af78b860388fa25f05dc471524e84d0ec8

Author: ericallam

.changeset/webhook-additional-signature-headers.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+HMAC webhook verifier configs accept `additionalSignatureHeaders`: more headers that can carry a signature, matched as alternatives to `signatureHeader`. Providers that send a second signature during a secret rotation (Hookdeck's `x-hookdeck-signature-2`) keep verifying through the rotation.

.server-changes/webhook-secret-rotation-grace.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: feature
+---
+
+Rotating a webhook endpoint's signing secret can keep the old one valid for a while, so you can rotate without failing deliveries that are still signed with it. The rotate and regenerate dialogs let you choose how long the old secret stays valid (up to 7 days, 24 hours by default). The `rotate-secret` and `secret` endpoints accept an optional `previousSecretTtlSeconds` and return `previousSecretExpiresAt`. With no value, the old secret stops working immediately, as before.

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.webhooks.endpoints.$endpointParam/route.tsx

@@ -29,6 +29,7 @@ import { Label } from "~/components/primitives/Label";
 import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { Paragraph } from "~/components/primitives/Paragraph";
 import * as Property from "~/components/primitives/PropertyTable";
+import { Select, SelectItem } from "~/components/primitives/Select";
 import {
   ResizableHandle,
   ResizablePanel,
@@ -68,6 +69,7 @@ import { docsPath, EnvironmentParamSchema, v3WebhooksPath } from "~/utils/pathBu
 import { throwPermissionDenied } from "~/utils/permissionDenied";
 import { parseFiniteInt } from "~/utils/searchParams";
 import { webhookEngine, webhookVerifyTokenKey } from "~/v3/webhookEngine.server";
+import { PreviousSecretTtlSeconds } from "~/v3/webhookSigningSecret";
 import {
   generateWebhookSigningSecret,
   storeWebhookSigningSecret,
@@ -209,7 +211,11 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   });
 };
 
-const SetSecretSchema = z.object({
+const PreviousSecretTtlFormField = z.object({
+  previousSecretTtlSeconds: z.coerce.number().pipe(PreviousSecretTtlSeconds).optional(),
+});
+
+const SetSecretSchema = PreviousSecretTtlFormField.extend({
   intent: z.literal("set-secret"),
   secret: z.string().trim().min(1, "A signing secret is required"),
 });
@@ -261,9 +267,17 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
 
   // Generate (integrator-supplied secret): the UI reveals it ONCE for pasting into the provider.
   if (intent === "generate-secret") {
-    const result = await generateWebhookSigningSecret(endpoint);
+    const ttl = PreviousSecretTtlFormField.safeParse(Object.fromEntries(formData));
+    if (!ttl.success) {
+      return { success: false as const, error: ttl.error.issues[0]?.message ?? "Invalid" };
+    }
+    const result = await generateWebhookSigningSecret(endpoint, ttl.data);
     return result.ok
-      ? { success: true as const, generatedSecret: result.secret }
+      ? {
+          success: true as const,
+          generatedSecret: result.secret,
+          previousSecretExpiresAt: result.previousSecretExpiresAt,
+        }
       : { success: false as const, error: result.error };
   }
 
@@ -287,7 +301,9 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
   if (!submission.success) {
     return { success: false as const, error: submission.error.issues[0]?.message ?? "Invalid" };
   }
-  await storeWebhookSigningSecret(endpoint, submission.data.secret);
+  await storeWebhookSigningSecret(endpoint, submission.data.secret, {
+    previousSecretTtlSeconds: submission.data.previousSecretTtlSeconds,
+  });
 
   return { success: true as const };
 };
@@ -683,7 +699,12 @@ function ProviderSetup({ verifier, source }: { verifier: LoaderData["verifier"];
             <Property.Item>
               <Property.Label>Signature header</Property.Label>
               <Property.Value>
-                <span className="font-mono text-sm">{config.signatureHeader}</span>
+                <span className="font-mono text-sm">
+                  {[
+                    config.signatureHeader,
+                    ...(config.scheme === "hmac" ? (config.additionalSignatureHeaders ?? []) : []),
+                  ].join(", ")}
+                </span>
               </Property.Value>
             </Property.Item>
             <Property.Item>
@@ -780,6 +801,39 @@ function describeTimestampSource(source: WebhookValueSource): string {
   }
 }
 
+const PREVIOUS_SECRET_TTL_OPTIONS = [
+  { value: "0", label: "Stop immediately" },
+  { value: "3600", label: "1 hour" },
+  { value: "86400", label: "24 hours" },
+  { value: "604800", label: "7 days" },
+];
+
+function PreviousSecretTtlField({ credentialNoun }: { credentialNoun: string }) {
+  return (
+    <div className="flex flex-col gap-1">
+      <Label>Keep accepting the old {credentialNoun} for</Label>
+      <Select
+        name="previousSecretTtlSeconds"
+        variant="tertiary/medium"
+        dropdownIcon
+        items={PREVIOUS_SECRET_TTL_OPTIONS}
+        defaultValue="86400"
+        text={(value) => PREVIOUS_SECRET_TTL_OPTIONS.find((o) => o.value === value)?.label}
+      >
+        {PREVIOUS_SECRET_TTL_OPTIONS.map((option) => (
+          <SelectItem key={option.value} value={option.value}>
+            {option.label}
+          </SelectItem>
+        ))}
+      </Select>
+      <Hint>
+        Deliveries signed with the old {credentialNoun} still verify until then, so you can update
+        the provider without failing any. Stop immediately if the old one leaked.
+      </Hint>
+    </div>
+  );
+}
+
 function SetSecretDialog({
   hasSigningSecret,
   credentialNoun,
@@ -832,6 +886,7 @@ function SetSecretDialog({
                 : "Stored encrypted and never shown again. Deliveries are verified against this secret."}
             </Hint>
           </div>
+          {hasSigningSecret ? <PreviousSecretTtlField credentialNoun={credentialNoun} /> : null}
           {fetcher.data && !fetcher.data.success ? (
             <Paragraph variant="small" className="text-error">
               {fetcher.data.error}
@@ -864,6 +919,10 @@ function GenerateSecretDialog({ hasSigningSecret }: { hasSigningSecret: boolean
   const isSubmitting = fetcher.state !== "idle";
   const generated =
     fetcher.data && "generatedSecret" in fetcher.data ? fetcher.data.generatedSecret : undefined;
+  const previousSecretExpiresAt =
+    fetcher.data && "previousSecretExpiresAt" in fetcher.data
+      ? fetcher.data.previousSecretExpiresAt
+      : undefined;
 
   return (
     <Dialog open={open} onOpenChange={setOpen}>
@@ -883,6 +942,12 @@ function GenerateSecretDialog({ hasSigningSecret }: { hasSigningSecret: boolean
             </Paragraph>
             <ClipboardField value={generated} variant="secondary/medium" />
             <Hint>Paste this into your provider's webhook signing-secret field.</Hint>
+            {previousSecretExpiresAt ? (
+              <Hint>
+                The previous secret keeps verifying deliveries until{" "}
+                <DateTime date={previousSecretExpiresAt} />.
+              </Hint>
+            ) : null}
             <div className="flex justify-end">
               <Button type="button" variant="primary/small" onClick={() => setOpen(false)}>
                 Done
@@ -896,6 +961,12 @@ function GenerateSecretDialog({ hasSigningSecret }: { hasSigningSecret: boolean
               Trigger.dev generates a strong signing secret, stores it encrypted, and shows it once
               so you can paste it into your provider.
             </Paragraph>
+            {hasSigningSecret ? <PreviousSecretTtlField credentialNoun="signing secret" /> : null}
+            {fetcher.data && !fetcher.data.success ? (
+              <Paragraph variant="small" className="text-error">
+                {fetcher.data.error}
+              </Paragraph>
+            ) : null}
             <div className="flex justify-end gap-2">
               <Button
                 type="button"

apps/webapp/app/routes/api.v1.webhooks.endpoints.$endpointId.rotate-secret.ts

@@ -3,10 +3,17 @@ import { z } from "zod";
 import { webhookPrisma } from "~/db.server";
 import { webhookEndpointLookup } from "~/v3/webhookEndpointLookup";
 import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
+import { safeJsonParse } from "~/utils/json";
+import { PreviousSecretTtlSeconds } from "~/v3/webhookSigningSecret";
 import { generateWebhookSigningSecret } from "~/v3/webhookSigningSecret.server";
 
 const ParamsSchema = z.object({ endpointId: z.string() });
 
+/** Optional: existing callers POST with no body, which keeps the replace-immediately behavior. */
+const RotateSecretRequestBody = z.object({
+  previousSecretTtlSeconds: PreviousSecretTtlSeconds.optional(),
+});
+
 // POST /api/v1/webhooks/endpoints/:endpointId/rotate-secret — mint a new signing secret and return
 // it ONCE. Only for schemes we generate (hmac / shared-secret); asymmetric endpoints set a public key.
 const { action, loader } = createActionApiRoute(
@@ -17,17 +24,28 @@ const { action, loader } = createActionApiRoute(
     corsStrategy: "all",
     authorization: { action: "write", resource: () => ({ type: "webhooks" }) },
   },
-  async ({ params, authentication }) => {
+  async ({ params, authentication, request }) => {
+    const rawBody = await request.text();
+    const body = RotateSecretRequestBody.safeParse(rawBody ? safeJsonParse(rawBody) : {});
+    if (!body.success) {
+      return json({ error: "Invalid request body", issues: body.error.issues }, { status: 400 });
+    }
+
     const endpoint = await webhookPrisma.webhookEndpoint.findFirst({
       where: webhookEndpointLookup(authentication.environment.id, params.endpointId),
       select: { id: true, friendlyId: true, verifierArtifact: true },
     });
     if (!endpoint) return json({ error: "Not found" }, { status: 404 });
 
-    const result = await generateWebhookSigningSecret(endpoint);
+    const result = await generateWebhookSigningSecret(endpoint, body.data);
     if (!result.ok) return json({ error: result.error }, { status: 400 });
 
-    return json({ id: endpoint.friendlyId, secretSet: true as const, secret: result.secret });
+    return json({
+      id: endpoint.friendlyId,
+      secretSet: true as const,
+      secret: result.secret,
+      previousSecretExpiresAt: result.previousSecretExpiresAt,
+    });
   }
 );
 

apps/webapp/app/routes/api.v1.webhooks.endpoints.$endpointId.secret.ts

@@ -4,14 +4,19 @@ import { z } from "zod";
 import { webhookPrisma } from "~/db.server";
 import { webhookEndpointLookup } from "~/v3/webhookEndpointLookup";
 import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
+import { PreviousSecretTtlSeconds } from "~/v3/webhookSigningSecret";
 import { storeWebhookSigningSecret } from "~/v3/webhookSigningSecret.server";
 
 const ParamsSchema = z.object({ endpointId: z.string() });
 
+const SetSecretRequestBody = SetWebhookEndpointSecretRequestBody.extend({
+  previousSecretTtlSeconds: PreviousSecretTtlSeconds.optional(),
+});
+
 const { action, loader } = createActionApiRoute(
   {
     params: ParamsSchema,
-    body: SetWebhookEndpointSecretRequestBody,
+    body: SetSecretRequestBody,
     method: "PUT",
     allowJWT: true,
     corsStrategy: "all",
@@ -24,9 +29,11 @@ const { action, loader } = createActionApiRoute(
     });
     if (!endpoint) return json({ error: "Not found" }, { status: 404 });
 
-    await storeWebhookSigningSecret(endpoint, body.secret);
+    const { previousSecretExpiresAt } = await storeWebhookSigningSecret(endpoint, body.secret, {
+      previousSecretTtlSeconds: body.previousSecretTtlSeconds,
+    });
 
-    return json({ id: endpoint.friendlyId, secretSet: true as const });
+    return json({ id: endpoint.friendlyId, secretSet: true as const, previousSecretExpiresAt });
   }
 );
 

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.webhooks.endpoints.$endpointParam.replay-source.ts

@@ -57,6 +57,9 @@ function signatureHeaderNames(artifact: unknown): Set<string> {
   const config = parsed.data.config;
   if (config.scheme === "hmac" || config.scheme === "asymmetric") {
     names.add(config.signatureHeader.toLowerCase());
+    if (config.scheme === "hmac") {
+      for (const header of config.additionalSignatureHeaders ?? []) names.add(header.toLowerCase());
+    }
     if (config.timestamp?.source.from === "header") {
       const { name, fallback } = config.timestamp.source;
       for (const header of [name, ...(fallback ?? [])]) names.add(header.toLowerCase());

apps/webapp/app/v3/webhookEngine.server.ts

@@ -21,6 +21,7 @@ import { engine as runEngine } from "./runEngine.server";
 import { ServiceValidationError } from "./services/common.server";
 import { TriggerTaskService } from "./services/triggerTask.server";
 import { resolveWebhookSession } from "./webhookSessionTarget.server";
+import { activeSigningSecrets, StoredSigningSecret } from "./webhookSigningSecret";
 import { webhookLimitsForEnvironment } from "./webhookLimits.server";
 import { webhookPortErrorMessage } from "./webhookPortErrors.server";
 import { webhookWaitpoints } from "./webhookWaitpoints.server";
@@ -77,10 +78,6 @@ function webhookRedisConnection(): WebhookRedisOptions {
   };
 }
 
-// The plaintext signing secret is stored under the "DATABASE" SecretStore
-// provider as { secret: string } (same shape as environment variables).
-const SigningSecretSchema = z.object({ secret: z.string() });
-
 /** The GET verification token (Meta hub.verify_token) an endpoint owner generated in the dashboard. */
 const VerifyTokenSchema = z.object({ token: z.string() });
 
@@ -141,9 +138,10 @@ function createWebhookEngine() {
     tracer,
     meter,
     resolveSigningSecret: async (key) => {
-      const value = await secretStore.getSecret(SigningSecretSchema, key);
+      const value = await secretStore.getSecret(StoredSigningSecret, key);
       // Fail closed: an unset/empty secret returns undefined so ingest rejects.
-      return value?.secret || undefined;
+      const secrets = activeSigningSecrets(value, new Date());
+      return secrets.length > 0 ? secrets : undefined;
     },
     resolveVerifyToken: async (endpointId) => {
       const value = await secretStore.getSecret(

apps/webapp/app/v3/webhookSigningSecret.server.ts

@@ -2,9 +2,14 @@ import { randomBytes } from "node:crypto";
 import { WebhookVerifierArtifact } from "@trigger.dev/core/v3";
 import { prisma, webhookPrisma } from "~/db.server";
 import { getSecretStore } from "~/services/secrets/secretStore.server";
+import { replaceStoredSigningSecret, StoredSigningSecret } from "./webhookSigningSecret";
 
 type SecretEndpoint = { id: string; verifierArtifact: unknown };
 
+type StoreOptions = { previousSecretTtlSeconds?: number };
+
+type StoreResult = { previousSecretExpiresAt?: string };
+
 export function webhookSigningSecretKey(endpointId: string): string {
   return `webhook:signing-secret:${endpointId}`;
 }
@@ -16,28 +21,39 @@ function isAsymmetricWebhookEndpoint(endpoint: SecretEndpoint): boolean {
 
 /**
  * Stores a signing secret (or, for an asymmetric source, the provider's public key) in the shape the
- * engine's `resolveSigningSecret` reads, and points the endpoint at it.
+ * engine's `resolveSigningSecret` reads, and points the endpoint at it. With
+ * `previousSecretTtlSeconds`, the secret it replaces keeps verifying deliveries for that long.
  */
-export async function storeWebhookSigningSecret(endpoint: SecretEndpoint, secret: string) {
+export async function storeWebhookSigningSecret(
+  endpoint: SecretEndpoint,
+  secret: string,
+  options: StoreOptions = {}
+): Promise<StoreResult> {
   const key = webhookSigningSecretKey(endpoint.id);
-  await getSecretStore("DATABASE", { prismaClient: prisma }).setSecret(key, { secret });
+  const secretStore = getSecretStore("DATABASE", { prismaClient: prisma });
+  const ttl = options.previousSecretTtlSeconds ?? 0;
+  const existing = ttl > 0 ? await secretStore.getSecret(StoredSigningSecret, key) : undefined;
+  const stored = replaceStoredSigningSecret(existing, secret, ttl, new Date());
+  await secretStore.setSecret(key, stored);
   await webhookPrisma.webhookEndpoint.update({
     where: { id: endpoint.id },
     data: { signingSecretKey: key },
   });
+  return stored.previous ? { previousSecretExpiresAt: stored.previous.expiresAt } : {};
 }
 
 /** Mints and stores a new shared secret, replacing any existing one. Refused for asymmetric sources. */
 export async function generateWebhookSigningSecret(
-  endpoint: SecretEndpoint
-): Promise<{ ok: true; secret: string } | { ok: false; error: string }> {
+  endpoint: SecretEndpoint,
+  options: StoreOptions = {}
+): Promise<({ ok: true; secret: string } & StoreResult) | { ok: false; error: string }> {
   if (isAsymmetricWebhookEndpoint(endpoint)) {
     return {
       ok: false,
       error: "Cannot generate a secret for an asymmetric endpoint; set its public key instead.",
     };
   }
   const secret = `whsec_${randomBytes(32).toString("hex")}`;
-  await storeWebhookSigningSecret(endpoint, secret);
-  return { ok: true, secret };
+  const stored = await storeWebhookSigningSecret(endpoint, secret, options);
+  return { ok: true, secret, ...stored };
 }