fix(webapp,run-engine): stop delayed runs firing after a project is deleted

Deleting a project didn't prevent delayed runs from being queued later.
When the delay elapsed the run was enqueued, which re-added the queue,
and it executed and alerted the project's channels long after the
project was gone. Cron schedules were already gated on deletion; the
delayed-run path had no equivalent guard.

Mono-RevId: 04d8246437c4cc661640b2a457455518dded7638

Author: matt-aitken

.server-changes/deleted-project-delayed-runs.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: fix
+---
+
+Deleting a project now stops its pending runs. Runs that were waiting on a `delay` or sitting in the queue are cancelled instead of executing later, and a deleted project no longer sends task failure alerts.

apps/webapp/app/services/deleteProject.server.ts

@@ -58,7 +58,7 @@ export class DeleteProjectService {
       },
     });
 
-    // project.deletedAt (which gates env resolution) changed; drop every cached env of this project.
+    /** project.deletedAt, which the engine gates enqueue and dequeue on, changed; drop every cached env of this project. */
     for (const environment of project.environments) {
       controlPlaneResolver.invalidateEnvironment(environment.id);
     }

apps/webapp/app/v3/runOpsMigration/controlPlaneCache.server.ts

@@ -40,8 +40,20 @@ export type ResolvedEnv = {
   // ignore these additive fields.
   maximumConcurrencyLimit: number;
   concurrencyLimitBurstFactor: Prisma.Decimal;
+  /**
+   * Soft-delete tombstones. The engine gates enqueue and dequeue on these, and alerts skip a
+   * deleted project, so a deleted project's pending work neither runs nor pages anyone.
+   */
+  projectDeletedAt: Date | null;
+  organizationDeletedAt: Date | null;
 };
 
+/**
+ * Just the soft-delete tombstones. Structurally identical to the run-engine
+ * `EnvDeletionState`; declared here so the app resolver stays engine-agnostic.
+ */
+export type EnvDeletionState = Pick<ResolvedEnv, "projectDeletedAt" | "organizationDeletedAt">;
+
 /**
  * The BackgroundWorkerTask columns the dequeue resolve path reads. Mirrors run-engine's
  * `ResolvedWorkerTask` exactly. The unread heavy JSON columns (`payloadSchema`, `config`,

apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.ts

@@ -14,6 +14,7 @@ import {
   resolvedWorkerDeploymentSelect,
   resolvedWorkerTaskSelect,
   type ResolvedAuthenticatedEnv,
+  type EnvDeletionState,
   type ResolvedEnv,
   type ResolvedWorkerVersion,
   type ResolvedRunLockedWorker,
@@ -140,7 +141,13 @@ export class ControlPlaneResolver {
         archivedAt: true,
         maximumConcurrencyLimit: true,
         concurrencyLimitBurstFactor: true,
-        project: { select: { organizationId: true } },
+        project: {
+          select: {
+            organizationId: true,
+            deletedAt: true,
+            organization: { select: { deletedAt: true } },
+          },
+        },
         parentEnvironment: { select: { type: true } },
       },
     });
@@ -158,6 +165,36 @@ export class ControlPlaneResolver {
       parentEnvironmentType: env.parentEnvironment?.type ?? null,
       maximumConcurrencyLimit: env.maximumConcurrencyLimit,
       concurrencyLimitBurstFactor: env.concurrencyLimitBurstFactor,
+      projectDeletedAt: env.project.deletedAt,
+      organizationDeletedAt: env.project.organization.deletedAt,
+    };
+  }
+
+  /**
+   * The soft-delete tombstones, always read from the control-plane primary and never cached.
+   *
+   * `resolveEnv` is cache-fronted per process when split is on, and `invalidateEnvironment` /
+   * `invalidateOrganization` only clear the process that handled the delete, so every other
+   * process can serve pre-deletion state until its entry expires. A caller whose decision must
+   * not be made on that stale value uses this instead.
+   */
+  async resolveEnvDeletionState(environmentId: string): Promise<EnvDeletionState | null> {
+    const env = await this.controlPlanePrimary.runtimeEnvironment.findFirst({
+      where: { id: environmentId },
+      select: {
+        project: {
+          select: { deletedAt: true, organization: { select: { deletedAt: true } } },
+        },
+      },
+    });
+
+    if (!env) {
+      return null;
+    }
+
+    return {
+      projectDeletedAt: env.project.deletedAt,
+      organizationDeletedAt: env.project.organization.deletedAt,
     };
   }
 

apps/webapp/app/v3/runOpsMigration/runEngineControlPlaneResolver.server.ts

@@ -1,5 +1,6 @@
 import type {
   ControlPlaneResolver as EngineControlPlaneResolver,
+  EnvDeletionState,
   ResolvedAuthenticatedEnv,
   ResolvedEngineEnv,
   ResolvedWorkerVersion,
@@ -40,11 +41,17 @@ export class RunEngineControlPlaneResolver implements EngineControlPlaneResolver
       concurrencyLimitBurstFactor: env.concurrencyLimitBurstFactor,
       projectId: env.projectId,
       organizationId: env.organizationId,
+      projectDeletedAt: env.projectDeletedAt,
+      organizationDeletedAt: env.organizationDeletedAt,
       project: { id: env.projectId },
       organization: { id: env.organizationId },
     };
   }
 
+  async resolveEnvDeletionState(environmentId: string): Promise<EnvDeletionState | null> {
+    return this.#resolver.resolveEnvDeletionState(environmentId);
+  }
+
   async resolveWorkerVersion(args: {
     environmentId: string;
     type: RuntimeEnvironmentType;

apps/webapp/app/v3/services/alerts/performTaskRunAlerts.server.ts

@@ -67,6 +67,18 @@ export class PerformTaskRunAlertsService extends BaseService {
       },
     });
 
+    if (alertChannels.length === 0) {
+      return;
+    }
+
+    const deletionState = await this.#controlPlaneResolver.resolveEnvDeletionState(
+      run.runtimeEnvironmentId
+    );
+
+    if (!deletionState || deletionState.projectDeletedAt || deletionState.organizationDeletedAt) {
+      return;
+    }
+
     for (const alertChannel of alertChannels) {
       await this.#createAndSendAlert(alertChannel, run);
     }

apps/webapp/test/performTaskRunAlertsStoreRouting.test.ts

@@ -245,3 +245,71 @@ describe("PerformTaskRunAlertsService passthrough (single-DB)", () => {
     }
   );
 });
+
+async function seedAlertableRun(prisma: PrismaClient, suffix: string) {
+  const id = generateRunOpsId();
+  const { project, organization, runtimeEnvironment } = await seedProject(prisma, suffix);
+
+  await seedRun(
+    prisma,
+    { id, friendlyId: `run_${id}` },
+    {
+      runtimeEnvironmentId: runtimeEnvironment.id,
+      projectId: project.id,
+      organizationId: organization.id,
+    }
+  );
+  await prisma.projectAlertChannel.create({
+    data: {
+      friendlyId: `alert_${id}`,
+      name: "test-channel",
+      projectId: project.id,
+      alertTypes: ["TASK_RUN"],
+      environmentTypes: ["PRODUCTION"],
+      type: "EMAIL",
+      properties: { type: "EMAIL", email: "test@example.com" },
+      enabled: true,
+    },
+  });
+
+  return { id, project, organization };
+}
+
+function buildAlertsService(prisma: PrismaClient) {
+  return new PerformTaskRunAlertsService({
+    prisma,
+    runStore: new PostgresRunStore({ prisma, readOnlyPrisma: prisma }),
+    controlPlaneResolver: buildControlPlaneResolver(prisma),
+  });
+}
+
+describe("PerformTaskRunAlertsService deleted project/organization", () => {
+  postgresTest("sends no alert once the project is deleted", async ({ prisma }) => {
+    const { id, project } = await seedAlertableRun(prisma, "del-project");
+
+    await prisma.project.update({ where: { id: project.id }, data: { deletedAt: new Date() } });
+
+    await buildAlertsService(prisma)
+      .call(id)
+      .catch(() => {});
+
+    const delivered = await prisma.projectAlert.findMany({ where: { projectId: project.id } });
+    expect(delivered.length).toBe(0);
+  });
+
+  postgresTest("sends no alert once the organization is deleted", async ({ prisma }) => {
+    const { id, project, organization } = await seedAlertableRun(prisma, "del-org");
+
+    await prisma.organization.update({
+      where: { id: organization.id },
+      data: { deletedAt: new Date() },
+    });
+
+    await buildAlertsService(prisma)
+      .call(id)
+      .catch(() => {});
+
+    const delivered = await prisma.projectAlert.findMany({ where: { projectId: project.id } });
+    expect(delivered.length).toBe(0);
+  });
+});

apps/webapp/test/v3/runOpsMigration/runEngineControlPlaneResolver.server.test.ts

@@ -154,11 +154,56 @@ describe("RunEngineControlPlaneResolver adapter", () => {
       expect(env!.maximumConcurrencyLimit).toBe(9);
       expect(env!.concurrencyLimitBurstFactor.toNumber()).toBe(2);
       expect(env!.archivedAt).toBeNull();
+      expect(env!.projectDeletedAt).toBeNull();
+      expect(env!.organizationDeletedAt).toBeNull();
 
       expect(await adapter.resolveEnv("env_missing")).toBeNull();
     }
   );
 
+  heteroPostgresTest(
+    "resolveEnv carries the project and organization soft-delete tombstones the engine guards on",
+    async ({ prisma14 }) => {
+      const { organization, project, environment } = await seedEnv(prisma14, "PRODUCTION");
+      const deletedAt = new Date();
+
+      await prisma14.project.update({ where: { id: project.id }, data: { deletedAt } });
+      await prisma14.organization.update({ where: { id: organization.id }, data: { deletedAt } });
+
+      const env = await new RunEngineControlPlaneResolver(buildAppResolver(prisma14)).resolveEnv(
+        environment.id
+      );
+
+      expect(env!.projectDeletedAt).toEqual(deletedAt);
+      expect(env!.organizationDeletedAt).toEqual(deletedAt);
+    }
+  );
+
+  /**
+   * The delete is applied WITHOUT calling invalidateEnvironment, which models it happening in a
+   * different process: that is the only kind this process's cache never hears about, and the
+   * reason the guards cannot make their decision on `resolveEnv`.
+   */
+  heteroPostgresTest(
+    "resolveEnvDeletionState sees a deletion the warm per-process env cache is still hiding",
+    async ({ prisma14 }) => {
+      const { project, environment } = await seedEnv(prisma14, "PRODUCTION");
+      const appResolver = buildAppResolver(prisma14, { splitEnabled: true });
+      const adapter = new RunEngineControlPlaneResolver(appResolver);
+
+      expect((await adapter.resolveEnv(environment.id))!.projectDeletedAt).toBeNull();
+
+      const deletedAt = new Date();
+      await prisma14.project.update({ where: { id: project.id }, data: { deletedAt } });
+
+      expect((await adapter.resolveEnv(environment.id))!.projectDeletedAt).toBeNull();
+
+      const deletionState = await adapter.resolveEnvDeletionState(environment.id);
+      expect(deletionState!.projectDeletedAt).toEqual(deletedAt);
+      expect(await appResolver.resolveEnvDeletionState("env_missing")).toBeNull();
+    }
+  );
+
   heteroPostgresTest(
     "resolveWorkerVersion (deployed, no workerId) resolves the promoted MANAGED deployment",
     async ({ prisma14 }) => {

internal-packages/run-engine/src/engine/controlPlaneResolver.ts

@@ -27,6 +27,11 @@ import type { AuthenticatedEnvironment } from "@trigger.dev/core/v3/auth/environ
  * so a `ResolvedEngineEnv` is a structural supertype of it and can be passed directly
  * to `enqueueSystem.enqueueRun({ env })`. `concurrencyLimitBurstFactor` stays a
  * `Prisma.Decimal` (do NOT coerce); `maximumConcurrencyLimit` is non-null per schema.
+ *
+ * `projectDeletedAt`/`organizationDeletedAt` are the soft-delete tombstones the engine gates
+ * enqueue and dequeue on, so a deleted project's work stops instead of running weeks later.
+ * Every implementation of this interface MUST populate them; a resolver that leaves them null
+ * silently disables both guards.
  */
 export type ResolvedEngineEnv = {
   id: string;
@@ -36,10 +41,48 @@ export type ResolvedEngineEnv = {
   concurrencyLimitBurstFactor: Prisma.Decimal;
   projectId: string;
   organizationId: string;
+  projectDeletedAt: Date | null;
+  organizationDeletedAt: Date | null;
   project: { id: string };
   organization: { id: string };
 };
 
+/**
+ * Just the soft-delete tombstones, read from the authoritative source rather than from
+ * whatever `resolveEnv` has cached.
+ */
+export type EnvDeletionState = Pick<
+  ResolvedEngineEnv,
+  "projectDeletedAt" | "organizationDeletedAt"
+>;
+
+/**
+ * Recorded when the environment row itself is absent, so there is no tombstone to read. Only
+ * ever reached from an authoritative read: a cached absence is not trustworthy enough to end
+ * a run on.
+ */
+export const MISSING_ENVIRONMENT_REASON = "The environment was deleted";
+
+/**
+ * The reason a run on this environment must not execute, or `null` when it may. Doubles as the
+ * enqueue/dequeue guard and as the cancellation reason recorded on the run, so the two call
+ * sites cannot drift apart.
+ */
+export function deletedEnvironmentReason(env: {
+  projectDeletedAt: Date | null;
+  organizationDeletedAt: Date | null;
+}): string | null {
+  if (env.organizationDeletedAt) {
+    return "The organization was deleted";
+  }
+
+  if (env.projectDeletedAt) {
+    return "The project was deleted";
+  }
+
+  return null;
+}
+
 /**
  * The richer control-plane env primitive: the slim, structural
  * `AuthenticatedEnvironment` (slug/branchName/project/organization/…) PLUS the
@@ -127,6 +170,14 @@ export type WorkerVersionDispatchFilter = {
 
 export interface ControlPlaneResolver {
   resolveEnv(environmentId: string): Promise<ResolvedEngineEnv | null>;
+  /**
+   * The deletion tombstones read straight from the authoritative control-plane source, never
+   * from a cache. An implementation may serve `resolveEnv` from a per-process cache whose
+   * invalidation does not reach other processes, so a decision that must not act on
+   * pre-deletion state for even a moment reads it here instead. Returns `null` when the
+   * environment does not exist.
+   */
+  resolveEnvDeletionState(environmentId: string): Promise<EnvDeletionState | null>;
   resolveAuthenticatedEnv(environmentId: string): Promise<ResolvedAuthenticatedEnv | null>;
   resolveWorkerVersion(
     args: {
@@ -174,8 +225,8 @@ export class PassthroughControlPlaneResolver implements ControlPlaneResolver {
         maximumConcurrencyLimit: true,
         concurrencyLimitBurstFactor: true,
         projectId: true,
-        project: { select: { id: true, organizationId: true } },
-        organization: { select: { id: true } },
+        project: { select: { id: true, organizationId: true, deletedAt: true } },
+        organization: { select: { id: true, deletedAt: true } },
       },
     });
 
@@ -191,11 +242,38 @@ export class PassthroughControlPlaneResolver implements ControlPlaneResolver {
       concurrencyLimitBurstFactor: env.concurrencyLimitBurstFactor,
       projectId: env.projectId,
       organizationId: env.project.organizationId,
+      projectDeletedAt: env.project.deletedAt,
+      organizationDeletedAt: env.organization.deletedAt,
       project: { id: env.project.id },
       organization: { id: env.organization.id },
     };
   }
 
+  /**
+   * Passthrough holds no cache, so `resolveEnv` is already authoritative and this exists to
+   * satisfy the seam. It reads the org tombstone through `project.organization` (the same path
+   * the webapp resolver uses) rather than the environment's own organization relation.
+   */
+  async resolveEnvDeletionState(environmentId: string): Promise<EnvDeletionState | null> {
+    const env = await this.#prisma.runtimeEnvironment.findFirst({
+      where: { id: environmentId },
+      select: {
+        project: {
+          select: { deletedAt: true, organization: { select: { deletedAt: true } } },
+        },
+      },
+    });
+
+    if (!env) {
+      return null;
+    }
+
+    return {
+      projectDeletedAt: env.project.deletedAt,
+      organizationDeletedAt: env.project.organization.deletedAt,
+    };
+  }
+
   async resolveAuthenticatedEnv(environmentId: string): Promise<ResolvedAuthenticatedEnv | null> {
     const env = await this.#prisma.runtimeEnvironment.findFirst({
       where: { id: environmentId },