refactor(webapp): move dashboard agent maintenance sweeps out

Drop the eval retention, soft-deleted-chat retention and investigation
sweeps, plus the retention half of the watch sweep. They now run in the
agent trigger project. Watch finalize/deliver, batch re-arm and the org
purge stay.

Author: kathiekiwi

.server-changes/dashboard-agent-maintenance-move.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: improvement
+---
+
+Routine cleanup of old dashboard agent data now runs on its own schedule, so it no longer competes with the dashboard for resources.

apps/webapp/app/services/dashboardAgentChatRetention.server.ts

@@ -1,71 +1,14 @@
 /**
- * Retention for soft-deleted chats. A deleted chat is kept for a grace window and then
- * hard-deleted with all its child rows; one bounded statement per run, oldest first.
- * Also the eventual purge behind organization deletion, which soft-deletes the org's
- * chats so this same sweep removes them.
+ * The purge behind organization deletion: it soft-deletes the org's chats, and retention
+ * hard-deletes them once the window passes.
  */
 
-import {
-  hardDeleteChatsSoftDeletedBefore,
-  softDeleteChatsForOrganization,
-} from "@internal/dashboard-agent-db";
+import { softDeleteChatsForOrganization } from "@internal/dashboard-agent-db";
 import { dashboardAgentDb } from "~/services/dashboardAgentDb.server";
-import { logger } from "~/services/logger.server";
 
 /**
- * How long a soft-deleted chat is kept before it and its children are hard-deleted.
- * Long enough that an accidental delete can still be investigated; org deletion soft-
- * deletes the org's chats, so those are removed the same way once the window passes.
- */
-export const CHAT_SOFT_DELETE_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
-
-/** Per-run cap. Retention is one bounded statement, not a row-at-a-time loop. */
-const RETENTION_BATCH_LIMIT = 500;
-
-export type ChatRetentionResult = {
-  /** Soft-deleted chats past the retention window dropped this run. */
-  purged: number;
-  failed: number;
-};
-
-export type ChatRetentionDeps = {
-  now?: () => Date;
-  limit?: number;
-  /** Hard-delete chats soft-deleted before `before`. Returns how many went. */
-  purge?: (params: { before: Date; limit: number }) => Promise<number>;
-};
-
-export async function sweepDashboardAgentSoftDeletedChats(
-  deps: ChatRetentionDeps = {}
-): Promise<ChatRetentionResult> {
-  const now = deps.now?.() ?? new Date();
-  const limit = deps.limit ?? RETENTION_BATCH_LIMIT;
-  const purge =
-    deps.purge ?? ((params) => hardDeleteChatsSoftDeletedBefore(dashboardAgentDb, params));
-
-  const result: ChatRetentionResult = { purged: 0, failed: 0 };
-
-  try {
-    result.purged = await purge({
-      before: new Date(now.getTime() - CHAT_SOFT_DELETE_RETENTION_MS),
-      limit,
-    });
-  } catch (error) {
-    result.failed++;
-    logger.error("Dashboard agent chat retention failed", { error });
-  }
-
-  if (result.failed > 0) {
-    throw new Error("The dashboard agent chat retention pass failed");
-  }
-
-  return result;
-}
-
-/**
- * Soft-delete every chat belonging to a deleted organization. The retention sweep above
- * hard-deletes them once the window passes, so the org-deletion request never runs a
- * cross-database hard delete.
+ * Soft-delete every chat belonging to a deleted organization. Retention hard-deletes them
+ * once the window passes, so the org-deletion request never runs a cross-database hard delete.
  */
 export async function purgeDashboardAgentChatsForOrganization(params: {
   organizationId: string;

apps/webapp/app/services/dashboardAgentEvalRetention.server.ts

@@ -6,8 +6,6 @@
 import {
   cancelWatch,
   claimWatchAlertDispatch,
-  deleteTerminalWatchesOlderThan,
-  deleteWatchSubmissionsOlderThan,
   listExpiredActiveWatches,
   listWatchBatchGroupsToArm,
   listWatchesAwaitingDelivery,
@@ -57,12 +55,6 @@ export const WATCH_DELIVERY_GRACE_MS = 5 * 60 * 1000;
 /** Per-run cap for each half of the sweep. Oldest first, so the rest land next run. */
 const SWEEP_BATCH_LIMIT = 100;
 
-/** How long a terminal watch is kept. Its outcome also lives in the chat transcript. */
-export const WATCH_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
-
-/** Higher than the other caps: retention is one statement, not a row-at-a-time loop. */
-const RETENTION_BATCH_LIMIT = 500;
-
 /**
  * How many rows one sweep handles at once. An incident expires a whole group together, and a
  * bound is what stops one slow tenant spending the entire visibility window.
@@ -91,10 +83,6 @@ export type WatchSweepResult = {
   redelivered: number;
   /** Decided but not handed over, with no agent project. They stay owed. */
   deliveryDeferred: number;
-  /** Long-terminal rows dropped by retention. */
-  purged: number;
-  /** Ledger rows dropped by retention. */
-  purgedSubmissions: number;
   failed: number;
 };
 
@@ -113,10 +101,6 @@ export type WatchSweepDeps = {
   deliver?: (watch: Watch) => Promise<void>;
   /** Gates the delivery half only. Finalization never depends on it. */
   configured?: () => boolean;
-  /** Drop terminal rows older than `before`. Returns how many went. */
-  purgeTerminal?: (params: { before: Date; limit: number }) => Promise<number>;
-  /** Drop submission-ledger rows older than `before`. */
-  purgeSubmissions?: (params: { before: Date; limit: number }) => Promise<number>;
   /** How many rows are handled at once. */
   concurrency?: number;
 };
@@ -332,11 +316,6 @@ export async function sweepDashboardAgentWatches(
   const listAwaitingDelivery =
     deps.listAwaitingDelivery ??
     ((params) => listWatchesAwaitingDelivery(dashboardAgentDb, params));
-  const purgeTerminal =
-    deps.purgeTerminal ?? ((params) => deleteTerminalWatchesOlderThan(dashboardAgentDb, params));
-  const purgeSubmissions =
-    deps.purgeSubmissions ??
-    ((params) => deleteWatchSubmissionsOlderThan(dashboardAgentDb, params));
 
   const result: WatchSweepResult = {
     overdue: 0,
@@ -347,8 +326,6 @@ export async function sweepDashboardAgentWatches(
     undelivered: 0,
     redelivered: 0,
     deliveryDeferred: 0,
-    purged: 0,
-    purgedSubmissions: 0,
     failed: 0,
   };
 
@@ -435,18 +412,6 @@ export async function sweepDashboardAgentWatches(
     }
   }
 
-  // Retention runs last, over rows both halves are finished with. Its own try/catch so a
-  // lost retention pass can't mask the other failures.
-  try {
-    const before = new Date(now.getTime() - WATCH_RETENTION_MS);
-    result.purged = await purgeTerminal({ before, limit: RETENTION_BATCH_LIMIT });
-    // The ledger's rows age out on the same window: past it no client is still retrying.
-    result.purgedSubmissions = await purgeSubmissions({ before, limit: RETENTION_BATCH_LIMIT });
-  } catch (error) {
-    result.failed++;
-    logger.error("Dashboard agent watch sweep: failed to purge terminal watches", { error });
-  }
-
   if (result.failed > 0) {
     throw new Error(`The dashboard agent watch sweep failed on ${result.failed} watches`);
   }