A dead worker redeploys once, then waits for Restart (CL-9913)

apps/web/src/agent-redeployer.tsx

@@ -0,0 +1,31 @@
+import { useEffect, useRef } from "react";
+
+import { claimAutoRedeploy } from "@/chat/redeploy-failures";
+import { useRestartAgent } from "./use-restart-agent";
+
+// Renders nothing; a mount's own ref plus the mutation's `isPending`/
+// `isSuccess` keep StrictMode's double effect from firing it twice. Fires
+// from an effect, never during render: mutating mid-render updates the
+// mutation's own state on a fiber that hasn't mounted yet (React warns).
+// The claim keeps the chat page and the workers list from both redeploying
+// the same agent.
+export function AgentRedeployer({
+  workbenchTenantId,
+  agent,
+}: {
+  readonly workbenchTenantId: string;
+  readonly agent: {
+    readonly id: string;
+    readonly name: string;
+    readonly assetName: string;
+  };
+}) {
+  const started = useRef(false);
+  const redeploy = useRestartAgent();
+  useEffect(() => {
+    if (started.current || redeploy.isPending || redeploy.isSuccess) return;
+    started.current = true;
+    if (claimAutoRedeploy(agent.id)) redeploy.mutate({ tenantId: workbenchTenantId, agent });
+  });
+  return null;
+}

apps/web/src/chat/agent-redeploy.test.ts

@@ -0,0 +1,53 @@
+import { describe, expect, test } from "bun:test";
+
+import { agentsToAutoDeploy, stoppedAgents } from "./agent-redeploy";
+import { redeployMode, workerState } from "./deployment-liveness";
+import type { WorkbenchParticipant } from "./threads-api";
+import { workerStatus } from "../pages/workers-page";
+
+const agent = (over: Partial<WorkbenchParticipant>): WorkbenchParticipant => ({
+  id: "a1",
+  kind: "agent",
+  name: "Worker",
+  address: "",
+  assetName: "worker",
+  ...over,
+});
+
+describe("agent redeploy selection", () => {
+  test("an agent marked auto is redeployed on its own", () => {
+    const released = agent({ redeploy: "auto" });
+    expect(agentsToAutoDeploy([released]).map((a) => a.id)).toEqual([released.id]);
+    expect(stoppedAgents([released])).toEqual([]);
+  });
+
+  test("an agent marked manual waits for Restart", () => {
+    const capped = agent({ redeploy: "manual" });
+    expect(agentsToAutoDeploy([capped])).toEqual([]);
+    expect(stoppedAgents([capped]).map((a) => a.id)).toEqual([capped.id]);
+  });
+
+  test("a coming-up or live agent is neither", () => {
+    for (const other of [agent({}), agent({ address: "worker@bench.example" })]) {
+      expect(agentsToAutoDeploy([other])).toEqual([]);
+      expect(stoppedAgents([other])).toEqual([]);
+    }
+  });
+});
+
+describe("workers list and chat agree", () => {
+  const shapes = [
+    { liveAddress: "w@bench.example", latest: "live", capped: false },
+    { liveAddress: null, latest: "starting", capped: false },
+    { liveAddress: null, latest: "stopped", capped: false },
+    { liveAddress: null, latest: "stopped", capped: true },
+  ] as const;
+
+  for (const shape of shapes) {
+    test(`${shape.latest} live=${shape.liveAddress !== null} capped=${shape.capped}`, () => {
+      const stopped = redeployMode(shape) === "manual";
+      expect(workerStatus(shape).tone === "ready").toBe(stopped);
+      expect(workerState(shape) === "stopped").toBe(stopped);
+    });
+  }
+});

apps/web/src/chat/agent-redeploy.ts

@@ -0,0 +1,16 @@
+import type { WorkbenchParticipant } from "./threads-api";
+
+type RedeployableAgent = WorkbenchParticipant & { readonly assetName: string };
+
+function withMode(mode: "auto" | "manual") {
+  return (agent: WorkbenchParticipant): agent is RedeployableAgent =>
+    agent.kind === "agent" && agent.assetName !== undefined && agent.redeploy === mode;
+}
+
+export function agentsToAutoDeploy(agents: readonly WorkbenchParticipant[]): RedeployableAgent[] {
+  return agents.filter(withMode("auto"));
+}
+
+export function stoppedAgents(agents: readonly WorkbenchParticipant[]): RedeployableAgent[] {
+  return agents.filter(withMode("manual"));
+}

apps/web/src/chat/deployment-liveness.test.ts

@@ -0,0 +1,100 @@
+import { describe, expect, test } from "bun:test";
+
+import { deploymentLiveness, diedQuickly, redeployMode, workerState } from "./deployment-liveness";
+
+describe("deploymentLiveness", () => {
+  test("a deployed projection over an errored run is stopped", () => {
+    expect(deploymentLiveness("deployed", "error")).toBe("stopped");
+  });
+
+  test("a deployed projection over a stopped run is stopped", () => {
+    expect(deploymentLiveness("deployed", "stopped")).toBe("stopped");
+  });
+
+  test("a deployed projection over a serving run is live", () => {
+    expect(deploymentLiveness("deployed", "running")).toBe("live");
+    expect(deploymentLiveness("deployed", "deployed")).toBe("live");
+    expect(deploymentLiveness("deployed", "updating")).toBe("live");
+  });
+
+  test("pending and recovering projections are starting", () => {
+    expect(deploymentLiveness("pending", "running")).toBe("starting");
+    expect(deploymentLiveness("recovering", "updating")).toBe("starting");
+  });
+
+  test("a deployment with no run is judged by its projection alone", () => {
+    expect(deploymentLiveness("deployed", undefined)).toBe("live");
+  });
+
+  test("terminal projections are stopped", () => {
+    for (const status of ["failed", "released", "releasing", "destroy_failed"] as const) {
+      expect(deploymentLiveness(status, "running")).toBe("stopped");
+    }
+  });
+});
+
+describe("diedQuickly", () => {
+  const run = { createdAt: "2026-10-04T10:00:00.000Z" };
+
+  test("a run that ended seconds after it started died quickly", () => {
+    expect(
+      diedQuickly({
+        ...run,
+        updatedAt: "2026-10-04T10:00:20.000Z",
+        endedAt: "2026-10-04T10:00:20.000Z",
+      }),
+    ).toBe(true);
+  });
+
+  test("a run that served for an hour before dying did not", () => {
+    expect(
+      diedQuickly({
+        ...run,
+        updatedAt: "2026-10-04T11:00:00.000Z",
+        endedAt: "2026-10-04T11:00:00.000Z",
+      }),
+    ).toBe(false);
+  });
+
+  test("falls back to updatedAt when endedAt is absent", () => {
+    expect(diedQuickly({ ...run, updatedAt: "2026-10-04T10:00:05.000Z" })).toBe(true);
+    expect(diedQuickly({ ...run, updatedAt: "2026-10-04T10:30:00.000Z", endedAt: null })).toBe(
+      false,
+    );
+  });
+});
+
+describe("2-minute boundary", () => {
+  const createdAt = "2026-10-04T10:00:00.000Z";
+
+  test("dying just inside two minutes is quick, at two minutes it is not", () => {
+    expect(diedQuickly({ createdAt, updatedAt: "2026-10-04T10:01:59.999Z" })).toBe(true);
+    expect(diedQuickly({ createdAt, updatedAt: "2026-10-04T10:02:00.000Z" })).toBe(false);
+  });
+});
+
+describe("redeployMode and workerState", () => {
+  const dead = { liveAddress: null, latest: "stopped", capped: false } as const;
+
+  test("a live agent is left alone", () => {
+    const live = { ...dead, liveAddress: "w@bench.example" };
+    expect(redeployMode(live)).toBeUndefined();
+    expect(workerState(live)).toBe("live");
+  });
+
+  test("a coming-up agent is left alone and reads starting", () => {
+    const coming = { liveAddress: null, latest: "starting", capped: false } as const;
+    expect(redeployMode(coming)).toBeUndefined();
+    expect(workerState(coming)).toBe("starting");
+  });
+
+  test("a dead agent redeploys on its own and reads starting", () => {
+    expect(redeployMode(dead)).toBe("auto");
+    expect(workerState(dead)).toBe("starting");
+  });
+
+  test("a capped dead agent waits for Restart and reads stopped", () => {
+    expect(redeployMode({ ...dead, capped: true })).toBe("manual");
+    expect(workerState({ ...dead, capped: true })).toBe("stopped");
+  });
+});

apps/web/src/chat/deployment-liveness.ts

@@ -0,0 +1,52 @@
+import type { WorkflowDeploymentResponse, WorkflowRunStatus } from "@intx/types";
+
+export type DeploymentLiveness = "live" | "starting" | "stopped";
+
+const DEAD_RUN_STATUSES = new Set<WorkflowRunStatus>(["error", "stopped"]);
+
+/** The deployments projection never consults run status, so a dead run still
+ * reads "deployed"; the run's own status settles it. */
+export function deploymentLiveness(
+  deploymentStatus: WorkflowDeploymentResponse["status"],
+  runStatus: WorkflowRunStatus | undefined,
+): DeploymentLiveness {
+  if (runStatus !== undefined && DEAD_RUN_STATUSES.has(runStatus)) return "stopped";
+  if (deploymentStatus === "deployed") return "live";
+  if (deploymentStatus === "pending" || deploymentStatus === "recovering") return "starting";
+  return "stopped";
+}
+
+const QUICK_DEATH_MS = 2 * 60 * 1000;
+
+/** A run that ended within moments of starting: redeploying it again would
+ * only loop, so the person restarts it instead. */
+export function diedQuickly(run: {
+  readonly createdAt: string;
+  readonly updatedAt: string;
+  readonly endedAt?: string | null;
+}): boolean {
+  const end = Date.parse(run.endedAt ?? run.updatedAt);
+  return end - Date.parse(run.createdAt) < QUICK_DEATH_MS;
+}
+
+export type RedeployMode = "auto" | "manual";
+export type WorkerState = "live" | "starting" | "stopped";
+
+type AgentLiveness = {
+  readonly liveAddress: string | null;
+  readonly latest: DeploymentLiveness;
+  readonly capped: boolean;
+};
+
+/** What happens to a worker with nothing live: a dead one redeploys on its
+ * own once, then, if that died again quickly, waits for the person. */
+export function redeployMode(agent: AgentLiveness): RedeployMode | undefined {
+  if (agent.liveAddress !== null || agent.latest !== "stopped") return undefined;
+  return agent.capped ? "manual" : "auto";
+}
+
+/** The one worker state every surface reads. */
+export function workerState(agent: AgentLiveness): WorkerState {
+  if (agent.liveAddress !== null) return "live";
+  return redeployMode(agent) === "manual" ? "stopped" : "starting";
+}

apps/web/src/chat/list-chat-agents.test.ts

@@ -0,0 +1,113 @@
+import { afterEach, describe, expect, spyOn, test } from "bun:test";
+
+import { redeployMode } from "./deployment-liveness";
+import { markRedeployFailed, clearRedeployFailure } from "./redeploy-failures";
+import { listChatAgents } from "./threads-api";
+
+const fetchSpy = spyOn(globalThis, "fetch");
+afterEach(() => {
+  fetchSpy.mockReset();
+  clearRedeployFailure("asset-1");
+});
+
+type Deployment = { id: string; status: string; createdAt: string };
+type Run = { id: string; status: string; createdAt: string; updatedAt: string; endedAt?: string };
+
+const T0 = "2026-10-04T10:00:00.000Z";
+const T1H = "2026-10-04T11:00:00.000Z";
+const T10S = "2026-10-04T11:00:10.000Z";
+
+function hub(deployments: Deployment[], runs: Run[]) {
+  fetchSpy.mockImplementation(((input: string | URL | Request) => {
+    const url = String(input);
+    const body = url.includes("/workflows/deployments")
+      ? deployments.map((d) => ({
+          ...d,
+          tenantId: "t1",
+          definitionAssetId: "asset-1",
+        }))
+      : url.includes("/workflows/runs")
+        ? {
+            data: runs.map((r) => ({
+              ...r,
+              definitionId: "def",
+              definitionName: "worker",
+              tenantId: "t1",
+              address: `${r.id}@bench.example`,
+            })),
+            nextCursor: null,
+          }
+        : [{ id: "asset-1", name: "worker" }];
+    return Promise.resolve(new Response(JSON.stringify(body)));
+  }) as typeof fetch);
+}
+
+async function agent() {
+  const [only] = await listChatAgents("t1");
+  if (only === undefined) throw new Error("no agent listed");
+  return only;
+}
+
+describe("listChatAgents self-heal join", () => {
+  test("a hub restart (released deployment, long-lived errored run) redeploys on its own", async () => {
+    hub(
+      [{ id: "r1", status: "released", createdAt: T0 }],
+      [{ id: "r1", status: "error", createdAt: T0, updatedAt: T1H, endedAt: T1H }],
+    );
+    const listed = await agent();
+    expect(listed.liveAddress).toBeNull();
+    expect(redeployMode(listed)).toBe("auto");
+  });
+
+  test("a deployed projection over a dead run is not live", async () => {
+    hub(
+      [{ id: "r1", status: "deployed", createdAt: T0 }],
+      [{ id: "r1", status: "stopped", createdAt: T0, updatedAt: T1H, endedAt: T1H }],
+    );
+    expect((await agent()).liveAddress).toBeNull();
+  });
+
+  test("a redeploy that died within moments waits for Restart", async () => {
+    hub(
+      [
+        { id: "r2", status: "deployed", createdAt: T1H },
+        { id: "r1", status: "released", createdAt: T0 },
+      ],
+      [
+        { id: "r2", status: "error", createdAt: T1H, updatedAt: T10S, endedAt: T10S },
+        { id: "r1", status: "error", createdAt: T0, updatedAt: T1H, endedAt: T1H },
+      ],
+    );
+    expect(redeployMode(await agent())).toBe("manual");
+  });
+
+  test("a fresh deployment whose run is not addressed yet is starting, not redeployed again", async () => {
+    hub(
+      [
+        { id: "r2", status: "pending", createdAt: T1H },
+        { id: "r1", status: "released", createdAt: T0 },
+      ],
+      [{ id: "r1", status: "error", createdAt: T0, updatedAt: T1H, endedAt: T1H }],
+    );
+    expect(redeployMode(await agent())).toBeUndefined();
+  });
+
+  test("a failed redeploy caps the agent", async () => {
+    hub(
+      [{ id: "r1", status: "released", createdAt: T0 }],
+      [{ id: "r1", status: "error", createdAt: T0, updatedAt: T1H, endedAt: T1H }],
+    );
+    markRedeployFailed("asset-1");
+    expect(redeployMode(await agent())).toBe("manual");
+  });
+
+  test("a live run is left alone", async () => {
+    hub(
+      [{ id: "r1", status: "deployed", createdAt: T0 }],
+      [{ id: "r1", status: "running", createdAt: T0, updatedAt: T1H }],
+    );
+    const listed = await agent();
+    expect(listed.liveAddress).toBe("r1@bench.example");
+    expect(redeployMode(listed)).toBeUndefined();
+  });
+});

apps/web/src/chat/redeploy-failures.test.ts

@@ -0,0 +1,15 @@
+import { describe, expect, test } from "bun:test";
+
+import { claimAutoRedeploy, releaseAutoRedeploy } from "./redeploy-failures";
+
+describe("auto-redeploy claim", () => {
+  test("only the first surface claims an asset until it settles", () => {
+    expect(claimAutoRedeploy("asset-x")).toBe(true);
+    expect(claimAutoRedeploy("asset-x")).toBe(false);
+    expect(claimAutoRedeploy("asset-y")).toBe(true);
+    releaseAutoRedeploy("asset-x");
+    releaseAutoRedeploy("asset-y");
+    expect(claimAutoRedeploy("asset-x")).toBe(true);
+    releaseAutoRedeploy("asset-x");
+  });
+});