Deliver first-turn mail once the provisioned sidecar is routable (CL-7476)

apps/hub/src/index.ts

@@ -2260,6 +2260,8 @@ export async function createHub(config: HubConfig) {
             repoStore: agentRepoStore.repoStore,
             workflowAllocationService,
             credentialCipher,
+            isRoutable: (address) =>
+              sidecarRouter.getRoutableAddresses().includes(address),
             cryptoProviderCache: cryptoProviders,
             persistLaunch: async (input) => {
               await workbenchLaunchPersistExtra(input)(db);

apps/hub/src/system-seed.ts

@@ -35,6 +35,12 @@ const log = getLogger(["hub", "system-seed"]);
 const DEFAULT_DEADLINE_MS = 60_000;
 const DEFAULT_POLL_INTERVAL_MS = 2_000;
 
+// Matches `packages/seeding/src/seed.ts`'s `resolveRealSourceOfferingIds`
+// failure verbatim — the one deploy failure that is expected, not
+// transient, when no operator seed key is configured.
+const NO_CATALOG_OFFERINGS_REASON =
+  "this tenant has no catalog offerings to deploy against";
+
 // Matches `workbench seed`'s own default model source (readSeedConfig in
 // the now-deleted `packages/cli/src/config.ts`): real anthropic/
 // claude-sonnet-5 when a hub-owned key is configured (`config.seedModel`),
@@ -150,6 +156,21 @@ export async function runSystemSeed(deps: SystemSeedDeps): Promise<void> {
     } catch (cause) {
       const reason = cause instanceof Error ? cause.message : String(cause);
       if (Date.now() >= deadline) {
+        // No operator seed key means `seedCatalog` can only ever plant a
+        // placeholder credential — never a real, launchable offering —
+        // so a default workflow's deploy permanently has nothing to
+        // deploy against until an operator configures one. That is the
+        // expected shape of an unconfigured dev/CI boot, not a fault the
+        // next boot can retry its way out of, so it is a logged skip
+        // rather than an error (which `reportError` would otherwise
+        // paint red on every single boot).
+        if (
+          deps.seedModel === undefined &&
+          reason.includes(NO_CATALOG_OFFERINGS_REASON)
+        ) {
+          log.info`root tenant seed skipped: no operator seed key configured, so the tenant has no catalog offerings to deploy against yet`;
+          return;
+        }
         reportError(cause, { operation: "system-seed.seedRootTenant" });
         log.error`root tenant seed did not complete before its deadline (last error: ${reason}); the next boot will retry`;
         return;

packages/webhook-triggers/src/launch.ts

@@ -14,6 +14,7 @@
 // reports through `@corbits/error-sink`, naming the run.
 import { reportError } from "@corbits/error-sink";
 import {
+  deliverWhenRoutable,
   readDefinitionProjection,
   readFoldedBody,
   WORKFLOW_SOURCE_ENTRY,
@@ -46,6 +47,15 @@ export type LaunchWebhookTriggerDeps = {
     "prepareProvisionedDeployment"
   >;
   sessionService: Pick<SessionService, "sendUserMessage">;
+  /**
+   * Reads the hub's live sidecar routing table. A freshly provisioned
+   * run's sidecar takes several seconds to boot and register (CL-7476)
+   * — a webhook's opening mail can land in that gap and fail with
+   * "agent is unreachable" even though the run itself deployed fine.
+   * `deliverWhenRoutable` polls this until the address is routable
+   * before its one retry.
+   */
+  isRoutable: (address: string) => boolean;
   cryptoProviderCache: CryptoProviderCache;
   /**
    * Host-supplied cipher. Interchange decrypts bindings inside
@@ -201,15 +211,19 @@ export async function launchWebhookTrigger(
     prepared.anchorRunId,
   );
   try {
-    await deps.sessionService.sendUserMessage({
-      agentAddress: prepared.deploymentAddress,
-      from: `webhook-trigger:${trigger.id}`,
-      messageId: `<${crypto.randomUUID()}@${tenantRow.domain}>`,
-      date: new Date(),
-      content,
-      sessionId,
-      tenantId: trigger.tenantId,
-      cryptoProvider,
+    await deliverWhenRoutable({
+      send: () =>
+        deps.sessionService.sendUserMessage({
+          agentAddress: prepared.deploymentAddress,
+          from: `webhook-trigger:${trigger.id}`,
+          messageId: `<${crypto.randomUUID()}@${tenantRow.domain}>`,
+          date: new Date(),
+          content,
+          sessionId,
+          tenantId: trigger.tenantId,
+          cryptoProvider,
+        }),
+      isRoutable: () => deps.isRoutable(prepared.deploymentAddress),
     });
   } catch (error) {
     reportError(error, {

packages/webhook-triggers/test/launch.test.ts

@@ -137,6 +137,7 @@ let sendUserMessageCalls: unknown[] = [];
 let sendUserMessageImpl: () => Promise<Uint8Array> = async () =>
   new Uint8Array([1]);
 let cryptoGetKeys: string[] = [];
+let isRoutableForTest = true;
 
 function baseDeps() {
   return {
@@ -177,6 +178,7 @@ function baseDeps() {
         return sendUserMessageImpl();
       },
     },
+    isRoutable: () => isRoutableForTest,
   };
 }
 
@@ -191,6 +193,7 @@ function resetLaunchSpies() {
   visibleOfferings = [...DEFAULT_VISIBLE_OFFERINGS];
   frozenProjection = INERT_PROJECTION;
   sendUserMessageImpl = async () => new Uint8Array([1]);
+  isRoutableForTest = true;
 }
 
 describe("launchWebhookTrigger", () => {
@@ -214,6 +217,39 @@ describe("launchWebhookTrigger", () => {
     expect(sendUserMessageCalls).toHaveLength(1);
   });
 
+  // CL-7476: a freshly provisioned run's sidecar takes several seconds to
+  // boot and register with the hub. The opening mail can land in that
+  // gap and fail "agent is unreachable" even though the run deployed
+  // fine — `deliverWhenRoutable` must retry once the address becomes
+  // routable rather than dropping the mail for good.
+  test("retries the opening mail once the freshly provisioned run becomes routable", async () => {
+    resetLaunchSpies();
+    isRoutableForTest = false;
+    let sendAttempts = 0;
+    sendUserMessageImpl = async () => {
+      sendAttempts += 1;
+      if (sendAttempts === 1) {
+        throw new Error("agent is unreachable: ins_1@ten1.workbench.test");
+      }
+      return new Uint8Array([1]);
+    };
+
+    const deps = baseDeps();
+    const sendPromise = launchWebhookTrigger(deps, TRIGGER, { status: "ok" });
+
+    await new Promise((resolve) => setTimeout(resolve, 10));
+    isRoutableForTest = true;
+
+    const result = await sendPromise;
+
+    expect(result).toEqual({
+      instanceId: INTERCHANGE_RUN_ID,
+      triggerAddress: INTERCHANGE_ADDRESS,
+    });
+    expect(sendUserMessageCalls).toHaveLength(2);
+    expect(reportErrorCalls).toHaveLength(0);
+  });
+
   test("reports the delivery failure with the run's context", async () => {
     resetLaunchSpies();
     const deliveryError = new Error("sidecar unreachable");

packages/workflows/src/deliver-when-routable.test.ts

@@ -0,0 +1,74 @@
+import { describe, expect, test } from "bun:test";
+import { deliverWhenRoutable } from "./deliver-when-routable";
+
+function unreachable(): Error {
+  return new Error("agent is unreachable: run_1@ten1.workbench.test");
+}
+
+describe("deliverWhenRoutable", () => {
+  test("retries once the address is routable after an unreachable send", async () => {
+    let sendCalls = 0;
+    let routablePolls = 0;
+    const sleeps: number[] = [];
+
+    const result = await deliverWhenRoutable({
+      send: async () => {
+        sendCalls++;
+        if (sendCalls === 1) {
+          throw unreachable();
+        }
+        return "delivered";
+      },
+      isRoutable: () => {
+        routablePolls++;
+        return routablePolls >= 2;
+      },
+      sleep: async (ms) => {
+        sleeps.push(ms);
+      },
+    });
+
+    expect(result).toBe("delivered");
+    expect(sendCalls).toBe(2);
+    expect(routablePolls).toBe(2);
+    expect(sleeps.length).toBe(1);
+  });
+
+  test("rethrows the original unreachable error once the deadline passes", async () => {
+    let now = 0;
+    const originalNow = Date.now;
+    Date.now = () => now;
+    try {
+      let sendCalls = 0;
+      await expect(
+        deliverWhenRoutable({
+          send: async () => {
+            sendCalls++;
+            throw unreachable();
+          },
+          isRoutable: () => false,
+          deadlineMs: 500,
+          pollIntervalMs: 100,
+          sleep: async (ms) => {
+            now += ms;
+          },
+        }),
+      ).rejects.toThrow("agent is unreachable");
+      expect(sendCalls).toBe(1);
+    } finally {
+      Date.now = originalNow;
+    }
+  });
+
+  test("does not retry an error other than unreachable", async () => {
+    const err = new Error("boom");
+    await expect(
+      deliverWhenRoutable({
+        send: async () => {
+          throw err;
+        },
+        isRoutable: () => true,
+      }),
+    ).rejects.toBe(err);
+  });
+});

packages/workflows/src/deliver-when-routable.ts

@@ -0,0 +1,62 @@
+// CL-7476: since the process provisioner started spawning one sidecar per
+// allocation, a freshly launched run's first mail can arrive before that
+// sidecar has finished booting and registering with the hub, so the send
+// fails with "agent is unreachable" even though nothing is actually wrong
+// — the address just isn't routable yet. The webhook trigger's ingress
+// delivery goes through this helper; chat keeps its own existing
+// wake-and-retry loop (`sendRunMailWithReclaimRetry`) instead.
+const DEFAULT_DEADLINE_MS = 20_000;
+const DEFAULT_POLL_INTERVAL_MS = 250;
+
+export function isAgentUnreachableError(err: unknown): boolean {
+  return err instanceof Error && err.message.includes("agent is unreachable");
+}
+
+export type DeliverWhenRoutableOptions<T> = {
+  /** Performs the actual delivery. Called once, and again after the
+   * address becomes routable if the first attempt fails unreachable. */
+  send: () => Promise<T>;
+  /** Reads the hub's live routing table for this delivery's address. */
+  isRoutable: () => boolean;
+  /** Defaults to matching Interchange's "agent is unreachable" message. */
+  isUnreachable?: (err: unknown) => boolean;
+  /** How long to keep polling before giving up. Defaults to a budget a
+   * touch past the process provisioner's own boot-and-register time. */
+  deadlineMs?: number;
+  pollIntervalMs?: number;
+  sleep?: (ms: number) => Promise<void>;
+};
+
+function defaultSleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Attempts `send`; if it fails with "agent is unreachable", polls
+ * `isRoutable` until the address is routable (or the deadline passes)
+ * and sends exactly once more. Any other failure, or an unreachable
+ * failure that never resolves before the deadline, is rethrown as-is —
+ * this never swallows the original error or retries indefinitely.
+ */
+export async function deliverWhenRoutable<T>(
+  opts: DeliverWhenRoutableOptions<T>,
+): Promise<T> {
+  const isUnreachable = opts.isUnreachable ?? isAgentUnreachableError;
+  try {
+    return await opts.send();
+  } catch (err) {
+    if (!isUnreachable(err)) {
+      throw err;
+    }
+    const deadline = Date.now() + (opts.deadlineMs ?? DEFAULT_DEADLINE_MS);
+    const pollIntervalMs = opts.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
+    const sleep = opts.sleep ?? defaultSleep;
+    while (!opts.isRoutable()) {
+      if (Date.now() >= deadline) {
+        throw err;
+      }
+      await sleep(pollIntervalMs);
+    }
+    return await opts.send();
+  }
+}

packages/workflows/src/index.ts

@@ -55,6 +55,11 @@ export {
   type ZonedParts,
 } from "./schedule/cron";
 export { scheduleCronFromProjection } from "./schedule/from-projection";
+export {
+  deliverWhenRoutable,
+  isAgentUnreachableError,
+  type DeliverWhenRoutableOptions,
+} from "./deliver-when-routable";
 export {
   listScheduledWorkflowDefinitions,
   scheduledDefinitionsFromRows,