fix(hub): bound coordinator start at a measured 60 seconds

.env.hub.example

@@ -94,6 +94,8 @@ PRODUCT_HELLO_WORDING=
 # Agents run in sidecars. By default the hub starts them as local processes.
 # Most local sidecars running at once.
 # SIDECAR_LOCAL_MAX_ACTIVE=16
+# Longest wait, in milliseconds, for a project's coordinator to start. Raise it on a slow machine; it must stay under 300000.
+# COORDINATOR_START_TIMEOUT_MS=60000
 # Optional Docker sidecars. deploy/compose.docker-sidecars.yml sets them for the hub
 # container, with the published sidecar image at BUILDER_VERSION. From source, build
 # it with `bun run docker:sidecar` and set:

apps/hub/src/app.ts

@@ -45,6 +45,7 @@ import { getLogger } from "@intx/log";
 import { hexDecode, hexEncode } from "@intx/types";
 import { createProductConfig } from "./config/index.ts";
 import {
+  readCoordinatorStartTimeoutMs,
   readHubEnvironment,
   readGoogleEmulateBaseUrl,
   required,
@@ -109,6 +110,8 @@ export async function createHubRuntime(
   rawEnvironment: Record<string, string | undefined> = process.env,
 ) {
   const env = readHubEnvironment(rawEnvironment);
+  const coordinatorStartTimeoutMs =
+    readCoordinatorStartTimeoutMs(rawEnvironment);
   const handle = createDB(env.database);
   let authEmail: ReturnType<typeof createAuthEmail>;
   try {
@@ -350,6 +353,8 @@ export async function createHubRuntime(
           throw error;
         }
       },
+      coordinatorStartTimeoutMs,
+      workflows.services.workflowLifecycleService.requestCancellation,
     );
     const coordinatorUpgrades = createCoordinatorUpgrades({
       db: handle.db,

apps/hub/src/environment.ts

@@ -131,6 +131,29 @@ export function readBaseURL(env: Environment): string {
   );
 }
 
+// Measured starts take seconds (3.6 s cold on a fresh install, 15 s worst over 115 dev
+// starts), so a minute leaves margin and a failed start reaches the person soon. Slow
+// machines raise it, but a failed start must still arrive before the web's 5-minute
+// request ceiling.
+export function readCoordinatorStartTimeoutMs(env: Environment): number {
+  const value = positiveInteger(env, "COORDINATOR_START_TIMEOUT_MS", 60_000);
+  if (value >= 300_000)
+    throw new Error("COORDINATOR_START_TIMEOUT_MS must be under 300000");
+  return value;
+}
+
+export function positiveInteger(
+  env: Environment,
+  name: string,
+  absent: number,
+): number {
+  const raw = env[name];
+  const value = raw === undefined ? absent : Number(raw);
+  if (!Number.isSafeInteger(value) || value <= 0)
+    throw new Error(`${name} must be a positive safe integer`);
+  return value;
+}
+
 export function readHubEnvironment(env: Environment) {
   const credentialKey = required(env, "CREDENTIAL_ENCRYPTION_KEY");
   const principalKey = required(env, "PRINCIPAL_KEY_ENCRYPTION_KEY");

apps/hub/src/projects/coordinator-startup.test.ts

@@ -36,20 +36,28 @@ afterEach(() => {
   setSystemTime();
 });
 
-test("a coordinator ready after more than 120 s still starts", async () => {
-  await awaitCoordinatorReady(
-    nativeStates(["provisioning", "allocated"], 121_000),
-    "tenant",
-    "run",
-  );
-});
-
-test("a start still pending past 270 s fails", async () => {
+test("a start still pending past the limit fails and cancels the run", async () => {
+  const cancelled: string[][] = [];
   await expect(
     awaitCoordinatorReady(
-      nativeStates(["provisioning"], 271_000),
+      nativeStates(["provisioning"], 61_000),
       "tenant",
       "run",
+      60_000,
+      async (...args) => {
+        cancelled.push(args);
+      },
     ),
   ).rejects.toMatchObject({ code: "coordinator_start_failed" });
+  expect(cancelled).toEqual([["tenant", "run", "coordinator_start_timeout"]]);
+});
+
+test("a raised limit lets a 90 s start finish", async () => {
+  await awaitCoordinatorReady(
+    nativeStates(["provisioning", "allocated"], 90_000),
+    "tenant",
+    "run",
+    120_000,
+    async () => {},
+  );
 });

apps/hub/src/projects/coordinator-startup.ts

@@ -22,12 +22,15 @@ import { ProjectError } from "./errors.ts";
 import type { Actor } from "../lib/actor.ts";
 
 const START_FAILED = "coordinator_start_failed";
-// Longer than the native sidecar ensure and connection stages (120 s each) normally take, and
-// shorter than the web's 5-minute request ceiling, so a failed start reaches the person who
-// created the project instead of archiving it after they stopped waiting.
-const READY_WAIT_MS = 270_000;
+const START_TIMEOUT = "coordinator_start_timeout";
 const READY_POLL_MS = 250;
 
+type CancelRun = (
+  tenantId: string,
+  runId: string,
+  reason: string,
+) => Promise<unknown>;
+
 type BoundRun = { runId: string; ended: boolean };
 
 type CoordinatorStartupPorts = {
@@ -174,7 +177,12 @@ async function attach(
     runId = created.runId;
     deployed.set(key, runId);
   }
-  await ports.awaitReady({ tenantId: actor.tenantId, runId });
+  try {
+    await ports.awaitReady({ tenantId: actor.tenantId, runId });
+  } catch (error) {
+    deployed.delete(key);
+    throw error;
+  }
   try {
     await ports.bind(actor, projectId, {
       runId,
@@ -195,8 +203,10 @@ export async function awaitCoordinatorReady(
   db: DB["db"],
   tenantId: string,
   runId: string,
+  readyWaitMs: number,
+  cancel: CancelRun,
 ): Promise<void> {
-  const deadline = Date.now() + READY_WAIT_MS;
+  const deadline = Date.now() + readyWaitMs;
   for (;;) {
     const [run] = await db
       .select()
@@ -231,7 +241,12 @@ export async function awaitCoordinatorReady(
       (run && workflowRunExecutability(run) === "terminal")
     )
       throw startFailed();
-    if (Date.now() >= deadline) throw startFailed();
+    if (Date.now() >= deadline) {
+      // The hub's limit can be shorter than the native start stages, so the hub
+      // cancels what it stops waiting for instead of leaving it to claim a slot later.
+      await cancel(tenantId, runId, START_TIMEOUT);
+      throw startFailed();
+    }
     await new Promise((resolve) => setTimeout(resolve, READY_POLL_MS));
   }
 }
@@ -248,6 +263,8 @@ export function createInstalledCoordinatorStartup(
     domain: string;
     deployment: NativeSpecialistDeployment;
   }) => Promise<{ runId: string }>,
+  readyWaitMs: number,
+  cancel: CancelRun,
 ) {
   const registry = createCoordinatorRegistry(db);
   return createProjectCoordinatorStartup({
@@ -302,7 +319,13 @@ export function createInstalledCoordinatorStartup(
       }
     },
     awaitReady(args) {
-      return awaitCoordinatorReady(db, args.tenantId, args.runId);
+      return awaitCoordinatorReady(
+        db,
+        args.tenantId,
+        args.runId,
+        readyWaitMs,
+        cancel,
+      );
     },
     bind(actor, projectId, body) {
       return registry.bind(actor, projectId, body);

apps/hub/src/provisioning/workflow-services.ts

@@ -38,6 +38,7 @@ import {
   createDockerSidecarProvisioner,
   defaultDockerLimits,
 } from "./docker.ts";
+import { positiveInteger } from "../environment.ts";
 import { localSidecarDataRoot } from "../sidecar/data-root.ts";
 import { createHostCapacity } from "../sidecar/host-capacity.ts";
 import {
@@ -86,13 +87,6 @@ export async function createWorkflowServices(options: {
     "/api/sidecars/ws",
     options.publicURL,
   ).href.replace(/^http/, "ws");
-  const integer = (key: string, fallback: number) => {
-    const raw = environment[key];
-    const value = raw === undefined ? fallback : Number(raw);
-    if (!Number.isSafeInteger(value) || value <= 0)
-      throw new Error(`${key} must be a positive safe integer`);
-    return value;
-  };
   const adapterManifest = parseAdapterManifestEnv(
     environment["SIDECAR_ADAPTER_MANIFEST"],
   ) ?? [
@@ -121,7 +115,11 @@ export async function createWorkflowServices(options: {
   const credentialEncryptionKey = environment["CREDENTIAL_ENCRYPTION_KEY"];
   if (!credentialEncryptionKey)
     throw new Error("CREDENTIAL_ENCRYPTION_KEY is required");
-  const maxActiveDeployments = integer("SIDECAR_LOCAL_MAX_ACTIVE", 16);
+  const maxActiveDeployments = positiveInteger(
+    environment,
+    "SIDECAR_LOCAL_MAX_ACTIVE",
+    16,
+  );
   const dataRoot = localSidecarDataRoot(
     environment["HUB_DATA_DIR"] ?? "data/hub",
   );
@@ -167,12 +165,21 @@ export async function createWorkflowServices(options: {
         image: image.Id,
         adapterManifest,
         limits: {
-          memoryBytes: integer(
+          memoryBytes: positiveInteger(
+            environment,
             "SIDECAR_MEMORY_BYTES",
             defaultDockerLimits.memoryBytes,
           ),
-          pids: integer("SIDECAR_PIDS", defaultDockerLimits.pids),
-          nanoCpus: integer("SIDECAR_NANO_CPUS", defaultDockerLimits.nanoCpus),
+          pids: positiveInteger(
+            environment,
+            "SIDECAR_PIDS",
+            defaultDockerLimits.pids,
+          ),
+          nanoCpus: positiveInteger(
+            environment,
+            "SIDECAR_NANO_CPUS",
+            defaultDockerLimits.nanoCpus,
+          ),
         },
         ...(environment["SIDECAR_HUB_WS_URL"]
           ? { hubWebSocketUrl: environment["SIDECAR_HUB_WS_URL"] }

docs/SELF_HOST.md

@@ -128,6 +128,9 @@ docker compose --env-file .env.hub -f compose.yml -f compose.docker-sidecars.yml
   when that address is not reachable from them.
 - Per-container limits: `SIDECAR_MEMORY_BYTES`, `SIDECAR_PIDS`,
   `SIDECAR_NANO_CPUS`.
+- `COORDINATOR_START_TIMEOUT_MS` (default `60000`) is the longest a project's
+  coordinator may take to start; raise it on a slow machine, but keep it under
+  `300000`.
 
 ## Logs