diff --git a/.env.hub.example b/.env.hub.example index 83625211..ae24ad41 100644 --- a/.env.hub.example +++ b/.env.hub.example @@ -94,6 +94,8 @@ PRODUCT_HELLO_WORDING= # Agents run in sidecars. By default the hub starts them as local processes. # Most local sidecars running at once. # SIDECAR_LOCAL_MAX_ACTIVE=16 +# Longest wait, in milliseconds, for a project's coordinator to start. Raise it on a slow machine; it must stay under 300000. +# COORDINATOR_START_TIMEOUT_MS=60000 # Optional Docker sidecars. deploy/compose.docker-sidecars.yml sets them for the hub # container, with the published sidecar image at BUILDER_VERSION. From source, build # it with `bun run docker:sidecar` and set: diff --git a/apps/hub/src/app.ts b/apps/hub/src/app.ts index 53ff4293..71ebfcd7 100644 --- a/apps/hub/src/app.ts +++ b/apps/hub/src/app.ts @@ -45,6 +45,7 @@ import { getLogger } from "@intx/log"; import { hexDecode, hexEncode } from "@intx/types"; import { createProductConfig } from "./config/index.ts"; import { + readCoordinatorStartTimeoutMs, readHubEnvironment, readGoogleEmulateBaseUrl, required, @@ -109,6 +110,8 @@ export async function createHubRuntime( rawEnvironment: Record = process.env, ) { const env = readHubEnvironment(rawEnvironment); + const coordinatorStartTimeoutMs = + readCoordinatorStartTimeoutMs(rawEnvironment); const handle = createDB(env.database); let authEmail: ReturnType; try { @@ -350,6 +353,8 @@ export async function createHubRuntime( throw error; } }, + coordinatorStartTimeoutMs, + workflows.services.workflowLifecycleService.requestCancellation, ); const coordinatorUpgrades = createCoordinatorUpgrades({ db: handle.db, diff --git a/apps/hub/src/environment.ts b/apps/hub/src/environment.ts index 50ee9f00..ae751868 100644 --- a/apps/hub/src/environment.ts +++ b/apps/hub/src/environment.ts @@ -131,6 +131,29 @@ export function readBaseURL(env: Environment): string { ); } +// Measured starts take seconds (3.6 s cold on a fresh install, 15 s worst over 115 dev +// starts), so a minute leaves margin and a failed start reaches the person soon. Slow +// machines raise it, but a failed start must still arrive before the web's 5-minute +// request ceiling. +export function readCoordinatorStartTimeoutMs(env: Environment): number { + const value = positiveInteger(env, "COORDINATOR_START_TIMEOUT_MS", 60_000); + if (value >= 300_000) + throw new Error("COORDINATOR_START_TIMEOUT_MS must be under 300000"); + return value; +} + +export function positiveInteger( + env: Environment, + name: string, + absent: number, +): number { + const raw = env[name]; + const value = raw === undefined ? absent : Number(raw); + if (!Number.isSafeInteger(value) || value <= 0) + throw new Error(`${name} must be a positive safe integer`); + return value; +} + export function readHubEnvironment(env: Environment) { const credentialKey = required(env, "CREDENTIAL_ENCRYPTION_KEY"); const principalKey = required(env, "PRINCIPAL_KEY_ENCRYPTION_KEY"); diff --git a/apps/hub/src/projects/coordinator-startup.test.ts b/apps/hub/src/projects/coordinator-startup.test.ts index 50e0a328..38a36bab 100644 --- a/apps/hub/src/projects/coordinator-startup.test.ts +++ b/apps/hub/src/projects/coordinator-startup.test.ts @@ -36,20 +36,28 @@ afterEach(() => { setSystemTime(); }); -test("a coordinator ready after more than 120 s still starts", async () => { - await awaitCoordinatorReady( - nativeStates(["provisioning", "allocated"], 121_000), - "tenant", - "run", - ); -}); - -test("a start still pending past 270 s fails", async () => { +test("a start still pending past the limit fails and cancels the run", async () => { + const cancelled: string[][] = []; await expect( awaitCoordinatorReady( - nativeStates(["provisioning"], 271_000), + nativeStates(["provisioning"], 61_000), "tenant", "run", + 60_000, + async (...args) => { + cancelled.push(args); + }, ), ).rejects.toMatchObject({ code: "coordinator_start_failed" }); + expect(cancelled).toEqual([["tenant", "run", "coordinator_start_timeout"]]); +}); + +test("a raised limit lets a 90 s start finish", async () => { + await awaitCoordinatorReady( + nativeStates(["provisioning", "allocated"], 90_000), + "tenant", + "run", + 120_000, + async () => {}, + ); }); diff --git a/apps/hub/src/projects/coordinator-startup.ts b/apps/hub/src/projects/coordinator-startup.ts index fa2e91f3..514aafe4 100644 --- a/apps/hub/src/projects/coordinator-startup.ts +++ b/apps/hub/src/projects/coordinator-startup.ts @@ -22,12 +22,15 @@ import { ProjectError } from "./errors.ts"; import type { Actor } from "../lib/actor.ts"; const START_FAILED = "coordinator_start_failed"; -// Longer than the native sidecar ensure and connection stages (120 s each) normally take, and -// shorter than the web's 5-minute request ceiling, so a failed start reaches the person who -// created the project instead of archiving it after they stopped waiting. -const READY_WAIT_MS = 270_000; +const START_TIMEOUT = "coordinator_start_timeout"; const READY_POLL_MS = 250; +type CancelRun = ( + tenantId: string, + runId: string, + reason: string, +) => Promise; + type BoundRun = { runId: string; ended: boolean }; type CoordinatorStartupPorts = { @@ -174,7 +177,12 @@ async function attach( runId = created.runId; deployed.set(key, runId); } - await ports.awaitReady({ tenantId: actor.tenantId, runId }); + try { + await ports.awaitReady({ tenantId: actor.tenantId, runId }); + } catch (error) { + deployed.delete(key); + throw error; + } try { await ports.bind(actor, projectId, { runId, @@ -195,8 +203,10 @@ export async function awaitCoordinatorReady( db: DB["db"], tenantId: string, runId: string, + readyWaitMs: number, + cancel: CancelRun, ): Promise { - const deadline = Date.now() + READY_WAIT_MS; + const deadline = Date.now() + readyWaitMs; for (;;) { const [run] = await db .select() @@ -231,7 +241,12 @@ export async function awaitCoordinatorReady( (run && workflowRunExecutability(run) === "terminal") ) throw startFailed(); - if (Date.now() >= deadline) throw startFailed(); + if (Date.now() >= deadline) { + // The hub's limit can be shorter than the native start stages, so the hub + // cancels what it stops waiting for instead of leaving it to claim a slot later. + await cancel(tenantId, runId, START_TIMEOUT); + throw startFailed(); + } await new Promise((resolve) => setTimeout(resolve, READY_POLL_MS)); } } @@ -248,6 +263,8 @@ export function createInstalledCoordinatorStartup( domain: string; deployment: NativeSpecialistDeployment; }) => Promise<{ runId: string }>, + readyWaitMs: number, + cancel: CancelRun, ) { const registry = createCoordinatorRegistry(db); return createProjectCoordinatorStartup({ @@ -302,7 +319,13 @@ export function createInstalledCoordinatorStartup( } }, awaitReady(args) { - return awaitCoordinatorReady(db, args.tenantId, args.runId); + return awaitCoordinatorReady( + db, + args.tenantId, + args.runId, + readyWaitMs, + cancel, + ); }, bind(actor, projectId, body) { return registry.bind(actor, projectId, body); diff --git a/apps/hub/src/provisioning/workflow-services.ts b/apps/hub/src/provisioning/workflow-services.ts index 3b69988a..a90b50cb 100644 --- a/apps/hub/src/provisioning/workflow-services.ts +++ b/apps/hub/src/provisioning/workflow-services.ts @@ -38,6 +38,7 @@ import { createDockerSidecarProvisioner, defaultDockerLimits, } from "./docker.ts"; +import { positiveInteger } from "../environment.ts"; import { localSidecarDataRoot } from "../sidecar/data-root.ts"; import { createHostCapacity } from "../sidecar/host-capacity.ts"; import { @@ -86,13 +87,6 @@ export async function createWorkflowServices(options: { "/api/sidecars/ws", options.publicURL, ).href.replace(/^http/, "ws"); - const integer = (key: string, fallback: number) => { - const raw = environment[key]; - const value = raw === undefined ? fallback : Number(raw); - if (!Number.isSafeInteger(value) || value <= 0) - throw new Error(`${key} must be a positive safe integer`); - return value; - }; const adapterManifest = parseAdapterManifestEnv( environment["SIDECAR_ADAPTER_MANIFEST"], ) ?? [ @@ -121,7 +115,11 @@ export async function createWorkflowServices(options: { const credentialEncryptionKey = environment["CREDENTIAL_ENCRYPTION_KEY"]; if (!credentialEncryptionKey) throw new Error("CREDENTIAL_ENCRYPTION_KEY is required"); - const maxActiveDeployments = integer("SIDECAR_LOCAL_MAX_ACTIVE", 16); + const maxActiveDeployments = positiveInteger( + environment, + "SIDECAR_LOCAL_MAX_ACTIVE", + 16, + ); const dataRoot = localSidecarDataRoot( environment["HUB_DATA_DIR"] ?? "data/hub", ); @@ -167,12 +165,21 @@ export async function createWorkflowServices(options: { image: image.Id, adapterManifest, limits: { - memoryBytes: integer( + memoryBytes: positiveInteger( + environment, "SIDECAR_MEMORY_BYTES", defaultDockerLimits.memoryBytes, ), - pids: integer("SIDECAR_PIDS", defaultDockerLimits.pids), - nanoCpus: integer("SIDECAR_NANO_CPUS", defaultDockerLimits.nanoCpus), + pids: positiveInteger( + environment, + "SIDECAR_PIDS", + defaultDockerLimits.pids, + ), + nanoCpus: positiveInteger( + environment, + "SIDECAR_NANO_CPUS", + defaultDockerLimits.nanoCpus, + ), }, ...(environment["SIDECAR_HUB_WS_URL"] ? { hubWebSocketUrl: environment["SIDECAR_HUB_WS_URL"] } diff --git a/docs/SELF_HOST.md b/docs/SELF_HOST.md index b27f2e88..c88ecb7b 100644 --- a/docs/SELF_HOST.md +++ b/docs/SELF_HOST.md @@ -128,6 +128,9 @@ docker compose --env-file .env.hub -f compose.yml -f compose.docker-sidecars.yml when that address is not reachable from them. - Per-container limits: `SIDECAR_MEMORY_BYTES`, `SIDECAR_PIDS`, `SIDECAR_NANO_CPUS`. +- `COORDINATOR_START_TIMEOUT_MS` (default `60000`) is the longest a project's + coordinator may take to start; raise it on a slow machine, but keep it under + `300000`. ## Logs