feat(deploy): start agents through a separate provisioner service

.env.hub.example

@@ -91,19 +91,23 @@ PRODUCT_NAME=
 PRODUCT_HELLO_WORDING=
 
 # --- Sidecars ----------------------------------------------------------------
-# Agents run in sidecars. By default the hub starts them as local processes.
+# Agents run in sidecars. Docker Compose starts each in its own container through
+# its sidecar-provisioner service; `bun run dev` starts them as local processes.
+# Secret, required by Docker Compose: what the hub sends sidecar-provisioner, 32+ characters.
+SIDECAR_PROVISIONER_TOKEN=
 # Most local sidecars running at once.
 # SIDECAR_LOCAL_MAX_ACTIVE=16
 # Longest wait, in milliseconds, for a project's coordinator to start. Raise it on a slow machine; it must stay under 300000.
 # COORDINATOR_START_TIMEOUT_MS=60000
-# Optional Docker sidecars. deploy/compose.docker-sidecars.yml sets them for the hub
-# container, with the published sidecar image at BUILDER_VERSION. From source, build
-# it with `bun run docker:sidecar` and set:
+# The sidecar image. Docker Compose defaults to the published one at BUILDER_VERSION.
+# Outside Docker it also adds Docker sidecars beside local ones; build it from source
+# with `bun run docker:sidecar` and set:
 # SIDECAR_IMAGE=corbits-builder-sidecar:local
 # SIDECAR_DOCKER_HOST=unix:///var/run/docker.sock
-# The group owning the Docker socket for deploy/compose.docker-sidecars.yml: 0 on Docker Desktop.
+# The group owning the Docker socket for sidecar-provisioner: 0 on Docker Desktop.
 # DOCKER_SOCKET_GID=0
-# A stable id for this hub on the Docker daemon. Defaults to the hub's own id.
+# A stable id for this install on the Docker daemon. Defaults to the hub's own id, or
+# the Compose project name under Docker Compose.
 # SIDECAR_DOCKER_HUB_ID=
 # The hub WebSocket URL Docker sidecars dial, when the default is not reachable from them.
 # SIDECAR_HUB_WS_URL=ws://host.docker.internal:8000/api/sidecars/ws

.github/workflows/ci.yml

@@ -67,6 +67,7 @@ jobs:
             e2e:
               - *shared
               - "apps/hub/**"
+              - "apps/sidecar-provisioner/**"
               - "packages/**"
               - "workflows/**"
               - "test/e2e/**"

apps/hub/package.json

@@ -20,6 +20,7 @@
     "@corbits-builder/designer": "workspace:*",
     "@corbits-builder/reviewer": "workspace:*",
     "@corbits-builder/schemas": "workspace:*",
+    "@corbits-builder/sidecar-provisioner": "workspace:*",
     "@corbits-builder/writer": "workspace:*",
     "@corbits/agent-token": "0.2.2",
     "@corbits/artifacts": "0.3.0",

apps/hub/src/config/sidecar-provisioner.ts

@@ -0,0 +1,14 @@
+// What the person reads when the Docker install's sidecar-provisioner service cannot start an agent. It reaches
+// them as the failed run's reason, or as the hub's startup error.
+export const sidecarProvisionerWording = {
+  unreachable:
+    "Builder could not reach its sidecar-provisioner service, so the agent could not start. Start it with `docker compose up -d`, then try again.",
+  unreachableAtStart:
+    "Builder could not reach its sidecar-provisioner service, so it cannot start agents. Start it with `docker compose up -d`, then start Builder again.",
+  unauthorized:
+    "Builder's sidecar-provisioner service refused its token. Set the same SIDECAR_PROVISIONER_TOKEN for Builder and the service, then restart both.",
+  invalidRequest:
+    "Builder's sidecar-provisioner service could not read the agent's start request. Run Builder and the service at the same BUILDER_VERSION, then restart both.",
+  bindingChanged:
+    "Builder's sidecar-provisioner service now runs with another Docker setup than when Builder started. Restart Builder, then try again.",
+};

apps/hub/src/provisioning/remote.test.ts

@@ -0,0 +1,138 @@
+import { expect, test } from "bun:test";
+import type { SidecarProvisioner } from "@intx/hub-sessions";
+import {
+  BINDING_HEADER,
+  createProvisionerRoutes,
+} from "@corbits-builder/sidecar-provisioner/routes";
+import { sidecarProvisionerWording } from "../config/sidecar-provisioner.ts";
+import { createRemoteSidecarProvisioner } from "./remote.ts";
+
+const ensureRequest = {
+  allocationId: "allocation-1",
+  generation: 1,
+  tenantId: "tenant-1",
+  anchorRunId: "run-1",
+  sidecarId: "sidecar-1",
+  token: "sidecar-token",
+  hubWebSocketUrl: "ws://hub:8000/api/sidecars/ws",
+};
+const destroyRequest = {
+  allocationId: "allocation-1",
+  generation: 1,
+  sidecarId: "sidecar-1",
+};
+
+const secret = "a".repeat(32);
+const capabilities = [
+  { capability: "runtime:posix", state: "available" },
+] as const;
+
+function serve(token: string) {
+  const calls: unknown[] = [];
+  let bindingFingerprint = "docker:v1:daemon:hub";
+  const provisioner: SidecarProvisioner = {
+    id: "service",
+    apiVersion: 2,
+    get bindingFingerprint() {
+      return bindingFingerprint;
+    },
+    capabilities,
+    async ensure({ signal: _signal, ...request }) {
+      calls.push(request);
+      return { kind: "accepted", externalRef: "container-1" };
+    },
+    async destroy({ signal: _signal, ...request }) {
+      calls.push(request);
+      return { kind: "destroyed", cleanup: "confirmed" };
+    },
+  };
+  const server = Bun.serve({
+    hostname: "127.0.0.1",
+    port: 0,
+    routes: createProvisionerRoutes({ provisioner, token }),
+  });
+  return {
+    calls,
+    server,
+    rebind(fingerprint: string) {
+      bindingFingerprint = fingerprint;
+    },
+  };
+}
+
+test("the hub's provisioner takes the service's binding and forwards ensure and destroy with its secret", async () => {
+  const { calls, server, rebind } = serve(secret);
+  try {
+    const remote = await createRemoteSidecarProvisioner({
+      url: server.url,
+      token: secret,
+    });
+    expect(remote.bindingFingerprint).toBe(
+      "sidecar-provisioner:v1:docker:v1:daemon:hub",
+    );
+    expect(remote.capabilities).toEqual(capabilities);
+    expect(await remote.ensure(ensureRequest)).toEqual({
+      kind: "accepted",
+      externalRef: "container-1",
+    });
+    expect(await remote.destroy(destroyRequest)).toEqual({
+      kind: "destroyed",
+      cleanup: "confirmed",
+    });
+    expect(calls).toEqual([ensureRequest, destroyRequest]);
+
+    rebind("docker:v1:other-daemon:hub");
+    expect(await remote.ensure(ensureRequest)).toEqual({
+      kind: "rejected",
+      code: "sidecar_provisioner_binding_changed",
+      message: sidecarProvisionerWording.bindingChanged,
+      retryable: false,
+    });
+    expect(calls).toHaveLength(2);
+
+    await expect(
+      createRemoteSidecarProvisioner({
+        url: server.url,
+        token: "b".repeat(32),
+      }),
+    ).rejects.toThrow(sidecarProvisionerWording.unauthorized);
+  } finally {
+    await server.stop(true);
+  }
+});
+
+test("the service answers a body that is not JSON with 400", async () => {
+  const { server } = serve(secret);
+  try {
+    const response = await fetch(new URL("/ensure", server.url), {
+      method: "POST",
+      headers: {
+        authorization: `Bearer ${secret}`,
+        [BINDING_HEADER]: "docker:v1:daemon:hub",
+      },
+      body: "{",
+    });
+    expect(response.status).toBe(400);
+  } finally {
+    await server.stop(true);
+  }
+});
+
+test("an unreachable service fails the hub's start, then rejects placement with the person's reason and never confirms destruction", async () => {
+  const { server } = serve(secret);
+  const url = server.url;
+  const remote = await createRemoteSidecarProvisioner({ url, token: secret });
+  await server.stop(true);
+  await expect(
+    createRemoteSidecarProvisioner({ url, token: secret }),
+  ).rejects.toThrow(sidecarProvisionerWording.unreachableAtStart);
+  expect(await remote.ensure(ensureRequest)).toEqual({
+    kind: "rejected",
+    code: "sidecar_provisioner_unreachable",
+    message: sidecarProvisionerWording.unreachable,
+    retryable: true,
+  });
+  await expect(remote.destroy(destroyRequest)).rejects.toThrow(
+    sidecarProvisionerWording.unreachable,
+  );
+});

apps/hub/src/provisioning/remote.ts

@@ -0,0 +1,165 @@
+import { type } from "arktype";
+import type {
+  DestroySidecarResult,
+  EnsureSidecarResult,
+  SidecarProvisioner,
+} from "@intx/hub-sessions";
+import { SidecarCapabilityDeclaration } from "@intx/types";
+import { BINDING_HEADER } from "@corbits-builder/sidecar-provisioner/routes";
+import { sidecarProvisionerWording } from "../config/sidecar-provisioner.ts";
+
+/** Fetch failures where no connection opened, so the request cannot have taken effect. */
+const NEVER_CONNECTED = new Set([
+  "ECONNREFUSED",
+  "ENOTFOUND",
+  "EAI_AGAIN",
+  "EHOSTUNREACH",
+]);
+
+type Refusal = { readonly code: string; readonly message: string };
+
+/** Answers the service gives before it acts on a call, so nothing took effect. */
+const REFUSALS: ReadonlyMap<number, Refusal> = new Map([
+  [
+    400,
+    {
+      code: "sidecar_provisioner_invalid_request",
+      message: sidecarProvisionerWording.invalidRequest,
+    },
+  ],
+  [
+    401,
+    {
+      code: "sidecar_provisioner_unauthorized",
+      message: sidecarProvisionerWording.unauthorized,
+    },
+  ],
+  [
+    409,
+    {
+      code: "sidecar_provisioner_binding_changed",
+      message: sidecarProvisionerWording.bindingChanged,
+    },
+  ],
+]);
+
+const Binding = type({
+  bindingFingerprint: "string > 0",
+  capabilities: SidecarCapabilityDeclaration.array(),
+});
+
+function neverConnected(error: unknown): boolean {
+  return (
+    error instanceof Error &&
+    "code" in error &&
+    typeof error.code === "string" &&
+    NEVER_CONNECTED.has(error.code)
+  );
+}
+
+class RefusedError extends Error {
+  constructor(readonly refusal: Refusal) {
+    super(refusal.message);
+  }
+}
+
+type CreateRemoteSidecarProvisionerOpts = {
+  /** The sidecar-provisioner service's origin. */
+  readonly url: URL;
+  /** The secret it expects as the bearer token. */
+  readonly token: string;
+};
+
+/**
+ * Forwards ensure and destroy to the sidecar-provisioner service, which starts each sidecar in its own container, so
+ * the hub itself never holds the Docker socket. Interchange reads a provisioner's binding once, when the hub starts,
+ * so this takes the service's Docker binding first and fails the start when the service cannot answer.
+ */
+export async function createRemoteSidecarProvisioner({
+  url,
+  token,
+}: CreateRemoteSidecarProvisionerOpts): Promise<SidecarProvisioner> {
+  const authorization = `Bearer ${token}`;
+
+  async function readBinding(): Promise<typeof Binding.infer> {
+    let response: Response;
+    try {
+      response = await fetch(new URL("/binding", url), {
+        headers: { authorization },
+      });
+    } catch (error) {
+      if (!neverConnected(error)) throw error;
+      throw new Error(sidecarProvisionerWording.unreachableAtStart, {
+        cause: error,
+      });
+    }
+    if (response.status === 401)
+      throw new Error(sidecarProvisionerWording.unauthorized);
+    if (!response.ok)
+      throw new Error(
+        `Sidecar provisioner /binding failed (${String(response.status)}): ${await response.text()}`,
+      );
+    return Binding.assert(await response.json());
+  }
+
+  const binding = await readBinding();
+
+  async function post(
+    path: "/ensure" | "/destroy",
+    { signal, ...body }: { readonly signal?: AbortSignal },
+  ): Promise<unknown> {
+    const response = await fetch(new URL(path, url), {
+      method: "POST",
+      headers: {
+        authorization,
+        [BINDING_HEADER]: binding.bindingFingerprint,
+        "content-type": "application/json",
+      },
+      body: JSON.stringify(body),
+      ...(signal !== undefined && { signal }),
+    });
+    const refusal = REFUSALS.get(response.status);
+    if (refusal !== undefined) throw new RefusedError(refusal);
+    if (!response.ok)
+      throw new Error(
+        `Sidecar provisioner ${path} failed (${String(response.status)}): ${await response.text()}`,
+      );
+    return response.json();
+  }
+
+  return {
+    id: "sidecar-provisioner",
+    apiVersion: 2,
+    bindingFingerprint: `sidecar-provisioner:v1:${binding.bindingFingerprint}`,
+    capabilities: binding.capabilities,
+    async ensure(request) {
+      try {
+        // The hub validates every provisioner result at runtime, so the service's answer passes through as sent.
+        return (await post("/ensure", request)) as EnsureSidecarResult;
+      } catch (error) {
+        if (error instanceof RefusedError)
+          return { kind: "rejected", ...error.refusal, retryable: false };
+        if (!neverConnected(error)) throw error;
+        return {
+          kind: "rejected",
+          code: "sidecar_provisioner_unreachable",
+          message: sidecarProvisionerWording.unreachable,
+          retryable: true,
+        };
+      }
+    },
+    async destroy(request) {
+      try {
+        return (await post("/destroy", request)) as DestroySidecarResult;
+      } catch (error) {
+        // Retryable, so cleanup resumes once the operator fixes the setup.
+        if (error instanceof RefusedError)
+          return { kind: "rejected", ...error.refusal, retryable: true };
+        if (!neverConnected(error)) throw error;
+        throw new Error(sidecarProvisionerWording.unreachable, {
+          cause: error,
+        });
+      }
+    },
+  };
+}