diff --git a/.env.hub.example b/.env.hub.example index ae24ad41..369c419f 100644 --- a/.env.hub.example +++ b/.env.hub.example @@ -91,19 +91,23 @@ PRODUCT_NAME= PRODUCT_HELLO_WORDING= # --- Sidecars ---------------------------------------------------------------- -# Agents run in sidecars. By default the hub starts them as local processes. +# Agents run in sidecars. Docker Compose starts each in its own container through +# its sidecar-provisioner service; `bun run dev` starts them as local processes. +# Secret, required by Docker Compose: what the hub sends sidecar-provisioner, 32+ characters. +SIDECAR_PROVISIONER_TOKEN= # Most local sidecars running at once. # SIDECAR_LOCAL_MAX_ACTIVE=16 # Longest wait, in milliseconds, for a project's coordinator to start. Raise it on a slow machine; it must stay under 300000. # COORDINATOR_START_TIMEOUT_MS=60000 -# Optional Docker sidecars. deploy/compose.docker-sidecars.yml sets them for the hub -# container, with the published sidecar image at BUILDER_VERSION. From source, build -# it with `bun run docker:sidecar` and set: +# The sidecar image. Docker Compose defaults to the published one at BUILDER_VERSION. +# Outside Docker it also adds Docker sidecars beside local ones; build it from source +# with `bun run docker:sidecar` and set: # SIDECAR_IMAGE=corbits-builder-sidecar:local # SIDECAR_DOCKER_HOST=unix:///var/run/docker.sock -# The group owning the Docker socket for deploy/compose.docker-sidecars.yml: 0 on Docker Desktop. +# The group owning the Docker socket for sidecar-provisioner: 0 on Docker Desktop. # DOCKER_SOCKET_GID=0 -# A stable id for this hub on the Docker daemon. Defaults to the hub's own id. +# A stable id for this install on the Docker daemon. Defaults to the hub's own id, or +# the Compose project name under Docker Compose. # SIDECAR_DOCKER_HUB_ID= # The hub WebSocket URL Docker sidecars dial, when the default is not reachable from them. # SIDECAR_HUB_WS_URL=ws://host.docker.internal:8000/api/sidecars/ws diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44d5ff6e..140dc445 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,6 +67,7 @@ jobs: e2e: - *shared - "apps/hub/**" + - "apps/sidecar-provisioner/**" - "packages/**" - "workflows/**" - "test/e2e/**" diff --git a/apps/hub/package.json b/apps/hub/package.json index ef66c1f7..24e7ec6f 100644 --- a/apps/hub/package.json +++ b/apps/hub/package.json @@ -20,6 +20,7 @@ "@corbits-builder/designer": "workspace:*", "@corbits-builder/reviewer": "workspace:*", "@corbits-builder/schemas": "workspace:*", + "@corbits-builder/sidecar-provisioner": "workspace:*", "@corbits-builder/writer": "workspace:*", "@corbits/agent-token": "0.2.2", "@corbits/artifacts": "0.3.0", diff --git a/apps/hub/src/config/sidecar-provisioner.ts b/apps/hub/src/config/sidecar-provisioner.ts new file mode 100644 index 00000000..1e20db12 --- /dev/null +++ b/apps/hub/src/config/sidecar-provisioner.ts @@ -0,0 +1,14 @@ +// What the person reads when the Docker install's sidecar-provisioner service cannot start an agent. It reaches +// them as the failed run's reason, or as the hub's startup error. +export const sidecarProvisionerWording = { + unreachable: + "Builder could not reach its sidecar-provisioner service, so the agent could not start. Start it with `docker compose up -d`, then try again.", + unreachableAtStart: + "Builder could not reach its sidecar-provisioner service, so it cannot start agents. Start it with `docker compose up -d`, then start Builder again.", + unauthorized: + "Builder's sidecar-provisioner service refused its token. Set the same SIDECAR_PROVISIONER_TOKEN for Builder and the service, then restart both.", + invalidRequest: + "Builder's sidecar-provisioner service could not read the agent's start request. Run Builder and the service at the same BUILDER_VERSION, then restart both.", + bindingChanged: + "Builder's sidecar-provisioner service now runs with another Docker setup than when Builder started. Restart Builder, then try again.", +}; diff --git a/apps/hub/src/provisioning/remote.test.ts b/apps/hub/src/provisioning/remote.test.ts new file mode 100644 index 00000000..06db1647 --- /dev/null +++ b/apps/hub/src/provisioning/remote.test.ts @@ -0,0 +1,138 @@ +import { expect, test } from "bun:test"; +import type { SidecarProvisioner } from "@intx/hub-sessions"; +import { + BINDING_HEADER, + createProvisionerRoutes, +} from "@corbits-builder/sidecar-provisioner/routes"; +import { sidecarProvisionerWording } from "../config/sidecar-provisioner.ts"; +import { createRemoteSidecarProvisioner } from "./remote.ts"; + +const ensureRequest = { + allocationId: "allocation-1", + generation: 1, + tenantId: "tenant-1", + anchorRunId: "run-1", + sidecarId: "sidecar-1", + token: "sidecar-token", + hubWebSocketUrl: "ws://hub:8000/api/sidecars/ws", +}; +const destroyRequest = { + allocationId: "allocation-1", + generation: 1, + sidecarId: "sidecar-1", +}; + +const secret = "a".repeat(32); +const capabilities = [ + { capability: "runtime:posix", state: "available" }, +] as const; + +function serve(token: string) { + const calls: unknown[] = []; + let bindingFingerprint = "docker:v1:daemon:hub"; + const provisioner: SidecarProvisioner = { + id: "service", + apiVersion: 2, + get bindingFingerprint() { + return bindingFingerprint; + }, + capabilities, + async ensure({ signal: _signal, ...request }) { + calls.push(request); + return { kind: "accepted", externalRef: "container-1" }; + }, + async destroy({ signal: _signal, ...request }) { + calls.push(request); + return { kind: "destroyed", cleanup: "confirmed" }; + }, + }; + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + routes: createProvisionerRoutes({ provisioner, token }), + }); + return { + calls, + server, + rebind(fingerprint: string) { + bindingFingerprint = fingerprint; + }, + }; +} + +test("the hub's provisioner takes the service's binding and forwards ensure and destroy with its secret", async () => { + const { calls, server, rebind } = serve(secret); + try { + const remote = await createRemoteSidecarProvisioner({ + url: server.url, + token: secret, + }); + expect(remote.bindingFingerprint).toBe( + "sidecar-provisioner:v1:docker:v1:daemon:hub", + ); + expect(remote.capabilities).toEqual(capabilities); + expect(await remote.ensure(ensureRequest)).toEqual({ + kind: "accepted", + externalRef: "container-1", + }); + expect(await remote.destroy(destroyRequest)).toEqual({ + kind: "destroyed", + cleanup: "confirmed", + }); + expect(calls).toEqual([ensureRequest, destroyRequest]); + + rebind("docker:v1:other-daemon:hub"); + expect(await remote.ensure(ensureRequest)).toEqual({ + kind: "rejected", + code: "sidecar_provisioner_binding_changed", + message: sidecarProvisionerWording.bindingChanged, + retryable: false, + }); + expect(calls).toHaveLength(2); + + await expect( + createRemoteSidecarProvisioner({ + url: server.url, + token: "b".repeat(32), + }), + ).rejects.toThrow(sidecarProvisionerWording.unauthorized); + } finally { + await server.stop(true); + } +}); + +test("the service answers a body that is not JSON with 400", async () => { + const { server } = serve(secret); + try { + const response = await fetch(new URL("/ensure", server.url), { + method: "POST", + headers: { + authorization: `Bearer ${secret}`, + [BINDING_HEADER]: "docker:v1:daemon:hub", + }, + body: "{", + }); + expect(response.status).toBe(400); + } finally { + await server.stop(true); + } +}); + +test("an unreachable service fails the hub's start, then rejects placement with the person's reason and never confirms destruction", async () => { + const { server } = serve(secret); + const url = server.url; + const remote = await createRemoteSidecarProvisioner({ url, token: secret }); + await server.stop(true); + await expect( + createRemoteSidecarProvisioner({ url, token: secret }), + ).rejects.toThrow(sidecarProvisionerWording.unreachableAtStart); + expect(await remote.ensure(ensureRequest)).toEqual({ + kind: "rejected", + code: "sidecar_provisioner_unreachable", + message: sidecarProvisionerWording.unreachable, + retryable: true, + }); + await expect(remote.destroy(destroyRequest)).rejects.toThrow( + sidecarProvisionerWording.unreachable, + ); +}); diff --git a/apps/hub/src/provisioning/remote.ts b/apps/hub/src/provisioning/remote.ts new file mode 100644 index 00000000..c2c65c67 --- /dev/null +++ b/apps/hub/src/provisioning/remote.ts @@ -0,0 +1,165 @@ +import { type } from "arktype"; +import type { + DestroySidecarResult, + EnsureSidecarResult, + SidecarProvisioner, +} from "@intx/hub-sessions"; +import { SidecarCapabilityDeclaration } from "@intx/types"; +import { BINDING_HEADER } from "@corbits-builder/sidecar-provisioner/routes"; +import { sidecarProvisionerWording } from "../config/sidecar-provisioner.ts"; + +/** Fetch failures where no connection opened, so the request cannot have taken effect. */ +const NEVER_CONNECTED = new Set([ + "ECONNREFUSED", + "ENOTFOUND", + "EAI_AGAIN", + "EHOSTUNREACH", +]); + +type Refusal = { readonly code: string; readonly message: string }; + +/** Answers the service gives before it acts on a call, so nothing took effect. */ +const REFUSALS: ReadonlyMap = new Map([ + [ + 400, + { + code: "sidecar_provisioner_invalid_request", + message: sidecarProvisionerWording.invalidRequest, + }, + ], + [ + 401, + { + code: "sidecar_provisioner_unauthorized", + message: sidecarProvisionerWording.unauthorized, + }, + ], + [ + 409, + { + code: "sidecar_provisioner_binding_changed", + message: sidecarProvisionerWording.bindingChanged, + }, + ], +]); + +const Binding = type({ + bindingFingerprint: "string > 0", + capabilities: SidecarCapabilityDeclaration.array(), +}); + +function neverConnected(error: unknown): boolean { + return ( + error instanceof Error && + "code" in error && + typeof error.code === "string" && + NEVER_CONNECTED.has(error.code) + ); +} + +class RefusedError extends Error { + constructor(readonly refusal: Refusal) { + super(refusal.message); + } +} + +type CreateRemoteSidecarProvisionerOpts = { + /** The sidecar-provisioner service's origin. */ + readonly url: URL; + /** The secret it expects as the bearer token. */ + readonly token: string; +}; + +/** + * Forwards ensure and destroy to the sidecar-provisioner service, which starts each sidecar in its own container, so + * the hub itself never holds the Docker socket. Interchange reads a provisioner's binding once, when the hub starts, + * so this takes the service's Docker binding first and fails the start when the service cannot answer. + */ +export async function createRemoteSidecarProvisioner({ + url, + token, +}: CreateRemoteSidecarProvisionerOpts): Promise { + const authorization = `Bearer ${token}`; + + async function readBinding(): Promise { + let response: Response; + try { + response = await fetch(new URL("/binding", url), { + headers: { authorization }, + }); + } catch (error) { + if (!neverConnected(error)) throw error; + throw new Error(sidecarProvisionerWording.unreachableAtStart, { + cause: error, + }); + } + if (response.status === 401) + throw new Error(sidecarProvisionerWording.unauthorized); + if (!response.ok) + throw new Error( + `Sidecar provisioner /binding failed (${String(response.status)}): ${await response.text()}`, + ); + return Binding.assert(await response.json()); + } + + const binding = await readBinding(); + + async function post( + path: "/ensure" | "/destroy", + { signal, ...body }: { readonly signal?: AbortSignal }, + ): Promise { + const response = await fetch(new URL(path, url), { + method: "POST", + headers: { + authorization, + [BINDING_HEADER]: binding.bindingFingerprint, + "content-type": "application/json", + }, + body: JSON.stringify(body), + ...(signal !== undefined && { signal }), + }); + const refusal = REFUSALS.get(response.status); + if (refusal !== undefined) throw new RefusedError(refusal); + if (!response.ok) + throw new Error( + `Sidecar provisioner ${path} failed (${String(response.status)}): ${await response.text()}`, + ); + return response.json(); + } + + return { + id: "sidecar-provisioner", + apiVersion: 2, + bindingFingerprint: `sidecar-provisioner:v1:${binding.bindingFingerprint}`, + capabilities: binding.capabilities, + async ensure(request) { + try { + // The hub validates every provisioner result at runtime, so the service's answer passes through as sent. + return (await post("/ensure", request)) as EnsureSidecarResult; + } catch (error) { + if (error instanceof RefusedError) + return { kind: "rejected", ...error.refusal, retryable: false }; + if (!neverConnected(error)) throw error; + return { + kind: "rejected", + code: "sidecar_provisioner_unreachable", + message: sidecarProvisionerWording.unreachable, + retryable: true, + }; + } + }, + async destroy(request) { + try { + return (await post("/destroy", request)) as DestroySidecarResult; + } catch (error) { + // Retryable, so cleanup resumes once the operator fixes the setup. + if (error instanceof RefusedError) + return { kind: "rejected", ...error.refusal, retryable: true }; + if (!neverConnected(error)) throw error; + throw new Error(sidecarProvisionerWording.unreachable, { + cause: error, + }); + } + }, + }; +} diff --git a/apps/hub/src/provisioning/workflow-services.ts b/apps/hub/src/provisioning/workflow-services.ts index a90b50cb..1a0adc0c 100644 --- a/apps/hub/src/provisioning/workflow-services.ts +++ b/apps/hub/src/provisioning/workflow-services.ts @@ -1,5 +1,4 @@ import { resolve } from "node:path"; -import { type } from "arktype"; import type { SidecarProvisioner } from "@intx/hub-sessions"; import { createSidecarAllocationStore, @@ -33,12 +32,10 @@ import { type ResolvedWorkflowLifecyclePolicy, } from "@intx/types"; import { getLogger } from "@intx/log"; -import { - connectDocker, - createDockerSidecarProvisioner, - defaultDockerLimits, -} from "./docker.ts"; import { positiveInteger } from "../environment.ts"; +import { dockerProvisionerFromEnv } from "@corbits-builder/sidecar-provisioner/docker"; +import { provisionerToken } from "@corbits-builder/sidecar-provisioner/routes"; +import { createRemoteSidecarProvisioner } from "./remote.ts"; import { localSidecarDataRoot } from "../sidecar/data-root.ts"; import { createHostCapacity } from "../sidecar/host-capacity.ts"; import { @@ -55,8 +52,6 @@ import { } from "../sidecar/spawn.ts"; const log = getLogger(["corbits-builder", "workflow-services"]); -const Daemon = type({ ID: "string > 0" }); -const Image = type({ Id: "string > 0" }); const ROOT = resolve(import.meta.dir, "../../../.."); const INTERCHANGE_DIR = resolve(ROOT, "vendor/interchange"); @@ -112,88 +107,89 @@ export async function createWorkflowServices(options: { export: "createXaiResponsesAdapter", }, ]; - const credentialEncryptionKey = environment["CREDENTIAL_ENCRYPTION_KEY"]; - if (!credentialEncryptionKey) - throw new Error("CREDENTIAL_ENCRYPTION_KEY is required"); - const maxActiveDeployments = positiveInteger( - environment, - "SIDECAR_LOCAL_MAX_ACTIVE", - 16, - ); - const dataRoot = localSidecarDataRoot( - environment["HUB_DATA_DIR"] ?? "data/hub", - ); - const local = createLocalProcessSidecarProvisioner({ - dataRoot, - spawnSidecar: createSpawnLocalSidecar({ - interchangeDir: INTERCHANGE_DIR, - credentialEncryptionKey, - maxActiveDeployments, - processEnv: childProcessEnv(environment), - }), - stopStaleSidecar: stopStaleLocalSidecar, - manifests: createLocalSidecarManifestStore({ - dataRoot, - cipher: options.cipher, - log: sidecarLog, - }), - capacity: createHostCapacity({ - cgroupRoot: "/sys/fs/cgroup", - meminfo: "/proc/meminfo", - }), - adapterManifest, - log: sidecarLog, - }); - const provisioners: SidecarProvisioner[] = [local.provisioner]; - const imageName = environment["SIDECAR_IMAGE"]; - if (imageName) { - const engine = connectDocker( - environment["SIDECAR_DOCKER_HOST"] ?? "unix:///var/run/docker.sock", + async function localSidecars() { + const credentialEncryptionKey = environment["CREDENTIAL_ENCRYPTION_KEY"]; + if (!credentialEncryptionKey) + throw new Error("CREDENTIAL_ENCRYPTION_KEY is required"); + const maxActiveDeployments = positiveInteger( + environment, + "SIDECAR_LOCAL_MAX_ACTIVE", + 16, ); - const daemon = Daemon.assert(await engine.request("GET", "/info")); - const image = Image.assert( - await engine.request( - "GET", - `/images/${encodeURIComponent(imageName)}/json`, - ), + const dataRoot = localSidecarDataRoot( + environment["HUB_DATA_DIR"] ?? "data/hub", ); - provisioners.push( - createDockerSidecarProvisioner({ - engine, - hubId: environment["SIDECAR_DOCKER_HUB_ID"] ?? options.hubId, - daemonId: daemon.ID, - image: image.Id, - adapterManifest, - limits: { - memoryBytes: positiveInteger( - environment, - "SIDECAR_MEMORY_BYTES", - defaultDockerLimits.memoryBytes, - ), - pids: positiveInteger( - environment, - "SIDECAR_PIDS", - defaultDockerLimits.pids, - ), - nanoCpus: positiveInteger( - environment, - "SIDECAR_NANO_CPUS", - defaultDockerLimits.nanoCpus, - ), - }, - ...(environment["SIDECAR_HUB_WS_URL"] - ? { hubWebSocketUrl: environment["SIDECAR_HUB_WS_URL"] } - : {}), + const local = createLocalProcessSidecarProvisioner({ + dataRoot, + spawnSidecar: createSpawnLocalSidecar({ + interchangeDir: INTERCHANGE_DIR, + credentialEncryptionKey, + maxActiveDeployments, + processEnv: childProcessEnv(environment), }), - ); + stopStaleSidecar: stopStaleLocalSidecar, + manifests: createLocalSidecarManifestStore({ + dataRoot, + cipher: options.cipher, + log: sidecarLog, + }), + capacity: createHostCapacity({ + cgroupRoot: "/sys/fs/cgroup", + meminfo: "/proc/meminfo", + }), + adapterManifest, + log: sidecarLog, + }); + const provisioners: SidecarProvisioner[] = [local.provisioner]; + if (environment["SIDECAR_IMAGE"]) + provisioners.push( + await dockerProvisionerFromEnv(environment, { + hubId: environment["SIDECAR_DOCKER_HUB_ID"] ?? options.hubId, + }), + ); + return { + local, + // The sidecar resolves its data dir from its own working directory. + dataDir: resolve(INTERCHANGE_DIR, localSidecarDataDir(dataRoot)), + provisioners, + chooser: createPlacementChooser({ + db, + local: local.provisioner, + maxDeployments: maxActiveDeployments, + }), + }; } + + // The Docker install starts every agent through its sidecar-provisioner service and none in the hub's container. + async function remoteSidecars(url: string) { + if (environment["SIDECAR_IMAGE"]) + throw new Error( + "SIDECAR_PROVISIONER_URL and SIDECAR_IMAGE cannot both be set", + ); + return { + local: undefined, + dataDir: undefined, + provisioners: [ + await createRemoteSidecarProvisioner({ + url: new URL(url), + token: provisionerToken(environment["SIDECAR_PROVISIONER_TOKEN"]), + }), + ], + chooser: undefined, + }; + } + + const provisionerUrl = environment["SIDECAR_PROVISIONER_URL"]; + const sidecars = provisionerUrl + ? await remoteSidecars(provisionerUrl) + : await localSidecars(); + const { local, provisioners, chooser } = sidecars; + log.info("Sidecar provisioners: {ids}", { + ids: provisioners.map((provisioner) => provisioner.id), + }); const plugins = createSidecarPluginRegistry({ provisioners, - chooser: createPlacementChooser({ - db, - local: local.provisioner, - maxDeployments: maxActiveDeployments, - }), + ...(chooser !== undefined && { chooser }), }); const duration = (key: string, fallback: string) => LifecycleDuration.assert(environment[key] ?? fallback); @@ -283,7 +279,8 @@ export async function createWorkflowServices(options: { retireProvisionerSidecar: allocationStore.retireProvisionerSidecar, isConnected: router.holdsAllocatedBinding, }); - await restoreLocalSidecars(db, local, adapterManifest); + if (local !== undefined) + await restoreLocalSidecars(db, local, adapterManifest); const pending = new Set>(); const track = (run: () => Promise): Promise => { const task = Promise.resolve().then(run); @@ -373,11 +370,7 @@ export async function createWorkflowServices(options: { workflowLifecycleService: lifecycle, }, configured: true as const, - // The sidecar resolves its data dir from its own working directory. - localSidecarDataDir: resolve( - INTERCHANGE_DIR, - localSidecarDataDir(dataRoot), - ), + localSidecarDataDir: sidecars.dataDir, start() { for (const scheduler of schedulers) scheduler.start(); }, @@ -387,7 +380,7 @@ export async function createWorkflowServices(options: { }, async drain() { await Promise.allSettled(pending); - await local.shutdown(); + await local?.shutdown(); }, }; } diff --git a/apps/hub/src/server.ts b/apps/hub/src/server.ts index 3e6e60b4..7d817988 100644 --- a/apps/hub/src/server.ts +++ b/apps/hub/src/server.ts @@ -15,9 +15,7 @@ if (import.meta.main) { websocket: runtime.websocket, }); await runtime.start(); - log.info( - "Hub ready: Postgres connected; local Interchange sidecar provisioner", - ); + log.info("Hub ready: Postgres connected"); let stopping = false; const stop = async () => { if (stopping) return; diff --git a/apps/sidecar-provisioner/package.json b/apps/sidecar-provisioner/package.json new file mode 100644 index 00000000..0eb1bb35 --- /dev/null +++ b/apps/sidecar-provisioner/package.json @@ -0,0 +1,18 @@ +{ + "name": "@corbits-builder/sidecar-provisioner", + "version": "0.1.0-rc.2", + "private": true, + "license": "SEE LICENSE IN ../../LICENSE.md", + "author": "Sawyer Cutler ", + "type": "module", + "exports": { + "./docker": "./src/docker.ts", + "./routes": "./src/routes.ts" + }, + "dependencies": { + "@intx/hub-sessions": "workspace:*", + "@intx/types": "workspace:*", + "@logtape/logtape": "2.4.2", + "arktype": "catalog:" + } +} diff --git a/apps/hub/src/provisioning/docker-engine.test.ts b/apps/sidecar-provisioner/src/docker-engine.test.ts similarity index 100% rename from apps/hub/src/provisioning/docker-engine.test.ts rename to apps/sidecar-provisioner/src/docker-engine.test.ts diff --git a/apps/hub/src/provisioning/docker-engine.ts b/apps/sidecar-provisioner/src/docker-engine.ts similarity index 100% rename from apps/hub/src/provisioning/docker-engine.ts rename to apps/sidecar-provisioner/src/docker-engine.ts diff --git a/apps/hub/src/provisioning/docker.ts b/apps/sidecar-provisioner/src/docker.ts similarity index 81% rename from apps/hub/src/provisioning/docker.ts rename to apps/sidecar-provisioner/src/docker.ts index ded45750..b4e14338 100644 --- a/apps/hub/src/provisioning/docker.ts +++ b/apps/sidecar-provisioner/src/docker.ts @@ -8,16 +8,14 @@ import type { EnsureSidecarResult, SidecarProvisioner, } from "@intx/hub-sessions"; -import type { AdapterManifest } from "@intx/types"; -import { settle } from "../lib/serial-queue.ts"; +import type { SidecarCapabilityDeclaration } from "@intx/types"; import { + connectDocker, DockerEngineError, isThisMachine, type DockerEngine, } from "./docker-engine.ts"; -export { connectDocker } from "./docker-engine.ts"; - const LABEL = { hub: "corbits-builder.hub", allocation: "corbits-builder.allocation", @@ -31,11 +29,16 @@ type DockerLimits = { readonly pids: number; readonly nanoCpus: number; }; -export const defaultDockerLimits: DockerLimits = { +const defaultDockerLimits: DockerLimits = { memoryBytes: 2048 * MiB, pids: 1024, nanoCpus: 2_000_000_000, }; +const dockerSidecarCapabilities: readonly SidecarCapabilityDeclaration[] = [ + { capability: "isolation:workload", state: "blocked" }, + { capability: "network:outbound", state: "available" }, + { capability: "runtime:posix", state: "available" }, +]; const Container = type({ Id: "string", @@ -50,6 +53,8 @@ type Container = typeof Container.infer; const Listed = type("unknown[]"); const Created = type({ Id: "string" }); const Info = type({ MemTotal: "number" }); +const Daemon = type({ ID: "string > 0" }); +const Image = type({ Id: "string > 0" }); type CreateDockerSidecarProvisionerOpts = { readonly engine: DockerEngine; @@ -60,8 +65,6 @@ type CreateDockerSidecarProvisionerOpts = { readonly daemonId: string; /** Runs the stock sidecar as its command. */ readonly image: string; - /** The hub's adapter manifest at the paths the image holds its modules. */ - readonly adapterManifest: AdapterManifest; /** Where containers reach the hub, for a daemon on another machine; unset, the hub's own address with loopback mapped to the Docker host. */ readonly hubWebSocketUrl?: string; }; @@ -133,7 +136,6 @@ export function createDockerSidecarProvisioner({ hubId, daemonId, image, - adapterManifest, hubWebSocketUrl, limits = defaultDockerLimits, }: CreateDockerSidecarProvisionerOpts): SidecarProvisioner { @@ -179,9 +181,12 @@ export function createDockerSidecarProvisioner({ SIDECAR_TOKEN: request.token, SIDECAR_DATA_DIR: DATA_DIR, SIDECAR_CREDENTIAL_ENCRYPTION_KEY: randomBytes(32).toString("hex"), - SIDECAR_ADAPTER_MANIFEST: JSON.stringify(adapterManifest), HOME: DATA_DIR, TMPDIR: "/tmp", + // The hub's manifest names modules at the paths the image holds them. + ...(request.adapterManifest !== undefined && { + SIDECAR_ADAPTER_MANIFEST: JSON.stringify(request.adapterManifest), + }), }; const body = { Image: image, @@ -229,6 +234,11 @@ export function createDockerSidecarProvisioner({ return Id; } + // The caller observes the failure; the queue only needs ordering. + async function settle(operation: Promise | undefined) { + await Promise.allSettled([operation]); + } + async function runAfter( previous: Promise | undefined, run: () => Promise, @@ -347,11 +357,7 @@ export function createDockerSidecarProvisioner({ apiVersion: 2, // The image is a launch setting: containers are found by label whatever image they run. bindingFingerprint: `docker:v1:${daemonId}:${hubId}:${limits.memoryBytes}:${limits.pids}:${limits.nanoCpus}`, - capabilities: [ - { capability: "isolation:workload", state: "blocked" }, - { capability: "network:outbound", state: "available" }, - { capability: "runtime:posix", state: "available" }, - ], + capabilities: dockerSidecarCapabilities, ensure(request) { return serialize(request.allocationId, () => ensureSidecar(request)); }, @@ -360,3 +366,48 @@ export function createDockerSidecarProvisioner({ }, }; } + +/** + * The Docker provisioner configured by `SIDECAR_IMAGE`, `SIDECAR_DOCKER_HOST`, `SIDECAR_HUB_WS_URL` and the + * `SIDECAR_MEMORY_BYTES`, `SIDECAR_PIDS` and `SIDECAR_NANO_CPUS` limits, bound to the daemon it reaches. + */ +export async function dockerProvisionerFromEnv( + environment: Record, + { hubId }: { readonly hubId: string }, +): Promise { + function limit(key: string, fallback: number): number { + const raw = environment[key]; + const value = raw === undefined ? fallback : Number(raw); + if (!Number.isSafeInteger(value) || value <= 0) + throw new Error(`${key} must be a positive safe integer`); + return value; + } + const imageName = environment["SIDECAR_IMAGE"]; + if (!imageName) throw new Error("SIDECAR_IMAGE is required"); + const engine = connectDocker( + environment["SIDECAR_DOCKER_HOST"] ?? "unix:///var/run/docker.sock", + ); + const daemon = Daemon.assert(await engine.request("GET", "/info")); + const image = Image.assert( + await engine.request( + "GET", + `/images/${encodeURIComponent(imageName)}/json`, + ), + ); + const hubWebSocketUrl = environment["SIDECAR_HUB_WS_URL"]; + return createDockerSidecarProvisioner({ + engine, + hubId, + daemonId: daemon.ID, + image: image.Id, + limits: { + memoryBytes: limit( + "SIDECAR_MEMORY_BYTES", + defaultDockerLimits.memoryBytes, + ), + pids: limit("SIDECAR_PIDS", defaultDockerLimits.pids), + nanoCpus: limit("SIDECAR_NANO_CPUS", defaultDockerLimits.nanoCpus), + }, + ...(hubWebSocketUrl ? { hubWebSocketUrl } : {}), + }); +} diff --git a/apps/sidecar-provisioner/src/routes.ts b/apps/sidecar-provisioner/src/routes.ts new file mode 100644 index 00000000..fe2a3b1c --- /dev/null +++ b/apps/sidecar-provisioner/src/routes.ts @@ -0,0 +1,114 @@ +import { createHash, timingSafeEqual } from "node:crypto"; +import { type } from "arktype"; +import type { SidecarProvisioner } from "@intx/hub-sessions"; +import { AdapterManifest } from "@intx/types"; + +const MIN_TOKEN_LENGTH = 32; +/** Carries the binding the hub registered, so a service restarted with another Docker setup refuses its calls. */ +export const BINDING_HEADER = "x-sidecar-binding"; + +const Identity = { + allocationId: "string > 0", + generation: "number.integer >= 0", + sidecarId: "string > 0", +} as const; +const EnsureBody = type({ + ...Identity, + tenantId: "string > 0", + anchorRunId: "string > 0", + token: "string > 0", + hubWebSocketUrl: "string > 0", + "adapterManifest?": AdapterManifest, +}); +const DestroyBody = type({ ...Identity, "externalRef?": "string" }); + +const digest = (value: string) => createHash("sha256").update(value).digest(); + +/** The secret the hub and the service share, checked wherever either reads it. */ +export function provisionerToken(token: string | undefined): string { + if (!token) throw new Error("SIDECAR_PROVISIONER_TOKEN is required"); + if (token.length < MIN_TOKEN_LENGTH) + throw new Error( + `SIDECAR_PROVISIONER_TOKEN must be at least ${String(MIN_TOKEN_LENGTH)} characters; generate one with \`openssl rand -hex 32\``, + ); + return token; +} + +async function readBody( + request: Request, + schema: (data: unknown) => T | type.errors, +): Promise { + let data: unknown; + try { + data = await request.json(); + } catch (error) { + if (!(error instanceof SyntaxError)) throw error; + return Response.json({ error: "Body is not JSON" }, { status: 400 }); + } + const body = schema(data); + if (body instanceof type.errors) + return Response.json({ error: body.summary }, { status: 400 }); + return body; +} + +/** + * The hub's provisioner calls over HTTP. `GET /binding` answers the provisioner's binding fingerprint and + * capabilities; `POST /ensure` and `POST /destroy` take the Interchange request without its signal and answer its + * result. A dropped hub connection aborts the call, as the hub's signal would. + */ +export function createProvisionerRoutes({ + provisioner, + token, +}: { + readonly provisioner: SidecarProvisioner; + /** The secret the hub sends as its bearer token. */ + readonly token: string; +}) { + const expected = digest(`Bearer ${provisionerToken(token)}`); + function authorized(request: Request): boolean { + const authorization = request.headers.get("authorization"); + return ( + authorization !== null && timingSafeEqual(digest(authorization), expected) + ); + } + function refusal(request: Request): Response | undefined { + if (!authorized(request)) return new Response(null, { status: 401 }); + if (request.headers.get(BINDING_HEADER) !== provisioner.bindingFingerprint) + return new Response(null, { status: 409 }); + return undefined; + } + + function binding(request: Request): Response { + if (!authorized(request)) return new Response(null, { status: 401 }); + return Response.json({ + bindingFingerprint: provisioner.bindingFingerprint, + capabilities: provisioner.capabilities, + }); + } + + async function ensure(request: Request): Promise { + const refused = refusal(request); + if (refused) return refused; + const body = await readBody(request, EnsureBody); + if (body instanceof Response) return body; + return Response.json( + await provisioner.ensure({ ...body, signal: request.signal }), + ); + } + + async function destroy(request: Request): Promise { + const refused = refusal(request); + if (refused) return refused; + const body = await readBody(request, DestroyBody); + if (body instanceof Response) return body; + return Response.json( + await provisioner.destroy({ ...body, signal: request.signal }), + ); + } + + return { + "/binding": { GET: binding }, + "/ensure": { POST: ensure }, + "/destroy": { POST: destroy }, + }; +} diff --git a/apps/sidecar-provisioner/src/server.ts b/apps/sidecar-provisioner/src/server.ts new file mode 100644 index 00000000..dfc4b454 --- /dev/null +++ b/apps/sidecar-provisioner/src/server.ts @@ -0,0 +1,47 @@ +import { + configure, + getConsoleSink, + getJsonLinesFormatter, + getLogger, +} from "@logtape/logtape"; +import { dockerProvisionerFromEnv } from "./docker.ts"; +import { createProvisionerRoutes, provisionerToken } from "./routes.ts"; + +await configure({ + sinks: { console: getConsoleSink({ formatter: getJsonLinesFormatter() }) }, + loggers: [ + { category: [], lowestLevel: "info", sinks: ["console"] }, + { + category: ["logtape", "meta"], + lowestLevel: "warning", + sinks: ["console"], + }, + ], +}); +const log = getLogger(["corbits-builder", "sidecar-provisioner"]); + +function required(key: string): string { + const value = process.env[key]; + if (!value) throw new Error(`${key} is required`); + return value; +} + +const token = provisionerToken(process.env["SIDECAR_PROVISIONER_TOKEN"]); +const provisioner = await dockerProvisionerFromEnv(process.env, { + hubId: required("SIDECAR_DOCKER_HUB_ID"), +}); + +const server = Bun.serve({ + hostname: required("HOST"), + port: Number(required("PORT")), + routes: createProvisionerRoutes({ provisioner, token }), + fetch: () => new Response(null, { status: 404 }), + // Docker errors leave out response bodies, so their messages are safe to log and return. + error(error) { + log.error("Sidecar provisioner request failed: {message}", { + message: error.message, + }); + return Response.json({ error: error.message }, { status: 500 }); + }, +}); +log.info("Sidecar provisioner listening on {url}", { url: server.url.href }); diff --git a/bun.lock b/bun.lock index 0d77bca2..c43c2f13 100644 --- a/bun.lock +++ b/bun.lock @@ -67,6 +67,7 @@ "@corbits-builder/designer": "workspace:*", "@corbits-builder/reviewer": "workspace:*", "@corbits-builder/schemas": "workspace:*", + "@corbits-builder/sidecar-provisioner": "workspace:*", "@corbits-builder/writer": "workspace:*", "@corbits/agent-token": "0.2.2", "@corbits/artifacts": "0.3.0", @@ -112,6 +113,16 @@ "@types/nodemailer": "8.0.2", }, }, + "apps/sidecar-provisioner": { + "name": "@corbits-builder/sidecar-provisioner", + "version": "0.1.0-rc.2", + "dependencies": { + "@intx/hub-sessions": "workspace:*", + "@intx/types": "workspace:*", + "@logtape/logtape": "2.4.2", + "arktype": "catalog:", + }, + }, "apps/web": { "name": "@corbits-builder/web", "version": "0.1.0-rc.3", @@ -732,6 +743,8 @@ "@corbits-builder/schemas": ["@corbits-builder/schemas@workspace:packages/schemas"], + "@corbits-builder/sidecar-provisioner": ["@corbits-builder/sidecar-provisioner@workspace:apps/sidecar-provisioner"], + "@corbits-builder/web": ["@corbits-builder/web@workspace:apps/web"], "@corbits-builder/writer": ["@corbits-builder/writer@workspace:workflows/writer"], diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 1cd736a4..e90d0bff 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -59,12 +59,13 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* # Docker sidecars load the hub's inference adapters at the paths the hub resolves them to, -# so this image keeps the hub's /app layout. +# so this image keeps the hub's /app layout. The sidecar-provisioner service runs from it too. FROM runtime AS sidecar WORKDIR /app COPY --from=deps --chown=bun:bun /app/package.json ./ COPY --from=deps --chown=bun:bun /app/node_modules ./node_modules COPY --from=deps --chown=bun:bun /app/apps/hub ./apps/hub +COPY --from=deps --chown=bun:bun /app/apps/sidecar-provisioner ./apps/sidecar-provisioner COPY --from=deps --chown=bun:bun /app/packages ./packages COPY --from=deps --chown=bun:bun /app/vendor/interchange ./vendor/interchange RUN mkdir -p /data && chown bun:bun /data @@ -78,6 +79,7 @@ WORKDIR /app COPY --from=deps --chown=bun:bun /app/package.json /app/bun.lock /app/tsconfig.json ./ COPY --from=deps --chown=bun:bun /app/node_modules ./node_modules COPY --from=deps --chown=bun:bun /app/apps/hub ./apps/hub +COPY --from=deps --chown=bun:bun /app/apps/sidecar-provisioner ./apps/sidecar-provisioner COPY --from=web --chown=bun:bun /app/apps/web/dist ./apps/web/dist COPY --from=deps --chown=bun:bun /app/packages ./packages COPY --from=deps --chown=bun:bun /app/workflows ./workflows diff --git a/deploy/compose.build.yml b/deploy/compose.build.yml index 7bc5931f..59b6fcfc 100644 --- a/deploy/compose.build.yml +++ b/deploy/compose.build.yml @@ -1,4 +1,4 @@ -# Builds the hub from this checkout instead of pulling the published image: +# Builds the hub and sidecar images from this checkout instead of pulling the published ones: # docker compose --env-file .env.hub --project-directory . -f deploy/compose.yml -f deploy/compose.build.yml up -d --build x-build: &build image: corbits-builder-hub:local @@ -10,3 +10,11 @@ x-build: &build services: hub: *build setup: *build + sidecar-provisioner: + image: corbits-builder-sidecar:local + build: + context: . + dockerfile: deploy/Dockerfile + target: sidecar + environment: + SIDECAR_IMAGE: corbits-builder-sidecar:local diff --git a/deploy/compose.docker-sidecars.yml b/deploy/compose.docker-sidecars.yml deleted file mode 100644 index 3cf42871..00000000 --- a/deploy/compose.docker-sidecars.yml +++ /dev/null @@ -1,25 +0,0 @@ -# Lets the hub also start sidecars as containers on the host's Docker daemon: -# docker compose --env-file .env.hub --project-directory . -f deploy/compose.yml -f deploy/compose.docker-sidecars.yml up -d -# `sidecar-image` pulls SIDECAR_IMAGE onto that daemon before the hub starts; -# `docker compose pull` refreshes it after a channel change. -# From source, `bun run docker:sidecar` builds corbits-builder-sidecar:local; -# set SIDECAR_IMAGE to that. -# Access to the socket is root on the host. -services: - sidecar-image: - image: ${SIDECAR_IMAGE:-ghcr.io/corbitsdev/corbits-builder-sidecar:${BUILDER_VERSION:-rc}} - entrypoint: ["true"] - restart: "no" - hub: - depends_on: - sidecar-image: - condition: service_completed_successfully - environment: - SIDECAR_IMAGE: ${SIDECAR_IMAGE:-ghcr.io/corbitsdev/corbits-builder-sidecar:${BUILDER_VERSION:-rc}} - SIDECAR_DOCKER_HOST: unix:///var/run/docker.sock - # The group that owns the socket inside the container: 0 on Docker Desktop, - # the host's docker group id (`getent group docker`) on Linux. - group_add: - - "${DOCKER_SOCKET_GID:-0}" - volumes: - - /var/run/docker.sock:/var/run/docker.sock diff --git a/deploy/compose.yml b/deploy/compose.yml index 183e79fb..92bd342f 100644 --- a/deploy/compose.yml +++ b/deploy/compose.yml @@ -4,6 +4,7 @@ # this file: # docker compose --env-file .env.hub --project-directory . -f deploy/compose.yml up -d # deploy/compose.build.yml builds the images from source instead. +x-sidecar-image: &sidecar-image ${SIDECAR_IMAGE:-ghcr.io/corbitsdev/corbits-builder-sidecar:${BUILDER_VERSION:-rc}} x-hub: &hub image: ghcr.io/corbitsdev/corbits-builder-hub:${BUILDER_VERSION:-rc} env_file: .env.hub @@ -18,9 +19,15 @@ x-hub: &hub MIGRATION_DATABASE_URL: "" HUB_WEB_DIR: /app/apps/web/dist BETTER_AUTH_BASE_URL: ${BETTER_AUTH_BASE_URL:-http://localhost:${PORT:-8000}} + # Agents run in containers sidecar-provisioner starts, never in the hub's. + SIDECAR_PROVISIONER_URL: http://sidecar-provisioner:8080 + SIDECAR_IMAGE: "" depends_on: db: condition: service_healthy + # The hub takes the provisioner's Docker binding when it starts. + sidecar-provisioner: + condition: service_healthy volumes: - hub-data:/app/data/hub @@ -57,6 +64,41 @@ services: - "127.0.0.1:${PORT:-8000}:${PORT:-8000}" - "127.0.0.1:1455:1455" - "127.0.0.1:1456:1456" + # The only service with the Docker socket, which is root on the host. It starts one hardened + # container per agent from the sidecar image, which it runs from itself. + sidecar-provisioner: + image: *sidecar-image + working_dir: /app + command: + [ + "bun", + "--no-env-file", + "--conditions=intx-src", + "apps/sidecar-provisioner/src/server.ts", + ] + environment: + HOST: 0.0.0.0 + PORT: 8080 + SIDECAR_PROVISIONER_TOKEN: ${SIDECAR_PROVISIONER_TOKEN:?Set SIDECAR_PROVISIONER_TOKEN} + SIDECAR_IMAGE: *sidecar-image + SIDECAR_DOCKER_HOST: unix:///var/run/docker.sock + SIDECAR_DOCKER_HUB_ID: ${SIDECAR_DOCKER_HUB_ID:-${COMPOSE_PROJECT_NAME}} + SIDECAR_HUB_WS_URL: + SIDECAR_MEMORY_BYTES: + SIDECAR_PIDS: + SIDECAR_NANO_CPUS: + # It listens only once it has reached Docker and found the sidecar image; any answer is healthy. + healthcheck: + test: ["CMD", "bun", "-e", "await fetch('http://127.0.0.1:8080/')"] + interval: 2s + timeout: 3s + retries: 30 + # The group that owns the socket inside the container: 0 on Docker Desktop, + # the host's docker group id (`getent group docker`) on Linux. + group_add: + - "${DOCKER_SOCKET_GID:-0}" + volumes: + - /var/run/docker.sock:/var/run/docker.sock setup: <<: *hub profiles: ["setup"] diff --git a/docs/DEV.md b/docs/DEV.md index 44c7bc5f..6d459f05 100644 --- a/docs/DEV.md +++ b/docs/DEV.md @@ -49,7 +49,9 @@ There is no default profile. When a project's coordinator starts, it runs on: The resolved profile is fixed for the life of the project. Agents run in sidecar processes the hub starts on this machine; their state -lives under `HUB_DATA_DIR` (`data/hub`). Docker sidecars are optional +lives under `HUB_DATA_DIR` (`data/hub`). Docker sidecars beside them are +optional (`SIDECAR_IMAGE` in `.env.hub.example`). The Docker install runs every +agent in its own container instead ([SELF_HOST.md](https://github.com/corbitsdev/corbits-builder/blob/main/docs/SELF_HOST.md#sidecars-in-docker)). ## Tests @@ -104,7 +106,7 @@ real API ([CONTRIBUTING.md](https://github.com/corbitsdev/corbits-builder/blob/m - Hub routes: mounted in `apps/hub/src/app.ts`; each project's routes in `apps/hub/src/projects/routes.ts`. - Starting and placing sidecars: `apps/hub/src/provisioning/`, - `apps/hub/src/sidecar/`. + `apps/hub/src/sidecar/`, `apps/sidecar-provisioner/`. - Each role's prompt: `workflows//src/`. Its `defineTool` tools: `apps/hub/src/` (`artifacts/`, `projects/`, `builds/`, `skills/`). - Interchange docs: `vendor/interchange/docs/`. diff --git a/docs/IMPLEMENTATION.md b/docs/IMPLEMENTATION.md index fe5c84d7..0168f438 100644 --- a/docs/IMPLEMENTATION.md +++ b/docs/IMPLEMENTATION.md @@ -42,17 +42,30 @@ so the vendored packages load from source. Agents run on sidecars, which the hub provisions ([apps/hub/src/provisioning/workflow-services.ts](https://github.com/corbitsdev/corbits-builder/blob/main/apps/hub/src/provisioning/workflow-services.ts)): -- **Local process sidecars** are always available. The hub spawns them from +- **Local process sidecars** run outside Docker. The hub spawns them from the vendored sidecar app on the same machine, keeps their state under `HUB_DATA_DIR`, and restores them after a restart. One holds up to `SIDECAR_LOCAL_MAX_ACTIVE` deployments. -- **Docker sidecars** are added when `SIDECAR_IMAGE` names an image (built - from the `sidecar` target of +- **Docker sidecars** are added beside them when `SIDECAR_IMAGE` names an + image (built from the `sidecar` target of [deploy/Dockerfile](https://github.com/corbitsdev/corbits-builder/blob/main/deploy/Dockerfile)), - with memory, process and CPU limits from `SIDECAR_*`. - -Placement uses the local sidecar until it is full, then the first matching -sandbox + with memory, process and CPU limits from `SIDECAR_*` + ([apps/sidecar-provisioner/src/docker.ts](https://github.com/corbitsdev/corbits-builder/blob/main/apps/sidecar-provisioner/src/docker.ts)). +- **In the Docker install**, `SIDECAR_PROVISIONER_URL` replaces both: the hub + forwards every ensure and destroy to the `sidecar-provisioner` service + ([apps/hub/src/provisioning/remote.ts](https://github.com/corbitsdev/corbits-builder/blob/main/apps/hub/src/provisioning/remote.ts)), + which runs the same Docker provisioner and is the only container with the + Docker socket. Interchange reads a provisioner's binding once, so the hub + takes the service's Docker binding fingerprint and capabilities from + `GET /binding` when it starts and does not start without them; Compose + starts the hub once the service is healthy. A service restarted with + another Docker setup refuses the hub's calls until the hub restarts. When + the service cannot be reached later, placement retries under + Interchange's provisioning budget and then fails the run with the reason + from `apps/hub/src/config/sidecar-provisioner.ts`. + +Outside Docker, placement uses the local sidecar until it is full, then the +first matching sandbox ([apps/hub/src/sidecar/placement.ts](https://github.com/corbitsdev/corbits-builder/blob/main/apps/hub/src/sidecar/placement.ts)). Default lifetimes and retention come from `WORKFLOW_DEFAULT_*`. @@ -115,22 +128,23 @@ Environment files, installs and build output never leave the workspace. ## Package layout -| Path | Package | What it holds | -| ------------------------- | ------------------------------ | ---------------------------------------------------------------- | -| `apps/hub` | `@corbits-builder/hub` | The server: routes, services, hub tools, provisioning, live app. | -| `packages/client` | `@corbits-builder/client` | The typed client for the hub's API. | -| `apps/web` | `@corbits-builder/web` | The interface, built only on the client and the schemas. | -| `workflows/coordinator` | `@corbits-builder/coordinator` | The coordinator's agent, prompt and workflow. | -| `workflows/writer` | `@corbits-builder/writer` | The document writer and its per-kind skills. | -| `workflows/reviewer` | `@corbits-builder/reviewer` | The document reviewer. | -| `workflows/designer` | `@corbits-builder/designer` | The designer, its starter, themes and skills. | -| `workflows/coder` | `@corbits-builder/coder` | The coder, its app starter and `check-app`. | -| `packages/schemas` | `@corbits-builder/schemas` | Types and schemas shared by the hub, client and workflows. | -| `vendor/interchange` | | Interchange at the pinned revision. | -| `deploy/` | | The Dockerfile and Compose files. | -| `scripts/` | | Dev and verification scripts. | -| `skills/` | | Agent skills to install Builder and drive it through its API. | -| `test/integration`, `e2e` | | Cross-module and full-stack tests. | +| Path | Package | What it holds | +| -------------------------- | -------------------------------------- | ---------------------------------------------------------------- | +| `apps/hub` | `@corbits-builder/hub` | The server: routes, services, hub tools, provisioning, live app. | +| `packages/client` | `@corbits-builder/client` | The typed client for the hub's API. | +| `apps/web` | `@corbits-builder/web` | The interface, built only on the client and the schemas. | +| `apps/sidecar-provisioner` | `@corbits-builder/sidecar-provisioner` | The Docker install's service that starts sidecar containers. | +| `workflows/coordinator` | `@corbits-builder/coordinator` | The coordinator's agent, prompt and workflow. | +| `workflows/writer` | `@corbits-builder/writer` | The document writer and its per-kind skills. | +| `workflows/reviewer` | `@corbits-builder/reviewer` | The document reviewer. | +| `workflows/designer` | `@corbits-builder/designer` | The designer, its starter, themes and skills. | +| `workflows/coder` | `@corbits-builder/coder` | The coder, its app starter and `check-app`. | +| `packages/schemas` | `@corbits-builder/schemas` | Types and schemas shared by the hub, client and workflows. | +| `vendor/interchange` | | Interchange at the pinned revision. | +| `deploy/` | | The Dockerfile and Compose files. | +| `scripts/` | | Dev and verification scripts. | +| `skills/` | | Agent skills to install Builder and drive it through its API. | +| `test/integration`, `e2e` | | Cross-module and full-stack tests. | Workflow packages declare their agents and workflows with Interchange's definition exports (`@intx/agent/authoring`, `@intx/workflow/definition`). diff --git a/docs/SELF_HOST.md b/docs/SELF_HOST.md index 569a2b82..28402b1a 100644 --- a/docs/SELF_HOST.md +++ b/docs/SELF_HOST.md @@ -125,32 +125,38 @@ or `bwrap: setting up uid map: Permission denied`. ## Sidecars in Docker -Agents run in sidecars. By default the hub starts them as processes beside -itself, inside the hub container. With `SIDECAR_IMAGE` set, it can also -start them as containers on the Docker daemon at `SIDECAR_DOCKER_HOST`. - -```sh -curl -fsSLO https://raw.githubusercontent.com/corbitsdev/corbits-builder/main/deploy/compose.docker-sidecars.yml -docker compose --env-file .env.hub -f compose.yml -f compose.docker-sidecars.yml up -d -``` - -- Compose pulls the sidecar image for your `BUILDER_VERSION` before the hub - starts, and `docker compose pull` refreshes it with the hub. From source, `bun run docker:sidecar` builds - `corbits-builder-sidecar:local`; set `SIDECAR_IMAGE` to it in `.env.hub`. -- `deploy/compose.docker-sidecars.yml` mounts `/var/run/docker.sock` into the hub and - adds the socket's group (`DOCKER_SOCKET_GID`: `0` on Docker Desktop, the +Agents run in sidecars, each in its own container. The `sidecar-provisioner` +service starts them on the host's Docker daemon when the hub asks over the +Compose network, authenticated with `SIDECAR_PROVISIONER_TOKEN` (at least 32 +characters). It is the only service that mounts `/var/run/docker.sock`; the +hub has no access to Docker and runs no agents itself. The hub starts once +`sidecar-provisioner` is healthy. If it stops later, agents do not start, and +the project shows why. + +- `sidecar-provisioner` runs from the sidecar image for your + `BUILDER_VERSION`, so `docker compose pull` refreshes both with the hub. + `deploy/compose.build.yml` builds it from source as + `corbits-builder-sidecar:local`. +- It adds the socket's group (`DOCKER_SOCKET_GID`: `0` on Docker Desktop, the host's `docker` group id on Linux). Access to that socket is root on the host. +- Each container has a read-only root, drops every capability, cannot gain + privileges, and gets its own data volume and the per-container limits + `SIDECAR_MEMORY_BYTES`, `SIDECAR_PIDS` and `SIDECAR_NANO_CPUS`. - Sidecar containers are not on the Compose network. They reach the hub - through `HUB_API_ORIGIN` and its WebSocket; on Docker Desktop the hub maps - a loopback origin to `host.docker.internal`. Set `SIDECAR_HUB_WS_URL` + through `HUB_API_ORIGIN` and its WebSocket; on Docker Desktop a loopback + origin is mapped to `host.docker.internal`. Set `SIDECAR_HUB_WS_URL` when that address is not reachable from them. -- Per-container limits: `SIDECAR_MEMORY_BYTES`, `SIDECAR_PIDS`, - `SIDECAR_NANO_CPUS`. - `COORDINATOR_START_TIMEOUT_MS` (default `60000`) is the longest a project's coordinator may take to start; raise it on a slow machine, but keep it under `300000`. +Upgrading from a release where agents ran inside the hub container: work that +was running when you upgraded fails within a few minutes with its reason, and a +project's coordinator starts again on `sidecar-provisioner`. If you used +`deploy/compose.docker-sidecars.yml`, remove its old `sb-*` sidecar containers +with `docker rm -f`; the new provisioner does not manage them. + ## Logs The hub writes JSON lines to stdout at `LOG_LEVEL` (default `info`); an diff --git a/package.json b/package.json index 8d33302f..f54e971c 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "format": "bun fmt --no-script", "format:check": "bun fmt --no-script --check", "typecheck": "bun check --no-script && bun check --no-script -p packages/client && bun check --no-script -p apps/web", - "test": "bun --conditions=intx-src test ./apps/hub ./apps/web/src ./packages ./workflows ./test/integration", + "test": "bun --conditions=intx-src test ./apps/hub ./apps/sidecar-provisioner ./apps/web/src ./packages ./workflows ./test/integration", "test:e2e": "bun --env-file=.env.hub --conditions=intx-src test ./test/e2e", "vendor:verify": "bun scripts/verify-vendor.ts", "pins:verify": "bun scripts/verify-pins.ts", diff --git a/tsconfig.json b/tsconfig.json index a11685ef..bdb29d5d 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -17,7 +17,14 @@ "skipLibCheck": true, "allowImportingTsExtensions": true }, - "include": ["apps/hub", "scripts", "packages", "workflows", "test"], + "include": [ + "apps/hub", + "apps/sidecar-provisioner", + "scripts", + "packages", + "workflows", + "test" + ], "exclude": [ "node_modules", "vendor",