diff --git a/.env.hub.example b/.env.hub.example index 6152d864..83625211 100644 --- a/.env.hub.example +++ b/.env.hub.example @@ -1,6 +1,11 @@ # Builder hub environment. Copy this to .env.hub (never committed) and fill in every # line marked "Secret" with its own random value: openssl rand -hex 32 +# --- Docker Compose ---------------------------------------------------------- +# The published images deploy/compose.yml runs: rc (newest release candidate), +# latest (newest stable), main (every merge), an exact version or a commit sha. +BUILDER_VERSION=rc + # --- Database (required) ----------------------------------------------------- # Postgres 17. Docker Compose starts one with these values. DB_HOST=127.0.0.1 @@ -89,8 +94,9 @@ PRODUCT_HELLO_WORDING= # Agents run in sidecars. By default the hub starts them as local processes. # Most local sidecars running at once. # SIDECAR_LOCAL_MAX_ACTIVE=16 -# Optional Docker sidecars: build the image with `bun run docker:sidecar`, then set it. -# deploy/compose.docker-sidecars.yml sets them for the hub container. +# Optional Docker sidecars. deploy/compose.docker-sidecars.yml sets them for the hub +# container, with the published sidecar image at BUILDER_VERSION. From source, build +# it with `bun run docker:sidecar` and set: # SIDECAR_IMAGE=corbits-builder-sidecar:local # SIDECAR_DOCKER_HOST=unix:///var/run/docker.sock # The group owning the Docker socket for deploy/compose.docker-sidecars.yml: 0 on Docker Desktop. diff --git a/README.md b/README.md index 15e0170f..88c4503a 100644 --- a/README.md +++ b/README.md @@ -41,10 +41,11 @@ Set up Builder on this machine with the corbits-builder-install skill. Use Docke **With Docker.** ```sh -git clone https://github.com/corbitsdev/corbits-builder.git -cd corbits-builder -cp .env.hub.example .env.hub # fill in each line marked Secret -docker compose --env-file .env.hub --project-directory . -f deploy/compose.yml up -d --build +mkdir builder && cd builder +base=https://raw.githubusercontent.com/corbitsdev/corbits-builder/main +curl -fsSLO "$base/deploy/compose.yml" +curl -fsSL "$base/.env.hub.example" -o .env.hub # fill in each line marked Secret +docker compose --env-file .env.hub -f compose.yml up -d ``` Open http://localhost:8000. Ports, logs and running on a server are in diff --git a/deploy/compose.build.yml b/deploy/compose.build.yml new file mode 100644 index 00000000..7bc5931f --- /dev/null +++ b/deploy/compose.build.yml @@ -0,0 +1,12 @@ +# Builds the hub from this checkout instead of pulling the published image: +# docker compose --env-file .env.hub --project-directory . -f deploy/compose.yml -f deploy/compose.build.yml up -d --build +x-build: &build + image: corbits-builder-hub:local + build: + context: . + dockerfile: deploy/Dockerfile + target: hub + +services: + hub: *build + setup: *build diff --git a/deploy/compose.docker-sidecars.yml b/deploy/compose.docker-sidecars.yml index 66e8ef00..3cf42871 100644 --- a/deploy/compose.docker-sidecars.yml +++ b/deploy/compose.docker-sidecars.yml @@ -1,11 +1,21 @@ # Lets the hub also start sidecars as containers on the host's Docker daemon: # docker compose --env-file .env.hub --project-directory . -f deploy/compose.yml -f deploy/compose.docker-sidecars.yml up -d -# Build the image on that daemon first with `bun run docker:sidecar`. +# `sidecar-image` pulls SIDECAR_IMAGE onto that daemon before the hub starts; +# `docker compose pull` refreshes it after a channel change. +# From source, `bun run docker:sidecar` builds corbits-builder-sidecar:local; +# set SIDECAR_IMAGE to that. # Access to the socket is root on the host. services: + sidecar-image: + image: ${SIDECAR_IMAGE:-ghcr.io/corbitsdev/corbits-builder-sidecar:${BUILDER_VERSION:-rc}} + entrypoint: ["true"] + restart: "no" hub: + depends_on: + sidecar-image: + condition: service_completed_successfully environment: - SIDECAR_IMAGE: ${SIDECAR_IMAGE:-corbits-builder-sidecar:local} + SIDECAR_IMAGE: ${SIDECAR_IMAGE:-ghcr.io/corbitsdev/corbits-builder-sidecar:${BUILDER_VERSION:-rc}} SIDECAR_DOCKER_HOST: unix:///var/run/docker.sock # The group that owns the socket inside the container: 0 on Docker Desktop, # the host's docker group id (`getent group docker`) on Linux. diff --git a/deploy/compose.yml b/deploy/compose.yml index 4f60a477..711859c2 100644 --- a/deploy/compose.yml +++ b/deploy/compose.yml @@ -1,12 +1,11 @@ -# Run from the repo root with `--project-directory .` so compose keeps the -# repo-named project and resolves paths from the root, and `--env-file .env.hub` -# so the values below interpolate from it: +# Runs the published images. Pass `--env-file .env.hub` so the values below, +# BUILDER_VERSION among them, interpolate from it. From a checkout, run from the +# repo root with `--project-directory .`; anywhere else, keep .env.hub beside +# this file: # docker compose --env-file .env.hub --project-directory . -f deploy/compose.yml up -d +# deploy/compose.build.yml builds the images from source instead. x-hub: &hub - build: - context: . - dockerfile: deploy/Dockerfile - target: hub + image: ghcr.io/corbitsdev/corbits-builder-hub:${BUILDER_VERSION:-rc} env_file: .env.hub environment: DB_HOST: db @@ -42,8 +41,6 @@ services: timeout: 3s retries: 30 hub: - # No image name: Compose tags it after the project, so two checkouts never - # share one image. <<: *hub ports: - "127.0.0.1:${PORT:-8000}:${PORT:-8000}" diff --git a/docs/SELF_HOST.md b/docs/SELF_HOST.md index de1c2f77..b27f2e88 100644 --- a/docs/SELF_HOST.md +++ b/docs/SELF_HOST.md @@ -5,16 +5,23 @@ in [DEV.md](https://github.com/corbitsdev/corbits-builder/blob/main/docs/DEV.md) You need Docker with Compose and an API key or subscription login for at least one [model provider](#models). -`deploy/compose.yml` runs Postgres 17 and the hub, which applies the -migrations, then serves the API and the UI. Run every compose command from the -repo root. +`deploy/compose.yml` runs Postgres 17 and the published hub image, which +applies the migrations, then serves the API and the UI. It needs no clone. -## Configure +## Install ```sh -cp .env.hub.example .env.hub +mkdir builder && cd builder +base=https://raw.githubusercontent.com/corbitsdev/corbits-builder/main +curl -fsSLO "$base/deploy/compose.yml" +curl -fsSL "$base/.env.hub.example" -o .env.hub ``` +Replace `main` with a release tag (`v`) to pin these files to that +release. Every command below runs from this folder. + +## Configure + Give each secret `.env.hub` marks its own random value, for example from `openssl rand -hex 32`; Builder refuses to start without them. Losing or changing the encryption keys makes stored provider keys unreadable. Every other @@ -26,6 +33,11 @@ people open, and `HUB_PUBLIC_URL`, `HUB_API_ORIGIN` and `UI_PUBLIC_URL` when they differ from it. If a port is taken, `PORT` moves Builder (default `8000`) and `DB_PORT` the database (default `5433`). +`BUILDER_VERSION` picks the published images +(`ghcr.io/corbitsdev/corbits-builder-hub` and `-sidecar`): `rc` (default, the +newest release candidate), `latest` (the newest stable release), `main` (every +merge), an exact version such as `0.1.0-rc.1`, or a full commit sha. + ## Models Connect a provider in the UI under **Settings → Providers**, or set its key in @@ -47,9 +59,9 @@ How profiles map models to roles is in ## Run it ```sh -docker compose --env-file .env.hub --project-directory . -f deploy/compose.yml up -d --build # start or update -docker compose --env-file .env.hub --project-directory . -f deploy/compose.yml logs -f hub # follow the logs -docker compose --env-file .env.hub --project-directory . -f deploy/compose.yml down # stop, keep the data +docker compose --env-file .env.hub -f compose.yml up -d # start +docker compose --env-file .env.hub -f compose.yml logs -f hub # follow the logs +docker compose --env-file .env.hub -f compose.yml down # stop, keep the data ``` Open http://localhost:8000 (or your `PORT`), create an account, then add a @@ -57,8 +69,13 @@ provider under **Settings → Providers**, or set a provider key in `.env.hub` b starting. With one provider, projects run on its profile; with several, each person picks one under **Settings → Profiles** ([how the choice works](https://github.com/corbitsdev/corbits-builder/blob/main/docs/DEV.md#model-profile-choice)). -To update, pull the new commits and start it again; the hub applies new -migrations as it starts. +To update, or to switch channel after changing `BUILDER_VERSION`, pull the +images and start it again; the hub applies new migrations as it starts: + +```sh +docker compose --env-file .env.hub -f compose.yml pull +docker compose --env-file .env.hub -f compose.yml up -d +``` Login with ChatGPT and Login with xAI finish by sending the browser to `localhost:1455` and `localhost:1456`, so they work only when the browser runs @@ -73,9 +90,20 @@ To create a machine account and workspace without the UI, set the `BOOTSTRAP_*` values in `.env.hub` and run the `setup` service once: ```sh -docker compose --env-file .env.hub --project-directory . -f deploy/compose.yml --profile setup run --rm setup +docker compose --env-file .env.hub -f compose.yml --profile setup run --rm setup ``` +## From source + +In a clone, `deploy/compose.build.yml` builds the hub image from the checkout +instead of pulling it. Run from the repo root with `.env.hub` there: + +```sh +docker compose --env-file .env.hub --project-directory . -f deploy/compose.yml -f deploy/compose.build.yml up -d --build +``` + +`bun run docker:up` runs the same. + ## Sidecars in Docker Agents run in sidecars. By default the hub starts them as processes beside @@ -83,12 +111,13 @@ itself, inside the hub container. With `SIDECAR_IMAGE` set, it can also start them as containers on the Docker daemon at `SIDECAR_DOCKER_HOST`. ```sh -docker build -f deploy/Dockerfile --target sidecar -t corbits-builder-sidecar:local . -docker compose --env-file .env.hub --project-directory . -f deploy/compose.yml -f deploy/compose.docker-sidecars.yml up -d --build +curl -fsSLO https://raw.githubusercontent.com/corbitsdev/corbits-builder/main/deploy/compose.docker-sidecars.yml +docker compose --env-file .env.hub -f compose.yml -f compose.docker-sidecars.yml up -d ``` -- Build the image on the daemon the hub uses; the hub checks that it exists - when it starts. +- Compose pulls the sidecar image for your `BUILDER_VERSION` before the hub + starts, and `docker compose pull` refreshes it with the hub. From source, `bun run docker:sidecar` builds + `corbits-builder-sidecar:local`; set `SIDECAR_IMAGE` to it in `.env.hub`. - `deploy/compose.docker-sidecars.yml` mounts `/var/run/docker.sock` into the hub and adds the socket's group (`DOCKER_SOCKET_GID`: `0` on Docker Desktop, the host's `docker` group id on Linux). Access to that socket is root on the diff --git a/package.json b/package.json index 74d7c5b9..c3a86834 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "start": "exec bun --env-file=.env.hub --conditions=intx-src apps/hub/src/server.ts", "db:migrate": "bun --env-file=.env.hub --conditions=intx-src apps/hub/src/migrate.ts", "db:generate": "bun --conditions=intx-src ./node_modules/drizzle-kit/bin.cjs generate --config=apps/hub/drizzle.config.ts", - "docker:up": "docker compose --env-file .env.hub --project-directory . -f deploy/compose.yml up -d --build", + "docker:up": "docker compose --env-file .env.hub --project-directory . -f deploy/compose.yml -f deploy/compose.build.yml up -d --build", "docker:sidecar": "docker build -f deploy/Dockerfile --target sidecar -t corbits-builder-sidecar:local .", "auth:bootstrap": "bun --env-file=.env.hub --conditions=intx-src apps/hub/src/setup.ts", "auth:emulate": "bun --env-file=.env.hub scripts/google-emulate.ts", diff --git a/skills/corbits-builder-install/SKILL.md b/skills/corbits-builder-install/SKILL.md index e94d5466..9c79dcae 100644 --- a/skills/corbits-builder-install/SKILL.md +++ b/skills/corbits-builder-install/SKILL.md @@ -1,6 +1,6 @@ --- name: corbits-builder-install -description: Gets Builder running on this machine, from finding or cloning the checkout to the person signing up and connecting a model provider. Use whenever someone wants to install, set up, run, start, self-host or try Builder (Corbits Builder, corbitsdev/corbits-builder), with Docker Compose or with their own Postgres, or when a Builder checkout won't start. For driving projects through Builder's API once it runs, use operate-corbits-builder instead. +description: Gets Builder running on this machine, from downloading the Compose files or cloning the checkout to the person signing up and connecting a model provider. Use whenever someone wants to install, set up, run, start, self-host or try Builder (Corbits Builder, corbitsdev/corbits-builder), with Docker Compose or with their own Postgres, or when a Builder checkout won't start. For driving projects through Builder's API once it runs, use operate-corbits-builder instead. --- # Install Builder @@ -17,13 +17,7 @@ instead. - **Keep an existing `.env.hub`.** Its keys already encrypt stored data; new ones lock Builder out of it. -## 1. Find the checkout - -Work from the root of a Builder checkout (a folder with `deploy/compose.yml`). -If the current folder is not one, ask where it is, or clone -`https://github.com/corbitsdev/corbits-builder.git` where the person chooses. - -## 2. Choose a path +## 1. Choose a path Suggest Docker Compose if `docker info` succeeds, otherwise their own Postgres 17. Let the person decide. @@ -32,6 +26,25 @@ Suggest Docker Compose if `docker info` succeeds, otherwise their own Postgres 1 | Docker Compose | Docker with Compose | `http://localhost:8000` | | Own Postgres | Bun 1.4.4, Postgres 17 | `http://localhost:5173` | +## 2. Get the files + +**Docker Compose** runs published images and needs no clone. In a folder the +person chooses (or the one holding their existing `compose.yml`), download +what is missing: + +```sh +base=https://raw.githubusercontent.com/corbitsdev/corbits-builder/main +curl -fsSLO "$base/deploy/compose.yml" +curl -fsSL "$base/.env.hub.example" -o .env.hub.example +``` + +In a Builder checkout, download nothing: use `--project-directory . -f deploy/compose.yml` +wherever the steps below say `-f compose.yml`. + +**Own Postgres** runs from a Builder checkout (a folder with `deploy/compose.yml` +and `package.json`). If the current folder is not one, ask where it is, or clone +`https://github.com/corbitsdev/corbits-builder.git` where the person chooses. + ## 3. Write `.env.hub` and `.env.web` If `.env.hub` exists, keep it. Otherwise `cp .env.hub.example .env.hub` and fill each @@ -50,9 +63,9 @@ and `DB_PORT` (5433) the Compose database's. If a port is taken, set `PORT` or ## 4. Start Builder **Docker Compose**: -`docker compose --env-file .env.hub --project-directory . -f deploy/compose.yml up -d --build` -builds and starts Builder and Postgres. It is up when the hub logs `Hub ready` -(`docker compose --env-file .env.hub --project-directory . -f deploy/compose.yml logs hub`). +`docker compose --env-file .env.hub -f compose.yml up -d` +pulls and starts Builder and Postgres. It is up when the hub logs `Hub ready` +(`docker compose --env-file .env.hub -f compose.yml logs hub`). **Own Postgres** ([CONTRIBUTING.md](https://github.com/corbitsdev/corbits-builder/blob/main/CONTRIBUTING.md), @@ -78,8 +91,8 @@ Wait for them to confirm before you call it done. **Docker Compose**: ```sh -docker compose --env-file .env.hub --project-directory . -f deploy/compose.yml stop # stop, keep everything -docker compose --env-file .env.hub --project-directory . -f deploy/compose.yml down # remove containers, keep data +docker compose --env-file .env.hub -f compose.yml stop # stop, keep everything +docker compose --env-file .env.hub -f compose.yml down # remove containers, keep data ``` `down -v` also deletes every account, project and stored key; run it only when