diff --git a/README.md b/README.md index 3be8549..c7f5987 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,9 @@ ```bash bun i # root Bun workspace: web-app, examiner-dashboard, desktop-app -bun run dev # web-app (13003) + examiner-dashboard client (13005) +bun run dev edd # slice in Docker: eDd + eDb + S3, migrated; `-d` background (`bun run dev help`) +bun run dev down # stop slices, keep data (`reset` deletes it) +bun run dev:client # web-app (13003) + examiner-dashboard client (13005), HMR bun run dev:desktop # desktop-app via tauri dev bun run check # astro check / tsc -b in every Bun workspace member bun run build diff --git a/apps/examiner-dashboard/Dockerfile b/apps/examiner-dashboard/Dockerfile index 7396ece..6d54097 100644 --- a/apps/examiner-dashboard/Dockerfile +++ b/apps/examiner-dashboard/Dockerfile @@ -25,11 +25,14 @@ COPY databases/curriculum databases/curriculum COPY databases/examiner databases/examiner # Checks `query!` macros against the committed .sqlx/, so the build needs no database. ENV SQLX_OFFLINE=true +# `dev` for local development only (compose.dev.yaml): debug builds allow MOCK_AUTH. Never deploy one. +ARG CARGO_PROFILE=release RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/src/target \ - cargo build --locked --release -p examiner-dashboard -p edb -p cdb \ - && cp target/release/server /usr/local/bin/examiner-dashboard \ - && cp target/release/edb-migrate target/release/cdb-migrate /usr/local/bin/ + cargo build --locked --profile "$CARGO_PROFILE" -p examiner-dashboard -p edb -p cdb \ + && out="target/$([ "$CARGO_PROFILE" = dev ] && echo debug || echo "$CARGO_PROFILE")" \ + && cp "$out/server" /usr/local/bin/examiner-dashboard \ + && cp "$out/edb-migrate" "$out/cdb-migrate" /usr/local/bin/ FROM debian:trixie-slim RUN apt-get update \ diff --git a/compose.dev.yaml b/compose.dev.yaml new file mode 100644 index 0000000..28b85a4 --- /dev/null +++ b/compose.dev.yaml @@ -0,0 +1,172 @@ +# Local development: every service in one project. Run through `bun run dev ` (dev/dev.ts), +# which passes the database `.env` files for interpolation. Starting a service starts its +# `depends_on` too: `up -d examiner-dashboard` brings up eDb, S3, the eDb migration and the bucket. +# Deploys never use this file: they use each directory's compose.yaml. +# +# Databases extend their own compose.yaml. Apps are defined here instead: extending theirs would +# interpolate their required (`:?`) variables for every slice. Apps read their own `.env` as +# `env_file`, then `environment` points them at in-network hosts. +name: exam-platform-dev + +x-edd-build: &edd-build + context: . + dockerfile: apps/examiner-dashboard/Dockerfile + # Debug build: MOCK_AUTH needs one, and it compiles faster. + args: { CARGO_PROFILE: dev } +x-aapi-build: &aapi-build + context: . + dockerfile: apps/auth-api/Dockerfile + +services: + examiner-db: + extends: { file: databases/examiner/compose.yaml, service: examiner-db } + + curriculum-db: + extends: { file: databases/curriculum/compose.yaml, service: curriculum-db } + + # One-shot: creates database `curriculum` (on branch `dev`, server.sh) if missing. + curriculum-db-init: + extends: { file: databases/curriculum/compose.yaml, service: curriculum-db } + entrypoint: ["/bin/sh", "-c"] + command: + - DOLT_CLI_PASSWORD="$$DOLT_ROOT_PASSWORD" exec dolt --host curriculum-db --port 3306 --user root --no-tls sql -q 'CREATE DATABASE IF NOT EXISTS curriculum' + restart: "no" + # The CLI needs a writable HOME; the server's lives on the volume this service does not mount. + environment: + HOME: /tmp + ports: !reset [] + volumes: !reset [] + healthcheck: { disable: true } + depends_on: + curriculum-db: { condition: service_healthy } + + # S3-compatible curriculum bucket. `s3.localhost` resolves to this container in the network and + # to loopback in browsers, so presigned URLs work from both. + s3: + image: chrislusf/seaweedfs:4.52@sha256:57877e32bb90073b37dc109b438ba66e84fd5ed9ced5f5c04f56444ef243ce59 + command: server -dir=/data -s3 -s3.port=18333 -s3.config=/etc/seaweedfs/s3.json + restart: unless-stopped + ports: ["127.0.0.1:18333:18333"] + volumes: + - s3:/data + - ./dev/seaweedfs-s3.json:/etc/seaweedfs/s3.json:ro + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:18333/healthz"] + interval: 5s + timeout: 5s + retries: 30 + start_period: 10s + networks: + platform: + aliases: [s3.localhost] + + # One-shot: creates bucket `curriculum` if missing. + s3-init: + image: chrislusf/seaweedfs:4.52@sha256:57877e32bb90073b37dc109b438ba66e84fd5ed9ced5f5c04f56444ef243ce59 + entrypoint: ["/bin/sh", "-c"] + command: + - echo 's3.bucket.list' | weed shell -master=s3:9333 | grep -qw curriculum || echo 's3.bucket.create -name curriculum' | weed shell -master=s3:9333 + restart: "no" + depends_on: + s3: { condition: service_healthy } + networks: [platform] + + migrate-edb: + build: *edd-build + image: exam-platform-dev/examiner-dashboard + entrypoint: [edb-migrate] + restart: "no" + environment: + EDB_MIGRATOR_URL: postgres://edb_migrator:${EDB_MIGRATOR_PASSWORD}@examiner-db:5432/examiner + depends_on: + examiner-db: { condition: service_healthy } + networks: [platform] + + # Same migrations from aAPI's image, so the auth slice never builds eDd. + migrate-edb-aapi: + extends: migrate-edb + build: *aapi-build + image: exam-platform-dev/auth-api + + migrate-cdb: + build: *edd-build + image: exam-platform-dev/examiner-dashboard + entrypoint: [cdb-migrate] + restart: "no" + environment: + CDB_MIGRATOR_URL: mysql://root:${CDB_ROOT_PASSWORD}@curriculum-db:3306/curriculum + depends_on: + curriculum-db-init: { condition: service_completed_successfully } + networks: [platform] + + examiner-dashboard: + build: *edd-build + image: exam-platform-dev/examiner-dashboard + restart: unless-stopped + ports: ["127.0.0.1:13002:13002"] + # Optional: RUST_LOG, SENTRY_*, and real GOOGLE_* to test Google login. Unset GOOGLE_* and + # COOKIE_KEY fall back to MOCK_AUTH defaults. + env_file: + - path: apps/examiner-dashboard/.env + required: false + environment: + ENVIRONMENT: development + # Mounts POST /auth/dev/login. + MOCK_AUTH: "true" + LISTEN_ADDR: 0.0.0.0:13002 + WEB_DIR: /app/web + EDB_DATABASE_URL: postgres://edd:${EDB_EDD_PASSWORD}@examiner-db:5432/examiner + CURRICULUM_S3_ENDPOINT: http://s3.localhost:18333 + CURRICULUM_S3_REGION: us-east-1 + CURRICULUM_S3_BUCKET: curriculum + CURRICULUM_S3_ACCESS_KEY_ID: dev + CURRICULUM_S3_SECRET_ACCESS_KEY: dev-secret-key + CURRICULUM_S3_FORCE_PATH_STYLE: "true" + depends_on: + migrate-edb: { condition: service_completed_successfully } + s3-init: { condition: service_completed_successfully } + networks: + platform: + aliases: [examiner-dashboard] + + auth-api: + build: *aapi-build + image: exam-platform-dev/auth-api + restart: unless-stopped + ports: ["127.0.0.1:13001:13001"] + env_file: + - path: apps/auth-api/.env + required: false + environment: + ENVIRONMENT: development + LISTEN_ADDR: 0.0.0.0:13001 + EDB_DATABASE_URL: postgres://aapi:${EDB_AAPI_PASSWORD}@examiner-db:5432/examiner + depends_on: + migrate-edb-aapi: { condition: service_completed_successfully } + networks: + platform: + aliases: [auth-api] + + web-app: + build: + context: . + dockerfile: apps/web-app/Dockerfile + image: exam-platform-dev/web-app + restart: unless-stopped + ports: ["127.0.0.1:13003:13003"] + environment: + AUTH_API_URL: http://127.0.0.1:13001 + depends_on: + auth-api: { condition: service_healthy } + networks: + platform: + aliases: [web-app] + +networks: + platform: + name: exam-platform-dev + +volumes: + examiner-db: + curriculum-db: + s3: diff --git a/dev/dev.ts b/dev/dev.ts new file mode 100644 index 0000000..57bbb6a --- /dev/null +++ b/dev/dev.ts @@ -0,0 +1,233 @@ +#!/usr/bin/env bun +// Local development front end for compose.dev.yaml. `bun run dev help` lists commands. +import { existsSync } from "node:fs"; +import { randomBytes } from "node:crypto"; +import { join } from "node:path"; + +const ROOT = join(import.meta.dir, ".."); + +// Slice -> compose services to start (compose starts each `depends_on` too) and long-running +// services whose logs to follow. +type Slice = { services: string[]; logs: string[]; about: string }; +const SLICES: Record = { + edb: { + services: ["migrate-edb"], + logs: ["examiner-db"], + about: "eDb, migrated", + }, + cdb: { + services: ["migrate-cdb"], + logs: ["curriculum-db"], + about: "cDb, database created and migrated", + }, + s3: { + services: ["s3-init"], + logs: ["s3"], + about: "SeaweedFS with bucket `curriculum`", + }, + edd: { + services: ["examiner-dashboard"], + logs: ["examiner-dashboard"], + about: "eDd server + built client, with eDb, S3", + }, + aapi: { services: ["auth-api"], logs: ["auth-api"], about: "aAPI, with eDb" }, + wapp: { + services: ["web-app"], + logs: ["web-app"], + about: "wApp, with aAPI and eDb", + }, + all: { + services: ["examiner-dashboard", "web-app", "migrate-cdb"], + logs: ["examiner-dashboard", "auth-api", "web-app"], + about: "everything", + }, +}; + +// Interpolation only: database passwords. Apps read their own `.env` as `env_file`. +const ENV_FILES = ["databases/examiner/.env", "databases/curriculum/.env"]; + +// Generated when a `.env` is created from `.env.example` and the value is empty. +const SECRETS: Record string> = { + EDB_PASSWORD: () => hex(32), + EDB_MIGRATOR_PASSWORD: () => hex(32), + EDB_EDD_PASSWORD: () => hex(32), + EDB_AAPI_PASSWORD: () => hex(32), + CDB_ROOT_PASSWORD: () => hex(32), +}; + +const hex = (bytes: number) => randomBytes(bytes).toString("hex"); + +function parseEnv(text: string): Record { + const vars: Record = {}; + for (const line of text.split("\n")) { + const m = line.match(/^([A-Z0-9_]+)=(.*)$/); + if (m) vars[m[1]!] = m[2]!; + } + return vars; +} + +// Creates `/.env` from `.env.example`: fills empty secrets, then `` placeholders from +// this file. Never touches an existing `.env`. +async function ensureEnv(dir: string): Promise { + const target = join(ROOT, dir, ".env"); + if (existsSync(target)) return false; + let text = await Bun.file(join(ROOT, dir, ".env.example")).text(); + text = text.replace(/^([A-Z0-9_]+)=$/gm, (line, key: string) => + SECRETS[key] ? `${key}=${SECRETS[key]()}` : line, + ); + const vars = parseEnv(text); + text = text.replace(/<([A-Z0-9_]+)>/g, (m, key: string) => vars[key] ?? m); + await Bun.write(target, text); + console.log(`created ${dir}/.env from .env.example`); + return true; +} + +// Database passwords. Apps need no `.env` in development: eDd runs with MOCK_AUTH. +async function bootstrap() { + await ensureEnv("databases/examiner"); + await ensureEnv("databases/curriculum"); +} + +function composeCmd(args: string[]): string[] { + const envFiles = ENV_FILES.filter((f) => existsSync(join(ROOT, f))).flatMap((f) => [ + "--env-file", + f, + ]); + return ["docker", "compose", "-f", "compose.dev.yaml", ...envFiles, ...args]; +} + +async function compose(args: string[]): Promise { + const proc = Bun.spawn(composeCmd(args), { + cwd: ROOT, + stdio: ["inherit", "inherit", "inherit"], + }); + return proc.exited; +} + +async function composeOutput(args: string[]): Promise { + const proc = Bun.spawn(composeCmd(args), { cwd: ROOT, stdio: ["ignore", "pipe", "inherit"] }); + const out = await proc.stdout.text(); + if ((await proc.exited) !== 0) process.exit(1); + return out; +} + +// `services` plus everything they transitively `depends_on`, split by restart policy: +// `restart: "no"` marks a one-shot (database init, migration, bucket creation). +async function withDependencies( + services: string[], +): Promise<{ all: string[]; oneShots: string[]; longRunning: string[] }> { + const config = JSON.parse(await composeOutput(["config", "--format", "json"])) as { + services: Record }>; + }; + const seen = new Set(); + const visit = (name: string) => { + if (seen.has(name)) return; + seen.add(name); + Object.keys(config.services[name]?.depends_on ?? {}).forEach(visit); + }; + services.forEach(visit); + const all = [...seen]; + const isOneShot = (name: string) => config.services[name]?.restart === "no"; + return { + all, + oneShots: all.filter(isOneShot), + longRunning: all.filter((name) => !isOneShot(name)), + }; +} + +// Like `docker compose up -d`, but one-shots count as ready once they exit 0. `--wait` treats an +// exited one-shot as a failure, or returns before a slow one has finished. +async function upDetached(services: string[]): Promise { + const { all, oneShots, longRunning } = await withDependencies(services); + if ((await compose(["up", "--detach", "--build", ...all])) !== 0) return false; + if (oneShots.length) { + await compose(["wait", ...oneShots]); + const containers = (await composeOutput(["ps", "--all", "--format", "json", ...oneShots])) + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as { Service: string; State: string; ExitCode: number }); + const failed = oneShots.filter( + (name) => + !containers.some((c) => c.Service === name && c.State === "exited" && c.ExitCode === 0), + ); + if (failed.length) { + console.error(`failed: ${failed.join(", ")}`); + return false; + } + } + return !longRunning.length || (await compose(["up", "--detach", "--wait", ...longRunning])) === 0; +} + +function resolve(slices: string[], key: "services" | "logs"): string[] { + const unknown = slices.filter((s) => !SLICES[s]); + if (unknown.length) { + console.error(`unknown slice: ${unknown.join(", ")}`); + help(); + process.exit(2); + } + return [...new Set(slices.flatMap((s) => SLICES[s]![key]))]; +} + +function help() { + const slices = Object.entries(SLICES) + .map(([name, { about }]) => ` ${name.padEnd(6)} ${about}`) + .join("\n"); + console.log(`Usage: bun run dev ... build, start, follow logs; Ctrl-C stops + bun run dev -d ... build, start in background, wait until healthy + bun run dev logs [slice]... follow logs + bun run dev ps list containers + bun run dev down stop everything, keep data + bun run dev reset stop everything, delete data volumes + bun run dev compose ... any docker compose command on compose.dev.yaml + +Slices: +${slices} + +Example: bun run dev edd`); +} + +let argv = process.argv.slice(2); +if (argv[0] === "up") argv = argv.slice(1); +const [cmd, ...rest] = argv; + +switch (cmd) { + case undefined: + case "help": + case "-h": + case "--help": + help(); + break; + case "down": + process.exit(await compose(["down", ...rest])); + case "reset": + process.exit(await compose(["down", "--volumes", ...rest])); + case "ps": + process.exit(await compose(["ps", "--all", ...rest])); + case "compose": + process.exit(await compose(rest)); + case "logs": + process.exit(await compose(["logs", "--follow", "--tail", "100", ...resolve(rest, "logs")])); + default: { + // Slices only: other commands pass `-d` through unchanged. + const DETACH = ["-d", "--detach"]; + const detach = argv.some((a) => DETACH.includes(a)); + const slices = argv.filter((a) => !DETACH.includes(a)); + const services = resolve(slices, "services"); + await bootstrap(); + if (detach) { + if (!(await upDetached(services))) { + console.error("startup failed: `bun run dev ps`, then `bun run dev logs `"); + process.exit(1); + } + process.exit(await compose(["ps", "--all"])); + } + // Like `docker compose up`: attached, Ctrl-C stops the slices and their dependencies. Logs + // from the slices' long-running services only. Ctrl-C reaches compose through the terminal: + // ignore it here and wait for compose to finish stopping. + process.on("SIGINT", () => {}); + const attach = resolve(slices, "logs").flatMap((s) => ["--attach", s]); + process.exit( + await compose(["up", "--build", ...attach, ...(await withDependencies(services)).all]), + ); + } +} diff --git a/dev/seaweedfs-s3.json b/dev/seaweedfs-s3.json new file mode 100644 index 0000000..44b2556 --- /dev/null +++ b/dev/seaweedfs-s3.json @@ -0,0 +1,9 @@ +{ + "identities": [ + { + "name": "dev", + "credentials": [{ "accessKey": "dev", "secretKey": "dev-secret-key" }], + "actions": ["Admin", "Read", "Write", "List", "Tagging"] + } + ] +} diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 74b58a7..1f09bc4 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -87,6 +87,36 @@ VS Code: install recommended extensions (`.vscode/extensions.json`). Settings ru ### Run +#### Slices (Docker) + +`bun run dev ...` builds and starts a slice and everything it depends on from `compose.dev.yaml`, attached like `docker compose up`: logs of the slice's own services follow, Ctrl-C stops the slice and its dependencies. `-d` runs it in the background like `docker compose up -d`: waits until healthy, then returns; stop with `bun run dev down`. No network setup and no `.env` export needed. + +| Slice | Starts | +| ------ | ------------------------------------------------------------------- | +| `edb` | eDb, migrated | +| `cdb` | cDb, database `curriculum` created and migrated | +| `s3` | SeaweedFS on `127.0.0.1:18333`, bucket `curriculum` | +| `edd` | eDd server with built client on `127.0.0.1:13002`, plus `edb`, `s3` | +| `aapi` | aAPI on `127.0.0.1:13001`, plus `edb` | +| `wapp` | wApp on `127.0.0.1:13003`, plus `aapi` | +| `all` | Everything | + +```bash +bun run dev edd # combine: bun run dev edd wapp +bun run dev -d edd # background +bun run dev logs edd # follow logs +bun run dev ps +bun run dev down # stop, keep data +bun run dev reset # stop, delete data volumes +bun run dev compose ... # any docker compose command on compose.dev.yaml +``` + +- **Env.** Missing `databases/*/.env` are created from `.env.example` with random passwords. Apps need no `.env`: eDd is a debug build with `MOCK_AUTH=true`, so log in through `POST /auth/dev/login` and Google needs no credentials. An app `.env`, if present, is read (e.g. real `GOOGLE_*` or `RUST_LOG`), with database and S3 URLs overridden to in-network hosts. +- **Rebuild.** Code changes need the slice run again (`--build`; eDd builds the `dev` profile). For client HMR, run `bun run dev:client` alongside `edd`: it proxies to the container on 13002. +- **S3.** `s3.localhost` resolves to SeaweedFS inside the network and to loopback in the browser, so presigned URLs work from both. Dev-only keys: `dev/seaweedfs-s3.json`. + +#### Native + | App | Command | Port | | ---------- | ----------------------------------------------- | ----------------- | | aAPI | `cargo run -p auth-api` | `127.0.0.1:13001` | @@ -97,7 +127,7 @@ VS Code: install recommended extensions (`.vscode/extensions.json`). Settings ru | eDb | `docker compose up -d` in `databases/examiner/` | `127.0.0.1:15432` | | Docs | `mdbook watch` (or `mdbook serve`) | `localhost:3000` | -`cargo run` rows need their `.env` exported first ([Environment](#environment)). `bun run dev` starts wApp and eDd client together. eDd client proxies `/api` to eDd server. wApp and dApp call aAPI at `AUTH_API_URL`. +`cargo run` rows need their `.env` exported first ([Environment](#environment)). `bun run dev:client` starts wApp and eDd client together. eDd client proxies `/api` to eDd server. wApp and dApp call aAPI at `AUTH_API_URL`. Images: `docker compose -f apps//compose.yaml up --build`. Build context is repo root. diff --git a/docs/apps/examiner-dashboard.md b/docs/apps/examiner-dashboard.md index c146368..a5d3328 100644 --- a/docs/apps/examiner-dashboard.md +++ b/docs/apps/examiner-dashboard.md @@ -9,7 +9,7 @@ Where examiners author, review, promote, and moderate. | Server port | `127.0.0.1:13002` | | Client dev port | `127.0.0.1:13005`, proxies `/api` to the server | -The S3 client is `libs/object-storage`, which knows no provider: `CURRICULUM_S3_*` sets the endpoint, region, bucket and key pair. To switch provider, change those values and copy the objects across with their keys. +The S3 client is `libs/object-storage`, which knows no provider: `CURRICULUM_S3_*` sets the endpoint, region, bucket and key pair. To switch provider, change those values and copy the objects across with their keys. Local development: `bun run dev edd` points eDd at SeaweedFS (`compose.dev.yaml`, [Run](../CONTRIBUTING.md#run)). The Dockerfile builds `client/` with Bun, `server/` with cargo, and copies `dist` to `/app/web`. diff --git a/package.json b/package.json index 45feaac..a862cf9 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,8 @@ } }, "scripts": { - "dev": "bun --filter web-app --filter examiner-dashboard-client dev", + "dev": "bun dev/dev.ts", + "dev:client": "bun --filter web-app --filter examiner-dashboard-client dev", "dev:desktop": "bun --filter desktop-app tauri:dev", "build": "bun --filter '*' build", "check": "bun --filter '*' check",