Release v0.1.0-rc7

.github/workflows/ci.yml

@@ -2,7 +2,11 @@ name: CI
 
 on:
   push:
-    branches: [main]
+    # dev 也要跑，为的是缓存：GitHub 只让 PR 读到自己、base 分支和默认分支的缓存，
+    # 而此前没有任何一次 CI 在 dev 上跑过，于是 PR 之间互相看不见对方存的东西，
+    # 每次都从零编 300 多个依赖——今天 12 次 backend 运行，恢复缓存全是 No cache found。
+    # 顺带把「合并后的 dev 从没被 CI 跑过」这个口子堵上（见下面 migrations job 的注释）
+    branches: [main, dev]
   pull_request:
 
 # **显式只读。** 没有这一块时它继承仓库级默认，而那个默认要为
@@ -15,29 +19,99 @@ env:
   CARGO_TERM_COLOR: always
 
 jobs:
-  backend:
+  # lint / test / build 三个 job 并行。它们互不共享产物：clippy 是 check 模式，build 是
+  # 非测试的 codegen，test 是 test profile——串在一个 job 里只是把 clippy 和 build
+  # 排到了关键路径上（缓存命中后各 36s 和 39s，未命中 143s 和 51s）。
+  # 分支保护按名字要求 backend 与 web 通过，所以 backend 保留为下面的汇总 job；
+  # 直接改名会让 clippy 和 build 悄悄退出合并门槛
+  lint:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: rustfmt, clippy
+      # 只有 push（dev / main）那次运行才写缓存，PR 只读。此前每个 PR 各存约 2GB，
+      # 仓库上限 10GB，五六个 PR 就挤满，按 LRU 淘汰——同一个 PR 第二次跑时自己上次
+      # 存的也已经没了：17 条缓存的最后访问时间全等于创建时间，从来没人读到过
       - uses: Swatinem/rust-cache@v2
-      - name: Install PDF text extraction tools
-        run: sudo apt-get update && sudo apt-get install -y poppler-utils poppler-data
+        with:
+          save-if: ${{ github.event_name == 'push' }}
       - name: Format
         run: cargo fmt --all --check
       - name: Clippy
         run: cargo clippy --workspace --all-targets -- -D warnings
+
+  # 连库测试在这里跑。从前它们放在下面的 migrations job：那边有库、这边没有，
+  # 于是同一批测试二进制被编了两遍——`cargo test --workspace` 编一遍然后全部跳过，
+  # migrations 再编一遍才真的跑。给这个 job 也起一个 Postgres，一次编译全部跑完，
+  # migrations job 只留迁移本身的检查。连库测试由此进了 backend 汇总，才真正成为合并门槛
+  test:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: pgvector/pgvector:pg16
+        env:
+          POSTGRES_USER: utopia
+          POSTGRES_PASSWORD: utopia
+          POSTGRES_DB: utopia
+        ports: ["5432:5432"]
+        options: >-
+          --health-cmd pg_isready --health-interval 5s
+          --health-timeout 5s --health-retries 10
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      # 只在 push 时写缓存，理由见 lint job 同一处的注释
+      - uses: Swatinem/rust-cache@v2
+        with:
+          save-if: ${{ github.event_name == 'push' }}
+      - name: Install PDF text extraction tools
+        run: sudo apt-get update && sudo apt-get install -y poppler-utils poppler-data
+      # 库要先迁移好：store 的一百多个集成测试和 server 的多数 fixture 不自己跑迁移，
+      # 对着空库并发起几百个测试会成片撞 relation "organizations" does not exist（#869）。
+      # 不用 sqlx-cli——装它要一分多钟，而这个 job 在关键路径上；这个 example 复用
+      # 测试本来就要编的 utopia-store，几乎不花额外时间
+      - name: 迁移测试库
+        run: cargo run -p utopia-store --example migrate
+        env:
+          UTOPIA_DATABASE_URL: postgres://utopia:utopia@localhost:5432/utopia
       - name: Test
         run: cargo test --workspace
         # 装了 Poppler 就必须真的用它测：没有它时 PDF 回退的那个测试会跳过，
-        # 而一个静静跳过的测试等于没写（#248）
+        # 而一个静静跳过的测试等于没写（#248）。库同理：有库就**必须**连
+        # （UTOPIA_TEST_REQUIRE_DB），连库测试没跑成要红，不能跳过显示绿色
         env:
           UTOPIA_TEST_REQUIRE_PDFTOTEXT: "1"
+          UTOPIA_DATABASE_URL: postgres://utopia:utopia@localhost:5432/utopia
+          UTOPIA_TEST_REQUIRE_DB: "1"
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      # 只在 push 时写缓存，理由见 lint job 同一处的注释
+      - uses: Swatinem/rust-cache@v2
+        with:
+          save-if: ${{ github.event_name == 'push' }}
       - name: Build
         run: cargo build --workspace
 
+  # 汇总。分支保护要求的检查名是 backend，上面三个 job 任何一个不是 success 它就红——
+  # `if: always()` 是为了上游失败或取消时它也跑，而不是被跳过后显示为「未报告」
+  backend:
+    runs-on: ubuntu-latest
+    needs: [lint, test, build]
+    if: always()
+    steps:
+      - name: 汇总
+        run: |
+          echo "lint=${{ needs.lint.result }} test=${{ needs.test.result }} build=${{ needs.build.result }}"
+          test "${{ needs.lint.result }}" = success
+          test "${{ needs.test.result }}" = success
+          test "${{ needs.build.result }}" = success
+
   # 迁移得真的在库上跑一遍。此前 CI 只有 fmt/clippy/test/build，于是两个 PR
   # 各带一个 0025、各自通过，合进 main 之后服务直接起不来——sqlx 按版本号索引，
   # 而**合并后的状态从来没有被任何一次 CI 跑过**。
@@ -66,45 +140,20 @@ jobs:
             exit 1
           fi
       - uses: dtolnay/rust-toolchain@stable
+      # 只在 push 时写缓存，理由见 backend job 同一处的注释
       - uses: Swatinem/rust-cache@v2
+        with:
+          save-if: ${{ github.event_name == 'push' }}
       - name: 装 sqlx-cli
         run: cargo install sqlx-cli --no-default-features --features rustls,postgres --locked
       - name: 全新库上跑一遍
         run: sqlx migrate run --database-url postgres://utopia:utopia@localhost:5432/utopia
       # 第二遍必须也过：迁移改号后要能在已经跑过旧号的库上安全重放
       - name: 再跑一遍
         run: sqlx migrate run --database-url postgres://utopia:utopia@localhost:5432/utopia
-      # 连库的测试只有这个 job 跑得动——rust job 没有数据库，那些测试在那边
-      # 会跳过并显示绿色。不在这里跑一遍，等于写了测试却永远没执行过，
-      # 比没写更糟：它会让人以为 SQL 有覆盖
-      # 整套 store 集成测试都在这里跑，而且**没有库就失败**（UTOPIA_TEST_REQUIRE_DB）：
-      # 从前只跑 graph_changes，其余二十几个在没有库的 backend job 里静默跳过，
-      # 绿色是假的（#248）
-      - name: 连库测试
-        run: |
-          set -o pipefail
-          cargo test -p utopia-store 2>&1 | tee store-tests.log
-        env:
-          UTOPIA_DATABASE_URL: postgres://utopia:utopia@localhost:5432/utopia
-          UTOPIA_TEST_REQUIRE_DB: "1"
-      - name: 摘要
-        if: always()
-        run: |
-          passed=$(grep -o '[0-9]* passed' store-tests.log | awk '{s+=$1} END {print s+0}')
-          failed=$(grep -o '[0-9]* failed' store-tests.log | awk '{s+=$1} END {print s+0}')
-          echo "utopia-store against Postgres: **${passed} passed**, ${failed} failed — a missing database fails this job instead of skipping" >> "$GITHUB_STEP_SUMMARY"
-
-      - name: MCP structured reads against Postgres
-        run: cargo test -p utopia-server api::mcp::tests
-        env:
-          UTOPIA_DATABASE_URL: postgres://utopia:utopia@localhost:5432/utopia
-          UTOPIA_TEST_REQUIRE_DB: "1"
-
-      - name: Hybrid retrieval against Postgres
-        run: "cargo test -p utopia-server retrieval::"
-        env:
-          UTOPIA_DATABASE_URL: postgres://utopia:utopia@localhost:5432/utopia
-          UTOPIA_TEST_REQUIRE_DB: "1"
+      # 连库测试不在这里：它们随 backend job 的 `cargo test --workspace` 一起跑，
+      # 那边同样有库、同样设了 UTOPIA_TEST_REQUIRE_DB。曾经放在这里是因为只有这个 job
+      # 有库，代价是同一批测试二进制编两遍；这个 job 现在只管迁移本身能不能跑、能不能重放
 
   web:
     runs-on: ubuntu-latest

.roadmap-proposals/backup-restore.md

@@ -0,0 +1,203 @@
+# Backup and restore commands
+
+Roadmap item (README §Roadmap): *"Enterprise: OIDC SSO, backup and
+restore commands, benchmarks at 100k documents."*
+
+This proposal adds a `utopia` operator CLI with two subcommands,
+`backup` and `restore`, that shell out to `pg_dump` / `pg_restore` and
+`tar` to produce and replay a self-describing archive of the database
+plus the on-disk `data/` directory.
+
+## Why a separate crate
+
+The server binary at `crates/utopia-server/src/main.rs` is the
+production runtime. It already loads `AppConfig`, opens the database
+pool, runs migrations, and starts the HTTP server. Adding
+`utopia backup` to it is *possible* — a subcommand dispatch in `main`
+would do — but two reasons argue for a separate `utopia-cli` crate:
+
+1. **The CLI doesn't need the HTTP server, the extractor, the chat
+   loop, the mapping engine, or any of the 50+ modules pulled in by
+   `utopia-server`.** Today `utopia-server` is the largest crate in
+   the workspace; making the operator compile it just to dump the
+   database is a real cost (CI time, image size, attack surface).
+
+2. **The CLI is its own surface.** It runs in containers, in
+   Kubernetes Jobs, in cron, on operators' laptops. Keeping it
+   separate makes the boundary explicit and lets us ship a thinner
+   image.
+
+The CLI reuses `utopia-core::config::AppConfig::load()` for connection
+info (so it picks up the same `.env`, the same `UTOPIA_DATABASE_URL`,
+the same `UTOPIA_DATA_DIR` that the server uses), but nothing else.
+
+## `utopia backup`
+
+```text
+$ utopia backup [flags]
+
+  --output <PATH>            # archive path; default utopia-<timestamp>.tar.gz
+  --include-data-dir         # also tar the data/ directory (without secret.key)
+  --include-secret-key       # …and the sealing key as well, deliberately
+  --dry-run                  # print the plan, don't write anything
+  --pg-dump <PATH>           # path to pg_dump binary
+  --tar <PATH>               # path to tar binary
+  --migration-url <URL>      # override connection string for pg_dump
+
+# defaults to .env / UTOPIA_DATABASE_URL for the connection
+```
+
+What it does:
+
+1. Resolves the connection string and `data_dir` from `AppConfig`.
+2. Runs `pg_dump -Fc` to a temporary file under the current directory.
+   `-Fc` (custom format) is the only format `pg_restore` consumes;
+   plain SQL would lose the ability to do parallel restore and would
+   fail on large objects.
+3. Optionally tars `data/` into a second temporary file.
+4. Builds a `manifest.json` with:
+   - `schema_version`: pulled from the running migrations
+   - `utopia_version`: from `Cargo.toml`
+   - `created_at`: UTC ISO 8601
+   - `components.pg_dump.{path, format, bytes}` and `data_dir.{path, present}`
+   - `checksums`: `sha256:<hex>` for each component
+5. Writes a single `*.tar.gz` containing `manifest.json` + `pg_dump.custom`
+   + (optional) `data/`. Refuses to overwrite an existing archive.
+
+Why a tarball wrapper instead of just shipping the `pg_dump` output?
+
+- One file to copy off-host.
+- `manifest.json` records what was backed up, when, and from which
+  schema version — without it, six months later, you can't tell
+  whether a `.dump` file came from a v0.1 or v0.2 server.
+- Checksums in the manifest let `restore` fail loudly on a corrupted
+  archive before touching the live database.
+
+## `utopia restore`
+
+```text
+$ utopia restore --from <PATH> [flags]
+
+  --from <PATH>              # archive to restore from (required)
+  --target-data-dir <PATH>   # override the data_dir path
+  --pg-restore <PATH>        # path to pg_restore binary
+  --dry-run                  # print the plan, don't write anything
+  --force                    # allow restore into a non-empty database
+  --yes                      # skip the "are you sure?" prompt
+```
+
+Both halves are implemented. `backup` landed first, and `restore`
+followed in the same crate, for two reasons worth keeping on the record:
+
+1. The backup side is independently useful — operators want to *take*
+   backups even before they have a working restore, because the
+   alternative (no backups at all) is worse.
+2. The restore side has design questions that benefit from running
+   backup in production for a while first (see Open question #2).
+
+## Manifest version policy
+
+`schema_version` is a `u32`. On read, `restore` will refuse a manifest
+whose `schema_version` is greater than the running server's
+`schema_version` — forward-incompatible backups are useless. Older
+manifests are accepted with a warning, since older schema versions
+might still restore cleanly into a newer server.
+
+## Docker compatibility
+
+The server ships as `ghcr.io/deeplethe/utopia`. Today the runtime
+image does *not* include `postgresql-client` (pg_dump lives in a
+separate `postgres:16` image). Two ways to close that gap:
+
+- **A: Bundle `pg_dump` in the server image.** Pro: one image, `utopia
+  backup` works in any container that the server runs in. Con:
+  ~30 MB extra for the postgres client.
+- **B: Ship a separate `utopia-cli` image with `pg_dump` and `pg_restore`.**
+  Pro: server image stays small. Con: two artifacts to publish.
+
+Recommendation: **B**. The cli image is also where future ops tools
+will live (`utopia migrate`, `utopia reindex`); it's the right
+home for `postgresql-client`.
+
+This proposal does **not** ship the docker image change — it lands the
+binary first, the image second.
+
+## Open questions for the maintainer
+
+1. **Where does the CLI live?** `crates/utopia-cli/` (separate crate,
+   separate `utopia` binary — this proposal) vs. a `bin/utopia.rs`
+   inside `utopia-server` (one less artifact, slower compile for
+   everyone, one less Cargo.toml to maintain). My read: separate
+   crate, separate binary.
+2. **Restore strategy.** Drop-and-recreate the database (requires
+   `CREATEDB` privilege on the operator's role) vs. in-place
+   `pg_restore --clean` (no extra privilege, but slower and brittle
+   on partial restores). My read: in-place `--clean`, since the
+   operator role that can dump is usually the same one that owns the
+   schema.
+3. **Should the app Docker image install `postgresql-client`?** My
+   read: no — keep the runtime image minimal; ship a separate
+   `utopia-cli` image with `pg_dump` and `pg_restore`.
+4. **Manifest version policy.** Refuse forward-incompatible
+   manifests on read? Accept older ones with a warning? My read:
+   refuse forward, warn on older.
+5. **Should `UTOPIA_BACKUP_DIR` become a new config knob** for the
+   default output location (today's default is the current working
+   directory)? My read: yes, but trivially — it's the kind of thing
+   operators will set once and forget.
+
+## What this cut does NOT do
+
+- No docker image change (see Open question #3).
+- No `UTOPIA_BACKUP_DIR` config (see Open question #5).
+- No automatic migration of older manifests — `restore` will only
+  read `schema_version == current`.
+- **The sealing key is left out of the archive by default.** The dump carries
+  credentials sealed with `data/secret.key`, and an archive is the artifact that
+  gets copied between hosts and handed to whoever runs the restore, so shipping
+  both halves in one file is not an unencrypted archive — it is no sealing at
+  all. `--include-secret-key` puts it in on purpose, the manifest records
+  whether it is there, and a restore without it says so rather than letting the
+  server fail to open its own credentials later.
+- No encryption-at-rest. Out of scope; the operator's filesystem
+  encryption (LUKS, EBS encryption, etc.) is the right layer.
+- No streaming upload to S3. Out of scope; the operator can pipe
+  `utopia backup --output -` to `aws s3 cp - s3://…` today.
+
+## Test plan
+
+8 unit tests, all pure (no live DB, no Docker):
+
+1. `parses_backup_minimal` — `utopia backup --dry-run` produces a
+   `BackupArgs` with all defaults.
+2. `parses_backup_full` — every flag set, every field populated.
+3. `parses_restore_requires_from` — `--from` is mandatory.
+4. `parses_restore_full` — every flag set, every field populated.
+5. `rejects_unknown_subcommand` — `utopia frobnicate` fails clearly.
+6. `redact_url_host_keeps_userinfo_at_host` — `postgres://u:***@h/db`
+   stays redacted on round-trip.
+7. `redact_url_host_handles_no_at` — `postgres://h/db` redacts
+   nothing (no userinfo).
+8. `hex_encode_known_value` — `hex_encode(&[0xde, 0xad]) == "dead"`.
+
+Live integration (smoke test, not part of CI):
+
+```
+$ createdb utopia_test_restore
+$ UTOPIA_DATABASE_URL=postgres://utopia:utopia@localhost:1543/utopia_test \
+    utopia backup --output /tmp/utopia.tar.gz --include-data-dir
+$ dropdb utopia_test_restore && createdb utopia_test_restore
+$ UTOPIA_DATABASE_URL=postgres://utopia:utopia@localhost:1543/utopia_test \
+    utopia restore --from /tmp/utopia.tar.gz --yes
+```
+
+Smoke-tested manually on the maintainer's local docker-compose stack
+once the restore side lands.
+
+## Migration concern: existing data is already there
+
+Backups are forward-looking. Anything written before this lands has
+no `manifest.json` and no `checksums`. The operator's first backup
+after upgrade will be the first one with a manifest; older `.dump`
+files (if any exist) are still valid `pg_dump -Fc` files but cannot
+be verified or version-checked by `utopia restore`.