diff --git a/Cargo.lock b/Cargo.lock index 4055249..b637b85 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4938,7 +4938,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix 1.1.5", "windows-sys 0.61.2", diff --git a/docs/scorpiofs-libra-complete-spec-v1.md b/docs/scorpiofs-libra-complete-spec-v1.md new file mode 100644 index 0000000..4de07cd --- /dev/null +++ b/docs/scorpiofs-libra-complete-spec-v1.md @@ -0,0 +1,747 @@ +# ScorpioFS × Libra 完整集成与传输协议 SPEC + +- **版本**：v1.1.0（v2 协议已实现并通过 E2E） +- **日期**：2026-09-21 +- **状态**：§6 的 Worktree Control Protocol v2（effective diff / commit-finalize / refresh）已实现并通过真实 mega2 + FUSE E2E；Libra `sync` 已接入 finalize。`.libra` 移出 mount 的 chain 模式已实现并 E2E；host-side metadata 已实现并 E2E。 + +### 实施进度（v1.1.0 增量） + +| 项 | 状态 | 证据 | +|---|---|---| +| Mega `refs` 语义实测 | ✅ 已确认 | `tree/blob/content-hash/latest-commit` 均接受 `refs=`；**仅接受 mega 内部 monorepo commit OID**（`latest-commit` 返回值），git tip OID 不可解析（空结果）、分支名 500。content-hash OID = 标准 git blob OID（`sha1("blob \0"+content)`，用 mega 自身回报值验证）。因此 lower switch 的 revision 键 = 内部 OID | +| Dicfuse revision pinning | ✅ 已实现 | `DictionaryStore.pinned_refs`（`store.rs`）→ `fetch_dir` 携带 `&refs=`（6 处调用点）；`Dicfuse::new_with_base_path_store_path_and_refs`；`DicfuseManager::for_base_path_and_refs`（缓存键含 refs，store 目录带 `-rev-<前缀>` 后缀，防跨 revision 复用持久化树库） | +| effective diff | ✅ 已实现并 E2E | `daemon/worktree_v2.rs::effective_changes`：扫描 upper（跳过 `.libra`、OCI whiteout 语义）→ 每路径计算 git blob OID 与 lower 对比 → Added/Modified/Deleted；upper==base 报 clean（消除假 dirty）；`generation_of` 路径排序后 FNV，顺序无关（单测抓出并修复过顺序敏感 bug） | +| `GET /worktrees/{id}/state` | ✅ 已实现并 E2E | 返回 `lower_revision`/`base_revision`（两个命名空间分开）/generation/dirty/changes | +| `POST /worktrees`（v2 attach） | ✅ **已实现并 E2E** | 单调用 attach：**挂载即钉定 lower**（client 传内部 OID 或留空由 daemon 解析 latest-commit），无 v1 的 tip 漂移窗口；mountpoint 可指定（空目录预校验，失败零副作用）；`base_revision` 原样记录（client 命名空间）；**daemon 不写任何 VCS metadata 进 mount**。Libra `worktree add --backend scorpiofs` 已迁移到该端点，指针文件由 Libra 经挂载自写（`commondir`/`worktree_id`/`index`/`scorpiofs_mount_id`） | +| **修复的真实缺陷：MountEntry 丢 pin** | ✅ 已修 | `create_mount` 构造 `MountEntry` 时未写入 `request.pinned_refs` → mount 用 pinned 实例但 entry 记录 None，finalize/state 会对比**错误的 lower**。E2E 的 attach 断言（lower_revision 非空）抓出 | +| **host-side `.libra` metadata（§3.2）** | ✅ **已实现并 E2E** | 实现形态：mount 内 `.libra` = **一个符号链接** → `
/.libra/worktrees//`（host 侧 per-worktree gitdir，含 commondir/worktree_id/index/scorpiofs_mount_id）。内核在 FUSE 之前解析 symlink → Libra 命令直接读写 host 侧状态，**discovery 代码零改动**；upper 只含这一个指针 inode。配套：Libra layout 检测（`is_legacy_symlink_worktree` + `detect_entry_layout`）学会区分「指向隔离 host gitdir 的 symlink（含 worktree_id 文件）」与「legacy 主库 symlink」——否则 mutation 门会 fail-closed 误拒 | +| **`POST /worktrees/{id}/commit-finalize`** | ✅ 已实现并 E2E | 事务：A 段（generation 乐观锁、base 校验、构建新 pinned Dicfuse、**逐路径哈希验证 committed 内容**）→ B 段（unmount → 精确删除 committed upper 条目含 whiteout + 空父目录剪枝 → remount → 换 entry）→ 失败回滚旧 lower + 幂等重试；`GENERATION_CHANGED`/`BASE_MISMATCH`/`TREE_MISMATCH`/`SWITCH_FAILED` 错误码 | +| `POST /worktrees/{id}/refresh` | ✅ 已实现 | require_clean → BlockedDirty；lower 切换到目标/最新内部 revision；状态机 Quiescing→Ready | +| 能力协商 | ✅ | `/health` 增 `worktree.state.v2`、`worktree.commit-finalize.v2`、`worktree.refresh.v2` | +| 重启恢复 | ✅ | `PersistedMountState.pinned_refs`；恢复时 pinned mount 用 pinned 实例重挂（防静默回退到移动 tip） | +| **E2E：v2 finalize 全链路** | ✅ **全部通过** | 挂载→（改/增/删三 kinds）→ state 报正确三种变更+正确 OID → git clone 复刻提交并 push → finalize → `lower pinned`、cleaned=3、upper 清空、mount 从新 lower 供给内容、dirty=false；过期 generation 正确拒绝 | +| **E2E：libra sync 接入 finalize** | ✅ **全部通过** | `worktree add --backend scorpiofs`（job_id 加随机后缀——修复确定性 job_id 幂等命中旧 mount 导致的 base 冲突）→ mount 内编辑 → `libra sync`（state→add→commit→push→finalize）→ 远端 tip 前进、dirty=false、lower 钉到新内部 revision | +| 修复的真实缺陷：确定性 job_id | ✅ 已修 | 同路径重复注册会幂等命中旧 mount（其 base/lower 与新 HEAD 不符）→ add 400。改为每次注册唯一 job_id；崩溃残留由 repair/GC 清理（spec Q10 的实现注记） | +| **`POST /worktrees`（v2 attach）** | ✅ 已实现并 E2E | 单调用 attach：**挂载即钉定 lower**（client 传内部 OID 或留空由 daemon 解析 latest-commit），无 v1 的 tip 漂移窗口；daemon 不写任何 VCS metadata 进 mount。Libra attach 已迁移至该端点 | +| **host-side `.libra` metadata（§3.2）** | ✅ **已实现并 E2E** | mount 内 `.libra` = 一个符号链接 → `
/.libra/worktrees//`（host 侧 gitdir）。内核在 FUSE 前解析 symlink，Libra 命令直接读写 host 侧，discovery 零改动；upper 只含这一个指针 inode。配套：Libra layout 检测识别「symlink 目标含 worktree_id = 隔离布局」，否则 mutation 门误拒 | +| **chain 模式（ADR-1 封存-接链）** | ✅ **已实现并 E2E** | `fork mode=chain`：源 upper 原子改名封存为共享只读层 + 父会话换新空 upper（**零拷贝、父可继续写**）；子 lower = [sealed] + 源链；`CreateMountRequest.sealed_chain`（内部字段）+ `AntaresFuse::with_frozen_layers` 多层 lower；链感知 effective diff（最近优先 + 白障穿透）；finalize/refresh 将链**摊平进 upper** 再切 lower（陈旧链不再遮挡新投影），摊平后链清空；delete_mount 回收无引用 sealed 层；源未钉定时安全降级 materialize 并回报。E2E：子继承父编辑/新增、双向不泄漏、父保持可写、child sync 推回远端 | +| 修复的真实缺陷：链泄漏父指针 | ✅ 已修 | sealed 层携带父 `.libra` symlink → 子经 overlay 解析到**父的** gitdir（错 index/身份）。修复：封存时摘除指针、父新 upper 重建同目标 symlink、子视图天然无指针（E2E EEXIST 抓出） | + +--- + +## 0. 一句话结论 + +```text +Libra 管 VCS 状态和 Git objects/pack +ScorpioFS 管 POSIX 文件视图和 upper +Mega 管远端 refs、Git pack、revision tree/blob + +每个 ScorpioFS mount 内不放 .libra +每个 linked worktree 的 .libra 放在 host-side metadata store +``` + +工作区同步的完成条件： + +```text +commit 成功 ++ push 成功 ++ ScorpioFS lower 切换到新 commit ++ 已提交 upper 安全清理 += sync 成功 +``` + +`push` 成功但 lower 没切换，不能报告 sync 成功。 + +--- + +# 1. 问题与设计原则 + +## 1.1 当前冲突 + +原始方案把 `.libra` 放入 ScorpioFS mount 内： + +```text +/worktree/.libra/ +/worktree/src/ +``` + +这会冲突： + +1. ScorpioFS mount 要求目标目录为空； +2. Libra worktree 初始化会先写 `.libra`、index、HEAD scope； +3. mount 之后 host-side `.libra` 会被 FUSE 覆盖； +4. `.libra` 会被误认为普通 upper 文件； +5. 按需加载 lower 不应该负责加载 Libra 的 index/objects/refs； +6. fork、remount、恢复时会出现 host metadata 与 FUSE view 双真相。 + +## 1.2 设计原则 + +1. **VCS metadata 与 POSIX projection 分离**。 +2. **Git pack 只在 Libra ↔ Mega 之间传输**。 +3. **ScorpioFS 不实现 Git object/index/commit/push**。 +4. **每个 worktree 不复制 objects，只共享主 Libra store**。 +5. **upper 是相对于 base revision 的临时覆盖层，不是永久 commit 存储**。 +6. **effective diff 由 upper 与 base tree 比较产生，不能把所有 upper entry 都当成修改**。 +7. **commit finalize 必须是幂等事务**。 +8. **materialize 是默认 fork 模式；chain 不是 v1 必需能力**。 + +--- + +# 2. 三层架构 + +```text +┌────────────────────────────────────────────────────────────┐ +│ Libra │ +│ │ +│ VCS 控制面 │ +│ HEAD / refs / index / objects / pack / commit / push │ +│ worktree registry / private worktree scope │ +│ worktree add / fork / sync │ +└───────────────┬────────────────────────────────────────────┘ + │ Worktree Control Protocol v2 + │ HTTP/JSON + ▼ +┌────────────────────────────────────────────────────────────┐ +│ ScorpioFS │ +│ │ +│ POSIX/FUSE 数据面 │ +│ upper(rw) + optional CL + revision-addressed lower(ro) │ +│ whiteout / effective diff / lower switch / fork materialize │ +└───────────────┬────────────────────────────────────────────┘ + │ Revision Projection API + ▼ +┌────────────────────────────────────────────────────────────┐ +│ Mega │ +│ │ +│ Git Smart HTTP / refs / upload-pack / receive-pack │ +│ revision tree / blob / content-hash │ +└────────────────────────────────────────────────────────────┘ +``` + +--- + +# 3. Libra metadata布局 + +## 3.1 主 Libra store + +```text +/.libra/ +├── libra.db +├── objects/ +│ ├── +│ └── pack/ +├── vault.db +├── hooks/ +└── worktrees.json +``` + +主 store 保存： + +```text +objects / pack +refs / HEAD 的 SQLite rows +config +vault +worktree registry +``` + +## 3.2 linked worktree metadata + +```text +/.libra/worktrees// +├── commondir +├── worktree_id +└── index +``` + +说明： + +- `commondir`：指向主 `.libra` store； +- `worktree_id`：稳定 linked worktree identity； +- `index`：当前 worktree 私有 index； +- 不放 `objects/`； +- 不放 `pack/`； +- 不放 `vault.db`； +- 不放在 ScorpioFS mount 内。 + +## 3.3 ScorpioFS mount 内 + +```text +/ +├── src/ +├── docs/ +└── ... +``` + +默认不出现： + +```text +.libra/ +.git/ +objects/ +pack/ +refs/ +``` + +ScorpioFS mount 只提供代码文件和 POSIX 语义。 + +## 3.4 当前 worktree 的发现 + +Libra 不再依赖： + +```text +cwd/.libra +``` + +而是统一使用： + +```rust +resolve_current_worktree() -> WorktreeContext +``` + +解析顺序： + +```text +1. canonicalize 当前 cwd +2. 读取 main .libra/worktrees.json +3. 找到最长匹配的 worktree path +4. 得到 worktree_id +5. 打开 .libra/worktrees//index +6. 读取 SQLite 中该 worktree scope 的 HEAD/refs +7. 读取 ScorpioFS backend/mount_id metadata +``` + +因此，`libra status`、`add`、`commit`、`push`、`fork`、`sync` 必须全部使用同一个 `WorktreeContext` resolver。 + +--- + +# 4. Git objects 与 pack 协议 + +## 4.1 Pack 协议边界 + +```text +Libra ───── Git Smart HTTP ───── Mega +``` + +ScorpioFS 不参与： + +```text +upload-pack +receive-pack +pack encode/decode +pack index +ref negotiation +``` + +## 4.2 Fetch + +```text +libra fetch origin +``` + +流程： + +```text +1. Libra 读取 remote URL +2. 请求 info/refs +3. 请求 upload-pack +4. 接收 pack stream +5. 解码 commit/tree/blob +6. 写入主 .libra/objects 或 pack +7. 更新 refs/remotes/origin/* +``` + +之后所有 linked worktree 共享这些对象。 + +## 4.3 Push + +```text +1. Libra 从当前 index/HEAD 构造 tree +2. 创建 commit object +3. 计算 commit OID +4. 找到远端缺失对象 +5. 生成 pack +6. 调用 receive-pack +7. 更新远端 ref +``` + +commit OID 由 Libra 本地确定性计算： + +```text +commit = hash( + "commit " + canonical_commit_payload_length + "\0" + payload +) +``` + +其中 payload 包含： + +```text +tree +parent +author +committer + + +``` + +hash 算法由： + +```text +core.objectformat = sha1 | sha256 +``` + +决定。 + +## 4.4 “最新 commit”不是 ScorpioFS 计算的 + +需要区分： + +```text +remote latest commit = Mega refs/heads/ +worktree base = ScorpioFS 当前 base_revision +new local commit = Libra 根据 index/tree/parent 创建的 commit +``` + +获取远端 branch 最新 commit： + +```text +libra fetch origin +读取 refs/remotes/origin/main +``` + +获取当前 worktree base： + +```text +GET /worktrees/{id}/state +读取 base_revision +``` + +计算本次新 commit： + +```text +ScorpioFS effective diff + ↓ +Libra add 更新 private index + ↓ +index 构造完整 tree T + ↓ +parent B + tree T + metadata + ↓ +Libra 计算 commit C +``` + +因此： + +```text +ScorpioFS 不决定 latest commit +ScorpioFS 不生成 commit OID +Libra 不从文件 mtime 推断 commit +``` + +--- + +# 5. Mega Revision Projection API + +Git pack 解决 VCS 对象传输；ScorpioFS 需要的是按 revision 读取文件树。 + +Mega 提供或适配以下能力： + +```http +GET /api/v1/tree?path=&revision= +GET /api/v1/tree/content-hash?path=&revision= +GET /api/v1/blob/ +``` + +## 5.1 ScorpioFS lower 读取流程 + +```text +open("src/main.rs") + ↓ +ScorpioFS 根据 base_revision 查询 tree entry + ↓ +得到 blob_oid / size / mode / content_hash + ↓ +按需读取 blob + ↓ +校验 content_hash + ↓ +返回 POSIX read +``` + +## 5.2 Libra 读取对象的优先级 + +Libra 自己需要 commit/tree/blob 时： + +```text +1. 本地 .libra/objects +2. 本地 pack/index +3. 本地 alternates（如果启用） +4. fetch/upload-pack +5. remote revision API（只作为受控读取适配器，不替代对象存储） +``` + +ScorpioFS 读取工作区文件时： + +```text +1. upper +2. lower cache +3. Mega revision tree/blob API +``` + +两个 resolver 不混合。 + +--- + +# 6. Worktree Control Protocol v2 + +有效基础路径： + +```text +/antares/worktrees +``` + +旧的 `/antares/mounts` 保留为兼容别名。 + +## 6.1 Capability + +```http +GET /health +``` + +最低能力： + +```json +{ + "protocol": "worktree.v2", + "capabilities": [ + "worktree.attach.v2", + "worktree.state.v2", + "worktree.commit.v2", + "worktree.refresh.v2", + "worktree.fork.materialize.v2", + "lower.switch.v2", + "whiteout.oci.v1" + ] +} +``` + +## 6.2 Attach + +```http +POST /antares/worktrees +``` + +```json +{ + "worktree_id": "wt-123", + "repo_id": "repo-a", + "repo_path": "/project", + "mountpoint": "/workspace/feature-a", + "base_revision": "" +} +``` + +ScorpioFS 必须： + +1. 检查 mountpoint 不存在或为空； +2. 创建 upper； +3. 将 lower 固定到 `base_revision`； +4. mount FUSE； +5. 保存 worktree_id、repo_id、base_revision、generation； +6. 不写 `.libra` 到 mountpoint。 + +响应： + +```json +{ + "worktree_id": "wt-123", + "mount_id": "mount-456", + "base_revision": "", + "generation": 1, + "state": "ready" +} +``` + +## 6.3 State / effective diff + +```http +GET /antares/worktrees/{worktree_id}/state +``` + +```json +{ + "worktree_id": "wt-123", + "mount_id": "mount-456", + "base_revision": "B", + "generation": 42, + "state": "dirty", + "changes": [ + { + "path": "src/main.rs", + "kind": "modified", + "content_hash": "sha256:new", + "base_hash": "sha256:old" + }, + { + "path": "src/old.rs", + "kind": "deleted", + "content_hash": null, + "base_hash": "sha256:old" + }, + { + "path": "src/new.rs", + "kind": "added", + "content_hash": "sha256:new", + "base_hash": null + } + ] +} +``` + +规则： + +```text +upper file == base content -> clean +upper file != base content -> modified +upper file + no base -> added +whiteout + base exists -> deleted +whiteout + no base -> no-op +``` + +`generation` 只在 effective diff 改变时递增。 + +## 6.4 Commit finalize + +```http +POST /antares/worktrees/{worktree_id}/commit-finalize +``` + +```json +{ + "expected_base_revision": "B", + "new_base_revision": "C", + "new_tree": "T", + "expected_generation": 42, + "committed_paths": [ + "src/main.rs", + "src/old.rs", + "src/new.rs" + ], + "committed_hashes": { + "src/main.rs": "sha256:new", + "src/old.rs": null, + "src/new.rs": "sha256:new" + } +} +``` + +ScorpioFS 必须按顺序： + +```text +1. 校验 base_revision +2. 校验 generation +3. 验证 new_tree / committed_hashes +4. 进入 switching +5. 切换 lower 到 C +6. 删除 committed_paths 对应的 upper/whiteout +7. 更新 base_revision=C +8. 返回 ready/clean +``` + +失败时： + +```text +不删除 upper +不推进 base_revision +不报告 clean +进入 recovery/conflict +``` + +该接口必须幂等：重复提交同一个 `C` 可以继续完成上次未完成的 lower switch/upper cleanup。 + +## 6.5 Refresh + +```http +POST /antares/worktrees/{worktree_id}/refresh +``` + +```json +{ + "expected_base_revision": "B", + "target_revision": "T", + "require_clean": true +} +``` + +- clean：切换 lower 到 T； +- dirty：返回 `blocked_dirty`； +- base 不匹配：返回 `base_mismatch`； +- 切换失败：保留 upper，进入 recovery。 + +## 6.6 Fork + +```http +POST /antares/worktrees/{worktree_id}/fork +``` + +```json +{ + "mountpoint": "/workspace/child", + "mode": "materialize" +} +``` + +处理： + +1. 读取 source effective upper delta； +2. 在 child FUSE session 启动前复制 delta； +3. child 继承 source base_revision； +4. child 创建独立 upper； +5. 返回 child mount_id。 + +v1 只实现 `materialize`。`chain` 不作为必要协议能力。 + +--- + +# 7. Libra 命令映射 + +## 7.1 `worktree add --backend scorpiofs` + +```text +Libra 注册 worktree scope + ↓ +不 restore host tree + ↓ +POST worktrees attach + ↓ +ScorpioFS 直接挂到目标路径 + ↓ +Libra 写 host-side private index +``` + +mount 内没有 `.libra`。 + +## 7.2 `libra fork` + +```text +Libra registry resolver 找到当前 worktree + ↓ +读取 mount_id + ↓ +POST worktrees/{id}/fork + ↓ +注册 child metadata + ↓ +写 child private index +``` + +## 7.3 `libra sync` + +```text +1. resolve_current_worktree() +2. GET state/effective diff +3. libra add -A +4. libra commit -> C/tree(C) +5. libra push -> Mega +6. POST commit-finalize(C) +7. ScorpioFS lower switch + upper cleanup +``` + +只有第 6 步成功，sync 才返回成功。 + +--- + +# 8. Failure and recovery + +## 8.1 Push 成功，finalize 失败 + +```text +remote/main = C +local base = B +upper 保留 +state = recovery +``` + +允许重试： + +```http +POST commit-finalize(C) +``` + +## 8.2 finalize 期间文件再次变化 + +```text +expected_generation != current_generation +``` + +返回： + +```json +{ + "state": "conflict", + "code": "GENERATION_CHANGED" +} +``` + +不得清理新 upper 内容。 + +## 8.3 upper 与 base 同内容 + +upper 可以存在，但 effective diff 必须为空： + +```text +upper/a == base/a +=> no change +``` + +可以在下一次 successful finalize 或后台 compaction 时安全删除该冗余 upper。 + +--- + +# 9. 迁移旧协议 + +| v1 接口 | v2 语义 | +|---|---| +| `POST /mounts` | `POST /worktrees` attach | +| `GET /mounts/{id}/worktree` | `GET /worktrees/{id}/state` | +| `POST /worktree/base` | attach 的一部分 | +| `POST /worktree/refresh-plan` | refresh 的 preflight/执行合并 | +| `POST /fork` | materialize fork | +| 新增 | `POST /worktrees/{id}/commit-finalize` | + +迁移期间： + +```text +v1 client 可以继续读写 +v2 client 才能获得 lower switch + upper cleanup 的完整一致性保证 +``` + +--- + +# 10. 最终简化边界 + +```text +Git pack: + Libra ↔ Mega + +Revision tree/blob: + ScorpioFS ↔ Mega + +Worktree control: + Libra ↔ ScorpioFS + +VCS metadata: + host-side Libra .libra + +POSIX workspace: + ScorpioFS mount，不包含 .libra +``` + +这套设计避免了： + +- `.libra` 被 FUSE mount 隐藏； +- 按需加载把 VCS metadata 当成代码文件； +- upper 文件与 lower 相同时产生假 dirty； +- commit 后旧 upper 永久遮挡新 lower； +- ScorpioFS 重复实现 Git pack； +- push 成功但本地 worktree 仍停留在旧 revision。 diff --git a/docs/scorpiofs-libra-protocol-v2.md b/docs/scorpiofs-libra-protocol-v2.md new file mode 100644 index 0000000..3098a53 --- /dev/null +++ b/docs/scorpiofs-libra-protocol-v2.md @@ -0,0 +1,591 @@ +# ScorpioFS × Libra 简化架构与传输协议 v2 + +- **版本**：v2.0.0-draft +- **状态**：设计稿，作为现有 `docs/worktree-api.md`、MST/2 workspace 接口和当前实现的收敛版。 +- **目标**：减少控制面接口数量，明确 Git pack 与 POSIX 文件传输的边界，并解决 upper 提交后继续遮挡 lower 的一致性问题。 + +--- + +## 1. 核心结论 + +### 1.1 Libra 是 VCS 权威 + +Libra 唯一拥有： + +```text +HEAD +index +refs +objects / pack +commit / tree / blob +fetch / push +merge / rebase / conflict stages +``` + +### 1.2 ScorpioFS 是工作区数据面 + +ScorpioFS 只拥有： + +```text +FUSE/POSIX mount +Dicfuse lower projection +Antares upper layer +whiteout +changed-path/effective-diff scan +lower revision switch +``` + +ScorpioFS 不拥有： + +```text +Git objects +Git pack +refs +HEAD +index +credentials +commit/push protocol +``` + +### 1.3 worktree 不复制 Git objects + +主 Libra store： + +```text +main/.libra/ +├── libra.db +├── objects/ +├── pack/ +├── vault.db +└── worktrees.json +``` + +linked/ScorpioFS worktree： + +```text +worktree/.libra/ +├── commondir +├── worktree_id +├── index +└── scorpiofs_mount_id +``` + +**每个 worktree 不保存 `objects/` 或 pack。** 所有 worktree 共享主 Libra store 的对象。 + +--- + +## 2. 三条完全分离的数据链 + +```text +┌─────────────────────────────────────────────────────────┐ +│ Git object/pack plane │ +│ Libra ───── Git Smart HTTP upload-pack/receive-pack ─▶ Mega │ +└─────────────────────────────────────────────────────────┘ + +┌─────────────────────────────────────────────────────────┐ +│ POSIX file plane │ +│ ScorpioFS ───── tree/blob/content-hash ───────────────▶ Mega │ +└─────────────────────────────────────────────────────────┘ + +┌─────────────────────────────────────────────────────────┐ +│ Worktree control plane │ +│ Libra ─────────── HTTP/JSON ─────────────────────────▶ ScorpioFS │ +└─────────────────────────────────────────────────────────┘ +``` + +### 2.1 Git pack 不经过 ScorpioFS + +`fetch`： + +```text +Libra -> upload-pack -> Mega + -> 解码 pack + -> 写入主 .libra/objects/pack + -> 更新 refs +``` + +`push`： + +```text +Libra -> 构造 commit/tree/blob + -> 生成 pack + -> receive-pack -> Mega +``` + +ScorpioFS 只从 Mega 读取指定 revision 的 tree/blob 内容，用于提供文件视图。它不理解 pack。 + +### 2.2 两条 Mega API 可以并存 + +| 用途 | 协议 | +|---|---| +| Libra 对象同步 | Git Smart HTTP / pack | +| ScorpioFS 文件投影 | `/api/v1/tree`、`/api/v1/blob`、`content-hash` | + +这是有意的双协议，不是重复实现： + +```text +pack = VCS 对象传输 +blob/tree = 文件系统按需读取 +``` + +--- + +## 3. 单一 Worktree 控制协议 + +有效前缀： + +```text +/antares/worktrees +``` + +现有 `/antares/mounts` 可以保留为兼容别名，但新客户端使用 `/worktrees` 语义。 + +### 3.1 Capability + +```http +GET /health +``` + +v2 服务至少声明： + +```json +{ + "protocol": "worktree.v2", + "capabilities": [ + "worktree.attach.v2", + "worktree.state.v2", + "worktree.commit.v2", + "worktree.fork.materialize.v2", + "whiteout.oci.v1", + "lower.switch.v2" + ] +} +``` + +如果缺少 `lower.switch.v2`，客户端必须拒绝把 commit 宣布为完整同步成功。 + +--- + +## 4. Worktree 生命周期 + +```text +Absent + │ attach + ▼ +Ready ── local write/delete ──▶ Dirty + │ │ + │ commit finalize │ commit finalize + ▼ ▼ +Switching ───────────────────▶ Ready + │ + └── failure ──▶ Conflict/Recovery +``` + +核心不变量： + +1. 一个 mount 只有一个 `base_revision`。 +2. `base_revision` 对应当前 lower 的真实 revision，不只是备注字段。 +3. upper 中与 base 内容相同的文件不算 dirty。 +4. commit finalize 成功前，upper 不删除任何内容。 +5. lower 切换失败时，HEAD/index/upper 都保持不变。 +6. upper 里的已提交内容不能无限期遮挡新的 lower。 + +--- + +## 5. v2 接口 + +### 5.1 Attach + +```http +POST /antares/worktrees +``` + +```json +{ + "worktree_id": "wt-123", + "repo_path": "/project", + "mountpoint": "/workspace/feature", + "base_revision": "", + "commondir": "/repo/main/.libra", + "mode": "materialize" +} +``` + +语义： + +1. `mountpoint` 必须不存在或为空目录； +2. ScorpioFS 创建 Antares mount； +3. lower 固定到 `base_revision`； +4. 写入 `.libra/commondir` 和 `.libra/worktree_id`； +5. 返回 `mount_id`、`base_revision`、`generation`。 + +```json +{ + "worktree_id": "wt-123", + "mount_id": "...", + "base_revision": "", + "generation": 1, + "state": "ready" +} +``` + +### 5.2 State / effective diff + +```http +GET /antares/worktrees/{worktree_id}/state +``` + +```json +{ + "worktree_id": "wt-123", + "mount_id": "...", + "base_revision": "", + "generation": 42, + "state": "dirty", + "changes": [ + { + "path": "src/main.rs", + "kind": "modified", + "content_hash": "sha256:...", + "base_hash": "sha256:..." + }, + { + "path": "src/old.rs", + "kind": "deleted", + "content_hash": null, + "base_hash": "sha256:..." + }, + { + "path": "src/new.rs", + "kind": "added", + "content_hash": "sha256:...", + "base_hash": null + } + ] +} +``` + +#### 关键变化：返回 effective diff，而不是 upper entry 列表 + +当前 upper 目录里有文件，不代表它一定相对于 lower 有变化： + +```text +upper/a = X +lower/a = X +``` + +v2 必须返回： + +```text +没有 change +``` + +而不是简单地把所有 upper 文件都标记为 modified。 + +effective diff 的判定： + +| upper/lower 情况 | 结果 | +|---|---| +| upper 文件存在，hash 等于 base | clean/no change | +| upper 文件存在，hash 不等于 base | modified | +| upper 文件存在，base 不存在 | added | +| OCI whiteout，base 存在 | deleted | +| OCI whiteout，base 不存在 | no-op，可清理 | + +`generation` 只在 effective diff 改变时递增。 + +### 5.3 Commit finalize + +```http +POST /antares/worktrees/{worktree_id}/commit +``` + +请求： + +```json +{ + "expected_base_revision": "", + "new_base_revision": "", + "commit_tree": "", + "expected_generation": 42, + "committed_paths": [ + "src/main.rs", + "src/old.rs", + "src/new.rs" + ] +} +``` + +这是 v2 最重要的接口。 + +服务端必须按以下顺序处理： + +```text +1. 校验 expected_base_revision == 当前 base_revision +2. 校验 expected_generation == 当前 generation +3. 读取 new_base_revision 对应的 tree/content hashes +4. 确认 committed_paths 在新 tree 中的内容一致 +5. 进入 Switching +6. 切换 Dicfuse lower 到 new_base_revision +7. 只删除 committed_paths 对应的 upper entries/whiteouts +8. 更新 base_revision = new_base_revision +9. generation += 1 +10. 返回 Ready/Clean +``` + +成功响应： + +```json +{ + "state": "ready", + "base_revision": "", + "generation": 43, + "cleaned_paths": [ + "src/main.rs", + "src/old.rs", + "src/new.rs" + ] +} +``` + +失败响应： + +```json +{ + "state": "conflict", + "code": "BASE_CHANGED|GENERATION_CHANGED|TREE_MISMATCH|SWITCH_FAILED", + "base_revision": "", + "generation": 42 +} +``` + +失败时： + +```text +不删除 upper +不推进 base_revision +不伪造 clean 状态 +``` + +### 5.4 Refresh / pull + +```http +POST /antares/worktrees/{worktree_id}/refresh +``` + +```json +{ + "expected_base_revision": "", + "target_revision": "", + "require_clean": true +} +``` + +处理规则： + +```text +clean -> 切换 lower 到 target_revision + dirty -> blocked_dirty + base 不匹配 -> base_mismatch +``` + +`refresh` 与 `commit` 都使用同一个 lower switch transaction，不再维护独立的“只做计划但不执行”的半协议。 + +### 5.5 Fork + +```http +POST /antares/worktrees/{worktree_id}/fork +``` + +```json +{ + "mountpoint": "/workspace/child", + "mode": "materialize" +} +``` + +当前唯一支持： + +```text +materialize +``` + +处理： + +1. 读取父 effective upper delta； +2. 在 child mount 启动前复制到 child upper； +3. child 继承相同 base_revision； +4. child 使用独立 upper； +5. 父后续写入不影响 child。 + +`chain` 暂不属于 v2 必需能力，避免为性能优化引入另一套持久化和恢复模型。 + +--- + +## 6. 简化后的 Libra 命令 + +### 6.1 `worktree add --backend scorpiofs` + +```text +Libra 登记 linked worktree + ↓ +POST worktrees attach + ↓ +目标目录被 ScorpioFS 挂载 + ↓ +写 pointer + private index +``` + +### 6.2 `fork` + +```text +Libra 读取当前 worktree mount_id + ↓ +POST worktrees/{id}/fork + ↓ +登记 child linked worktree + ↓ +写 child pointer + index +``` + +### 6.3 `sync` + +```text +1. GET state/effective diff +2. Libra add -A +3. Libra commit，得到 C/tree(C) +4. Libra push C 到 Mega +5. POST worktrees/{id}/commit +6. ScorpioFS 切 lower + 清 committed upper +``` + +注意： + +```text +push 成功 != worktree 同步完成 +``` + +只有 `POST .../commit` 成功，才算本地工作区完成同步。 + +如果 push 成功但 lower switch 失败： + +```text +远端已有 commit +本地 worktree 进入 Conflict/Recovery +upper 保留 +不能继续伪装成 clean +``` + +--- + +## 7. Upper 文件的正确语义 + +upper 不是 Git index，也不是永久提交记录。 + +它表示： + +```text +当前 base_revision 之上的未确认工作区覆盖层 +``` + +因此每个 upper path 逻辑上应携带： + +```text +base_revision_at_copyup +current_content_hash +kind +``` + +如果无法在首次 copy-up 时保存 base hash，至少在 `GET state` 时使用： + +```text +Mega tree/content-hash(base_revision, path) +``` + +比较 upper 与 base。 + +禁止以下简化： + +```text +upper 有文件 => 一定 modified +sync 成功 => 直接清 upper +push 成功 => 直接推进 base_revision +``` + +这三条都会制造一致性错误。 + +--- + +## 8. 最小实现范围 + +v2 只要求： + +```text +attach +state/effective diff +commit finalize +refresh +materialize fork +``` + +明确不要求： + +```text +chain frozen layers +ScorpioFS 自己实现 Git pack +ScorpioFS 自己实现 commit/push +透明 FUSE 读时 Git object hydration +``` + +这样系统只有三条清晰边界： + +```text +Git pack Libra ↔ Mega +文件 projection ScorpioFS ↔ Mega +worktree state Libra ↔ ScorpioFS +``` + +--- + +## 9. 迁移现有协议 + +现有接口保留兼容： + +| 旧接口 | v2 映射 | +|---|---| +| `POST /mounts` | `POST /worktrees` | +| `GET /worktree` | `GET /worktrees/{id}/state` | +| `POST /worktree/base` | attach 的一部分 | +| `POST /worktree/refresh-plan` | `POST /worktrees/{id}/refresh` 的 preflight/执行合并 | +| `POST /fork` | 保留，增加 `materialize` 明确语义 | +| 新增 `POST /worktrees/{id}/commit` | commit 后 lower switch + upper cleanup | + +迁移期间： + +```text +v1 client -> 继续使用旧 API，但不能声称 commit 后 lower 已同步 +v2 client -> 使用 commit finalize，得到完整一致性保证 +``` + +--- + +## 10. 结论 + +简化后的正确模型是： + +```text +Libra 管 VCS 对象和 pack +ScorpioFS 管 POSIX 文件和 upper +Mega 管远端仓库与按 revision 读取 + +commit 的完成条件不是 push 成功， +而是 push 成功 + lower switch 成功 + committed upper 清理成功。 +``` + +这样可以同时解决： + +- upper 文件与 lower 内容相同导致的假 dirty； +- commit 后 upper 永久残留； +- 旧 upper 遮挡远端新 lower； +- push 成功但本地工作区仍显示旧内容； +- ScorpioFS 和 Libra 各自重复实现 Git pack 的问题。 diff --git a/docs/scorpiofs-tree-pack-sync-spec-v1.md b/docs/scorpiofs-tree-pack-sync-spec-v1.md new file mode 100644 index 0000000..0dec3e0 --- /dev/null +++ b/docs/scorpiofs-tree-pack-sync-spec-v1.md @@ -0,0 +1,201 @@ +# ScorpioFS Tree-Pack 同步（commit+tree 一次下载）SPEC + +- **版本**：v1.0.0（设计稿，未实现） +- **日期**：2026-09-22 +- **状态**：待评审。本文只做设计与实测论证；实现分两步（§5 mega2 侧、§6 ScorpioFS 侧）。 +- **关联**：`docs/scorpiofs-libra-complete-spec-v1.md`（Worktree v2 协议）、`docs/scorpiofs-libra-protocol-v2.md` + +--- + +## 1. 背景与实测 + +ScorpioFS 当前的 lower 投影是**逐目录惰性拉取**：`Dicfuse` 按需对每个目录调用 +`GET /api/v1/tree`（`fetch_dir`），一次 HTTP 往返换取一个目录的条目。挂载/refresh +之后第一次全树浏览要付出 N 次往返（N = 目录数）。 + +实测（本机 WSL2 + docker mega2，monorepo `/project` = mega + rk8s 源码； +2783 个 tree / ~5140 个 blob / 5177 个文件 / 2743 个目录）： + +| 场景 | 耗时 | 传输 | +|---|---|---| +| `git clone`（全对象，一次 pack） | **4.9s** | 8.7 MB pack | +| `git clone --filter=blob:none`（mega2 忽略 filter，退化为全量） | 2.8s | 8.6 MB pack，**5126 个 blob 仍被下发** | +| ScorpioFS 冷全树 metadata 遍历（2743 目录逐个 fetch_dir） | **22.2s** | 2743 次 HTTP，每次仅一个目录 | +| ScorpioFS 热遍历（store 已加载） | 2.5s | 0 | +| ScorpioFS 单目录 fetch 延迟 | ~8ms | 本地 docker；广域部署会显著放大 | + +结论： +1. **一次 pack 传输完胜逐目录往返**——git 全量（含全部 blob）都比 ScorpioFS 的纯 + metadata 惰性遍历快 4.5 倍。 +2. **逐目录方案的成本随目录数线性增长**，且对网络延迟敏感（本机 8ms，远程 50ms+ + 时 22s 会变成分钟级）。 +3. mega2 目前**不支持** git partial clone 的 object filter（实测 `--filter=blob:none` + 被忽略、仍下发全部 blob；协议调查见 §5.1）。 + +## 2. 目标 / 非目标 + +**目标** +- G1：挂载或 refresh 后，**全树的 commit+tree 元数据通过一次（或极少次）pack 传输 + 到位**；之后任意路径的 readdir/lookup 零网络。 +- G2：**异步**——pack 下载与导入不阻塞挂载 Ready，也不阻塞 commit-finalize； + 导入完成前既有惰性 `fetch_dir` 路径照常兜底，两层并存、去重。 +- G3：blob 继续**按需**逐个拉取（`fetch_file_content` 语义不变）；pack 里不含 blob。 +- G4：mega2 侧实现 **git 标准的 partial clone filter**（`blob:none` / `blob:limit`）， + 让标准 git 客户端同样受益，而非私有端点。 +- G5：优雅降级——对端 mega2 不支持 filter 时，ScorpioFS 行为退化为现状（惰性 + + 可选的低深度预热），不报错。 + +**非目标** +- 不做 blob 的批量预取/缓存预热（blob:limit 可作为后续增强）。 +- 不改变 Worktree v2 的挂载/finalize/refresh 语义与 generation 乐观锁。 +- 不改变 upper/CL/frozen 层次与白障语义（pack 只影响 lower 的加载速度）。 + +## 3. git partial clone 原理（协议对照） + +1. **能力广告**：upload-pack 在 capability 列表中广告 `filter`；客户端未见 `filter` + 即退化为全量 clone（实测正是如此：blobless clone 拿到 5126 个 blob）。 +2. **请求**：`git clone --filter=blob:none` 的客户端在 want 行后发送 `filter blob:none`。 +3. **服务端 pack 生成**：遍历 want 对象图时按 filter 排除对象——`blob:none` 时 + traverse 到 blob 直接跳过（tree 条目引用 blob OID 但不递归、不下发）。 +4. **filter 语法族**：`blob:none`（只要 commit/tag/tree）、`blob:limit=`（小 blob + 照发）、`tree:`（连 tree 都省，tree:0 只发 commit）、`object:type=`、 + `combine:+`。 +5. **客户端 promisor 机制**：clone 产物为 partial repo（`extensions.partialclone`）， + checkout/log/diff 缺失 blob 时向 promisor remote 发批量 fetch 并缓存；对上层透明。 + +ScorpioFS 的数据模型（tree 全量、blob 按需）与 `blob:none` 语义天然对齐。 + +## 4. 总体方案 + +``` +attach / refresh + │ + ├─(同步，现状不变)── wait_for_ready(root) → FUSE Ready + │ │ + │ 用户请求 → 惰性 fetch_dir 兜底（现状） + │ + └─(异步，新增)──── tree-pack sync worker + │ ① GET pack（commit+tree only） + │ ② 解析 pack → 对象 + │ ③ 构建 per-revision store 的路径↔inode↔条目 + │ ④ 完成标记（store 元数据），后续 readdir 零网络 + ▼ + blob 仍按需 fetch_file_content +``` + +- pack 导入**不改变** store 的对外语义：导入前后 `fetch_dir` / `get_inode` 返回 + 同样的内容；导入只是把"将来要惰性拉的目录"提前批量填好。 +- 并发安全：导入走 store 现有的 `dir_locks` / `ensure_dir_loaded` 通道（与惰性 + fetch 同一把锁），或以"先到先得、后到跳过"的条目合并规则写入（§6.3）。 + +## 5. mega2 侧：实现 git 标准 object filter + +### 5.1 现状（协议调查结论） + +- Smart HTTP 入口：`mono/src/git_protocol/http.rs`（`git_info_refs` / `git_upload_pack` + / `git_receive_pack`），路由分发 `mono/src/server/http_server.rs:445`。 +- 能力广告仅 `multi_ack_detailed no-done include-tag side-band-64k ofs-delta + agent=mega/0.1.0`（`ceres/src/transport/protocol/smart.rs:34-41, 79-83`）。 +- upload-pack 命令循环只处理 want/have/done（`smart.rs:120-135`）。 +- **object filter：不存在**——Capability 枚举无 `filter`（`ceres/src/transport/ + protocol/mod.rs:86-117`），不广告、不解析。 +- **shallow clone：不存在**（`full_pack` 遍历全部父提交，`pack/monorepo.rs:236, + 262-279`）。 +- pack 生成：`ceres/src/transport/pack/`——trait `RepoHandler`（`mod.rs`，共享递归 + `traverse`:332）、`monorepo.rs::full_pack:236 / incremental_pack:240`；编码用 + git-internal 0.8.6 的 `PackEncoder::encode_async`（`mod.rs:320`）。 +- 树快照/批量 tree 导出 API：不存在（仅逐级 `GET /api/v1/tree*` 与原始单 tree 的 + `/api/v1/file/tree`）。 + +### 5.2 改动清单（标准 filter，非私有端点） + +| # | 改动 | 位置 | +|---|---|---| +| M1 | Capability 枚举增加 `Filter`；info/refs 广告 `filter` | `ceres/src/transport/protocol/mod.rs:86-117`、`smart.rs:34-41,79-83` | +| M2 | upload-pack 命令循环解析 `filter ` 行（session 状态携带 filter） | `smart.rs:120-135` | +| M3 | `full_pack` / `traverse` 接受 filter：`blob:none` 跳过 blob（不递归、不编码）；`blob:limit=` 按 size 判断 | `ceres/src/transport/pack/monorepo.rs:236`、`mod.rs:332` | +| M4 | （可选，本期不做）shallow/deepen | — | + +兼容性约束： +- 仅当客户端**显式发送** filter 才启用；不带 filter 的请求行为完全不变。 +- `blob:none` 的 pack 不含 blob → git 客户端标 promisor，checkout 时按需回取 + （走 mega2 现有 blob 端点，标准协议）。 +- LFS：>1MB 文件在 mega2 中本就以 LFS 指针形式存在，tree 条目引用的 blob 即指针 + 内容——filter 不影响指针下发（它也是 blob，只是小）。**ScorpioFS 不消费这些 + 指针 blob**（仍走按需），故语义无冲突。 + +### 5.3 验收 + +- `git clone --filter=blob:none $M2/project` 后 `git cat-file --batch-all-objects | + grep -c blob` 为 **0**（或 `blob:limit` 场景下符合 limit）。 +- pack 大小：全树 tree-only pack ≤ ~2MB（对照全量 8.7MB）。 +- 全量 clone（无 filter）回归不变。 + +## 6. ScorpioFS 侧：异步 tree-pack 导入器 + +### 6.1 触发点 + +- `create_mount` / `attach_worktree`：Ready 返回**之后**后台启动。 +- `commit_finalize` Phase B 与 `refresh_lower`：remount 完成后，对新 pinned store + 后台启动（同一 store 最多一个在跑，幂等去重）。 +- 手动触发（运维/测试）：`POST /antares/worktrees/{id}/sync-tree-pack`（可选， + 返回当前导入状态）。 + +### 6.2 下载与解析 + +- 下载：**复用 git 标准 Smart HTTP**——ScorpioFS 内嵌一个最小 upload-pack 客户端 + （want `` + `filter blob:none` + done），或直接 + 起子进程 `git fetch --filter=blob:none`（二选一，见 §7 决策点 D1）。 +- 解析：git-internal 已在依赖树内（`PackDecoder` / 对象遍历）。 +- 导入目标：per-revision `DictionaryStore`（`-rev-` 目录）。tree 对象按 + 「tree-hash → 目录条目」重建：与 `fetch_dir` 相同的 StorageItem 结构 + （inode/parent/name/children/hash），路径↔inode 映射沿用 radix_trie + 持久层。 + +### 6.3 并发与一致性 + +- 导入以 store 为单位持锁（`global_import_semaphore` 限并发，8 worker 解析）。 +- **先到先得合并**：某目录已被惰性 `fetch_dir` 加载 → 跳过（不覆盖 inode 分配， + 避免已打开 fd 的 inode 漂移）；未加载 → 批量填入。 +- tree 之外的对象（commit/tag）仅用于确定根与父子关系，不做 FUSE 投影。 +- 失败重试：指数退避（1s/4s/16s），三次后放弃并保留惰性路径（G5 降级）。 + +### 6.4 状态与可观测 + +- `GET /antares/worktrees/{id}/state` 增加可选字段： + `tree_pack: { state: idle|syncing|done|failed|unsupported, trees_imported, elapsed_ms }` + （`unsupported` = 对端无 filter 能力，已降级）。 +- daemon 日志：开始/完成/失败各一条 INFO/WARN，含 trees/blobs 计数。 + +### 6.5 与既有修复的关系 + +- 本方案**不回退**"pinned store 跳过深预热"的修复：该修复禁用的是同步整树 + `load_dir_depth`（会阻塞 Ready 且风暴式压垮远端）；tree-pack 是**异步单请求** + 传输，远端压力为一次顺序读，二者不冲突。 +- 保留现有惰性 fetch 与 `fetch_file_size` 重试：pack 导入完成前的兜底不变。 + +## 7. 决策点（评审时定） + +| # | 问题 | 候选 | 倾向 | +|---|---|---|---| +| D1 | ScorpioFS 侧 pack 客户端形态 | (a) 内嵌最小 upload-pack 客户端；(b) 子进程 `git fetch --filter=blob:none` + 解析本地对象 | (a)：避免外部 git 依赖，协议面窄（want+filter+done） | +| D2 | filter 能力探测 | 从 info/refs 的 capability 广告判断；无 `filter` → 降级 | 按标准协议探测 | +| D3 | pack 内容是否持久化复用 | per-revision store 已按 `-rev-` 目录持久化，导入结果随 store 复用 | 随 store 持久化，不额外存 pack | +| D4 | `blob:limit` 增强 | 后续版本 | 本期只做 `blob:none` | + +## 8. 预期收益（基于 §1 实测外推） + +| 指标 | 现状（惰性） | tree-pack 后 | +|---|---|---| +| 冷全树 metadata 就绪 | 22.2s（串行）/ 广域分钟级 | **≈1s**（传输 ~1-2MB + 解析），且后台异步 | +| 冷 readdir/lookup 网络往返 | 每目录 1 次 | 0（导入完成后） | +| blob 首次读 | 按需 1 次 HTTP | 不变 | +| mega2 远端负载 | 每目录 1 请求 × N 客户端 | 每客户端 1 请求 | + +## 9. 实施切分 + +1. **P1（mega2）**：M1-M3 + 验收（§5.3）。独立可发布，标准 git 客户端即可受益。 +2. **P2（ScorpioFS）**：下载/解析/导入（§6.2-6.3）+ 状态字段（§6.4）+ 降级（§2 G5）。 +3. **P3（可选）**：`blob:limit`、手动触发端点、`blob` 批量预热。 + +每步独立可回归：P1 落地后用标准 git 验证；P2 落地前后各跑一轮 +`bench-scorpio-vs-git.sh` 对比。 diff --git a/scorpio.toml b/scorpio.toml index e128354..9f8c34a 100644 --- a/scorpio.toml +++ b/scorpio.toml @@ -19,7 +19,11 @@ dicfuse_stat_mode = "accurate" dicfuse_open_buff_max_bytes = "268435456" dicfuse_open_buff_max_files = "4096" antares_load_dir_depth = "3" -antares_dicfuse_stat_mode = "fast" +# "fast" skips remote size probes, but then getattr reports size 0 for files +# never read in this revision; the kernel caches that attr and truncates every +# subsequent read to empty (git status sees bogus diffs against the index). +# "accurate" probes once per file per revision and caches the result in size.db. +antares_dicfuse_stat_mode = "accurate" antares_dicfuse_open_buff_max_bytes = "67108864" antares_dicfuse_open_buff_max_files = "1024" antares_dicfuse_dir_sync_ttl_secs = "120" diff --git a/src/antares/fuse.rs b/src/antares/fuse.rs index c157a5c..ee6b7c8 100644 --- a/src/antares/fuse.rs +++ b/src/antares/fuse.rs @@ -2,18 +2,102 @@ use std::{path::PathBuf, sync::Arc}; use asyncfuse::raw::{logfs::LoggingFileSystem, MountHandle}; use libfuse_fs::{ - passthrough::new_antares_passthroughfs_layer, + passthrough::{config::Config as PassthroughConfig, PassthroughFs}, unionfs::{config::Config, layer::Layer, OverlayFs}, + util::whiteout::WhiteoutFormat, }; use crate::{server::mount_filesystem_with_antares_cache, util::fuse_platform}; +/// Antares records deletions with the OCI whiteout form (`.wh.`) instead of the +/// Linux kernel-overlayfs character-device form. +/// +/// Two reasons, both load-bearing: +/// +/// 1. A char-device whiteout is created with `mknod`, which requires `CAP_MKNOD`. ScorpioFS +/// runs with `CAP_SYS_ADMIN` only, so under the char-device form deleting a file that +/// exists solely in the Dicfuse lower layer fails with `EPERM`. +/// 2. `scan_layer_changes` classifies `.wh.` entries as `ChangeKind::Deleted` (and char +/// devices as well), so either form is *readable* — but only the OCI form is *writable* +/// without extra privileges. +/// +/// Changing this value changes the on-disk upper-layer representation, so an upper directory +/// written under one form must not be reused under the other. +const ANTARES_WHITEOUT_FORMAT: WhiteoutFormat = WhiteoutFormat::OciWhiteout; + +/// Hand a daemon-created directory to the invoking user when running via sudo. +/// +/// `sudo`-launched daemons run as root, but FUSE passthrough writes are performed +/// with the requesting user's credentials; a root-owned rw layer rejects them. +/// Ownership is derived from `SUDO_USER` (best-effort, no-op as non-root). +#[cfg(unix)] +fn chown_to_invoking_user(dir: &std::path::Path) { + use std::os::unix::fs::{chown, MetadataExt}; + if unsafe { libc::geteuid() } != 0 { + return; + } + let Some(user) = std::env::var_os("SUDO_USER") else { + return; + }; + let Ok(meta) = std::fs::metadata(dir) else { + return; + }; + if meta.uid() != 0 { + return; // already owned by a non-root user; leave it alone + } + let Ok(cuser) = std::ffi::CString::new(user.as_os_str().as_encoded_bytes()) else { + return; + }; + let pw = unsafe { libc::getpwnam(cuser.as_ptr()) }; + if pw.is_null() { + return; + } + let (uid, gid) = unsafe { ((*pw).pw_uid, (*pw).pw_gid) }; + if let Err(e) = chown(dir, Some(uid), Some(gid)) { + tracing::warn!(dir = %dir.display(), error = %e, "failed to chown upper layer"); + } +} +#[cfg(not(unix))] +fn chown_to_invoking_user(_dir: &std::path::Path) {} + +/// Build a passthrough layer for an Antares upper/CL/frozen directory. +/// +/// Expressed with the published libfuse-fs 0.2.0 API: explicit whiteout format, +/// xattr on, import at construction, writeback off (Antares keeps the safer +/// non-writeback path — append-mode writes fail with EBADF under writeback). +async fn new_antares_passthrough_layer( + dir: &std::path::Path, +) -> std::io::Result> { + let config = PassthroughConfig { + root_dir: dir.to_path_buf(), + xattr: true, + do_import: true, + writeback: false, + whiteout_format: ANTARES_WHITEOUT_FORMAT, + ..Default::default() + }; + let fs = PassthroughFs::<()>::new(config)?; + #[cfg(target_os = "linux")] + fs.import().await?; + Ok(fs) +} + /// Antares union-fs wrapper: dicfuse lower + passthrough upper/CL. pub struct AntaresFuse { pub mountpoint: PathBuf, pub upper_dir: PathBuf, pub dic: Arc, + /// Lower-projection override: when set, this layer takes the Dicfuse slot as + /// the overlay's base layer (spec 12 §1 — "现有 user-space Layer 适配到 + /// SnapshotReader"). `dic` stays available for the paths that still speak + /// the Dicfuse store (effective diff, verify-committed, refresh). + pub lower_override: Option>, pub cl_dir: Option, + /// Sealed read-only delta layers from `chain` forks, **nearest first** (they + /// shadow the Dicfuse projection below them). Plain host directories: they are + /// part of the overlay lookup order but are never mounted themselves, so no + /// unmount-ordering constraint applies. + pub frozen_dirs: Vec, /// Live FUSE session. Drop / [`MountHandle::unmount`] tears the mount down. mount_handle: Option, } @@ -30,33 +114,79 @@ impl AntaresFuse { } std::fs::create_dir_all(&upper_dir)?; std::fs::create_dir_all(&mountpoint)?; + // The passthrough write path executes with the *requesting* user's + // credentials (setfsuid per FUSE request), so the rw upper layer must be + // owned by that user. Under `sudo` the daemon creates it as root instead, + // which makes every user write fail with EACCES. + chown_to_invoking_user(&upper_dir); Ok(Self { mountpoint, upper_dir, dic, + lower_override: None, cl_dir, + frozen_dirs: Vec::new(), mount_handle: None, }) } + /// Serve `lower` in place of the Dicfuse projection as the overlay's base + /// layer (MST/2 snapshot view; spec 12 §1). + pub fn with_lower_override(mut self, lower: Arc) -> Self { + self.lower_override = Some(lower); + self + } + + /// Attach sealed chain layers (chain forks). Each path must be an existing + /// directory — sealed layers are renamed-in formers uppers, never created + /// fresh; creating one accidentally would silently serve a wrong projection. + /// Order: nearest first (they shadow the layers below them). + pub fn with_frozen_layers(mut self, frozen_dirs: Vec) -> std::io::Result { + for frozen in &frozen_dirs { + if !frozen.is_dir() { + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("sealed layer {} does not exist", frozen.display()), + )); + } + } + self.frozen_dirs = frozen_dirs; + Ok(self) + } + /// Compose the union filesystem instance. pub async fn build_overlay(&self) -> std::io::Result { - // Build lower layers: - // - Optional CL dir sits above Dicfuse to override base files for the CL view. - // - Dicfuse remains the base read-only monorepo layer. + // Build lower layers, nearest first: + // - Optional CL dir sits above everything to override base files for the CL view. + // - Sealed chain layers follow, most recent first (they shadow what is below). + // - Dicfuse remains the base read-only monorepo projection. let mut lower_layers: Vec> = Vec::new(); if let Some(cl_dir) = &self.cl_dir { - let cl_layer = new_antares_passthroughfs_layer(cl_dir).await?; + let cl_layer = + new_antares_passthrough_layer(cl_dir).await?; lower_layers.push(Arc::new(cl_layer) as Arc); } - lower_layers.push(self.dic.clone() as Arc); + // Sealed chain layers, nearest first — each shadows the layers below it. + for frozen in &self.frozen_dirs { + let frozen_layer = + new_antares_passthrough_layer(frozen).await?; + lower_layers.push(Arc::new(frozen_layer) as Arc); + } + + // Base projection: an explicit override (MST/2 snapshot view) takes the + // Dicfuse slot when present (spec 12 §1). + match &self.lower_override { + Some(lower) => lower_layers.push(lower.clone()), + None => lower_layers.push(self.dic.clone() as Arc), + } // Upper layer mirrors upper_dir to keep writes separated from lower layers. - let upper_layer: Arc = - Arc::new(new_antares_passthroughfs_layer(&self.upper_dir).await?); + let upper_layer: Arc = Arc::new( + new_antares_passthrough_layer(&self.upper_dir).await?, + ); // passthrough Upper - readwrite file system over upper dir // passthrough CL - readwrite file system over upper dir @@ -2260,4 +2390,48 @@ mod tests { let _ = std::fs::remove_dir_all(&base); } + + /// Guard the Antares whiteout format. + /// + /// Regression: with the platform default (`CharDev` on Linux) libfuse-fs records a + /// deletion by creating a character device with `mknod`, which requires `CAP_MKNOD`. + /// ScorpioFS deployments grant `CAP_SYS_ADMIN` only, so under the char-device form + /// deleting a file that exists solely in the Dicfuse lower layer fails with `EPERM` — + /// and a failure to record the deletion means the deletion is invisible in `changes`. + #[test] + fn antares_uses_oci_whiteout_format() { + assert_eq!( + super::ANTARES_WHITEOUT_FORMAT, + super::WhiteoutFormat::OciWhiteout, + "Antares must use the OCI whiteout form; the char-device form needs CAP_MKNOD", + ); + } + + /// Wiring guard: a writable Antares layer must report the OCI whiteout format through the + /// `Layer` trait, because that is the value libfuse-fs consults when creating a whiteout + /// and when detecting one. Setting the format on the `Config` is only effective if it + /// reaches this accessor. + #[tokio::test] + async fn antares_passthrough_layer_reports_oci_whiteout() { + let dir = tempfile::tempdir().unwrap(); + let layer = super::new_antares_passthrough_layer(dir.path()).await.unwrap(); + + assert_eq!( + libfuse_fs::unionfs::layer::Layer::whiteout_format(&layer), + super::WhiteoutFormat::OciWhiteout, + ); + } + + /// The OCI form must be the one that `scan_layer_changes` already understands: a `.wh.` + /// entry in an upper layer is a deletion of the stripped name, not a file of its own. + #[test] + fn oci_whiteout_name_is_what_the_change_scanner_expects() { + assert!(libfuse_fs::util::whiteout::is_oci_whiteout_name( + std::ffi::OsStr::new(".wh.gone.rs") + )); + assert_eq!( + libfuse_fs::util::whiteout::oci_whiteout_target(std::ffi::OsStr::new(".wh.gone.rs")), + Some(std::ffi::OsStr::new("gone.rs")), + ); + } } diff --git a/src/daemon/antares.rs b/src/daemon/antares.rs index 301702a..f5c72ff 100644 --- a/src/daemon/antares.rs +++ b/src/daemon/antares.rs @@ -40,9 +40,84 @@ use uuid::Uuid; use crate::{ antares::fuse::AntaresFuse, + daemon::lower_view::DicfuseLower, + daemon::upper_fork::{fork_upper, ForkCopyError, ForkCopyStats}, + daemon::worktree_v2::{ + effective_changes, flatten_chain_into_upper, generation_of, lower_item_for, + remove_committed_upper_entries, + resolve_latest_revision, AttachWorktreeRequest, AttachWorktreeResponse, + CommitFinalizeRequest, CommitFinalizeResponse, CommittedPath, EffectiveKind, + RefreshDisposition, RefreshRequest, RefreshResponse, WorktreeStateV2, + }, + dicfuse::store::DictionaryStore, dicfuse::{Dicfuse, DicfuseManager}, + snapshot::fuse::Mst2Fuse, + snapshot::{Mst2Client, SnapshotReader}, + util::config, }; +/// Retention window requested for an MST/2 snapshot view backing a mount. The +/// server clamps to `1..=3600`; a mount outliving the window renews lazily. +const MST2_LEASE_SECONDS: u64 = 3600; + +/// Build the MST/2 snapshot-view lower layer when `mst2_lower_enabled` is set +/// (spec 12 §1). Returns `None` in the default Dicfuse mode, so the legacy +/// reader stays the only path unless the operator opted in explicitly +/// (spec 15 §3: no silent fallback in either direction). +async fn mst2_lower_layer() -> Result>, ServiceError> { + if !config::mst2_lower_enabled() { + return Ok(None); + } + let token = config::mst2_auth_token(); + let client = Mst2Client::with_token( + config::mst2_base_url(), + (!token.is_empty()).then(|| token.to_string()), + ); + let reader = SnapshotReader::resolve(client, config::mst2_scope(), MST2_LEASE_SECONDS) + .await + .map_err(|e| { + ServiceError::Internal(format!( + "mst2 lower: resolve({}) failed: {e}", + config::mst2_scope() + )) + })?; + let fuse = Mst2Fuse::from_reader_lazy(reader, None) + .await + .map_err(|e| ServiceError::Internal(format!("mst2 lower: build view failed: {e}")))?; + tracing::info!( + scope = config::mst2_scope(), + snapshot = ?fuse.snapshot_id(), + "antares svc: serving MST/2 snapshot view as the lower layer" + ); + Ok(Some(Arc::new(fuse))) +} + +/// The lower projection a mount's effective diff must compare against: the +/// MST/2 snapshot view when the mount serves one, otherwise the Dicfuse +/// projection (P3; see `mst2-impl/P3-HASH-DOMAIN-DESIGN.md`). +fn lower_view_for(entry: &MountEntry) -> Arc { + use crate::daemon::lower_view::{DicfuseLower, Mst2Lower}; + match &entry.mst2_lower { + Some(view) => Arc::new(Mst2Lower(view.clone())), + None => Arc::new(DicfuseLower(entry.fuse.dic.store.clone())), + } +} + +/// MST/2-lowered mounts do not support the worktree-v2 mutations yet: their +/// lower moves by resolving a new snapshot, not by re-pinning the Dicfuse +/// projection, and the finalize/refresh plumbing for that is not in place. +/// Refuse explicitly rather than executing the Dicfuse semantics against the +/// wrong projection (spec 15 §3). +fn reject_mst2_mutation(entry: &MountEntry, op: &str) -> Result<(), ServiceError> { + if entry.mst2_lower.is_some() { + return Err(ServiceError::InvalidRequest(format!( + "{op} is not supported on an MST/2-lowered mount yet; re-attach without \ + mst2_lower_enabled or wait for the snapshot-side finalize/refresh" + ))); + } + Ok(()) +} + /// High-level HTTP daemon that exposes Antares orchestration capabilities. pub struct AntaresDaemon { service: Arc, @@ -90,6 +165,15 @@ where "/mounts/{mount_id}/worktree/refresh-plan", post(Self::plan_worktree_refresh), ) + .route("/mounts/{mount_id}/fork", post(Self::fork_mount)) + // Worktree Control Protocol v2 (docs/scorpiofs-libra-complete-spec-v1.md). + .route("/worktrees", post(Self::attach_worktree)) + .route("/worktrees/{mount_id}/state", get(Self::worktree_state_v2)) + .route( + "/worktrees/{mount_id}/commit-finalize", + post(Self::commit_finalize), + ) + .route("/worktrees/{mount_id}/refresh", post(Self::refresh_lower)) .with_state(self.service.clone()) } @@ -349,6 +433,51 @@ where service.plan_worktree_refresh(mount_id, request).await?, )) } + + /// Derive a new worktree mount from an existing one. + async fn fork_mount( + State(service): State>, + AxumPath(mount_id): AxumPath, + Json(request): Json, + ) -> Result<(StatusCode, Json), ApiError> { + let response = service.fork_mount(mount_id, request).await?; + Ok((StatusCode::CREATED, Json(response))) + } + + /// Worktree v2: attach a worktree with its lower pinned from the first request. + async fn attach_worktree( + State(service): State>, + Json(request): Json, + ) -> Result<(StatusCode, Json), ApiError> { + let response = service.attach_worktree(request).await?; + Ok((StatusCode::CREATED, Json(response))) + } + + /// Worktree v2: effective diff of a mount. + async fn worktree_state_v2( + State(service): State>, + AxumPath(mount_id): AxumPath, + ) -> Result, ApiError> { + Ok(Json(service.worktree_state_v2(mount_id).await?)) + } + + /// Worktree v2: finalize a commit (pin lower, clean committed upper entries). + async fn commit_finalize( + State(service): State>, + AxumPath(mount_id): AxumPath, + Json(request): Json, + ) -> Result, ApiError> { + Ok(Json(service.commit_finalize(mount_id, request).await?)) + } + + /// Worktree v2: move the lower projection to a newer revision. + async fn refresh_lower( + State(service): State>, + AxumPath(mount_id): AxumPath, + Json(request): Json, + ) -> Result, ApiError> { + Ok(Json(service.refresh_lower(mount_id, request).await?)) + } } /// Asynchronous service boundary that the HTTP layer depends on. @@ -423,6 +552,60 @@ pub trait AntaresService: Send + Sync { )) } + /// Derive a new worktree mount from an existing one. + /// + /// Default implementation reports the capability as absent, following the same + /// pattern as the other worktree operations: a client must be able to tell "this + /// daemon cannot fork" from "this fork failed". + async fn fork_mount( + &self, + _source_mount_id: Uuid, + _request: ForkMountRequest, + ) -> Result { + Err(ServiceError::Unsupported( + "fork is not implemented by this Antares service".into(), + )) + } + + /// Worktree Control Protocol v2: effective diff, commit finalize, lower switch. + /// + /// See `docs/scorpiofs-libra-complete-spec-v1.md` and `worktree_v2.rs` for the + /// failure modes these operations eliminate over the v1 contract. + async fn attach_worktree( + &self, + _request: AttachWorktreeRequest, + ) -> Result { + Err(ServiceError::Unsupported( + "worktree v2 attach is not implemented by this Antares service".into(), + )) + } + + async fn worktree_state_v2(&self, _mount_id: Uuid) -> Result { + Err(ServiceError::Unsupported( + "worktree v2 state is not implemented by this Antares service".into(), + )) + } + + async fn commit_finalize( + &self, + _mount_id: Uuid, + _request: CommitFinalizeRequest, + ) -> Result { + Err(ServiceError::Unsupported( + "commit finalize is not implemented by this Antares service".into(), + )) + } + + async fn refresh_lower( + &self, + _mount_id: Uuid, + _request: RefreshRequest, + ) -> Result { + Err(ServiceError::Unsupported( + "lower refresh is not implemented by this Antares service".into(), + )) + } + async fn health_info(&self) -> HealthResponse; async fn shutdown_cleanup(&self) -> Result<(), ServiceError>; } @@ -459,6 +642,41 @@ pub struct CreateMountRequest { /// Optional CL (changelist) identifier for the CL layer #[serde(default)] pub cl: Option, + /// Optional absolute filesystem mountpoint. When omitted, Antares allocates + /// one under `antares_mount_root`; when present, this directory is mounted + /// directly. The caller must provide an empty directory. + #[serde(default)] + pub mountpoint: Option, + /// **Internal only — not part of the HTTP contract.** A pre-populated upper + /// directory to adopt instead of generating one. + /// + /// `fork` needs this because an upper layer is only imported into the FUSE layer + /// when the session starts: a delta written into the upper *after* mounting is + /// invisible through the mount (and can even make writes fail with `EEXIST`), + /// while `GET /worktree` — which scans the directory directly — reports it, + /// leaving the API and the filesystem disagreeing. So the child's delta has to be + /// on disk before the mount exists. + /// + /// `skip_deserializing` is load-bearing: if a client could set this, mount + /// creation would become arbitrary directory creation, and the failure paths call + /// `remove_dir_all` on it. + #[serde(default, skip_serializing, skip_deserializing)] + pub upper_dir: Option, + /// **Internal only — not part of the HTTP contract.** Pin the mount's Dicfuse + /// lower to this Mega *internal* commit OID instead of the moving trunk tip. + /// + /// The Worktree-v2 attach sets it so the projection is immutable from the first + /// request; v1 mounts stay unpinned. Like `upper_dir`, `skip_deserializing` is + /// load-bearing — a client able to pin an arbitrary revision would be able to + /// serve content the mount's owner never asked for. + #[serde(default, skip_serializing, skip_deserializing)] + pub pinned_refs: Option, + /// **Internal only.** Sealed chain layers for a `chain`-fork child, nearest + /// first (host paths). Same `skip_deserializing` rationale as `pinned_refs`: + /// a client able to stack arbitrary directories below its view would read + /// content its mount never projected. + #[serde(default, skip_serializing, skip_deserializing)] + pub sealed_chain: Vec, } /// Request payload for building/rebuilding a CL layer. @@ -553,11 +771,33 @@ pub struct MountChangesResponse { pub changes: Vec, } +/// Where a worktree sits inside its VCS repository, so that a process running *in* +/// the mount can find the repository it belongs to. +/// +/// Deliberately tiny. The authoritative state (HEAD, index, refs, objects) lives +/// host-side under `commondir`, keyed by `worktree_id`, and must never be +/// materialized in the upper layer: what gets written into a mount is a *pointer*, +/// not a repository. ScorpioFS does not interpret either value. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct VcsPointer { + /// Absolute path of the repository's shared state directory. The caller owns + /// canonicalization — the value is written verbatim, and a VCS that compares it + /// against its own canonical storage will reject a messy one. + pub commondir: String, + /// Stable identifier of this worktree within that repository. + pub worktree_id: String, +} + /// Request used by Libra immediately after attaching a clean worktree mount. #[derive(Debug, Clone, Deserialize, Serialize)] pub struct BindWorktreeBaseRequest { /// Immutable commit/revision that Dicfuse is expected to project for this mount. pub base_revision: String, + /// Optional. When present, the pointer files are written into the mount. + /// + /// Optional so an older client keeps working unchanged. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vcs_pointer: Option, } /// Current ScorpioFS contribution to a Git-compatible worktree state. @@ -627,6 +867,73 @@ pub enum ChangeKind { Deleted, } +/// How `fork` should produce the child worktree's layers. +#[derive(Debug, Clone, Copy, Default, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ForkMode { + /// Copy the parent's writable delta into a fresh upper layer. The child ends up + /// with its own independent layers and the parent keeps working. Cost is + /// proportional to the parent's *delta*, not to the repository size. + #[default] + Materialize, + /// Share a frozen copy of the parent's upper layer as a lower layer instead of + /// copying it, for an O(1) fork. **Not implemented**: a request is downgraded to + /// `Materialize` and the response reports `mode_downgraded_from`, so a caller is + /// never silently given a different mechanism than the one it asked for. + Chain, +} + +/// Request to derive a new worktree mount from an existing one. +/// +/// The monorepo path is inherited from the source: a fork is a second worktree over +/// the *same* subtree, so there is nothing to choose. The child's mountpoint is +/// generated by the daemon, exactly as on `POST /mounts`; read it back from the +/// response (`path`) or from `GET /mounts/{id}`. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ForkMountRequest { + /// Optional task identifier, as on mount creation. + #[serde(default)] + pub job_id: Option, + /// Optional target mountpoint. When omitted, ScorpioFS allocates one below + /// its configured mount root; Libra supplies the linked worktree path. + #[serde(default)] + pub mountpoint: Option, + /// Layer strategy. Defaults to `materialize`. + #[serde(default)] + pub mode: ForkMode, + /// Carry the source's CL layer into the child. Off by default: a CL is a build + /// baseline, not a worktree edit (see `docs/worktree-state-transitions.md`). + #[serde(default)] + pub inherit_cl: bool, +} + +/// Result of a `fork`. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ForkMountResponse { + pub mount_id: Uuid, + #[serde(skip_serializing_if = "Option::is_none")] + pub job_id: Option, + pub path: String, + /// The worktree this one was derived from. + pub source_mount_id: Uuid, + /// The mode actually used. + pub mode: ForkMode, + /// Set when the requested mode could not be honoured. + #[serde(skip_serializing_if = "Option::is_none")] + pub mode_downgraded_from: Option, + /// Lower layers, **nearest first**. Clients must not reorder this. + pub lower_chain: Vec, + /// The source's base binding, inherited. `None` until `POST .../worktree/base`. + pub base_revision: Option, + pub mount_state: MountLifecycle, + /// Always `None` under `materialize`; reserved for `chain`. + #[serde(skip_serializing_if = "Option::is_none")] + pub source_frozen_layer: Option, + /// What the delta copy actually did. `None` under `chain`. + #[serde(skip_serializing_if = "Option::is_none")] + pub copy_stats: Option, +} + /// Health check response payload. #[derive(Debug, Clone, Deserialize, Serialize)] pub struct HealthResponse { @@ -744,6 +1051,15 @@ struct MountEntry { cl: Option, /// Immutable revision selected by Libra for an interactive worktree. base_revision: Option, + /// Mega *internal* commit OID the Dicfuse lower is pinned to (from + /// `/api/v1/latest-commit`). `None` = legacy mount whose lower tracks the + /// moving trunk tip. Distinct from `base_revision`, which lives in the VCS + /// client's identifier space (a git commit OID). + pinned_refs: Option, + /// Sealed delta layers from `chain` forks, **nearest first** (host directory + /// paths). They shadow the Dicfuse projection below them and are flattened + /// into the upper at the next finalize/refresh. + sealed_chain: Vec, /// Auto-generated mountpoint path mountpoint: String, /// Auto-generated upper directory @@ -751,6 +1067,12 @@ struct MountEntry { /// Auto-generated CL directory (if cl is provided) cl_dir: Option, fuse: AntaresFuse, + /// The MST/2 snapshot view backing this mount's lower projection, when the + /// mount was created with `mst2_lower_enabled` (spec 12 §1). Not persisted: + /// a restarted daemon refuses to restore such a mount rather than silently + /// serving the Dicfuse projection instead (spec 15 §3 — explicit modes, no + /// silent fallback). + mst2_lower: Option>, state: MountLifecycle, created_at_epoch_ms: u64, last_seen_epoch_ms: u64, @@ -799,6 +1121,73 @@ impl MountEntry { } } +/// Directory inside a mount that holds the VCS pointer. +const VCS_POINTER_DIR: &str = ".libra"; + +/// Write (or verify) the VCS pointer inside a mount. +/// +/// Idempotent for identical content, and refuses to silently rewrite a *different* +/// identity: two worktrees claiming the same directory would be worse than an error. +/// +/// The two files are written into the mount, so they land in the writable upper layer. +/// `scan_layer_changes` skips this directory, so the pointer never shows up as a local +/// change — which is what lets the VCS layer treat the worktree as clean. +fn write_vcs_pointer(mountpoint: &Path, pointer: &VcsPointer) -> Result<(), ServiceError> { + let commondir = pointer.commondir.trim(); + let worktree_id = pointer.worktree_id.trim(); + + if commondir.is_empty() || worktree_id.is_empty() { + return Err(ServiceError::InvalidRequest( + "vcs_pointer.commondir and vcs_pointer.worktree_id cannot be empty".into(), + )); + } + // One value per line. An embedded newline would forge an extra line rather than + // describing a path. + if commondir.contains('\n') || worktree_id.contains('\n') { + return Err(ServiceError::InvalidRequest( + "vcs_pointer values must not contain newlines".into(), + )); + } + if !Path::new(commondir).is_absolute() { + return Err(ServiceError::InvalidRequest( + "vcs_pointer.commondir must be an absolute path".into(), + )); + } + + let dir = mountpoint.join(VCS_POINTER_DIR); + std::fs::create_dir_all(&dir) + .map_err(|e| ServiceError::Internal(format!("failed to create {}: {e}", dir.display())))?; + + for (name, value) in [("commondir", commondir), ("worktree_id", worktree_id)] { + let path = dir.join(name); + let wanted = format!("{value}\n"); + match std::fs::read_to_string(&path) { + // Already correct: a repeated bind is a no-op. + Ok(existing) if existing == wanted => continue, + Ok(existing) => { + return Err(ServiceError::InvalidRequest(format!( + "worktree pointer {} already reads {:?}; refusing to rewrite it to {:?}", + path.display(), + existing.trim_end(), + value + ))); + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => { + return Err(ServiceError::Internal(format!( + "failed to read {}: {e}", + path.display() + ))); + } + } + std::fs::write(&path, wanted).map_err(|e| { + ServiceError::Internal(format!("failed to write {}: {e}", path.display())) + })?; + } + + Ok(()) +} + /// OCI overlay whiteout prefix (libfuse-fs default on macOS). const OCI_WHITEOUT_PREFIX: &str = ".wh."; /// OCI opaque-directory marker; not a per-file delete. @@ -953,10 +1342,23 @@ pub struct PersistedMountState { pub cl: Option, #[serde(default)] pub base_revision: Option, + /// Mega internal commit OID the Dicfuse lower is pinned to. + #[serde(default)] + pub pinned_refs: Option, + /// Sealed delta layers, nearest first (host paths). Flattened away at the + /// next finalize/refresh; persisted so recovery rebuilds the same view. + #[serde(default)] + pub sealed_chain: Vec, pub mountpoint: String, pub upper_dir: String, pub cl_dir: Option, pub created_at_epoch_ms: u64, + /// Whether this mount's lower projection is an MST/2 snapshot view. Such a + /// mount is not restored across restarts: the view is resolved per process, + /// and rebuilding it with the Dicfuse projection instead would silently + /// serve a different base (spec 15 §3). + #[serde(default)] + pub mst2_lower: bool, } /// Persisted state file structure. @@ -1481,9 +1883,32 @@ impl AntaresServiceImpl { /// - Support incremental directory tree loading to reduce initial wait time /// - Add progress callback for long-running initialization /// - Consider lazy loading for very large subdirectory mounts - async fn get_or_create_dicfuse(&self, path: &str) -> Result, ServiceError> { + async fn get_or_create_dicfuse( + &self, + path: &str, + pinned_refs: Option<&str>, + ) -> Result, ServiceError> { const INIT_TIMEOUT_SECS: u64 = 120; + // A pinned mount must never share the path-keyed (unpinned) instances: its + // projection is a fixed revision, theirs tracks the moving trunk tip. + if let Some(refs) = pinned_refs { + let dicfuse = DicfuseManager::for_base_path_and_refs(path, refs).await; + if tokio::time::timeout( + Duration::from_secs(INIT_TIMEOUT_SECS), + dicfuse.store.wait_for_ready(), + ) + .await + .is_err() + { + return Err(ServiceError::FuseFailure(format!( + "pinned Dicfuse for {path} at {refs} did not become ready within \ + {INIT_TIMEOUT_SECS}s" + ))); + } + return Ok(dicfuse); + } + // For root path, use the shared global instance (but ensure it's initialized first). if path.is_empty() || path == "/" { tracing::info!( @@ -1587,65 +2012,765 @@ impl AntaresServiceImpl { } /// Persist current mount state to file. - async fn persist_state(&self) { - if self.state_ownership == StateOwnership::External { - return; + /// Resolve the Dicfuse instance backing a mount's lower projection. + /// + /// Pinned mounts get their pinned instance; legacy mounts share the path-keyed + /// unpinned instance. Callers must NOT cache the result across a finalize: the + /// pin changes the instance. + async fn lower_dicfuse_for( + &self, + path: &str, + pinned_refs: Option<&str>, + ) -> Result, ServiceError> { + match pinned_refs { + Some(refs) => { + let dicfuse = DicfuseManager::for_base_path_and_refs(path, refs).await; + if tokio::time::timeout(Duration::from_secs(120), dicfuse.store.wait_for_ready()) + .await + .is_err() + { + return Err(ServiceError::FuseFailure(format!( + "pinned Dicfuse for {path} at {refs} did not become ready" + ))); + } + Ok(dicfuse) + } + None => self.get_or_create_dicfuse(path, None).await, } + } - let mounts = self.mounts.read().await; - let state = PersistedState { - mounts: mounts - .values() - .filter(|e| matches!(e.state, MountLifecycle::Mounted | MountLifecycle::Ready)) - .map(|e| PersistedMountState { - mount_id: e.mount_id, - job_id: e.job_id.clone(), - path: e.path.clone(), - cl_path: e.cl_path.clone(), - cl: e.cl.clone(), - base_revision: e.base_revision.clone(), - mountpoint: e.mountpoint.clone(), - upper_dir: e.upper_dir.clone(), - cl_dir: e.cl_dir.clone(), - created_at_epoch_ms: e.created_at_epoch_ms, - }) - .collect(), - }; - drop(mounts); + /// Verify that a candidate lower actually serves what a VCS commit claims. + /// + /// Per-path optimistic lock: the committed hashes must be exactly what the new + /// lower serves. This proves the push landed AND that no path was concurrently + /// edited between staging and finalize. + async fn verify_committed_paths( + new_store: &DictionaryStore, + committed: &[CommittedPath], + ) -> Result<(), String> { + for path in committed { + let item = lower_item_for(new_store, &path.path).await; + match (&path.kind, item) { + (EffectiveKind::Deleted, None) => {} + (EffectiveKind::Deleted, Some(item)) => { + return Err(format!( + "committed deletion of {} does not match the new revision \ + (lower still serves blob {})", + path.path, item.hash + )); + } + (EffectiveKind::Added | EffectiveKind::Modified, None) => { + return Err(format!( + "committed path {} is absent from the new revision", + path.path + )); + } + (EffectiveKind::Added | EffectiveKind::Modified, Some(item)) => { + if let Some(expected) = &path.content_hash { + if item.hash != *expected { + return Err(format!( + "committed content of {} does not match the new revision \ + (commit {}, lower {})", + path.path, expected, item.hash + )); + } + } + } + } + } + Ok(()) + } - // Write state to file - if let Some(parent) = self.state_file.parent() { - if let Err(e) = std::fs::create_dir_all(parent) { - tracing::warn!("Failed to create state directory: {}", e); - return; + /// Rebuild a mount's FUSE session over a new lower projection. + /// + /// The caller must have already unmounted the old session. The upper directory + /// and mountpoint are reused unchanged; only the Dicfuse instance moves. + async fn remount_with_lower( + mountpoint: &Path, + dicfuse: Arc, + upper_dir: &Path, + cl_dir: Option<&Path>, + sealed_chain: &[String], + ) -> Result { + let frozen = sealed_chain.iter().map(PathBuf::from).collect::>(); + let mut fuse = AntaresFuse::new( + mountpoint.to_path_buf(), + dicfuse, + upper_dir.to_path_buf(), + cl_dir.map(PathBuf::from), + ) + .await + .and_then(|fuse| fuse.with_frozen_layers(frozen)) + .map_err(|e| ServiceError::FuseFailure(format!("failed to rebuild overlay: {e}")))?; + fuse.mount() + .await + .map_err(|e| ServiceError::FuseFailure(format!("failed to remount: {e}")))?; + Ok(fuse) + } + + /// Rebuild the mount over an MST/2 snapshot view instead of the Dicfuse + /// projection (spec 12 §1). The Dicfuse instance is still constructed — it + /// backs the effective-diff/verify paths for Dicfuse-lowered mounts and is + /// required by `AntaresFuse::new` — but the overlay's base layer is the + /// snapshot view. + async fn remount_with_mst2_lower( + mountpoint: &Path, + view: Arc, + dicfuse: Arc, + upper_dir: &Path, + cl_dir: Option<&Path>, + ) -> Result { + let mut fuse = AntaresFuse::new( + mountpoint.to_path_buf(), + dicfuse, + upper_dir.to_path_buf(), + cl_dir.map(PathBuf::from), + ) + .await + .map_err(|e| ServiceError::FuseFailure(format!("failed to rebuild overlay: {e}")))? + .with_lower_override(view as Arc); + fuse.mount() + .await + .map_err(|e| ServiceError::FuseFailure(format!("failed to remount: {e}")))?; + Ok(fuse) + } + + /// MST/2 refresh: resolve the latest snapshot view and remount over it. + /// The upper layer and mountpoint are reused; only the view moves. The + /// Dicfuse semantics of re-pinning a per-revision store do not apply here. + #[allow(clippy::too_many_arguments)] + async fn mst2_refresh_lower( + &self, + mount_id: Uuid, + old_view: Arc, + path: &str, + upper_dir: &Path, + cl_dir: Option<&Path>, + mountpoint: &Path, + base_revision: &Option, + start: Instant, + ) -> Result { + let dicfuse = self.lower_dicfuse_for(path, None).await?; + + // Dirty check against the *current* view: a refresh must not discard + // uncommitted upper content (same contract as the Dicfuse path). + let view_before = Arc::new(crate::daemon::lower_view::Mst2Lower(old_view.clone())); + let changes = effective_changes(view_before.as_ref(), upper_dir, &[]) + .await + .map_err(|e| ServiceError::Internal(format!("effective scan failed: {e}")))?; + let generation = generation_of(&changes); + + // Resolve the latest snapshot view (the MST/2 analogue of re-pinning). + let new_view = mst2_lower_layer() + .await? + .ok_or_else(|| ServiceError::Internal("mst2 lower disabled mid-flight".into()))?; + + // Idempotency: re-resolving the same snapshot is a no-op. + let new_id = new_view.snapshot_id().map(str::to_string); + if old_view.snapshot_id() == new_view.snapshot_id().as_deref() { + return Ok(RefreshResponse { + disposition: RefreshDisposition::AlreadyAtTarget, + base_revision: base_revision.clone().unwrap_or_default(), + lower_revision: new_id, + generation, + code: None, + detail: None, + }); + } + + // Quiesce, remount over the new view, roll back on failure. + { + let mut mounts = self.mounts.write().await; + let entry = mounts + .get_mut(&mount_id) + .ok_or(ServiceError::NotFound(mount_id))?; + entry.state = MountLifecycle::Quiescing; + if let Err(e) = entry.fuse.unmount().await { + entry.state = MountLifecycle::Ready; + return Err(ServiceError::FuseFailure(format!( + "failed to quiesce mount {mount_id}: {e}" + ))); } } - match toml::to_string_pretty(&state) { - Ok(content) => { - if let Err(e) = std::fs::write(&self.state_file, content) { - tracing::warn!("Failed to write state file: {}", e); + match Self::remount_with_mst2_lower( + mountpoint, + new_view.clone(), + dicfuse.clone(), + upper_dir, + cl_dir, + ) + .await + { + Ok(new_fuse) => { + { + let mut mounts = self.mounts.write().await; + let entry = mounts + .get_mut(&mount_id) + .ok_or(ServiceError::NotFound(mount_id))?; + entry.fuse = new_fuse; + entry.mst2_lower = Some(new_view.clone()); + entry.pinned_refs = new_id.clone(); + entry.state = MountLifecycle::Ready; + entry.last_seen_epoch_ms = current_epoch_ms(); } + self.persist_state().await; + tracing::info!( + mount_id = %mount_id, + snapshot = ?new_id, + elapsed_ms = start.elapsed().as_millis() as u64, + "antares svc: mst2 refresh success" + ); + Ok(RefreshResponse { + disposition: RefreshDisposition::Switched, + base_revision: base_revision.clone().unwrap_or_default(), + lower_revision: new_id, + generation, + code: None, + detail: None, + }) } Err(e) => { - tracing::warn!("Failed to serialize state: {}", e); + // Roll back to the old view so the mount keeps serving bytes. + let _ = Self::remount_with_mst2_lower( + mountpoint, + old_view, + self.lower_dicfuse_for(path, None).await?, + upper_dir, + cl_dir, + ) + .await; + Err(e) } } } - /// Recover mounts from persisted state file. - async fn recover_mounts(&self) { - if !self.state_file.exists() { - tracing::debug!( - "No state file found at {:?}, skipping recovery", - self.state_file - ); - return; + /// MST/2 commit-finalize: verify the committed paths against the *latest* + /// snapshot (the push has already landed there), clear exactly those upper + /// entries, and remount over the new view. + #[allow(clippy::too_many_arguments)] + async fn mst2_commit_finalize( + &self, + mount_id: Uuid, + request: CommitFinalizeRequest, + old_view: Arc, + path: &str, + upper_dir: &Path, + cl_dir: Option<&Path>, + mountpoint: &Path, + base_revision: &Option, + start: Instant, + ) -> Result { + let dicfuse = self.lower_dicfuse_for(path, None).await?; + + // Optimistic lock against the effective diff over the current view. + let view_before = Arc::new(crate::daemon::lower_view::Mst2Lower(old_view.clone())); + let changes = effective_changes(view_before.as_ref(), upper_dir, &[]) + .await + .map_err(|e| ServiceError::Internal(format!("effective scan failed: {e}")))?; + let generation = generation_of(&changes); + if let Some(expected) = request.expected_generation { + if expected != generation { + return Ok(CommitFinalizeResponse { + state: "conflict".into(), + code: Some("GENERATION_CHANGED".into()), + detail: Some(format!( + "state generation {expected} no longer matches {generation}; \ + re-read state and re-commit" + )), + base_revision: base_revision.clone().unwrap_or_default(), + lower_revision: None, + generation, + cleaned_paths: Vec::new(), + }); + } } - let content = match std::fs::read_to_string(&self.state_file) { - Ok(c) => c, - Err(e) => { + // Resolve the latest snapshot: the client pushed its commit, so the + // newest view must already contain the committed content. + let new_view = mst2_lower_layer() + .await? + .ok_or_else(|| ServiceError::Internal("mst2 lower disabled mid-flight".into()))?; + + // Verify the committed set against the new view in the view's own + // domain: the upper file's sha256 must equal the view's digest, which + // proves the commit landed and guards concurrent edits (spec 12 §1: + // sizes/identities come from verified entries, never a 0 placeholder). + for committed in &request.committed_paths { + let lower_digest = new_view.digest_for_path(&committed.path).await; + match (&committed.kind, lower_digest) { + (EffectiveKind::Deleted, None) => {} + (EffectiveKind::Deleted, Some(_)) => { + return Ok(CommitFinalizeResponse { + state: "conflict".into(), + code: Some("TREE_MISMATCH".into()), + detail: Some(format!( + "committed deletion of {} is still served by the new snapshot", + committed.path + )), + base_revision: base_revision.clone().unwrap_or_default(), + lower_revision: None, + generation, + cleaned_paths: Vec::new(), + }); + } + (EffectiveKind::Added | EffectiveKind::Modified, None) => { + return Ok(CommitFinalizeResponse { + state: "conflict".into(), + code: Some("TREE_MISMATCH".into()), + detail: Some(format!( + "committed path {} is absent from the new snapshot", + committed.path + )), + base_revision: base_revision.clone().unwrap_or_default(), + lower_revision: None, + generation, + cleaned_paths: Vec::new(), + }); + } + (EffectiveKind::Added | EffectiveKind::Modified, Some(lower_digest)) => { + let bytes = tokio::fs::read(Path::new(upper_dir).join(&committed.path)) + .await + .map_err(|e| { + ServiceError::Internal(format!( + "failed to read upper content of {}: {e}", + committed.path + )) + })?; + let upper_digest = crate::snapshot::durable::digest_of(&bytes); + if upper_digest != *lower_digest { + return Ok(CommitFinalizeResponse { + state: "conflict".into(), + code: Some("TREE_MISMATCH".into()), + detail: Some(format!( + "upper content of {} does not match the new snapshot \ + ({upper_digest} vs {lower_digest})", + committed.path + )), + base_revision: base_revision.clone().unwrap_or_default(), + lower_revision: None, + generation, + cleaned_paths: Vec::new(), + }); + } + } + } + } + + // Phase B — quiesce, clear exactly the committed entries, remount over + // the new view. Rollbacks restore the previous projection. + { + let mut mounts = self.mounts.write().await; + let entry = mounts + .get_mut(&mount_id) + .ok_or(ServiceError::NotFound(mount_id))?; + entry.state = MountLifecycle::Quiescing; + if let Err(e) = entry.fuse.unmount().await { + entry.state = MountLifecycle::Ready; + return Err(ServiceError::FuseFailure(format!( + "failed to quiesce mount {mount_id}: {e}" + ))); + } + } + + let cleaned = match remove_committed_upper_entries(Path::new(upper_dir), &request.committed_paths) + { + Ok(cleaned) => cleaned, + Err(e) => { + let _ = Self::remount_with_mst2_lower( + mountpoint, + old_view.clone(), + dicfuse.clone(), + upper_dir, + cl_dir, + ) + .await; + return Ok(CommitFinalizeResponse { + state: "failed".into(), + code: Some("SWITCH_FAILED".into()), + detail: Some(format!("upper cleanup failed: {e}")), + base_revision: base_revision.clone().unwrap_or_default(), + lower_revision: None, + generation, + cleaned_paths: Vec::new(), + }); + } + }; + + match Self::remount_with_mst2_lower( + mountpoint, + new_view.clone(), + dicfuse.clone(), + upper_dir, + cl_dir, + ) + .await + { + Ok(new_fuse) => { + { + let mut mounts = self.mounts.write().await; + let entry = mounts + .get_mut(&mount_id) + .ok_or(ServiceError::NotFound(mount_id))?; + entry.fuse = new_fuse; + entry.mst2_lower = Some(new_view.clone()); + entry.pinned_refs = new_view.snapshot_id().map(str::to_string); + entry.state = MountLifecycle::Ready; + entry.last_seen_epoch_ms = current_epoch_ms(); + } + self.persist_state().await; + tracing::info!( + mount_id = %mount_id, + snapshot = ?new_view.snapshot_id(), + cleaned = cleaned.len(), + elapsed_ms = start.elapsed().as_millis() as u64, + "antares svc: mst2 commit_finalize success" + ); + Ok(CommitFinalizeResponse { + state: "ready".into(), + code: None, + detail: None, + base_revision: base_revision.clone().unwrap_or_default(), + lower_revision: new_view.snapshot_id().map(str::to_string), + generation, + cleaned_paths: cleaned, + }) + } + Err(e) => { + let _ = Self::remount_with_mst2_lower( + mountpoint, + old_view, + dicfuse.clone(), + upper_dir, + cl_dir, + ) + .await; + Ok(CommitFinalizeResponse { + state: "failed".into(), + code: Some("SWITCH_FAILED".into()), + detail: Some(format!( + "remount over the new snapshot failed after cleanup: {e}; \ + worktree remounted on the previous projection" + )), + base_revision: base_revision.clone().unwrap_or_default(), + lower_revision: None, + generation, + cleaned_paths: Vec::new(), + }) + } + } + } + + /// `chain` fork: seal the source's upper into a shared read-only layer, give the + /// source a fresh upper (its view is byte-identical and it stays writable), and + /// stack the sealed layer under the child — zero bytes copied. + /// + /// Requires the source to be pinned: a sealed layer over a moving trunk tip + /// would give the child a base with no stable identity. The caller downgrades + /// unpinned sources to `materialize` before getting here. + #[allow(clippy::too_many_arguments)] + async fn fork_mount_chain( + &self, + source_mount_id: Uuid, + request: ForkMountRequest, + source_path: String, + source_upper: PathBuf, + source_pinned: String, + mut source_chain: Vec, + inherited_base: String, + start: Instant, + ) -> Result { + let upper_root = PathBuf::from(crate::util::config::antares_upper_root()); + let frozen = upper_root.join(format!("sealed-{}", Uuid::new_v4())); + let source_new_upper = upper_root.join(Uuid::new_v4().to_string()); + // Parent's VCS pointer target (host gitdir), captured while sealing. + let mut source_pointer_target: Option = None; + + // 1. Quiesce the source and seal its upper with one rename (same filesystem, + // atomic). Rollback restores the rename and the mount. + { + let mut mounts = self.mounts.write().await; + let entry = mounts + .get_mut(&source_mount_id) + .ok_or(ServiceError::NotFound(source_mount_id))?; + entry.state = MountLifecycle::Quiescing; + if let Err(e) = entry.fuse.unmount().await { + entry.state = MountLifecycle::Ready; + return Err(ServiceError::FuseFailure(format!( + "chain fork: failed to quiesce the source mount: {e}" + ))); + } + if let Err(e) = std::fs::rename(&source_upper, &frozen) { + entry.state = MountLifecycle::Ready; + return Err(ServiceError::FuseFailure(format!( + "chain fork: failed to seal the source upper: {e}" + ))); + } + // The sealed layer must not carry the parent's VCS pointer: the child + // inherits the layer read-only and would otherwise resolve the PARENT's + // `.libra` (wrong index, wrong identity). Capture the pointer target so + // the parent's rebuilt upper can serve the same link, then drop it from + // the sealed layer. + source_pointer_target = std::fs::read_link(frozen.join(".libra")).ok(); + let pointer_path = frozen.join(".libra"); + match std::fs::symlink_metadata(&pointer_path) { + Ok(meta) if meta.file_type().is_symlink() => { + if let Err(e) = std::fs::remove_file(&pointer_path) { + entry.state = MountLifecycle::Ready; + return Err(ServiceError::FuseFailure(format!( + "chain fork: failed to strip the VCS pointer from the sealed layer: {e}" + ))); + } + } + Ok(meta) if meta.is_dir() => { + if let Err(e) = std::fs::remove_dir_all(&pointer_path) { + entry.state = MountLifecycle::Ready; + return Err(ServiceError::FuseFailure(format!( + "chain fork: failed to strip the VCS pointer dir from the sealed layer: {e}" + ))); + } + } + _ => {} + } + } + + // 2. Rebuild the source: fresh empty upper over [frozen] + its old chain. + // Its view is unchanged and it keeps accepting writes; the child shares + // the same sealed layer read-only. + let source_dicfuse = + DicfuseManager::for_base_path_and_refs(&source_path, &source_pinned).await; + if tokio::time::timeout( + Duration::from_secs(180), + source_dicfuse.store.wait_for_ready(), + ) + .await + .is_err() + { + // Roll back the seal: the source keeps its original upper, unchained. + if let Err(rename_err) = std::fs::rename(&frozen, &source_upper) { + tracing::error!( + "chain fork: rollback rename failed: {rename_err}; the source upper is now at {}", + frozen.display() + ); + } + { + let mut mounts = self.mounts.write().await; + if let Some(entry) = mounts.get_mut(&source_mount_id) { + entry.state = MountLifecycle::Ready; + } + } + return Err(ServiceError::FuseFailure( + "chain fork: the sealed Dicfuse projection did not become ready; the source was restored unchanged".into(), + )); + } + + let mut new_chain: Vec = vec![frozen.to_string_lossy().to_string()]; + new_chain.extend(source_chain.iter().cloned()); + + let (source_mountpoint, source_cl_dir) = { + let mounts = self.mounts.read().await; + match mounts.get(&source_mount_id) { + Some(entry) => ( + entry.mountpoint.clone(), + entry.cl_dir.clone().map(PathBuf::from), + ), + None => { + let _ = std::fs::rename(&frozen, &source_upper); + return Err(ServiceError::Internal( + "chain fork: source mount vanished mid-fork".into(), + )); + } + } + }; + + let source_fuse = match AntaresFuse::new( + PathBuf::from(&source_mountpoint), + source_dicfuse.clone(), + source_new_upper.clone(), + source_cl_dir.clone(), + ) + .await + .and_then(|fuse| fuse.with_frozen_layers(new_chain.iter().map(PathBuf::from).collect())) + .map_err(|e| ServiceError::FuseFailure(format!("chain fork source rebuild: {e}"))) + { + Ok(mut fuse) => match fuse.mount().await { + Ok(()) => fuse, + Err(e) => { + let _ = Self::remount_with_lower( + Path::new(&source_mountpoint), + source_dicfuse, + &source_upper, + source_cl_dir.as_deref(), + &source_chain, + ) + .await; + let _ = std::fs::rename(&frozen, &source_upper); + return Err(ServiceError::FuseFailure(format!( + "chain fork: source remount failed: {e}" + ))); + } + }, + Err(e) => { + let _ = std::fs::rename(&frozen, &source_upper); + return Err(ServiceError::FuseFailure(format!( + "chain fork: source rebuild failed: {e}" + ))); + } + }; + + // Serve the same VCS pointer from the parent's new upper: the sealed layer + // no longer carries it, and Libra expects `/.libra` to resolve. + if let Some(target) = &source_pointer_target { + std::os::unix::fs::symlink(target, source_new_upper.join(".libra")).map_err(|e| { + ServiceError::FuseFailure(format!( + "chain fork: failed to re-create the parent VCS pointer: {e}" + )) + })?; + } + + { + let mut mounts = self.mounts.write().await; + if let Some(entry) = mounts.get_mut(&source_mount_id) { + entry.fuse = source_fuse; + entry.upper_dir = source_new_upper.to_string_lossy().to_string(); + entry.sealed_chain = new_chain.clone(); + entry.state = MountLifecycle::Ready; + entry.last_seen_epoch_ms = current_epoch_ms(); + } + } + self.persist_state().await; + + // 3. The child: fresh upper over [frozen] + the source's old chain, the same + // pinned Dicfuse instance (shared through the manager cache), the same + // bound base. create_mount handles duplicates and job binding. + let job_id = request + .job_id + .clone() + .unwrap_or_else(|| format!("chain-fork-{}-{}", source_mount_id, Uuid::new_v4())); + let created = match self + .create_mount(CreateMountRequest { + job_id: Some(job_id), + build_id: None, + path: source_path, + cl_path: None, + cl: None, + mountpoint: request.mountpoint.clone(), + upper_dir: None, + pinned_refs: Some(source_pinned.clone()), + sealed_chain: new_chain.clone(), + }) + .await + { + Ok(created) => created, + Err(e) => { + // The source is already rebuilt and writable; only the child failed. + return Err(ServiceError::Internal(format!( + "chain fork: child mount failed (source unchanged): {e}" + ))); + } + }; + + // 4. Bind the child's base to the inherited revision (fresh clean mount). + self.bind_worktree_base( + created.mount_id, + BindWorktreeBaseRequest { + base_revision: inherited_base.clone(), + vcs_pointer: None, + }, + ) + .await?; + + // 5. Report the child's lower chain: the sealed dirs, nearest first, then + // the pinned Dicfuse projection. + let mut lower_chain: Vec = new_chain.clone(); + lower_chain.push(format!("dicfuse@{source_pinned}")); + + tracing::info!( + source_mount_id = %source_mount_id, + mount_id = %created.mount_id, + sealed_layer = %frozen.display(), + elapsed_ms = start.elapsed().as_millis() as u64, + "antares svc: fork_mount_chain success" + ); + + Ok(ForkMountResponse { + mount_id: created.mount_id, + job_id: request.job_id, + path: created.mountpoint, + source_mount_id, + mode: ForkMode::Chain, + mode_downgraded_from: None, + lower_chain, + // Inherited: a fork is a copy of the same revision, not a new one. + base_revision: Some(inherited_base), + mount_state: MountLifecycle::Ready, + source_frozen_layer: Some(frozen.to_string_lossy().to_string()), + copy_stats: None, + }) + } + + async fn persist_state(&self) { + if self.state_ownership == StateOwnership::External { + return; + } + + let mounts = self.mounts.read().await; + let state = PersistedState { + mounts: mounts + .values() + .filter(|e| matches!(e.state, MountLifecycle::Mounted | MountLifecycle::Ready)) + .map(|e| PersistedMountState { + mount_id: e.mount_id, + job_id: e.job_id.clone(), + path: e.path.clone(), + cl_path: e.cl_path.clone(), + cl: e.cl.clone(), + base_revision: e.base_revision.clone(), + pinned_refs: e.pinned_refs.clone(), + sealed_chain: e.sealed_chain.clone(), + mountpoint: e.mountpoint.clone(), + upper_dir: e.upper_dir.clone(), + cl_dir: e.cl_dir.clone(), + created_at_epoch_ms: e.created_at_epoch_ms, + mst2_lower: e.mst2_lower.is_some(), + }) + .collect(), + }; + drop(mounts); + + // Write state to file + if let Some(parent) = self.state_file.parent() { + if let Err(e) = std::fs::create_dir_all(parent) { + tracing::warn!("Failed to create state directory: {}", e); + return; + } + } + + match toml::to_string_pretty(&state) { + Ok(content) => { + if let Err(e) = std::fs::write(&self.state_file, content) { + tracing::warn!("Failed to write state file: {}", e); + } + } + Err(e) => { + tracing::warn!("Failed to serialize state: {}", e); + } + } + } + + /// Recover mounts from persisted state file. + async fn recover_mounts(&self) { + if !self.state_file.exists() { + tracing::debug!( + "No state file found at {:?}, skipping recovery", + self.state_file + ); + return; + } + + let content = match std::fs::read_to_string(&self.state_file) { + Ok(c) => c, + Err(e) => { tracing::warn!("Failed to read state file: {}", e); return; } @@ -1667,6 +2792,18 @@ impl AntaresServiceImpl { tracing::info!("Recovering {} mounts from state file", state.mounts.len()); for persisted in state.mounts { + // An MST/2-lowered mount is not restored: its view is resolved per + // process, and rebuilding the mount with the Dicfuse projection + // instead would silently serve a different base (spec 15 §3). + if persisted.mst2_lower { + tracing::warn!( + mount_id = %persisted.mount_id, + "not restoring an MST/2-lowered mount after restart; re-attach it \ + (the Dicfuse projection would be a different base)" + ); + continue; + } + // Check if mountpoint still exists let mountpoint = PathBuf::from(&persisted.mountpoint); if !mountpoint.exists() { @@ -1677,24 +2814,56 @@ impl AntaresServiceImpl { continue; } - // Get or create Dicfuse instance (uses cache for subdirectory paths) - let dicfuse = match self.get_or_create_dicfuse(&persisted.path).await { - Ok(d) => d, - Err(e) => { - tracing::warn!( - "Failed to get Dicfuse for {} during recovery: {}", - persisted.mount_id, - e - ); - continue; + // Get or create Dicfuse instance (uses cache for subdirectory paths). + // A pinned mount must come back pinned: an unpinned instance would serve + // the moving trunk tip, silently breaking the lower_revision invariant. + let dicfuse = match &persisted.pinned_refs { + Some(refs) => { + let pinned = + DicfuseManager::for_base_path_and_refs(&persisted.path, refs).await; + match tokio::time::timeout( + Duration::from_secs(120), + pinned.store.wait_for_ready(), + ) + .await + { + Ok(()) => pinned, + Err(_) => { + tracing::warn!( + "Pinned Dicfuse for {} at {} did not become ready during recovery", + persisted.mount_id, + refs + ); + continue; + } + } } + None => match self.get_or_create_dicfuse(&persisted.path, None).await { + Ok(d) => d, + Err(e) => { + tracing::warn!( + "Failed to get Dicfuse for {} during recovery: {}", + persisted.mount_id, + e + ); + continue; + } + }, }; let upper_dir = PathBuf::from(&persisted.upper_dir); let cl_dir = persisted.cl_dir.as_ref().map(PathBuf::from); // Try to create and mount AntaresFuse - match AntaresFuse::new(mountpoint.clone(), dicfuse, upper_dir, cl_dir.clone()).await { + let frozen = persisted + .sealed_chain + .iter() + .map(PathBuf::from) + .collect::>(); + match AntaresFuse::new(mountpoint.clone(), dicfuse, upper_dir, cl_dir.clone()) + .await + .and_then(|fuse| fuse.with_frozen_layers(frozen)) + { Ok(mut fuse) => { if let Err(e) = fuse.mount().await { tracing::warn!( @@ -1717,10 +2886,15 @@ impl AntaresServiceImpl { cl_path: Some(cl_path.clone()), cl: persisted.cl.clone(), base_revision: persisted.base_revision.clone(), + pinned_refs: persisted.pinned_refs.clone(), + sealed_chain: persisted.sealed_chain.clone(), mountpoint: persisted.mountpoint.clone(), upper_dir: persisted.upper_dir.clone(), cl_dir: persisted.cl_dir.clone(), fuse, + // Recovery only restores Dicfuse-lowered mounts; the + // MST/2 ones are skipped above and must be re-attached. + mst2_lower: None, // Dicfuse is ready after AntaresFuse::new() completes import_arc. state: MountLifecycle::Ready, created_at_epoch_ms: persisted.created_at_epoch_ms, @@ -1790,6 +2964,24 @@ impl AntaresServiceImpl { "changes.v1".to_string(), "worktree-base.v1".to_string(), "refresh-plan.v1".to_string(), + // Advertises that this daemon records upper-layer deletions in the OCI + // whiteout form (`.wh.`), so a deletion of a lower-layer file is + // observable in `changes` without requiring `CAP_MKNOD`. A VCS client that + // relies on deletions (e.g. a `sync` that must delete remote files) must + // refuse to operate when this capability is absent rather than silently + // miss deletions. + "whiteout.oci.v1".to_string(), + // Derive a new worktree mount from an existing one + // (`POST /mounts/{mount_id}/fork`). + "fork.v1".to_string(), + // Worktree Control Protocol v2: effective diff, commit finalize, + // lower refresh (docs/scorpiofs-libra-complete-spec-v1.md). + "worktree.state.v2".to_string(), + "worktree.attach.v2".to_string(), + // Chain forks: sealed layers shared between parent and child. + "worktree.fork-chain.v2".to_string(), + "worktree.commit-finalize.v2".to_string(), + "worktree.refresh.v2".to_string(), ], status: "healthy".to_string(), mount_count: mounts.len(), @@ -1929,8 +3121,28 @@ impl AntaresService for AntaresServiceImpl { let cl_root = crate::util::config::antares_cl_root(); // Auto-generate paths based on UUID - let mountpoint_str = format!("{}/{}", mount_root, id_str); - let upper_dir_str = format!("{}/{}", upper_root, id_str); + let mountpoint_str = request + .mountpoint + .clone() + .unwrap_or_else(|| format!("{}/{}", mount_root, id_str)); + // `fork` supplies a pre-populated upper (see `CreateMountRequest::upper_dir`); + // everything else gets a fresh one. The hint is validated to sit under the + // configured upper root so a stray value cannot point the failure-path + // `remove_dir_all` at an unrelated directory. + let upper_dir_str = match request.upper_dir.as_deref() { + Some(hint) => { + let hint_path = Path::new(hint.trim_end_matches('/')); + let root = Path::new(upper_root.trim_end_matches('/')); + if hint_path.parent() != Some(root) { + return Err(ServiceError::InvalidRequest(format!( + "upper_dir must be a direct child of {}", + root.display() + ))); + } + hint_path.to_string_lossy().to_string() + } + None => format!("{}/{}", upper_root, id_str), + }; let cl_dir_str = request .cl .as_ref() @@ -1972,13 +3184,30 @@ impl AntaresService for AntaresServiceImpl { // If a specific base path is requested (not root), get from cache or create a dedicated // Dicfuse with path remapping. Otherwise, use the shared global instance. // This may take time for new subdirectory paths as it waits for import_arc to complete. - let dicfuse = self.get_or_create_dicfuse(&request.path).await?; + let dicfuse = self + .get_or_create_dicfuse(&request.path, request.pinned_refs.as_deref()) + .await?; // 6. Create AntaresFuse instance (may take time, not holding lock) + let sealed = request + .sealed_chain + .iter() + .map(PathBuf::from) + .collect::>(); let mut fuse = AntaresFuse::new(mountpoint, dicfuse, upper_dir, cl_dir) .await + .and_then(|fuse| fuse.with_frozen_layers(sealed)) .map_err(|e| ServiceError::FuseFailure(format!("failed to create fuse: {}", e)))?; + // MST/2 lower projection (spec 12 §1): when the operator enabled it, + // the snapshot view takes the Dicfuse slot as the overlay's base layer. + // The view is kept on the mount entry so the effective diff compares + // against the projection that is actually being served. + let mst2_lower = mst2_lower_layer().await?; + if let Some(view) = &mst2_lower { + fuse = fuse.with_lower_override(view.clone() as Arc); + } + // 7. Mount the filesystem fuse.mount() .await @@ -2059,10 +3288,13 @@ impl AntaresService for AntaresServiceImpl { cl_path: request.cl_path.clone(), cl: request.cl.clone(), base_revision: None, + pinned_refs: request.pinned_refs.clone(), + sealed_chain: request.sealed_chain.clone(), mountpoint: mountpoint_str.clone(), upper_dir: upper_dir_str.clone(), cl_dir: cl_dir_str.clone(), fuse, + mst2_lower, state: MountLifecycle::Mounted, created_at_epoch_ms: now, last_seen_epoch_ms: now, @@ -2172,166 +3404,1033 @@ impl AntaresService for AntaresServiceImpl { let entry = mounts .get(&mount_id) .ok_or(ServiceError::NotFound(mount_id))?; - if matches!(entry.state, MountLifecycle::Quiescing) { - return Err(ServiceError::InvalidRequest(format!( - "mount {} is quiescing while its CL layer is reconfigured", - mount_id + if matches!(entry.state, MountLifecycle::Quiescing) { + return Err(ServiceError::InvalidRequest(format!( + "mount {} is quiescing while its CL layer is reconfigured", + mount_id + ))); + } + ( + PathBuf::from(&entry.upper_dir), + entry.cl_dir.as_deref().map(PathBuf::from), + ) + }; + let scan_result = tokio::task::spawn_blocking(move || { + scan_mount_changes(mount_id, &upper_dir, cl_dir.as_deref()) + }) + .await + .map_err(|error| { + ServiceError::Internal(format!( + "Antares changed-path scan task failed for mount {}: {}", + mount_id, error + )) + })?; + drop(mounts); + scan_result + } + + async fn worktree_state(&self, mount_id: Uuid) -> Result { + let (path, base_revision, mount_state, upper_dir) = { + let mounts = self.mounts.read().await; + let entry = mounts + .get(&mount_id) + .ok_or(ServiceError::NotFound(mount_id))?; + ( + entry.path.clone(), + entry.base_revision.clone(), + entry.state.clone(), + PathBuf::from(&entry.upper_dir), + ) + }; + + // A VCS worktree treats an optional CL layer as part of its supplied + // base, not as a user edit. Only the private upper layer is dirty. + let changes = + tokio::task::spawn_blocking(move || scan_mount_changes(mount_id, &upper_dir, None)) + .await + .map_err(|error| { + ServiceError::Internal(format!( + "Antares worktree-state scan task failed for mount {}: {}", + mount_id, error + )) + })??; + + Ok(WorktreeStateResponse { + mount_id, + path, + base_revision, + mount_state, + dirty: !changes.changes.is_empty(), + changes, + }) + } + + async fn bind_worktree_base( + &self, + mount_id: Uuid, + request: BindWorktreeBaseRequest, + ) -> Result { + let base_revision = request.base_revision.trim(); + if base_revision.is_empty() { + return Err(ServiceError::InvalidRequest( + "base_revision cannot be empty".into(), + )); + } + + // Collect what validation needs first: the pointer write below is filesystem + // I/O and should not run while holding the mount-table lock. + let (existing_base, mountpoint, cl_present, mount_state) = { + let mounts = self.mounts.read().await; + let entry = mounts + .get(&mount_id) + .ok_or(ServiceError::NotFound(mount_id))?; + ( + entry.base_revision.clone(), + PathBuf::from(&entry.mountpoint), + entry.cl.is_some(), + entry.state.clone(), + ) + }; + + if let Some(existing) = &existing_base { + if existing != base_revision { + return Err(ServiceError::InvalidRequest(format!( + "mount {} is already bound to base revision {}", + mount_id, existing + ))); + } + // Repeating the same binding is idempotent, and re-asserts the pointer so a + // caller whose first attempt lost the write can simply ask again. + if let Some(pointer) = &request.vcs_pointer { + write_vcs_pointer(&mountpoint, pointer)?; + } + return self.worktree_state(mount_id).await; + } + + if !matches!( + &mount_state, + MountLifecycle::Mounted | MountLifecycle::Ready + ) { + return Err(ServiceError::InvalidRequest(format!( + "mount {} is currently in state {:?}; cannot bind a Libra worktree base", + mount_id, mount_state + ))); + } + if cl_present { + return Err(ServiceError::InvalidRequest( + "cannot bind a Libra worktree base to a mount with a CL layer".into(), + )); + } + + let state = self.worktree_state(mount_id).await?; + if state.dirty { + return Err(ServiceError::InvalidRequest( + "cannot bind a worktree base after local upper-layer changes exist".into(), + )); + } + + // Pointer first, binding second: if the pointer cannot be written, no binding + // has been advertised on the strength of state that is not on disk. + if let Some(pointer) = &request.vcs_pointer { + write_vcs_pointer(&mountpoint, pointer)?; + } + + { + let mut mounts = self.mounts.write().await; + let entry = mounts + .get_mut(&mount_id) + .ok_or(ServiceError::NotFound(mount_id))?; + // Re-check under the write lock: the mount may have moved on while the + // pointer was being written. + if !matches!(entry.state, MountLifecycle::Mounted | MountLifecycle::Ready) { + return Err(ServiceError::InvalidRequest(format!( + "mount {} is currently in state {:?}; cannot bind a Libra worktree base", + mount_id, entry.state + ))); + } + if entry.cl.is_some() { + return Err(ServiceError::InvalidRequest( + "cannot bind a Libra worktree base to a mount with a CL layer".into(), + )); + } + match &entry.base_revision { + Some(existing) if existing != base_revision => { + return Err(ServiceError::InvalidRequest(format!( + "mount {} is already bound to base revision {}", + mount_id, existing + ))); + } + Some(_) => {} + None => { + entry.base_revision = Some(base_revision.to_string()); + entry.update_last_seen(); + } + } + } + + self.persist_state().await; + self.worktree_state(mount_id).await + } + + async fn plan_worktree_refresh( + &self, + mount_id: Uuid, + request: RefreshPlanRequest, + ) -> Result { + let expected = request.expected_base_revision.trim(); + let target = request.target_revision.trim(); + if expected.is_empty() || target.is_empty() { + return Err(ServiceError::InvalidRequest( + "expected_base_revision and target_revision cannot be empty".into(), + )); + } + + let worktree = self.worktree_state(mount_id).await?; + let disposition = match worktree.base_revision.as_deref() { + None => RefreshPlanDisposition::Unbound, + Some(current) if current != expected => RefreshPlanDisposition::BaseMismatch, + Some(current) if current == target => RefreshPlanDisposition::AlreadyAtTarget, + Some(_) if request.require_clean && worktree.dirty => { + RefreshPlanDisposition::BlockedDirty + } + Some(_) => RefreshPlanDisposition::Ready, + }; + + Ok(RefreshPlanResponse { + mount_id, + current_base_revision: worktree.base_revision.clone(), + target_revision: target.to_string(), + disposition, + worktree, + }) + } + + async fn fork_mount( + &self, + source_mount_id: Uuid, + request: ForkMountRequest, + ) -> Result { + let start = Instant::now(); + + // Resolve what we need from the source, then drop the lock: the delta copy + // below does blocking filesystem I/O and must not hold the mount table. + // + // The monorepo path is *inherited*: a fork is a second worktree over the same + // subtree, so the caller does not get to point it somewhere else. Note that + // `path` here is the monorepo path (as on `POST /mounts`), **not** a filesystem + // location — the child's mountpoint is generated by `create_mount`. + let (source_path, source_upper, source_cl, source_cl_path, source_base, source_pinned, source_chain) = { + let mounts = self.mounts.read().await; + let entry = mounts + .get(&source_mount_id) + .ok_or(ServiceError::NotFound(source_mount_id))?; + + if matches!( + &entry.state, + MountLifecycle::Quiescing | MountLifecycle::Unmounting | MountLifecycle::Unmounted + ) { + return Err(ServiceError::InvalidRequest(format!( + "source mount {} is in state {:?} and cannot be forked from", + source_mount_id, entry.state + ))); + } + + ( + entry.path.clone(), + PathBuf::from(&entry.upper_dir), + entry.cl.clone(), + entry.cl_path.clone(), + entry.base_revision.clone(), + entry.pinned_refs.clone(), + entry.sealed_chain.clone(), + ) + }; + + let inherited_base = source_base.clone().ok_or_else(|| { + ServiceError::InvalidRequest( + "cannot fork a worktree without a bound base revision".into(), + ) + })?; + + // Chain requires a pinned source: a sealed layer over a MOVING trunk tip + // would make the child's base meaningless. Unpinned sources fall back to + // materialize and the response says so. + let (mode, mode_downgraded_from) = match (&request.mode, &source_pinned) { + (ForkMode::Chain, None) => (ForkMode::Materialize, Some(ForkMode::Chain)), + (ForkMode::Chain, Some(_)) => (ForkMode::Chain, None), + (ForkMode::Materialize, _) => (ForkMode::Materialize, None), + }; + // The child is a distinct mount: never inherit the source's job binding, + // and never let the (path, cl) duplicate check reject a second fork. + let job_id = request + .job_id + .clone() + .unwrap_or_else(|| format!("fork-{}-{}", source_mount_id, Uuid::new_v4())); + let sealed_fork = mode == ForkMode::Chain; + + if sealed_fork { + return self + .fork_mount_chain( + source_mount_id, + request.clone(), + source_path.clone(), + source_upper.clone(), + source_pinned + .clone() + .expect("checked above: chain requires a pinned source"), + source_chain.clone(), + inherited_base.clone(), + start, + ) + .await; + } + + // The child's delta must be on disk *before* the FUSE session starts, because + // the upper layer is only imported at mount time. So it is copied into a + // staging upper directory, which `create_mount` then adopts (the internal + // `upper_dir` field) instead of generating an empty one. + let staging = PathBuf::from(crate::util::config::antares_upper_root()) + .join(Uuid::new_v4().to_string()); + + let (src, dst) = (source_upper.clone(), staging.clone()); + let copied = tokio::task::spawn_blocking(move || fork_upper(&src, &dst)) + .await + .map_err(|e| ServiceError::Internal(format!("fork delta copy task failed: {e}")))?; + + let copy_stats = match copied { + Ok(stats) => stats, + Err(err) => { + let _ = std::fs::remove_dir_all(&staging); + return Err(match err { + ForkCopyError::SourceBusy { .. } => { + ServiceError::InvalidRequest(format!("fork aborted: {err}")) + } + ForkCopyError::UnsupportedEntry { .. } => ServiceError::InvalidRequest( + format!("fork cannot copy this worktree: {err}"), + ), + ForkCopyError::Io { .. } => { + ServiceError::Internal(format!("fork failed while copying: {err}")) + } + }); + } + }; + + let inherit_cl = request.inherit_cl; + let created = match self + .create_mount(CreateMountRequest { + job_id: Some(job_id), + build_id: None, + path: source_path, + cl_path: if inherit_cl { source_cl_path } else { None }, + cl: if inherit_cl { source_cl } else { None }, + mountpoint: request.mountpoint.clone(), + upper_dir: Some(staging.to_string_lossy().to_string()), + // The child inherits the source's pin: a fork of a pinned worktree + // must not silently fall back to the moving trunk tip. + pinned_refs: source_pinned, + sealed_chain: Vec::new(), + }) + .await + { + Ok(created) => created, + Err(err) => { + // `create_mount` rolls back the paths it owns, but the staging + // directory is ours to clean up. + let _ = std::fs::remove_dir_all(&staging); + return Err(err); + } + }; + + { + let mut mounts = self.mounts.write().await; + let entry = mounts + .get_mut(&created.mount_id) + .ok_or(ServiceError::NotFound(created.mount_id))?; + entry.base_revision = Some(inherited_base.clone()); + entry.update_last_seen(); + } + self.persist_state().await; + + let status = self.describe_mount(created.mount_id).await?; + + // Nearest-first: the CL layer (a build baseline) sits above the shared + // Dicfuse projection. Callers must not reorder this. + let mut lower_chain = Vec::new(); + if let Some(cl) = status.layers.cl.clone() { + lower_chain.push(cl); + } + lower_chain.push(status.layers.dicfuse.clone()); + + tracing::info!( + source_mount_id = %source_mount_id, + mount_id = %created.mount_id, + mode = ?mode, + files = copy_stats.files, + bytes = copy_stats.bytes, + reflink_used = copy_stats.reflink_used, + retries = copy_stats.retries, + elapsed_ms = start.elapsed().as_millis() as u64, + "antares svc: fork_mount success" + ); + + Ok(ForkMountResponse { + mount_id: created.mount_id, + job_id: status.job_id.clone(), + path: status.mountpoint.clone(), + source_mount_id, + mode, + mode_downgraded_from, + lower_chain, + // Inherited: a fork is a copy of the same revision, not a new one. + base_revision: Some(inherited_base), + mount_state: status.state.clone(), + // Reserved for `chain`; always absent under `materialize`. + source_frozen_layer: None, + copy_stats: Some(copy_stats), + }) + } + + async fn attach_worktree( + &self, + request: AttachWorktreeRequest, + ) -> Result { + let start = Instant::now(); + let repo_path = Self::normalize_mount_path(&request.repo_path); + + // Fail before creating anything if the mountpoint cannot serve a FUSE + // session — the same rule v1's mount path enforces, just earlier. + let mountpoint = PathBuf::from(Self::normalize_mount_path(&request.mountpoint)); + crate::server::prepare_mountpoint(&mountpoint).map_err(|e| { + ServiceError::InvalidRequest(format!( + "attach target {} is not usable as a mountpoint: {e}", + mountpoint.display() + )) + })?; + + // Pin the lower now: an explicit internal OID from the client, or the + // monorepo's latest commit for the path. Either way the projection is + // immutable from the first request — no tip-drift window before bind. + let lower_revision = match request.lower_revision.as_deref().map(str::trim) { + Some(r) if !r.is_empty() => r.to_string(), + _ => resolve_latest_revision(config::base_url(), &repo_path) + .await + .map_err(ServiceError::Internal)?, + }; + + let created = self + .create_mount(CreateMountRequest { + job_id: request.job_id.clone(), + build_id: None, + path: repo_path.clone(), + cl_path: None, + cl: None, + mountpoint: Some(mountpoint.to_string_lossy().to_string()), + upper_dir: None, + pinned_refs: Some(lower_revision.clone()), + sealed_chain: Vec::new(), + }) + .await?; + + // Record the client-side binding on the fresh (clean) mount: the same + // validation v1's bind endpoint runs, inlined for the single-call attach. + // The pointer files stay the client's business — the daemon never writes + // VCS metadata into a mount. + if let Some(base) = request + .base_revision + .as_deref() + .map(str::trim) + .filter(|b| !b.is_empty()) + { + self.bind_worktree_base( + created.mount_id, + BindWorktreeBaseRequest { + base_revision: base.to_string(), + vcs_pointer: None, + }, + ) + .await?; + } + + let generation = { + let mounts = self.mounts.read().await; + let entry = mounts + .get(&created.mount_id) + .ok_or(ServiceError::NotFound(created.mount_id))?; + let chain_dirs = entry + .sealed_chain + .iter() + .map(PathBuf::from) + .collect::>(); + let changes = effective_changes(lower_view_for(entry).as_ref(), Path::new(&entry.upper_dir), &chain_dirs) + .await + .map_err(|e| ServiceError::Internal(format!("effective scan failed: {e}")))?; + generation_of(&changes) + }; + + tracing::info!( + mount_id = %created.mount_id, + worktree_id = ?request.worktree_id, + repo_path = %repo_path, + lower_revision = %lower_revision, + elapsed_ms = start.elapsed().as_millis() as u64, + "antares svc: attach_worktree success" + ); + + Ok(AttachWorktreeResponse { + mount_id: created.mount_id.to_string(), + worktree_id: request.worktree_id, + mountpoint: created.mountpoint, + base_revision: request.base_revision, + lower_revision, + state: "ready".into(), + generation, + }) + } + + async fn worktree_state_v2(&self, mount_id: Uuid) -> Result { + let (path, upper_dir, base_revision, pinned_refs, sealed_chain, mount_state, mst2_lower) = { + let mounts = self.mounts.read().await; + let entry = mounts + .get(&mount_id) + .ok_or(ServiceError::NotFound(mount_id))?; + ( + entry.path.clone(), + PathBuf::from(&entry.upper_dir), + entry.base_revision.clone(), + entry.pinned_refs.clone(), + entry.sealed_chain.clone(), + entry.state.clone(), + entry.mst2_lower.clone(), + ) + }; + + let dicfuse = self + .lower_dicfuse_for(&path, pinned_refs.as_deref()) + .await?; + let chain_dirs: Vec = sealed_chain.iter().map(PathBuf::from).collect(); + let view: Arc = match &mst2_lower { + Some(view) => Arc::new(crate::daemon::lower_view::Mst2Lower(view.clone())), + None => Arc::new(DicfuseLower(dicfuse.store.clone())), + }; + let changes = effective_changes(view.as_ref(), &upper_dir, &chain_dirs) + .await + .map_err(|e| ServiceError::Internal(format!("effective scan failed: {e}")))?; + let generation = generation_of(&changes); + + Ok(WorktreeStateV2 { + mount_id: mount_id.to_string(), + lower_revision: pinned_refs, + base_revision, + state: format!("{mount_state:?}").to_lowercase(), + generation, + dirty: !changes.is_empty(), + changes, + }) + } + + async fn commit_finalize( + &self, + mount_id: Uuid, + request: CommitFinalizeRequest, + ) -> Result { + let start = Instant::now(); + + // Phase A — reads and builds only. Every failure below this point leaves + // the mount, the upper layer, and the bound revision untouched. + let (path, upper_dir, cl_dir, mountpoint, base_revision, pinned_refs, sealed_chain, mount_state, mst2_lower) = { + let mounts = self.mounts.read().await; + let entry = mounts + .get(&mount_id) + .ok_or(ServiceError::NotFound(mount_id))?; + ( + entry.path.clone(), + PathBuf::from(&entry.upper_dir), + entry.cl_dir.clone().map(PathBuf::from), + PathBuf::from(&entry.mountpoint), + entry.base_revision.clone(), + entry.pinned_refs.clone(), + entry.sealed_chain.clone(), + entry.state.clone(), + entry.mst2_lower.clone(), + ) + }; + if !matches!(mount_state, MountLifecycle::Mounted | MountLifecycle::Ready) { + return Err(ServiceError::InvalidRequest(format!( + "mount {mount_id} is in state {mount_state:?}; cannot finalize a commit" + ))); + } + // MST/2-lowered mounts move their lower by resolving a new snapshot, not + // by re-pinning the Dicfuse projection (P3; see + // mst2-impl/P3-HASH-DOMAIN-DESIGN.md). + if let Some(old_view) = mst2_lower.clone() { + return self + .mst2_commit_finalize( + mount_id, + request, + old_view, + &path, + &upper_dir, + cl_dir.as_deref(), + &mountpoint, + &base_revision, + start, + ) + .await; + } + + let chain_dirs: Vec = sealed_chain.iter().map(PathBuf::from).collect(); + let current = self + .lower_dicfuse_for(&path, pinned_refs.as_deref()) + .await?; + let changes = effective_changes(&DicfuseLower(current.store.clone()), &upper_dir, &chain_dirs) + .await + .map_err(|e| ServiceError::Internal(format!("effective scan failed: {e}")))?; + let generation = generation_of(&changes); + + if let Some(expected) = request.expected_generation { + if expected != generation { + return Ok(CommitFinalizeResponse { + state: "conflict".into(), + code: Some("GENERATION_CHANGED".into()), + detail: Some(format!( + "state generation {expected} no longer matches {generation}; \ + re-read state and re-commit" + )), + base_revision: base_revision.unwrap_or_default(), + lower_revision: pinned_refs, + generation, + cleaned_paths: Vec::new(), + }); + } + } + if let Some(expected) = &request.expected_base_revision { + match &base_revision { + Some(actual) if actual != expected => { + return Ok(CommitFinalizeResponse { + state: "conflict".into(), + code: Some("BASE_MISMATCH".into()), + detail: Some(format!( + "bound base revision is {actual}, client expected {expected}" + )), + base_revision: base_revision.unwrap_or_default(), + lower_revision: pinned_refs, + generation, + cleaned_paths: Vec::new(), + }); + } + _ => {} + } + } + + let new_refs = match request.new_base_revision.as_deref().map(str::trim) { + Some(r) if !r.is_empty() => r.to_string(), + _ => resolve_latest_revision(config::base_url(), &path) + .await + .map_err(ServiceError::Internal)?, + }; + + // A sealed chain is a delta against the OLD revision: keeping it across a + // lower switch would let stale chain entries shadow the new projection. + // Flatten it into the upper (nearest wins, upper wins over everything); + // the chain's O(1) fork cost becomes a one-time O(uncommitted delta) here. + let chain_empty = chain_dirs.is_empty(); + if !chain_empty { + let dirs = chain_dirs.clone(); + let upper = upper_dir.clone(); + tokio::task::spawn_blocking(move || flatten_chain_into_upper(&dirs, &upper)) + .await + .map_err(|e| ServiceError::Internal(format!("chain flatten task failed: {e}")))? + .map_err(|e| ServiceError::Internal(format!("chain flatten failed: {e}")))?; + } + + // Already at the requested revision: only the upper cleanup remains, which + // makes a retried finalize idempotent. The chain is already flattened. + if pinned_refs.as_deref() == Some(new_refs.as_str()) { + let cleaned = remove_committed_upper_entries(&upper_dir, &request.committed_paths) + .map_err(|e| ServiceError::Internal(format!("upper cleanup failed: {e}")))?; + { + let mut mounts = self.mounts.write().await; + if let Some(entry) = mounts.get_mut(&mount_id) { + entry.sealed_chain = Vec::new(); + entry.last_seen_epoch_ms = current_epoch_ms(); + } + } + self.persist_state().await; + let changes = effective_changes(&DicfuseLower(current.store.clone()), &upper_dir, &[]) + .await + .map_err(|e| ServiceError::Internal(format!("effective scan failed: {e}")))?; + return Ok(CommitFinalizeResponse { + state: "ready".into(), + code: None, + detail: None, + base_revision: base_revision.unwrap_or_default(), + lower_revision: pinned_refs, + generation: generation_of(&changes), + cleaned_paths: cleaned, + }); + } + + let new_dicfuse = DicfuseManager::for_base_path_and_refs(&path, &new_refs).await; + if tokio::time::timeout( + Duration::from_secs(180), + new_dicfuse.store.wait_for_ready(), + ) + .await + .is_err() + { + return Ok(CommitFinalizeResponse { + state: "failed".into(), + code: Some("SWITCH_FAILED".into()), + detail: Some(format!( + "pinned lower for revision {new_refs} did not become ready" + )), + base_revision: base_revision.unwrap_or_default(), + lower_revision: pinned_refs, + generation, + cleaned_paths: Vec::new(), + }); + } + + if let Err(detail) = + Self::verify_committed_paths(&new_dicfuse.store, &request.committed_paths).await + { + return Ok(CommitFinalizeResponse { + state: "conflict".into(), + code: Some("TREE_MISMATCH".into()), + detail: Some(detail), + base_revision: base_revision.unwrap_or_default(), + lower_revision: pinned_refs, + generation, + cleaned_paths: Vec::new(), + }); + } + + // Phase B — mutation. Quiesce, remove exactly the committed entries, then + // remount over the new lower. Rollbacks restore the original chain: the + // flatten was view-neutral, so the original stack serves the same bytes. + let old_dicfuse = current; + { + let mut mounts = self.mounts.write().await; + let entry = mounts + .get_mut(&mount_id) + .ok_or(ServiceError::NotFound(mount_id))?; + entry.state = MountLifecycle::Quiescing; + if let Err(e) = entry.fuse.unmount().await { + entry.state = MountLifecycle::Ready; + return Err(ServiceError::FuseFailure(format!( + "failed to quiesce mount {mount_id}: {e}" ))); } - ( - PathBuf::from(&entry.upper_dir), - entry.cl_dir.as_deref().map(PathBuf::from), - ) + } + + let cleaned = match remove_committed_upper_entries(&upper_dir, &request.committed_paths) + { + Ok(cleaned) => cleaned, + Err(e) => { + let _ = Self::remount_with_lower( + &mountpoint, + old_dicfuse, + &upper_dir, + cl_dir.as_deref(), + &sealed_chain, + ) + .await; + return Ok(CommitFinalizeResponse { + state: "failed".into(), + code: Some("SWITCH_FAILED".into()), + detail: Some(format!("upper cleanup failed: {e}")), + base_revision: base_revision.unwrap_or_default(), + lower_revision: pinned_refs, + generation, + cleaned_paths: Vec::new(), + }); + } }; - let scan_result = tokio::task::spawn_blocking(move || { - scan_mount_changes(mount_id, &upper_dir, cl_dir.as_deref()) - }) + + let mut new_fuse = match Self::remount_with_lower( + &mountpoint, + new_dicfuse.clone(), + &upper_dir, + cl_dir.as_deref(), + &[], + ) .await - .map_err(|error| { - ServiceError::Internal(format!( - "Antares changed-path scan task failed for mount {}: {}", - mount_id, error - )) - })?; - drop(mounts); - scan_result + { + Ok(fuse) => fuse, + Err(e) => { + let _ = Self::remount_with_lower( + &mountpoint, + old_dicfuse, + &upper_dir, + cl_dir.as_deref(), + &sealed_chain, + ) + .await; + return Ok(CommitFinalizeResponse { + state: "failed".into(), + code: Some("SWITCH_FAILED".into()), + detail: Some(format!( + "remount over revision {new_refs} failed after cleanup: {e}; \ + worktree remounted on the previous projection" + )), + base_revision: base_revision.unwrap_or_default(), + lower_revision: pinned_refs, + generation, + cleaned_paths: Vec::new(), + }); + } + }; + + { + let mut mounts = self.mounts.write().await; + let entry = mounts + .get_mut(&mount_id) + .ok_or(ServiceError::NotFound(mount_id))?; + entry.fuse = new_fuse; + entry.pinned_refs = Some(new_refs.clone()); + // Flattened above: the chain no longer applies to the new revision. + entry.sealed_chain = Vec::new(); + entry.state = MountLifecycle::Ready; + entry.last_seen_epoch_ms = current_epoch_ms(); + } + self.persist_state().await; + + // Warm the committed paths into the new store so the first read after a + // sync cannot race store initialization and come back empty. + for committed in &request.committed_paths { + let Some(item) = lower_item_for(&new_dicfuse.store, &committed.path).await else { + continue; + }; + if item.is_dir() { + continue; + } + let ino = item.get_inode(); + let oid = item.hash.clone(); + let _ = new_dicfuse.store.fetch_file_content(ino, &oid).await; + } + + let changes = effective_changes(&DicfuseLower(new_dicfuse.store.clone()), &upper_dir, &[]) + .await + .map_err(|e| ServiceError::Internal(format!("effective scan failed: {e}")))?; + let generation = generation_of(&changes); + + tracing::info!( + mount_id = %mount_id, + lower_revision = %new_refs, + cleaned = cleaned.len(), + dirty = !changes.is_empty(), + elapsed_ms = start.elapsed().as_millis() as u64, + "antares svc: commit_finalize success" + ); + + Ok(CommitFinalizeResponse { + state: "ready".into(), + code: None, + detail: None, + base_revision: base_revision.unwrap_or_default(), + lower_revision: Some(new_refs), + generation, + cleaned_paths: cleaned, + }) } - async fn worktree_state(&self, mount_id: Uuid) -> Result { - let (path, base_revision, mount_state, upper_dir) = { + async fn refresh_lower( + &self, + mount_id: Uuid, + request: RefreshRequest, + ) -> Result { + let start = Instant::now(); + let (path, upper_dir, cl_dir, mountpoint, base_revision, pinned_refs, sealed_chain, mount_state, mst2_lower) = { let mounts = self.mounts.read().await; let entry = mounts .get(&mount_id) .ok_or(ServiceError::NotFound(mount_id))?; ( entry.path.clone(), + PathBuf::from(&entry.upper_dir), + entry.cl_dir.clone().map(PathBuf::from), + PathBuf::from(&entry.mountpoint), entry.base_revision.clone(), + entry.pinned_refs.clone(), + entry.sealed_chain.clone(), entry.state.clone(), - PathBuf::from(&entry.upper_dir), + entry.mst2_lower.clone(), ) }; + if !matches!(mount_state, MountLifecycle::Mounted | MountLifecycle::Ready) { + return Err(ServiceError::InvalidRequest(format!( + "mount {mount_id} is in state {mount_state:?}; cannot refresh the lower" + ))); + } + // MST/2-lowered mounts move their lower by resolving a new snapshot, not + // by re-pinning the Dicfuse projection (P3; see + // mst2-impl/P3-HASH-DOMAIN-DESIGN.md). + if let Some(old_view) = mst2_lower.clone() { + return self + .mst2_refresh_lower( + mount_id, + old_view, + &path, + &upper_dir, + cl_dir.as_deref(), + &mountpoint, + &base_revision, + start, + ) + .await; + } - // A VCS worktree treats an optional CL layer as part of its supplied - // base, not as a user edit. Only the private upper layer is dirty. - let changes = - tokio::task::spawn_blocking(move || scan_mount_changes(mount_id, &upper_dir, None)) + let chain_dirs: Vec = sealed_chain.iter().map(PathBuf::from).collect(); + let current = self + .lower_dicfuse_for(&path, pinned_refs.as_deref()) + .await?; + let changes = effective_changes(&DicfuseLower(current.store.clone()), &upper_dir, &chain_dirs) + .await + .map_err(|e| ServiceError::Internal(format!("effective scan failed: {e}")))?; + let generation = generation_of(&changes); + + if request.require_clean && !changes.is_empty() { + return Ok(RefreshResponse { + disposition: RefreshDisposition::BlockedDirty, + base_revision: base_revision.unwrap_or_default(), + lower_revision: pinned_refs, + generation, + code: Some("BLOCKED_DIRTY".into()), + detail: Some(format!( + "{} effective change(s) exist; commit or stash before refreshing", + changes.len() + )), + }); + } + + let target = match request.target_revision.as_deref().map(str::trim) { + Some(r) if !r.is_empty() => r.to_string(), + _ => resolve_latest_revision(config::base_url(), &path) .await - .map_err(|error| { - ServiceError::Internal(format!( - "Antares worktree-state scan task failed for mount {}: {}", - mount_id, error - )) - })??; + .map_err(ServiceError::Internal)?, + }; - Ok(WorktreeStateResponse { - mount_id, - path, - base_revision, - mount_state, - dirty: !changes.changes.is_empty(), - changes, - }) - } + if pinned_refs.as_deref() == Some(target.as_str()) { + return Ok(RefreshResponse { + disposition: RefreshDisposition::AlreadyAtTarget, + base_revision: base_revision.unwrap_or_default(), + lower_revision: pinned_refs, + generation, + code: None, + detail: None, + }); + } - async fn bind_worktree_base( - &self, - mount_id: Uuid, - request: BindWorktreeBaseRequest, - ) -> Result { - let base_revision = request.base_revision.trim(); - if base_revision.is_empty() { - return Err(ServiceError::InvalidRequest( - "base_revision cannot be empty".into(), - )); + // Flatten before the switch: a sealed chain is a delta against the OLD + // revision and must not shadow the new projection. + if !chain_dirs.is_empty() { + let dirs = chain_dirs.clone(); + let upper = upper_dir.clone(); + tokio::task::spawn_blocking(move || flatten_chain_into_upper(&dirs, &upper)) + .await + .map_err(|e| ServiceError::Internal(format!("chain flatten task failed: {e}")))? + .map_err(|e| ServiceError::Internal(format!("chain flatten failed: {e}")))?; } - let existing_base = { - let mounts = self.mounts.read().await; + let new_dicfuse = DicfuseManager::for_base_path_and_refs(&path, &target).await; + if tokio::time::timeout( + Duration::from_secs(180), + new_dicfuse.store.wait_for_ready(), + ) + .await + .is_err() + { + return Ok(RefreshResponse { + disposition: RefreshDisposition::BaseMismatch, + base_revision: base_revision.unwrap_or_default(), + lower_revision: pinned_refs, + generation, + code: Some("SWITCH_FAILED".into()), + detail: Some(format!("lower for revision {target} did not become ready")), + }); + } + + let old_dicfuse = current; + { + let mut mounts = self.mounts.write().await; let entry = mounts - .get(&mount_id) + .get_mut(&mount_id) .ok_or(ServiceError::NotFound(mount_id))?; - entry.base_revision.clone() - }; - if let Some(existing) = existing_base { - if existing != base_revision { - return Err(ServiceError::InvalidRequest(format!( - "mount {} is already bound to base revision {}", - mount_id, existing + entry.state = MountLifecycle::Quiescing; + if let Err(e) = entry.fuse.unmount().await { + entry.state = MountLifecycle::Ready; + return Err(ServiceError::FuseFailure(format!( + "failed to quiesce mount {mount_id}: {e}" ))); } - return self.worktree_state(mount_id).await; } - let state = self.worktree_state(mount_id).await?; - if state.dirty { - return Err(ServiceError::InvalidRequest( - "cannot bind a worktree base after local upper-layer changes exist".into(), - )); - } + let mut new_fuse = match Self::remount_with_lower( + &mountpoint, + new_dicfuse.clone(), + &upper_dir, + cl_dir.as_deref(), + &[], + ) + .await + { + Ok(fuse) => fuse, + Err(e) => { + let _ = Self::remount_with_lower( + &mountpoint, + old_dicfuse, + &upper_dir, + cl_dir.as_deref(), + &sealed_chain, + ) + .await; + return Ok(RefreshResponse { + disposition: RefreshDisposition::BaseMismatch, + base_revision: base_revision.unwrap_or_default(), + lower_revision: pinned_refs, + generation, + code: Some("SWITCH_FAILED".into()), + detail: Some(format!( + "remount over revision {target} failed: {e}; \ + worktree remounted on the previous projection" + )), + }); + } + }; { let mut mounts = self.mounts.write().await; let entry = mounts .get_mut(&mount_id) .ok_or(ServiceError::NotFound(mount_id))?; - if !matches!(entry.state, MountLifecycle::Mounted | MountLifecycle::Ready) { - return Err(ServiceError::InvalidRequest(format!( - "mount {} is currently in state {:?}; cannot bind a Libra worktree base", - mount_id, entry.state - ))); - } - if entry.cl.is_some() { - return Err(ServiceError::InvalidRequest( - "cannot bind a Libra worktree base to a mount with a CL layer".into(), - )); - } - if let Some(existing) = &entry.base_revision { - if existing != base_revision { - return Err(ServiceError::InvalidRequest(format!( - "mount {} is already bound to base revision {}", - mount_id, existing - ))); - } - } else { - entry.base_revision = Some(base_revision.to_string()); - entry.update_last_seen(); - } + entry.fuse = new_fuse; + entry.pinned_refs = Some(target.clone()); + // Flattened above: the chain no longer applies to the new revision. + entry.sealed_chain = Vec::new(); + entry.state = MountLifecycle::Ready; + entry.last_seen_epoch_ms = current_epoch_ms(); } - self.persist_state().await; - self.worktree_state(mount_id).await - } - async fn plan_worktree_refresh( - &self, - mount_id: Uuid, - request: RefreshPlanRequest, - ) -> Result { - let expected = request.expected_base_revision.trim(); - let target = request.target_revision.trim(); - if expected.is_empty() || target.is_empty() { - return Err(ServiceError::InvalidRequest( - "expected_base_revision and target_revision cannot be empty".into(), - )); - } + let changes = effective_changes(&DicfuseLower(new_dicfuse.store.clone()), &upper_dir, &[]) + .await + .map_err(|e| ServiceError::Internal(format!("effective scan failed: {e}")))?; + let generation = generation_of(&changes); - let worktree = self.worktree_state(mount_id).await?; - let disposition = match worktree.base_revision.as_deref() { - None => RefreshPlanDisposition::Unbound, - Some(current) if current != expected => RefreshPlanDisposition::BaseMismatch, - Some(current) if current == target => RefreshPlanDisposition::AlreadyAtTarget, - Some(_) if request.require_clean && worktree.dirty => { - RefreshPlanDisposition::BlockedDirty - } - Some(_) => RefreshPlanDisposition::Ready, - }; + tracing::info!( + mount_id = %mount_id, + lower_revision = %target, + elapsed_ms = start.elapsed().as_millis() as u64, + "antares svc: refresh_lower success" + ); - Ok(RefreshPlanResponse { - mount_id, - current_base_revision: worktree.base_revision.clone(), - target_revision: target.to_string(), - disposition, - worktree, + Ok(RefreshResponse { + disposition: RefreshDisposition::Switched, + base_revision: base_revision.unwrap_or_default(), + lower_revision: Some(target), + generation, + code: None, + detail: None, }) } @@ -2368,6 +4467,7 @@ impl AntaresService for AntaresServiceImpl { let cl = entry.cl.clone(); let job_id = entry.job_id.clone(); let job_id_for_log = job_id.clone(); + let sealed_chain = entry.sealed_chain.clone(); tracing::info!( mount_id = %mount_id, task_id = ?job_id_for_log, @@ -2455,6 +4555,21 @@ impl AntaresService for AntaresServiceImpl { } else { index.remove(&(path, cl, cl_path)); } + // Sealed layers of a chain fork are shared: reclaim one only when no + // surviving mount still references it. Computed before the locks drop. + let orphaned = if sealed_chain.is_empty() { + Vec::new() + } else { + let referenced: std::collections::HashSet<&str> = mounts + .values() + .flat_map(|e| e.sealed_chain.iter().map(String::as_str)) + .collect(); + sealed_chain + .iter() + .filter(|d| !referenced.contains(d.as_str())) + .cloned() + .collect() + }; drop(mounts); drop(index); drop(job_index); @@ -2468,6 +4583,11 @@ impl AntaresService for AntaresServiceImpl { // The instance is gone from the maps and cannot be reattached, so // reclaim its mountpoint and private layers (outside the locks). Self::remove_mount_dirs(mount_id, &mountpoint, &upper_dir, cl_dir.as_deref()); + for dir in orphaned { + if let Err(e) = tokio::fs::remove_dir_all(&dir).await { + tracing::warn!("sealed layer {} cleanup failed: {e}", dir); + } + } // Persist state to file for recovery self.persist_state().await; @@ -3846,6 +5966,14 @@ mod tests { cl_path: None, path: format!("/project/path{}", i), cl: None, + + upper_dir: None, + + mountpoint: None, + + pinned_refs: None, + + sealed_chain: Vec::new(), }) .await }) @@ -3871,6 +5999,14 @@ mod tests { cl_path: None, path: "/third-party/mega".into(), cl: Some("CL123".into()), + + upper_dir: None, + + mountpoint: None, + + pinned_refs: None, + + sealed_chain: Vec::new(), }; // First mount should succeed @@ -3892,6 +6028,14 @@ mod tests { cl_path: None, path: "/third-party/mega".into(), cl: Some("CL123".into()), + + upper_dir: None, + + mountpoint: None, + + pinned_refs: None, + + sealed_chain: Vec::new(), }; let first = service.create_mount(request.clone()).await.unwrap(); @@ -3911,6 +6055,14 @@ mod tests { cl_path: None, path: "/third-party/mega".into(), cl: Some("CL123".into()), + + upper_dir: None, + + mountpoint: None, + + pinned_refs: None, + + sealed_chain: Vec::new(), }; let first = service.create_mount(request.clone()).await.unwrap(); @@ -3936,6 +6088,14 @@ mod tests { cl_path: None, path: "/third-party/mega".into(), cl: Some("CL123".into()), + + upper_dir: None, + + mountpoint: None, + + pinned_refs: None, + + sealed_chain: Vec::new(), }; let req2 = CreateMountRequest { job_id: Some("job-b".into()), @@ -3943,6 +6103,14 @@ mod tests { cl_path: None, path: "/third-party/mega".into(), cl: Some("CL123".into()), + + upper_dir: None, + + mountpoint: None, + + pinned_refs: None, + + sealed_chain: Vec::new(), }; let r1 = service.create_mount(req1).await; @@ -3966,6 +6134,14 @@ mod tests { cl_path: None, path: "/third-party/mega".into(), cl: None, + + upper_dir: None, + + mountpoint: None, + + pinned_refs: None, + + sealed_chain: Vec::new(), }) .await .unwrap(); @@ -3993,6 +6169,14 @@ mod tests { cl_path: None, path: "/third-party/mega".into(), cl: Some("CL1".into()), + + upper_dir: None, + + mountpoint: None, + + pinned_refs: None, + + sealed_chain: Vec::new(), }) .await; assert!(result1.is_ok()); @@ -4005,6 +6189,14 @@ mod tests { cl_path: None, path: "/third-party/mega".into(), cl: Some("CL2".into()), + + upper_dir: None, + + mountpoint: None, + + pinned_refs: None, + + sealed_chain: Vec::new(), }) .await; assert!(result2.is_ok()); @@ -4032,6 +6224,14 @@ mod tests { cl_path: None, path: format!("/concurrent-path-{}", i), cl: None, + + upper_dir: None, + + mountpoint: None, + + pinned_refs: None, + + sealed_chain: Vec::new(), }; svc.create_mount(request).await }); @@ -4077,6 +6277,14 @@ mod tests { cl_path: None, path: "/test-concurrent-ops".to_string(), cl: None, + + upper_dir: None, + + mountpoint: None, + + pinned_refs: None, + + sealed_chain: Vec::new(), }; let created = service.create_mount(request).await.unwrap(); let mount_id = created.mount_id; @@ -4113,6 +6321,14 @@ mod tests { cl_path: None, path: "/third-party/mega".into(), cl: None, + + upper_dir: None, + + mountpoint: None, + + pinned_refs: None, + + sealed_chain: Vec::new(), }) .await .unwrap(); @@ -4136,6 +6352,14 @@ mod tests { cl_path: None, path: "/third-party/mega".into(), cl: None, + + upper_dir: None, + + mountpoint: None, + + pinned_refs: None, + + sealed_chain: Vec::new(), }) .await .unwrap(); @@ -4161,6 +6385,14 @@ mod tests { cl_path: None, path: "/third-party/mega".into(), cl: None, + + upper_dir: None, + + mountpoint: None, + + pinned_refs: None, + + sealed_chain: Vec::new(), }) .await .unwrap(); @@ -4198,6 +6430,14 @@ mod tests { cl_path: None, path: "/third-party/mega".into(), cl: Some("CL123".into()), + + upper_dir: None, + + mountpoint: None, + + pinned_refs: None, + + sealed_chain: Vec::new(), }) .await .unwrap(); @@ -4223,6 +6463,14 @@ mod tests { cl_path: None, path: "/third-party/mega".into(), cl: None, + + upper_dir: None, + + mountpoint: None, + + pinned_refs: None, + + sealed_chain: Vec::new(), }) .await .unwrap(); @@ -4245,6 +6493,14 @@ mod tests { cl_path: None, path: "/third-party/mega".into(), cl: Some("CL123".into()), + + upper_dir: None, + + mountpoint: None, + + pinned_refs: None, + + sealed_chain: Vec::new(), }) .await .unwrap(); @@ -4272,6 +6528,14 @@ mod tests { cl_path: None, path: "/test/path".into(), cl: None, + + upper_dir: None, + + mountpoint: None, + + pinned_refs: None, + + sealed_chain: Vec::new(), }) .await .unwrap(); @@ -4313,6 +6577,14 @@ mod tests { cl_path: None, path: "/test/path".into(), cl: Some("CL123".into()), + + upper_dir: None, + + mountpoint: None, + + pinned_refs: None, + + sealed_chain: Vec::new(), }) .await .unwrap(); diff --git a/src/daemon/lower_view.rs b/src/daemon/lower_view.rs new file mode 100644 index 0000000..cbdc0ac --- /dev/null +++ b/src/daemon/lower_view.rs @@ -0,0 +1,107 @@ +//! Content-identity domains for the effective diff (P3, spec 12 §1). +//! +//! The lower projection answers content identity in its own domain: the +//! Dicfuse path speaks the mega content-hash (a git blob OID), while an MST/2 +//! snapshot view speaks `sha256(raw bytes)` (spec 03 §1). The effective diff +//! must hash the *upper* layer in the same domain, otherwise "edited back to +//! base" and "modified" cannot be told apart. +//! +//! [`LowerView`] makes that domain explicit; [`DicfuseLower`] is the legacy +//! implementation, and the MST/2 one lives next to the snapshot view. + +use std::sync::Arc; + +use async_trait::async_trait; + +use crate::daemon::worktree_v2::{git_blob_oid, lower_item_for}; +use crate::dicfuse::store::DictionaryStore; +use crate::snapshot::fuse::Mst2Fuse; + +/// Which content-identity domain a lower projection speaks. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum LowerHashKind { + /// `sha1("blob \0" + content)` — the mega content-hash, i.e. a git + /// blob OID (bare 40 hex). + GitBlobOid, + /// `sha256:<64 hex>` over the raw bytes — the MST/2 `content_digest` + /// (spec 03 §1). Symlinks hash their target bytes, matching spec 07 §1. + Sha256Raw, +} + +/// Content identity of `bytes` in `kind`'s domain. +pub fn hash_content(kind: LowerHashKind, bytes: &[u8]) -> String { + match kind { + LowerHashKind::GitBlobOid => git_blob_oid(bytes), + LowerHashKind::Sha256Raw => crate::snapshot::durable::digest_of(bytes), + } +} + +/// A read-only lower projection as the effective diff sees it. +#[async_trait] +pub trait LowerView: Send + Sync { + /// Content identity of `rel_path` as the lower serves it; `None` = the + /// path is absent from this projection (deleted or never existed). + async fn base_hash(&self, rel_path: &str) -> Option; + + /// The domain [`Self::base_hash`] answers in. + fn hash_kind(&self) -> LowerHashKind; +} + +/// The Dicfuse projection as a [`LowerView`]: mega content-hash (git blob OID). +pub struct DicfuseLower(pub Arc); + +#[async_trait] +impl LowerView for DicfuseLower { + async fn base_hash(&self, rel_path: &str) -> Option { + lower_item_for(&self.0, rel_path).await.map(|item| item.hash) + } + + fn hash_kind(&self) -> LowerHashKind { + LowerHashKind::GitBlobOid + } +} + +/// An MST/2 snapshot view as a [`LowerView`]: `sha256(raw)` content digests +/// (spec 03 §1). Symlinks answer the digest of their target bytes, which is +/// what the view stores and what the upper side hashes too (spec 07 §1). +pub struct Mst2Lower(pub Arc); + +#[async_trait] +impl LowerView for Mst2Lower { + async fn base_hash(&self, rel_path: &str) -> Option { + self.0.digest_for_path(rel_path).await + } + + fn hash_kind(&self) -> LowerHashKind { + LowerHashKind::Sha256Raw + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The git-blob domain must reproduce the value the monorepo server + /// reports for the same content (the existing cross-check), and the sha256 + /// domain must use the view's wire form. + #[test] + fn both_domains_hash_their_wire_forms() { + let content = b"formal-backend-sync\n"; + assert_eq!( + hash_content(LowerHashKind::GitBlobOid, content), + "d93a4449096163af2b8789689d1923f96b182671" + ); + let sha = hash_content(LowerHashKind::Sha256Raw, content); + assert!(sha.starts_with("sha256:"), "{sha}"); + assert_eq!(sha.len(), "sha256:".len() + 64); + } + + #[test] + fn empty_content_hashes_are_well_formed() { + assert_eq!( + hash_content(LowerHashKind::GitBlobOid, b""), + "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391" + ); + assert!(hash_content(LowerHashKind::Sha256Raw, b"").starts_with("sha256:")); + } +} diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index a267f5c..254f0b5 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -19,6 +19,9 @@ use crate::{ util::{config, GPath}, }; pub mod antares; +pub mod lower_view; +pub mod upper_fork; +pub mod worktree_v2; //mod git; const SUCCESS: &str = "Success"; diff --git a/src/daemon/upper_fork.rs b/src/daemon/upper_fork.rs new file mode 100644 index 0000000..bcc57ef --- /dev/null +++ b/src/daemon/upper_fork.rs @@ -0,0 +1,505 @@ +//! Copying an Antares upper layer, for `fork`. +//! +//! An upper layer is a passthrough directory holding a worktree's private edits: +//! the files a user modified, plus a *whiteout* entry for every file they deleted +//! from the read-only lower layer. +//! +//! `fork` derives a second worktree from a first one, so the child must start out +//! seeing exactly what the parent saw at fork time. Copying the upper layer is how +//! that state is transferred when the two worktrees are not sharing layers. +//! +//! Two properties are load-bearing and easy to get wrong: +//! +//! * **A hard link is not a copy here.** The Antares upper is a *passthrough* +//! directory: the overlay writes to upper files in place, through the same +//! inode. Hard-linking a child's file to the parent's would make the child +//! observe every later write in the parent, silently and with no error. So a +//! file is either *reflinked* (`FICLONE`, a copy-on-write clone with its own +//! inode) or really copied — never linked. +//! +//! * **The parent may be writing while we read.** A file can be captured +//! half-written. Every file is therefore re-`stat`ed after it is copied and the +//! copy is retried if the source moved; if it will not settle, the fork fails +//! rather than producing a child that never existed. + +use std::collections::BTreeSet; +use std::fs; +use std::io; +use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt}; +use std::os::unix::io::AsRawFd; +use std::path::{Path, PathBuf}; + +/// The VCS metadata directory inside a mount. It is a reconstructable pointer, not +/// part of the writable delta, so it is never copied — the child gets its own. +const VCS_POINTER_DIR: &str = ".libra"; + +/// `FICLONE` from `linux/fs.h` (values `_IOW(0x94, 9, int)`). +const FICLONE: libc::c_ulong = 0x4004_9409; + +/// Per-file retries before a run of the delta copy is declared torn. +const FILE_ATTEMPTS: u32 = 3; + +/// Whole-delta passes before the source is declared too busy to fork from. +const DELTA_ATTEMPTS: u32 = 3; + +/// What a completed fork actually copied. +#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct ForkCopyStats { + /// Number of files written into the child upper layer. + pub files: u64, + /// Total bytes of those files (reflinked files count their logical size). + pub bytes: u64, + /// How many files were cloned with `FICLONE` instead of byte-copied. + pub reflink_used: u64, + /// How many per-file retries were needed because the source was moving. + pub retries: u64, +} + +#[derive(Debug, thiserror::Error)] +pub enum ForkCopyError { + #[error("upper layer I/O error at {path}: {source}")] + Io { + path: PathBuf, + #[source] + source: io::Error, + }, + #[error( + "the source upper layer contains a {kind} at {path}, which fork does not know how to copy" + )] + UnsupportedEntry { path: PathBuf, kind: &'static str }, + #[error( + "the source upper layer kept changing while it was copied ({attempts} attempts); \ + quiesce the source worktree and fork again" + )] + SourceBusy { attempts: u32 }, +} + +fn io_err(path: &Path) -> impl FnOnce(io::Error) -> ForkCopyError + '_ { + move |source| ForkCopyError::Io { + path: path.to_path_buf(), + source, + } +} + +/// Copy the writable delta of `src_upper` into `dst_upper`. +/// +/// `dst_upper` is created if missing; it must not already contain entries, because +/// merging a second delta into a non-empty layer has no defined precedence. +/// +/// `.libra` is skipped at the top level only — the child owns its own pointer. +pub fn fork_upper(src_upper: &Path, dst_upper: &Path) -> Result { + fs::create_dir_all(dst_upper).map_err(io_err(dst_upper))?; + ensure_destination_empty(dst_upper)?; + + let mut total = ForkCopyStats::default(); + + for attempt in 1..=DELTA_ATTEMPTS { + let delta = list_delta(src_upper)?; + let mut stats = ForkCopyStats::default(); + + for rel in &delta { + copy_one(src_upper, dst_upper, rel, &mut stats)?; + } + + // Re-list: a file added or removed while we walked would otherwise be lost. + // (Content changes within a stable path set are caught per-file instead.) + let after = list_delta(src_upper)?; + if after == delta { + stats.retries += total.retries; + return Ok(stats); + } + + total.retries += stats.retries + 1; + if attempt == DELTA_ATTEMPTS { + return Err(ForkCopyError::SourceBusy { + attempts: DELTA_ATTEMPTS, + }); + } + // Retry with a fresh listing; already-copied paths are re-verified. + reset_destination(dst_upper)?; + } + + unreachable!("the loop either returns Ok or Err") +} + +fn ensure_destination_empty(dst: &Path) -> Result<(), ForkCopyError> { + let mut entries = fs::read_dir(dst).map_err(io_err(dst))?; + if let Some(entry) = entries.next() { + let entry = entry.map_err(io_err(dst))?; + return Err(ForkCopyError::Io { + path: entry.path(), + source: io::Error::new( + io::ErrorKind::AlreadyExists, + "fork destination upper layer is not empty", + ), + }); + } + Ok(()) +} + +/// Empty the destination between delta attempts. +fn reset_destination(dst: &Path) -> Result<(), ForkCopyError> { + for entry in fs::read_dir(dst).map_err(io_err(dst))? { + let entry = entry.map_err(io_err(dst))?; + let path = entry.path(); + let meta = fs::symlink_metadata(&path).map_err(io_err(&path))?; + if meta.is_dir() { + fs::remove_dir_all(&path).map_err(io_err(&path))?; + } else { + fs::remove_file(&path).map_err(io_err(&path))?; + } + } + Ok(()) +} + +/// Every path in the layer, relative to its root, excluding the VCS pointer dir. +/// +/// Directories are included so that empty directories survive the copy. +fn list_delta(root: &Path) -> Result, ForkCopyError> { + let mut out = BTreeSet::new(); + let mut pending = vec![PathBuf::new()]; + + while let Some(dir) = pending.pop() { + for entry in fs::read_dir(root.join(&dir)).map_err(io_err(&root.join(&dir)))? { + let entry = entry.map_err(io_err(&root.join(&dir)))?; + let name = entry.file_name(); + let rel = dir.join(&name); + + if dir.as_os_str().is_empty() && name == VCS_POINTER_DIR { + continue; + } + + let meta = fs::symlink_metadata(entry.path()).map_err(io_err(&entry.path()))?; + if meta.is_dir() { + pending.push(rel.clone()); + } + out.insert(rel); + } + } + Ok(out) +} + +fn copy_one( + src_root: &Path, + dst_root: &Path, + rel: &Path, + stats: &mut ForkCopyStats, +) -> Result<(), ForkCopyError> { + // `rel` comes from our own walk, but this function is a public-facing seam: + // refuse anything that could escape the layer root. + if rel.is_absolute() + || rel + .components() + .any(|c| c == std::path::Component::ParentDir) + { + return Err(ForkCopyError::UnsupportedEntry { + path: rel.to_path_buf(), + kind: "path outside the layer", + }); + } + + let src = src_root.join(rel); + let dst = dst_root.join(rel); + let meta = fs::symlink_metadata(&src).map_err(io_err(&src))?; + + if meta.is_dir() { + fs::create_dir_all(&dst).map_err(io_err(&dst))?; + fs::set_permissions(&dst, fs::Permissions::from_mode(meta.mode() & 0o7777)) + .map_err(io_err(&dst))?; + return Ok(()); + } + + if let Some(parent) = dst.parent() { + fs::create_dir_all(parent).map_err(io_err(parent))?; + } + + if meta.file_type().is_symlink() { + let target = fs::read_link(&src).map_err(io_err(&src))?; + let _ = fs::remove_file(&dst); + std::os::unix::fs::symlink(&target, &dst).map_err(io_err(&dst))?; + return Ok(()); + } + + if !meta.is_file() { + // A char/block device here is a Linux char-device whiteout: the form + // Antares does not use (see `ANTARES_WHITEOUT_FORMAT`). Refuse loudly + // rather than copy something whose meaning we would be guessing at. + let kind = if meta.file_type().is_char_device() { + "character device" + } else if meta.file_type().is_block_device() { + "block device" + } else if meta.file_type().is_fifo() { + "fifo" + } else if meta.file_type().is_socket() { + "socket" + } else { + "special file" + }; + return Err(ForkCopyError::UnsupportedEntry { + path: rel.to_path_buf(), + kind, + }); + } + + copy_file_verified(&src, &dst, stats) +} + +/// Copy one regular file, retrying while the source is being written. +/// +/// Reflink first (an O(1) CoW clone with its own inode), falling back to a byte +/// copy on filesystems without `FICLONE` (e.g. ext4 without reflink support). +fn copy_file_verified( + src: &Path, + dst: &Path, + stats: &mut ForkCopyStats, +) -> Result<(), ForkCopyError> { + for attempt in 0..FILE_ATTEMPTS { + let before = fs::symlink_metadata(src).map_err(io_err(src))?; + if !before.is_file() { + // Replaced by a non-file while we looked at it; let the next delta pass + // see whatever it became. + return Err(ForkCopyError::SourceBusy { + attempts: FILE_ATTEMPTS, + }); + } + + let cloned = try_reflink(src, dst, before.size()); + let written_bytes = if cloned { + stats.reflink_used += 1; + before.size() + } else { + fs::copy(src, dst).map_err(io_err(dst))? + }; + fs::set_permissions(dst, fs::Permissions::from_mode(before.mode() & 0o7777)) + .map_err(io_err(dst))?; + + let after = fs::symlink_metadata(src).map_err(io_err(src))?; + if same_file_state(&before, &after) { + stats.files += 1; + stats.bytes += written_bytes; + return Ok(()); + } + + // The source moved under us; discard and retry. + stats.retries += 1; + let _ = fs::remove_file(dst); + if attempt + 1 == FILE_ATTEMPTS { + return Err(ForkCopyError::SourceBusy { + attempts: FILE_ATTEMPTS, + }); + } + } + unreachable!("the loop either returns Ok or Err") +} + +/// Everything that must be unchanged for a copy to be a real snapshot of one instant. +fn same_file_state(a: &fs::Metadata, b: &fs::Metadata) -> bool { + a.ino() == b.ino() + && a.size() == b.size() + && a.mtime() == b.mtime() + && a.mtime_nsec() == b.mtime_nsec() + && a.ctime() == b.ctime() + && a.ctime_nsec() == b.ctime_nsec() +} + +/// Try `FICLONE`. Returns whether the file was cloned into `dst`. +/// +/// Any failure (unsupported filesystem, cross-device, ...) is reported as "not +/// cloned" so the caller falls back to a byte copy. +fn try_reflink(src: &Path, dst: &Path, size: u64) -> bool { + let Ok(src_file) = fs::File::open(src) else { + return false; + }; + let Ok(dst_file) = fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(dst) + else { + return false; + }; + + // SAFETY: both fds are valid for the duration of the call; FICLONE takes an + // int argument (the source fd) and reads no user memory. + let rc = unsafe { libc::ioctl(dst_file.as_raw_fd(), FICLONE, src_file.as_raw_fd()) }; + if rc != 0 { + return false; + } + + // A clone that did not carry the data would be worse than no clone at all. + match dst_file.metadata() { + Ok(meta) => meta.size() == size, + Err(_) => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn write(path: &Path, contents: &str) { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).unwrap(); + } + fs::write(path, contents).unwrap(); + } + + #[test] + fn copies_modified_files_and_directories() { + let tmp = tempfile::tempdir().unwrap(); + let src = tmp.path().join("src"); + let dst = tmp.path().join("dst"); + + write(&src.join("a.txt"), "alpha"); + write(&src.join("deep/nested/b.txt"), "beta"); + fs::create_dir_all(src.join("empty-dir")).unwrap(); + + let stats = fork_upper(&src, &dst).unwrap(); + + assert_eq!(stats.files, 2); + assert_eq!(fs::read_to_string(dst.join("a.txt")).unwrap(), "alpha"); + assert_eq!( + fs::read_to_string(dst.join("deep/nested/b.txt")).unwrap(), + "beta" + ); + assert!(dst.join("empty-dir").is_dir(), "empty dir must survive"); + } + + #[test] + fn skips_the_vcs_pointer_at_the_top_level_only() { + let tmp = tempfile::tempdir().unwrap(); + let src = tmp.path().join("src"); + let dst = tmp.path().join("dst"); + + write(&src.join(".libra/commondir"), "/host/main/.libra\n"); + write(&src.join(".libra/worktree_id"), "wt-a\n"); + // A *nested* `.libra` is an ordinary directory and must be copied. + write(&src.join("vendor/.libra/keep.txt"), "keep"); + + fork_upper(&src, &dst).unwrap(); + + assert!( + !dst.join(".libra").exists(), + "the child must get its own pointer, not the parent's" + ); + assert_eq!( + fs::read_to_string(dst.join("vendor/.libra/keep.txt")).unwrap(), + "keep" + ); + } + + #[test] + fn copies_whiteouts_as_ordinary_files() { + let tmp = tempfile::tempdir().unwrap(); + let src = tmp.path().join("src"); + let dst = tmp.path().join("dst"); + + // The OCI whiteout form: an empty regular `.wh.` file. + write(&src.join("src/.wh.gone.rs"), ""); + write(&src.join("src/.wh..wh..opq"), ""); + + fork_upper(&src, &dst).unwrap(); + + let wh = dst.join("src/.wh.gone.rs"); + let meta = fs::symlink_metadata(&wh).unwrap(); + assert!( + meta.is_file(), + "whiteout must stay a regular file, not become a device" + ); + assert_eq!(meta.size(), 0, "an OCI whiteout is empty"); + assert!( + dst.join("src/.wh..wh..opq").exists(), + "opaque marker copied" + ); + } + + #[test] + fn copies_symlinks_without_following_them() { + let tmp = tempfile::tempdir().unwrap(); + let src = tmp.path().join("src"); + let dst = tmp.path().join("dst"); + + write(&src.join("real.txt"), "real"); + std::os::unix::fs::symlink("real.txt", src.join("link.txt")).unwrap(); + + fork_upper(&src, &dst).unwrap(); + + let meta = fs::symlink_metadata(dst.join("link.txt")).unwrap(); + assert!(meta.file_type().is_symlink()); + assert_eq!( + fs::read_link(dst.join("link.txt")).unwrap(), + PathBuf::from("real.txt") + ); + } + + #[test] + fn refuses_a_non_empty_destination() { + let tmp = tempfile::tempdir().unwrap(); + let src = tmp.path().join("src"); + let dst = tmp.path().join("dst"); + + write(&src.join("a.txt"), "alpha"); + write(&dst.join("already-there.txt"), "clash"); + + let err = fork_upper(&src, &dst).unwrap_err(); + assert!(matches!(err, ForkCopyError::Io { .. }), "got {err:?}"); + } + + #[test] + fn preserves_the_executable_bit() { + let tmp = tempfile::tempdir().unwrap(); + let src = tmp.path().join("src"); + let dst = tmp.path().join("dst"); + + write(&src.join("run.sh"), "#!/bin/sh\n"); + fs::set_permissions(src.join("run.sh"), fs::Permissions::from_mode(0o755)).unwrap(); + + fork_upper(&src, &dst).unwrap(); + + let mode = fs::symlink_metadata(dst.join("run.sh")) + .unwrap() + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o755, "mode was {mode:o}"); + } + + #[test] + fn copied_files_are_separate_inodes_not_hard_links() { + // The whole point of the hard-link ban: a later in-place write to the + // parent's file must not appear in the child. + let tmp = tempfile::tempdir().unwrap(); + let src = tmp.path().join("src"); + let dst = tmp.path().join("dst"); + + write(&src.join("shared.txt"), "parent-before"); + fork_upper(&src, &dst).unwrap(); + + let src_ino = fs::symlink_metadata(src.join("shared.txt")).unwrap().ino(); + let dst_ino = fs::symlink_metadata(dst.join("shared.txt")).unwrap().ino(); + assert_ne!(src_ino, dst_ino, "copy must not be a hard link"); + + // Simulate the overlay writing the parent's file in place. + fs::write(src.join("shared.txt"), "parent-after-write").unwrap(); + + assert_eq!( + fs::read_to_string(dst.join("shared.txt")).unwrap(), + "parent-before", + "the child must not observe later writes in the parent" + ); + } + + #[test] + fn rejects_a_path_escaping_the_layer() { + let tmp = tempfile::tempdir().unwrap(); + let src = tmp.path().join("src"); + let dst = tmp.path().join("dst"); + fs::create_dir_all(&src).unwrap(); + + let mut stats = ForkCopyStats::default(); + let err = copy_one(&src, &dst, Path::new("../escape.txt"), &mut stats).unwrap_err(); + assert!( + matches!(err, ForkCopyError::UnsupportedEntry { .. }), + "got {err:?}" + ); + } +} diff --git a/src/daemon/worktree_v2.rs b/src/daemon/worktree_v2.rs new file mode 100644 index 0000000..8ae019d --- /dev/null +++ b/src/daemon/worktree_v2.rs @@ -0,0 +1,863 @@ +//! Worktree Control Protocol v2: effective diff, commit finalize, lower switch. +//! +//! The v1 contract reported every upper-layer entry as a change and never moved the +//! lower projection, which produced two failures this module fixes: +//! +//! 1. **Phantom dirt.** An upper file whose content equals the base (edited back, or +//! left by an earlier commit) was reported `modified` forever. +//! 2. **Stale upper shadowing a newer lower.** After a commit was pushed, the upper +//! kept the committed content indefinitely, so a later remote update to the same +//! path would be invisible behind the stale overlay. +//! +//! The fix is a per-path comparison of upper content against the *lower* projection +//! ([`effective_changes`]), plus a transactional finalize ([`CommitFinalizeRequest`]) +//! that pins the lower to the committed revision and removes exactly the committed +//! upper entries. See `docs/scorpiofs-libra-complete-spec-v1.md`. + +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +use crate::daemon::lower_view::{hash_content, LowerHashKind, LowerView}; +use crate::dicfuse::store::DictionaryStore; +use crate::dicfuse::tree_store::StorageItem; + +/// Effective change kinds, in Git terms. `added`/`modified` compare upper content +/// against the lower projection; `deleted` is an OCI whiteout over a lower entry. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EffectiveKind { + Added, + Modified, + Deleted, +} + +/// One path whose effective content differs from the lower projection. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct EffectiveChange { + /// Path relative to the mount root, `/`-separated, no leading slash. + pub path: String, + pub kind: EffectiveKind, + /// Git blob OID of the current upper content (`null` for deletions). + pub content_hash: Option, + /// Git blob OID of the base (lower) content (`null` when the path is new). + pub base_hash: Option, +} + +/// A path a VCS client says its commit has absorbed. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CommittedPath { + /// Path relative to the mount root, `/`-separated, no leading slash. + pub path: String, + pub kind: EffectiveKind, + /// Git blob OID the path was committed with. `None` for deletions. The finalize + /// verifies this is exactly what the new lower serves, which both proves the + /// commit landed and acts as a per-path optimistic lock against concurrent edits. + #[serde(default)] + pub content_hash: Option, +} + +// --------------------------------------------------------------------------- +// Hashing +// --------------------------------------------------------------------------- + +/// Compute the git blob OID of `content`, matching what Mega's `content-hash` +/// entries contain for the same bytes. +/// +/// Uses the process-wide hash kind (SHA-1 unless someone reconfigured it), which is +/// the format the monorepo server serves for the repositories this integration +/// targets. A length check against the lower's OID at comparison time catches a +/// mismatched configuration instead of silently reporting phantom changes. +pub fn git_blob_oid(content: &[u8]) -> String { + git_internal::internal::object::blob::Blob::from_content_bytes(content.to_vec()) + .id + .to_string() +} + +// --------------------------------------------------------------------------- +// Upper-layer scanning +// --------------------------------------------------------------------------- + +/// One physical entry found in the upper directory. +#[derive(Debug)] +struct UpperEntry { + /// Path relative to the upper root, `/`-separated, no leading or trailing slash. + rel_path: String, + kind: UpperEntryKind, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum UpperEntryKind { + File, + Symlink, + /// OCI whiteout: the name it hides is `rel_path`. + Whiteout, +} + +const WHITEOUT_PREFIX: &str = ".wh."; +const WHITEOUT_OPAQUE: &str = ".wh..wh..opq"; +/// VCS metadata inside the mount; a reconstructable pointer, never a worktree edit. +const VCS_POINTER_DIR: &str = ".libra"; + +fn classify_upper_entry(name: &str) -> Option { + if name == WHITEOUT_OPAQUE { + // Directory-wide marker: it does not name one path, and the entries it + // hides are handled when the scanner walks the (now absent) children. + return None; + } + if let Some(hidden) = name.strip_prefix(WHITEOUT_PREFIX) { + if hidden.is_empty() { + return None; + } + return Some(UpperEntryKind::Whiteout); + } + None +} + +/// Walk the upper directory and return every physical entry. +/// +/// Whiteouts are returned under the *hidden* name (`src/.wh.a.rs` scans as +/// `src/a.rs` with kind `Whiteout`). The VCS pointer directory is skipped at the +/// top level, mirroring the change scanner. +fn scan_upper_entries(upper_dir: &Path) -> std::io::Result> { + let mut out = Vec::new(); + let mut pending = vec![PathBuf::new()]; + + while let Some(dir) = pending.pop() { + for entry in std::fs::read_dir(upper_dir.join(&dir))? { + let entry = entry?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + + if dir.as_os_str().is_empty() && name == VCS_POINTER_DIR { + continue; + } + + // file_type() does not follow symlinks: a link must be hashed as its + // target bytes, never traversed. + let file_type = entry.file_type()?; + let rel = if dir.as_os_str().is_empty() { + name.to_string() + } else { + format!("{}/{}", dir.display(), name) + }; + + if file_type.is_dir() { + pending.push(PathBuf::from(&rel)); + continue; + } + + if let Some(kind) = classify_upper_entry(&name) { + let hidden = if let Some(hidden) = name.strip_prefix(WHITEOUT_PREFIX) { + if dir.as_os_str().is_empty() { + hidden.to_string() + } else { + format!("{}/{}", dir.display(), hidden) + } + } else { + rel.clone() + }; + out.push(UpperEntry { + rel_path: hidden, + kind, + }); + continue; + } + + let kind = if file_type.is_symlink() { + UpperEntryKind::Symlink + } else { + UpperEntryKind::File + }; + out.push(UpperEntry { + rel_path: rel, + kind, + }); + } + } + + Ok(out) +} + +/// Git blob OID of an upper entry's current content. +/// +/// Symlinks hash their target bytes (git stores the target as the blob), matching +/// how the monorepo serves them. +fn upper_content_oid( + upper_dir: &Path, + entry: &UpperEntry, + kind: LowerHashKind, +) -> std::io::Result { + let bytes = match entry.kind { + UpperEntryKind::Symlink => std::fs::read_link(upper_dir.join(&entry.rel_path))? + .as_os_str() + .as_encoded_bytes() + .to_vec(), + _ => std::fs::read(upper_dir.join(&entry.rel_path))?, + }; + Ok(hash_content(kind, &bytes)) +} + +/// Resolve the lower projection's item for a mount-relative path. +/// +/// The tree store loads lazily, so a miss on a deep path may just mean "the parent +/// directory was never fetched". This walks the ancestors top-down, forcing each +/// directory load, and retries before concluding the path is absent from the lower. +pub(crate) async fn lower_item_for(store: &DictionaryStore, rel_path: &str) -> Option { + let user_path = format!("/{rel_path}"); + if let Ok(item) = store.get_by_path(&user_path).await { + return Some(item); + } + + let parts: Vec<&str> = rel_path.split('/').filter(|p| !p.is_empty()).collect(); + if parts.len() <= 1 { + return None; + } + + // Ensure the tree root is loaded, then each ancestor down to the parent. + let _ = store.ensure_dir_loaded(1).await; + let mut walked = String::new(); + for part in &parts[..parts.len() - 1] { + walked = format!("{walked}/{part}"); + match store.get_inode_from_path(&walked).await { + Ok(inode) => { + let _ = store.ensure_dir_loaded(inode).await; + } + Err(_) => { + let _ = store.ensure_dir_loaded(1).await; + } + } + } + + store.get_by_path(&user_path).await.ok() +} + +/// The base content a chain layer or the projection serves for one path. +#[derive(Debug, Clone, PartialEq, Eq)] +enum ChainBase { + /// The path exists with this git blob OID. + Hash(String), + /// A nearer layer whiteouts the path, or it is absent everywhere. + Absent, +} + +/// Look up `rel_path` through the sealed chain (nearest first) and finally the +/// Dicfuse projection, honoring per-layer whiteouts along the way. +/// +/// The sealed layers are plain host directories (renamed former uppers), so every +/// lookup is a local filesystem probe — no FUSE, no network. +fn chain_base_for(chain: &[PathBuf], rel_path: &str, kind: LowerHashKind) -> Option { + let rel = Path::new(rel_path); + let name = rel.file_name()?.to_string_lossy().to_string(); + let parent = rel.parent().unwrap_or_else(|| Path::new("")); + let whiteout_name = format!("{WHITEOUT_PREFIX}{name}"); + + for layer in chain { + let layer_parent = layer.join(parent); + if layer_parent.join(&whiteout_name).is_file() { + // A nearer chain layer hides the path for everything above it. + return Some(ChainBase::Absent); + } + let entry = layer.join(rel); + match fs::symlink_metadata(&entry) { + Ok(meta) if meta.is_dir() => { + // A directory cannot be the base of a file query; keep looking. + continue; + } + Ok(meta) if meta.file_type().is_symlink() => { + let bytes = fs::read_link(&entry).ok()?.as_os_str().as_encoded_bytes().to_vec(); + return Some(ChainBase::Hash(hash_content(kind, &bytes))); + } + Ok(_) => { + let bytes = fs::read(&entry).ok()?; + return Some(ChainBase::Hash(hash_content(kind, &bytes))); + } + Err(_) => continue, + } + } + None +} + +/// Compare the upper layer against the lower projection. +/// +/// `chain` holds the sealed layers, nearest first. A call performs directory loads +/// for paths whose parents were not yet fetched, so the first scan of a cold deep +/// path may hit the network; later scans are local. +pub async fn effective_changes( + lower: &dyn LowerView, + upper_dir: &Path, + chain: &[PathBuf], +) -> std::io::Result> { + let kind = lower.hash_kind(); + let mut by_path: BTreeMap = BTreeMap::new(); + + for entry in scan_upper_entries(upper_dir)? { + // The base is the chain first (nearest layer wins, whiteouts honored); + // the lower projection answers only what the chain does not. Both sides + // are hashed in the lower's own identity domain. + let base = match chain_base_for(chain, &entry.rel_path, kind) { + Some(base) => base, + None => match lower.base_hash(&entry.rel_path).await { + Some(hash) => ChainBase::Hash(hash), + None => ChainBase::Absent, + }, + }; + match entry.kind { + UpperEntryKind::Whiteout => match base { + ChainBase::Hash(base_hash) => { + by_path.insert( + entry.rel_path.clone(), + EffectiveChange { + path: entry.rel_path.clone(), + kind: EffectiveKind::Deleted, + content_hash: None, + base_hash: Some(base_hash), + }, + ); + } + // Whiteout over an absent base is a no-op; dropping it happens + // during the next finalize, not here. + ChainBase::Absent => {} + }, + UpperEntryKind::File | UpperEntryKind::Symlink => { + let content_hash = upper_content_oid(upper_dir, &entry, kind)?; + match base { + ChainBase::Hash(base_hash) if base_hash == content_hash => { + // Edited back to the base content: effectively clean. The + // redundant upper entry is left in place; finalize/compaction + // may drop it. + by_path.remove(&entry.rel_path); + } + ChainBase::Hash(base_hash) => { + by_path.insert( + entry.rel_path.clone(), + EffectiveChange { + path: entry.rel_path.clone(), + kind: EffectiveKind::Modified, + content_hash: Some(content_hash), + base_hash: Some(base_hash), + }, + ); + } + ChainBase::Absent => { + by_path.insert( + entry.rel_path.clone(), + EffectiveChange { + path: entry.rel_path.clone(), + kind: EffectiveKind::Added, + content_hash: Some(content_hash), + base_hash: None, + }, + ); + } + } + } + } + } + + Ok(by_path.into_values().collect()) +} + +/// Merge every sealed chain layer into the upper layer (nearest wins, upper wins +/// over everything), honoring whiteouts in both directions. +/// +/// This runs when a chained mount's lower is about to move (finalize/refresh): a +/// chain is a delta against the OLD revision, so keeping it across a lower switch +/// would make stale chain entries shadow the new projection. Flattening converts +/// the O(1) fork cost into a one-time O(uncommitted delta) at finalize. +/// +/// Returns the number of entries copied. Chain directories themselves are NOT +/// deleted here — other mounts may still reference them; unreferenced ones are +/// collected by the mount-delete path. +pub fn flatten_chain_into_upper(chain: &[PathBuf], upper_dir: &Path) -> std::io::Result { + let mut copied = 0; + + // Farthest first, nearest last: a nearer layer's entry must overwrite a + // farther layer's copy of the same path. The upper itself wins over all of + // them — an existing upper entry is never touched. + for layer in chain.iter().rev() { + for entry in scan_upper_entries(layer)? { + let upper_target = upper_dir.join(&entry.rel_path); + if upper_target.exists() || upper_target.is_symlink() { + continue; + } + let upper_parent = upper_target + .parent() + .map(Path::to_path_buf) + .unwrap_or_else(|| PathBuf::from("")); + fs::create_dir_all(&upper_parent)?; + let physical_name = match entry.kind { + UpperEntryKind::Whiteout => format!( + "{WHITEOUT_PREFIX}{}", + Path::new(&entry.rel_path) + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_default() + ), + _ => Path::new(&entry.rel_path) + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_default(), + }; + let layer_entry = layer.join(parent_of(&entry.rel_path)).join(&physical_name); + let meta = fs::symlink_metadata(&layer_entry)?; + if meta.is_dir() { + continue; // directories carry no content of their own + } + if meta.file_type().is_symlink() { + let target = fs::read_link(&layer_entry)?; + std::os::unix::fs::symlink(target, &upper_target)?; + } else { + fs::copy(&layer_entry, &upper_target)?; + } + copied += 1; + } + } + + Ok(copied) +} + +fn parent_of(rel_path: &str) -> PathBuf { + Path::new(rel_path) + .parent() + .map(Path::to_path_buf) + .unwrap_or_else(|| PathBuf::from("")) +} + +/// Stable fingerprint of an effective change set (FNV-1a over path/kind/hashes). +/// +/// Order-independent: the changes are sorted by path before hashing, so the same +/// change set always produces the same generation regardless of how the caller +/// collected it. Libra uses this as an optimistic lock: it reads state, stages, +/// commits, then sends the generation back with the finalize; if anything changed +/// underneath, the finalize is refused before any destructive step. +pub fn generation_of(changes: &[EffectiveChange]) -> u64 { + let mut ordered: Vec<&EffectiveChange> = changes.iter().collect(); + ordered.sort_by(|x, y| x.path.cmp(&y.path)); + + let mut generation: u64 = 0xcbf29ce484222325; + for change in ordered { + for field in [ + change.path.as_str(), + match change.kind { + EffectiveKind::Added => "added", + EffectiveKind::Modified => "modified", + EffectiveKind::Deleted => "deleted", + }, + change.content_hash.as_deref().unwrap_or(""), + change.base_hash.as_deref().unwrap_or(""), + ] { + for byte in field.as_bytes() { + generation ^= u64::from(*byte); + generation = generation.wrapping_mul(0x100000001b3); + } + generation ^= 0xff; + generation = generation.wrapping_mul(0x100000001b3); + } + } + generation +} + +// --------------------------------------------------------------------------- +// Mega revision resolution +// --------------------------------------------------------------------------- + +/// Resolve the monorepo's current internal commit OID for `repo_path`. +/// +/// Mega's tree/blob API addresses revisions by *internal* commit OID (the one +/// `/api/v1/latest-commit` returns), not by git commit OID. A VCS client that just +/// pushed a commit therefore cannot name its revision directly — it either passes +/// the OID it obtained from `latest-commit`, or leaves [`CommitFinalizeRequest:: +/// new_base_revision`] empty and lets the daemon resolve it here. +pub async fn resolve_latest_revision(base_url: &str, repo_path: &str) -> Result { + let url = format!( + "{}/api/v1/latest-commit?path={}", + base_url.trim_end_matches('/'), + // Mega's examples carry the leading slash (`?path=/project`); keep it. + repo_path.trim_end_matches('/') + ); + let response = reqwest::Client::new() + .get(&url) + .timeout(std::time::Duration::from_secs(30)) + .send() + .await + .map_err(|e| format!("latest-commit request failed for {repo_path}: {e}"))?; + if !response.status().is_success() { + return Err(format!( + "latest-commit returned HTTP {} for {repo_path}", + response.status() + )); + } + #[derive(serde::Deserialize)] + struct LatestCommit { + oid: String, + } + let parsed: LatestCommit = response + .json() + .await + .map_err(|e| format!("latest-commit response parse failed for {repo_path}: {e}"))?; + if parsed.oid.trim().is_empty() { + return Err(format!( + "latest-commit returned an empty OID for {repo_path}" + )); + } + Ok(parsed.oid) +} + +// --------------------------------------------------------------------------- +// Protocol payloads +// --------------------------------------------------------------------------- + +/// `POST /antares/worktrees` — Worktree v2 attach. +/// +/// One call provisions the mount with its lower **pinned from the first request**, +/// closing the v1 gap where `POST /mounts` served the moving trunk tip and the +/// base binding happened only afterwards. The daemon never writes VCS metadata +/// into the mount; the client owns `.libra` and writes it through the FUSE view. +#[derive(Debug, Clone, Deserialize)] +pub struct AttachWorktreeRequest { + /// Client-side worktree identity, echoed back. The daemon keys mounts by + /// `mount_id`; this field is provenance for the caller's registry. + #[serde(default)] + pub worktree_id: Option, + /// Monorepo path to project (e.g. `/project`). + pub repo_path: String, + /// Filesystem mountpoint. Must be absent or an empty directory. + pub mountpoint: String, + /// The revision the client binds (its own identifier space, e.g. a git commit + /// OID). Recorded verbatim; it does NOT drive the lower pin. Empty/omitted + /// leaves the mount unbound until a later v1 `worktree/base` or finalize. + #[serde(default)] + pub base_revision: Option, + /// Pin the lower to this Mega *internal* commit OID instead of the trunk tip + /// at attach time. Empty/omitted resolves `latest-commit` for `repo_path`. + #[serde(default)] + pub lower_revision: Option, + /// Optional task identifier for idempotent job binding, as on v1 mounts. + #[serde(default)] + pub job_id: Option, +} + +/// `POST /antares/worktrees` response. +#[derive(Debug, Clone, Serialize)] +pub struct AttachWorktreeResponse { + pub mount_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub worktree_id: Option, + pub mountpoint: String, + /// Bound client-side revision (echoed, unchanged). + pub base_revision: Option, + /// The internal revision the lower was pinned to. + pub lower_revision: String, + pub state: String, + pub generation: u64, +} + +/// `GET /antares/worktrees/{mount_id}/state` +#[derive(Debug, Clone, Serialize)] +pub struct WorktreeStateV2 { + pub mount_id: String, + /// The revision the lower projection is pinned to. `null` means the mount + /// predates pinning and its lower tracks the moving trunk tip. + pub lower_revision: Option, + /// The revision the VCS client bound (v1 `worktree/base`). May differ from + /// `lower_revision` on legacy mounts; finalize aligns them. + pub base_revision: Option, + pub state: String, + pub generation: u64, + pub dirty: bool, + pub changes: Vec, +} + +/// `POST /antares/worktrees/{mount_id}/commit-finalize` +#[derive(Debug, Clone, Deserialize)] +pub struct CommitFinalizeRequest { + /// The revision the client believes the lower is at. Verified before anything + /// is touched. + pub expected_base_revision: Option, + /// The revision to pin the lower to. Empty/omitted resolves the monorepo's + /// latest commit for the mounted path. + #[serde(default)] + pub new_base_revision: Option, + /// Optimistic lock over the whole change set, from the state call. + #[serde(default)] + pub expected_generation: Option, + /// Paths the commit absorbed; their upper entries are removed once the lower + /// verifiably serves the committed content. + #[serde(default)] + pub committed_paths: Vec, +} + +/// `POST /antares/worktrees/{mount_id}/commit-finalize` response. +#[derive(Debug, Clone, Serialize)] +pub struct CommitFinalizeResponse { + pub state: String, + /// Error code for a non-ready outcome: `GENERATION_CHANGED`, `TREE_MISMATCH`, + /// `BASE_MISMATCH`, `SWITCH_FAILED`. + #[serde(skip_serializing_if = "Option::is_none")] + pub code: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub detail: Option, + pub base_revision: String, + pub lower_revision: Option, + pub generation: u64, + /// Paths whose upper entries were removed (verified committed content only). + pub cleaned_paths: Vec, +} + +/// `POST /antares/worktrees/{mount_id}/refresh` +#[derive(Debug, Clone, Deserialize)] +pub struct RefreshRequest { + pub expected_base_revision: Option, + /// Revision to move the lower to. Empty/omitted resolves the latest. + #[serde(default)] + pub target_revision: Option, + /// Refuse when the effective diff is non-empty (Git-style overwrite protection). + #[serde(default = "default_require_clean")] + pub require_clean: bool, +} + +fn default_require_clean() -> bool { + true +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum RefreshDisposition { + Switched, + AlreadyAtTarget, + BlockedDirty, + BaseMismatch, +} + +/// `POST /antares/worktrees/{mount_id}/refresh` response. +#[derive(Debug, Clone, Serialize)] +pub struct RefreshResponse { + pub disposition: RefreshDisposition, + pub base_revision: String, + pub lower_revision: Option, + pub generation: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub code: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + +// --------------------------------------------------------------------------- +// Filesystem cleanup +// --------------------------------------------------------------------------- + +/// Remove the upper-layer entries for a set of committed paths, then prune any +/// parent directories left empty (up to the upper root). +pub fn remove_committed_upper_entries( + upper_dir: &Path, + committed: &[CommittedPath], +) -> std::io::Result> { + let mut removed = Vec::new(); + let mut touched_parents: Vec = Vec::new(); + + for path in committed { + let rel = Path::new(&path.path); + let physical = match path.kind { + EffectiveKind::Deleted => { + let parent = rel.parent().unwrap_or_else(|| Path::new("")); + let file_name = rel + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_default(); + upper_dir + .join(parent) + .join(format!("{WHITEOUT_PREFIX}{file_name}")) + } + EffectiveKind::Added | EffectiveKind::Modified => upper_dir.join(rel), + }; + + match std::fs::symlink_metadata(&physical) { + Ok(meta) => { + if meta.is_dir() { + std::fs::remove_dir_all(&physical)?; + } else { + std::fs::remove_file(&physical)?; + } + removed.push(path.path.clone()); + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + // Already gone: idempotent retry of an earlier finalize. + } + Err(e) => return Err(e), + } + + if let Some(parent) = rel.parent() { + if !parent.as_os_str().is_empty() { + touched_parents.push(parent.to_path_buf()); + } + } + } + + // Deepest first so a chain of emptied directories collapses in one pass. + touched_parents.sort_by_key(|p| std::cmp::Reverse(p.components().count())); + for parent in touched_parents { + let _ = std::fs::remove_dir(upper_dir.join(parent)); // fails harmlessly when non-empty + } + + Ok(removed) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn write_file(path: &Path, contents: &str) { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + #[test] + fn git_blob_oid_matches_the_monorepo_server() { + // The E2E worktree committed "formal-backend-sync\n" to src/alpha.txt and + // Mega's content-hash API reported exactly this OID for the path — the + // formula is the standard git blob header + content. + assert_eq!( + git_blob_oid(b"formal-backend-sync\n"), + "d93a4449096163af2b8789689d1923f96b182671" + ); + // Well-known git blob OIDs. + assert_eq!( + git_blob_oid(b"hello\n"), + "ce013625030ba8dba906f756967f9e9ca394464a" + ); + assert_eq!( + git_blob_oid(b""), + "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391" + ); + } + + #[test] + fn classifies_whiteouts_and_files() { + assert_eq!( + classify_upper_entry(".wh.gone.rs"), + Some(UpperEntryKind::Whiteout) + ); + assert_eq!(classify_upper_entry("normal.rs"), None); + assert_eq!(classify_upper_entry(WHITEOUT_OPAQUE), None); + assert_eq!(classify_upper_entry(".wh."), None); + } + + #[test] + fn generation_is_stable_and_order_independent() { + let a = EffectiveChange { + path: "src/a.rs".into(), + kind: EffectiveKind::Modified, + content_hash: Some("aaa".into()), + base_hash: Some("bbb".into()), + }; + let b = EffectiveChange { + path: "src/b.rs".into(), + kind: EffectiveKind::Added, + content_hash: Some("ccc".into()), + base_hash: None, + }; + let one = generation_of(&[a.clone(), b.clone()]); + let two = generation_of(&[b.clone(), a.clone()]); + assert_eq!( + one, two, + "BTreeMap-fed inputs sort by path; order must not matter" + ); + let changed = generation_of(std::slice::from_ref(&b)); + assert_ne!(one, changed); + } + + #[test] + fn generation_distinguishes_content_hash_from_base_hash() { + let base = EffectiveChange { + path: "p".into(), + kind: EffectiveKind::Modified, + content_hash: Some("c1".into()), + base_hash: Some("b1".into()), + }; + let same_base_diff_content = EffectiveChange { + content_hash: Some("c2".into()), + ..base.clone() + }; + let same_content_diff_base = EffectiveChange { + base_hash: Some("b2".into()), + ..base.clone() + }; + let g1 = generation_of(std::slice::from_ref(&base)); + assert_ne!( + g1, + generation_of(std::slice::from_ref(&same_base_diff_content)) + ); + assert_ne!( + g1, + generation_of(std::slice::from_ref(&same_content_diff_base)) + ); + } + + #[test] + fn removes_files_whiteouts_and_prunes_empty_parents() { + let tmp = tempfile::tempdir().unwrap(); + let upper = tmp.path(); + + write_file(&upper.join("src/keep.rs"), "keep"); + write_file(&upper.join("src/deep/committed.rs"), "content"); + write_file(&upper.join("src/deep/.wh.gone.rs"), ""); + write_file(&upper.join("docs/.wh.gone.md"), ""); + + let committed = vec![ + CommittedPath { + path: "src/deep/committed.rs".into(), + kind: EffectiveKind::Modified, + content_hash: Some("x".into()), + }, + CommittedPath { + path: "src/deep/gone.rs".into(), + kind: EffectiveKind::Deleted, + content_hash: None, + }, + CommittedPath { + path: "docs/gone.md".into(), + kind: EffectiveKind::Deleted, + content_hash: None, + }, + ]; + + let removed = remove_committed_upper_entries(upper, &committed).unwrap(); + let mut sorted = removed.clone(); + sorted.sort(); + assert_eq!( + sorted, + vec!["docs/gone.md", "src/deep/committed.rs", "src/deep/gone.rs"] + ); + + // Committed entries are gone. + assert!( + !upper.join("src/deep").exists(), + "emptied parents are pruned" + ); + assert!(!upper.join("docs").exists()); + // Sibling content survives. + assert!(upper.join("src/keep.rs").exists()); + } + + #[test] + fn removal_is_idempotent() { + let tmp = tempfile::tempdir().unwrap(); + let upper = tmp.path(); + write_file(&upper.join("a.rs"), "x"); + + let committed = vec![CommittedPath { + path: "a.rs".into(), + kind: EffectiveKind::Modified, + content_hash: None, + }]; + remove_committed_upper_entries(upper, &committed).unwrap(); + // Second pass over an already-clean tree must not error. + let removed = remove_committed_upper_entries(upper, &committed).unwrap(); + assert!(removed.is_empty()); + } +} diff --git a/src/dicfuse/async_io.rs b/src/dicfuse/async_io.rs index 05e5cda..eaf0b24 100644 --- a/src/dicfuse/async_io.rs +++ b/src/dicfuse/async_io.rs @@ -441,6 +441,7 @@ impl Filesystem for Dicfuse { size: u32, ) -> Result { if !self.readable { + tracing::warn!(inode, "dicfuse read: readable=false, returning empty"); return Ok(ReplyData { data: Bytes::from("".as_bytes()), }); @@ -487,6 +488,9 @@ impl Filesystem for Dicfuse { .ok_or_else(|| std::io::Error::from_raw_os_error(libc::ENOENT))?; let _offset = offset as usize; let end = (_offset + size as usize).min(datas.len()); + if datas.is_empty() { + tracing::warn!(inode, "dicfuse read: memory cache still empty after refetch"); + } let slice = &datas[_offset..end]; return Ok(ReplyData { data: Bytes::copy_from_slice(slice), @@ -536,6 +540,9 @@ impl Filesystem for Dicfuse { let _offset = offset as usize; let end = (_offset + size as usize).min(buf.len()); + if buf.is_empty() { + tracing::warn!(inode, "dicfuse read: persisted content empty after refetch"); + } let slice = &buf[_offset..end]; return Ok(ReplyData { data: Bytes::copy_from_slice(slice), diff --git a/src/dicfuse/manager.rs b/src/dicfuse/manager.rs index 4aa0a3c..8ca903d 100644 --- a/src/dicfuse/manager.rs +++ b/src/dicfuse/manager.rs @@ -48,11 +48,15 @@ static DICFUSE_CACHE: OnceCell bool { - self.store_root == other.store_root && self.base_path == other.base_path + self.store_root == other.store_root + && self.base_path == other.base_path + && self.refs == other.refs } } @@ -60,6 +64,7 @@ impl Hash for DicfuseCacheKey { fn hash(&self, state: &mut H) { self.store_root.hash(state); self.base_path.hash(state); + self.refs.hash(state); } } @@ -131,6 +136,72 @@ impl DicfuseManager { Self::for_base_path_with_store_root(base_path, &store_root).await } + /// Get or initialize a Dicfuse instance for a base path **pinned to a revision**. + /// + /// `refs` must be a Mega internal monorepo commit OID (from + /// `/api/v1/latest-commit`); raw git commit OIDs do not resolve. Instances are + /// cached per (base_path, refs): a pinned projection is immutable for that + /// revision, so sharing one across mounts of the same revision is free, while + /// different revisions never share tree caches. + /// + /// The on-disk store directory embeds the refs prefix — the persisted tree DB + /// is only valid for the revision that built it. + pub async fn for_base_path_and_refs(base_path: &str, refs: &str) -> Arc { + let store_root = config::store_path().to_string(); + Self::for_base_path_refs_with_store_root(base_path, &store_root, refs).await + } + + /// Test-friendly variant of [`Self::for_base_path_and_refs`] with an explicit store root. + pub async fn for_base_path_refs_with_store_root( + base_path: &str, + store_root: &str, + refs: &str, + ) -> Arc { + let normalized = normalize_base_path(base_path); + let refs = refs.trim(); + debug_assert!( + !refs.is_empty(), + "pinned Dicfuse requires a non-empty refs; use for_base_path for the moving tip" + ); + + let cache = DICFUSE_CACHE.get_or_init(|| async { DashMap::new() }).await; + let key = DicfuseCacheKey { + store_root: store_root.to_string(), + base_path: normalized.clone(), + refs: refs.to_string(), + }; + + let cell = cache + .entry(key) + .or_insert_with(|| Arc::new(OnceCell::new())) + .clone(); + + cell.get_or_init(|| async move { + // Same layout as the unpinned dir plus a refs suffix: the persisted + // tree DB is revision-specific, so it must never be shared across refs. + let base_store_dir = + super::compute_store_dir_for_base_path_with_store_root(store_root, &normalized); + let refs_tag: String = refs.chars().take(12).collect(); + let store_path = format!("{base_store_dir}-rev-{refs_tag}"); + let _ = std::fs::create_dir_all(&store_path); + + let dicfuse = Arc::new( + Dicfuse::new_with_base_path_store_path_and_refs( + &normalized, + &store_path, + Some(refs.to_string()), + ) + .await, + ); + // The tree is immutable at this revision: prefetch eagerly so the mount + // becomes ready once and stays consistent. + dicfuse.start_import(); + dicfuse + }) + .await + .clone() + } + /// Same as `for_base_path`, but allows explicitly specifying the store root directory. /// Useful for tests that want isolated on-disk state. pub async fn for_base_path_with_store_root(base_path: &str, store_root: &str) -> Arc { @@ -146,6 +217,7 @@ impl DicfuseManager { let key = DicfuseCacheKey { store_root: store_root.to_string(), base_path: normalized.clone(), + refs: String::new(), }; let cell = cache diff --git a/src/dicfuse/mod.rs b/src/dicfuse/mod.rs index e1fad35..7ee2a0c 100644 --- a/src/dicfuse/mod.rs +++ b/src/dicfuse/mod.rs @@ -4,7 +4,7 @@ mod content_store; pub mod manager; mod size_store; pub mod store; -mod tree_store; +pub(crate) mod tree_store; use std::{ffi::OsStr, sync::Arc, time::Duration}; @@ -243,6 +243,31 @@ impl Dicfuse { } } + /// Create a Dicfuse pinned to a specific monorepo revision. + /// + /// `refs` must be a Mega *internal* commit OID (from `/api/v1/latest-commit`); + /// raw git commit OIDs are not accepted by the tree API. Every directory + /// listing this store performs is pinned to that revision, so the projection + /// is immutable while the trunk moves on. Blob contents are fetched by OID and + /// need no pin. + /// + /// The caller must give a `store_path` that is unique per (base_path, refs): + /// the persisted tree DB bakes in the revision, so sharing a directory across + /// revisions would serve stale listings. + pub async fn new_with_base_path_store_path_and_refs( + base_path: &str, + store_path: &str, + refs: Option, + ) -> Self { + Self { + readable: config::dicfuse_readable(), + store: DictionaryStore::new_with_base_path_and_store_path(base_path, store_path) + .await + .with_pinned_refs(refs) + .into(), + } + } + /// Create a new Dicfuse instance with a base path for subdirectory mounting. /// /// When `base_path` is set (e.g., "/third-party/mega"), the filesystem will: diff --git a/src/dicfuse/store.rs b/src/dicfuse/store.rs index a6a87e2..5d4bd70 100644 --- a/src/dicfuse/store.rs +++ b/src/dicfuse/store.rs @@ -233,6 +233,22 @@ fn encode_api_path(path: &str) -> String { normalized.replace('+', "%2B").replace('#', "%23") } +/// Percent-encode a raw query-string value (used for `refs` commit OIDs). +/// Monorepo OIDs are hex, so this is normally a no-op — the encoding exists so a +/// future change to the identifier format cannot inject query syntax. +fn urlencode_component(value: &str) -> String { + let mut out = String::with_capacity(value.len()); + for byte in value.as_bytes() { + match byte { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(*byte as char) + } + other => out.push_str(&format!("%{other:02X}")), + } + } + out +} + // Get Mega dictionary tree from server #[allow(unused)] async fn fetch_tree(path: &str) -> Result { @@ -478,7 +494,12 @@ async fn fetch_file_size(oid: &str) -> Option { None } -async fn fetch_dir(path: &str) -> Result { +/// Directory listing against Mega's `content-hash` API. +/// +/// `refs` pins the listing to a specific monorepo revision (a Mega internal commit +/// OID from `/api/v1/latest-commit`). `None` keeps the legacy behavior: the moving +/// trunk tip. Blob contents are fetched separately by immutable OID and need no pin. +async fn fetch_dir(path: &str, refs: Option<&str>) -> Result { let start = Instant::now(); // NOTE: Timeout values are captured once on first use and cannot be changed at runtime. static CLIENT: Lazy = Lazy::new(|| { @@ -492,11 +513,15 @@ async fn fetch_dir(path: &str) -> Result { // Encode path for URL safety (e.g., '+' in crate versions → '%2B'). let encoded_path = encode_api_path(path); - let url = format!( + let mut url = format!( "{}/api/v1/tree/content-hash?path={}", config::base_url(), encoded_path ); + if let Some(refs) = refs { + url.push_str("&refs="); + url.push_str(&urlencode_component(refs)); + } let max_retries = config::dicfuse_fetch_dir_max_retries().max(1); // Base delay for linear backoff: 100ms, 200ms, 300ms for attempts 0, 1, 2 @@ -799,6 +824,13 @@ pub struct DictionaryStore { open_buff_max_bytes: u64, open_buff_max_files: usize, open_buff_bytes: AtomicU64, + /// Revision this store is pinned to (a Mega monorepo commit OID from + /// `/api/v1/latest-commit`). When set, every directory listing request carries + /// `&refs=` so the store serves the tree of that exact revision instead of + /// the moving trunk tip. `None` keeps the legacy behavior: always the latest. + /// + /// File contents need no pinning: blobs are fetched by immutable OID. + pinned_refs: Option, } #[allow(unused)] @@ -842,6 +874,7 @@ impl DictionaryStore { open_buff_max_bytes: config::dicfuse_open_buff_max_bytes(), open_buff_max_files: config::dicfuse_open_buff_max_files(), open_buff_bytes: AtomicU64::new(0), + pinned_refs: None, } } @@ -875,6 +908,7 @@ impl DictionaryStore { open_buff_max_bytes: config::dicfuse_open_buff_max_bytes(), open_buff_max_files: config::dicfuse_open_buff_max_files(), open_buff_bytes: AtomicU64::new(0), + pinned_refs: None, } } @@ -939,6 +973,7 @@ impl DictionaryStore { open_buff_max_bytes, open_buff_max_files, open_buff_bytes: AtomicU64::new(0), + pinned_refs: None, } } @@ -947,6 +982,21 @@ impl DictionaryStore { !self.import_started.swap(true, Ordering::AcqRel) } + /// Pin this store to a specific monorepo revision. + /// + /// Must be called BEFORE the first import/fetch: once tree data is loaded, the + /// store's persisted DB reflects whatever revision was current at fetch time. + /// See the `pinned_refs` field docs for semantics. + pub fn with_pinned_refs(mut self, refs: Option) -> Self { + self.pinned_refs = refs; + self + } + + /// The revision this store is pinned to, if any. + pub fn pinned_refs(&self) -> Option<&str> { + self.pinned_refs.as_deref() + } + /// Create a new DictionaryStore with a base path for subdirectory mounting. /// /// When `base_path` is set (e.g., "/third-party/mega"), the store will: @@ -1173,7 +1223,7 @@ impl DictionaryStore { // Fetch remote listing and populate children. let real_parent_path = self.to_real_path(&parent_user_path); - let fetched = fetch_dir(&real_parent_path) + let fetched = fetch_dir(&real_parent_path, self.pinned_refs.as_deref()) .await .map_err(|e| io::Error::other(e.to_string()))?; if !fetched._req_result { @@ -1511,7 +1561,10 @@ impl DictionaryStore { } pub async fn import(&self) { - let items = fetch_dir("").await.unwrap().data; + let items = fetch_dir("", self.pinned_refs.as_deref()) + .await + .unwrap() + .data; //let root_inode = self.inodes.lock().await.get(&1).unwrap().clone(); // deque for bus. @@ -1549,7 +1602,10 @@ impl DictionaryStore { let path = it.to_string(); debug!("fetch path :{path}"); // get tree by parent inode. - new_items = fetch_dir(&path).await.unwrap().data; + new_items = fetch_dir(&path, self.pinned_refs.as_deref()) + .await + .unwrap() + .data; // Insert all new inode. for newit in new_items { @@ -1733,6 +1789,13 @@ impl DictionaryStore { /// - As a last resort, fetch size from remote by hash/oid (HEAD/Range) and persist it pub async fn get_or_fetch_file_size(&self, inode: u64, oid: &str) -> u64 { if let Some(persisted) = self.get_persisted_size(inode) { + if persisted == 0 && oid != EMPTY_BLOB_OID { + tracing::warn!( + inode, + oid, + "get_or_fetch_file_size: persisted size 0 for non-empty blob" + ); + } // NOTE: 0 is a valid cached size (empty file). We treat "not cached" as None. return persisted; } @@ -1764,6 +1827,7 @@ impl DictionaryStore { } if oid.is_empty() { + tracing::warn!(inode, "get_or_fetch_file_size: empty oid, reporting size 0"); return 0; } @@ -1772,6 +1836,15 @@ impl DictionaryStore { return sz; } + // A transient remote hiccup must not surface as an empty file: the size + // result is cached by callers, so retry once before giving up. + tokio::time::sleep(Duration::from_millis(200)).await; + if let Some(sz) = fetch_file_size(oid).await { + self.set_persisted_size(inode, sz); + return sz; + } + + tracing::warn!(inode, oid, "get_or_fetch_file_size: probe failed, reporting size 0"); 0 } @@ -1782,6 +1855,13 @@ impl DictionaryStore { pub async fn file_size_for_stat(&self, inode: u64, oid: &str) -> u64 { match self.stat_mode() { config::DicfuseStatMode::Fast => { + if !oid.is_empty() && oid != EMPTY_BLOB_OID { + tracing::warn!( + inode, + oid, + "file_size_for_stat: FAST mode reporting unprobed size 0" + ); + } if !oid.is_empty() && oid == EMPTY_BLOB_OID { self.set_persisted_size(inode, 0); return 0; @@ -1815,6 +1895,9 @@ impl DictionaryStore { } /// Save to db and then save in the memory. pub fn save_file(&self, inode: u64, content: Vec) { + if content.is_empty() { + tracing::warn!(inode, "save_file: storing EMPTY content for inode"); + } // Persist size metadata so getattr can report correct size even with lazy content. let _ = self .persistent_size_store @@ -2019,7 +2102,7 @@ pub async fn load_dir_depth(store: Arc, parent_path: String, ma let _dir_guard = dir_lock.lock().await; let queue = Arc::new(SegQueue::new()); - let fetched = match fetch_dir(&real_parent_path).await { + let fetched = match fetch_dir(&real_parent_path, store.pinned_refs.as_deref()).await { Ok(r) => r, Err(e) => { warn!( @@ -2188,7 +2271,7 @@ pub async fn load_dir_depth(store: Arc, parent_path: String, ma tokio::time::sleep(Duration::from_millis(REQUEST_DELAY_MS)).await; // get all children inode - let result = fetch_dir(&real_path).await; + let result = fetch_dir(&real_path, store.pinned_refs.as_deref()).await; match result { Ok(resp) => { if !resp._req_result { @@ -2401,8 +2484,12 @@ pub async fn import_arc(store: Arc) { // Antares mount) can now safely resolve root-level lookups without blocking on network IO. store.mark_ready(); - // Optional deep prewarm: disabled by default for Antares subdir mounts (max_depth=0). - if store.max_depth() > 0 { + // Pinned stores (Antares lower projections) skip the deep prewarm: the + // revision is immutable, so on-demand fetches never go stale, and warming a + // large monorepo saturates the remote while a finalize/remount is serving + // user fetches concurrently — those fail fast, cache size 0, and reads come + // back empty for seconds. + if store.max_depth() > 0 && store.pinned_refs().is_none() { let max_depth = store.max_depth() + 2; info!( "[import_arc] Prewarming directory tree (user_root={user_root:?} real_root={real_root:?} max_depth={max_depth} load_dir_depth={})", @@ -2622,7 +2709,7 @@ pub async fn load_dir( } //last, if the dir's hash is different from the parent dir's hash, //then fetch the dir from the server. - let fetched = fetch_dir(&real_parent_path) + let fetched = fetch_dir(&real_parent_path, store.pinned_refs.as_deref()) .await .map_err(|e| io::Error::other(e.to_string()))?; if !fetched._req_result { @@ -3012,6 +3099,7 @@ mod tests { open_buff_max_bytes, open_buff_max_files, open_buff_bytes: AtomicU64::new(0), + pinned_refs: None, } } diff --git a/src/server/mod.rs b/src/server/mod.rs index b1aef30..a011e60 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -77,24 +77,18 @@ pub async fn mount_filesystem( - fs: F, - mountpoint: &OsStr, - enable_antares_cache: bool, -) -> std::io::Result { +/// Make `path` ready to serve as a FUSE mountpoint, or explain why it cannot. +/// +/// Creates the directory if it is missing, then requires it to be a directory and to +/// be **empty**. An unreadable directory counts as non-empty: mounting over contents +/// that cannot even be enumerated is not a recoverable mistake. +/// +/// Split out of [`mount_filesystem_with_antares_cache`] so that an operation which +/// creates a worktree *before* mounting it (such as `fork`) can run the identical +/// check up front and fail without leaving a half-created worktree behind. +pub fn prepare_mountpoint(path: &std::path::Path) -> std::io::Result<()> { use std::io::{Error, ErrorKind}; - // This library function does not install a logger. The scorpio/antares - // binaries call `util::logging::init` once at startup, which installs the - // tracing subscriber and the `log` -> `tracing` bridge; a library consumer - // that wants `log::` records captured must initialize tracing itself. - //let logfs = LoggingFileSystem::new(fs); - - let mount_path: OsString = OsString::from(mountpoint); - let path = std::path::Path::new(&mount_path); if !path.exists() { std::fs::create_dir_all(path).map_err(|e| { Error::new( @@ -124,6 +118,28 @@ pub async fn mount_filesystem_with_antares_cache< path.display() ))); } + Ok(()) +} + +#[allow(unused)] +pub async fn mount_filesystem_with_antares_cache< + F: Filesystem + std::marker::Sync + Send + 'static, +>( + fs: F, + mountpoint: &OsStr, + enable_antares_cache: bool, +) -> std::io::Result { + use std::io::{Error, ErrorKind}; + + // This library function does not install a logger. The scorpio/antares + // binaries call `util::logging::init` once at startup, which installs the + // tracing subscriber and the `log` -> `tracing` bridge; a library consumer + // that wants `log::` records captured must initialize tracing itself. + //let logfs = LoggingFileSystem::new(fs); + + let mount_path: OsString = OsString::from(mountpoint); + let path = std::path::Path::new(&mount_path); + prepare_mountpoint(path)?; let uid = unsafe { libc::getuid() }; let gid = unsafe { libc::getgid() }; diff --git a/src/snapshot/fuse.rs b/src/snapshot/fuse.rs index 1527cab..9182359 100644 --- a/src/snapshot/fuse.rs +++ b/src/snapshot/fuse.rs @@ -32,11 +32,11 @@ use crate::{ util::file_attr::make_file_attr, }; -const ROOT_INODE: u64 = 1; -const TTL: Duration = Duration::from_secs(60); +pub(crate) const ROOT_INODE: u64 = 1; +pub(crate) const TTL: Duration = Duration::from_secs(60); #[derive(Clone)] -struct DirNode { +pub(crate) struct DirNode { /// Scope-relative path, no leading slash ("" for root). #[allow(dead_code)] path: String, @@ -53,7 +53,7 @@ struct DirNode { } #[derive(Clone)] -struct FileNode { +pub(crate) struct FileNode { path: String, fs_kind: String, size: u64, @@ -61,7 +61,7 @@ struct FileNode { } #[derive(Clone)] -enum Node { +pub(crate) enum Node { Dir(DirNode), File(FileNode), } @@ -422,7 +422,7 @@ impl Mst2Fuse { Ok(()) } - fn build( + pub(crate) fn build( reader: Option, store: Option>, manifest: Vec, @@ -543,7 +543,7 @@ impl Mst2Fuse { reader.read_file(&f.path, &f.digest).await.map_err(io_err) } - fn node(&self, inode: u64) -> Result { + pub(crate) fn node(&self, inode: u64) -> Result { self.state .lock() .unwrap() @@ -552,6 +552,33 @@ impl Mst2Fuse { .cloned() .ok_or_else(|| Errno::from(libc::ENOENT)) } + + /// Content digest of `rel_path` in the fixed view, in the view's wire form + /// (`sha256:`). Directory pages are loaded on demand, so a lazy mount + /// resolves the path with the same metadata requests a lookup would make. + /// `None` = the path is absent from the view, or is a directory (which has + /// no content identity). + pub(crate) async fn digest_for_path(&self, rel_path: &str) -> Option { + let parts: Vec<&str> = rel_path.split('/').filter(|p| !p.is_empty()).collect(); + let mut inode = ROOT_INODE; + for part in parts { + if self.ensure_loaded(inode).await.is_err() { + return None; + } + let next = { + let state = self.state.lock().unwrap(); + match state.nodes.get(&inode) { + Some(Node::Dir(d)) => d.children.get(part).copied(), + _ => None, + } + }; + inode = next?; + } + match self.node(inode).ok()? { + Node::File(f) => Some(f.digest.clone()), + Node::Dir(_) => None, + } + } } type EnsureResult = std::result::Result; @@ -627,7 +654,8 @@ fn ensure_child( Ok(inode) } -fn dir_attr(inode: u64) -> FileAttr { +pub(crate) fn dir_attr(inode: u64) -> FileAttr { + let owner = crate::util::mount_owner::mount_owner(); make_file_attr( inode, 0, @@ -638,15 +666,16 @@ fn dir_attr(inode: u64) -> FileAttr { FileType::Directory, 0o755, 2, - 0, - 0, + owner.uid, + owner.gid, 0, 4096, ) } -fn file_attr(inode: u64, f: &FileNode) -> FileAttr { +pub(crate) fn file_attr(inode: u64, f: &FileNode) -> FileAttr { let symlink = f.fs_kind == "symlink"; + let owner = crate::util::mount_owner::mount_owner(); make_file_attr( inode, // For a symlink this is the target's length, per POSIX. @@ -668,8 +697,8 @@ fn file_attr(inode: u64, f: &FileNode) -> FileAttr { 0o644 }, 1, - 0, - 0, + owner.uid, + owner.gid, 0, 4096, ) @@ -1009,6 +1038,164 @@ impl Filesystem for Mst2Fuse { ) -> Result<()> { Err(Errno::from(libc::EROFS)) } + + // ---- Read-only layer: deny every mutation with EROFS (spec 12 §7). + // + // The snapshot view is immutable; writes belong to the upper layer of the + // overlay. Answering EROFS (not the trait default ENOSYS) keeps the + // behaviour identical to the Dicfuse lower layer, so the union filesystem + // and the kernel treat this layer as read-only rather than unsupported. + + async fn setattr( + &self, + _req: Request, + _inode: Inode, + _fh: Option, + _set_attr: SetAttr, + ) -> Result { + Err(Errno::from(libc::EROFS)) + } + + async fn symlink( + &self, + _req: Request, + _parent: Inode, + _name: &OsStr, + _link: &OsStr, + ) -> Result { + Err(Errno::from(libc::EROFS)) + } + + async fn mknod( + &self, + _req: Request, + _parent: Inode, + _name: &OsStr, + _mode: u32, + _rdev: u32, + ) -> Result { + Err(Errno::from(libc::EROFS)) + } + + async fn mkdir( + &self, + _req: Request, + _parent: Inode, + _name: &OsStr, + _mode: u32, + _umask: u32, + ) -> Result { + Err(Errno::from(libc::EROFS)) + } + + async fn link( + &self, + _req: Request, + _inode: Inode, + _new_parent: Inode, + _new_name: &OsStr, + ) -> Result { + Err(Errno::from(libc::EROFS)) + } + + async fn unlink(&self, _req: Request, _parent: Inode, _name: &OsStr) -> Result<()> { + Err(Errno::from(libc::EROFS)) + } + + async fn rmdir(&self, _req: Request, _parent: Inode, _name: &OsStr) -> Result<()> { + Err(Errno::from(libc::EROFS)) + } + + async fn rename( + &self, + _req: Request, + _parent: Inode, + _name: &OsStr, + _new_parent: Inode, + _new_name: &OsStr, + ) -> Result<()> { + Err(Errno::from(libc::EROFS)) + } + + async fn rename2( + &self, + _req: Request, + _parent: Inode, + _name: &OsStr, + _new_parent: Inode, + _new_name: &OsStr, + _flags: u32, + ) -> Result<()> { + Err(Errno::from(libc::EROFS)) + } + + async fn write( + &self, + _req: Request, + _inode: Inode, + _fh: u64, + _offset: u64, + _data: &[u8], + _write_flags: u32, + _flags: u32, + ) -> Result { + Err(Errno::from(libc::EROFS)) + } + + async fn create( + &self, + _req: Request, + _parent: Inode, + _name: &OsStr, + _mode: u32, + _flags: u32, + ) -> Result { + Err(Errno::from(libc::EROFS)) + } + + async fn fallocate( + &self, + _req: Request, + _inode: Inode, + _fh: u64, + _offset: u64, + _length: u64, + _mode: u32, + ) -> Result<()> { + Err(Errno::from(libc::EROFS)) + } + + #[allow(clippy::too_many_arguments)] + async fn copy_file_range( + &self, + _req: Request, + _inode: Inode, + _fh_in: u64, + _off_in: u64, + _inode_out: Inode, + _fh_out: u64, + _off_out: u64, + _length: u64, + _flags: u64, + ) -> Result { + Err(Errno::from(libc::EROFS)) + } + + async fn setxattr( + &self, + _req: Request, + _inode: Inode, + _name: &OsStr, + _value: &[u8], + _flags: u32, + _position: u32, + ) -> Result<()> { + Err(Errno::from(libc::EROFS)) + } + + async fn removexattr(&self, _req: Request, _inode: Inode, _name: &OsStr) -> Result<()> { + Err(Errno::from(libc::EROFS)) + } } fn io_err(e: crate::snapshot::SnapshotError) -> Errno { diff --git a/src/snapshot/layer.rs b/src/snapshot/layer.rs new file mode 100644 index 0000000..f053f64 --- /dev/null +++ b/src/snapshot/layer.rs @@ -0,0 +1,205 @@ +//! MST/2 snapshot view as an Antares overlay **lower layer** (spec 12 §1). +//! +//! [`Mst2Fuse`] already implements the read-only FUSE semantics over a fixed +//! snapshot view (lookup/getattr/read/readdir/readlink, T10-verified). This +//! module adds the [`Layer`] impl so the Antares `OverlayFs` can stack it in +//! the position `Dicfuse` occupies today — spec 12: "现有 user-space Layer +//! 适配到 SnapshotReader". +//! +//! The layer is read-only by construction: every mutation answers `EROFS` +//! (see `fuse.rs`), so the overlay routes all writes to its upper layer and +//! the whiteout format stays the Antares-wide OCI convention. + +use std::ffi::OsStr; + +use asyncfuse::{ + raw::reply::{ReplyCreated, ReplyEntry}, + FileType, Result, +}; +use libfuse_fs::{ + context::OperationContext, + unionfs::{layer::Layer, Inode}, + util::whiteout::WhiteoutFormat, +}; + +use super::fuse::{self, Mst2Fuse, Node}; + +#[cfg(target_os = "linux")] +type Stat64 = libc::stat64; +#[cfg(target_os = "macos")] +type Stat64 = libc::stat; + +#[async_trait::async_trait] +impl Layer for Mst2Fuse { + fn root_inode(&self) -> Inode { + fuse::ROOT_INODE + } + + /// Same convention as every other Antares layer: deletions are recorded as + /// OCI `.wh.` markers, never character devices (no `CAP_MKNOD`). + fn whiteout_format(&self) -> WhiteoutFormat { + WhiteoutFormat::OciWhiteout + } + + /// The union filesystem's copy-up path asks the lower layer for a raw + /// `stat64` when it materializes a node in the upper layer. Answering the + /// trait default (ENOSYS) makes every write that needs a copy-up fail with + /// "Function not implemented", so a writable worktree over this lower must + /// serve this. Sizes come from the verified snapshot entries — never a 0 + /// placeholder (spec 12 §1). + async fn getattr_with_mapping( + &self, + inode: Inode, + _handle: Option, + _mapping: bool, + ) -> std::io::Result<(Stat64, std::time::Duration)> { + let node = self.node(inode).map_err(|e| { + let raw = i32::from(e); + tracing::warn!( + inode, + errno = raw, + "mst2 lower: getattr_with_mapping on unknown inode" + ); + std::io::Error::from_raw_os_error(raw.saturating_abs()) + })?; + let attr = match &node { + Node::Dir(_) => fuse::dir_attr(inode), + Node::File(f) => fuse::file_attr(inode, f), + }; + let type_bits: libc::mode_t = match attr.kind { + FileType::Directory => libc::S_IFDIR, + FileType::Symlink => libc::S_IFLNK, + _ => libc::S_IFREG, + }; + let mut stat: Stat64 = unsafe { std::mem::zeroed() }; + stat.st_dev = 0; + stat.st_ino = inode; + stat.st_nlink = attr.nlink as _; + stat.st_mode = type_bits | attr.perm as libc::mode_t; + stat.st_uid = attr.uid; + stat.st_gid = attr.gid; + stat.st_rdev = 0; + stat.st_size = attr.size as i64; + stat.st_blksize = 4096; + stat.st_blocks = attr.blocks as i64; + stat.st_atime = attr.atime.sec; + stat.st_atime_nsec = attr.atime.nsec.into(); + stat.st_mtime = attr.mtime.sec; + stat.st_mtime_nsec = attr.mtime.nsec.into(); + stat.st_ctime = attr.ctime.sec; + stat.st_ctime_nsec = attr.ctime.nsec.into(); + Ok((stat, fuse::TTL)) + } + + // Everything else keeps the trait defaults: + // - `host_path_of` → None: the view has no 1:1 host-fs mapping. + // - `create_whiteout`/`delete_whiteout` → default bodies reach `lookup` + // and `mknod`/`unlink`, which answer EROFS here — a whiteout can never + // be recorded in an immutable snapshot view. + + /// The overlay's copy-up asks each layer, in order, to create the node. + /// A read-only lower must answer EROFS (not the trait default ENOSYS) so + /// the union filesystem moves on to the *upper* layer and creates the + /// copy-up node with the requesting user's credentials. Answering ENOSYS + /// makes copy-up fall back to a daemon-owned (root) creation, after which + /// the user cannot write into the copied-up directory (EACCES). + async fn create_with_context( + &self, + _ctx: OperationContext, + _parent: Inode, + _name: &OsStr, + _mode: u32, + _flags: u32, + ) -> Result { + Err(std::io::Error::from_raw_os_error(libc::EROFS).into()) + } + + async fn mkdir_with_context( + &self, + _ctx: OperationContext, + _parent: Inode, + _name: &OsStr, + _mode: u32, + _umask: u32, + ) -> Result { + Err(std::io::Error::from_raw_os_error(libc::EROFS).into()) + } + + async fn symlink_with_context( + &self, + _ctx: OperationContext, + _parent: Inode, + _name: &OsStr, + _link: &OsStr, + ) -> Result { + Err(std::io::Error::from_raw_os_error(libc::EROFS).into()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use asyncfuse::{raw::prelude::*, Errno}; + use std::ffi::OsStr; + + /// An empty view: no reader, no store, no entries. Enough to exercise the + /// layer surface without a server. + fn empty_view() -> Mst2Fuse { + Mst2Fuse::build(None, None, Vec::new()).expect("empty view builds") + } + + fn erofs(e: Errno) -> bool { + i32::from(e) == -libc::EROFS + } + + #[test] + fn lower_layer_identity_and_whiteout_convention() { + let fs = empty_view(); + assert_eq!(Layer::root_inode(&fs), 1); + assert_eq!(Layer::whiteout_format(&fs), WhiteoutFormat::OciWhiteout); + } + + /// Every mutation must answer EROFS — not the trait default ENOSYS — so the + /// overlay treats the layer as read-only exactly like Dicfuse (spec 12 §7). + #[tokio::test] + async fn every_mutation_answers_erofs() { + let fs = empty_view(); + let req = Request::default(); + let name = OsStr::new("probe"); + + let err = fs.create(req, 1, name, 0o644, 0).await.unwrap_err(); + assert!(erofs(err), "create: {err:?}"); + let err = fs.mkdir(req, 1, name, 0o755, 0).await.unwrap_err(); + assert!(erofs(err), "mkdir: {err:?}"); + let err = fs.mknod(req, 1, name, 0o644, 0).await.unwrap_err(); + assert!(erofs(err), "mknod: {err:?}"); + let err = fs.symlink(req, 1, name, OsStr::new("t")).await.unwrap_err(); + assert!(erofs(err), "symlink: {err:?}"); + let err = fs.link(req, 1, 1, name).await.unwrap_err(); + assert!(erofs(err), "link: {err:?}"); + let err = fs.unlink(req, 1, name).await.unwrap_err(); + assert!(erofs(err), "unlink: {err:?}"); + let err = fs.rmdir(req, 1, name).await.unwrap_err(); + assert!(erofs(err), "rmdir: {err:?}"); + let err = fs.rename(req, 1, name, 1, name).await.unwrap_err(); + assert!(erofs(err), "rename: {err:?}"); + let err = fs.rename2(req, 1, name, 1, name, 0).await.unwrap_err(); + assert!(erofs(err), "rename2: {err:?}"); + let err = fs.write(req, 1, 0, 0, b"x", 0, 0).await.unwrap_err(); + assert!(erofs(err), "write: {err:?}"); + let err = fs + .setattr(req, 1, None, SetAttr::default()) + .await + .unwrap_err(); + assert!(erofs(err), "setattr: {err:?}"); + let err = fs + .setxattr(req, 1, name, b"v", 0, 0) + .await + .unwrap_err(); + assert!(erofs(err), "setxattr: {err:?}"); + let err = fs.removexattr(req, 1, name).await.unwrap_err(); + assert!(erofs(err), "removexattr: {err:?}"); + let err = fs.fallocate(req, 1, 0, 0, 0, 0).await.unwrap_err(); + assert!(erofs(err), "fallocate: {err:?}"); + } +} diff --git a/src/snapshot/mod.rs b/src/snapshot/mod.rs index ee3bf0b..b02fa5e 100644 --- a/src/snapshot/mod.rs +++ b/src/snapshot/mod.rs @@ -12,6 +12,7 @@ pub mod durable; pub mod frames; pub mod fuse; pub mod incremental; +pub mod layer; pub mod range; pub mod reader; pub mod types; diff --git a/src/util/config.rs b/src/util/config.rs index d0c1530..ced46b5 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -47,6 +47,18 @@ pub struct ScorpioConfig { pub antares_cl_root: String, pub antares_mount_root: String, pub antares_state_file: String, + /// MST/2 lower projection (spec 12 §1). When enabled, mounts resolve a + /// fixed snapshot view from the MST/2 surface and serve it as the overlay's + /// base layer instead of the Dicfuse tree/blob path. Off by default: the + /// legacy reader and the snapshot reader are explicit modes, never a + /// silent fallback (spec 15 §3). + pub mst2_lower_enabled: bool, + /// MST/2 surface base URL (the `/api/v2/snapshots` service). + pub mst2_base_url: String, + /// Bearer token for the MST/2 surface (spec 04 §1). Empty = no auth header. + pub mst2_auth_token: String, + /// Namespace scope to resolve (`/project` for the trunk monorepo). + pub mst2_scope: String, } /// Filesystem paths the installer must prepare for the effective config. @@ -167,13 +179,23 @@ fn defaults() -> ScorpioConfig { antares_load_dir_depth: DEFAULT_ANTARES_LOAD_DIR_DEPTH, antares_dicfuse_dir_sync_ttl_secs: DEFAULT_ANTARES_DICFUSE_DIR_SYNC_TTL_SECS, antares_dicfuse_reply_ttl_secs: DEFAULT_ANTARES_DICFUSE_REPLY_TTL_SECS, - antares_dicfuse_stat_mode: DicfuseStatMode::Fast, + // Accurate, not Fast: under Fast, getattr reports size 0 for files never + // read in this revision, the kernel caches that attr, and the FUSE read + // of the real bytes gets truncated to empty — every consumer (git-style + // status scans, builds) then sees a bogus empty file until the attr TTL + // lapses. Size probes are cached in size.db, so the Accurate cost is one + // HEAD/Range per file per revision, not per access. + antares_dicfuse_stat_mode: DicfuseStatMode::Accurate, antares_dicfuse_open_buff_max_bytes: DEFAULT_ANTARES_DICFUSE_OPEN_BUFF_MAX_BYTES, antares_dicfuse_open_buff_max_files: DEFAULT_ANTARES_DICFUSE_OPEN_BUFF_MAX_FILES, antares_upper_root: format!("{base_path}/{DEFAULT_ANTARES_SUBDIR}/upper"), antares_cl_root: format!("{base_path}/{DEFAULT_ANTARES_SUBDIR}/cl"), antares_mount_root: format!("{base_path}/{DEFAULT_ANTARES_SUBDIR}/mnt"), antares_state_file: format!("{base_path}/{DEFAULT_ANTARES_SUBDIR}/state.toml"), + mst2_lower_enabled: false, + mst2_base_url: "http://127.0.0.1:19700".to_string(), + mst2_auth_token: String::new(), + mst2_scope: "/project".to_string(), } } @@ -536,6 +558,22 @@ fn build_config(r: &RawResolver) -> ConfigResult { "state_file", d.antares_state_file, )?, + mst2_lower_enabled: parse_bool( + r, + "mst2_lower_enabled", + "mst2", + "lower_enabled", + d.mst2_lower_enabled, + )?, + mst2_base_url: optional_string(r, "mst2_base_url", "mst2", "base_url", d.mst2_base_url)?, + mst2_auth_token: optional_string( + r, + "mst2_auth_token", + "mst2", + "auth_token", + d.mst2_auth_token, + )?, + mst2_scope: optional_string(r, "mst2_scope", "mst2", "scope", d.mst2_scope)?, }; validate(&cfg)?; @@ -1236,6 +1274,24 @@ pub fn antares_state_file() -> &'static str { get_config().antares_state_file.as_str() } +/// Whether mounts should serve an MST/2 snapshot view as the overlay's lower +/// layer instead of the Dicfuse projection (spec 12 §1). +pub fn mst2_lower_enabled() -> bool { + get_config().mst2_lower_enabled +} + +pub fn mst2_base_url() -> &'static str { + get_config().mst2_base_url.as_str() +} + +pub fn mst2_auth_token() -> &'static str { + get_config().mst2_auth_token.as_str() +} + +pub fn mst2_scope() -> &'static str { + get_config().mst2_scope.as_str() +} + pub fn load_dir_depth() -> usize { get_config().load_dir_depth } diff --git a/src/util/file_attr.rs b/src/util/file_attr.rs index 15b9fca..f452765 100644 --- a/src/util/file_attr.rs +++ b/src/util/file_attr.rs @@ -42,7 +42,13 @@ pub fn make_file_attr( } /// Zeroed attributes used for default / negative FUSE entries. +/// +/// Ownership is the mount owner's, not the daemon's: the overlay's copy-up +/// preserves the lower layer's uid/gid when it materializes a node in the +/// upper layer, so reporting root here would create root-owned upper +/// directories the user cannot write into. pub fn empty_file_attr(ino: u64, kind: FileType, perm: u16) -> FileAttr { + let owner = crate::util::mount_owner::mount_owner(); make_file_attr( ino, 0, @@ -53,8 +59,8 @@ pub fn empty_file_attr(ino: u64, kind: FileType, perm: u16) -> FileAttr { kind, perm, 0, - 0, - 0, + owner.uid, + owner.gid, 0, 0, ) diff --git a/src/util/mod.rs b/src/util/mod.rs index 4b1dc09..86da23c 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -8,6 +8,7 @@ pub mod config; pub mod file_attr; pub mod fuse_platform; pub mod logging; +pub mod mount_owner; #[derive(Debug, Deserialize, Serialize, Clone, Default)] pub struct GPath { diff --git a/src/util/mount_owner.rs b/src/util/mount_owner.rs new file mode 100644 index 0000000..1d92ab2 --- /dev/null +++ b/src/util/mount_owner.rs @@ -0,0 +1,59 @@ +//! Ownership for files surfaced through a ScorpioFS mount. +//! +//! A daemon started via `sudo` runs as root, but the mount serves a *user's* +//! worktree. Reporting the daemon's own uid/gid would make the union +//! filesystem's copy-up create root-owned upper nodes (the overlay preserves +//! the lower layer's ownership), after which the user cannot create files +//! inside a copied-up directory. The mount owner is therefore derived from +//! `SUDO_USER` (best-effort) and falls back to the effective ids. + +use std::sync::OnceLock; + +/// uid/gid reported for every node of a mount. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MountOwner { + pub uid: u32, + pub gid: u32, +} + +impl Default for MountOwner { + fn default() -> Self { + resolve() + } +} + +fn resolve() -> MountOwner { + let euid = unsafe { libc::geteuid() }; + let egid = unsafe { libc::getegid() }; + if euid != 0 { + return MountOwner { uid: euid, gid: egid }; + } + let Some(user) = std::env::var_os("SUDO_USER") else { + return MountOwner { uid: euid, gid: egid }; + }; + let Ok(cuser) = std::ffi::CString::new(user.as_os_str().as_encoded_bytes()) else { + return MountOwner { uid: euid, gid: egid }; + }; + let pw = unsafe { libc::getpwnam(cuser.as_ptr()) }; + if pw.is_null() { + return MountOwner { uid: euid, gid: egid }; + } + let (uid, gid) = unsafe { ((*pw).pw_uid, (*pw).pw_gid) }; + MountOwner { uid, gid } +} + +/// The uid/gid a mount should report. Resolved once per process. +pub fn mount_owner() -> MountOwner { + static OWNER: OnceLock = OnceLock::new(); + *OWNER.get_or_init(resolve) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn owner_is_stable_across_calls() { + assert_eq!(mount_owner(), mount_owner()); + } +}