From 478353d28bb0509ab97690f8c02b4b8755b6a277 Mon Sep 17 00:00:00 2001 From: callumalpass Date: Wed, 16 Sep 2026 19:42:00 +1000 Subject: [PATCH 1/5] fix(sync): replicate opaque Markdown losslessly by default --- docs/lossless-opaque-sync-acceptance.md | 74 ++++++++ packages/sync/README.md | 18 +- packages/sync/src/mirror-format.ts | 4 +- packages/sync/src/mirror-state.ts | 3 +- packages/sync/src/mirror.test.ts | 240 ++++++++++++++++++++---- packages/sync/src/node.test.ts | 26 +++ packages/sync/src/node.ts | 3 +- packages/sync/src/sync-inspector.ts | 4 +- 8 files changed, 324 insertions(+), 48 deletions(-) create mode 100644 docs/lossless-opaque-sync-acceptance.md diff --git a/docs/lossless-opaque-sync-acceptance.md b/docs/lossless-opaque-sync-acceptance.md new file mode 100644 index 000000000..02957647b --- /dev/null +++ b/docs/lossless-opaque-sync-acceptance.md @@ -0,0 +1,74 @@ +# Default lossless opaque-document sync + +## Contract + +Readable UTF-8 Markdown is replicated exactly even when its leading YAML is +malformed, duplicated, null, scalar, or a sequence. Structural diagnostics are +nonblocking; validation does not repair or normalize documents. No option, +public TypeScript shape, protocol field, or persisted-state layout is added. + +Unreadable files and invalid UTF-8 still fence work. Conflicts and receive-only +local divergence still require an explicit decision. A readable malformed local +edit is no longer mistaken for corrupt bytes that a receive-only mirror may +replace automatically. Existing authority scope/configuration checks remain. + +Both Node and embedding adapters must preserve a UTF-8 BOM. Node's decoder now +retains it; the new real-filesystem regression caught the previous BOM loss. +The YAML parser's default duplicate-key rejection is retained and tested. + +## Consumer audit + +- Portable and Node directory mirrors share the inspector. Exact `document` + bytes were already separate from the structured projection; opaque records + already have a valid representation without changing generic/public types. +- CLI synchronization executes the engine plan, not `local_issues.length`. + Diagnostic `attention` can remain after successful synchronization. +- Server/reference-authority consumers and every Connect TypeScript workspace + typecheck with this change. Application SDK, editor, devkit, testing and other + non-desktop workspace tests pass. +- Obsidian's apply button already uses the plan's blocking count, but its + diagnostic prose incorrectly described every warning as a pause. A separate + consumer branch updates that prose, tests nonblocking reviews, and preserves + BOMs in its Vault adapter. Its release pin must not move to an unpublished + npm version. +- `tasknotes-app` only pins `connect-sync` as an immutable SDK override; no direct + directory-mirror or `local_issues` consumer was found in its application code. +- Desktop renderer status is owned by the Rust mirror, not this JavaScript SDK + inspector. This change does not claim to update that separate implementation. + +There is no persisted local-issue gate to migrate. Existing plan-only checkpoints +are reused; unsupported older engine layouts remain rejected as before. + +## Automated evidence + +Node 24.19.0 / pnpm 11.15.1: + +- `pnpm typecheck`: all workspaces pass. +- `pnpm --filter @mdbase-dev/connect-sync test`: 193 tests pass. +- `pnpm --filter '!@mdbase/connect-desktop' -r test`: passes, including server + (582 passed, 35 intentionally skipped) and editor (473 passed). +- `pnpm check:mirror:mobile`: passes unchanged budgets, 187,632 raw / 55,991 + gzip bytes; no Node-only references in the portable bundle. + +New/updated regressions cover exact uploads and second-mirror downloads, +BOM/CRLF/no-final-newline preservation, updates/moves/deletes, both conflict +choices, lost-reply replay after restart, existing checkpoint reuse, stale +reviews, receive-only divergence, invalid UTF-8 and failed rereads. + +Authorities in these SDK tests are `MemoryAuthority`, not deployed hosted +infrastructure. The Node round-trip uses real temporary filesystem directories; +the portable fixtures use in-memory adapters. + +## Remaining qualification + +- Full `pnpm test` reached the desktop tests, where 124 passed and one failed + while importing Electron: its binary installation could not create an + already-existing `dist/locales` directory. Non-desktop suites were run + separately rather than represented as a full-workspace pass. +- `cargo fmt --all --check` and `cargo test --locked --workspace` cannot resolve + the required sibling `../mdbase-rs` in this isolated workspace. Rust/system + end-to-end qualification remains outstanding; no Rust sources changed. +- This is not a published SDK or deployed-hosted acceptance result. The prior + LAB loopback-port ownership failure and disposable-Obsidian-vault prerequisite + have not been bypassed. A real Obsidian → hosted → second-mirror round trip + remains required before claiming live acceptance. diff --git a/packages/sync/README.md b/packages/sync/README.md index ebf33c982..7486f7c23 100644 --- a/packages/sync/README.md +++ b/packages/sync/README.md @@ -12,7 +12,19 @@ unclosed `---` fence are preserved byte-for-byte as body content. An explicitly empty frontmatter block is also accepted. Malformed, scalar, null, or list frontmatter is synchronized byte-for-byte as opaque Markdown with `{}` persisted fields; structured queries and field operations ignore its invalid -frontmatter until the document is repaired. +frontmatter until the document is repaired. This is the default for directory +mirrors, not an opt-in mode. Record schema validation is separate from exact +replication: missing required fields, wrong field types, and enum violations +do not by themselves prevent synchronization. Authority scope, configuration, +conflict, and safety checks still apply. + +Local frontmatter diagnostics remain visible in `local_issues` and can make +status `attention` even after a successful sync. Consumers must use the reviewed +plan's `issues[].blocking` / `summary.blocking_issues` to decide whether applying +that plan is allowed, not the diagnostic code or `local_issues.length`. +Unreadable files and invalid UTF-8 still fence writes and checkpoint advancement. +Receive-only mirrors also fence local divergence, including readable malformed +edits; malformed YAML is not permission to overwrite those edits from authority. Application replicas resolve a stale record explicitly with `resolveConflict(recordId, "local" | "remote")`. Keeping the local version @@ -92,7 +104,9 @@ await mirror.sync(); ``` The filesystem adapter deals only in collection-relative POSIX paths and -ordinary UTF-8 strings. The state store must live in device-local application +ordinary UTF-8 strings. Text reads must preserve every byte representable by +UTF-8, including a leading BOM (use `TextDecoder` with `fatal: true` and +`ignoreBOM: true`, which retains the BOM character). The state store must live in device-local application state, not inside the mirrored collection. A host should provide a lease that excludes concurrent mirror owners for the same vault; the default memory lease only protects overlapping calls in one JavaScript process. diff --git a/packages/sync/src/mirror-format.ts b/packages/sync/src/mirror-format.ts index d7c32b12d..a4d771a6a 100644 --- a/packages/sync/src/mirror-format.ts +++ b/packages/sync/src/mirror-format.ts @@ -18,14 +18,14 @@ export type LocalRecordStructuralOutcome = { outcome: "parsed" | "invalid_yaml" | "non_mapping_frontmatter"; }; -/** Strict structural check for local mirror records; authority parsing remains opaque-compatible. */ +/** Structural diagnostics only. Readable documents synchronize exactly, including opaque frontmatter. */ export function classifyLocalRecord(document: string): LocalRecordStructuralOutcome { const yaml = leadingFrontmatterYaml(document); if (yaml === null) return { outcome: "parsed" }; let frontmatter: unknown; try { - frontmatter = parse(yaml, { mapAsMap: true, uniqueKeys: true }); + frontmatter = parse(yaml, { mapAsMap: true }); } catch { return { outcome: "invalid_yaml" }; } diff --git a/packages/sync/src/mirror-state.ts b/packages/sync/src/mirror-state.ts index 2b6f49fc7..fb0aa42ea 100644 --- a/packages/sync/src/mirror-state.ts +++ b/packages/sync/src/mirror-state.ts @@ -31,6 +31,7 @@ export interface MirrorFileEntry { file: CollectionFileDescriptor; } +/** Diagnostics, not an apply gate. Use the inspected plan's blocking issues. */ export interface MirrorLocalIssue { path: string; code: "invalid_frontmatter" | "file_read_failed"; @@ -148,7 +149,7 @@ export interface MirrorFileSystem { /** True when any filesystem entry occupies this exact portable path. */ exists(path: string): Promise; read(path: string): Promise; - /** Classified byte-aware read; expected I/O failures reject as `SyncError("file_read_failed")`. */ + /** Exact UTF-8 read, preserving a leading BOM; expected I/O failures reject as `SyncError("file_read_failed")`. */ readText(path: string): Promise; write(path: string, value: string): Promise; /** Atomically rename one managed path without changing its bytes. */ diff --git a/packages/sync/src/mirror.test.ts b/packages/sync/src/mirror.test.ts index 12babd5ba..c94205423 100644 --- a/packages/sync/src/mirror.test.ts +++ b/packages/sync/src/mirror.test.ts @@ -1,7 +1,7 @@ import { createHash } from "node:crypto"; import { describe, expect, it } from "vitest"; import { MemoryAuthority, type SyncTransport } from "./index.js"; -import { documentRevision } from "./mirror-format.js"; +import { classifyLocalRecord, documentRevision } from "./mirror-format.js"; import { SyncError } from "./sync-error.js"; import { DirectoryMirror, @@ -1350,9 +1350,11 @@ describe("platform-neutral directory mirror", () => { await expect(mirror.status()).resolves.toMatchObject({ state: "up_to_date" }); }); - it("fences invalid local records with valid siblings until repair", async () => { + it("syncs malformed frontmatter exactly with valid siblings, but fences invalid UTF-8", async () => { const invalidCases: Array<[string, string | Uint8Array, string]> = [ ["broken.md", "---\nbroken: [\n---\nBody", "invalid_yaml"], + ["bom.md", "\uFEFF---\r\nbroken: [\r\n---\r\nBody\r\n", "invalid_yaml"], + ["no-newline.md", "---\nbroken: [\n---", "invalid_yaml"], ["duplicate.md", "---\na: 1\na: 2\n---\nBody", "invalid_yaml"], ["scalar.md", "---\nhello\n---\nBody", "non_mapping_frontmatter"], ["null.md", "---\nnull\n---\nBody", "non_mapping_frontmatter"], @@ -1373,20 +1375,42 @@ describe("platform-neutral directory mirror", () => { runtime: deterministicRuntime() }); - const blocked = await mirror.inspect(); - expect(blocked.actions).toEqual([]); - expect(blocked.issues).toEqual(expect.arrayContaining([ - expect.objectContaining({ code: "invalid_frontmatter", path, message: expect.stringContaining(reason) }) + const plan = await mirror.inspect(); + expect(plan.issues).toEqual(expect.arrayContaining([ + expect.objectContaining({ code: "invalid_frontmatter", path, blocking: typeof invalid !== "string" }) ])); + if (typeof invalid === "string") { + expect(classifyLocalRecord(invalid).outcome).toBe(reason); + expect(plan.summary.blocking_issues).toBe(0); + expect(plan.summary.uploads).toBe(2); + await expect(mirror.apply(plan)).resolves.toMatchObject({ status: "applied" }); + } else { + expect(plan.actions).toEqual([]); + } await expect(mirror.status()).resolves.toMatchObject({ state: "attention", local_issues: [expect.objectContaining({ code: "invalid_frontmatter", path })] }); expect(typeof invalid === "string" ? fileSystem.files.get(path) : fileSystem.rawFiles.get(path)) .toEqual(invalid); - expect((await hosted.transport(replicaId).snapshot( + const uploaded = (await hosted.transport(replicaId).snapshot( (await hosted.transport(replicaId).openSession()).snapshot_id - )).records).toEqual([]); + )).records; + if (typeof invalid === "string") { + expect(uploaded.find((record) => record.path === path)?.document).toBe(invalid); + expect(uploaded.find((record) => record.path === "valid.md")?.document).toBe("# Valid body-only note"); + const receiverId = hosted.registerReplica({ name: "Opaque receiver", mode: "read_only" }); + const receiverFiles = new TestFileSystem(); + const receiver = new DirectoryMirror(receiverId, hosted.transport(receiverId), { + fileSystem: receiverFiles, stateStore: new MemoryMirrorStateStore(), runtime: deterministicRuntime() + }); + await receiver.sync(); + expect(receiverFiles.files.get(path)).toBe(invalid); + expect((await receiver.inspect()).summary.blocking_issues).toBe(0); + expect((await receiver.inspect()).actions).toEqual([]); + } else { + expect(uploaded).toEqual([]); + } fileSystem.rawFiles.delete(path); fileSystem.files.set(path, "# Fixed body-only note"); @@ -1402,7 +1426,7 @@ describe("platform-neutral directory mirror", () => { } }); - it("fences a malformed managed file before resuming normal conflict handling", async () => { + it.each(["local", "remote"] as const)("resolves an opaque conflict by keeping %s without requiring YAML repair", async (resolution) => { const hosted = new MemoryAuthority(); hosted.seed([{ record_id: "managed", @@ -1441,36 +1465,27 @@ describe("platform-neutral directory mirror", () => { await mirror.sync(); expect(fileSystem.files.get("managed.md")).toBe(malformed); - await expect(mirror.status()).resolves.toMatchObject({ - state: "attention", - conflicts: [], - local_issues: [{ code: "invalid_frontmatter", path: "managed.md" }] - }); - - fileSystem.files.set("managed.md", "# Repaired local body"); - await mirror.sync(); await expect(mirror.status()).resolves.toMatchObject({ state: "attention", conflicts: [{ entity: "record", object_id: "managed", kind: "conflicted" }], - local_issues: [] + local_issues: [{ code: "invalid_frontmatter", path: "managed.md" }] }); await mirror.resolveConflict( "managed", (await mirror.status()).conflicts[0]!.decision_id, - "local" + resolution ); await mirror.sync(); const session = await hosted.transport(replicaId).openSession(); const snapshot = await hosted.transport(replicaId).snapshot(session.snapshot_id); - expect(snapshot.records.find((record) => record.record_id === "managed")).toMatchObject({ - frontmatter: {}, - body: "# Repaired local body" - }); + const expected = resolution === "local" ? malformed : "---\ntitle: Remote\n---\n\nRemote body"; + expect(snapshot.records.find((record) => record.record_id === "managed")?.document).toBe(expected); + expect(fileSystem.files.get("managed.md")).toBe(expected); await expect(mirror.status()).resolves.toMatchObject({ - state: "up_to_date", + state: resolution === "local" ? "attention" : "up_to_date", conflicts: [], - local_issues: [] + local_issues: resolution === "local" ? [{ code: "invalid_frontmatter", path: "managed.md" }] : [] }); }); @@ -1522,7 +1537,7 @@ describe("platform-neutral directory mirror", () => { const blocked = await mirror.inspect(); expect(blocked.issues).toEqual(expect.arrayContaining([ - expect.objectContaining({ code: "invalid_frontmatter", path: "invalid.md", blocking: true }), + expect.objectContaining({ code: "invalid_frontmatter", path: "invalid.md", blocking: false }), expect.objectContaining({ code: "file_read_failed", path: "eio.md", blocking: true }) ])); expect(blocked.actions).toEqual([]); @@ -1531,9 +1546,7 @@ describe("platform-neutral directory mirror", () => { it("repairs authority records over exactly sealed invalid receive-only bytes", async () => { const invalidCases: Array<[string, string | Uint8Array]> = [ - ["invalid UTF-8", Uint8Array.from([0x62, 0x61, 0x64, 0xff])], - ["invalid YAML", "---\na: [broken\n---\n"], - ["nonmapping", "---\n- item\n---\n"] + ["invalid UTF-8", Uint8Array.from([0x62, 0x61, 0x64, 0xff])] ]; for (const [label, invalid] of invalidCases) { const hosted = new MemoryAuthority(); @@ -1593,8 +1606,9 @@ describe("platform-neutral directory mirror", () => { runtime: deterministicRuntime() }); await mirror.sync(); - const invalid = "---\na: [broken\n---\n"; - fileSystem.files.set("managed.md", invalid); + const invalid = Uint8Array.from([0xff]); + fileSystem.files.delete("managed.md"); + fileSystem.rawFiles.set("managed.md", invalid); const plan = await mirror.inspect(); const stateBefore = await stateStore.read(); const writesBefore = fileSystem.writes; @@ -1609,7 +1623,7 @@ describe("platform-neutral directory mirror", () => { failure: { code: "sync_plan_stale" } }); expect(fileSystem.writes).toBe(writesBefore); - expect(fileSystem.files.get("managed.md")).toBe(invalid); + expect(fileSystem.rawFiles.get("managed.md")).toEqual(invalid); expect(await stateStore.read()).toEqual(stateBefore); expect(hosted.serialize().receipts).toEqual([]); }); @@ -1626,8 +1640,9 @@ describe("platform-neutral directory mirror", () => { runtime: deterministicRuntime() }); await mirror.sync(); - const invalid = "---\na: [broken\n---\n"; - fileSystem.files.set("managed.md", invalid); + const invalid = Uint8Array.from([0xff]); + fileSystem.files.delete("managed.md"); + fileSystem.rawFiles.set("managed.md", invalid); const plan = await mirror.inspect(); const stateBefore = await stateStore.read(); const writesBefore = fileSystem.writes; @@ -1642,7 +1657,7 @@ describe("platform-neutral directory mirror", () => { failure: { code: "sync_revalidation_failed", message: "programmer bug" } }); expect(fileSystem.writes).toBe(writesBefore); - expect(fileSystem.files.get("managed.md")).toBe(invalid); + expect(fileSystem.rawFiles.get("managed.md")).toEqual(invalid); expect(await stateStore.read()).toEqual(stateBefore); expect(hosted.serialize().receipts).toEqual([]); }); @@ -1687,12 +1702,13 @@ describe("platform-neutral directory mirror", () => { runtime: deterministicRuntime() }); await mirror.sync(); - fileSystem.files.set("managed.md", "---\na: [broken\n---\n"); + fileSystem.files.delete("managed.md"); + fileSystem.rawFiles.set("managed.md", Uint8Array.from([0xff])); const plan = await mirror.inspect(); - const concurrent = "---\na: [different\n---\n"; - fileSystem.files.set("managed.md", concurrent); + const concurrent = Uint8Array.from([0xfe]); + fileSystem.rawFiles.set("managed.md", concurrent); await expect(mirror.apply(plan)).resolves.toMatchObject({ status: "stale" }); - expect(fileSystem.files.get("managed.md")).toBe(concurrent); + expect(fileSystem.rawFiles.get("managed.md")).toEqual(concurrent); fileSystem.readFailures.set("managed.md", Object.assign(new Error("EIO"), { code: "EIO" })); const failed = await mirror.inspect(); @@ -1705,7 +1721,7 @@ describe("platform-neutral directory mirror", () => { expect(hosted.serialize().receipts).toEqual([]); }); - it("fences receive-only downloads and checkpointing on an invalid local record", async () => { + it("fences receive-only divergence even when its document is opaque", async () => { const hosted = new MemoryAuthority(); hosted.seed([{ record_id: "remote", path: "remote.md", frontmatter: {}, body: "Remote", types: [] }]); const replicaId = hosted.registerReplica({ name: "Receiver", mode: "read_only" }); @@ -1724,6 +1740,150 @@ describe("platform-neutral directory mirror", () => { expect(await stateStore.read()).toBeNull(); }); + it("preserves opaque documents across updates, moves and deletes on a second mirror", async () => { + const hosted = new MemoryAuthority(); + const writerId = hosted.registerReplica({ name: "Writer", mode: "read_write" }); + const readerId = hosted.registerReplica({ name: "Reader", mode: "read_only" }); + const source = new TestFileSystem(); + const destination = new TestFileSystem(); + const writer = new WritableDirectoryMirror(writerId, hosted.transport(writerId), { + fileSystem: source, stateStore: new MemoryMirrorStateStore(), runtime: deterministicRuntime() + }); + const reader = new DirectoryMirror(readerId, hosted.transport(readerId), { + fileSystem: destination, stateStore: new MemoryMirrorStateStore(), runtime: deterministicRuntime() + }); + const original = "---\na: [broken\n---\n\nOriginal\r\n"; + const updated = "---\n- nonmapping\n---\n\nUpdated without final newline"; + source.files.set("opaque.md", original); + await writer.sync(); + await reader.sync(); + expect(destination.files.get("opaque.md")).toBe(original); + source.files.set("opaque.md", updated); + await writer.sync(); + await reader.sync(); + expect(destination.files.get("opaque.md")).toBe(updated); + await source.move("opaque.md", "moved.md"); + expect((await writer.inspect()).actions).toEqual(expect.arrayContaining([ + expect.objectContaining({ command: "move_remote" }) + ])); + await writer.sync(); + await reader.sync(); + expect(destination.files.has("opaque.md")).toBe(false); + expect(destination.files.get("moved.md")).toBe(updated); + await source.remove("moved.md"); + await writer.sync(); + await reader.sync(); + expect(destination.files.has("moved.md")).toBe(false); + expect((await reader.inspect()).summary.blocking_issues).toBe(0); + }); + + it("replays a lost opaque-upload receipt after restart without duplicating records", async () => { + const hosted = new MemoryAuthority(); + const replicaId = hosted.registerReplica({ name: "Writer", mode: "read_write" }); + const transport = hosted.transport(replicaId); + const source = new TestFileSystem(); + const stateStore = new MemoryMirrorStateStore(); + const document = "---\na: [broken\n---\nExact recovery bytes\n"; + source.files.set("opaque.md", document); + source.files.set("valid.md", "A valid sibling\n"); + let loseReply = true; + const unreliable: SyncTransport = { + ...transport, + mutate: async (mutation) => { + const receipt = await transport.mutate(mutation); + if (loseReply && mutation.path === "opaque.md") { + loseReply = false; + throw new Error("Injected reply loss after authority commit"); + } + return receipt; + } + }; + const mirror = new WritableDirectoryMirror(replicaId, unreliable, { + fileSystem: source, stateStore, runtime: deterministicRuntime() + }); + expect((await mirror.sync()).status).toBe("failed"); + expect((await stateStore.read())?.batch).toBeDefined(); + const restarted = new WritableDirectoryMirror(replicaId, transport, { + fileSystem: source, stateStore, runtime: deterministicRuntime() + }); + expect((await restarted.sync()).status).toBe("applied"); + const snapshot = await transport.snapshot((await transport.openSession()).snapshot_id); + expect(snapshot.records).toHaveLength(2); + expect(snapshot.records.find((record) => record.path === "opaque.md")?.document).toBe(document); + expect(hosted.serialize().receipts).toHaveLength(2); + expect((await stateStore.read())?.batch).toBeUndefined(); + }); + + it("reuses an existing checkpoint after managed YAML becomes malformed", async () => { + const hosted = new MemoryAuthority(); + const replicaId = hosted.registerReplica({ name: "Writer", mode: "read_write" }); + const transport = hosted.transport(replicaId); + const source = new TestFileSystem(); + const stateStore = new MemoryMirrorStateStore(); + const options = { fileSystem: source, stateStore, runtime: deterministicRuntime() }; + source.files.set("managed.md", "---\ntitle: Original\n---\nBody"); + await new WritableDirectoryMirror(replicaId, transport, options).sync(); + const before = await stateStore.read(); + const originalId = Object.keys(before!.records)[0]!; + const malformed = "---\ntitle: [broken\n---\nUpdated body"; + source.files.set("managed.md", malformed); + const restarted = new WritableDirectoryMirror(replicaId, transport, options); + const plan = await restarted.inspect(); + expect(plan.summary).toMatchObject({ uploads: 1, blocking_issues: 0 }); + expect((await restarted.apply(plan)).status).toBe("applied"); + const snapshot = await transport.snapshot((await transport.openSession()).snapshot_id); + expect(snapshot.records).toHaveLength(1); + expect(snapshot.records[0]).toMatchObject({ record_id: originalId, document: malformed }); + expect((await stateStore.read())!.generation).toBeGreaterThan(before!.generation!); + }); + + it("stops a stale opaque-document review before uploading any sibling", async () => { + const hosted = new MemoryAuthority(); + const replicaId = hosted.registerReplica({ name: "Writer", mode: "read_write" }); + const transport = hosted.transport(replicaId); + const source = new TestFileSystem(); + const stateStore = new MemoryMirrorStateStore(); + source.files.set("opaque.md", "---\na: [broken\n---\nReviewed"); + source.files.set("valid.md", "Valid sibling"); + const mirror = new WritableDirectoryMirror(replicaId, transport, { + fileSystem: source, stateStore, runtime: deterministicRuntime() + }); + const reviewed = await mirror.inspect(); + expect(reviewed.summary.blocking_issues).toBe(0); + const changed = "---\na: [still broken\n---\nEdited after review"; + source.files.set("opaque.md", changed); + expect((await mirror.apply(reviewed)).status).toBe("stale"); + expect(hosted.serialize().receipts).toHaveLength(0); + expect(await stateStore.read()).toBeNull(); + await mirror.sync(); + const snapshot = await transport.snapshot((await transport.openSession()).snapshot_id); + expect(snapshot.records.find((record) => record.path === "opaque.md")?.document).toBe(changed); + }); + + it("does not silently overwrite readable malformed edits in receive-only mirrors", async () => { + const hosted = new MemoryAuthority(); + hosted.seed([{ record_id: "managed", path: "managed.md", frontmatter: {}, body: "Authority", types: [] }]); + const replicaId = hosted.registerReplica({ name: "Reader", mode: "read_only" }); + const source = new TestFileSystem(); + const stateStore = new MemoryMirrorStateStore(); + const mirror = new DirectoryMirror(replicaId, hosted.transport(replicaId), { + fileSystem: source, stateStore, runtime: deterministicRuntime() + }); + await mirror.sync(); + const before = await stateStore.read(); + const edited = "---\na: [broken\n---\nLocal edit must survive"; + source.files.set("managed.md", edited); + const plan = await mirror.inspect(); + expect(plan.issues).toEqual(expect.arrayContaining([ + expect.objectContaining({ code: "invalid_frontmatter", blocking: false }), + expect.objectContaining({ code: "mirror_diverged", blocking: true }) + ])); + expect(plan.actions).toEqual([]); + await mirror.sync(); + expect(source.files.get("managed.md")).toBe(edited); + expect(await stateStore.read()).toEqual(before); + }); + it("makes a 2,000-record no-op sync a zero-write operation", async () => { const hosted = new MemoryAuthority({ snapshotPageSize: 100 }); hosted.seed(records(2_000)); diff --git a/packages/sync/src/node.test.ts b/packages/sync/src/node.test.ts index 6d8102d99..ed98960ef 100644 --- a/packages/sync/src/node.test.ts +++ b/packages/sync/src/node.test.ts @@ -297,6 +297,32 @@ describe("receive-only Markdown mirror", () => { }); describe("writable Markdown mirror", () => { + it("round-trips opaque Markdown through real Node filesystem adapters without normalizing bytes", async () => { + const root = await mkdtemp(join(tmpdir(), "mdbase-opaque-writer-")); + const receiverRoot = await mkdtemp(join(tmpdir(), "mdbase-opaque-reader-")); + try { + const hosted = new MemoryAuthority(); + const writerId = hosted.registerReplica({ name: "Writer", mode: "read_write" }); + const readerId = hosted.registerReplica({ name: "Reader", mode: "read_only" }); + const bytes = Buffer.from("\uFEFF---\r\ntitle: [broken\r\n---\r\n\r\nExact body — no final newline", "utf8"); + await writeFile(join(root, "opaque.md"), bytes); + await writeFile(join(root, "valid.md"), "Valid sibling\n"); + const writer = new WritableDirectoryMirror(root, writerId, hosted.transport(writerId), deviceState()); + const reader = new DirectoryMirror(receiverRoot, readerId, hosted.transport(readerId), deviceState()); + const plan = await writer.inspect(); + expect(plan.summary).toMatchObject({ uploads: 2, blocking_issues: 0 }); + await expect(writer.apply(plan)).resolves.toMatchObject({ status: "applied" }); + await reader.sync(); + expect(await readFile(join(root, "opaque.md"))).toEqual(bytes); + expect(await readFile(join(receiverRoot, "opaque.md"))).toEqual(bytes); + expect(await readFile(join(receiverRoot, "valid.md"), "utf8")).toBe("Valid sibling\n"); + expect((await reader.inspect()).actions).toEqual([]); + } finally { + await rm(root, { recursive: true, force: true }); + await rm(receiverRoot, { recursive: true, force: true }); + } + }); + it("builds a stable canonical authority manifest", async () => { expect(authorityManifestDigest([ { diff --git a/packages/sync/src/node.ts b/packages/sync/src/node.ts index d8037f852..1191f2bbc 100644 --- a/packages/sync/src/node.ts +++ b/packages/sync/src/node.ts @@ -331,7 +331,8 @@ export class NodeMirrorFileSystem implements MirrorFileSystem { try { const bytes = await readFile(target); try { - return new TextDecoder("utf-8", { fatal: true }).decode(bytes); + // Preserve the BOM as a character instead of consuming its bytes. + return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes); } catch { return { kind: "invalid", diff --git a/packages/sync/src/sync-inspector.ts b/packages/sync/src/sync-inspector.ts index 833f65f67..2b9926634 100644 --- a/packages/sync/src/sync-inspector.ts +++ b/packages/sync/src/sync-inspector.ts @@ -177,7 +177,7 @@ export class PlanOnlyMirrorInspector issue.code === "invalid_frontmatter") + && local.issues.some((issue) => issue.code === "invalid_frontmatter" && issue.blocking) ) return this.inspectSnapshot("rebuild", state); return this.finish({ kind: "incremental", @@ -347,7 +347,7 @@ export class PlanOnlyMirrorInspector Date: Wed, 16 Sep 2026 19:47:59 +1000 Subject: [PATCH 2/5] docs: record Obsidian candidate compatibility evidence --- docs/lossless-opaque-sync-acceptance.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/lossless-opaque-sync-acceptance.md b/docs/lossless-opaque-sync-acceptance.md index 02957647b..ba64ded68 100644 --- a/docs/lossless-opaque-sync-acceptance.md +++ b/docs/lossless-opaque-sync-acceptance.md @@ -30,7 +30,11 @@ The YAML parser's default duplicate-key rejection is retained and tested. diagnostic prose incorrectly described every warning as a pause. A separate consumer branch updates that prose, tests nonblocking reviews, and preserves BOMs in its Vault adapter. Its release pin must not move to an unpublished - npm version. + npm version. Immutable artifacts from commit `478353d28bb0` pass all 96 + Obsidian unit tests, TypeScript/build and unchanged mobile budgets, plus eight + strict exact round trips through the Obsidian adapter and reference authority. + The same strict check fails against released beta.91 with seven YAML blockers. + Its released dependencies and generated bundle were restored afterward. - `tasknotes-app` only pins `connect-sync` as an immutable SDK override; no direct directory-mirror or `local_issues` consumer was found in its application code. - Desktop renderer status is owned by the Rust mirror, not this JavaScript SDK @@ -43,7 +47,8 @@ are reused; unsupported older engine layouts remain rejected as before. Node 24.19.0 / pnpm 11.15.1: -- `pnpm typecheck`: all workspaces pass. +- `pnpm typecheck`: all workspaces pass. The guarded consumer-artifact packaging + command also completed a full `pnpm -r build`. - `pnpm --filter @mdbase-dev/connect-sync test`: 193 tests pass. - `pnpm --filter '!@mdbase/connect-desktop' -r test`: passes, including server (582 passed, 35 intentionally skipped) and editor (473 passed). From 87e28391233c9cfbe2dca0c3bf1c88a526404373 Mon Sep 17 00:00:00 2001 From: callumalpass Date: Wed, 16 Sep 2026 19:53:51 +1000 Subject: [PATCH 3/5] fix(sync): align BOM and leading-fence projections with authority --- docs/lossless-opaque-sync-acceptance.md | 12 +++++++++-- packages/sync/README.md | 5 ++++- packages/sync/src/mirror-format.ts | 26 ++++++++++++++---------- packages/sync/src/mirror.test.ts | 27 ++++++++++++++++++++++++- packages/sync/src/node.test.ts | 8 ++++++-- 5 files changed, 61 insertions(+), 17 deletions(-) diff --git a/docs/lossless-opaque-sync-acceptance.md b/docs/lossless-opaque-sync-acceptance.md index ba64ded68..8a4ecf69d 100644 --- a/docs/lossless-opaque-sync-acceptance.md +++ b/docs/lossless-opaque-sync-acceptance.md @@ -15,6 +15,9 @@ replace automatically. Existing authority scope/configuration checks remain. Both Node and embedding adapters must preserve a UTF-8 BOM. Node's decoder now retains it; the new real-filesystem regression caught the previous BOM loss. The YAML parser's default duplicate-key rejection is retained and tested. +Structural diagnostics and projection parsing now share Rust-compatible +leading-fence recognition, including BOM-prefixed mappings/body-only records, +non-leading fences, and the authority's closing-delimiter/body boundary. ## Consumer audit @@ -49,10 +52,10 @@ Node 24.19.0 / pnpm 11.15.1: - `pnpm typecheck`: all workspaces pass. The guarded consumer-artifact packaging command also completed a full `pnpm -r build`. -- `pnpm --filter @mdbase-dev/connect-sync test`: 193 tests pass. +- `pnpm --filter @mdbase-dev/connect-sync test`: 196 tests pass. - `pnpm --filter '!@mdbase/connect-desktop' -r test`: passes, including server (582 passed, 35 intentionally skipped) and editor (473 passed). -- `pnpm check:mirror:mobile`: passes unchanged budgets, 187,632 raw / 55,991 +- `pnpm check:mirror:mobile`: passes unchanged budgets, 187,664 raw / 55,970 gzip bytes; no Node-only references in the portable bundle. New/updated regressions cover exact uploads and second-mirror downloads, @@ -60,6 +63,11 @@ BOM/CRLF/no-final-newline preservation, updates/moves/deletes, both conflict choices, lost-reply replay after restart, existing checkpoint reuse, stale reviews, receive-only divergence, invalid UTF-8 and failed rereads. +The canonical Rust runtime's five `frontmatter::parser` tests and +`schema_invalid_record_remains_semantically_projectable` test also pass with +`cargo test --locked --lib`; these are source/runtime evidence, not a hosted +round trip or full Connect Rust-workspace qualification. + Authorities in these SDK tests are `MemoryAuthority`, not deployed hosted infrastructure. The Node round-trip uses real temporary filesystem directories; the portable fixtures use in-memory adapters. diff --git a/packages/sync/README.md b/packages/sync/README.md index 7486f7c23..30c6cf734 100644 --- a/packages/sync/README.md +++ b/packages/sync/README.md @@ -12,7 +12,10 @@ unclosed `---` fence are preserved byte-for-byte as body content. An explicitly empty frontmatter block is also accepted. Malformed, scalar, null, or list frontmatter is synchronized byte-for-byte as opaque Markdown with `{}` persisted fields; structured queries and field operations ignore its invalid -frontmatter until the document is repaired. This is the default for directory +frontmatter until the document is repaired. Leading-fence and BOM recognition +matches the Rust authority: a body-only record's structured body projection +omits one leading BOM, but the exact `document` always retains it. Opaque +fallback bodies retain the whole document. This is the default for directory mirrors, not an opt-in mode. Record schema validation is separate from exact replication: missing required fields, wrong field types, and enum violations do not by themselves prevent synchronization. Authority scope, configuration, diff --git a/packages/sync/src/mirror-format.ts b/packages/sync/src/mirror-format.ts index a4d771a6a..b8e445761 100644 --- a/packages/sync/src/mirror-format.ts +++ b/packages/sync/src/mirror-format.ts @@ -20,8 +20,9 @@ export type LocalRecordStructuralOutcome = { /** Structural diagnostics only. Readable documents synchronize exactly, including opaque frontmatter. */ export function classifyLocalRecord(document: string): LocalRecordStructuralOutcome { - const yaml = leadingFrontmatterYaml(document); - if (yaml === null) return { outcome: "parsed" }; + const block = leadingFrontmatter(document); + if (block === null) return { outcome: "parsed" }; + const { yaml } = block; let frontmatter: unknown; try { @@ -37,7 +38,7 @@ export function classifyLocalRecord(document: string): LocalRecordStructuralOutc } /** Mirrors mdbase-rs parse_document_without_bom delimiter recognition. */ -function leadingFrontmatterYaml(document: string): string | null { +function leadingFrontmatter(document: string): { yaml: string; body: string } | null { const content = document.startsWith("\uFEFF") ? document.slice(1) : document; const openingEnd = content.indexOf("\n"); if (openingEnd < 0 || content.slice(0, openingEnd).trimEnd() !== "---") return null; @@ -46,7 +47,10 @@ function leadingFrontmatterYaml(document: string): string | null { let lineEnd = content.indexOf("\n", lineStart); if (lineEnd < 0) lineEnd = content.length; if (content.slice(lineStart, lineEnd).trimEnd() === "---") { - return content.slice(yamlStart, lineStart); + return { + yaml: content.slice(yamlStart, lineStart), + body: content.slice(lineStart + 3).replace(/^\r?\n/, "") + }; } lineStart = lineEnd + 1; } @@ -148,24 +152,24 @@ export function fastRecordDocumentMatches(document: string, record: SyncRecord): } export function parseMarkdown(document: string, _path: string): { frontmatter: JsonObject; body: string } { - const match = document.match(/^---[ \t]*\r?\n([\s\S]*?)^---[ \t]*(?:\r?\n|$)([\s\S]*)$/m); - if (!match) { - return { frontmatter: {}, body: document }; + const block = leadingFrontmatter(document); + if (block === null) { + return { frontmatter: {}, body: document.startsWith("\uFEFF") ? document.slice(1) : document }; } let frontmatter: unknown; try { - frontmatter = parse(match[1]!, { mapAsMap: true }); + frontmatter = parse(block.yaml, { mapAsMap: true }); } catch { return { frontmatter: {}, body: document }; } - if (frontmatter === null && match[1]!.trim() === "") { - return { frontmatter: {}, body: match[2] ?? "" }; + if (frontmatter === null && block.yaml.trim() === "") { + return { frontmatter: {}, body: block.body }; } const projection = jsonProjection(frontmatter, new Set()); if (projection === INVALID_JSON_PROJECTION || !isJsonObject(projection)) { return { frontmatter: {}, body: document }; } - return { frontmatter: projection, body: match[2] ?? "" }; + return { frontmatter: projection, body: block.body }; } function jsonProjection( diff --git a/packages/sync/src/mirror.test.ts b/packages/sync/src/mirror.test.ts index c94205423..a93483ff2 100644 --- a/packages/sync/src/mirror.test.ts +++ b/packages/sync/src/mirror.test.ts @@ -1,7 +1,7 @@ import { createHash } from "node:crypto"; import { describe, expect, it } from "vitest"; import { MemoryAuthority, type SyncTransport } from "./index.js"; -import { classifyLocalRecord, documentRevision } from "./mirror-format.js"; +import { classifyLocalRecord, documentRevision, parseMarkdown } from "./mirror-format.js"; import { SyncError } from "./sync-error.js"; import { DirectoryMirror, @@ -148,6 +148,31 @@ describe("platform-neutral directory mirror", () => { } }); + it("matches Rust BOM and leading-fence projections without changing authority bytes", async () => { + const cases: Array<{ document: string; frontmatter: Record; body: string }> = [ + { document: "\uFEFF---\r\ntitle: Present\r\n---\r\nBody", frontmatter: { title: "Present" }, body: "Body" }, + { document: "\uFEFF# Body only\r\n", frontmatter: {}, body: "# Body only\r\n" }, + { document: "\uFEFF\uFEFF---\ntitle: Not frontmatter\n---\n", frontmatter: {}, body: "\uFEFF---\ntitle: Not frontmatter\n---\n" }, + { document: "Intro\n---\ntitle: Not frontmatter\n---\nBody", frontmatter: {}, body: "Intro\n---\ntitle: Not frontmatter\n---\nBody" }, + { document: "--- \t\r\ntitle: Present\r\n--- \t\r\nBody", frontmatter: { title: "Present" }, body: " \t\r\nBody" }, + ]; + for (const expected of cases) { + expect(parseMarkdown(expected.document, "record.md")).toEqual({ frontmatter: expected.frontmatter, body: expected.body }); + const hosted = new MemoryAuthority(); + // Supply the Rust-compatible projection explicitly, rather than deriving + // both sides from the SDK parser under test. + hosted.seed([{ record_id: "record", path: "record.md", types: [], ...expected }]); + const replicaId = hosted.registerReplica({ name: "Reader", mode: "read_only" }); + const fileSystem = new TestFileSystem(); + const mirror = new DirectoryMirror(replicaId, hosted.transport(replicaId), { + fileSystem, stateStore: new MemoryMirrorStateStore(), runtime: deterministicRuntime() + }); + expect((await mirror.sync()).status).toBe("applied"); + expect(fileSystem.files.get("record.md")).toBe(expected.document); + expect((await mirror.inspect()).actions).toEqual([]); + } + }); + it("materializes the authority's exact snapshot Markdown bytes", async () => { const hosted = new MemoryAuthority(); hosted.seed([{ diff --git a/packages/sync/src/node.test.ts b/packages/sync/src/node.test.ts index ed98960ef..132dcf6d3 100644 --- a/packages/sync/src/node.test.ts +++ b/packages/sync/src/node.test.ts @@ -297,14 +297,18 @@ describe("receive-only Markdown mirror", () => { }); describe("writable Markdown mirror", () => { - it("round-trips opaque Markdown through real Node filesystem adapters without normalizing bytes", async () => { + it.each([ + "\uFEFF---\r\ntitle: [broken\r\n---\r\n\r\nExact body — no final newline", + "\uFEFF---\r\ntitle: Present\r\n---\r\n\r\nExact body — no final newline", + "\uFEFF# Body-only note\r\n" + ])("round-trips BOM-prefixed Markdown through real Node filesystem adapters: %j", async (document) => { const root = await mkdtemp(join(tmpdir(), "mdbase-opaque-writer-")); const receiverRoot = await mkdtemp(join(tmpdir(), "mdbase-opaque-reader-")); try { const hosted = new MemoryAuthority(); const writerId = hosted.registerReplica({ name: "Writer", mode: "read_write" }); const readerId = hosted.registerReplica({ name: "Reader", mode: "read_only" }); - const bytes = Buffer.from("\uFEFF---\r\ntitle: [broken\r\n---\r\n\r\nExact body — no final newline", "utf8"); + const bytes = Buffer.from(document, "utf8"); await writeFile(join(root, "opaque.md"), bytes); await writeFile(join(root, "valid.md"), "Valid sibling\n"); const writer = new WritableDirectoryMirror(root, writerId, hosted.transport(writerId), deviceState()); From a0c89c50a6571a341a416683b10f6659b8316f3e Mon Sep 17 00:00:00 2001 From: callumalpass Date: Wed, 16 Sep 2026 19:55:24 +1000 Subject: [PATCH 4/5] docs: qualify final BOM-compatible consumer artifacts --- docs/lossless-opaque-sync-acceptance.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/lossless-opaque-sync-acceptance.md b/docs/lossless-opaque-sync-acceptance.md index 8a4ecf69d..4452f4490 100644 --- a/docs/lossless-opaque-sync-acceptance.md +++ b/docs/lossless-opaque-sync-acceptance.md @@ -33,8 +33,8 @@ non-leading fences, and the authority's closing-delimiter/body boundary. diagnostic prose incorrectly described every warning as a pause. A separate consumer branch updates that prose, tests nonblocking reviews, and preserves BOMs in its Vault adapter. Its release pin must not move to an unpublished - npm version. Immutable artifacts from commit `478353d28bb0` pass all 96 - Obsidian unit tests, TypeScript/build and unchanged mobile budgets, plus eight + npm version. Immutable artifacts from commit `87e28391233c` pass all 96 + Obsidian unit tests, TypeScript/build and unchanged mobile budgets, plus ten strict exact round trips through the Obsidian adapter and reference authority. The same strict check fails against released beta.91 with seven YAML blockers. Its released dependencies and generated bundle were restored afterward. From 273f71e155fe3ad85fe954eb72134f5381648781 Mon Sep 17 00:00:00 2001 From: callumalpass Date: Wed, 16 Sep 2026 23:03:14 +1000 Subject: [PATCH 5/5] fix(ci): refresh verified upgrade predecessor to beta102 --- .github/previous-release.env | 8 +++--- docs/release-contract.md | 6 ++-- scripts/lib/upgrade-release-identity.test.mjs | 28 +++++++++---------- scripts/lib/upgrade-test-contract.test.mjs | 8 +++--- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/previous-release.env b/.github/previous-release.env index 05aea88ae..96de8760f 100644 --- a/.github/previous-release.env +++ b/.github/previous-release.env @@ -1,6 +1,6 @@ # Exact server image from the release immediately preceding this candidate. # Update this file as part of each release-preparation change. -MDBASE_CONNECT_PREVIOUS_RELEASE=v0.1.0-beta.101 -MDBASE_CONNECT_PREVIOUS_RELEASE_COMMIT=4d0bdf9ec566a15c117249ca8d5b24107f1faecb -MDBASE_CONNECT_PREVIOUS_SERVER_IMAGE=ghcr.io/mdbase-dev/mdbase-connect-server@sha256:836962e2322701360aadaf1b1873447803cf4603e003803e9880ff9ca85b1e9c -MDBASE_CONNECT_PREVIOUS_PROVIDER_IMAGE=ghcr.io/mdbase-dev/mdbase-connect-hosted-provider@sha256:dfc5fb9d37e2017bb08f12dde9eb9fc4864103ffd0f73f41d9141cdd6d47dd2e +MDBASE_CONNECT_PREVIOUS_RELEASE=v0.1.0-beta.102 +MDBASE_CONNECT_PREVIOUS_RELEASE_COMMIT=ea65ae4f13d0e0621fe9654618e13b505cdebf21 +MDBASE_CONNECT_PREVIOUS_SERVER_IMAGE=ghcr.io/mdbase-dev/mdbase-connect-server@sha256:bfb2ca0beb8e63005b89354f7b57e3568fb438f40f4948963a723c1aeb815f1e +MDBASE_CONNECT_PREVIOUS_PROVIDER_IMAGE=ghcr.io/mdbase-dev/mdbase-connect-hosted-provider@sha256:6b5c732b7ba21b6166acbaccd3b81e9a2ad4007e7e502f99bc5739d1d37585db diff --git a/docs/release-contract.md b/docs/release-contract.md index cf1dc8cc2..1fe14353a 100644 --- a/docs/release-contract.md +++ b/docs/release-contract.md @@ -55,9 +55,9 @@ components. `.github/previous-release.env` identifies the immediate published predecessor. Update its annotated tag, full commit and immutable server/provider digests as part of release preparation; ordinary upgrade qualification still requires it to -be the unique newest non-draft GitHub release. Following beta100 publication, -qualification uses beta100 (`6bd9420690295d3c5f65e68edcbf8e3e55612614`) and its -exact signed server/provider images. Advancing this fixture preserves the +be the unique newest non-draft GitHub release. Following beta102 publication, +qualification uses beta102 (`ea65ae4f13d0e0621fe9654618e13b505cdebf21`) and its +exact signed server/provider images from verified publication run `35059159880`. Advancing this fixture preserves the unique-newest-release check; the retained beta95 and beta94 lanes do not move. This fixture refresh does not itself publish or deploy a release. Historical regressions and candidate qualification are separate. The beta94 diff --git a/scripts/lib/upgrade-release-identity.test.mjs b/scripts/lib/upgrade-release-identity.test.mjs index eeb94d83d..8d083be94 100644 --- a/scripts/lib/upgrade-release-identity.test.mjs +++ b/scripts/lib/upgrade-release-identity.test.mjs @@ -9,9 +9,9 @@ import { promisify } from "node:util"; const execute = promisify(execFile); const root = resolve(import.meta.dirname, "../.."); const beta95 = "408c67bc10f128e0833f0da62cb3efb9d94657d7"; -const beta101 = "4d0bdf9ec566a15c117249ca8d5b24107f1faecb"; +const predecessorCommit = "ea65ae4f13d0e0621fe9654618e13b505cdebf21"; const release95 = { id: 95, tag_name: "v0.1.0-beta.95", draft: false, published_at: "2026-09-08T00:00:00Z" }; -const release101 = { ...release95, id: 101, tag_name: "v0.1.0-beta.101" }; +const predecessorRelease = { ...release95, id: 102, tag_name: "v0.1.0-beta.102" }; const annotated = (tag, commit) => `${"a".repeat(40)}\trefs/tags/${tag}\n${commit}\trefs/tags/${tag}^{}\n`; async function verify(context, { historical = false, metadata, refs, override = "", network = "ok", image = false, inspection } = {}) { @@ -41,8 +41,8 @@ printf '%s' \"$INSPECTION\" ` }; for (const [name, content] of Object.entries(scripts)) await writeFile(join(bin, name), content, { mode: 0o700 }); - const tag = historical ? release95.tag_name : release101.tag_name; - const commit = historical ? beta95 : beta101; + const tag = historical ? release95.tag_name : predecessorRelease.tag_name; + const commit = historical ? beta95 : predecessorCommit; const calls = join(work, "calls"); const pin = historical ? "retained-v2-predecessor.env" : "previous-release.env"; const verifier = historical ? "upgrade_verify_retained_v2_release" : "upgrade_verify_previous_release"; @@ -62,7 +62,7 @@ ${image ? 'upgrade_verify_previous_image "$MDBASE_CONNECT_PREVIOUS_SERVER_IMAGE" EXPECTED_URL: historical ? "https://api.github.com/repos/mdbase-dev/mdbase-connect/releases/tags/v0.1.0-beta.95" : "https://api.github.com/repos/mdbase-dev/mdbase-connect/releases?per_page=100", - METADATA: typeof metadata === "string" ? metadata : JSON.stringify(metadata ?? (historical ? release95 : [release101, release95])), + METADATA: typeof metadata === "string" ? metadata : JSON.stringify(metadata ?? (historical ? release95 : [predecessorRelease, release95])), REFS: refs ?? annotated(tag, commit), INSPECTION: JSON.stringify(inspection ?? [{ Config: { Labels: { "org.opencontainers.image.source": "https://github.com/mdbase-dev/mdbase-connect", @@ -78,13 +78,13 @@ ${image ? 'upgrade_verify_previous_image "$MDBASE_CONNECT_PREVIOUS_SERVER_IMAGE" return { code, stderr, calls: await readFile(calls, "utf8").catch(() => "") }; } -test("ordinary predecessor verifies newest beta101 and its annotated origin tag", async (context) => { +test("ordinary predecessor verifies newest beta102 and its annotated origin tag", async (context) => { const result = await verify(context); assert.equal(result.code, 0, result.stderr); assert.match(result.calls, /releases\?per_page=100\ngit\n$/); }); -test("ordinary qualification cannot substitute the historical beta95 pin for newest beta101", async (context) => { +test("ordinary qualification cannot substitute the historical beta95 pin for newest beta102", async (context) => { const result = await verify(context, { override: 'source "$ROOT/.github/retained-v2-predecessor.env"' }); assert.equal(result.code, 1); assert.match(result.stderr, /not the unique newest non-draft/); @@ -92,11 +92,11 @@ test("ordinary qualification cannot substitute the historical beta95 pin for new }); for (const [name, metadata] of [ - ["older pin", [release95, release101]], - ["duplicate release", [release101, release101]], - ["draft only", [{ ...release101, draft: true }]], + ["older pin", [release95, predecessorRelease]], + ["duplicate release", [predecessorRelease, predecessorRelease]], + ["draft only", [{ ...predecessorRelease, draft: true }]], ["missing release", []], - ["object instead of inventory", release101], + ["object instead of inventory", predecessorRelease], ["malformed JSON", "not-json"] ]) test(`ordinary predecessor rejects ${name}`, async (context) => { const result = await verify(context, { metadata }); @@ -105,8 +105,8 @@ for (const [name, metadata] of [ }); for (const historical of [false, true]) { - const tag = historical ? release95.tag_name : release101.tag_name; - const commit = historical ? beta95 : beta101; + const tag = historical ? release95.tag_name : predecessorRelease.tag_name; + const commit = historical ? beta95 : predecessorCommit; for (const [name, refs] of [ ["wrong peeled commit", annotated(tag, "b".repeat(40))], ["lightweight tag", `${commit}\trefs/tags/${tag}\n`], @@ -132,7 +132,7 @@ test("historical regression verifies published beta95 by exact tag, independentl }); for (const metadata of [ - { ...release95, draft: true }, { ...release95, tag_name: release101.tag_name }, + { ...release95, draft: true }, { ...release95, tag_name: predecessorRelease.tag_name }, { ...release95, published_at: null }, { ...release95, published_at: "" }, { ...release95, id: "95" }, [release95], {}, "not-json" ]) test(`historical rejects unqualified release metadata ${JSON.stringify(metadata)}`, async (context) => { diff --git a/scripts/lib/upgrade-test-contract.test.mjs b/scripts/lib/upgrade-test-contract.test.mjs index 3cbb02432..a9da6fd1f 100644 --- a/scripts/lib/upgrade-test-contract.test.mjs +++ b/scripts/lib/upgrade-test-contract.test.mjs @@ -27,10 +27,10 @@ test("upgrade pins the exact immediate predecessor", async () => { ); assert.equal(fixture, `# Exact server image from the release immediately preceding this candidate. # Update this file as part of each release-preparation change. -MDBASE_CONNECT_PREVIOUS_RELEASE=v0.1.0-beta.101 -MDBASE_CONNECT_PREVIOUS_RELEASE_COMMIT=4d0bdf9ec566a15c117249ca8d5b24107f1faecb -MDBASE_CONNECT_PREVIOUS_SERVER_IMAGE=ghcr.io/mdbase-dev/mdbase-connect-server@sha256:836962e2322701360aadaf1b1873447803cf4603e003803e9880ff9ca85b1e9c -MDBASE_CONNECT_PREVIOUS_PROVIDER_IMAGE=ghcr.io/mdbase-dev/mdbase-connect-hosted-provider@sha256:dfc5fb9d37e2017bb08f12dde9eb9fc4864103ffd0f73f41d9141cdd6d47dd2e +MDBASE_CONNECT_PREVIOUS_RELEASE=v0.1.0-beta.102 +MDBASE_CONNECT_PREVIOUS_RELEASE_COMMIT=ea65ae4f13d0e0621fe9654618e13b505cdebf21 +MDBASE_CONNECT_PREVIOUS_SERVER_IMAGE=ghcr.io/mdbase-dev/mdbase-connect-server@sha256:bfb2ca0beb8e63005b89354f7b57e3568fb438f40f4948963a723c1aeb815f1e +MDBASE_CONNECT_PREVIOUS_PROVIDER_IMAGE=ghcr.io/mdbase-dev/mdbase-connect-hosted-provider@sha256:6b5c732b7ba21b6166acbaccd3b81e9a2ad4007e7e502f99bc5739d1d37585db `); });