From 9af2b2ae66baf6bf4c6e79ec888857df95e5e4e7 Mon Sep 17 00:00:00 2001 From: callumalpass Date: Sun, 13 Sep 2026 19:40:25 +1000 Subject: [PATCH 01/16] fix(desktop): retain update transactions until recovery is healthy --- apps/desktop/src/main/update-coordinator.ts | 31 +++++++-- apps/desktop/test/update-coordinator.test.mjs | 69 +++++++++++++++++++ 2 files changed, 93 insertions(+), 7 deletions(-) diff --git a/apps/desktop/src/main/update-coordinator.ts b/apps/desktop/src/main/update-coordinator.ts index c5fca2429..5c123fc00 100644 --- a/apps/desktop/src/main/update-coordinator.ts +++ b/apps/desktop/src/main/update-coordinator.ts @@ -90,6 +90,13 @@ export class UpdateCoordinator { return structuredClone(this.statusValue); } + daemonStartupBlock(): string | null { + const status = this.statusValue; + return status.phase === "installing" || + (["recovery", "failed"].includes(status.phase) && !status.can_check) + ? status.message : null; + } + subscribe(listener: (status: DesktopUpdateStatus) => void): () => void { this.listeners.add(listener); listener(this.status()); @@ -134,8 +141,9 @@ export class UpdateCoordinator { }); try { const result = await this.backend.recover(persisted.transaction); + if (!result.healthy) throw new Error(result.message); await this.store.update((state) => { - if (result.healthy && !result.rolledBack) { + if (!result.rolledBack) { state.highest_trusted_version = maxVersion( state.highest_trusted_version, persisted.transaction?.target_version @@ -150,7 +158,7 @@ export class UpdateCoordinator { delete state.transaction; }); this.setStatus({ - phase: result.healthy && !result.rolledBack ? "idle" : "recovery", + phase: result.rolledBack ? "recovery" : "idle", message: result.message, can_check: this.backend.packaged, can_install: false @@ -218,18 +226,27 @@ export class UpdateCoordinator { }); this.backend.installAutomatic(); } catch (error) { - const recovered = await this.backend.recover(transaction).catch(() => null); await this.store.update((state) => { - delete state.transaction; + if (state.transaction?.id === transaction.id) state.transaction.phase = "recovering"; + }); + const recovered = await this.backend.recover(transaction).catch((recoveryError) => ({ + healthy: false, + rolledBack: false, + message: message(recoveryError) + })); + await this.store.update((state) => { + if (state.transaction?.id !== transaction.id) return; + if (recovered.healthy) delete state.transaction; + else state.transaction.error = recovered.message; }); this.candidate = null; this.setStatus({ phase: "failed", target_version: transaction.target_version, - message: recovered?.healthy + message: recovered.healthy ? `The update was not installed; the connector was restored. ${message(error)}` - : `The update was not installed and connector recovery failed: ${message(error)}`, - can_check: this.backend.packaged, + : `The update was not installed: ${message(error)}. Connector recovery needs attention: ${recovered.message}`, + can_check: recovered.healthy && this.backend.packaged, can_install: false }); throw error; diff --git a/apps/desktop/test/update-coordinator.test.mjs b/apps/desktop/test/update-coordinator.test.mjs index a3ab30b16..e302b2d40 100644 --- a/apps/desktop/test/update-coordinator.test.mjs +++ b/apps/desktop/test/update-coordinator.test.mjs @@ -270,6 +270,75 @@ test("download and verification failures never enable installation", async () => assert.equal(JSON.parse(await readFile(path, "utf8")).transaction, undefined); }); +for (const cut of ["stop", "install"]) { + for (const outcome of ["throws", "unhealthy"]) { + test(`${cut} failure retains exact transaction when recovery ${outcome}`, async () => { + const { path, store } = await fixture(); + const runtime = backend({ + async stopDaemon() { + if (cut === "stop") throw new Error("stop failed"); + }, + installAutomatic() { + throw new Error("install failed"); + }, + async recover() { + if (outcome === "throws") throw new Error("recovery unavailable"); + return { healthy: false, rolledBack: false, message: "recovery unavailable" }; + } + }); + const coordinator = new UpdateCoordinator(store, runtime); + await coordinator.initialize(); + await coordinator.check(); + const original = JSON.parse(await readFile(path, "utf8")).transaction; + await assert.rejects(coordinator.install(), new RegExp(`${cut} failed`)); + const retained = JSON.parse(await readFile(path, "utf8")).transaction; + assert.equal(retained?.id, original.id); + assert.equal(retained.phase, "recovering"); + assert.equal(retained.previous_runtime, original.previous_runtime); + assert.equal(retained.target_version, original.target_version); + assert.match(retained.error, /recovery unavailable/); + assert.equal(coordinator.status().can_check, false); + const failed = coordinator.status(); + await coordinator.check(true); + assert.deepEqual(coordinator.status(), failed); + await assert.rejects(coordinator.install(), /No update is ready/); + + let resumed; + const restarted = new UpdateCoordinator(new UpdateStateStore(path), backend({ + async recover(transaction) { + resumed = transaction; + return { healthy: true, rolledBack: true, message: "Prior runtime restored." }; + } + })); + assert.equal((await restarted.initialize()).can_check, true); + assert.equal(resumed.id, original.id); + assert.equal(JSON.parse(await readFile(path, "utf8")).transaction, undefined); + }); + } +} + +test("startup retains the transaction when recovery returns unhealthy", async () => { + const { path, store } = await fixture(); + const initial = new UpdateCoordinator(store, backend()); + await initial.initialize(); + await initial.check(); + await initial.install(); + const original = JSON.parse(await readFile(path, "utf8")).transaction; + const restarted = new UpdateCoordinator(new UpdateStateStore(path), backend({ + async recover() { + return { healthy: false, rolledBack: true, message: "Rollback is not healthy." }; + } + })); + const status = await restarted.initialize(); + assert.equal(status.phase, "failed"); + assert.equal(status.can_check, false); + const persisted = JSON.parse(await readFile(path, "utf8")); + assert.equal(persisted.transaction?.id, original.id); + assert.equal(persisted.transaction.phase, "recovering"); + assert.match(persisted.transaction.error, /Rollback is not healthy/); + assert.equal(persisted.last_known_good_runtime, undefined); +}); + test("a stop failure invokes recovery and clears the transaction", async () => { const { path, store } = await fixture(); const runtime = backend({ From 271bbe147171c6a8ccdaf1ae6708d633fb031606 Mon Sep 17 00:00:00 2001 From: callumalpass Date: Sun, 13 Sep 2026 19:41:37 +1000 Subject: [PATCH 02/16] fix(auth): confirm local revocation only after exact policy acknowledgement --- apps/editor/src/ConnectApp.test.tsx | 8 +- apps/editor/src/ConnectApp.tsx | 2 +- .../src/runtime_notifications/tests.rs | 1 + .../src/server/files_scope_tests.rs | 1 + .../connect-agent/src/server/files_tests.rs | 1 + .../connect-agent/src/server/setup_binding.rs | 1 + crates/connect-core/src/registry/grants.rs | 2 + crates/connect-protocol/src/applications.rs | 3 + crates/connect-runtime/src/lib.rs | 1 + crates/connect-runtime/src/timers.rs | 1 + crates/connect-testbed-adapter/src/main.rs | 1 + .../0031_local_revocation_confirmation.sql | 5 + services/server/src/db.test.ts | 6 +- .../server/src/features/account/me-routes.ts | 2 + .../authorizations/grant-revocation-route.ts | 55 +---- .../src/features/connectors/control-routes.ts | 4 +- .../src/features/grants/connector-routes.ts | 24 +-- .../server/src/local-grant-revocation.test.ts | 188 ++++++++++++++++++ services/server/src/local-grant-revocation.ts | 74 +++++++ services/server/src/relay-policy-session.ts | 7 +- services/server/src/relay-policy.ts | 9 + 21 files changed, 328 insertions(+), 68 deletions(-) create mode 100644 services/server/migrations/0031_local_revocation_confirmation.sql create mode 100644 services/server/src/local-grant-revocation.test.ts create mode 100644 services/server/src/local-grant-revocation.ts diff --git a/apps/editor/src/ConnectApp.test.tsx b/apps/editor/src/ConnectApp.test.tsx index 730023132..3f440a5f7 100644 --- a/apps/editor/src/ConnectApp.test.tsx +++ b/apps/editor/src/ConnectApp.test.tsx @@ -448,14 +448,14 @@ describe("ConnectApp", () => { expect(screen.getByText("Entire collection")).toBeInTheDocument(); }); - it("keeps provider-pending revocations visible without claiming success", async () => { + it.each(["local", "hosted"] as const)("keeps %s pending revocations visible without claiming success", async (kind) => { overview.grants = [{ id: "grant", operations: ["read"], scope: { contracts: [], access: "full_collection" }, created_at: new Date().toISOString(), revoked_at: new Date().toISOString(), revocation_status: "revoking", reauthorization_required_at: null, reauthorization_reason: null, collection_id: "collection", - collection_name: "Garden notes", collection_kind: "hosted", + collection_name: "Garden notes", collection_kind: kind, application_id: "app", application_name: "Photo catalog", distribution: "web", homepage: "https://photos.example", project_url: null, application_origin: "https://photos.example", icon: null @@ -467,7 +467,9 @@ describe("ConnectApp", () => { await user.click(screen.getByRole("link", { name: /Applications/ })); expect(await screen.findByText("Revoking…")).toBeInTheDocument(); - expect(screen.getByText(/Waiting for the hosted authority to confirm revocation/)).toBeInTheDocument(); + expect(screen.getByText(kind === "hosted" + ? /Waiting for the hosted authority to confirm enforcement/ + : /Waiting for the computer holding this collection to confirm enforcement/)).toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Revoke" })).not.toBeInTheDocument(); }); diff --git a/apps/editor/src/ConnectApp.tsx b/apps/editor/src/ConnectApp.tsx index 135a692fb..1b7024e2c 100644 --- a/apps/editor/src/ConnectApp.tsx +++ b/apps/editor/src/ConnectApp.tsx @@ -632,7 +632,7 @@ function GrantEditor({ grant, busy, perform }: { perform: PerformOperation; }) { if (grant.revocation_status === "revoking") { - return
{grant.collection_name}Local access is disabled. Waiting for the hosted authority to confirm revocation.Revoking…
; + return
{grant.collection_name}Revocation is pending. Waiting for {grant.collection_kind === "local" ? "the computer holding this collection" : "the hosted authority"} to confirm enforcement.Revoking…
; } if (grant.scope.access !== "full_collection" || grant.scope.contracts.length > 0) { return
{grant.collection_name}Legacy scoped access is revoked. Reauthorize this application for the entire collection.Reauthorization required
; diff --git a/crates/connect-agent/src/runtime_notifications/tests.rs b/crates/connect-agent/src/runtime_notifications/tests.rs index 9d404d235..fa48c21d5 100644 --- a/crates/connect-agent/src/runtime_notifications/tests.rs +++ b/crates/connect-agent/src/runtime_notifications/tests.rs @@ -115,6 +115,7 @@ async fn recovery_keeps_idle_registered_collections_cold() { #[test] fn compiled_workflows_keep_record_data_out_of_action_input() { let grant = GrantSummary { + revocation_status: None, application_declaration: None, contracts: mdbase_connect_protocol::ConnectContractRequirements::current(true), id: Uuid::new_v4(), diff --git a/crates/connect-agent/src/server/files_scope_tests.rs b/crates/connect-agent/src/server/files_scope_tests.rs index da50b3611..3bebc2955 100644 --- a/crates/connect-agent/src/server/files_scope_tests.rs +++ b/crates/connect-agent/src/server/files_scope_tests.rs @@ -9,6 +9,7 @@ use uuid::Uuid; fn file_grant(collection_id: Uuid, actions: Vec, scope: FileScope) -> GrantSummary { GrantSummary { + revocation_status: None, application_declaration: None, contracts: mdbase_connect_protocol::ConnectContractRequirements::current(true), id: Uuid::now_v7(), diff --git a/crates/connect-agent/src/server/files_tests.rs b/crates/connect-agent/src/server/files_tests.rs index 3c331eeee..27631e60c 100644 --- a/crates/connect-agent/src/server/files_tests.rs +++ b/crates/connect-agent/src/server/files_tests.rs @@ -117,6 +117,7 @@ fn local_list_pages_share_one_index_revision_and_expire_after_refresh() { fn file_grant(collection_id: Uuid, actions: Vec) -> GrantSummary { GrantSummary { + revocation_status: None, application_declaration: None, contracts: mdbase_connect_protocol::ConnectContractRequirements::current(true), id: Uuid::now_v7(), diff --git a/crates/connect-agent/src/server/setup_binding.rs b/crates/connect-agent/src/server/setup_binding.rs index 421c6e040..b57eebad1 100644 --- a/crates/connect-agent/src/server/setup_binding.rs +++ b/crates/connect-agent/src/server/setup_binding.rs @@ -467,6 +467,7 @@ mod tests { fn legacy_grant() -> GrantSummary { GrantSummary { + revocation_status: None, application_declaration: None, id: Uuid::new_v4(), application_id: Uuid::new_v4(), diff --git a/crates/connect-core/src/registry/grants.rs b/crates/connect-core/src/registry/grants.rs index dc9620ebf..c6cc8897a 100644 --- a/crates/connect-core/src/registry/grants.rs +++ b/crates/connect-core/src/registry/grants.rs @@ -506,6 +506,7 @@ impl CollectionRegistry { let application_declaration = authenticated_summary_declaration(&proof, application_declaration.as_deref())?; Ok(GrantSummary { + revocation_status: None, application_declaration, id: parse_registry_uuid(&id)?, application_id: parse_registry_uuid(&application_id)?, @@ -668,6 +669,7 @@ impl CollectionRegistry { .collect(); Ok(Some(GrantReplayContext { grant: GrantSummary { + revocation_status: None, application_declaration: authenticated_summary_declaration( &proof, application_declaration.as_deref(), diff --git a/crates/connect-protocol/src/applications.rs b/crates/connect-protocol/src/applications.rs index 0afcef277..a83f89d8d 100644 --- a/crates/connect-protocol/src/applications.rs +++ b/crates/connect-protocol/src/applications.rs @@ -383,6 +383,9 @@ impl GrantScope { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GrantSummary { + /// Presentation only; never installs or restores authorization. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub revocation_status: Option, /// Complete normalized declaration evidence authenticated before presentation. #[serde(default, skip_serializing_if = "Option::is_none")] pub application_declaration: Option, diff --git a/crates/connect-runtime/src/lib.rs b/crates/connect-runtime/src/lib.rs index f24084ed9..9a622ca4d 100644 --- a/crates/connect-runtime/src/lib.rs +++ b/crates/connect-runtime/src/lib.rs @@ -672,6 +672,7 @@ mod tests { fn grant(criterion_id: &str, event_id: &str, condition: Option<&str>) -> GrantSummary { GrantSummary { + revocation_status: None, application_declaration: None, contracts: mdbase_connect_protocol::ConnectContractRequirements::current(true), id: Uuid::new_v4(), diff --git a/crates/connect-runtime/src/timers.rs b/crates/connect-runtime/src/timers.rs index 647c26818..55c0b7037 100644 --- a/crates/connect-runtime/src/timers.rs +++ b/crates/connect-runtime/src/timers.rs @@ -900,6 +900,7 @@ mod tests { fn grant(criterion_id: &str) -> GrantSummary { GrantSummary { + revocation_status: None, application_declaration: None, contracts: mdbase_connect_protocol::ConnectContractRequirements::current(true), id: Uuid::new_v4(), diff --git a/crates/connect-testbed-adapter/src/main.rs b/crates/connect-testbed-adapter/src/main.rs index 5874dc6b0..b993f955a 100644 --- a/crates/connect-testbed-adapter/src/main.rs +++ b/crates/connect-testbed-adapter/src/main.rs @@ -272,6 +272,7 @@ impl DispatchAuthorizer for CountingAuthorizer { fn test_grant(collection_id: Uuid) -> GrantSummary { GrantSummary { + revocation_status: None, application_declaration: None, contracts: mdbase_connect_protocol::ConnectContractRequirements::current(true), id: Uuid::parse_str("0d57894d-9a5a-477a-95ac-a8b4d77839d9").expect("fixed UUID"), diff --git a/services/server/migrations/0031_local_revocation_confirmation.sql b/services/server/migrations/0031_local_revocation_confirmation.sql new file mode 100644 index 000000000..56c7f824b --- /dev/null +++ b/services/server/migrations/0031_local_revocation_confirmation.sql @@ -0,0 +1,5 @@ +-- Reuse the connector's monotonic policy sequence, not a second delivery journal. +-- Historical/unbound revocations remain pending until snapshot construction binds +-- them under the connector lock and the enforcing connector acknowledges it. +ALTER TABLE grants ADD COLUMN revocation_policy_sequence bigint; +ALTER TABLE grants ADD COLUMN revocation_confirmed_at timestamptz; diff --git a/services/server/src/db.test.ts b/services/server/src/db.test.ts index a36fdec1f..9e0ff76ad 100644 --- a/services/server/src/db.test.ts +++ b/services/server/src/db.test.ts @@ -69,7 +69,8 @@ describe("database migrations", () => { "0027_connector_policy_lease_adoption", "0028_application_declaration", "0029_external_signup", - "0030_sharing_cleanup_seat_reservations" + "0030_sharing_cleanup_seat_reservations", + "0031_local_revocation_confirmation" ]); const columns = await db.query<{ column_name: string }>( `SELECT column_name FROM information_schema.columns @@ -661,7 +662,8 @@ describe("database migrations", () => { "0027_connector_policy_lease_adoption", "0028_application_declaration", "0029_external_signup", - "0030_sharing_cleanup_seat_reservations" + "0030_sharing_cleanup_seat_reservations", + "0031_local_revocation_confirmation" ]); }); diff --git a/services/server/src/features/account/me-routes.ts b/services/server/src/features/account/me-routes.ts index 3c413af90..009fec56c 100644 --- a/services/server/src/features/account/me-routes.ts +++ b/services/server/src/features/account/me-routes.ts @@ -176,6 +176,8 @@ export function registerAccountOverviewRoute( g.reauthorization_required_at, g.reauthorization_reason, CASE WHEN g.revoked_at IS NULL THEN 'active' + WHEN g.hosted_replica_id IS NULL AND g.hosted_collection_id IS NULL + AND g.revocation_confirmed_at IS NULL THEN 'revoking' WHEN g.id IN ( SELECT job.grant_id FROM provider_revocation_jobs job WHERE job.grant_id IS NOT NULL AND job.completed_at IS NULL diff --git a/services/server/src/features/authorizations/grant-revocation-route.ts b/services/server/src/features/authorizations/grant-revocation-route.ts index 0f5d42660..2964c1289 100644 --- a/services/server/src/features/authorizations/grant-revocation-route.ts +++ b/services/server/src/features/authorizations/grant-revocation-route.ts @@ -10,6 +10,7 @@ import { audit } from "../../platform/audit-events.js"; import { apiError } from "../../platform/http-errors.js"; import { requireUser } from "../../platform/request-authentication.js"; import type { RelayHub } from "../../relay.js"; +import { localGrantRevocationStatus, queueLocalGrantRevocations } from "../../local-grant-revocation.js"; interface GrantRevocationRouteOptions { db: DatabasePool; @@ -51,34 +52,8 @@ export function registerGrantRevocationRoute( )); } - const localIds = found.rows - .filter((grant) => !grant.hosted_replica_id && !grant.revoked_at) - .map((grant) => grant.id); - if (localIds.length > 0) { - const localParameters = localIds.map((_id, index) => `$${index + 1}`).join(", "); - const connection = await options.db.connect(); - try { - await connection.query("BEGIN"); - await connection.query( - `UPDATE grants SET revoked_at = COALESCE(revoked_at, now()) WHERE id IN (${localParameters})`, - localIds - ); - await connection.query( - `UPDATE access_tokens SET revoked_at = COALESCE(revoked_at, now()) WHERE grant_id IN (${localParameters})`, - localIds - ); - await connection.query( - `UPDATE refresh_tokens SET revoked_at = COALESCE(revoked_at, now()) WHERE grant_id IN (${localParameters})`, - localIds - ); - await connection.query("COMMIT"); - } catch (error) { - await connection.query("ROLLBACK"); - throw error; - } finally { - connection.release(); - } - } + const localIds = found.rows.filter((grant) => !grant.hosted_replica_id).map((grant) => grant.id); + await queueLocalGrantRevocations(options.db, user.id, localIds); const results: Array<{ grant_id: string; @@ -86,7 +61,7 @@ export function registerGrantRevocationRoute( }> = []; for (const grant of found.rows) { if (!grant.hosted_replica_id) { - results.push({ grant_id: grant.id, status: "revoked" }); + results.push({ grant_id: grant.id, status: await localGrantRevocationStatus(options.db, user.id, grant.id) }); continue; } if (!grant.revoked_at) { @@ -109,7 +84,7 @@ export function registerGrantRevocationRoute( } await options.drainProviderRevocations(); for (const result of results) { - if (result.status !== "revoking") continue; + if (result.status !== "revoking" || localIds.includes(result.grant_id)) continue; const status = await hostedGrantRevocationStatus(options.db, user.id, result.grant_id); if (status === "revoked") result.status = "revoked"; } @@ -117,6 +92,9 @@ export function registerGrantRevocationRoute( await options.relay.pushPolicy(connectorId!); } for (const result of results) { + if (localIds.includes(result.grant_id)) { + result.status = await localGrantRevocationStatus(options.db, user.id, result.grant_id); + } await audit(options.db, user.id, result.status === "revoked" ? "grant.revoked" : "grant.revocation_requested", result.grant_id, { @@ -182,23 +160,10 @@ export function registerGrantRevocationRoute( } revocationStatus = current; } else { - if (grant.revoked_at) { - return { ok: true, revocation_status: "revoked" as const }; - } - await options.db.query( - "UPDATE grants SET revoked_at = now() WHERE id = $1", - [grantId] - ); - await options.db.query( - "UPDATE access_tokens SET revoked_at = now() WHERE grant_id = $1", - [grantId] - ); - await options.db.query( - "UPDATE refresh_tokens SET revoked_at = now() WHERE grant_id = $1", - [grantId] - ); + await queueLocalGrantRevocations(options.db, user.id, [grantId]); } if (grant.connector_id) await options.relay.pushPolicy(grant.connector_id); + if (!grant.hosted_replica_id) revocationStatus = await localGrantRevocationStatus(options.db, user.id, grantId); await audit( options.db, user.id, diff --git a/services/server/src/features/connectors/control-routes.ts b/services/server/src/features/connectors/control-routes.ts index 1392cf532..fbd93078b 100644 --- a/services/server/src/features/connectors/control-routes.ts +++ b/services/server/src/features/connectors/control-routes.ts @@ -47,6 +47,7 @@ export function registerConnectorControlRoutes( col.local_id AS collection_id, col.display_name AS collection_name, g.operations, g.scope, g.encryption, g.file_capability, g.created_at, + CASE WHEN g.revoked_at IS NULL THEN 'active' ELSE 'revoking' END AS revocation_status, g.notification_criteria, g.application_authorization->'binding'->>'application_declaration_id' AS application_declaration_id, @@ -56,7 +57,8 @@ export function registerConnectorControlRoutes( FROM grants g JOIN applications a ON a.id = g.application_id JOIN collections col ON col.id = g.collection_id - WHERE col.connector_id = $1 AND g.revoked_at IS NULL + WHERE col.connector_id = $1 + AND (g.revoked_at IS NULL OR g.revocation_confirmed_at IS NULL) AND g.activated_at IS NOT NULL ORDER BY a.name, col.display_name`, [connector.id] diff --git a/services/server/src/features/grants/connector-routes.ts b/services/server/src/features/grants/connector-routes.ts index ec258b3c8..1d76441d1 100644 --- a/services/server/src/features/grants/connector-routes.ts +++ b/services/server/src/features/grants/connector-routes.ts @@ -11,6 +11,7 @@ import type { DatabasePool } from "../../db.js"; import { fileCapabilityForRequirements } from "../../grant-planner.js"; import { collectionContractDescriptorSchema } from "../../protocol-schemas.js"; import type { RelayHub } from "../../relay.js"; +import { localGrantRevocationStatus, queueLocalGrantRevocations } from "../../local-grant-revocation.js"; import { audit } from "../../platform/audit-events.js"; import { apiError } from "../../platform/http-errors.js"; import { requireConnector } from "../../platform/request-authentication.js"; @@ -184,11 +185,10 @@ export function registerConnectorGrantRoutes( if (!connector) return; const { grantId } = z.object({ grantId: z.uuid() }).parse(request.params); const active = await options.db.query( - `UPDATE grants SET revoked_at = now() - WHERE id = $1 AND revoked_at IS NULL AND activated_at IS NOT NULL + `SELECT id FROM grants + WHERE id = $1 AND activated_at IS NOT NULL AND collection_id IN - (SELECT id FROM collections WHERE connector_id = $2) - RETURNING id`, + (SELECT id FROM collections WHERE connector_id = $2)`, [grantId, connector.id] ); if (!active.rows[0]) { @@ -197,18 +197,12 @@ export function registerConnectorGrantRoutes( "Active grant not found." )); } - await options.db.query( - "UPDATE access_tokens SET revoked_at = now() WHERE grant_id = $1", - [grantId] - ); - await options.db.query( - "UPDATE refresh_tokens SET revoked_at = now() WHERE grant_id = $1", - [grantId] - ); + await queueLocalGrantRevocations(options.db, connector.user_id, [grantId]); await options.relay.pushPolicy(connector.id); - await audit(options.db, connector.user_id, "grant.revoked", grantId, { - connector_id: connector.id + const status = await localGrantRevocationStatus(options.db, connector.user_id, grantId); + await audit(options.db, connector.user_id, status === "revoked" ? "grant.revoked" : "grant.revocation_requested", grantId, { + connector_id: connector.id, revocation_status: status }); - return { ok: true }; + return { ok: true, revocation_status: status }; }); } diff --git a/services/server/src/local-grant-revocation.test.ts b/services/server/src/local-grant-revocation.test.ts new file mode 100644 index 000000000..5e4c4d5ce --- /dev/null +++ b/services/server/src/local-grant-revocation.test.ts @@ -0,0 +1,188 @@ +import { randomUUID } from "node:crypto"; +import Fastify from "fastify"; +import pg from "pg"; +import { afterEach, describe, expect, it } from "vitest"; +import { createDatabase, type DatabasePool } from "./db.js"; +import { buildPolicySnapshot } from "./relay-policy.js"; +import { ExactPolicyPublisher } from "./relay-policy-session.js"; +import { RelayHub } from "./relay.js"; +import { registerGrantRevocationRoute } from "./features/authorizations/grant-revocation-route.js"; +import { confirmLocalGrantRevocations, localGrantRevocationStatus, queueLocalGrantRevocations } from "./local-grant-revocation.js"; + +const databases: DatabasePool[] = []; +const schemaCleanups: Array<() => Promise> = []; +afterEach(async () => { + await Promise.all(databases.splice(0).map((db) => db.end())); + await Promise.all(schemaCleanups.splice(0).map((cleanup) => cleanup())); +}); + +export async function revocationFixture(db: DatabasePool) { + const id = randomUUID(); + await db.query("INSERT INTO users(id,email,name) VALUES($1,$2,'Recovery test')", [id, `${id}@example.test`]); + await db.query("INSERT INTO connectors(id,user_id,name,token_hash,relay_generation) VALUES($1,$1,'Test connector',$2,1)", [id, id]); + await db.query("INSERT INTO collections(id,user_id,connector_id,local_id,display_name,spec_version) VALUES($1,$1,$1,$1,'Test collection','0.3.0')", [id]); + await db.query("INSERT INTO applications(id,canonical_identity,name,homepage,redirect_uris) VALUES($1,$2,'Test app','https://example.test','[]')", [id, id]); + await db.query(`INSERT INTO grants(id,user_id,application_id,collection_id,operations,scope,application_installation_id,application_authorization) + VALUES($1,$1,$1,$1,'["read"]','{"access":"full_collection","contracts":[]}','test-installation', + '{"binding":{"protocol_version":4,"contracts":{"semantic_capabilities":1}}}')`, [id]); + for (const table of ["access_tokens", "refresh_tokens"]) { + await db.query(`INSERT INTO ${table}(id,token_hash,grant_id,expires_at) VALUES($1,$2,$1,now() + interval '1 day')`, [id, id]); + } + return id; +} + +async function fixture() { + let url = process.env.MDBASE_CONNECT_TEST_DATABASE_URL; + if (url) { + const parsed = new URL(url); + if (process.env.MDBASE_CONNECT_DESTRUCTIVE_TEST_APPROVAL !== "I APPROVE MDBASE CONNECT DESTRUCTIVE POSTGRES TESTS" + || !["localhost", "127.0.0.1", "::1"].includes(parsed.hostname) || !/test/i.test(parsed.pathname)) { + throw new Error("A dedicated approved local test database is required"); + } + const admin = new pg.Pool({ connectionString: url }); + const schema = `revocation_test_${randomUUID().replaceAll("-", "")}`; + await admin.query(`CREATE SCHEMA "${schema}"`); + schemaCleanups.push(async () => { await admin.query(`DROP SCHEMA "${schema}" CASCADE`); await admin.end(); }); + parsed.searchParams.set("options", `-csearch_path=${schema}`); + url = parsed.toString(); + } + const db = await createDatabase(url ?? "memory"); + databases.push(db); + return { db, id: await revocationFixture(db) }; +} + +export async function exerciseRevocationBarrier(db: DatabasePool, id: string) { + const old = await buildPolicySnapshot(db, id, 55_000, "1"); + if (!old || !("sequence" in old)) throw new Error("Expected lease snapshot"); + expect(old.grants).toHaveLength(1); + await queueLocalGrantRevocations(db, id, [id]); + expect(await localGrantRevocationStatus(db, id, id)).toBe("revoking"); + for (const table of ["access_tokens", "refresh_tokens"]) { + expect((await db.query(`SELECT revoked_at FROM ${table} WHERE grant_id=$1`, [id])).rows[0].revoked_at).not.toBeNull(); + } + const barrier = (await db.query("SELECT revocation_policy_sequence FROM grants WHERE id=$1", [id])).rows[0].revocation_policy_sequence; + await queueLocalGrantRevocations(db, id, [id]); + expect((await db.query("SELECT revocation_policy_sequence FROM grants WHERE id=$1", [id])).rows[0].revocation_policy_sequence).toBe(barrier); + await confirmLocalGrantRevocations(db, id, "1", old.sequence); + expect(await localGrantRevocationStatus(db, id, id)).toBe("revoking"); + const newer = await buildPolicySnapshot(db, id, 55_000, "1"); + if (!newer || !("sequence" in newer)) throw new Error("Expected lease snapshot"); + expect(newer.grants).toHaveLength(0); + await confirmLocalGrantRevocations(db, randomUUID(), "1", newer.sequence); + await confirmLocalGrantRevocations(db, id, "0", newer.sequence); + expect(await localGrantRevocationStatus(db, id, id)).toBe("revoking"); + await confirmLocalGrantRevocations(db, id, "1", newer.sequence); + expect(await localGrantRevocationStatus(db, id, id)).toBe("revoked"); +} + +describe("truthful local revocation", () => { + it.skipIf(!process.env.MDBASE_CONNECT_TEST_DATABASE_URL)("PostgreSQL serializes revocation behind an in-flight snapshot cut", async () => { + const { db, id } = await fixture(); + const snapshotLocked = Promise.withResolvers(); + const releaseSnapshot = Promise.withResolvers(); + const revocationIssued = Promise.withResolvers(); + let firstConnection = true; + const wrapped: DatabasePool = { + query: db.query.bind(db), end: async () => {}, + async connect() { + const connection = await db.connect(); + const snapshot = firstConnection; + firstConnection = false; + return { + release: () => connection.release(), + async query(text, parameters) { + if (!snapshot && text.startsWith("UPDATE connectors SET policy_sequence")) { + const result = await connection.query("SELECT pg_backend_pid() AS pid"); + revocationIssued.resolve(result.rows[0].pid); + } + const result = await connection.query(text, parameters); + if (snapshot && text.includes("RETURNING policy_sequence, now()")) { + snapshotLocked.resolve(); + await releaseSnapshot.promise; + } + return result; + } + }; + } + }; + const snapshot = buildPolicySnapshot(wrapped, id, 55_000, "1"); + await snapshotLocked.promise; + const revocation = queueLocalGrantRevocations(wrapped, id, [id]); + try { + const pid = await revocationIssued.promise; + const deadline = Date.now() + 4_000; + let blocked = false; + while (Date.now() < deadline) { + blocked = (await db.query("SELECT cardinality(pg_blocking_pids($1)) > 0 AS blocked", [pid])).rows[0].blocked; + if (blocked) break; + await new Promise((resolve) => setImmediate(resolve)); + } + expect(blocked).toBe(true); + releaseSnapshot.resolve(); + const old = await snapshot; + await revocation; + expect(old?.grants).toHaveLength(1); + await confirmLocalGrantRevocations(db, id, "1", (old as { sequence: number }).sequence); + expect(await localGrantRevocationStatus(db, id, id)).toBe("revoking"); + const fresh = await buildPolicySnapshot(db, id, 55_000, "1"); + expect(fresh?.grants).toHaveLength(0); + await confirmLocalGrantRevocations(db, id, "1", (fresh as { sequence: number }).sequence); + expect(await localGrantRevocationStatus(db, id, id)).toBe("revoked"); + } finally { + releaseSnapshot.resolve(); + await Promise.allSettled([snapshot, revocation]); + } + }); + it("requires a post-revocation policy barrier from the exact connector generation", async () => { + const { db, id } = await fixture(); + await exerciseRevocationBarrier(db, id); + }); + + it.each(["single", "batch"])("%s offline API retains Revoking across repeated requests", async (mode) => { + const { db, id } = await fixture(); + const relay = new RelayHub(db); + const app = Fastify(); + registerGrantRevocationRoute(app, { db, relay, tailscaleAuth: true, drainProviderRevocations: async () => {} }); + try { + for (let attempt = 0; attempt < 2; attempt++) { + const response = await app.inject({ + method: mode === "single" ? "DELETE" : "POST", + url: mode === "single" ? `/v1/grants/${id}` : "/v1/grants/revoke-batch", + headers: { "tailscale-user-login": `${id}@example.test` }, + ...(mode === "single" ? {} : { payload: { grant_ids: [id] } }) + }); + expect(response.statusCode).toBe(200); + expect(response.json()).toMatchObject(mode === "single" + ? { revocation_status: "revoking" } : { results: [{ grant_id: id, status: "revoking" }] }); + } + } finally { await app.close(); await relay.close(); } + }); + + it("publisher never confirms mismatched or legacy acknowledgements", async () => { + const { db, id } = await fixture(); + await queueLocalGrantRevocations(db, id, [id]); + const authority = { connectorId: id, generation: "1", isStillCurrent: () => true }; + const publisher = new ExactPolicyPublisher(db, 55_000, async () => "1", () => true); + await expect(publisher.push(authority, async (message) => ({ + type: "policy_applied", protocol_version: 1, request_id: message.request_id, revision: "wrong", ok: true + }))).rejects.toThrow(); + expect(await localGrantRevocationStatus(db, id, id)).toBe("revoking"); + const legacy = new ExactPolicyPublisher(db, 55_000, async () => "1", () => true, "legacy_ack_v0"); + const ack = async (message: { request_id: string; revision: string }) => ({ type: "policy_applied", protocol_version: 1, request_id: message.request_id, revision: message.revision, ok: true }); + await legacy.push(authority, ack); + expect(await localGrantRevocationStatus(db, id, id)).toBe("revoking"); + await publisher.push(authority, ack); + expect(await localGrantRevocationStatus(db, id, id)).toBe("revoked"); + }); + + it("binds historical revocation to the first new snapshot rather than an older ack", async () => { + const { db, id } = await fixture(); + const old = await buildPolicySnapshot(db, id, 55_000, "1"); + await db.query("UPDATE grants SET revoked_at=now() WHERE id=$1", [id]); + await confirmLocalGrantRevocations(db, id, "1", (old as { sequence: number }).sequence); + expect(await localGrantRevocationStatus(db, id, id)).toBe("revoking"); + const fresh = await buildPolicySnapshot(db, id, 55_000, "1"); + await confirmLocalGrantRevocations(db, id, "1", (fresh as { sequence: number }).sequence); + expect(await localGrantRevocationStatus(db, id, id)).toBe("revoked"); + }); +}); diff --git a/services/server/src/local-grant-revocation.ts b/services/server/src/local-grant-revocation.ts new file mode 100644 index 000000000..87dd5bcde --- /dev/null +++ b/services/server/src/local-grant-revocation.ts @@ -0,0 +1,74 @@ +import type { DatabasePool } from "./db.js"; + +/** Lock the same connector row as snapshot construction before narrowing grants. + * Every policy at/above this barrier is therefore built after the revocation. + * Repeating a request never replaces its original barrier or restores credentials. + */ +export async function queueLocalGrantRevocations(db: DatabasePool, userId: string, grantIds: string[]): Promise { + if (!grantIds.length) return; + const ids = grantIds.map((_, index) => `$${index + 2}`).join(", "); + const parameters = [userId, ...grantIds]; + const connection = await db.connect(); + try { + await connection.query("BEGIN"); + const connectors = await connection.query<{ connector_id: string }>( + `SELECT DISTINCT col.connector_id FROM grants g + JOIN collections col ON col.id = g.collection_id + WHERE g.user_id = $1 AND g.id IN (${ids}) AND g.hosted_replica_id IS NULL + AND col.connector_id IS NOT NULL + ORDER BY col.connector_id`, parameters + ); + for (const { connector_id } of connectors.rows) { + const sequence = await connection.query<{ policy_sequence: string | number }>( + `UPDATE connectors SET policy_sequence = policy_sequence + 1 + WHERE id = $1 AND policy_sequence < $2::bigint RETURNING policy_sequence`, + [connector_id, Number.MAX_SAFE_INTEGER.toString()] + ); + if (!sequence.rows[0]) throw new Error("The connector policy sequence cannot advance; revocation needs attention."); + await connection.query( + `UPDATE grants SET revocation_policy_sequence = COALESCE(revocation_policy_sequence, $${parameters.length + 1}::bigint) + WHERE user_id = $1 AND id IN (${ids}) AND hosted_replica_id IS NULL + AND collection_id IN (SELECT id FROM collections WHERE connector_id = $${parameters.length + 2})`, + [...parameters, sequence.rows[0].policy_sequence, connector_id] + ); + } + await connection.query( + `UPDATE grants SET revoked_at = COALESCE(revoked_at, now()) + WHERE user_id = $1 AND id IN (${ids}) AND hosted_replica_id IS NULL`, parameters + ); + for (const table of ["access_tokens", "refresh_tokens"] as const) { + await connection.query( + `UPDATE ${table} SET revoked_at = COALESCE(revoked_at, now()) + WHERE grant_id IN (SELECT id FROM grants WHERE user_id = $1 AND id IN (${ids}) AND hosted_replica_id IS NULL)`, + parameters + ); + } + await connection.query("COMMIT"); + } catch (error) { + await connection.query("ROLLBACK"); + throw error; + } finally { connection.release(); } +} + +/** Called only after the exact snapshot acknowledgement and current-generation checks. */ +export async function confirmLocalGrantRevocations(db: DatabasePool, connectorId: string, generation: string, sequence: number): Promise { + if (!Number.isSafeInteger(sequence) || sequence < 1) throw new Error("Invalid policy acknowledgement sequence."); + await db.query( + `UPDATE grants SET revocation_confirmed_at = COALESCE(revocation_confirmed_at, now()) + WHERE revoked_at IS NOT NULL AND hosted_replica_id IS NULL + AND revocation_policy_sequence IS NOT NULL AND revocation_policy_sequence <= $3::bigint + AND collection_id IN ( + SELECT col.id FROM collections col JOIN connectors c ON c.id = col.connector_id + WHERE c.id = $1 AND c.relay_generation = $2::bigint AND c.revoked_at IS NULL + )`, [connectorId, generation, sequence] + ); +} + +export async function localGrantRevocationStatus(db: DatabasePool, userId: string, grantId: string): Promise<"revoking" | "revoked"> { + const result = await db.query<{ revocation_confirmed_at: string | null }>( + `SELECT revocation_confirmed_at FROM grants + WHERE id = $1 AND user_id = $2 AND revoked_at IS NOT NULL AND hosted_replica_id IS NULL`, + [grantId, userId] + ); + return result.rows[0]?.revocation_confirmed_at ? "revoked" : "revoking"; +} diff --git a/services/server/src/relay-policy-session.ts b/services/server/src/relay-policy-session.ts index 4a139850e..ffc18a9bb 100644 --- a/services/server/src/relay-policy-session.ts +++ b/services/server/src/relay-policy-session.ts @@ -1,5 +1,6 @@ import type { WebSocket } from "ws"; import type { DatabasePool } from "./db.js"; +import { confirmLocalGrantRevocations } from "./local-grant-revocation.js"; import { RelayBrokerUnavailableError, type RelayBroker, @@ -246,7 +247,11 @@ export class ExactPolicyPublisher { )) { throw new StalePolicyAuthorityError(); } - return exactPolicyAcknowledgement(settled, message); + const acknowledgement = exactPolicyAcknowledgement(settled, message); + if ("sequence" in message) { + await confirmLocalGrantRevocations(this.db, authority.connectorId, authority.generation, message.sequence); + } + return acknowledgement; } private async isCurrent(authority: ExactPolicyAuthority): Promise { diff --git a/services/server/src/relay-policy.ts b/services/server/src/relay-policy.ts index af680dbe3..284e6fc7f 100644 --- a/services/server/src/relay-policy.ts +++ b/services/server/src/relay-policy.ts @@ -285,6 +285,15 @@ export async function buildPolicySnapshot( } return null; } + // Bind historical or externally requested revocations to this exact snapshot + // while holding the same connector lock used by explicit revocation routes. + await connection.query( + `UPDATE grants SET revocation_policy_sequence = $2::bigint + WHERE revoked_at IS NOT NULL AND hosted_replica_id IS NULL + AND revocation_policy_sequence IS NULL + AND collection_id IN (SELECT id FROM collections WHERE connector_id = $1)`, + [connectorId, String(active.rows[0].policy_sequence)] + ); const grants = await observeConnectorPolicyStage("grant_inventory", () => connection.query<{ id: string; application_id: string; application_name: string; application_distribution: "web" | "portable"; application_homepage: string; From 0e18d73c60f73278e6146f33205273b558b7a877 Mon Sep 17 00:00:00 2001 From: callumalpass Date: Sun, 13 Sep 2026 19:42:43 +1000 Subject: [PATCH 03/16] fix(daemon): expose canonical readiness and fail closed on bootstrap failure Check initialization and critical worker liveness, require readiness in CLI startup and doctor, and propagate credential failures instead of reporting an unconfigured account. Keep relay attention separate from local readiness. --- crates/connect-agent/src/lib.rs | 13 +++- crates/connect-agent/src/server.rs | 80 ++++++++++++++++++---- crates/connect-agent/src/server/account.rs | 3 + crates/connect-agent/src/server/control.rs | 18 +++-- crates/connect-agent/src/server/policy.rs | 8 +++ crates/connect-agent/src/server/tests.rs | 52 ++++++++++++++ crates/connect-agent/src/watcher.rs | 9 +++ crates/connect-cli/src/daemon.rs | 67 ++++++++++++++---- crates/connect-cli/tests/unified_cli.rs | 58 ++++++++++++++++ crates/connect-protocol/src/control.rs | 23 +++++++ 10 files changed, 301 insertions(+), 30 deletions(-) diff --git a/crates/connect-agent/src/lib.rs b/crates/connect-agent/src/lib.rs index f2a54bf13..ac5edc995 100644 --- a/crates/connect-agent/src/lib.rs +++ b/crates/connect-agent/src/lib.rs @@ -160,6 +160,8 @@ pub async fn run(options: DaemonOptions) -> Result<(), Box Some((server_url, connector_token)), (None, None) => None, @@ -167,6 +169,8 @@ pub async fn run(options: DaemonOptions) -> Result<(), Box Result<(), Box tracing::error!(%error, "failed to initialize collection runtimes"), + Err(error) => { + tracing::error!(%error, "failed to initialize collection runtimes"); + initialization_state.mark_initialization_failed(); + return; + } } initialization_state.mark_initialized(); if let Some((server_url, connector_token)) = relay { relay::run(server_url, connector_token, relay_state).await; } }); + if has_relay { + worker_health_state.monitor_critical_worker(worker.abort_handle()); + } *initialization_worker_on_listening .lock() .expect("initialization worker lock poisoned") = Some(worker); diff --git a/crates/connect-agent/src/server.rs b/crates/connect-agent/src/server.rs index f8496f30e..92aa4364c 100644 --- a/crates/connect-agent/src/server.rs +++ b/crates/connect-agent/src/server.rs @@ -15,11 +15,11 @@ use mdbase_connect_protocol::crypto::{ }; use mdbase_connect_protocol::{ mutation_fingerprint, mutation_operation_identifier, operation_input_schema_version, - validate_operation_discriminators, AgentConnectionState, AgentStatus, ApplicationAccess, - AuthorityTarget, AuthorizationCollectionOffer, AuthorizationCollectionTypes, + validate_operation_discriminators, AgentConnectionState, AgentReadiness, AgentStatus, + ApplicationAccess, AuthorityTarget, AuthorizationCollectionOffer, AuthorizationCollectionTypes, ConnectOperationOutcome, ConnectProblem, ContractSetupChoice, ControlCommand, ControlError, - ControlRequest, ControlResponse, RelayMessage, SyncReplicaMode, CONTROL_PROTOCOL_VERSION, - LOCAL_CONTROL_PROTOCOL_VERSION, + ControlRequest, ControlResponse, ReadinessReason, RelayMessage, SyncReplicaMode, + CONTROL_PROTOCOL_VERSION, LOCAL_CONTROL_PROTOCOL_VERSION, }; use std::io; use std::sync::atomic::{AtomicU8, Ordering}; @@ -71,11 +71,20 @@ impl OperationExecutionState { } } +#[repr(u8)] +enum Initialization { + Starting, + Ready, + Failed, +} + pub struct AgentState { registry: CollectionRegistry, watcher: CollectionWatchService, connection_state: std::sync::RwLock, - initialized: std::sync::atomic::AtomicBool, + relay_problem: std::sync::RwLock>, + initialization: AtomicU8, + critical_workers: std::sync::Mutex>, loopback_port: std::sync::atomic::AtomicU16, cloud: Option, credential_store_error: Option, @@ -134,7 +143,9 @@ impl AgentState { registry, watcher, connection_state: std::sync::RwLock::new(AgentConnectionState::LocalOnly), - initialized: std::sync::atomic::AtomicBool::new(false), + relay_problem: std::sync::RwLock::new(None), + initialization: AtomicU8::new(Initialization::Starting as u8), + critical_workers: std::sync::Mutex::new(Vec::new()), loopback_port: std::sync::atomic::AtomicU16::new(0), cloud, credential_store_error: None, @@ -228,8 +239,50 @@ impl AgentState { } pub fn mark_initialized(&self) { - self.initialized - .store(true, std::sync::atomic::Ordering::Release); + self.initialization + .store(Initialization::Ready as u8, Ordering::Release); + } + + pub fn mark_initialization_failed(&self) { + self.initialization + .store(Initialization::Failed as u8, Ordering::Release); + } + + pub fn monitor_critical_worker(&self, worker: tokio::task::AbortHandle) { + self.critical_workers + .lock() + .expect("worker health lock poisoned") + .push(worker); + } + + fn critical_workers_alive(&self) -> bool { + self.watcher.is_alive() + && self + .critical_workers + .lock() + .expect("worker health lock poisoned") + .iter() + .all(|worker| !worker.is_finished()) + } + + fn readiness(&self) -> AgentReadiness { + let safe_reason = if !self.critical_workers_alive() { + Some(ReadinessReason::CriticalWorkerFailed) + } else if self.initialization.load(Ordering::Acquire) == Initialization::Failed as u8 { + Some(ReadinessReason::InitializationFailed) + } else if self.credential_store_error.is_some() { + Some(ReadinessReason::CredentialStoreUnavailable) + } else if self.initialization.load(Ordering::Acquire) != Initialization::Ready as u8 { + Some(ReadinessReason::Starting) + } else { + None + }; + AgentReadiness { + schema_version: 1, + ready: safe_reason.is_none(), + binary_version: env!("CARGO_PKG_VERSION").to_string(), + safe_reason, + } } pub fn set_loopback_port(&self, port: u16) { @@ -292,10 +345,6 @@ impl AgentState { self.shutdown.notified().await; } - fn initialized(&self) -> bool { - self.initialized.load(std::sync::atomic::Ordering::Acquire) - } - fn refresh_watchers(&self) { match self.registry.list() { Ok(collections) => self.watcher.refresh(&collections), @@ -303,6 +352,13 @@ impl AgentState { } } + pub fn set_relay_problem(&self, reason: Option<&'static str>) { + *self + .relay_problem + .write() + .expect("relay problem lock poisoned") = reason; + } + pub fn set_connection_state(&self, state: AgentConnectionState) { *self .connection_state diff --git a/crates/connect-agent/src/server/account.rs b/crates/connect-agent/src/server/account.rs index ff2ac62f2..ed4ea1c46 100644 --- a/crates/connect-agent/src/server/account.rs +++ b/crates/connect-agent/src/server/account.rs @@ -342,6 +342,9 @@ impl AgentState { } pub(super) async fn access_snapshot(&self) -> Result { + if let Some(error) = self.credential_store_unavailable() { + return Err(error); + } let Some(cloud) = &self.cloud else { return serde_json::to_value(mdbase_connect_protocol::AccessSnapshot { configured: false, diff --git a/crates/connect-agent/src/server/control.rs b/crates/connect-agent/src/server/control.rs index 54c22ea03..d5689b050 100644 --- a/crates/connect-agent/src/server/control.rs +++ b/crates/connect-agent/src/server/control.rs @@ -16,12 +16,22 @@ impl AgentState { ); } let result = match request.command { - ControlCommand::Ping => Ok(serde_json::json!({ - "pong": true, - "ready": self.initialized(), - })), + ControlCommand::Ping => { + let readiness = self.readiness(); + Ok(serde_json::json!({ + "pong": true, + "ready": readiness.ready, + "readiness": readiness, + })) + } ControlCommand::Status => self.registry.count().map(|registered_collections| { serde_json::to_value(AgentStatus { + readiness: Some(self.readiness()), + relay_problem: self + .relay_problem + .read() + .expect("relay problem lock poisoned") + .map(str::to_string), protocol_version: LOCAL_CONTROL_PROTOCOL_VERSION, binary_version: env!("CARGO_PKG_VERSION").to_string(), state: self diff --git a/crates/connect-agent/src/server/policy.rs b/crates/connect-agent/src/server/policy.rs index 34074033d..39ee6070b 100644 --- a/crates/connect-agent/src/server/policy.rs +++ b/crates/connect-agent/src/server/policy.rs @@ -337,6 +337,14 @@ impl AgentState { /// Capture before queueing. Admission later rechecks the authority digest /// and instance-scoped continuity epoch under the shared replacement gate. pub(crate) fn capture_policy_revision(&self) -> Result { + if !self.critical_workers_alive() + || self + .initialization + .load(std::sync::atomic::Ordering::Acquire) + == super::Initialization::Failed as u8 + { + return Err(policy_changed()); + } let mut gate = self .policy_revision_gate .0 diff --git a/crates/connect-agent/src/server/tests.rs b/crates/connect-agent/src/server/tests.rs index 715231cdb..c27947c8b 100644 --- a/crates/connect-agent/src/server/tests.rs +++ b/crates/connect-agent/src/server/tests.rs @@ -23,6 +23,24 @@ async fn access_snapshot_response( .await } +#[tokio::test] +async fn credential_bootstrap_failure_is_not_an_unconfigured_account_snapshot() { + let root = tempfile::tempdir().unwrap(); + let registry = CollectionRegistry::open(root.path()).unwrap(); + let watcher = CollectionWatchService::start(registry.clone()); + let mut state = AgentState::new(registry, watcher, None); + state.credential_store_error = + Some("Unlock the credential store and restart the connector.".to_string()); + state.mark_initialized(); + assert!(!state.readiness().ready); + let response = Arc::new(state) + .execute(ControlRequest::new(ControlCommand::AccessSnapshot)) + .await; + assert!(!response.ok); + assert!(response.result.is_none()); + assert_eq!(response.error.unwrap().code, "credential_store_unavailable"); +} + #[tokio::test] async fn access_snapshot_falls_back_only_when_the_control_plane_is_unavailable() { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -263,6 +281,40 @@ async fn listening_callback_runs_after_the_control_socket_is_reachable() { let _ = server.await; } +#[tokio::test] +async fn readiness_reports_initialization_and_critical_worker_failure_consistently() { + let root = tempfile::tempdir().unwrap(); + let registry = CollectionRegistry::open(root.path()).unwrap(); + let watcher = CollectionWatchService::start(registry.clone()); + let state = Arc::new(AgentState::new(registry, watcher, None)); + state.mark_initialization_failed(); + for command in [ControlCommand::Ping, ControlCommand::Status] { + let result = state + .execute(ControlRequest::new(command)) + .await + .result + .unwrap(); + assert_eq!(result["readiness"]["schema_version"], 1); + assert_eq!(result["readiness"]["ready"], false); + assert_eq!(result["readiness"]["safe_reason"], "initialization_failed"); + } + state.mark_initialized(); + let worker = tokio::spawn(std::future::pending::<()>()); + state.monitor_critical_worker(worker.abort_handle()); + assert!(state.readiness().ready); + worker.abort(); + let _ = worker.await; + for command in [ControlCommand::Ping, ControlCommand::Status] { + let result = state + .execute(ControlRequest::new(command)) + .await + .result + .unwrap(); + assert_eq!(result["readiness"]["ready"], false); + assert_eq!(result["readiness"]["safe_reason"], "critical_worker_failed"); + } +} + #[tokio::test] async fn local_control_refuses_to_replace_a_non_socket_endpoint() { let test_root = std::env::temp_dir().join(format!( diff --git a/crates/connect-agent/src/watcher.rs b/crates/connect-agent/src/watcher.rs index b97ce65cd..0d80c87b2 100644 --- a/crates/connect-agent/src/watcher.rs +++ b/crates/connect-agent/src/watcher.rs @@ -84,6 +84,15 @@ impl CollectionWatchService { } } + pub fn is_alive(&self) -> bool { + self.inner + .worker + .lock() + .expect("finalizer worker lock poisoned") + .as_ref() + .is_some_and(|worker| !worker.is_finished()) + } + pub fn refresh(&self, collections: &[CollectionSummary]) { let (ready, receiver) = mpsc::sync_channel(0); let active = collections diff --git a/crates/connect-cli/src/daemon.rs b/crates/connect-cli/src/daemon.rs index fc562bbdd..8e8dd2fd8 100644 --- a/crates/connect-cli/src/daemon.rs +++ b/crates/connect-cli/src/daemon.rs @@ -132,6 +132,7 @@ pub(super) async fn execute_daemon_command( .await .is_ok_and(|response| response.ok) { + wait_until_ready(endpoint).await?; return Ok(serde_json::json!({ "started": false, "already_running": true @@ -207,7 +208,10 @@ pub(super) async fn doctor(state_dir: &Path, endpoint: &str, target: DaemonTarge }; let response = send(endpoint, ControlRequest::new(ControlCommand::Status)).await; let (daemon, status) = match response { - Ok(response) if response.ok => ("ready", response.result), + Ok(response) if response.ok => { + let ready = response.result.as_ref().is_some_and(canonically_ready); + (if ready { "ready" } else { "attention" }, response.result) + } _ => ("unavailable", None), }; serde_json::json!({ @@ -309,20 +313,39 @@ fn daemon_lease_released(state_dir: &Path) -> bool { true } +fn canonically_ready(value: &Value) -> bool { + let health = &value["readiness"]; + health["schema_version"] == 1 + && health["ready"] == true + && health["binary_version"] == env!("CARGO_PKG_VERSION") +} + pub(super) async fn wait_until_ready(endpoint: &str) -> Result<(), CliError> { - for _ in 0..200 { - if send(endpoint, ControlRequest::new(ControlCommand::Ping)) - .await - .is_ok_and(|response| { - response.ok - && response - .result - .as_ref() - .and_then(|value| value["ready"].as_bool()) - .unwrap_or(false) - }) + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10); + while tokio::time::Instant::now() < deadline { + if let Ok(Ok(response)) = tokio::time::timeout( + std::time::Duration::from_millis(200), + send(endpoint, ControlRequest::new(ControlCommand::Ping)), + ) + .await { - return Ok(()); + if response.ok && response.result.as_ref().is_some_and(canonically_ready) { + return Ok(()); + } + if let Some(reason) = response + .result + .as_ref() + .and_then(|value| value["readiness"]["safe_reason"].as_str()) + { + if matches!( + reason, + "initialization_failed" + | "critical_worker_failed" + | "credential_store_unavailable" + ) { + return Err(CliError::unavailable(format!("The Connect daemon needs attention: {reason}. Restart after resolving this condition."))); + } + } } tokio::time::sleep(std::time::Duration::from_millis(50)).await; } @@ -440,6 +463,24 @@ pub(super) fn control_request_timeout(command: &ControlCommand) -> std::time::Du mod tests { use super::*; + #[test] + fn readiness_requires_the_canonical_contract_and_expected_version() { + assert!(!canonically_ready(&serde_json::json!({"ready": true}))); + for (version, ready, schema, expected) in [ + (env!("CARGO_PKG_VERSION"), true, 1, true), + (env!("CARGO_PKG_VERSION"), false, 1, false), + (env!("CARGO_PKG_VERSION"), true, 2, false), + ("wrong", true, 1, false), + ] { + assert_eq!( + canonically_ready(&serde_json::json!({"readiness": { + "schema_version": schema, "ready": ready, "binary_version": version + }})), + expected + ); + } + } + #[test] fn shutdown_waits_for_the_daemon_lease_after_the_socket_disappears() { let temporary = tempfile::tempdir().unwrap(); diff --git a/crates/connect-cli/tests/unified_cli.rs b/crates/connect-cli/tests/unified_cli.rs index 250f18ad6..e9c899de5 100644 --- a/crates/connect-cli/tests/unified_cli.rs +++ b/crates/connect-cli/tests/unified_cli.rs @@ -63,6 +63,64 @@ fn wait_for_daemon(endpoint: &Path) { panic!("daemon did not become reachable"); } +#[test] +fn credential_bootstrap_failure_is_unhealthy_even_with_a_live_control_endpoint() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + let endpoint = scratch.path().join("control.sock"); + std::fs::create_dir_all(&state).unwrap(); + std::fs::write(state.join("test-secrets.json"), "[").unwrap(); + let child = Command::new(binary()) + .args([ + "--state-dir", + state.to_str().unwrap(), + "--endpoint", + endpoint.to_str().unwrap(), + "connect", + "daemon", + "run", + "--loopback-port", + "0", + ]) + .env("MDBASE_CONNECT_ENV", "test") + .env("MDBASE_CONNECT_SECRET_BACKEND", "insecure-test-file") + .env_remove("MDBASE_CONNECT_SERVER_URL") + .env_remove("MDBASE_CONNECT_CONNECTOR_TOKEN") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let _daemon = Daemon { child }; + wait_for_daemon(&endpoint); + let call = |command: &str| { + run(&[ + "--state-dir", + state.to_str().unwrap(), + "--endpoint", + endpoint.to_str().unwrap(), + "--json", + "connect", + command, + ]) + }; + for command in ["ping", "status"] { + let result = json(&call(command)); + assert_eq!(result["readiness"]["ready"], false); + assert_eq!( + result["readiness"]["safe_reason"], + "credential_store_unavailable" + ); + } + assert_eq!(json(&call("doctor"))["healthy"], false); + let whoami = call("whoami"); + assert!(!whoami.status.success()); + assert_eq!( + diagnostic_json(&whoami)["error"]["code"], + "credential_store_unavailable" + ); +} + #[test] fn isolated_restart_preserves_the_bound_loopback_port() { let scratch = tempfile::tempdir().unwrap(); diff --git a/crates/connect-protocol/src/control.rs b/crates/connect-protocol/src/control.rs index 241cc6958..6af69f548 100644 --- a/crates/connect-protocol/src/control.rs +++ b/crates/connect-protocol/src/control.rs @@ -370,8 +370,31 @@ pub struct ControlError { pub details: Option, } +/// Payload-free local readiness contract. Missing/unknown versions are not Ready. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AgentReadiness { + pub schema_version: u32, + pub ready: bool, + pub binary_version: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub safe_reason: Option, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ReadinessReason { + Starting, + InitializationFailed, + CriticalWorkerFailed, + CredentialStoreUnavailable, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AgentStatus { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub readiness: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub relay_problem: Option, pub protocol_version: u32, #[serde(default)] pub binary_version: String, From b285418630cbe6b88be84e701c1bac1cd11b9f2b Mon Sep 17 00:00:00 2001 From: callumalpass Date: Sun, 13 Sep 2026 19:44:59 +1000 Subject: [PATCH 04/16] fix(relay): classify reconnect failures and bound retry pacing --- crates/connect-agent/src/relay.rs | 128 ++++++++++++---- crates/connect-agent/src/relay/retry.rs | 190 ++++++++++++++++++++++++ 2 files changed, 291 insertions(+), 27 deletions(-) create mode 100644 crates/connect-agent/src/relay/retry.rs diff --git a/crates/connect-agent/src/relay.rs b/crates/connect-agent/src/relay.rs index e86763700..fee19eb6c 100644 --- a/crates/connect-agent/src/relay.rs +++ b/crates/connect-agent/src/relay.rs @@ -1,3 +1,4 @@ +mod retry; use crate::admission::{ classify_operation, queue_deadline, AdmissionPermit, AdmissionRequest, WorkClass, }; @@ -9,7 +10,12 @@ use mdbase_connect_protocol::{ RelayFileFrame, RelayMessage, CONTROL_PROTOCOL_VERSION, PROTOCOL_USAGE_REPORT_CAPABILITY, RELAY_CAPABILITIES, RELAY_HANDSHAKE_TIMEOUT_SECONDS, RELAY_REQUIRED_CAPABILITIES, }; +use rand_core::{OsRng, RngCore}; use reqwest::Client; +use retry::{ + pacing_delay, retry_after, retry_delay, terminal_http_status, terminal_reason, RelayPacing, + TerminalRelayFailure, +}; use std::sync::Arc; use std::time::Duration; use tokio_tungstenite::connect_async; @@ -41,17 +47,54 @@ impl Drop for AbortOnDrop { pub async fn run(server_url: String, connector_token: String, state: Arc) { crate::ensure_tls_crypto_provider(); - let client = Client::new(); - let mut retry_delay = 1u64; + let client = Client::builder() + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(30)) + .build() + .expect("relay HTTP client"); + let mut failures = 0u32; loop { state.set_connection_state(AgentConnectionState::Connecting); - let result = connect_once(&client, &server_url, &connector_token, state.clone()).await; + let healthy_since = Arc::new(std::sync::Mutex::new(None)); + let result = connect_once( + &client, + &server_url, + &connector_token, + state.clone(), + healthy_since.clone(), + ) + .await; state.set_connection_state(AgentConnectionState::Offline); - if let Err(error) = result { - tracing::warn!(%error, retry_seconds = retry_delay, "cloud relay disconnected"); + if let Err(error) = &result { + if let Some(reason) = terminal_reason(error.as_ref()) { + state.set_relay_problem(Some(reason)); + tracing::warn!( + code = reason, + "relay requires user action; reconnect stopped" + ); + // Remain a live, explicitly blocked owner until controlled restart. + std::future::pending::<()>().await; + return; + } } - tokio::time::sleep(Duration::from_secs(retry_delay)).await; - retry_delay = (retry_delay * 2).min(30); + let uptime = healthy_since + .lock() + .expect("relay health lock poisoned") + .map(|since: std::time::Instant| since.elapsed()) + .unwrap_or_default(); + let delay = retry_delay(&mut failures, uptime, OsRng.next_u32()).max( + result + .as_ref() + .err() + .and_then(|error| pacing_delay(error.as_ref())) + .unwrap_or_default(), + ); + tracing::info!( + code = "relay_transport_interrupted", + retry_ms = delay.as_millis() as u64, + "relay reconnect scheduled" + ); + tokio::time::sleep(delay).await; } } @@ -60,6 +103,7 @@ async fn connect_once( server_url: &str, connector_token: &str, state: Arc, + healthy_since: Arc>>, ) -> Result<(), Box> { sync_collections(client, server_url, connector_token, &state).await?; let websocket_url = websocket_url(server_url)?; @@ -68,7 +112,8 @@ async fn connect_once( "authorization", HeaderValue::from_str(&format!("Bearer {connector_token}"))?, ); - let (mut socket, _) = connect_async(request).await?; + let (mut socket, _) = + tokio::time::timeout(Duration::from_secs(15), connect_async(request)).await??; socket .send(Message::Text( serde_json::to_string(&RelayMessage::RelayHello { @@ -94,12 +139,15 @@ async fn connect_once( .await .map_err(|_| "relay handshake timed out")? .ok_or("relay closed during handshake")??; - let Message::Text(welcome) = welcome else { - return Err("relay returned a non-text handshake response".into()); + let welcome = match welcome { + Message::Text(welcome) => welcome, + Message::Close(_) => return Err("relay closed during handshake".into()), + _ => return Err(TerminalRelayFailure("incompatible_version").into()), }; let (usage_reporting, server_semantics, declaration_evidence) = match serde_json::from_str::< RelayMessage, - >(welcome.as_ref())? + >(welcome.as_ref()) + .map_err(|_| TerminalRelayFailure("incompatible_version"))? { RelayMessage::RelayWelcome { protocol_version, @@ -122,8 +170,10 @@ async fn connect_once( }), ) } - RelayMessage::RelayIncompatible { message, .. } => return Err(message.into()), - _ => return Err("relay returned an incompatible handshake response".into()), + RelayMessage::RelayIncompatible { .. } => { + return Err(TerminalRelayFailure("incompatible_version").into()) + } + _ => return Err(TerminalRelayFailure("incompatible_version").into()), }; let (mut writer, mut reader) = socket.split(); let (responses, mut response_rx) = tokio::sync::mpsc::channel::(64); @@ -151,6 +201,10 @@ async fn connect_once( let ready = policy_state.finish_policy_update(generation, usable); if usable && ready { policy_state.set_connection_state(AgentConnectionState::Connected); + healthy_since + .lock() + .expect("relay health lock poisoned") + .get_or_insert_with(std::time::Instant::now); } policy_applied.send_replace((generation, usable && ready)); } @@ -164,7 +218,9 @@ async fn connect_once( let sync_period = Duration::from_secs(15); let mut sync_interval = tokio::time::interval_at(tokio::time::Instant::now() + sync_period, sync_period); - let mut inventory_sync = tokio::task::JoinSet::new(); + let mut inventory_sync: tokio::task::JoinSet< + Result<(), Box>, + > = tokio::task::JoinSet::new(); loop { tokio::select! { @@ -172,7 +228,8 @@ async fn connect_once( let Some(message) = message else { return Err("relay closed the connection".into()); }; match message? { Message::Text(text) => { - let relay_message: RelayMessage = serde_json::from_str(text.as_ref())?; + let relay_message: RelayMessage = serde_json::from_str(text.as_ref()) + .map_err(|_| TerminalRelayFailure("incompatible_version"))?; let supported_grant = |grant: &mdbase_connect_protocol::GrantPolicy| { let version = grant.application_authorization.binding.contracts.semantic_capabilities; server_semantics.contains(&version) && (version != 2 || declaration_evidence) @@ -183,7 +240,7 @@ async fn connect_once( _ => true, }; if !supported { - return Err("relay attempted an unadvertised authorization contract".into()); + return Err(TerminalRelayFailure("incompatible_version").into()); } if matches!(&relay_message, RelayMessage::PolicySnapshot { .. }) { // A dedicated single consumer preserves snapshot order without @@ -382,7 +439,12 @@ async fn connect_once( }); } Message::Ping(payload) => writer.send(Message::Pong(payload)).await?, - Message::Close(_) => return Err("relay closed the connection".into()), + Message::Close(frame) => { + if frame.as_ref().is_some_and(|frame| u16::from(frame.code) == 4002) { + return Err(TerminalRelayFailure("incompatible_version").into()); + } + return Err("relay closed the connection".into()); + } _ => {} } } @@ -434,8 +496,14 @@ async fn connect_once( } } result = inventory_sync.join_next(), if !inventory_sync.is_empty() => { - if let Some(Err(error)) = result { - tracing::warn!(%error, "collection sync task failed"); + match result { + Some(Ok(Err(error))) => { + if terminal_reason(error.as_ref()).is_some() { return Err(error); } + if let Some(delay) = pacing_delay(error.as_ref()) { sync_interval.reset_after(delay); } + tracing::warn!(code = "relay_inventory_unavailable", "collection inventory will retry"); + } + Some(Err(_)) => tracing::warn!(code = "relay_inventory_worker_failed", "collection inventory will retry"), + _ => {} } } _ = sync_interval.tick() => { @@ -456,14 +524,7 @@ async fn connect_once( let connector_token = connector_token.to_string(); let state = state.clone(); inventory_sync.spawn(async move { - if let Err(error) = sync_collections( - &client, - &server_url, - &connector_token, - &state, - ).await { - tracing::warn!(%error, "collection sync failed"); - } + sync_collections(&client, &server_url, &connector_token, &state).await }); } } @@ -703,6 +764,19 @@ async fn sync_collections( .send() .await?; if !response.status().is_success() { + if matches!(response.status().as_u16(), 429 | 503) { + if let Some(delay) = response + .headers() + .get("retry-after") + .and_then(|value| value.to_str().ok()) + .and_then(|value| retry_after(value, chrono::Utc::now())) + { + return Err(RelayPacing(delay).into()); + } + } + if let Some(reason) = terminal_http_status(response.status().as_u16()) { + return Err(TerminalRelayFailure(reason).into()); + } return Err(format!("collection sync failed with HTTP {}", response.status()).into()); } Ok(()) diff --git a/crates/connect-agent/src/relay/retry.rs b/crates/connect-agent/src/relay/retry.rs new file mode 100644 index 000000000..f8996b6ee --- /dev/null +++ b/crates/connect-agent/src/relay/retry.rs @@ -0,0 +1,190 @@ +use std::time::Duration; + +#[derive(Debug)] +pub(super) struct TerminalRelayFailure(pub &'static str); +impl std::fmt::Display for TerminalRelayFailure { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.0) + } +} +impl std::error::Error for TerminalRelayFailure {} + +pub(super) fn terminal_http_status(status: u16) -> Option<&'static str> { + match status { + 401 | 403 => Some("authentication_required"), + 400 | 404 | 405 | 426 => Some("incompatible_version"), + _ => None, + } +} + +pub(super) fn terminal_reason( + error: &(dyn std::error::Error + Send + Sync + 'static), +) -> Option<&'static str> { + if let Some(error) = error.downcast_ref::() { + return Some(error.0); + } + if let Some(tokio_tungstenite::tungstenite::Error::Http(response)) = + error.downcast_ref::() + { + return terminal_http_status(response.status().as_u16()); + } + None +} + +#[derive(Debug)] +pub(super) struct RelayPacing(pub Duration); +impl std::fmt::Display for RelayPacing { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("relay server requested pacing") + } +} +impl std::error::Error for RelayPacing {} + +pub(super) fn retry_after(value: &str, now: chrono::DateTime) -> Option { + let seconds = value.parse::().ok().or_else(|| { + chrono::DateTime::parse_from_rfc2822(value) + .ok() + .map(|date| date.signed_duration_since(now).num_seconds().max(0) as u64) + })?; + Some(Duration::from_secs(seconds.clamp(1, 300))) +} + +pub(super) fn pacing_delay( + error: &(dyn std::error::Error + Send + Sync + 'static), +) -> Option { + if let Some(pacing) = error.downcast_ref::() { + return Some(pacing.0); + } + if let Some(tokio_tungstenite::tungstenite::Error::Http(response)) = + error.downcast_ref::() + { + if matches!(response.status().as_u16(), 429 | 503) { + return response + .headers() + .get("retry-after") + .and_then(|value| value.to_str().ok()) + .and_then(|value| retry_after(value, chrono::Utc::now())); + } + } + None +} + +/// Equal jitter avoids zero-delay spin while preserving a hard 30-second cap. +/// Reset only after policy-authorized healthy uptime, not a successful handshake. +pub(super) fn retry_delay(failures: &mut u32, healthy_uptime: Duration, random: u32) -> Duration { + if healthy_uptime >= Duration::from_secs(30) { + *failures = 0; + } + let cap_ms = (1_000u64 << (*failures).min(5)).min(30_000); + *failures = failures.saturating_add(1); + Duration::from_millis(cap_ms / 2 + u64::from(random) % (cap_ms / 2 + 1)) +} + +#[cfg(test)] +mod tests { + use super::*; + #[tokio::test] + async fn terminal_http_rejection_stops_the_real_relay_owner() { + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }; + let requests = Arc::new(AtomicUsize::new(0)); + let observed = requests.clone(); + let app = axum::Router::new().route( + "/v1/connectors/sync", + axum::routing::post(move |headers: axum::http::HeaderMap| { + assert_eq!( + headers.get("authorization").unwrap(), + "Bearer unchanged-test-identity" + ); + observed.fetch_add(1, Ordering::SeqCst); + async { axum::http::StatusCode::UNAUTHORIZED } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + let root = tempfile::tempdir().unwrap(); + let registry = mdbase_connect_core::CollectionRegistry::open(root.path()).unwrap(); + let watcher = crate::watcher::CollectionWatchService::start(registry.clone()); + let state = Arc::new(crate::server::AgentState::new(registry, watcher, None)); + let relay = tokio::spawn(super::super::run( + format!("http://{address}"), + "unchanged-test-identity".to_string(), + state, + )); + tokio::time::timeout(Duration::from_secs(2), async { + while requests.load(Ordering::SeqCst) == 0 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(1200)).await; + assert_eq!(requests.load(Ordering::SeqCst), 1); + assert!( + !relay.is_finished(), + "terminal route remains explicitly blocked until controlled restart" + ); + relay.abort(); + let _ = relay.await; + server.abort(); + let _ = server.await; + } + + #[test] + fn retry_is_bounded_jittered_and_resets_only_after_healthy_uptime() { + let mut failures = 0; + assert_eq!( + retry_delay(&mut failures, Duration::ZERO, 0), + Duration::from_millis(500) + ); + assert_eq!( + retry_delay(&mut failures, Duration::ZERO, 500), + Duration::from_millis(1500) + ); + for _ in 0..100 { + let delay = retry_delay(&mut failures, Duration::from_secs(29), u32::MAX); + assert!(delay <= Duration::from_secs(30)); + assert!(delay >= Duration::from_secs(2)); + } + assert_eq!( + retry_delay(&mut failures, Duration::from_secs(30), 0), + Duration::from_millis(500) + ); + } + #[test] + fn server_pacing_accepts_seconds_and_dates_with_a_hard_bound() { + let now = chrono::DateTime::parse_from_rfc2822("Sun, 13 Sep 2026 00:00:00 +0000") + .unwrap() + .with_timezone(&chrono::Utc); + assert_eq!(retry_after("45", now), Some(Duration::from_secs(45))); + assert_eq!(retry_after("999999", now), Some(Duration::from_secs(300))); + assert_eq!( + retry_after("Sun, 13 Sep 2026 00:02:00 +0000", now), + Some(Duration::from_secs(120)) + ); + assert_eq!(retry_after("invalid", now), None); + } + + #[test] + fn only_terminal_auth_or_protocol_outcomes_stop_reconnect() { + assert_eq!(terminal_http_status(401), Some("authentication_required")); + assert_eq!(terminal_http_status(403), Some("authentication_required")); + assert_eq!(terminal_http_status(426), Some("incompatible_version")); + for status in [408, 429, 500, 502, 503, 504] { + assert_eq!(terminal_http_status(status), None); + } + assert_eq!( + terminal_reason(&TerminalRelayFailure("incompatible_version")), + Some("incompatible_version") + ); + assert_eq!( + terminal_reason(&std::io::Error::from(std::io::ErrorKind::ConnectionReset)), + None + ); + } +} From f1a73202f470d09ea820cbc7956bfe18f59df449 Mon Sep 17 00:00:00 2001 From: callumalpass Date: Sun, 13 Sep 2026 19:45:25 +1000 Subject: [PATCH 05/16] fix(mirrors): rearm transient credential reads in the existing scheduler --- crates/connect-agent/src/mirrors/runtime.rs | 2 +- crates/connect-agent/src/mirrors/support.rs | 11 +-- crates/connect-agent/src/mirrors/tests.rs | 9 ++- crates/connect-cli/tests/unified_cli.rs | 87 +++++++++++++++++++++ 4 files changed, 100 insertions(+), 9 deletions(-) diff --git a/crates/connect-agent/src/mirrors/runtime.rs b/crates/connect-agent/src/mirrors/runtime.rs index 9b95ef727..4e25f34c8 100644 --- a/crates/connect-agent/src/mirrors/runtime.rs +++ b/crates/connect-agent/src/mirrors/runtime.rs @@ -101,7 +101,7 @@ impl MirrorManager { Some(Ok((replica_id, Err(error)))) if error.code() == "mirror_sync_skipped" => { active_workers.remove(&replica_id); } - Some(Ok((replica_id, Err(error)))) if terminal_background_error(&error) => { + Some(Ok((replica_id, Err(error)))) if terminal_background_error(&error, manager.credential_store_error.is_some()) => { active_workers.remove(&replica_id); retries.remove(&replica_id); blocked.insert(replica_id); diff --git a/crates/connect-agent/src/mirrors/support.rs b/crates/connect-agent/src/mirrors/support.rs index b6db4929c..ecffdce38 100644 --- a/crates/connect-agent/src/mirrors/support.rs +++ b/crates/connect-agent/src/mirrors/support.rs @@ -170,11 +170,12 @@ pub(super) fn background_retry_delay(replica_id: Uuid, failures: u32) -> Duratio Duration::from_millis(millis as u64) } -pub(super) fn terminal_background_error(error: &ConnectError) -> bool { - matches!( - error.code(), - "mirror_state_upgrade_required" | "credential_store_unavailable" - ) +pub(super) fn terminal_background_error( + error: &ConnectError, + startup_credentials_unavailable: bool, +) -> bool { + error.code() == "mirror_state_upgrade_required" + || (error.code() == "credential_store_unavailable" && startup_credentials_unavailable) } pub(super) fn computer_name() -> String { diff --git a/crates/connect-agent/src/mirrors/tests.rs b/crates/connect-agent/src/mirrors/tests.rs index f3f86bd18..502b5ab31 100644 --- a/crates/connect-agent/src/mirrors/tests.rs +++ b/crates/connect-agent/src/mirrors/tests.rs @@ -75,9 +75,12 @@ fn prerelease_state_upgrade_blocks_background_retry() { let transient = mirror_error("mirror_transport_failed", "Try again."); let credentials = ConnectError::CredentialStore("The login keyring is locked.".into()); - assert!(terminal_background_error(&upgrade)); - assert!(terminal_background_error(&credentials)); - assert!(!terminal_background_error(&transient)); + assert!(terminal_background_error(&upgrade, false)); + // A read failure after a successful bootstrap reuses the existing bounded + // background retry schedule; it must not park this replica permanently. + assert!(!terminal_background_error(&credentials, false)); + assert!(terminal_background_error(&credentials, true)); + assert!(!terminal_background_error(&transient, false)); } #[test] diff --git a/crates/connect-cli/tests/unified_cli.rs b/crates/connect-cli/tests/unified_cli.rs index e9c899de5..e50af855d 100644 --- a/crates/connect-cli/tests/unified_cli.rs +++ b/crates/connect-cli/tests/unified_cli.rs @@ -121,6 +121,93 @@ fn credential_bootstrap_failure_is_unhealthy_even_with_a_live_control_endpoint() ); } +#[test] +fn running_mirror_retries_after_transient_credential_store_failure_without_restart() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + std::fs::create_dir_all(&state).unwrap(); + let endpoint = scratch.path().join("control.sock"); + let logs = scratch.path().join("daemon.log"); + let log = std::fs::File::create(&logs).unwrap(); + std::fs::write(state.join("mirrors.json"), serde_json::to_vec(&serde_json::json!({ + "version": 2, "mirrors": [{ + "collection_id": "00000000-0000-4000-8000-000000000001", + "replica_id": "00000000-0000-4000-8000-000000000002", + "enrollment_id": "00000000-0000-4000-8000-000000000003", + "name": "isolated credential recovery fixture", "mode": "read_only", + "path": scratch.path().join("mirror"), "lifecycle": "active", + "sync_url": "http://127.0.0.1:1/v1/authorities/00000000-0000-4000-8000-000000000001/sync", "control_url": "http://127.0.0.1:1", + "access_token_expires_at": "2099-01-01T00:00:00Z", "created_at": "2026-01-01T00:00:00Z" + }] + })).unwrap()).unwrap(); + let child = Command::new(binary()) + .args([ + "--state-dir", + state.to_str().unwrap(), + "--endpoint", + endpoint.to_str().unwrap(), + "connect", + "daemon", + "run", + "--loopback-port", + "0", + ]) + .env("MDBASE_CONNECT_ENV", "test") + .env("MDBASE_CONNECT_SECRET_BACKEND", "insecure-test-file") + .env_remove("MDBASE_CONNECT_SERVER_URL") + .env_remove("MDBASE_CONNECT_CONNECTOR_TOKEN") + .stdin(Stdio::null()) + .stdout(log.try_clone().unwrap()) + .stderr(log) + .spawn() + .unwrap(); + let mut daemon = Daemon { child }; + thread::sleep(Duration::from_millis(100)); + assert!( + daemon.child.try_wait().unwrap().is_none(), + "{}", + std::fs::read_to_string(&logs).unwrap() + ); + wait_for_daemon(&endpoint); + let observe = |offset: usize, marker: &str| { + let deadline = Instant::now() + Duration::from_secs(60); + loop { + let content = std::fs::read_to_string(&logs).unwrap(); + if content + .get(offset..) + .is_some_and(|tail| tail.contains(marker)) + { + return content.len(); + } + assert!( + Instant::now() < deadline, + "mirror worker did not reach {marker}" + ); + thread::sleep(Duration::from_millis(50)); + } + }; + observe(0, "mirror_credentials_missing"); + let secrets = state.join("test-secrets.json"); + let original = std::fs::read(&secrets).unwrap(); + // Fault the existing test-only store after successful bootstrap, never the OS keyring. + std::fs::write(&secrets, "[").unwrap(); + let failed_at = observe(0, "credential_store_unavailable"); + std::fs::write(&secrets, &original).unwrap(); + // A new missing-credential result proves the SAME worker re-read the restored store. + // This deliberately does not claim a successful authorized mirror synchronization. + observe(failed_at, "mirror_credentials_missing"); + assert!(daemon.child.try_wait().unwrap().is_none()); + assert_eq!(std::fs::read(&secrets).unwrap(), original); + let status = run(&[ + "--endpoint", + endpoint.to_str().unwrap(), + "--json", + "connect", + "status", + ]); + assert_eq!(json(&status)["readiness"]["ready"], true); +} + #[test] fn isolated_restart_preserves_the_bound_loopback_port() { let scratch = tempfile::tempdir().unwrap(); From 569bab68f9c0e50cc1ef7f6ee34656300f951287 Mon Sep 17 00:00:00 2001 From: callumalpass Date: Sun, 13 Sep 2026 19:46:55 +1000 Subject: [PATCH 06/16] fix(desktop): gate startup and updates on canonical daemon readiness --- apps/desktop/scripts/build-main.mjs | 1 + apps/desktop/src/main/agent-startup.ts | 35 ++++--- apps/desktop/src/main/boot-gate.ts | 62 ++++++++++++ apps/desktop/src/main/daemon-lifecycle.ts | 16 ++- .../src/main/electron-update-backend.ts | 10 +- apps/desktop/src/main/main.ts | 71 +++++++------ .../desktop/src/renderer/connection-state.mts | 10 ++ apps/desktop/src/renderer/global.d.ts | 4 +- apps/desktop/src/shared/readiness.ts | 32 ++++++ apps/desktop/test/agent-startup.test.mjs | 9 +- apps/desktop/test/boot-gate.test.mjs | 99 +++++++++++++++++++ apps/desktop/test/daemon-lifecycle.test.mjs | 66 ++++--------- apps/desktop/test/readiness.test.mjs | 30 ++++++ apps/desktop/tsconfig.main.json | 6 +- 14 files changed, 350 insertions(+), 101 deletions(-) create mode 100644 apps/desktop/src/main/boot-gate.ts create mode 100644 apps/desktop/src/shared/readiness.ts create mode 100644 apps/desktop/test/boot-gate.test.mjs create mode 100644 apps/desktop/test/readiness.test.mjs diff --git a/apps/desktop/scripts/build-main.mjs b/apps/desktop/scripts/build-main.mjs index e1c8eb766..8b16cc5e6 100644 --- a/apps/desktop/scripts/build-main.mjs +++ b/apps/desktop/scripts/build-main.mjs @@ -7,6 +7,7 @@ await build({ "src/main/main.ts", "src/main/preload.ts", "src/main/agent-startup.ts", + "src/main/boot-gate.ts", "src/main/daemon-lifecycle.ts", "src/main/deep-link.ts", "src/main/editor-url.ts", diff --git a/apps/desktop/src/main/agent-startup.ts b/apps/desktop/src/main/agent-startup.ts index 310f60ce9..aaadb197d 100644 --- a/apps/desktop/src/main/agent-startup.ts +++ b/apps/desktop/src/main/agent-startup.ts @@ -1,6 +1,10 @@ +import { presentReadiness, type AgentReadiness } from "../shared/readiness"; +export { presentReadiness } from "../shared/readiness"; + export interface AgentPing { pong: boolean; ready?: boolean; + readiness?: AgentReadiness; } export interface AgentStartupOptions { @@ -8,10 +12,21 @@ export interface AgentStartupOptions { launch(): Promise; endpointIsUnavailable(error: unknown): boolean; incompatibleDaemon(error: unknown): boolean; + expectedVersion: string; readinessTimeoutMs?: number; pollIntervalMs?: number; } +class ReadinessError extends Error {} + +function isReady(ping: AgentPing, options: AgentStartupOptions): boolean { + const health = presentReadiness(ping.readiness, options.expectedVersion); + if (!ping.pong || health.state === "attention") throw new ReadinessError(health.label); + return health.state === "ready"; +} + +const terminal = (error: unknown, options: AgentStartupOptions) => + error instanceof ReadinessError || options.incompatibleDaemon(error); const delay = (durationMs: number) => new Promise((resolve) => setTimeout(resolve, durationMs)); @@ -20,10 +35,9 @@ export async function waitForAgentReady(options: AgentStartupOptions): Promise { try { - const ping = await options.ping(400); - if (ping.ready !== false) return; + if (isReady(await options.ping(400), options)) return; return waitForAgentReady(options); } catch (error) { - if (options.incompatibleDaemon(error)) throw error; + if (terminal(error, options)) throw error; if (!options.endpointIsUnavailable(error)) return waitForAgentReady(options); } @@ -45,17 +58,13 @@ export async function ensureAgentReady(options: AgentStartupOptions): Promise; + start(): Promise; + blockedReason(): string | null; +} + +/** One owner for update recovery, startup and daemon-backed IPC admission. */ +export class BootGate { + private initialization: Promise | undefined; + private startup: Promise | undefined; + private installing = false; + + constructor(private readonly options: BootGateOptions) {} + + private initialize(): Promise { + // A failed boot stays failed until the application restarts. Ordinary IPC + // must not silently bypass a failed persisted update recovery. + return this.initialization ??= this.options.initialize(); + } + + private assertAdmission(): void { + const reason = this.installing + ? "The application update is in progress." + : this.options.blockedReason(); + if (reason) throw new Error(reason); + } + + async ready(): Promise { + await this.initialize(); + this.assertAdmission(); + this.startup ??= this.options.start().finally(() => { this.startup = undefined; }); + await this.startup; + this.assertAdmission(); + } + + async request(operation: () => Promise): Promise { + await this.ready(); + this.assertAdmission(); + return operation(); + } + + async check(operation: () => Promise): Promise { + await this.initialize(); + this.assertAdmission(); + return operation(); + } + + async install(operation: () => Promise): Promise { + if (this.installing) throw new Error("The application update is in progress."); + this.installing = true; + try { + await this.initialize(); + // Never race the CLI startup already admitted before installation began. + await this.startup; + const reason = this.options.blockedReason(); + if (reason) throw new Error(reason); + return await operation(); + } finally { + this.installing = false; + } + } +} diff --git a/apps/desktop/src/main/daemon-lifecycle.ts b/apps/desktop/src/main/daemon-lifecycle.ts index e372787d8..ecc74f95a 100644 --- a/apps/desktop/src/main/daemon-lifecycle.ts +++ b/apps/desktop/src/main/daemon-lifecycle.ts @@ -9,15 +9,27 @@ export function connectCliEnvironment( return sanitized; } +export type DaemonTarget = "installed_service" | "isolated_profile"; +export interface DaemonPaths { stateDir: string; endpoint: string; target: DaemonTarget } + +export function parseDaemonPaths(value: unknown): DaemonPaths { + const paths = value as { state_dir?: unknown; endpoint?: unknown; target?: unknown } | null; + if (!paths || typeof paths.state_dir !== "string" || typeof paths.endpoint !== "string" || + (paths.target !== "installed_service" && paths.target !== "isolated_profile")) { + throw new Error("The connector runtime returned invalid path information."); + } + return { stateDir: paths.state_dir, endpoint: paths.endpoint, target: paths.target }; +} + export function daemonCliArguments( - packaged: boolean, + target: DaemonTarget, stateDirectory: string, endpoint: string, command: string[], json = false ): string[] { return [ - ...(packaged ? [] : ["--state-dir", stateDirectory, "--endpoint", endpoint]), + ...(target === "isolated_profile" ? ["--state-dir", stateDirectory, "--endpoint", endpoint] : []), ...(json ? ["--json"] : []), "connect", "daemon", diff --git a/apps/desktop/src/main/electron-update-backend.ts b/apps/desktop/src/main/electron-update-backend.ts index 1f1a86205..7ea0b7493 100644 --- a/apps/desktop/src/main/electron-update-backend.ts +++ b/apps/desktop/src/main/electron-update-backend.ts @@ -1,4 +1,5 @@ import { autoUpdater, shell } from "electron"; +import { presentReadiness, type AgentReadiness } from "../shared/readiness"; import { execFile as execFileCallback } from "node:child_process"; import { createReadStream } from "node:fs"; import { @@ -25,7 +26,7 @@ import { } from "./update-policy"; import type { UpdateTransaction } from "./update-state"; import { artifactMatches, downloadArtifact, downloadBytes } from "./update-download"; -import { connectCliEnvironment, daemonCliArguments } from "./daemon-lifecycle"; +import { connectCliEnvironment, daemonCliArguments, type DaemonTarget } from "./daemon-lifecycle"; const execFile = promisify(execFileCallback); const AUTO_UPDATER_TIMEOUT_MS = 180_000; @@ -39,6 +40,7 @@ export interface ElectronUpdateBackendOptions { userDataDirectory: string; binaryPath: () => string; stateDirectory: () => string; + target: () => DaemonTarget; endpoint: () => string; } @@ -236,7 +238,7 @@ export class ElectronUpdateBackend implements UpdateBackend { while (Date.now() < deadline) { const status = await this.daemonStatus(binary).catch(() => null); lastVersion = status?.binaryVersion; - if (status?.running && status.binaryVersion === expectedVersion) return; + if (status?.running && status.ready && status.binaryVersion === expectedVersion) return; await new Promise((resolve) => setTimeout(resolve, 150)); } throw new Error( @@ -249,12 +251,14 @@ export class ElectronUpdateBackend implements UpdateBackend { private async daemonStatus(binary = this.options.binaryPath()): Promise<{ installed: boolean; running: boolean; + ready: boolean; binaryVersion?: string; }> { const value = await this.runCli(binary, ["status"], 10_000); return { installed: value.installed === true, running: value.running === true, + ready: presentReadiness((value.status as { readiness?: AgentReadiness } | undefined)?.readiness).state === "ready", binaryVersion: value.status && typeof value.status === "object" && @@ -273,7 +277,7 @@ export class ElectronUpdateBackend implements UpdateBackend { const { stdout } = await execFile( binary, daemonCliArguments( - this.packaged, + this.options.target(), this.options.stateDirectory(), this.options.endpoint(), command, diff --git a/apps/desktop/src/main/main.ts b/apps/desktop/src/main/main.ts index a6347bd92..dec2e289d 100644 --- a/apps/desktop/src/main/main.ts +++ b/apps/desktop/src/main/main.ts @@ -17,7 +17,8 @@ import { hostname } from "node:os"; import { promisify } from "node:util"; import { ensureAgentReady, type AgentPing } from "./agent-startup"; import { AgentControlError, requestAgent } from "./control-client"; -import { connectCliEnvironment, daemonCliArguments } from "./daemon-lifecycle"; +import { BootGate } from "./boot-gate"; +import { connectCliEnvironment, daemonCliArguments, parseDaemonPaths, type DaemonPaths } from "./daemon-lifecycle"; import { routeForDeepLink, shouldRegisterDeepLinks } from "./deep-link"; import { buildEditorUrl } from "./editor-url"; import { ElectronUpdateBackend } from "./electron-update-backend"; @@ -32,8 +33,9 @@ guardDesktopProcessOutput(); let mainWindow: BrowserWindow | null = null; let tray: Tray | null = null; -let agentStartup: Promise | null = null; -let daemonPaths: { stateDir: string; endpoint: string } | null = null; +let bootGate: BootGate; +let localHealthLabel = "Local connector starting"; +let daemonPaths: DaemonPaths | null = null; let updater: UpdateCoordinator | null = null; let quitting = false; const activePairings = new Map(); @@ -77,19 +79,7 @@ async function resolveDaemonPaths(): Promise { timeout: 10_000, windowsHide: true }); - const paths = JSON.parse(stdout) as { state_dir?: unknown; endpoint?: unknown }; - if (typeof paths.state_dir !== "string" || typeof paths.endpoint !== "string") { - throw new Error("The connector runtime returned invalid path information."); - } - daemonPaths = { stateDir: paths.state_dir, endpoint: paths.endpoint }; -} - -async function ensureAgent(): Promise { - if (agentStartup) return agentStartup; - agentStartup = startAgent().finally(() => { - agentStartup = null; - }); - return agentStartup; + daemonPaths = parseDaemonPaths(JSON.parse(stdout)); } async function requestReadyAgent( @@ -97,8 +87,7 @@ async function requestReadyAgent( params?: unknown, timeoutMs = 5_000 ): Promise { - await ensureAgent(); - return requestAgent(controlEndpoint(), method, params, timeoutMs); + return bootGate.request(() => requestAgent(controlEndpoint(), method, params, timeoutMs)); } function endpointIsUnavailable(error: unknown): boolean { @@ -113,6 +102,7 @@ function incompatibleDaemon(error: unknown): boolean { async function startAgent(): Promise { await ensureAgentReady({ + expectedVersion: app.getVersion(), ping: (timeoutMs) => requestAgent(controlEndpoint(), "ping", undefined, timeoutMs), endpointIsUnavailable, @@ -126,7 +116,7 @@ async function startAgent(): Promise { await execFile( binary, daemonCliArguments( - app.isPackaged, + daemonPaths!.target, stateDirectory(), controlEndpoint(), ["start"] @@ -176,12 +166,12 @@ function registerIpc(): void { ipcMain.handle("connect:updates:check", async (event) => { trustedIpc(event); if (!updater) throw new Error("The updater has not been initialized."); - return updater.check(true); + return bootGate.check(() => updater!.check(true)); }); ipcMain.handle("connect:updates:install", async (event) => { trustedIpc(event); if (!updater) throw new Error("The updater has not been initialized."); - return updater.install(); + return bootGate.install(() => updater!.install()); }); ipcMain.handle("connect:collections:list", async (event) => { trustedIpc(event); @@ -878,7 +868,7 @@ function refreshTrayMenu(): void { Menu.buildFromTemplate([ { label: "Show mdbase connect", click: () => mainWindow?.show() }, { type: "separator" }, - { label: "Local connector running", enabled: false }, + { label: localHealthLabel, enabled: false }, { label: updateReady ? update?.phase === "ready" @@ -889,14 +879,14 @@ function refreshTrayMenu(): void { click: () => { if (!updater) return; if (updateReady) { - void updater.install().catch((error) => { + void bootGate.install(() => updater!.install()).catch((error) => { dialog.showErrorBox( "mdbase connect could not install the update", error instanceof Error ? error.message : String(error) ); }); } else { - void updater.check(true); + void bootGate.check(() => updater!.check(true)).catch((error) => dialog.showErrorBox("Could not check for updates", String(error))); } } }, @@ -939,11 +929,33 @@ app.whenReady().then(async () => { userDataDirectory: app.getPath("userData"), binaryPath: connectBinary, stateDirectory, + target: () => daemonPaths!.target, endpoint: controlEndpoint }) ); - const recoveryStatus = await updater.initialize(); + bootGate = new BootGate({ + initialize: async () => { await updater!.initialize(); }, + start: async () => { + localHealthLabel = "Local connector starting"; + refreshTrayMenu(); + try { + await startAgent(); + localHealthLabel = "Local connector ready"; + } catch (error) { + localHealthLabel = error instanceof Error ? error.message : "Local connector needs attention"; + throw error; + } finally { + refreshTrayMenu(); + } + }, + blockedReason: () => updater!.daemonStartupBlock() + }); updater.subscribe((status) => { + if (status.phase === "installing" || status.phase === "recovery") { + localHealthLabel = "Local connector updating"; + } else if (status.phase === "failed" && !status.can_check) { + localHealthLabel = "Local connector needs attention"; + } mainWindow?.webContents.send("connect:update-status", status); refreshTrayMenu(); }); @@ -952,17 +964,18 @@ app.whenReady().then(async () => { createTray(); handleDeepLink(process.argv.find((value) => value.startsWith("mdbase-connect://"))); try { - if (recoveryStatus.phase === "failed") throw new Error(recoveryStatus.message); - await ensureAgent(); + await bootGate.ready(); } catch (error) { + localHealthLabel = "Local connector needs attention"; + refreshTrayMenu(); dialog.showErrorBox( "mdbase connect could not start", error instanceof Error ? error.message : String(error) ); } - const initialUpdateCheck = setTimeout(() => void updater?.check(false), 30_000); + const initialUpdateCheck = setTimeout(() => void bootGate.check(() => updater!.check(false)).catch(() => undefined), 30_000); initialUpdateCheck.unref(); - const updateChecks = setInterval(() => void updater?.check(false), 6 * 60 * 60 * 1000); + const updateChecks = setInterval(() => void bootGate.check(() => updater!.check(false)).catch(() => undefined), 6 * 60 * 60 * 1000); updateChecks.unref(); }); diff --git a/apps/desktop/src/renderer/connection-state.mts b/apps/desktop/src/renderer/connection-state.mts index 50832531b..69b23c8d9 100644 --- a/apps/desktop/src/renderer/connection-state.mts +++ b/apps/desktop/src/renderer/connection-state.mts @@ -3,6 +3,7 @@ export type ConnectionDotState = "connected" | "connecting" | "paused" | "danger export interface ConnectionStatus { state: "local_only" | "connecting" | "connected" | "offline"; paused: boolean; + relay_problem?: string; } export interface CloudConnection { @@ -28,6 +29,15 @@ export function presentConnection( if (status?.paused) { return { label: "Remote access paused", settingsLabel: "Paused", dot: "paused" }; } + if (status?.relay_problem) { + return { + label: status.relay_problem === "authentication_required" + ? "Account connection needs authorization; reconnect this computer" + : "Relay version incompatible; update the connector", + settingsLabel: "Needs attention", + dot: "danger" + }; + } if (status === null || status.state === "connecting" || status.state === "local_only") { return { label: "Connecting securely…", settingsLabel: "Connecting", dot: "connecting" }; } diff --git a/apps/desktop/src/renderer/global.d.ts b/apps/desktop/src/renderer/global.d.ts index a06c0ab78..bf7c195a7 100644 --- a/apps/desktop/src/renderer/global.d.ts +++ b/apps/desktop/src/renderer/global.d.ts @@ -1,4 +1,6 @@ interface AgentStatus { + readiness?: import("../shared/readiness").AgentReadiness; + relay_problem?: string; protocol_version: number; binary_version?: string; state: "local_only" | "connecting" | "connected" | "offline"; @@ -363,7 +365,7 @@ interface Window { renameComputer(name: string): Promise<{ connector: { id: string; name: string } }>; createGrant(input: { applicationId: string; collectionId: string; operations: string[] }): Promise; updateGrant(input: { grantId: string; operations: string[] }): Promise; - revokeGrant(grantId: string): Promise; + revokeGrant(grantId: string): Promise<{ ok: boolean; revocation_status: "revoking" | "revoked" }>; listActivity(limit?: number): Promise; hostedSnapshot(): Promise; createHostedCollection(input: { name: string; timezone: string }): Promise<{ collection: HostedCollectionSummary }>; diff --git a/apps/desktop/src/shared/readiness.ts b/apps/desktop/src/shared/readiness.ts new file mode 100644 index 000000000..4965417c9 --- /dev/null +++ b/apps/desktop/src/shared/readiness.ts @@ -0,0 +1,32 @@ +export interface AgentReadiness { + schema_version: number; + ready: boolean; + binary_version: string; + safe_reason?: string; +} + +export function presentReadiness(health: AgentReadiness | undefined, expectedVersion?: string): { + state: "starting" | "ready" | "attention"; + reason?: string; + label: string; +} { + if (!health || health.schema_version !== 1 || typeof health.ready !== "boolean" || + typeof health.binary_version !== "string" || + (expectedVersion !== undefined && health.binary_version !== expectedVersion)) { + return { state: "attention", reason: "incompatible_version", label: "Update or restart the local connector" }; + } + if (health.ready === true && !health.safe_reason) { + return { state: "ready", label: "Local connector ready" }; + } + const labels: Record = { + starting: "Local connector starting", + initialization_failed: "Local connector initialization failed; restart the connector", + critical_worker_failed: "Local connector worker stopped; restart the connector", + credential_store_unavailable: "Unlock the credential store, then restart the connector" + }; + return { + state: health.safe_reason === "starting" ? "starting" : "attention", + reason: health.safe_reason ?? "invalid_readiness", + label: labels[health.safe_reason ?? ""] ?? "Local connector needs attention" + }; +} diff --git a/apps/desktop/test/agent-startup.test.mjs b/apps/desktop/test/agent-startup.test.mjs index a8f4020eb..1907d924f 100644 --- a/apps/desktop/test/agent-startup.test.mjs +++ b/apps/desktop/test/agent-startup.test.mjs @@ -9,8 +9,13 @@ function unavailable() { return Object.assign(new Error("No local connector"), { code: "ENOENT" }); } +const pingResult = (ready = true) => ({ + pong: true, ready, + readiness: { schema_version: 1, ready, binary_version: "test", ...(ready ? {} : { safe_reason: "starting" }) } +}); const options = (overrides = {}) => ({ - ping: async () => ({ pong: true, ready: true }), + expectedVersion: "test", + ping: async () => pingResult(), launch: async () => {}, endpointIsUnavailable: (error) => error && typeof error === "object" && ["ENOENT", "ECONNREFUSED"].includes(error.code), @@ -30,7 +35,7 @@ test("a slow daemon keeps initializing after its launch command times out", asyn pingCount += 1; if (pingCount === 1) throw unavailable(); if (pingCount === 2) throw new Error("The local connector did not respond in time."); - return { pong: true, ready: pingCount >= 4 }; + return pingResult(pingCount >= 4); }, launch: async () => { throw launchError; diff --git a/apps/desktop/test/boot-gate.test.mjs b/apps/desktop/test/boot-gate.test.mjs new file mode 100644 index 000000000..ba5ce00ab --- /dev/null +++ b/apps/desktop/test/boot-gate.test.mjs @@ -0,0 +1,99 @@ +import assert from "node:assert/strict"; +import { createRequire } from "node:module"; +import test from "node:test"; +const { BootGate } = createRequire(import.meta.url)("../dist/main/boot-gate.js"); + +function deferred() { + let resolve; + const promise = new Promise((done) => { resolve = done; }); + return { promise, resolve }; +} + +test("all IPC waits for one recovery and one concurrent startup", async () => { + const recovery = deferred(); + const start = deferred(); + const events = []; + const gate = new BootGate({ + async initialize() { events.push("recover"); await recovery.promise; }, + async start() { events.push("start"); await start.promise; }, + blockedReason: () => null + }); + const requests = [1, 2, 3].map((id) => gate.request(async () => events.push(id))); + assert.deepEqual(events, ["recover"]); + recovery.resolve(); + await new Promise(setImmediate); + assert.deepEqual(events, ["recover", "start"]); + start.resolve(); + await Promise.all(requests); + assert.deepEqual(events, ["recover", "start", 1, 2, 3]); +}); + +test("update checks cannot race persisted recovery or start a daemon", async () => { + const recovery = deferred(); + let blocked = null; + let checks = 0; + const gate = new BootGate({ + async initialize() { await recovery.promise; }, + async start() { assert.fail("checking updates must not require daemon startup"); }, + blockedReason: () => blocked + }); + const checking = gate.check(async () => ++checks); + await new Promise(setImmediate); + assert.equal(checks, 0); + recovery.resolve(); + assert.equal(await checking, 1); + blocked = "recovery failed"; + await assert.rejects(gate.check(async () => assert.fail()), /recovery failed/); +}); + +test("failed boot stays closed to later IPC without replaying initialization", async () => { + let attempts = 0; + const gate = new BootGate({ + async initialize() { attempts++; throw new Error("rollback failed"); }, + async start() { assert.fail("must not start"); }, + blockedReason: () => null + }); + for (let i = 0; i < 3; i++) await assert.rejects(gate.request(async () => assert.fail()), /rollback failed/); + await assert.rejects(gate.check(async () => assert.fail()), /rollback failed/); + assert.equal(attempts, 1); +}); + +test("install waits for admitted startup and blocks racing IPC", async () => { + const start = deferred(); + const install = deferred(); + let blocked = null; + let installed = false; + const gate = new BootGate({ + async initialize() {}, + async start() { await start.promise; }, + blockedReason: () => blocked + }); + const request = gate.request(async () => assert.fail("must not send after install admission")); + const rejected = assert.rejects(request, /update is in progress/); + await new Promise(setImmediate); + const installing = gate.install(async () => { + installed = true; + await install.promise; + blocked = "recovery failed"; + throw new Error("install failed"); + }); + await new Promise(setImmediate); + assert.equal(installed, false); + await assert.rejects(gate.request(async () => assert.fail()), /update is in progress/); + await assert.rejects(gate.install(async () => assert.fail()), /update is in progress/); + await assert.rejects(gate.check(async () => assert.fail()), /update is in progress/); + start.resolve(); + await rejected; + await new Promise(setImmediate); + assert.equal(installed, true); + const failed = assert.rejects(installing, /install failed/); + install.resolve(); + await failed; + await assert.rejects(gate.ready(), /recovery failed/); +}); + +test("verified recovery after failed installation permits ordinary IPC", async () => { + const gate = new BootGate({ async initialize() {}, async start() {}, blockedReason: () => null }); + await assert.rejects(gate.install(async () => { throw new Error("restored old runtime"); }), /restored/); + assert.equal(await gate.request(async () => "ok"), "ok"); +}); diff --git a/apps/desktop/test/daemon-lifecycle.test.mjs b/apps/desktop/test/daemon-lifecycle.test.mjs index 43edfa0c2..949dea8d8 100644 --- a/apps/desktop/test/daemon-lifecycle.test.mjs +++ b/apps/desktop/test/daemon-lifecycle.test.mjs @@ -1,61 +1,31 @@ import assert from "node:assert/strict"; import { createRequire } from "node:module"; import test from "node:test"; +const { connectCliEnvironment, daemonCliArguments, parseDaemonPaths } = createRequire(import.meta.url)("../dist/main/daemon-lifecycle.js"); -const require = createRequire(import.meta.url); -const { - connectCliEnvironment, - daemonCliArguments -} = require("../dist/main/daemon-lifecycle.js"); +test("default daemon commands preserve CLI service targeting without inferred overrides", () => { + const paths = parseDaemonPaths({ state_dir: "/default/state", endpoint: "/default/socket", target: "installed_service" }); + assert.deepEqual(daemonCliArguments(paths.target, paths.stateDir, paths.endpoint, ["start"]), ["connect", "daemon", "start"]); + assert.deepEqual(daemonCliArguments(paths.target, paths.stateDir, paths.endpoint, ["status"], true), ["--json", "connect", "daemon", "status"]); +}); -test("packaged daemon commands target the installed service", () => { - assert.deepEqual( - daemonCliArguments( - true, - "/tmp/isolated-state", - "/tmp/isolated.sock", - ["start"] - ), - ["connect", "daemon", "start"] - ); - assert.deepEqual( - daemonCliArguments( - true, - "/tmp/isolated-state", - "/tmp/isolated.sock", - ["status"], - true - ), - ["--json", "connect", "daemon", "status"] - ); +test("explicit isolated profile never controls the default installed service", () => { + const paths = parseDaemonPaths({ state_dir: "/isolated/state", endpoint: "/isolated/socket", target: "isolated_profile" }); + for (const command of ["start", "stop", "status"]) { + assert.deepEqual(daemonCliArguments(paths.target, paths.stateDir, paths.endpoint, [command]), [ + "--state-dir", "/isolated/state", "--endpoint", "/isolated/socket", "connect", "daemon", command + ]); + } }); -test("development daemon commands retain their isolated profile", () => { - assert.deepEqual( - daemonCliArguments( - false, - "/tmp/isolated-state", - "/tmp/isolated.sock", - ["start"] - ), - [ - "--state-dir", - "/tmp/isolated-state", - "--endpoint", - "/tmp/isolated.sock", - "connect", - "daemon", - "start" - ] - ); +test("missing or unknown profile information cannot silently select a lifecycle owner", () => { + for (const target of [undefined, "detached", "unknown"]) { + assert.throws(() => parseDaemonPaths({ state_dir: "/state", endpoint: "/socket", target }), /invalid path/); + } }); test("packaged CLI calls cannot inherit isolated-profile selectors", () => { - const environment = { - PATH: "/usr/bin", - MDBASE_CONNECT_HOME: "/tmp/isolated-state", - MDBASE_CONNECT_SOCKET: "/tmp/isolated.sock" - }; + const environment = { PATH: "/usr/bin", MDBASE_CONNECT_HOME: "/isolated/state", MDBASE_CONNECT_SOCKET: "/isolated/socket" }; assert.deepEqual(connectCliEnvironment(true, environment), { PATH: "/usr/bin" }); assert.equal(connectCliEnvironment(false, environment), environment); }); diff --git a/apps/desktop/test/readiness.test.mjs b/apps/desktop/test/readiness.test.mjs new file mode 100644 index 000000000..3c92cb84f --- /dev/null +++ b/apps/desktop/test/readiness.test.mjs @@ -0,0 +1,30 @@ +import assert from "node:assert/strict"; +import { createRequire } from "node:module"; +import test from "node:test"; +const { ensureAgentReady, presentReadiness } = createRequire(import.meta.url)("../dist/main/agent-startup.js"); + +const health = { schema_version: 1, ready: true, binary_version: "expected" }; +for (const [name, value, state] of [ + ["ready", health, "ready"], + ["missing", undefined, "attention"], + ["future schema", { ...health, schema_version: 2 }, "attention"], + ["wrong version", { ...health, binary_version: "old" }, "attention"], + ["missing ready", { ...health, ready: undefined }, "attention"], + ["starting", { ...health, ready: false, safe_reason: "starting" }, "starting"], + ...["initialization_failed", "critical_worker_failed", "credential_store_unavailable"].map((reason) => + [reason, { ...health, ready: false, safe_reason: reason }, "attention"]) +]) { + test(`canonical readiness: ${name}`, async () => { + assert.equal(presentReadiness(value, "expected").state, state); + if (state !== "attention") return; + let probes = 0; + await assert.rejects(ensureAgentReady({ + expectedVersion: "expected", + async ping() { probes++; return { pong: true, ready: true, readiness: value }; }, + async launch() { assert.fail("must not launch over incompatible or failed daemon"); }, + endpointIsUnavailable: () => false, + incompatibleDaemon: () => false + })); + assert.equal(probes, 1); + }); +} diff --git a/apps/desktop/tsconfig.main.json b/apps/desktop/tsconfig.main.json index c077f22ed..2b16cf290 100644 --- a/apps/desktop/tsconfig.main.json +++ b/apps/desktop/tsconfig.main.json @@ -5,10 +5,10 @@ "moduleResolution": "Node16", "strict": true, "esModuleInterop": true, - "outDir": "dist/main", - "rootDir": "src/main", + "noEmit": true, + "rootDir": "src", "types": ["node"], "skipLibCheck": true }, - "include": ["src/main/**/*.ts"] + "include": ["src/main/**/*.ts", "src/shared/**/*.ts"] } From 19e9c3d9a69a1eaecc3e3fc37929889217a2cc88 Mon Sep 17 00:00:00 2001 From: callumalpass Date: Sun, 13 Sep 2026 19:46:57 +1000 Subject: [PATCH 07/16] fix(desktop): retain resource inventories and action errors during refresh --- apps/desktop/src/main/hosted-snapshot.ts | 22 ++--- apps/desktop/src/renderer/main.tsx | 80 ++++++++++++------- apps/desktop/src/renderer/resource-health.mts | 21 +++++ apps/desktop/test/hosted-snapshot.test.mjs | 12 +-- apps/desktop/test/resource-health.test.mjs | 32 ++++++++ 5 files changed, 115 insertions(+), 52 deletions(-) create mode 100644 apps/desktop/src/renderer/resource-health.mts create mode 100644 apps/desktop/test/resource-health.test.mjs diff --git a/apps/desktop/src/main/hosted-snapshot.ts b/apps/desktop/src/main/hosted-snapshot.ts index 3352c9a3a..968d9a2b2 100644 --- a/apps/desktop/src/main/hosted-snapshot.ts +++ b/apps/desktop/src/main/hosted-snapshot.ts @@ -12,24 +12,15 @@ const credentialStoreUnavailable = (error: unknown): boolean => ( && error.code === "credential_store_unavailable" ); -const offlineSnapshot = (): HostedControlSnapshot => ({ - online: false, - hosted_collections_available: false, - hosted_collections: [], - grants: [], - pending_authorizations: [] -}); - interface HostedSnapshotLoaderOptions { retryAfterMs?: number; now?: () => number; } /** - * A hosted snapshot is status data, so a known unavailable credential store is - * represented as an offline snapshot rather than a rejected Electron IPC call. - * Repeated polls are served locally during a short retry cooldown. Other - * failures remain visible to the renderer and preserve its last snapshot. + * Preserve failure as failure, never manufacture an empty success. The renderer + * owns last-known data. A credential error is cached only for a short cooldown, + * replacing repeated keyring pressure without becoming a second data cache. */ export function createHostedSnapshotLoader( request: () => Promise, @@ -38,21 +29,24 @@ export function createHostedSnapshotLoader( const retryAfterMs = options.retryAfterMs ?? 30_000; const now = options.now ?? Date.now; let retryAt = 0; + let credentialError: unknown; let inFlight: Promise | undefined; return () => { - if (now() < retryAt) return Promise.resolve(offlineSnapshot()); + if (now() < retryAt) return Promise.reject(credentialError); if (inFlight) return inFlight; const pending = (async () => { try { const snapshot = await request(); retryAt = 0; + credentialError = undefined; return snapshot; } catch (error) { if (!credentialStoreUnavailable(error)) throw error; retryAt = now() + retryAfterMs; - return offlineSnapshot(); + credentialError = error; + throw error; } })(); const tracked = pending.finally(() => { diff --git a/apps/desktop/src/renderer/main.tsx b/apps/desktop/src/renderer/main.tsx index 9cf05cfa4..6415651f0 100644 --- a/apps/desktop/src/renderer/main.tsx +++ b/apps/desktop/src/renderer/main.tsx @@ -33,6 +33,8 @@ import { NotificationAccess, RequestPermissionChoices } from "./authorization-co import { hasSupportedCapabilityDeclaration, requestCapabilityGroups } from "./application-capabilities"; import { ConnectionProgress, Overview } from "./overview-view"; import { singleFlight } from "./single-flight.mjs"; +import { refreshResources, presentResourceFailures, retainOfflineInventory } from "./resource-health.mjs"; +import { presentReadiness } from "../shared/readiness"; import { AccessControl, Empty, @@ -99,6 +101,7 @@ function App() { const [activity, setActivity] = useState([]); const [busy, setBusy] = useState(false); const [error, setError] = useState(null); + const [resourceFailures, setResourceFailures] = useState>({}); const [notice, setNotice] = useState(null); const [createOpen, setCreateOpen] = useState(false); const [copiedCollectionPath, setCopiedCollectionPath] = useState(null); @@ -112,27 +115,36 @@ function App() { const [initialRefreshComplete, setInitialRefreshComplete] = useState(false); const [navigationOpen, setNavigationOpen] = useState(false); - const runRefresh = useCallback(async (quiet = false) => { - try { - const results = await Promise.allSettled([ - window.mdbaseConnect.status().then(setStatus), - window.mdbaseConnect.updateStatus().then(setUpdateStatus), - window.mdbaseConnect.listCollections().then(setCollections), - window.mdbaseConnect.getLaunchAtLogin().then(setStartup), - window.mdbaseConnect.getCloudConfig().then(setCloud), - window.mdbaseConnect.accessSnapshot().then(setAccess), - window.mdbaseConnect.listActivity(100).then(setActivity), - window.mdbaseConnect.hostedSnapshot().then(setHosted).catch(() => { - setHosted((current) => ({ ...current, online: false })); - }), - window.mdbaseConnect.listMirrors().then(setMirrors) - ]); - const failed = results.find((result): result is PromiseRejectedResult => result.status === "rejected"); - if (failed) throw failed.reason; - setError(null); - } catch (refreshError) { - if (!quiet) setError(message(refreshError)); + const runRefresh = useCallback(async (_quiet = false) => { + let configured: boolean | undefined; + const failures = await refreshResources({ + connector: () => window.mdbaseConnect.status().then((next) => { + setStatus(next); + const health = presentReadiness(next.readiness); + if (health.state !== "ready") throw new Error(health.label); + }), + collections: () => window.mdbaseConnect.listCollections().then(setCollections), + startup: () => window.mdbaseConnect.getLaunchAtLogin().then(setStartup), + account: () => window.mdbaseConnect.getCloudConfig().then((next) => { + configured = next.configured; + setCloud(next); + }), + access: () => window.mdbaseConnect.accessSnapshot().then((next) => { + setAccess((current) => !next.configured ? next : retainOfflineInventory(current, next)); + if (next.configured && !next.online) throw new Error("Application access is offline."); + }), + activity: () => window.mdbaseConnect.listActivity(100).then(setActivity), + hosted: () => window.mdbaseConnect.hostedSnapshot().then((next) => { + setHosted((current) => retainOfflineInventory(current, next)); + if (!next.online) throw new Error("Hosted collections are offline."); + }), + mirrors: () => window.mdbaseConnect.listMirrors().then(setMirrors) + }); + if (configured === false) { + setHosted({ online: false, hosted_collections_available: false, hosted_collections: [], grants: [], pending_authorizations: [] }); + delete failures.hosted; } + setResourceFailures(failures); setInitialRefreshComplete(true); }, []); const refresh = useMemo(() => singleFlight(runRefresh), [runRefresh]); @@ -163,7 +175,15 @@ function App() { setRoute(next as Route); } }); - const removeUpdateStatus = window.mdbaseConnect.onUpdateStatus(setUpdateStatus); + // Updates already have a push subscription; do not poll them with daemon data. + let updatePushed = false; + const removeUpdateStatus = window.mdbaseConnect.onUpdateStatus((next) => { + updatePushed = true; + setUpdateStatus(next); + }); + void window.mdbaseConnect.updateStatus().then((next) => { + if (!updatePushed) setUpdateStatus(next); + }).catch(() => undefined); return () => { window.clearInterval(timer); removeNavigation(); @@ -194,7 +214,6 @@ function App() { async function act(action: () => Promise) { setBusy(true); - setError(null); setNotice(null); try { await action(); @@ -207,7 +226,6 @@ function App() { } async function transferAct(action: () => Promise) { - setError(null); setNotice(null); try { await action(); @@ -344,7 +362,8 @@ function App() {
- {error &&
{error}
} + {error &&
{error}
} + {presentResourceFailures(resourceFailures) &&
{presentResourceFailures(resourceFailures)}
} {notice &&
{notice}
}
@@ -620,11 +639,12 @@ function ApplicationGrantGroup({ group, busy, onAct, onNotice }: { const result = await window.mdbaseConnect.revokeHostedGrant(grant.id); providerConfirmationPending ||= result.revocation_status === "revoking"; } else { - await window.mdbaseConnect.revokeGrant(grant.id); + const result = await window.mdbaseConnect.revokeGrant(grant.id); + providerConfirmationPending ||= result.revocation_status !== "revoked"; } } onNotice(providerConfirmationPending - ? `${group.applicationName} access is disabled here; hosted revocation confirmation is pending.` + ? `${group.applicationName} revocation is pending authority confirmation.` : `${group.applicationName} collection access was revoked.`); }); }}>Revoke all access @@ -649,7 +669,7 @@ function GrantEditor({ grant, busy, onAct, onNotice }: { grant: GrantSummary; bu useEffect(() => setOperations(grant.operations), [grant.operations]); const authority = grant.collection_kind === "hosted" ? "Hosted by mdbase" : "On this computer"; if (grant.revocation_status === "revoking") { - return

Hosted by mdbase

{grant.collection_name}

Access is disabled here. Waiting for the hosted authority to confirm revocation.
Revoking…
; + return

{authority}

{grant.collection_name}

Revocation is pending. Waiting for {grant.collection_kind === "hosted" ? "the hosted authority" : "this computer"} to confirm enforcement.
Revoking…
; } if (permissionDetailsAvailable && (grant.scope.access !== "full_collection" || grant.scope.contracts.length > 0)) { return

{authority}

{grant.collection_name}

Legacy scoped access is revoked. Reauthorize this application for the entire collection.
Reauthorization required
; @@ -676,8 +696,10 @@ function GrantEditor({ grant, busy, onAct, onNotice }: { grant: GrantSummary; bu ? `${grant.application_name} access is disabled here; hosted revocation confirmation is pending.` : `${grant.application_name} access was revoked.`); } else { - await window.mdbaseConnect.revokeGrant(grant.id); - onNotice(`${grant.application_name} access was revoked.`); + const result = await window.mdbaseConnect.revokeGrant(grant.id); + onNotice(result.revocation_status === "revoked" + ? `${grant.application_name} access was revoked.` + : `${grant.application_name} revocation is pending confirmation from this computer.`); } }); }}>Revoke }; } diff --git a/apps/editor/src/demo-gateway.ts b/apps/editor/src/demo-gateway.ts index 59a4e5a80..af206d6bb 100644 --- a/apps/editor/src/demo-gateway.ts +++ b/apps/editor/src/demo-gateway.ts @@ -36,6 +36,10 @@ import type { } from "./model"; export class DemoCollectionGateway implements CollectionGateway { + pendingNoteMutations(): readonly import("@mdbase-dev/connect").PendingMutationSummary[] { return []; } + async recoverNoteMutation(_requestId: string): Promise { + throw new Error("No interrupted note operation is available."); + } private notes: NoteDocument[]; private files: CollectionFile[] = demoFiles(); private fileContents = new Map(demoFileContents()); diff --git a/apps/editor/src/gateway.test.ts b/apps/editor/src/gateway.test.ts index 35c034968..490cf3242 100644 --- a/apps/editor/src/gateway.test.ts +++ b/apps/editor/src/gateway.test.ts @@ -87,6 +87,46 @@ describe("ConnectCollectionGateway collection index", () => { }); describe("ConnectCollectionGateway recovery operations", () => { + it("recovers a committed autosave by original ID without touching unrelated pending work", async () => { + const saved = { ...summary("note.md"), revision: "r2", body: "Accepted" } as NoteDocument; + const recover = vi.fn(async () => connectSuccess(saved)); + const unrelated = vi.fn(); + const pendingMutation = vi.fn((id: string) => id === "original" ? { operation: "update", recover } : null); + const update = vi.fn(async () => connectFailure(connectProblem("operation_outcome_unknown", "Response lost", { + operationOutcome: "unknown", details: { request_id: "original" } + }))); + const gateway = new ConnectCollectionGateway("https://connect.example"); + injectConnection(gateway, { + update, pendingMutation, + pendingMutations: () => [{ requestId: "unrelated-file", operation: "file_control:delete", recover: unrelated }] + }); + await expect(gateway.update({ path: "note.md", revision: "r1", frontmatter: {}, title: "Note", body: "Accepted", source: { kind: "heading" } })).resolves.toEqual(saved); + expect(update).toHaveBeenCalledOnce(); + expect(pendingMutation).toHaveBeenCalledExactlyOnceWith("original"); + expect(recover).toHaveBeenCalledOnce(); + expect(unrelated).not.toHaveBeenCalled(); + }); + + it("fresh gateways expose durable pending work and block reconstructed updates or renames", async () => { + const saved = { ...summary("renamed.md"), revision: "r2" } as NoteDocument; + const recover = vi.fn(async () => connectSuccess(saved)); + const pending = { requestId: "original", operation: "rename", recover }; + const update = vi.fn(); + const renameWithProgress = vi.fn(); + const connection = { update, renameWithProgress, pendingMutations: () => [pending], pendingMutation: (id: string) => id === "original" ? pending : null }; + const first = new ConnectCollectionGateway("https://connect.example"); + injectConnection(first, connection); + const reloaded = new ConnectCollectionGateway("https://connect.example"); + injectConnection(reloaded, connection); + expect(reloaded.pendingNoteMutations()).toMatchObject([{ requestId: "original", operation: "rename" }]); + await expect(reloaded.rename("note.md", "changed.md", "r1")).rejects.toThrow("No new write"); + await expect(reloaded.update({ path: "note.md", revision: "r1", frontmatter: {}, title: "Changed", body: "Changed", source: { kind: "heading" } })).rejects.toThrow("No new write"); + await expect(reloaded.recoverNoteMutation("missing")).rejects.toThrow("No new write"); + await expect(reloaded.recoverNoteMutation("original")).resolves.toEqual(saved); + expect(recover).toHaveBeenCalledOnce(); + expect(update).not.toHaveBeenCalled(); + expect(renameWithProgress).not.toHaveBeenCalled(); + }); it("checks and requests direct access through the active SDK connection", async () => { let directAccess: DirectAccessStatus = "permission_required"; const checkDirectAccess = vi.fn(async () => "permission_required" as const); @@ -430,6 +470,8 @@ function injectConnection( route?: "remote" | "direct" | "relay"; directAccess?: "disabled" | "permission_required" | "checking" | "available" | "unavailable" | "denied"; authorizationCapabilities?: () => { missingOperations: string[] }; + pendingMutations?: MdbaseConnection["pendingMutations"]; + pendingMutation?: MdbaseConnection["pendingMutation"]; }; bound.collectionId ??= "collection"; bound.displayName ??= "Notes"; @@ -437,6 +479,8 @@ function injectConnection( bound.route ??= "relay"; bound.directAccess ??= "unavailable"; bound.authorizationCapabilities ??= () => ({ missingOperations: [] }); + bound.pendingMutations ??= () => []; + bound.pendingMutation ??= () => null; for (const name of [ "checkDirectAccess", "requestDirectAccess", "read", "create", "update", "renameWithProgress", "preflightRename", "preflightDelete", "assessTypePack", "applyTypePack" diff --git a/apps/editor/src/gateway.ts b/apps/editor/src/gateway.ts index 291bc4f95..3a249afea 100644 --- a/apps/editor/src/gateway.ts +++ b/apps/editor/src/gateway.ts @@ -265,22 +265,57 @@ export class ConnectCollectionGateway implements CollectionGateway { })); } + pendingNoteMutations() { + return this.activeConnection()?.pendingMutations() + .filter((pending) => pending.operation === "update" || pending.operation === "rename") ?? []; + } + + async recoverNoteMutation(requestId: string): Promise { + const pending = this.requireConnection().pendingMutation(requestId); + if (!pending || (pending.operation !== "update" && pending.operation !== "rename")) { + throw new Error("The exact interrupted note operation is unavailable. No new write was attempted."); + } + return requireOutcome(await pending.recover()); + } + + private assertNoPendingNoteMutation(connection: MdbaseConnection): void { + if (connection.pendingMutations().some((pending) => pending.operation === "update" || pending.operation === "rename")) { + throw new Error("Recover the interrupted note operation before making another change. No new write was attempted."); + } + } + async update(input: SaveNoteInput): Promise { - return requireOutcome(await this.requireConnection().update({ + const connection = this.requireConnection(); + this.assertNoPendingNoteMutation(connection); + const outcome = await connection.update({ path: input.path, patch: titlePatch(input.title, input.source, input.frontmatter), body: persistedBody(input.title, input.body, input.source), ifRevision: input.revision, includeDocument: true - })); + }); + if (!outcome.ok && outcome.problem.code === "operation_outcome_unknown") { + const pending = connection.pendingMutation(outcome.problem.details.request_id); + // One exact continuation, not a transport retry or a newly constructed update. + if (pending?.operation === "update") { + const recovered = await pending.recover().catch(() => outcome); + if (recovered.ok) return recovered.value; + } + // Preserve the original unknown identity even if its recovery is offline. + } + return requireOutcome(outcome); } async updateProperties(path: string, patch: JsonObject, revision: string): Promise { - return requireOutcome(await this.requireConnection().update({ path, patch, ifRevision: revision, includeDocument: true })); + const connection = this.requireConnection(); + this.assertNoPendingNoteMutation(connection); + return requireOutcome(await connection.update({ path, patch, ifRevision: revision, includeDocument: true })); } async updateDocument(path: string, document: string, revision: string): Promise { - return requireOutcome(await this.requireConnection().update({ + const connection = this.requireConnection(); + this.assertNoPendingNoteMutation(connection); + return requireOutcome(await connection.update({ path, document, ifRevision: revision @@ -303,10 +338,12 @@ export class ConnectCollectionGateway implements CollectionGateway { } async rename(from: string, to: string, revision: string, updateRefs = true, options: MutationOperationOptions = {}): Promise { + const connection = this.requireConnection(); + this.assertNoPendingNoteMutation(connection); const key = mutationKey(from, to, revision); let retainPreflight = false; try { - return requireOutcome(await this.requireConnection().renameWithProgress({ + return requireOutcome(await connection.renameWithProgress({ from, to, ifRevision: revision, diff --git a/apps/editor/src/model.ts b/apps/editor/src/model.ts index e36227360..130cfebe2 100644 --- a/apps/editor/src/model.ts +++ b/apps/editor/src/model.ts @@ -155,6 +155,8 @@ export interface FileUploadRequest extends FileReadRequest { } export interface CollectionGateway { + pendingNoteMutations(): readonly import("@mdbase-dev/connect").PendingMutationSummary[]; + recoverNoteMutation(requestId: string): Promise; sessionSnapshot(): CollectionSessionSnapshot; startSession(): Promise; onSessionChange(listener: (snapshot: CollectionSessionSnapshot) => void): () => void; diff --git a/apps/editor/src/note-mutation-presentation.ts b/apps/editor/src/note-mutation-presentation.ts index ce8e8e3ed..49d4b55ad 100644 --- a/apps/editor/src/note-mutation-presentation.ts +++ b/apps/editor/src/note-mutation-presentation.ts @@ -21,6 +21,7 @@ export function updateMutationActivity( export function noteRowStatus(session: NoteSession): NoteRowStatus | undefined { if (session.deleted) return { label: "Deleting", tone: "busy", busy: true, disabled: true }; + if (session.pendingSave) return { label: "Recovery pending", tone: "error", busy: false }; if (session.remoteDocument) return { label: "Changed elsewhere", tone: "error", busy: false }; if (session.activity) { const labels: Record = { diff --git a/apps/editor/src/note-operation-coordinator.test.ts b/apps/editor/src/note-operation-coordinator.test.ts index 08de702eb..07d7686a7 100644 --- a/apps/editor/src/note-operation-coordinator.test.ts +++ b/apps/editor/src/note-operation-coordinator.test.ts @@ -1,5 +1,7 @@ import { describe, expect, it, vi } from "vitest"; import { NoteOperationCoordinator } from "./note-operation-coordinator"; +import { MdbaseConnectError } from "@mdbase-dev/connect"; +import { connectProblem } from "@mdbase-dev/connect-testing"; import { createNoteSession } from "./note-session"; import type { NoteDocument } from "./model"; @@ -16,6 +18,48 @@ function document(revision = "1"): NoteDocument { } describe("NoteOperationCoordinator", () => { + it("recovers the original autosave snapshot before saving changed input", async () => { + const unknown = new MdbaseConnectError(connectProblem("operation_outcome_unknown", "Response lost", { + operationOutcome: "unknown", details: { request_id: "original-update" } + })); + const update = vi.fn(async (input) => { + if (update.mock.calls.length === 1) throw unknown; + return { ...document("3"), body: input.body }; + }); + const recover = vi.fn(async () => ({ ...document("2"), body: "First" })); + const session = createNoteSession(document(), []); + const coordinator = new NoteOperationCoordinator({ update, recover, onSaved() {}, onSaveError() {}, onChange() {} }); + session.draft.body = "First"; + await expect(coordinator.requestSave(session)).rejects.toBe(unknown); + expect(session.saveState).toBe("recovery"); + session.draft.body = "Second"; + session.remoteDocument = { ...document("2"), body: "First" }; + await coordinator.requestSave(session); + expect(recover).toHaveBeenCalledExactlyOnceWith("original-update"); + expect(update).toHaveBeenCalledTimes(1); + expect(session.persistedDraft.body).toBe("First"); + expect(session.draft.body).toBe("Second"); + expect(session.remoteDocument).toBeUndefined(); + await coordinator.flush(session); + expect(update.mock.calls[1][0]).toMatchObject({ revision: "2", body: "Second" }); + expect(session.saveState).toBe("saved"); + }); + + it("failed recovery retains the pending identity and never calls update again", async () => { + const unknown = new MdbaseConnectError(connectProblem("operation_outcome_unknown", "Response lost", { + operationOutcome: "unknown", details: { request_id: "original-update" } + })); + const update = vi.fn(async () => { throw unknown; }); + const recover = vi.fn(async () => { throw new Error("offline"); }); + const session = createNoteSession(document(), []); + session.draft.body = "Accepted"; + const coordinator = new NoteOperationCoordinator({ update, recover, onSaved() {}, onSaveError() {}, onChange() {} }); + await expect(coordinator.requestSave(session)).rejects.toBe(unknown); + for (let i = 0; i < 2; i++) await expect(coordinator.requestSave(session)).rejects.toThrow("offline"); + expect(update).toHaveBeenCalledTimes(1); + expect(session.pendingSave).toMatchObject({ requestId: "original-update", draft: { body: "Accepted" } }); + expect(session.saveState).toBe("recovery"); + }); it("serializes a newer draft behind an in-flight save", async () => { let releaseFirst!: () => void; const firstBlocked = new Promise((resolve) => { releaseFirst = resolve; }); diff --git a/apps/editor/src/note-operation-coordinator.ts b/apps/editor/src/note-operation-coordinator.ts index d2a79dc53..9bb30cbc8 100644 --- a/apps/editor/src/note-operation-coordinator.ts +++ b/apps/editor/src/note-operation-coordinator.ts @@ -2,9 +2,11 @@ import type { NoteDocument, SaveNoteInput } from "./model"; import type { NoteSession } from "./note-session"; import { sessionDirty } from "./note-session"; import { KeyedOperationQueue } from "./operation-queue"; +import { pendingNoteRequestId } from "./pending-note-mutation"; interface NoteOperationCoordinatorOptions { update(input: SaveNoteInput): Promise; + recover?(requestId: string): Promise; onSaved(session: NoteSession, document: NoteDocument): void; onSaveError(session: NoteSession, error: unknown): void; onChange(session: NoteSession): void; @@ -18,7 +20,7 @@ export class NoteOperationCoordinator { requestSave(session: NoteSession): Promise { if (session.deleted) return Promise.resolve(); - if (session.remoteDocument) { + if (session.remoteDocument && !session.pendingSave) { session.saveState = "conflict"; this.options.onChange(session); return Promise.resolve(); @@ -27,7 +29,7 @@ export class NoteOperationCoordinator { if (sessionDirty(session)) session.saveAgain = true; return session.savePromise; } - if (!sessionDirty(session)) { + if (!sessionDirty(session) && !session.pendingSave) { session.saveState = "saved"; this.options.onChange(session); return Promise.resolve(); @@ -36,34 +38,41 @@ export class NoteOperationCoordinator { const promise = this.queue.run(session, async () => { do { session.saveAgain = false; - if (!sessionDirty(session) || session.deleted) break; - const snapshot = structuredClone(session.draft); + if ((!sessionDirty(session) && !session.pendingSave) || session.deleted) break; + const pending = session.pendingSave; + const snapshot = pending?.draft ?? structuredClone(session.draft); session.activity = "saving"; session.saveState = "saving"; this.options.onChange(session); try { - const document = await this.options.update({ + const document = pending + ? await this.recover(pending.requestId) + : await this.options.update({ path: session.document.path, revision: session.document.revision, frontmatter: session.document.frontmatter, ...snapshot }); + session.pendingSave = undefined; session.document = document; session.persistedDraft = snapshot; + if (session.remoteDocument?.revision === document.revision) session.remoteDocument = undefined; session.error = undefined; - session.saveState = sessionDirty(session) ? "waiting" : "saved"; + session.saveState = session.remoteDocument ? "conflict" : sessionDirty(session) ? "waiting" : "saved"; this.options.onSaved(session, document); this.options.onChange(session); } catch (error) { - session.saveState = "conflict"; + const requestId = pendingNoteRequestId(error); + if (!session.pendingSave && requestId) session.pendingSave = { requestId, draft: snapshot }; + session.saveState = session.pendingSave ? "recovery" : "conflict"; session.activity = undefined; this.options.onSaveError(session, error); this.options.onChange(session); throw error; } - } while (session.saveAgain && sessionDirty(session)); + } while (session.saveAgain && sessionDirty(session) && !session.remoteDocument); session.activity = undefined; - session.saveState = sessionDirty(session) ? "waiting" : "saved"; + session.saveState = session.remoteDocument ? "conflict" : sessionDirty(session) ? "waiting" : "saved"; this.options.onChange(session); }); @@ -76,7 +85,13 @@ export class NoteOperationCoordinator { return promise; } + private async recover(requestId: string): Promise { + if (!this.options.recover) throw new Error("Exact mutation recovery is unavailable. No new write was attempted."); + return this.options.recover(requestId); + } + async flush(session: NoteSession): Promise { + if (session.pendingSave) await this.requestSave(session); if (session.remoteDocument) throw new Error("Resolve the version changed elsewhere before continuing."); while (!session.deleted && !session.remoteDocument && (sessionDirty(session) || session.savePromise)) { await this.requestSave(session); diff --git a/apps/editor/src/note-session.ts b/apps/editor/src/note-session.ts index 42960370c..db7414b25 100644 --- a/apps/editor/src/note-session.ts +++ b/apps/editor/src/note-session.ts @@ -2,7 +2,7 @@ import type { CollectionTypeDescriptor } from "@mdbase-dev/connect"; import type { NoteDocument, TitleSource } from "./model"; import { editableNote } from "./note"; -export type SaveState = "saved" | "waiting" | "saving" | "conflict"; +export type SaveState = "saved" | "waiting" | "saving" | "conflict" | "recovery"; export type NoteActivity = "saving" | "properties" | "renaming" | "moving" | "deleting" | "validating"; export interface Draft { @@ -26,6 +26,7 @@ export interface NoteSession { deleted?: boolean; saveAgain?: boolean; savePromise?: Promise; + pendingSave?: { requestId: string; draft: Draft }; } let editorSessionSequence = 0; diff --git a/apps/editor/src/pending-note-mutation.ts b/apps/editor/src/pending-note-mutation.ts new file mode 100644 index 000000000..b6154ed72 --- /dev/null +++ b/apps/editor/src/pending-note-mutation.ts @@ -0,0 +1,68 @@ +import { MdbaseConnectError, type PendingMutationSummary } from "@mdbase-dev/connect"; +import type { CollectionGateway } from "./model"; +import type { NoteSession, NoteSessionStore } from "./note-session"; +import type { CollectionMutationScope } from "./collection-mutation-scope"; +import type { ToastItem } from "./Toasts"; +import { gatewayError } from "./gateway"; + +export interface RenamePlan { + session: NoteSession; + from: string; + to: string; + affectedPaths: string[]; + warnings: string[]; +} +export interface PendingRenameRecovery { + plan: RenamePlan; + updateRefs: boolean; + requestId: string; +} + +export function pendingNoteRequestId(error: unknown): string | undefined { + return error instanceof MdbaseConnectError && error.problem.code === "operation_outcome_unknown" + ? error.problem.details.request_id : undefined; +} + +export function pendingNoteToasts(pending: readonly PendingMutationSummary[], renameId: string | undefined, busy: boolean, recover: (id: string) => Promise): ToastItem[] { + return pending.filter((operation) => operation.requestId !== renameId).map((operation) => ({ + id: `pending-${operation.requestId}`, + message: `An interrupted ${operation.operation} from ${new Date(operation.createdAt).toLocaleString()} needs exact recovery. No new write will be attempted.`, + tone: "error", sticky: true, dismissible: false, + action: { label: `Recover ${operation.operation}`, busy, onAction: () => void recover(operation.requestId) } + })); +} + +export async function recoverPendingNoteOperation(requestId: string, context: { + busy: boolean; + scope: CollectionMutationScope; + sessions: NoteSessionStore; + gateway: CollectionGateway; + save(session: NoteSession): Promise; + rename?: PendingRenameRecovery; + resumeRename(plan: RenamePlan, updateRefs: boolean, requestId: string): Promise; + refresh(path: string): Promise; + reload(): Promise; + setBusy(busy: boolean): void; + onError(message: string): void; +}): Promise { + const { scope } = context; + if (context.busy || scope.isFrozen) return; + const token = scope.token(); + context.setBusy(true); + try { + const saving = [...context.sessions.values()].find((session) => session.pendingSave?.requestId === requestId); + if (saving) await context.save(saving); + else if (context.rename?.requestId === requestId) { + await context.resumeRename(context.rename.plan, context.rename.updateRefs, requestId); + } else { + const recovered = await scope.register(token, context.gateway.recoverNoteMutation(requestId)); + if (!scope.isCurrent(token)) return; + await context.refresh(recovered.path); + } + if (scope.isCurrent(token)) await context.reload(); + } catch (error) { + if (scope.isCurrent(token)) context.onError(gatewayError(error)); + } finally { + if (scope.isCurrent(token)) context.setBusy(false); + } +} From 8ceb5583a31ee6894964e1018a2a75b3f6b27e54 Mon Sep 17 00:00:00 2001 From: callumalpass Date: Sun, 13 Sep 2026 19:47:46 +1000 Subject: [PATCH 09/16] docs(recovery): document ownership, qualification gates and measured surface --- config/architecture-budgets.json | 16 ++-- docs/cli-daemon.md | 18 +++-- docs/code-quality.md | 15 ++++ docs/desktop-updates.md | 12 ++- docs/exact-recovery-and-health.md | 117 ++++++++++++++++++++++++++++++ 5 files changed, 161 insertions(+), 17 deletions(-) create mode 100644 docs/exact-recovery-and-health.md diff --git a/config/architecture-budgets.json b/config/architecture-budgets.json index 46b77a84f..66f5b15b8 100644 --- a/config/architecture-budgets.json +++ b/config/architecture-budgets.json @@ -5,10 +5,10 @@ "apps/editor/src/TypeBrowser.tsx": 1998 }, "productionFileBudgetsByPackage": { - "apps/desktop": 40, - "apps/editor": 110, + "apps/desktop": 43, + "apps/editor": 111, "apps/portal": 16, - "crates/connect-agent": 34, + "crates/connect-agent": 35, "crates/connect-cli": 7, "crates/connect-core": 34, "crates/connect-hosted-provider": 105, @@ -28,7 +28,7 @@ "packages/webhooks": 1, "services/feedback": 1, "services/mcp": 11, - "services/server": 138 + "services/server": 139 }, "deadCodeReferencesByFile": { "crates/connect-hosted-provider/src/workspace.rs": 1, @@ -43,11 +43,11 @@ "semanticProjectionFormatVersion": 6 }, "reviewBudgets": { - "productionFiles": 685, - "relativeImports": 1487, + "productionFiles": 691, + "relativeImports": 1504, "workspacePackages": 24, - "rustPublicDeclarations": 3163, - "typeScriptExportDeclarations": 2444, + "rustPublicDeclarations": 3185, + "typeScriptExportDeclarations": 2462, "mdbaseCollectionReferences": 16, "typedCollectionReferences": 1 } diff --git a/docs/cli-daemon.md b/docs/cli-daemon.md index b9d6f843d..100b155a4 100644 --- a/docs/cli-daemon.md +++ b/docs/cli-daemon.md @@ -174,12 +174,18 @@ mirror registry, cloud credential, retry loop, or agent child process. At startup the desktop: -1. compares a reachable daemon's binary version with the bundled CLI; -2. replaces and re-registers a stale or stopped installed runtime; -3. connects to the standard daemon endpoint; -4. asks a matching installed per-user service to start if the endpoint is absent; -5. presents a repair action if the service cannot start; -6. subscribes or polls for versioned status. +1. enters the shared boot gate and completes any persisted update recovery; +2. resolves installed-service versus isolated-profile targeting through CLI paths; +3. reconciles a stale installed runtime without controlling a different profile; +4. asks the selected daemon target to start if its endpoint is absent; +5. requires canonical readiness and the expected binary version before admitting + daemon-backed IPC, with a bounded wait and an attention state on failure; +6. subscribes or polls for versioned status, keeping local health separate from + account reachability and resource-specific failures. + +See [exact recovery and health](exact-recovery-and-health.md) for the readiness +contract, revocation confirmation, and bounded owner-specific recovery. Critical +worker detection does not imply automatic restart or a verified service owner. Desktop exit closes only its control connection. diff --git a/docs/code-quality.md b/docs/code-quality.md index 0cd7e6139..b4c5c38bd 100644 --- a/docs/code-quality.md +++ b/docs/code-quality.md @@ -124,6 +124,21 @@ does not relax file-size or cycle checks. Route tests and real PostgreSQL replay cross-provider email-race, same-subject concurrency and rollback tests cover the new persisted boundary. +The [exact-recovery changes](exact-recovery-and-health.md) add six narrow modules: +three desktop modules for boot admission, canonical readiness presentation, and +resource-local refresh results; one editor helper for existing SDK pending +handles; one relay retry policy within the existing owner; and one server +transaction boundary for local revocation. They replace bypasses/duplicated +policy rather than adding a supervisor, reconciliation bus, credential store, +or mutation journal. The measured surface adjustment is six production files, +17 relative imports, 22 Rust public declarations and 18 TypeScript exports: +691 files, 1,504 imports, 3,185 Rust declarations and 2,462 TypeScript exports. +Per-package file limits become desktop 43, editor 111, daemon 35 and server 139. +File-size, cycle, package-dependency and dead-code limits are unchanged. Evidence +belongs at each boundary: boot/update races, pending-handle/component tests, +exact-ACK and real PostgreSQL serialization tests, and real isolated-process +credential retry. These limits do not replace review or platform qualification. + Composition roots and package facades should approach these end-state shapes: - server `app.ts`: registration and lifecycle wiring only; diff --git a/docs/desktop-updates.md b/docs/desktop-updates.md index d4b00c8ed..c2895bb93 100644 --- a/docs/desktop-updates.md +++ b/docs/desktop-updates.md @@ -95,11 +95,14 @@ On first launch of the target app, before ordinary daemon startup, it: 1. marks the recorded transaction `recovering`; 2. atomically copies the new bundled CLI into the private stable service runtime and refreshes service registration, or starts a new transient daemon; -3. waits for the daemon to open its state and report the exact target version; -4. commits only after that health check. +3. waits for canonical daemon readiness and the exact target binary version; +4. commits only after that health check. Ordinary daemon-backed IPC and update + checks share the boot gate and cannot bypass recovery. If replacement was interrupted, the old app restarts its daemon and clears the -transaction. If the target daemon cannot start or migrate state, the new app +transaction only after verified health. A failed or thrown recovery retains the +same transaction and preserved runtime for the next process, blocking further +installation and ordinary startup rather than claiming restoration. If the target daemon cannot start or migrate state, the new app re-registers and health-checks the preserved previous daemon. Recovery remains visible so the user can install a higher signed recovery release. @@ -115,6 +118,9 @@ quarantined. A crash at every boundary is safe to retry: before stop there is no service impact; after stop the previous runtime is recorded; after replacement recovery is idempotent. +See [exact recovery and health](exact-recovery-and-health.md) for readiness +compatibility and the first-adoption gate for older rollback runtimes. + ## Release and recovery drills Before enabling a non-zero automatic rollout: diff --git a/docs/exact-recovery-and-health.md b/docs/exact-recovery-and-health.md new file mode 100644 index 000000000..e1c31f256 --- /dev/null +++ b/docs/exact-recovery-and-health.md @@ -0,0 +1,117 @@ +# Exact recovery and truthful health + +Recovery uses the existing owner of each operation. A response timeout is not a +new mutation, a reachable process is not initialized, and server-side revocation +is not proof that a local authority received it. + +## Daemon readiness and desktop admission + +The daemon owns the additive `readiness` object returned by `ping` and `status`: + +```json +{"schema_version":1,"ready":true,"binary_version":""} +``` + +When not ready, `safe_reason` is one of `starting`, `initialization_failed`, +`critical_worker_failed`, or `credential_store_unavailable`. These are safe +classification values, not raw exception strings. The legacy `ping.ready` is a +projection of this same state. Initialization is not committed after registry +loading fails. Readiness checks the existing watcher, runtime-notification, +mirror, and configured relay task handles; a dead critical task cannot leave a +ready result behind. This detects failure; it does **not** add an automatic +worker supervisor or prove that a platform service manager will restart it. + +CLI startup/doctor and desktop startup, tray, renderer, and updater consume this +contract. A missing/unknown readiness schema, an incompatible binary version, +or a terminal readiness reason is attention, never inferred success from PID +existence. Polling and individual startup probes have bounded deadlines. +Account reachability, remote access pause, and individual hosted resources remain +separate from local process readiness. + +`BootGate` serializes update recovery, ordinary daemon startup, and installation +admission. A daemon-backed request cannot initiate a second startup path while +boot recovery or installation owns that boundary. The CLI's resolved `connect +paths.target`, not whether Electron is packaged, selects `installed_service` or +`isolated_profile`. No isolated-profile action controls the default service. + +An unhealthy or thrown updater recovery retains the same transaction and its +last-known-good runtime in `recovering`. Update checks/reinstallation and ordinary +startup remain blocked until recovery verifies readiness and the expected binary +version. The next app process resumes the same transaction. No additional update +journal or repair owner is introduced. + +## Editor continuation + +The editor obtains pending note mutations from the existing SDK handles and +recovers by the original request ID. **Resume rename** never issues another +rename with reconstructed inputs. Unknown autosave completion retains the exact +original draft snapshot; that continuation must finish before a changed draft +can be sent. A watcher observation matching the recovered revision is not a +second conflicting mutation. + +Pending SDK inputs may exist only as ciphertext. The editor does not add a +plaintext path/draft index or another durable journal. After reload, pending +handles are listed by operation/time for explicit recovery. New note updates, +property/document edits, and renames are blocked while a pending note mutation +remains. The editor does not claim **Saved** while that work is unresolved. +Collection epochs prevent recovery from publishing into a subsequently selected +collection. Failure preserves the original identity and the local draft. + +## Revocation has an authority-specific completion point + +For a local grant, server admission and tokens are revoked transactionally with +an immutable barrier in the existing connector policy sequence. The response is +`revoking` until the local connector acknowledges an exact current-generation +policy snapshot at or above that barrier. Lease expiry, relay disconnect, an old +sequence, a retired/wrong connector generation, a mismatched digest/acknowledgement, +or a legacy snapshot does not establish confirmation. + +Migration `0031_local_revocation_confirmation` adds the barrier and confirmation +timestamp to grants. Historical unbound revocations receive a barrier under the +same connector lock used to build policy snapshots. Concurrent snapshots cannot +accidentally acknowledge a later revocation. Repeated single/batch revocations +reuse their existing barrier. Pending rows remain visible in account/connector +inventories. `revocation_status` is presentation-only, never an authorization +input. Hosted grants continue to complete at their own hosted authority; the UI +must not describe them as waiting for a local computer. + +## Resource-scoped desktop refresh + +Each resource refresh publishes independently. A failed/offline refresh retains +that resource's last known inventory rather than inventing a successful empty +list. Successful unrelated refreshes do not clear action errors; those have an +explicit dismissal. Cascading local-connector failures collapse into one message. +Update status uses its existing push subscription, not a second polling owner. +A configured account's credential-store failure is cached and rethrown during +cooldown; an explicitly unconfigured account can clear hosted inventory. + +## Relay and credential recovery + +The existing relay owner uses capped equal jitter (1–30 second exponential +ceilings), resets only after 30 seconds of policy-authorized healthy uptime, and +honours bounded server pacing (seconds or HTTP-date, capped at 300 seconds). +Transient transport failures retry without changing credentials or identity. +Authentication/protocol rejection parks the owner with a stable `relay_problem` +until a controlled restart after repair; it does not spin or mint credentials. +Handshake and inventory HTTP/WebSocket boundaries have timeouts. + +A running mirror whose previously usable credential store becomes unavailable +retries through its existing bounded mirror scheduler. Immutable bootstrap +credential failure remains blocked: unlock/repair the store and explicitly +restart the correct daemon target. CLI `whoami`/access snapshots propagate the +credential failure instead of claiming that the account is unconfigured. No unconditional retry or credential issuance +replay is added. Test-file-store recovery is not native OS-keyring qualification. + +## Qualification boundaries + +Local unit, PostgreSQL, component, and hermetic process tests establish their +specific boundaries, not signed updater or service-manager certification. Before +release, qualify real installed-service ownership and restart behaviour on each +supported platform, native keyring lock/unlock, and signed interrupted updates. +In particular, older preserved daemons lacking canonical readiness cannot be +silently accepted as healthy rollback targets. Exercise that first-adoption +boundary before enabling automatic rollout. + +See [CLI/daemon architecture](cli-daemon.md), [desktop updates](desktop-updates.md), +and [code-quality requirements](code-quality.md). Merge, release, and deployment +remain separate approval gates. From 7a8a430237101ce5218897fcf6511c33fe1725de Mon Sep 17 00:00:00 2001 From: callumalpass Date: Sun, 13 Sep 2026 19:55:27 +1000 Subject: [PATCH 10/16] test(relay): await committed readiness before asserting reconnect routing A fake connector sending policy_applied does not mean the server has finished its PostgreSQL confirmation and published session readiness. Wait for the existing server-owned signal, retaining a single strict routing assertion after that barrier. --- scripts/relay-e2e.mjs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/scripts/relay-e2e.mjs b/scripts/relay-e2e.mjs index 966370f94..4c24854cf 100644 --- a/scripts/relay-e2e.mjs +++ b/scripts/relay-e2e.mjs @@ -110,12 +110,13 @@ try { const config = { servers: [`nats://127.0.0.1:${natsPort}`], token: natsToken }; const brokerA = await createRelayBroker(config); const brokerB = await createRelayBroker(config); - ({ app: appA } = await buildApp({ + const builtA = await buildApp({ db: databaseA, devAuth: true, publicUrl: "http://127.0.0.1", relayBroker: brokerA - })); + }); + appA = builtA.app; const builtB = await buildApp({ db: databaseB, devAuth: true, @@ -150,6 +151,10 @@ try { }); socketA = connectorA.socket; await connectorA.waitForPolicy(); + // Sending the ACK is not the server's commit point: exact policy confirmation + // and readiness publication can still be awaiting PostgreSQL. + await poll(() => builtA.relay.isConnected(fixture.connectorId), + "Initial connector policy was not committed on instance A"); const initialPolicy = connectorA.policies.at(-1); const initialPolicyPredicates = { one_grant: initialPolicy?.grants?.length === 1, @@ -245,6 +250,8 @@ try { }); socketB = connectorB.socket; await connectorB.waitForPolicy(); + await poll(() => builtB.relay.isConnected(fixture.connectorId), + "Replacement connector policy was not committed on instance B"); const [closeCode] = await closedA; assert(closeCode === 4001, `Older cross-instance connector closed with ${closeCode}, not 4001`); @@ -484,6 +491,8 @@ try { }); socketA = connectorA2.socket; await connectorA2.waitForPolicy(); + await poll(() => builtA.relay.isConnected(fixture.connectorId), + "Reconnected connector policy was not committed on instance A"); const reconnected = await operation(urlB, fixture, "read", {}); assert(reconnected.status === 200 && reconnected.body.result?.owner === "instance-a-reconnected", `Relay did not follow a post-outage reconnect: ${JSON.stringify(reconnected)}`); From f80cb1951734fe629919ad79ce8820fc9854ecb5 Mon Sep 17 00:00:00 2001 From: callumalpass Date: Sun, 13 Sep 2026 21:39:57 +1000 Subject: [PATCH 11/16] fix(editor): settle definitively rejected recovery without losing drafts --- apps/editor/src/gateway.ts | 4 +- apps/editor/src/note-operation-coordinator.ts | 6 ++ apps/editor/src/note-recovery.test.ts | 88 +++++++++++++++++++ docs/exact-recovery-and-health.md | 5 +- 4 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 apps/editor/src/note-recovery.test.ts diff --git a/apps/editor/src/gateway.ts b/apps/editor/src/gateway.ts index 3a249afea..ad9fec0b7 100644 --- a/apps/editor/src/gateway.ts +++ b/apps/editor/src/gateway.ts @@ -299,9 +299,9 @@ export class ConnectCollectionGateway implements CollectionGateway { // One exact continuation, not a transport retry or a newly constructed update. if (pending?.operation === "update") { const recovered = await pending.recover().catch(() => outcome); - if (recovered.ok) return recovered.value; + return requireOutcome(recovered); } - // Preserve the original unknown identity even if its recovery is offline. + // Without a matching handle, the original outcome remains unknown. } return requireOutcome(outcome); } diff --git a/apps/editor/src/note-operation-coordinator.ts b/apps/editor/src/note-operation-coordinator.ts index 9bb30cbc8..e3a449b7c 100644 --- a/apps/editor/src/note-operation-coordinator.ts +++ b/apps/editor/src/note-operation-coordinator.ts @@ -1,3 +1,4 @@ +import { MdbaseConnectError } from "@mdbase-dev/connect"; import type { NoteDocument, SaveNoteInput } from "./model"; import type { NoteSession } from "./note-session"; import { sessionDirty } from "./note-session"; @@ -62,6 +63,11 @@ export class NoteOperationCoordinator { this.options.onSaved(session, document); this.options.onChange(session); } catch (error) { + // A definitive rejection settles the original intent; a failed probe + // (including not_sent) does not prove the earlier attempt was rejected. + if (error instanceof MdbaseConnectError && error.problem.operation_outcome === "rejected") { + session.pendingSave = undefined; + } const requestId = pendingNoteRequestId(error); if (!session.pendingSave && requestId) session.pendingSave = { requestId, draft: snapshot }; session.saveState = session.pendingSave ? "recovery" : "conflict"; diff --git a/apps/editor/src/note-recovery.test.ts b/apps/editor/src/note-recovery.test.ts new file mode 100644 index 000000000..9e48d6e6f --- /dev/null +++ b/apps/editor/src/note-recovery.test.ts @@ -0,0 +1,88 @@ +import { expect, it } from 'vitest'; +import { MdbaseConnectError } from '@mdbase-dev/connect'; +import { connectFailure, connectProblem, connectSuccess } from '@mdbase-dev/connect-testing'; +import { ConnectCollectionGateway } from './gateway'; +import { NoteOperationCoordinator } from './note-operation-coordinator'; +import { createNoteSession } from './note-session'; +import type { NoteDocument } from './model'; + +const document: NoteDocument = { + path: 'note.md', revision: 'r1', body: '# Note\n', types: [], + frontmatter: {}, effectiveFrontmatter: {}, file: { path: 'note.md' } +}; +const unknown = connectFailure(connectProblem('operation_outcome_unknown', 'Response lost', { + operationOutcome: 'unknown', details: { request_id: 'original-update' } +})); +const rejected = connectFailure(connectProblem('concurrent_modification', 'Revision changed', { + operationOutcome: 'rejected' +})); + +for (const mode of ['success', 'automatic-rejection', 'deferred-rejection'] as const) { + it(`${mode}: exact continuation settles the original pending identity`, async () => { + let pending = false; + let resolveNow = mode !== 'deferred-rejection'; + let updates = 0; + const handle = { + requestId: 'original-update', operation: 'update', + async recover() { + if (!resolveNow) return unknown; + // The SDK removes durable pending records on a definitive response. + pending = false; + return mode === 'success' ? connectSuccess({ ...document, revision: 'r2' }) : rejected; + } + }; + const connection = { + pendingMutations: () => pending ? [handle] : [], + pendingMutation: (id: string) => pending && id === handle.requestId ? handle : null, + async update() { updates++; pending = true; return unknown; } + }; + const gateway = new ConnectCollectionGateway('https://connect.example'); + Object.defineProperty(gateway, 'session', { value: { connection: () => connection }, configurable: true }); + const session = createNoteSession(document, []); + session.draft.body = 'Keep this draft'; + const coordinator = new NoteOperationCoordinator({ + update: input => gateway.update(input), recover: id => gateway.recoverNoteMutation(id), + onSaved() {}, onChange() {}, onSaveError() {} + }); + if (mode === 'success') { + await coordinator.requestSave(session); + expect(session.pendingSave).toBeUndefined(); + } else { + if (mode === 'deferred-rejection') { + await expect(coordinator.requestSave(session)).rejects.toMatchObject({ problem: { code: 'operation_outcome_unknown' } }); + resolveNow = true; + } + let error: unknown; + try { await coordinator.requestSave(session); } catch (cause) { error = cause; } + expect(error).toBeInstanceOf(MdbaseConnectError); + expect(gateway.pendingNoteMutations()).toEqual([]); + expect(updates).toBe(1); + expect(session.draft.body).toBe('Keep this draft'); + expect((error as MdbaseConnectError).problem.code).toBe('concurrent_modification'); + expect(session.pendingSave).toBeUndefined(); + expect(session.saveState).toBe('conflict'); + } + }); +} + +it.each([ + ['still unknown', new MdbaseConnectError(unknown.problem)], + ['probe not sent', new MdbaseConnectError(connectProblem('temporarily_unavailable', 'Probe unavailable', { operationOutcome: 'not_sent' }))], + ['unstructured failure', new Error('Offline')] +])('retains the original intent when recovery is %s', async (_name, failure) => { + let updates = 0; + const session = createNoteSession(document, []); + session.draft.body = 'Original accepted intent'; + const coordinator = new NoteOperationCoordinator({ + async update() { updates++; throw new MdbaseConnectError(unknown.problem); }, + async recover() { throw failure; }, + onSaved() {}, onChange() {}, onSaveError() {} + }); + await expect(coordinator.requestSave(session)).rejects.toBeInstanceOf(MdbaseConnectError); + session.draft.body = 'Newer unsent draft'; + await expect(coordinator.requestSave(session)).rejects.toBe(failure); + expect(updates).toBe(1); + expect(session.pendingSave).toMatchObject({ requestId: 'original-update', draft: { body: 'Original accepted intent' } }); + expect(session.draft.body).toBe('Newer unsent draft'); + expect(session.saveState).toBe('recovery'); +}); diff --git a/docs/exact-recovery-and-health.md b/docs/exact-recovery-and-health.md index e1c31f256..a4e43de5b 100644 --- a/docs/exact-recovery-and-health.md +++ b/docs/exact-recovery-and-health.md @@ -55,7 +55,10 @@ handles are listed by operation/time for explicit recovery. New note updates, property/document edits, and renames are blocked while a pending note mutation remains. The editor does not claim **Saved** while that work is unresolved. Collection epochs prevent recovery from publishing into a subsequently selected -collection. Failure preserves the original identity and the local draft. +collection. Unknown outcomes and failed recovery probes preserve the original +identity and local draft. A definitive recovery rejection is propagated and +settles that pending identity without discarding the draft or automatically +resending it; the editor returns to its conflict/error state. ## Revocation has an authority-specific completion point From 4112aafe1de256930813145bfaeb1b53848a498d Mon Sep 17 00:00:00 2001 From: callumalpass Date: Sun, 13 Sep 2026 21:59:20 +1000 Subject: [PATCH 12/16] fix(editor): use SDK settlement rather than probe error classification --- apps/editor/src/App.tsx | 1 + apps/editor/src/gateway.ts | 6 +++-- .../src/note-operation-coordinator.test.ts | 5 ++-- apps/editor/src/note-operation-coordinator.ts | 8 ++++--- apps/editor/src/note-recovery.test.ts | 24 ++++++++++++++----- 5 files changed, 31 insertions(+), 13 deletions(-) diff --git a/apps/editor/src/App.tsx b/apps/editor/src/App.tsx index 9c8fdeffc..f2779e26c 100644 --- a/apps/editor/src/App.tsx +++ b/apps/editor/src/App.tsx @@ -415,6 +415,7 @@ export function App({ gateway }: { gateway: CollectionGateway }) { const noteOperations = useMemo(() => new NoteOperationCoordinator({ recover: (requestId) => mutationScope.current.register(mutationScope.current.token(), gateway.recoverNoteMutation(requestId)), + isPending: (requestId) => gateway.pendingNoteMutations().some((pending) => pending.requestId === requestId), update: (input) => mutationScope.current.register(mutationScope.current.token(), gateway.update(input)), onSaved: (session, next) => { if (noteSessions.current.get(session.document.path) !== session) return; diff --git a/apps/editor/src/gateway.ts b/apps/editor/src/gateway.ts index ad9fec0b7..b4dec14aa 100644 --- a/apps/editor/src/gateway.ts +++ b/apps/editor/src/gateway.ts @@ -299,9 +299,11 @@ export class ConnectCollectionGateway implements CollectionGateway { // One exact continuation, not a transport retry or a newly constructed update. if (pending?.operation === "update") { const recovered = await pending.recover().catch(() => outcome); - return requireOutcome(recovered); + // A failed probe may not have reached the authority. Only the SDK + // settling its handle proves that rejection resolved the original intent. + if (recovered.ok || !connection.pendingMutation(pending.requestId)) return requireOutcome(recovered); } - // Without a matching handle, the original outcome remains unknown. + // Without a settled handle, the original outcome remains unknown. } return requireOutcome(outcome); } diff --git a/apps/editor/src/note-operation-coordinator.test.ts b/apps/editor/src/note-operation-coordinator.test.ts index 07d7686a7..032e1641e 100644 --- a/apps/editor/src/note-operation-coordinator.test.ts +++ b/apps/editor/src/note-operation-coordinator.test.ts @@ -28,7 +28,7 @@ describe("NoteOperationCoordinator", () => { }); const recover = vi.fn(async () => ({ ...document("2"), body: "First" })); const session = createNoteSession(document(), []); - const coordinator = new NoteOperationCoordinator({ update, recover, onSaved() {}, onSaveError() {}, onChange() {} }); + const coordinator = new NoteOperationCoordinator({ update, recover, isPending: () => true, onSaved() {}, onSaveError() {}, onChange() {} }); session.draft.body = "First"; await expect(coordinator.requestSave(session)).rejects.toBe(unknown); expect(session.saveState).toBe("recovery"); @@ -53,7 +53,7 @@ describe("NoteOperationCoordinator", () => { const recover = vi.fn(async () => { throw new Error("offline"); }); const session = createNoteSession(document(), []); session.draft.body = "Accepted"; - const coordinator = new NoteOperationCoordinator({ update, recover, onSaved() {}, onSaveError() {}, onChange() {} }); + const coordinator = new NoteOperationCoordinator({ update, recover, isPending: () => true, onSaved() {}, onSaveError() {}, onChange() {} }); await expect(coordinator.requestSave(session)).rejects.toBe(unknown); for (let i = 0; i < 2; i++) await expect(coordinator.requestSave(session)).rejects.toThrow("offline"); expect(update).toHaveBeenCalledTimes(1); @@ -70,6 +70,7 @@ describe("NoteOperationCoordinator", () => { const session = createNoteSession(document(), []); const coordinator = new NoteOperationCoordinator({ update, + isPending: () => false, onSaved: () => undefined, onSaveError: () => undefined, onChange: () => undefined diff --git a/apps/editor/src/note-operation-coordinator.ts b/apps/editor/src/note-operation-coordinator.ts index e3a449b7c..8620e0739 100644 --- a/apps/editor/src/note-operation-coordinator.ts +++ b/apps/editor/src/note-operation-coordinator.ts @@ -8,6 +8,7 @@ import { pendingNoteRequestId } from "./pending-note-mutation"; interface NoteOperationCoordinatorOptions { update(input: SaveNoteInput): Promise; recover?(requestId: string): Promise; + isPending(requestId: string): boolean; onSaved(session: NoteSession, document: NoteDocument): void; onSaveError(session: NoteSession, error: unknown): void; onChange(session: NoteSession): void; @@ -63,9 +64,10 @@ export class NoteOperationCoordinator { this.options.onSaved(session, document); this.options.onChange(session); } catch (error) { - // A definitive rejection settles the original intent; a failed probe - // (including not_sent) does not prove the earlier attempt was rejected. - if (error instanceof MdbaseConnectError && error.problem.operation_outcome === "rejected") { + // The SDK owns settlement. Probe errors alone cannot tell us whether + // the original request was rejected or remains durably pending. + if (session.pendingSave && error instanceof MdbaseConnectError && !error.outcomeUnknown && + !this.options.isPending(session.pendingSave.requestId)) { session.pendingSave = undefined; } const requestId = pendingNoteRequestId(error); diff --git a/apps/editor/src/note-recovery.test.ts b/apps/editor/src/note-recovery.test.ts index 9e48d6e6f..4a4fd623c 100644 --- a/apps/editor/src/note-recovery.test.ts +++ b/apps/editor/src/note-recovery.test.ts @@ -17,18 +17,22 @@ const rejected = connectFailure(connectProblem('concurrent_modification', 'Revis operationOutcome: 'rejected' })); -for (const mode of ['success', 'automatic-rejection', 'deferred-rejection'] as const) { +for (const mode of ['success', 'automatic-rejection', 'deferred-rejection', 'automatic-not-sent', 'deferred-not-sent', 'deferred-unmarked', 'probe-rejection'] as const) { it(`${mode}: exact continuation settles the original pending identity`, async () => { let pending = false; - let resolveNow = mode !== 'deferred-rejection'; + let resolveNow = !mode.startsWith('deferred'); + const failure = mode.endsWith('not-sent') + ? connectFailure(connectProblem('temporarily_unavailable', 'Not admitted', { operationOutcome: 'not_sent' })) + : mode.endsWith('unmarked') ? connectFailure(connectProblem('not_authorized', 'Grant expired')) : rejected; let updates = 0; const handle = { requestId: 'original-update', operation: 'update', async recover() { if (!resolveNow) return unknown; + if (mode === 'probe-rejection') return rejected; // The SDK removes durable pending records on a definitive response. pending = false; - return mode === 'success' ? connectSuccess({ ...document, revision: 'r2' }) : rejected; + return mode === 'success' ? connectSuccess({ ...document, revision: 'r2' }) : failure; } }; const connection = { @@ -42,13 +46,19 @@ for (const mode of ['success', 'automatic-rejection', 'deferred-rejection'] as c session.draft.body = 'Keep this draft'; const coordinator = new NoteOperationCoordinator({ update: input => gateway.update(input), recover: id => gateway.recoverNoteMutation(id), + isPending: id => gateway.pendingNoteMutations().some(pending => pending.requestId === id), onSaved() {}, onChange() {}, onSaveError() {} }); - if (mode === 'success') { + if (mode === 'probe-rejection') { + await expect(coordinator.requestSave(session)).rejects.toMatchObject({ problem: { code: 'operation_outcome_unknown' } }); + expect(session.pendingSave?.requestId).toBe('original-update'); + expect(pending).toBe(true); + expect(updates).toBe(1); + } else if (mode === 'success') { await coordinator.requestSave(session); expect(session.pendingSave).toBeUndefined(); } else { - if (mode === 'deferred-rejection') { + if (mode.startsWith('deferred')) { await expect(coordinator.requestSave(session)).rejects.toMatchObject({ problem: { code: 'operation_outcome_unknown' } }); resolveNow = true; } @@ -58,7 +68,7 @@ for (const mode of ['success', 'automatic-rejection', 'deferred-rejection'] as c expect(gateway.pendingNoteMutations()).toEqual([]); expect(updates).toBe(1); expect(session.draft.body).toBe('Keep this draft'); - expect((error as MdbaseConnectError).problem.code).toBe('concurrent_modification'); + expect((error as MdbaseConnectError).problem.code).toBe(failure.problem.code); expect(session.pendingSave).toBeUndefined(); expect(session.saveState).toBe('conflict'); } @@ -68,6 +78,7 @@ for (const mode of ['success', 'automatic-rejection', 'deferred-rejection'] as c it.each([ ['still unknown', new MdbaseConnectError(unknown.problem)], ['probe not sent', new MdbaseConnectError(connectProblem('temporarily_unavailable', 'Probe unavailable', { operationOutcome: 'not_sent' }))], + ['probe rejected but original still pending', new MdbaseConnectError(rejected.problem)], ['unstructured failure', new Error('Offline')] ])('retains the original intent when recovery is %s', async (_name, failure) => { let updates = 0; @@ -76,6 +87,7 @@ it.each([ const coordinator = new NoteOperationCoordinator({ async update() { updates++; throw new MdbaseConnectError(unknown.problem); }, async recover() { throw failure; }, + isPending: () => true, onSaved() {}, onChange() {}, onSaveError() {} }); await expect(coordinator.requestSave(session)).rejects.toBeInstanceOf(MdbaseConnectError); From ecc2d45705bcb7edba618d5358b2a578d2dbdac8 Mon Sep 17 00:00:00 2001 From: callumalpass Date: Sun, 13 Sep 2026 22:10:33 +1000 Subject: [PATCH 13/16] fix(desktop): preserve verified rollback admission across restarts --- apps/desktop/src/main/control-client.ts | 2 +- .../src/main/electron-update-backend.ts | 78 +++--- apps/desktop/src/main/main.ts | 6 +- apps/desktop/src/main/update-coordinator.ts | 77 ++++-- apps/desktop/src/main/update-state.ts | 27 +- apps/desktop/test/rollback-admission.test.mjs | 234 ++++++++++++++++++ apps/desktop/test/update-coordinator.test.mjs | 4 +- apps/editor/src/App.tsx | 5 +- config/architecture-budgets.json | 4 +- docs/code-quality.md | 7 + docs/desktop-updates.md | 18 +- docs/exact-recovery-and-health.md | 13 +- 12 files changed, 398 insertions(+), 77 deletions(-) create mode 100644 apps/desktop/test/rollback-admission.test.mjs diff --git a/apps/desktop/src/main/control-client.ts b/apps/desktop/src/main/control-client.ts index 4ac332c50..c85f5ccca 100644 --- a/apps/desktop/src/main/control-client.ts +++ b/apps/desktop/src/main/control-client.ts @@ -1,7 +1,7 @@ import { createConnection } from "node:net"; import { randomUUID } from "node:crypto"; -const LOCAL_CONTROL_PROTOCOL_VERSION = 4; +export const LOCAL_CONTROL_PROTOCOL_VERSION = 4; const MAX_LOCAL_CONTROL_RESPONSE_BYTES = 32 * 1024 * 1024; export interface ControlResponse { diff --git a/apps/desktop/src/main/electron-update-backend.ts b/apps/desktop/src/main/electron-update-backend.ts index 7ea0b7493..765d7c0ed 100644 --- a/apps/desktop/src/main/electron-update-backend.ts +++ b/apps/desktop/src/main/electron-update-backend.ts @@ -24,7 +24,8 @@ import { type UpdateManifest, type UpdateTarget } from "./update-policy"; -import type { UpdateTransaction } from "./update-state"; +import type { PersistedUpdateState, UpdateTransaction } from "./update-state"; +import { LOCAL_CONTROL_PROTOCOL_VERSION } from "./control-client"; import { artifactMatches, downloadArtifact, downloadBytes } from "./update-download"; import { connectCliEnvironment, daemonCliArguments, type DaemonTarget } from "./daemon-lifecycle"; @@ -59,13 +60,25 @@ export class ElectronUpdateBackend implements UpdateBackend { this.packaged = options.packaged; } - async reconcileInstalledRuntime(): Promise { - if (!this.packaged) return null; - const status = await this.daemonStatus(); - const needsReconciliation = runtimeNeedsReconciliation(status, this.currentVersion); - if (!needsReconciliation) return null; - await this.activateRuntime(this.options.binaryPath(), this.currentVersion); - return `Connector runtime ${this.currentVersion} was reconciled with this application.`; + async reconcileInstalledRuntime(rollback?: PersistedUpdateState["last_known_good_runtime"]): Promise { + if (!this.packaged && !rollback) return null; + if (rollback) { + this.assertPrivateRuntime(rollback.path, rollback.version); + if (!(await stat(rollback.path)).isFile()) throw new Error("The saved rollback runtime is missing."); + } + const binary = rollback?.path ?? this.options.binaryPath(); + const version = rollback?.version ?? this.currentVersion; + const status = await this.daemonStatus(binary); + if (runtimeNeedsReconciliation(status, version)) { + await this.activateRuntime(binary, version); + } else if (rollback && !status.ready) { + throw new Error("The saved rollback connector is not ready or uses an incompatible local protocol."); + } else if (!rollback) { + return null; + } + return rollback + ? `Using verified rollback connector ${version} with application ${this.currentVersion}.` + : `Connector runtime ${version} was reconciled with this application.`; } async findLatest(): Promise<{ manifest: UpdateManifest } | null> { @@ -110,7 +123,15 @@ export class ElectronUpdateBackend implements UpdateBackend { await stageMacUpdate(archive, manifest, onProgress); } - async prepareDaemonHandoff(previousVersion: string): Promise { + async prepareDaemonHandoff(previousVersion: string, previousRuntime?: string | null): Promise { + if (previousRuntime) { + this.assertPrivateRuntime(previousRuntime, previousVersion); + if (!(await stat(previousRuntime)).isFile()) throw new Error("The saved rollback runtime is missing."); + const status = await this.daemonStatus(previousRuntime); + // Already preserved: never overwrite the active fallback with the newer bundle. + return { serviceInstalled: status.installed, previousRuntime }; + } + if (previousVersion !== this.currentVersion) throw new Error("The previous runtime binary is missing."); const status = await this.daemonStatus(); const source = this.options.binaryPath(); const directory = join( @@ -155,19 +176,8 @@ export class ElectronUpdateBackend implements UpdateBackend { } async recover(transaction: UpdateTransaction): Promise { - if (transaction.previous_runtime) { - const extension = this.options.platform === "win32" ? ".exe" : ""; - const expected = join( - this.options.userDataDirectory, - "updates", - "runtimes", - transaction.previous_version, - `mdbase${extension}` - ); - if (transaction.previous_runtime !== expected) { - throw new Error("The recorded recovery runtime is outside the private update directory."); - } - } + const previousVersion = transaction.previous_runtime_version ?? transaction.previous_version; + if (transaction.previous_runtime) this.assertPrivateRuntime(transaction.previous_runtime, previousVersion); const runningTarget = this.currentVersion === transaction.target_version; const runningPrevious = this.currentVersion === transaction.previous_version; if (runningTarget) { @@ -185,21 +195,21 @@ export class ElectronUpdateBackend implements UpdateBackend { if (!transaction.previous_runtime) throw error; await this.activateRuntime( transaction.previous_runtime, - transaction.previous_version + previousVersion ); return { healthy: true, rolledBack: true, message: `Version ${transaction.target_version} could not start its connector. ` + - `The last-known-good ${transaction.previous_version} connector was restored.` + `The last-known-good ${previousVersion} connector was restored.` }; } } if (runningPrevious) { await this.activateRuntime( - this.options.binaryPath(), - transaction.previous_version + previousVersion === this.currentVersion ? this.options.binaryPath() : transaction.previous_runtime!, + previousVersion ); return { healthy: true, @@ -215,20 +225,26 @@ export class ElectronUpdateBackend implements UpdateBackend { } await this.activateRuntime( transaction.previous_runtime, - transaction.previous_version + previousVersion ); return { healthy: true, rolledBack: true, - message: `An unexpected application version was detected; connector ${transaction.previous_version} was restored.` + message: `An unexpected application version was detected; connector ${previousVersion} was restored.` }; } + private assertPrivateRuntime(binary: string, version: string): void { + const extension = this.options.platform === "win32" ? ".exe" : ""; + const expected = join(this.options.userDataDirectory, "updates", "runtimes", version, `mdbase${extension}`); + if (binary !== expected) throw new Error("The recorded recovery runtime is outside the private update directory."); + } + private async activateRuntime( binary: string, expectedVersion: string ): Promise { - const current = await this.daemonStatus().catch(() => ({ running: false })); + const current = await this.daemonStatus(binary).catch(() => ({ running: false })); if (current.running) { await this.runCli(binary, ["stop"], 35_000).catch(() => undefined); } @@ -255,10 +271,12 @@ export class ElectronUpdateBackend implements UpdateBackend { binaryVersion?: string; }> { const value = await this.runCli(binary, ["status"], 10_000); + const status = value.status as { readiness?: AgentReadiness; protocol_version?: number } | undefined; return { installed: value.installed === true, running: value.running === true, - ready: presentReadiness((value.status as { readiness?: AgentReadiness } | undefined)?.readiness).state === "ready", + ready: status?.protocol_version === LOCAL_CONTROL_PROTOCOL_VERSION && + presentReadiness(status.readiness).state === "ready", binaryVersion: value.status && typeof value.status === "object" && diff --git a/apps/desktop/src/main/main.ts b/apps/desktop/src/main/main.ts index dec2e289d..56541a862 100644 --- a/apps/desktop/src/main/main.ts +++ b/apps/desktop/src/main/main.ts @@ -100,15 +100,15 @@ function incompatibleDaemon(error: unknown): boolean { return error instanceof AgentControlError && error.code === "unsupported_local_protocol"; } -async function startAgent(): Promise { +async function startAgent(runtime = updater!.daemonStartupRuntime()): Promise { await ensureAgentReady({ - expectedVersion: app.getVersion(), + expectedVersion: runtime.version, ping: (timeoutMs) => requestAgent(controlEndpoint(), "ping", undefined, timeoutMs), endpointIsUnavailable, incompatibleDaemon, launch: async () => { - const binary = connectBinary(); + const binary = runtime.binary ?? connectBinary(); if (!existsSync(binary)) { throw new Error(`Connector runtime is missing: ${binary}`); } diff --git a/apps/desktop/src/main/update-coordinator.ts b/apps/desktop/src/main/update-coordinator.ts index 5c123fc00..904c4f3f9 100644 --- a/apps/desktop/src/main/update-coordinator.ts +++ b/apps/desktop/src/main/update-coordinator.ts @@ -6,7 +6,7 @@ import { type UpdateManifest, type UpdateTarget } from "./update-policy"; -import { UpdateStateStore, type UpdateTransaction } from "./update-state"; +import { UpdateStateStore, type PersistedUpdateState, type UpdateTransaction } from "./update-state"; export type UpdatePhase = | "unavailable" @@ -49,14 +49,14 @@ export interface UpdateBackend { channel: UpdateChannel; platformKey: string; packaged: boolean; - reconcileInstalledRuntime(): Promise; + reconcileInstalledRuntime(rollback?: PersistedUpdateState["last_known_good_runtime"]): Promise; findLatest(): Promise<{ manifest: UpdateManifest } | null>; stageAutomatic( manifest: UpdateManifest, target: UpdateTarget, onProgress: (progress: number) => void ): Promise; - prepareDaemonHandoff(previousVersion: string): Promise; + prepareDaemonHandoff(previousVersion: string, previousRuntime?: string | null): Promise; stopDaemon(): Promise; installAutomatic(): void; openExternal(url: string): Promise; @@ -70,10 +70,12 @@ export class UpdateCoordinator { private statusValue: DesktopUpdateStatus; private candidate: { manifest: UpdateManifest; target: UpdateTarget } | null = null; private operation: Promise | null = null; + private runtime: { version: string; binary: string | null }; constructor(store: UpdateStateStore, backend: UpdateBackend) { this.store = store; this.backend = backend; + this.runtime = { version: backend.currentVersion, binary: null }; this.statusValue = { phase: backend.packaged ? "idle" : "unavailable", current_version: backend.currentVersion, @@ -90,6 +92,10 @@ export class UpdateCoordinator { return structuredClone(this.statusValue); } + daemonStartupRuntime(): { version: string; binary: string | null } { + return { ...this.runtime }; + } + daemonStartupBlock(): string | null { const status = this.statusValue; return status.phase === "installing" || @@ -110,7 +116,10 @@ export class UpdateCoordinator { } if (!persisted.transaction) { try { - const message = await this.backend.reconcileInstalledRuntime(); + const rollback = persisted.last_known_good_runtime?.for_app_version === this.backend.currentVersion + ? persisted.last_known_good_runtime : undefined; + const message = await this.backend.reconcileInstalledRuntime(rollback); + if (rollback) this.runtime = { version: rollback.version, binary: rollback.path }; if (message) { this.setStatus({ phase: "idle", @@ -141,22 +150,7 @@ export class UpdateCoordinator { }); try { const result = await this.backend.recover(persisted.transaction); - if (!result.healthy) throw new Error(result.message); - await this.store.update((state) => { - if (!result.rolledBack) { - state.highest_trusted_version = maxVersion( - state.highest_trusted_version, - persisted.transaction?.target_version - ); - if (persisted.transaction?.previous_runtime) { - state.last_known_good_runtime = { - version: persisted.transaction.previous_version, - path: persisted.transaction.previous_runtime - }; - } - } - delete state.transaction; - }); + await this.completeRecovery(persisted.transaction, result); this.setStatus({ phase: result.rolledBack ? "recovery" : "idle", message: result.message, @@ -229,15 +223,16 @@ export class UpdateCoordinator { await this.store.update((state) => { if (state.transaction?.id === transaction.id) state.transaction.phase = "recovering"; }); - const recovered = await this.backend.recover(transaction).catch((recoveryError) => ({ + const recovered = await this.backend.recover(transaction).then(async (result) => { + await this.completeRecovery(transaction, result); + return result; + }).catch((recoveryError) => ({ healthy: false, rolledBack: false, message: message(recoveryError) })); - await this.store.update((state) => { - if (state.transaction?.id !== transaction.id) return; - if (recovered.healthy) delete state.transaction; - else state.transaction.error = recovered.message; + if (!recovered.healthy) await this.store.update((state) => { + if (state.transaction?.id === transaction.id) state.transaction.error = recovered.message; }); this.candidate = null; this.setStatus({ @@ -254,6 +249,34 @@ export class UpdateCoordinator { return this.status(); } + private async completeRecovery(transaction: UpdateTransaction, result: RecoveryResult): Promise { + if (!result.healthy) throw new Error(result.message); + const previousVersion = transaction.previous_runtime_version ?? transaction.previous_version; + const version = result.rolledBack ? previousVersion : transaction.target_version; + const binary = result.rolledBack && version !== this.backend.currentVersion ? transaction.previous_runtime : null; + if (version !== this.backend.currentVersion && !binary) { + throw new Error("Verified recovery did not preserve its runtime binary."); + } + await this.store.update((state) => { + if (state.transaction?.id !== transaction.id) throw new Error("Update recovery transaction changed."); + if (!result.rolledBack) { + state.highest_trusted_version = maxVersion(state.highest_trusted_version, transaction.target_version); + } + if (transaction.previous_runtime) { + state.last_known_good_runtime = { + version: previousVersion, + path: transaction.previous_runtime, + ...(result.rolledBack && version !== this.backend.currentVersion + ? { for_app_version: this.backend.currentVersion } : {}) + }; + } else if (state.last_known_good_runtime) { + delete state.last_known_good_runtime.for_app_version; + } + delete state.transaction; + }); + this.runtime = { version, binary }; + } + private async checkExclusive(manual: boolean): Promise { if (!this.backend.packaged) return this.status(); this.setStatus({ @@ -340,7 +363,7 @@ export class UpdateCoordinator { can_check: false, can_install: false }); - const handoff = await this.backend.prepareDaemonHandoff(this.backend.currentVersion); + const handoff = await this.backend.prepareDaemonHandoff(this.runtime.version, this.runtime.binary); const transaction: UpdateTransaction = { id: randomUUID(), phase: "prepared", @@ -348,6 +371,8 @@ export class UpdateCoordinator { previous_version: this.backend.currentVersion, service_installed: handoff.serviceInstalled, previous_runtime: handoff.previousRuntime, + ...(this.runtime.version !== this.backend.currentVersion + ? { previous_runtime_version: this.runtime.version } : {}), started_at: new Date().toISOString() }; await this.store.update((state) => { diff --git a/apps/desktop/src/main/update-state.ts b/apps/desktop/src/main/update-state.ts index 5c4888f85..cc33f58a9 100644 --- a/apps/desktop/src/main/update-state.ts +++ b/apps/desktop/src/main/update-state.ts @@ -10,6 +10,7 @@ export interface UpdateTransaction { previous_version: string; service_installed: boolean; previous_runtime: string | null; + previous_runtime_version?: string; started_at: string; error?: string; } @@ -22,6 +23,7 @@ export interface PersistedUpdateState { last_known_good_runtime?: { version: string; path: string; + for_app_version?: string; }; transaction?: UpdateTransaction; } @@ -55,9 +57,10 @@ export class UpdateStateStore { ): Promise { const current = await this.load(); const next = change(current) ?? current; - this.state = parsePersistedState(next); - await this.write(); - return structuredClone(this.state); + const validated = parsePersistedState(next); + await this.write(validated); + this.state = validated; + return structuredClone(validated); } async remove(): Promise { @@ -65,13 +68,13 @@ export class UpdateStateStore { await rm(this.path, { force: true }); } - private async write(): Promise { - if (!this.state) throw new Error("Update state has not been initialized."); + private async write(state = this.state): Promise { + if (!state) throw new Error("Update state has not been initialized."); const parent = dirname(this.path); await mkdir(parent, { recursive: true, mode: 0o700 }); await chmod(parent, 0o700).catch(() => undefined); const temporary = `${this.path}.tmp-${process.pid}-${randomUUID()}`; - await writeFile(temporary, `${JSON.stringify(this.state, null, 2)}\n`, { + await writeFile(temporary, `${JSON.stringify(state, null, 2)}\n`, { encoding: "utf8", mode: 0o600, flag: "wx" @@ -116,6 +119,11 @@ export function parsePersistedState(value: unknown): PersistedUpdateState { } compareVersions(runtime.version, runtime.version); parsed.last_known_good_runtime = { version: runtime.version, path: runtime.path }; + if (runtime.for_app_version !== undefined) { + if (typeof runtime.for_app_version !== "string") throw new Error("Rollback app version is invalid."); + compareVersions(runtime.for_app_version, runtime.for_app_version); + parsed.last_known_good_runtime.for_app_version = runtime.for_app_version; + } } if (state.transaction !== undefined) parsed.transaction = parseTransaction(state.transaction); return parsed; @@ -143,6 +151,12 @@ function parseTransaction(value: unknown): UpdateTransaction { if (transaction.previous_runtime !== null && typeof transaction.previous_runtime !== "string") { throw new Error("Update transaction runtime path is invalid."); } + if (transaction.previous_runtime_version !== undefined) { + if (typeof transaction.previous_runtime_version !== "string" || !transaction.previous_runtime) { + throw new Error("Previous runtime version is invalid."); + } + compareVersions(transaction.previous_runtime_version, transaction.previous_runtime_version); + } if (transaction.error !== undefined && typeof transaction.error !== "string") { throw new Error("Update transaction error is invalid."); } @@ -153,6 +167,7 @@ function parseTransaction(value: unknown): UpdateTransaction { previous_version: transaction.previous_version as string, service_installed: transaction.service_installed, previous_runtime: transaction.previous_runtime as string | null, + ...(transaction.previous_runtime_version ? { previous_runtime_version: transaction.previous_runtime_version as string } : {}), started_at: new Date(transaction.started_at as string).toISOString(), ...(transaction.error ? { error: transaction.error as string } : {}) }; diff --git a/apps/desktop/test/rollback-admission.test.mjs b/apps/desktop/test/rollback-admission.test.mjs new file mode 100644 index 000000000..1779f979e --- /dev/null +++ b/apps/desktop/test/rollback-admission.test.mjs @@ -0,0 +1,234 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { createRequire } from "node:module"; +import { mkdtemp, mkdir, readFile, writeFile, rm, rename } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, dirname } from "node:path"; + +const require = createRequire(import.meta.url); +const { UpdateCoordinator } = require("../dist/main/update-coordinator.js"); +const { UpdateStateStore, parsePersistedState } = require("../dist/main/update-state.js"); +const { ElectronUpdateBackend } = require("../dist/main/electron-update-backend.js"); +const { BootGate } = require("../dist/main/boot-gate.js"); +const { ensureAgentReady } = require("../dist/main/agent-startup.js"); +const previousVersion = "0.1.0-beta.98"; +const currentVersion = "0.1.0-beta.99"; +const nextVersion = "0.1.0-beta.100"; +const nextRelease = { manifest: { + version: nextVersion, tag: `v${nextVersion}`, channel: "beta", published_at: new Date().toISOString(), + release_url: "https://example.com/release", rollout: { percentage: 100, seed: "next" }, blocked_versions: [], + targets: { "darwin-arm64": { mode: "automatic", action_url: "https://example.com/release", artifacts: [] } } +} }; + +async function fixture(t, failTarget = true) { + const directory = await mkdtemp(join(tmpdir(), "mdbase-rollback-admission-")); + t.after(() => rm(directory, { recursive: true, force: true })); + const binary = join(directory, "bundled-mdbase"); + const previous = join(directory, "updates", "runtimes", previousVersion, "mdbase"); + await mkdir(dirname(previous), { recursive: true }); + await writeFile(previous, "preserved previous runtime"); + await writeFile(binary, "new bundled runtime"); + const path = join(directory, "state.json"); + const store = new UpdateStateStore(path); + const transaction = { id: "exact-update", phase: "installing", previous_version: previousVersion, + target_version: currentVersion, service_installed: true, previous_runtime: previous, + started_at: new Date().toISOString() }; + await store.update(state => { state.transaction = transaction; }); + const process = { running: false, version: previousVersion, protocol: 4, schema: 1, ready: true, failTarget }; + const commands = []; + function backend(version = currentVersion) { + const result = new ElectronUpdateBackend({ currentVersion: version, packaged: true, + platform: "darwin", arch: "arm64", userDataDirectory: directory, + binaryPath: () => binary, stateDirectory: () => directory, + target: () => "installed_service", endpoint: () => join(directory, "control.sock") }); + // Only the native process boundary is simulated. Recovery, reconciliation, + // persisted state, version checks and gate admission are production code. + result.runCli = async (file, command) => { + commands.push([file, command[0]]); + if (command[0] === "stop") process.running = false; + if (command[0] === "install") { + if (file === binary && process.failTarget) throw new Error("New runtime could not start"); + process.running = true; + process.version = file === previous ? previousVersion : version; + } + return { installed: true, running: process.running, + status: { protocol_version: process.protocol, binary_version: process.version, + readiness: { schema_version: process.schema, ready: process.ready, binary_version: process.version } } }; + }; + return result; + } + function gate(coordinator) { + return new BootGate({ + initialize: async () => { await coordinator.initialize(); }, + blockedReason: () => coordinator.daemonStartupBlock(), + start: () => ensureAgentReady({ + expectedVersion: coordinator.daemonStartupRuntime().version, + async ping() { + if (!process.running) throw new Error("Unavailable"); + return { pong: true, readiness: { + schema_version: process.schema, ready: process.ready, binary_version: process.version } }; + }, + async launch() { + const selected = coordinator.daemonStartupRuntime(); + assert.equal(selected.binary, previous); + process.running = true; + process.version = previousVersion; + }, + endpointIsUnavailable: error => error.message === "Unavailable", + incompatibleDaemon: () => false + }) + }); + } + return { store, path, transaction, previous, binary, process, commands, backend, gate }; +} + +for (const rollback of [false, true]) { + test(`verified ${rollback ? "rollback" : "target"} admits ordinary IPC and survives fresh startup`, async t => { + const f = await fixture(t, rollback); + const coordinator = new UpdateCoordinator(f.store, f.backend()); + assert.equal(await f.gate(coordinator).request(async () => "admitted"), "admitted"); + assert.equal((await f.store.load()).transaction, undefined); + assert.deepEqual(coordinator.daemonStartupRuntime(), rollback + ? { version: previousVersion, binary: f.previous } : { version: currentVersion, binary: null }); + const cold = new UpdateCoordinator(new UpdateStateStore(f.path), f.backend()); + f.commands.length = 0; + const gate = f.gate(cold); + await gate.request(async () => {}); + assert.equal(f.commands.some(([, command]) => command === "install"), false); + assert.equal(f.process.version, rollback ? previousVersion : currentVersion); + if (rollback) { + // A later crash must launch the preserved CLI, not the failing new bundle. + f.process.running = false; + await gate.request(async () => {}); + assert.equal(f.process.version, previousVersion); + f.process.version = "0.1.0-beta.97"; + await assert.rejects(gate.request(async () => assert.fail("Unexpected IPC")), /Update or restart/); + } + }); +} + +test("a stopped saved rollback is reconciled using only its preserved binary", async t => { + const f = await fixture(t); + await f.gate(new UpdateCoordinator(f.store, f.backend())).ready(); + f.process.running = false; + f.commands.length = 0; + await f.gate(new UpdateCoordinator(new UpdateStateStore(f.path), f.backend())).ready(); + assert.ok(f.commands.some(([file, command]) => file === f.previous && command === "install")); + assert.equal(f.commands.some(([file]) => file === f.binary), false); +}); + +for (const failure of ["unhealthy", "protocol", "unknown-schema", "missing", "outside"]) { + test(`a ${failure} saved rollback blocks IPC without silently installing the new bundle`, async t => { + const f = await fixture(t); + await f.gate(new UpdateCoordinator(f.store, f.backend())).ready(); + if (failure === "unhealthy") f.process.ready = false; + if (failure === "protocol") f.process.protocol = 3; + if (failure === "unknown-schema") f.process.schema = 2; + if (failure === "missing") await rm(f.previous); + if (failure === "outside") await f.store.update(s => { s.last_known_good_runtime.path = f.binary; }); + f.commands.length = 0; + const cold = new UpdateCoordinator(new UpdateStateStore(f.path), f.backend()); + await assert.rejects(f.gate(cold).request(async () => assert.fail("Unexpected IPC")), /Could not reconcile/); + assert.equal(f.commands.some(([, command]) => command === "install"), false); + assert.equal(cold.status().can_check, false); + }); +} + +test("a newer app does not inherit another app version's selected fallback", async t => { + const f = await fixture(t, false); + await f.store.update(s => { + delete s.transaction; + s.last_known_good_runtime = { version: previousVersion, path: f.previous, for_app_version: currentVersion }; + }); + const coordinator = new UpdateCoordinator(f.store, f.backend(nextVersion)); + await f.gate(coordinator).ready(); + assert.equal(f.process.version, nextVersion); + assert.deepEqual(coordinator.daemonStartupRuntime(), { version: nextVersion, binary: null }); +}); + +test("the next update preserves the selected fallback through stage and install failures", async t => { + const f = await fixture(t); + const native = f.backend(); + const coordinator = new UpdateCoordinator(f.store, native); + await f.gate(coordinator).ready(); + native.findLatest = async () => nextRelease; + native.stageAutomatic = async () => { throw new Error("Download interrupted"); }; + assert.equal((await coordinator.check(true)).phase, "failed"); + assert.equal((await f.store.load()).transaction, undefined); + assert.equal((await f.store.load()).last_known_good_runtime.for_app_version, currentVersion); + native.stageAutomatic = async () => {}; + assert.equal((await coordinator.check(true)).phase, "ready"); + const prepared = (await f.store.load()).transaction; + assert.equal(prepared.previous_version, currentVersion); + assert.equal(prepared.previous_runtime_version, previousVersion); + assert.equal(prepared.previous_runtime, f.previous); + assert.equal(await readFile(f.previous, "utf8"), "preserved previous runtime"); + native.installAutomatic = () => { throw new Error("Installer interrupted"); }; + await assert.rejects(coordinator.install(), /Installer interrupted/); + assert.equal(coordinator.daemonStartupBlock(), null); + await f.gate(coordinator).ready(); + assert.equal(f.process.version, previousVersion); + assert.equal((await f.store.load()).transaction, undefined); + // The next installed app can still roll back to 98, not mislabeled app 99. + await f.store.update(s => { s.transaction = prepared; }); + await f.gate(new UpdateCoordinator(new UpdateStateStore(f.path), f.backend(nextVersion))).ready(); + assert.equal((await new UpdateStateStore(f.path).load()).last_known_good_runtime.for_app_version, nextVersion); +}); + +test("a healthy subsequent upgrade clears the app-bound fallback selection", async t => { + const f = await fixture(t); + const native = f.backend(); + const coordinator = new UpdateCoordinator(f.store, native); + await f.gate(coordinator).ready(); + native.findLatest = async () => nextRelease; + native.stageAutomatic = async () => {}; + assert.equal((await coordinator.check(true)).phase, "ready"); + await f.store.update(s => { s.transaction.phase = "installing"; }); + f.process.failTarget = false; + const next = new UpdateCoordinator(new UpdateStateStore(f.path), f.backend(nextVersion)); + await f.gate(next).ready(); + const persisted = await new UpdateStateStore(f.path).load(); + assert.equal(persisted.transaction, undefined); + assert.equal(persisted.last_known_good_runtime.for_app_version, undefined); + assert.equal(persisted.last_known_good_runtime.version, previousVersion); + assert.equal(persisted.highest_trusted_version, nextVersion); + assert.deepEqual(next.daemonStartupRuntime(), { version: nextVersion, binary: null }); +}); + +test("a verified rollback is not admitted if its durable selection cannot be written", async t => { + const f = await fixture(t); + const native = f.backend(); + const recover = native.recover.bind(native); + native.recover = async transaction => { + const result = await recover(transaction); + await rename(f.path, `${f.path}.saved`); + await mkdir(f.path); + return result; + }; + const coordinator = new UpdateCoordinator(f.store, native); + await assert.rejects(f.gate(coordinator).request(async () => assert.fail("Unexpected IPC"))); + assert.equal((await f.store.load()).transaction.id, f.transaction.id); + assert.equal((await f.store.load()).transaction.phase, "recovering"); + assert.equal((await f.store.load()).last_known_good_runtime, undefined); + assert.equal(coordinator.daemonStartupRuntime().binary, null); +}); + +test("failed recovery-state persistence does not publish settlement in the store cache", async t => { + const f = await fixture(t); + await rename(f.path, `${f.path}.saved`); + await mkdir(f.path); + await assert.rejects(f.store.update(s => { delete s.transaction; })); + assert.deepEqual((await f.store.load()).transaction, f.transaction); +}); + +test("version bindings round-trip and legacy transactions keep their original version meaning", async t => { + const f = await fixture(t); + const state = await f.store.load(); + assert.equal(parsePersistedState(state).transaction.previous_runtime_version, undefined); + for (const field of ["previous_runtime_version", "for_app_version"]) { + const invalid = structuredClone(state); + if (field === "previous_runtime_version") invalid.transaction[field] = "not-a-version"; + else invalid.last_known_good_runtime = { version: previousVersion, path: f.previous, [field]: "not-a-version" }; + assert.throws(() => parsePersistedState(invalid)); + } +}); diff --git a/apps/desktop/test/update-coordinator.test.mjs b/apps/desktop/test/update-coordinator.test.mjs index e302b2d40..3e416c32a 100644 --- a/apps/desktop/test/update-coordinator.test.mjs +++ b/apps/desktop/test/update-coordinator.test.mjs @@ -76,9 +76,9 @@ function backend(overrides = {}) { async openExternal(url) { events.push(`open:${url}`); }, - async recover() { + async recover(transaction) { events.push("recover"); - return { healthy: true, rolledBack: false, message: "Recovered." }; + return { healthy: true, rolledBack: transaction.target_version !== this.currentVersion, message: "Recovered." }; }, ...overrides }; diff --git a/apps/editor/src/App.tsx b/apps/editor/src/App.tsx index f2779e26c..8920e614e 100644 --- a/apps/editor/src/App.tsx +++ b/apps/editor/src/App.tsx @@ -366,9 +366,8 @@ export function App({ gateway }: { gateway: CollectionGateway }) { } }, [fileController, loadIndex, refreshDescription]); - const updateNoteSummary = useCallback((next: NoteDocument, previousPath = next.path) => { - indexController.upsert(summaryFromDocument(next), previousPath); - }, [indexController]); + const updateNoteSummary = useCallback((next: NoteDocument, previousPath = next.path) => + indexController.upsert(summaryFromDocument(next), previousPath), [indexController]); const publishNoteHistory = useCallback((next: NoteNavigationHistory) => { noteHistory.current = next; diff --git a/config/architecture-budgets.json b/config/architecture-budgets.json index 66f5b15b8..5cab18683 100644 --- a/config/architecture-budgets.json +++ b/config/architecture-budgets.json @@ -44,10 +44,10 @@ }, "reviewBudgets": { "productionFiles": 691, - "relativeImports": 1504, + "relativeImports": 1505, "workspacePackages": 24, "rustPublicDeclarations": 3185, - "typeScriptExportDeclarations": 2462, + "typeScriptExportDeclarations": 2463, "mdbaseCollectionReferences": 16, "typedCollectionReferences": 1 } diff --git a/docs/code-quality.md b/docs/code-quality.md index b4c5c38bd..d41115ed4 100644 --- a/docs/code-quality.md +++ b/docs/code-quality.md @@ -139,6 +139,13 @@ belongs at each boundary: boot/update races, pending-handle/component tests, exact-ACK and real PostgreSQL serialization tests, and real isolated-process credential retry. These limits do not replace review or platform qualification. +The rollback-admission correction reuses the control client's existing protocol +constant in updater health verification: one additional relative import and one +export, making the reviewed totals 1,505 imports and 2,463 TypeScript exports. +No production module, file-size limit, or cycle allowance is added. The existing +update record now binds a verified fallback to its app version and distinguishes +that daemon version during subsequent handoff; it is not a second journal. + Composition roots and package facades should approach these end-state shapes: - server `app.ts`: registration and lifecycle wiring only; diff --git a/docs/desktop-updates.md b/docs/desktop-updates.md index c2895bb93..070a7f08f 100644 --- a/docs/desktop-updates.md +++ b/docs/desktop-updates.md @@ -104,7 +104,20 @@ transaction only after verified health. A failed or thrown recovery retains the same transaction and preserved runtime for the next process, blocking further installation and ordinary startup rather than claiming restoration. If the target daemon cannot start or migrate state, the new app re-registers and health-checks the preserved previous daemon. Recovery remains -visible so the user can install a higher signed recovery release. +visible so the user can install a higher signed recovery release. Startup uses +that exact verified runtime's version and preserved CLI, not the newer app's +version or bundled CLI. Canonical readiness and the desktop's local control +protocol must both match; unrelated or incompatible daemons remain blocked. + +A healthy fallback is recorded in the existing `last_known_good_runtime` with +`for_app_version` binding it to the app that required rollback before the +transaction is cleared. Fresh launches reverify that selection rather than +silently reinstalling the failing bundled daemon. A newer app does not inherit +another app version's fallback selection. Subsequent automatic transactions +keep `previous_version` as the app version and add `previous_runtime_version` +when the preserved daemon differs; staging never overwrites the active fallback +with the failing bundle. Legacy schema-1 records omit these optional bindings +and retain their original coupled app/runtime version meaning. The platform installer rolls back a failed application-bundle replacement. After a new signed app has launched, mdbase uses publish-forward app recovery @@ -113,7 +126,8 @@ collection access available. This avoids a second privileged installer and keeps signing authority with macOS, Microsoft Store, or the Linux package manager. -Update state uses user-only permissions and atomic rename. Invalid state is +Update state uses user-only permissions and atomic rename; its in-memory cache +publishes a transition only after the durable write succeeds. Invalid state is quarantined. A crash at every boundary is safe to retry: before stop there is no service impact; after stop the previous runtime is recorded; after replacement recovery is idempotent. diff --git a/docs/exact-recovery-and-health.md b/docs/exact-recovery-and-health.md index a4e43de5b..450b6dfc5 100644 --- a/docs/exact-recovery-and-health.md +++ b/docs/exact-recovery-and-health.md @@ -38,7 +38,12 @@ An unhealthy or thrown updater recovery retains the same transaction and its last-known-good runtime in `recovering`. Update checks/reinstallation and ordinary startup remain blocked until recovery verifies readiness and the expected binary version. The next app process resumes the same transaction. No additional update -journal or repair owner is introduced. +journal or repair owner is introduced. Healthy rollback binds the verified +preserved binary/version to that app in the existing last-known-good record. +Admission and later CLI startup use that selection, including after a fresh +process. The next update preserves the selected daemon rather than copying the +failing bundle over it. Readiness schema, exact version, private runtime path +and local control protocol checks remain fail-closed. ## Editor continuation @@ -58,7 +63,11 @@ Collection epochs prevent recovery from publishing into a subsequently selected collection. Unknown outcomes and failed recovery probes preserve the original identity and local draft. A definitive recovery rejection is propagated and settles that pending identity without discarding the draft or automatically -resending it; the editor returns to its conflict/error state. +resending it; the editor returns to its conflict/error state. Settlement is +checked against the SDK's existing pending handles, not inferred from a probe's +error code or outcome marker. Even a rejected recovery probe can leave the +original attempt unknown; conversely an authoritative not-sent response can +settle it. No second mutation owner is introduced. ## Revocation has an authority-specific completion point From d3fd97a957d6e74ec61c5fa2a4f8632c82e24bce Mon Sep 17 00:00:00 2001 From: callumalpass Date: Tue, 15 Sep 2026 08:11:25 +1000 Subject: [PATCH 14/16] Prepare beta100 exact recovery and truthful health release --- Cargo.lock | 16 ++++++++-------- Cargo.toml | 2 +- apps/desktop/package.json | 2 +- apps/portal/package.json | 2 +- deploy/docker/Cargo.lock.hosted-provider | 16 ++++++++-------- package.json | 2 +- packages/client/package.json | 2 +- packages/devkit/package.json | 2 +- packages/management/package.json | 2 +- packages/pickle/package.json | 2 +- packages/protocol/package.json | 2 +- packages/sync/package.json | 2 +- packages/testing/package.json | 2 +- packages/ui/package.json | 2 +- packages/webhooks/package.json | 2 +- services/mcp/package.json | 2 +- services/mcp/src/mcp.ts | 2 +- services/server/package.json | 2 +- 18 files changed, 32 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a8caeb833..b3b64a3e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1222,7 +1222,7 @@ dependencies = [ [[package]] name = "connect-hosted-storage-benchmark" -version = "0.1.0-beta.99" +version = "0.1.0-beta.100" dependencies = [ "aes-gcm", "chrono", @@ -2680,7 +2680,7 @@ dependencies = [ [[package]] name = "mdbase-cli" -version = "0.1.0-beta.99" +version = "0.1.0-beta.100" dependencies = [ "chrono-tz", "clap", @@ -2717,7 +2717,7 @@ dependencies = [ [[package]] name = "mdbase-connect-core" -version = "0.1.0-beta.99" +version = "0.1.0-beta.100" dependencies = [ "chrono", "chrono-tz", @@ -2746,7 +2746,7 @@ dependencies = [ [[package]] name = "mdbase-connect-daemon" -version = "0.1.0-beta.99" +version = "0.1.0-beta.100" dependencies = [ "async-trait", "axum", @@ -2787,7 +2787,7 @@ dependencies = [ [[package]] name = "mdbase-connect-hosted-provider" -version = "0.1.0-beta.99" +version = "0.1.0-beta.100" dependencies = [ "aes-gcm", "async-trait", @@ -2834,7 +2834,7 @@ dependencies = [ [[package]] name = "mdbase-connect-mirror" -version = "0.1.0-beta.99" +version = "0.1.0-beta.100" dependencies = [ "async-trait", "axum", @@ -2861,7 +2861,7 @@ dependencies = [ [[package]] name = "mdbase-connect-protocol" -version = "0.1.0-beta.99" +version = "0.1.0-beta.100" dependencies = [ "aes-gcm", "base64", @@ -2880,7 +2880,7 @@ dependencies = [ [[package]] name = "mdbase-connect-runtime" -version = "0.1.0-beta.99" +version = "0.1.0-beta.100" dependencies = [ "chrono", "mdbase", diff --git a/Cargo.toml b/Cargo.toml index 664689880..73d55a7ca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.1.0-beta.99" +version = "0.1.0-beta.100" edition = "2021" license = "MIT" repository = "https://github.com/mdbase-dev/mdbase-connect" diff --git a/apps/desktop/package.json b/apps/desktop/package.json index f54e940d7..a55d66319 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -1,7 +1,7 @@ { "name": "@mdbase/connect-desktop", "productName": "mdbase connect", - "version": "0.1.0-beta.99", + "version": "0.1.0-beta.100", "description": "Connect applications to authorized mdbase collections.", "author": "mdbase", "private": true, diff --git a/apps/portal/package.json b/apps/portal/package.json index c599da1ca..44e9d2608 100644 --- a/apps/portal/package.json +++ b/apps/portal/package.json @@ -1,6 +1,6 @@ { "name": "@mdbase/connect-portal", - "version": "0.1.0-beta.99", + "version": "0.1.0-beta.100", "private": true, "type": "module", "scripts": { diff --git a/deploy/docker/Cargo.lock.hosted-provider b/deploy/docker/Cargo.lock.hosted-provider index a8caeb833..b3b64a3e6 100644 --- a/deploy/docker/Cargo.lock.hosted-provider +++ b/deploy/docker/Cargo.lock.hosted-provider @@ -1222,7 +1222,7 @@ dependencies = [ [[package]] name = "connect-hosted-storage-benchmark" -version = "0.1.0-beta.99" +version = "0.1.0-beta.100" dependencies = [ "aes-gcm", "chrono", @@ -2680,7 +2680,7 @@ dependencies = [ [[package]] name = "mdbase-cli" -version = "0.1.0-beta.99" +version = "0.1.0-beta.100" dependencies = [ "chrono-tz", "clap", @@ -2717,7 +2717,7 @@ dependencies = [ [[package]] name = "mdbase-connect-core" -version = "0.1.0-beta.99" +version = "0.1.0-beta.100" dependencies = [ "chrono", "chrono-tz", @@ -2746,7 +2746,7 @@ dependencies = [ [[package]] name = "mdbase-connect-daemon" -version = "0.1.0-beta.99" +version = "0.1.0-beta.100" dependencies = [ "async-trait", "axum", @@ -2787,7 +2787,7 @@ dependencies = [ [[package]] name = "mdbase-connect-hosted-provider" -version = "0.1.0-beta.99" +version = "0.1.0-beta.100" dependencies = [ "aes-gcm", "async-trait", @@ -2834,7 +2834,7 @@ dependencies = [ [[package]] name = "mdbase-connect-mirror" -version = "0.1.0-beta.99" +version = "0.1.0-beta.100" dependencies = [ "async-trait", "axum", @@ -2861,7 +2861,7 @@ dependencies = [ [[package]] name = "mdbase-connect-protocol" -version = "0.1.0-beta.99" +version = "0.1.0-beta.100" dependencies = [ "aes-gcm", "base64", @@ -2880,7 +2880,7 @@ dependencies = [ [[package]] name = "mdbase-connect-runtime" -version = "0.1.0-beta.99" +version = "0.1.0-beta.100" dependencies = [ "chrono", "mdbase", diff --git a/package.json b/package.json index 61b87a86b..d505dd0c8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mdbase-connect", - "version": "0.1.0-beta.99", + "version": "0.1.0-beta.100", "private": true, "packageManager": "pnpm@11.15.1", "engines": { diff --git a/packages/client/package.json b/packages/client/package.json index f8eb1b20c..32b639e48 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@mdbase-dev/connect", - "version": "0.1.0-beta.99", + "version": "0.1.0-beta.100", "private": false, "type": "module", "license": "MIT", diff --git a/packages/devkit/package.json b/packages/devkit/package.json index 423501146..7d8e9b401 100644 --- a/packages/devkit/package.json +++ b/packages/devkit/package.json @@ -1,6 +1,6 @@ { "name": "@mdbase-dev/connect-dev", - "version": "0.1.0-beta.99", + "version": "0.1.0-beta.100", "private": false, "type": "module", "license": "MIT", diff --git a/packages/management/package.json b/packages/management/package.json index 48679b655..8f0576ada 100644 --- a/packages/management/package.json +++ b/packages/management/package.json @@ -1,6 +1,6 @@ { "name": "@mdbase/connect-management", - "version": "0.1.0-beta.99", + "version": "0.1.0-beta.100", "private": true, "type": "module", "scripts": { diff --git a/packages/pickle/package.json b/packages/pickle/package.json index a7655398c..3c1a9633b 100644 --- a/packages/pickle/package.json +++ b/packages/pickle/package.json @@ -1,6 +1,6 @@ { "name": "@mdbase-dev/pickle", - "version": "0.1.0-beta.99", + "version": "0.1.0-beta.100", "private": false, "type": "module", "license": "MIT", diff --git a/packages/protocol/package.json b/packages/protocol/package.json index 374712aa9..7926b0625 100644 --- a/packages/protocol/package.json +++ b/packages/protocol/package.json @@ -1,6 +1,6 @@ { "name": "@mdbase-dev/connect-protocol", - "version": "0.1.0-beta.99", + "version": "0.1.0-beta.100", "private": false, "type": "module", "license": "MIT", diff --git a/packages/sync/package.json b/packages/sync/package.json index 0802a7b95..69e253824 100644 --- a/packages/sync/package.json +++ b/packages/sync/package.json @@ -1,6 +1,6 @@ { "name": "@mdbase-dev/connect-sync", - "version": "0.1.0-beta.99", + "version": "0.1.0-beta.100", "private": false, "type": "module", "license": "MIT", diff --git a/packages/testing/package.json b/packages/testing/package.json index 830b30e87..825d0aaff 100644 --- a/packages/testing/package.json +++ b/packages/testing/package.json @@ -1,6 +1,6 @@ { "name": "@mdbase-dev/connect-testing", - "version": "0.1.0-beta.99", + "version": "0.1.0-beta.100", "private": false, "type": "module", "license": "MIT", diff --git a/packages/ui/package.json b/packages/ui/package.json index 4561318f5..26277f5f4 100644 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -1,6 +1,6 @@ { "name": "@mdbase/connect-ui", - "version": "0.1.0-beta.99", + "version": "0.1.0-beta.100", "private": true, "type": "module", "scripts": { diff --git a/packages/webhooks/package.json b/packages/webhooks/package.json index 6d36f7871..d95e7bdc3 100644 --- a/packages/webhooks/package.json +++ b/packages/webhooks/package.json @@ -1,6 +1,6 @@ { "name": "@mdbase-dev/connect-webhooks", - "version": "0.1.0-beta.99", + "version": "0.1.0-beta.100", "private": false, "type": "module", "license": "MIT", diff --git a/services/mcp/package.json b/services/mcp/package.json index 002731ce7..feb1726e3 100644 --- a/services/mcp/package.json +++ b/services/mcp/package.json @@ -1,6 +1,6 @@ { "name": "@mdbase/connect-mcp", - "version": "0.1.0-beta.99", + "version": "0.1.0-beta.100", "private": true, "type": "module", "scripts": { diff --git a/services/mcp/src/mcp.ts b/services/mcp/src/mcp.ts index b5497b027..e7a98f2fa 100644 --- a/services/mcp/src/mcp.ts +++ b/services/mcp/src/mcp.ts @@ -16,7 +16,7 @@ export function createMcpServer( gateway: ConnectGateway, oauth: OAuthService ): McpServer { - const server = new McpServer({ name: "mdbase", version: "0.1.0-beta.99" }); + const server = new McpServer({ name: "mdbase", version: "0.1.0-beta.100" }); server.registerTool("list_connections", { title: "List mdbase collections", diff --git a/services/server/package.json b/services/server/package.json index 0e3c1a3c7..1f0d694f2 100644 --- a/services/server/package.json +++ b/services/server/package.json @@ -1,6 +1,6 @@ { "name": "@mdbase/connect-server", - "version": "0.1.0-beta.99", + "version": "0.1.0-beta.100", "private": true, "type": "module", "scripts": { From 0be25a2802611e0ed060392bcdee13c1a2d8c952 Mon Sep 17 00:00:00 2001 From: callumalpass Date: Tue, 15 Sep 2026 08:34:44 +1000 Subject: [PATCH 15/16] fix(desktop): keep action error text distinct from its dismiss control --- apps/desktop/src/renderer/main.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/desktop/src/renderer/main.tsx b/apps/desktop/src/renderer/main.tsx index 6415651f0..226f03397 100644 --- a/apps/desktop/src/renderer/main.tsx +++ b/apps/desktop/src/renderer/main.tsx @@ -362,7 +362,7 @@ function App() {
- {error &&
{error}
} + {error &&
{error}
} {presentResourceFailures(resourceFailures) &&
{presentResourceFailures(resourceFailures)}
} {notice &&
{notice}
}
From 71bc56861533a5b2c654cf613fb4a25145c41370 Mon Sep 17 00:00:00 2001 From: callumalpass Date: Tue, 15 Sep 2026 08:43:57 +1000 Subject: [PATCH 16/16] test(editor): observe collection freeze independently of autosave --- apps/editor/src/App.collection-switch.test.tsx | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/apps/editor/src/App.collection-switch.test.tsx b/apps/editor/src/App.collection-switch.test.tsx index 1ccea39cc..54ab8f444 100644 --- a/apps/editor/src/App.collection-switch.test.tsx +++ b/apps/editor/src/App.collection-switch.test.tsx @@ -8,12 +8,12 @@ import { DemoCollectionGateway } from "./demo-gateway"; import type { CollectionAuthorizationTarget, CollectionFile, CollectionSessionSnapshot, ConnectionSummary, CreateNoteInput, FileUploadRequest, MutationOperationOptions, NoteDocument, NoteIndexRequest, NoteIndexResult, SaveNoteInput } from "./model"; vi.mock("./CodeEditor", () => ({ CodeEditor: ({ value, onChange, label }: { value: string; onChange?: (value: string) => void; label: string }) =>