diff --git a/src/chat-swarm-runtime.test.ts b/src/chat-swarm-runtime.test.ts index 113c08ed..2719b746 100644 --- a/src/chat-swarm-runtime.test.ts +++ b/src/chat-swarm-runtime.test.ts @@ -7,6 +7,7 @@ import test from "node:test"; import { CdpMacWebDriver, OpenCliMacWebDriver, + MacWebChatCarrierAdapter, ChatSwarmRuntimeManager, ChatSwarmRuntimeStore, loadChatSwarmRuntimeConfig, @@ -139,7 +140,13 @@ class FakeManagedAdapter implements ChatSwarmManagedCarrierAdapter { } } -function fixture(workerLimit = 5) { +function fixture( + workerLimit = 5, + options: { + provisionStaggerMs?: number; + sleep?: (ms: number) => Promise; + } = {}, +) { const root = mkdtempSync(join(tmpdir(), "devspace-runtime-117-")); const store = new ChatSwarmStore(root); const coordinator = new ChatSwarmCoordinator(store); @@ -153,11 +160,12 @@ function fixture(workerLimit = 5) { DEVSPACE_CHAT_SWARM_POOL_DEFAULT: "3", DEVSPACE_CHAT_SWARM_RUNTIME_TIMEOUT_MS: "5000", DEVSPACE_CHAT_SWARM_BOOTSTRAP_WAIT_MS: "5000", + DEVSPACE_CHAT_SWARM_PROVISION_STAGGER_MS: String(options.provisionStaggerMs ?? 0), }; const manager = new ChatSwarmRuntimeManager( coordinator, { stateDir: root, chatSwarmMaxWorkers: workerLimit }, - { env, registry, adapter }, + { env, registry, adapter, sleep: options.sleep }, ); adapter.onBootstrap = (operationId, rawIdentity) => { manager.bootstrap({ "openai/session": rawIdentity }, operationId); @@ -185,6 +193,7 @@ function cdpDriverForSelectorTest() { appLabel: "dev", operationTimeoutMs: 5_000, bootstrapWaitMs: 5_000, + provisionStaggerMs: 0, }); } @@ -236,6 +245,7 @@ function openCliDriverForTest() { appLabel: "dev", operationTimeoutMs: 5_000, bootstrapWaitMs: 5_000, + provisionStaggerMs: 0, }); } @@ -248,6 +258,7 @@ test("runtime config selects OpenCLI explicitly while preserving CDP as the defa assert.equal(opencli.transport, "opencli"); assert.equal(opencli.openCliExecutable, "/Users/test/.npm-global/bin/opencli"); assert.equal(opencli.appLabel, "dev"); + assert.equal(opencli.provisionStaggerMs, 8_000); const fallback = loadChatSwarmRuntimeConfig(base, {}); assert.equal(fallback.transport, "cdp"); }); @@ -323,6 +334,114 @@ test("OpenCLI provisioning fails closed when the authenticated peer probe is mal ); }); +test("OpenCLI provisioning durably records the exact conversation before peer probe completion", async () => { + const f = fixture(); + try { + const driver = openCliDriverForTest(); + (driver as any).runJson = async (args: string[]) => { + if (args[1] === "detail") { + throw new Error("OpenCLI command exceeded deadline"); + } + return [{ + conversationId: "opencli-managed-partial", + conversationUrl: "https://chatgpt.com/g/g-p-runtime-test/c/opencli-managed-partial", + response: "", + }]; + }; + const adapter = new MacWebChatCarrierAdapter( + f.manager.runtimeConfig, + f.registry, + driver, + ); + const slot = f.registry.ensureSlot( + f.swarm.id, + 1, + "https://chatgpt.com/g/g-p-runtime-test/project", + "1".repeat(64), + ); + const prepared = f.registry.prepareProvision(slot, 5_000); + assert.ok(prepared.operation); + assert.equal(f.registry.claimProvision(prepared.operation.operationId), true); + + await assert.rejects( + () => adapter.provision({ + operationId: prepared.operation!.operationId, + swarmId: f.swarm.id, + runtimeSlot: 1, + projectUrl: "https://chatgpt.com/g/g-p-runtime-test/project", + deadlineAt: new Date(Date.now() + 1_000).toISOString(), + }), + /OPENCLI_PEER_PROBE_FAILED:OpenCLI command exceeded deadline/, + ); + + const observed = f.registry.getProvision(prepared.operation.operationId)!; + assert.equal(observed.status, "transport_observed"); + assert.equal(observed.receipt?.disposition, "TRANSPORT_OBSERVED"); + assert.equal( + observed.receipt?.conversationUrl, + "https://chatgpt.com/g/g-p-runtime-test/c/opencli-managed-partial", + ); + assert.equal( + observed.receipt?.conversationFingerprint, + fingerprint("opencli-managed-partial"), + ); + + f.registry.markProvisionUnknown( + prepared.operation.operationId, + "OPENCLI_PEER_PROBE_FAILED:OpenCLI command exceeded deadline", + ); + const reconciled = f.registry.getSlot(f.swarm.id, 1)!; + assert.equal(reconciled.state, "RECONCILE_REQUIRED"); + assert.equal( + reconciled.conversationUrl, + "https://chatgpt.com/g/g-p-runtime-test/c/opencli-managed-partial", + ); + assert.equal( + reconciled.conversationFingerprint, + fingerprint("opencli-managed-partial"), + ); + } finally { + cleanup(f); + } +}); + +test("transport-observed provision state fails closed on replay without reprovision or bootstrap", async () => { + const f = fixture(); + try { + const slot = f.registry.ensureSlot( + f.swarm.id, + 1, + "https://chatgpt.com/g/g-p-runtime-test/project", + "1".repeat(64), + ); + const prepared = f.registry.prepareProvision(slot, 5_000); + assert.ok(prepared.operation); + assert.equal(f.registry.claimProvision(prepared.operation.operationId), true); + f.registry.markTransportObserved(prepared.operation.operationId, { + conversationUrl: "https://chatgpt.com/g/g-p-runtime-test/c/crash-window-conversation", + conversationFingerprint: fingerprint("crash-window-conversation"), + }); + + const replay = await f.manager.ensure(f.owner, f.swarm.id, 1); + assert.equal(replay.state, "RECONCILE_REQUIRED"); + assert.equal(replay.slots[0]?.state, "RECONCILE_REQUIRED"); + assert.equal( + replay.slots[0]?.conversationUrl, + "https://chatgpt.com/g/g-p-runtime-test/c/crash-window-conversation", + ); + assert.equal(f.adapter.provisionCalls, 0); + assert.equal(f.adapter.bootstrapCalls, 0); + const operation = f.registry.getProvision(prepared.operation.operationId)!; + assert.equal(operation.status, "outcome_unknown"); + assert.equal( + replay.slots[0]?.blocker, + "TRANSPORT_IDENTITY_OBSERVED_PEER_IDENTITY_UNVERIFIED", + ); + } finally { + cleanup(f); + } +}); + test("OpenCLI wake reopens the exact conversation", async () => { const driver = openCliDriverForTest(); const calls: string[][] = []; @@ -429,6 +548,27 @@ test("concurrent runtime ensure creates only missing managed workers and exact r } }); +test("fresh multi-worker ensure staggers external carrier creation without delaying healthy replay", async () => { + const delays: number[] = []; + const f = fixture(5, { + provisionStaggerMs: 8_000, + sleep: async (ms) => { delays.push(ms); }, + }); + try { + const first = await f.manager.ensure(f.owner, f.swarm.id, 3); + assert.equal(first.slots.filter((slot) => slot.state === "PARKED").length, 3); + assert.deepEqual(delays, [8_000, 8_000]); + assert.equal(f.adapter.provisionCalls, 3); + + const replay = await f.manager.ensure(f.owner, f.swarm.id, 3); + assert.equal(replay.slots.filter((slot) => slot.state === "PARKED").length, 3); + assert.deepEqual(delays, [8_000, 8_000]); + assert.equal(f.adapter.provisionCalls, 3); + } finally { + cleanup(f); + } +}); + test("cold ensure reconciles exact existing carriers before declaring the pool healthy", async () => { const f = fixture(); try { diff --git a/src/chat-swarm-runtime.ts b/src/chat-swarm-runtime.ts index c2a820c9..1c7fc50c 100644 --- a/src/chat-swarm-runtime.ts +++ b/src/chat-swarm-runtime.ts @@ -34,6 +34,7 @@ const MAX_RUNTIME_WORKERS = 64; const DEFAULT_RUNTIME_WORKERS = 3; const DEFAULT_OPERATION_TIMEOUT_MS = 60_000; const DEFAULT_BOOTSTRAP_WAIT_MS = 45_000; +const DEFAULT_PROVISION_STAGGER_MS = 8_000; const MAX_RUNTIME_TIMEOUT_MS = 120_000; type Row = Record; @@ -71,6 +72,7 @@ export interface ChatSwarmRuntimeConfig { appLabel: string; operationTimeoutMs: number; bootstrapWaitMs: number; + provisionStaggerMs: number; } export interface ManagedCarrierSlot { @@ -128,6 +130,7 @@ interface ProvisionReceipt { schema: typeof PROVISION_RECEIPT_SCHEMA; disposition: | "PREPARED" + | "TRANSPORT_OBSERVED" | "CARRIER_CREATED" | "BOOTSTRAPPING" | "BOUND" @@ -197,9 +200,12 @@ export interface RuntimeStatusResult { slots: ManagedCarrierSlot[]; } -export interface ManagedConversationEvidence { +export interface TransportConversationEvidence { conversationUrl: string; conversationFingerprint: string; +} + +export interface ManagedConversationEvidence extends TransportConversationEvidence { authenticatedPeerFingerprint?: string; appBinding: "READY" | "UNKNOWN" | "DISABLED" | "STALE"; } @@ -240,6 +246,7 @@ export interface MacWebDriver { createManagedConversation( projectUrl: string, deadlineAt: string, + onTransportObserved?: (evidence: TransportConversationEvidence) => void, ): Promise; sendPrompt( conversationUrl: string, @@ -342,6 +349,13 @@ export function loadChatSwarmRuntimeConfig( MAX_RUNTIME_TIMEOUT_MS, "DEVSPACE_CHAT_SWARM_BOOTSTRAP_WAIT_MS", ), + provisionStaggerMs: boundedInt( + env.DEVSPACE_CHAT_SWARM_PROVISION_STAGGER_MS, + DEFAULT_PROVISION_STAGGER_MS, + 0, + MAX_RUNTIME_TIMEOUT_MS, + "DEVSPACE_CHAT_SWARM_PROVISION_STAGGER_MS", + ), }; } @@ -646,6 +660,68 @@ export class ChatSwarmRuntimeStore { return result.changes === 1; } + markTransportObserved( + operationId: string, + evidence: TransportConversationEvidence, + ): ManagedCarrierSlot { + assertFingerprint(evidence.conversationFingerprint, "conversation fingerprint"); + const tx = this.database.sqlite.transaction(() => { + const operation = this.requireProvision(operationId); + if (operation.status === "outcome_unknown") { + throw new ChatSwarmError( + "RECONCILIATION_REQUIRED", + "provision outcome is unknown; do not create another carrier", + ); + } + if (operation.status !== "started") { + if ( + operation.receipt?.conversationFingerprint === evidence.conversationFingerprint && + operation.receipt?.conversationUrl === evidence.conversationUrl + ) { + return this.getSlot(operation.request.swarmId, operation.request.runtimeSlot)!; + } + throw new ChatSwarmError("INVALID_STATE", "provision operation is not active"); + } + const conflicting = this.getSlotByFingerprint(evidence.conversationFingerprint); + if ( + conflicting && + (conflicting.swarmId !== operation.request.swarmId || + conflicting.runtimeSlot !== operation.request.runtimeSlot) + ) { + throw new ChatSwarmError( + "OWNERSHIP_CONFLICT", + "conversation is already managed by another runtime slot", + ); + } + const observedAt = nowIso(); + const receipt: ProvisionReceipt = { + schema: PROVISION_RECEIPT_SCHEMA, + disposition: "TRANSPORT_OBSERVED", + conversationUrl: evidence.conversationUrl, + conversationFingerprint: evidence.conversationFingerprint, + remoteMayContinue: true, + observedAt, + }; + this.updateProvision(operationId, "transport_observed", receipt); + const slot = this.getSlot(operation.request.swarmId, operation.request.runtimeSlot)!; + this.updateSlotReceipt( + slot, + { + schema: SLOT_RECEIPT_SCHEMA, + generation: operation.request.generation, + state: "PROVISIONING", + conversationUrl: evidence.conversationUrl, + conversationFingerprint: evidence.conversationFingerprint, + lastOperationId: operationId, + updatedAt: observedAt, + }, + "started", + ); + return this.getSlot(operation.request.swarmId, operation.request.runtimeSlot)!; + }); + return tx.immediate(); + } + markCarrierCreated( operationId: string, evidence: ManagedConversationEvidence, @@ -668,7 +744,7 @@ export class ChatSwarmRuntimeStore { "provision outcome is unknown; do not create another carrier", ); } - if (operation.status !== "started") { + if (operation.status !== "started" && operation.status !== "transport_observed") { throw new ChatSwarmError("INVALID_STATE", "provision operation is not active"); } const conflicting = this.getSlotByFingerprint(evidence.conversationFingerprint); @@ -1409,43 +1485,61 @@ export class OpenCliMacWebDriver implements MacWebDriver { async createManagedConversation( projectUrl: string, deadlineAt: string, + onTransportObserved?: (evidence: TransportConversationEvidence) => void, ): Promise { const probePrompt = this.peerIdentityProbePrompt(); - const rows = await this.runJson( - [ - "chatgpt", - "ask", - probePrompt, - "--project", - openCliProjectId(projectUrl), - "--new", - "true", - "--wait", - "false", - "--site-session", - "ephemeral", - "--keep-tab", - "false", - "-f", - "json", - ], - deadlineAt, - ); + let rows: OpenCliConversationRow[]; + try { + rows = await this.runJson( + [ + "chatgpt", + "ask", + probePrompt, + "--project", + openCliProjectId(projectUrl), + "--new", + "true", + "--wait", + "false", + "--site-session", + "ephemeral", + "--keep-tab", + "false", + "-f", + "json", + ], + deadlineAt, + ); + } catch (error) { + throw new Error( + `OPENCLI_CREATE_CONVERSATION_FAILED:${error instanceof Error ? error.message : String(error)}`, + ); + } const conversationUrl = rows[0]?.conversationUrl?.trim(); if (!conversationUrl) { - throw new Error("OpenCLI did not return a ChatGPT conversation URL"); + throw new Error("OPENCLI_CREATE_CONVERSATION_FAILED:missing conversation URL"); } - await this.waitForConversationIdle(conversationUrl, deadlineAt); - const authenticatedPeerFingerprint = this.peerFingerprintFromDetail( - await this.detail(conversationUrl, deadlineAt), - probePrompt, - ); - return { + const transportEvidence: TransportConversationEvidence = { conversationUrl, conversationFingerprint: conversationFingerprintFromUrl(conversationUrl), - authenticatedPeerFingerprint, - appBinding: "READY", }; + onTransportObserved?.(transportEvidence); + try { + await this.waitForConversationIdle(conversationUrl, deadlineAt); + const authenticatedPeerFingerprint = this.peerFingerprintFromDetail( + await this.detail(conversationUrl, deadlineAt), + probePrompt, + ); + return { + ...transportEvidence, + authenticatedPeerFingerprint, + appBinding: "READY", + }; + } catch (error) { + throw new Error( + `OPENCLI_PEER_PROBE_FAILED:${error instanceof Error ? error.message : String(error)}`, + ); + } } async sendPrompt( @@ -1702,6 +1796,7 @@ export class CdpMacWebDriver implements MacWebDriver { async createManagedConversation( projectUrl: string, deadlineAt: string, + onTransportObserved?: (evidence: TransportConversationEvidence) => void, ): Promise { await this.ensureRuntime(deadlineAt); const target = await this.newTarget(projectUrl, deadlineAt); @@ -1713,9 +1808,13 @@ export class CdpMacWebDriver implements MacWebDriver { deadlineAt, ); const conversationUrl = await this.waitForConversationUrl(target, deadlineAt); - return { + const evidence = { conversationUrl, conversationFingerprint: conversationFingerprintFromUrl(conversationUrl), + }; + onTransportObserved?.(evidence); + return { + ...evidence, appBinding, }; } @@ -2035,7 +2134,13 @@ export class MacWebChatCarrierAdapter implements ChatSwarmManagedCarrierAdapter projectUrl: string; deadlineAt: string; }): Promise { - return this.driver.createManagedConversation(input.projectUrl, input.deadlineAt); + return this.driver.createManagedConversation( + input.projectUrl, + input.deadlineAt, + (evidence) => { + this.registry.markTransportObserved(input.operationId, evidence); + }, + ); } async bootstrap(input: { @@ -2167,6 +2272,7 @@ export class ChatSwarmRuntimeManager { readonly registry: ChatSwarmRuntimeStore; readonly adapter: ChatSwarmManagedCarrierAdapter; readonly carrierManager: ChatSwarmCarrierManager; + private readonly sleepFn: (ms: number) => Promise; constructor( readonly coordinator: ChatSwarmCoordinator, @@ -2175,12 +2281,14 @@ export class ChatSwarmRuntimeManager { env?: NodeJS.ProcessEnv; adapter?: ChatSwarmManagedCarrierAdapter; registry?: ChatSwarmRuntimeStore; + sleep?: (ms: number) => Promise; } = {}, ) { this.runtimeConfig = loadChatSwarmRuntimeConfig(serverConfig, options.env); this.registry = options.registry ?? new ChatSwarmRuntimeStore(this.runtimeConfig.stateDir); this.adapter = options.adapter ?? new MacWebChatCarrierAdapter(this.runtimeConfig, this.registry); + this.sleepFn = options.sleep ?? ((ms) => new Promise((resolvePromise) => setTimeout(resolvePromise, ms))); this.carrierManager = new ChatSwarmCarrierManager( coordinator.store, coordinator, @@ -2259,6 +2367,7 @@ export class ChatSwarmRuntimeManager { const projectUrl = this.runtimeConfig.projectUrl!; const browserProfileId = profileId(this.runtimeConfig.browserProfileDir); + let createdCarriersThisEnsure = 0; for (let runtimeSlot = 1; runtimeSlot <= desiredWorkers; runtimeSlot += 1) { let slot = this.registry.ensureSlot( swarmId, @@ -2289,6 +2398,13 @@ export class ChatSwarmRuntimeManager { if (operation.status === "outcome_unknown" || slot.state === "RECONCILE_REQUIRED") { break; } + if (operation.status === "transport_observed") { + this.registry.markProvisionUnknown( + operation.operationId, + "TRANSPORT_IDENTITY_OBSERVED_PEER_IDENTITY_UNVERIFIED", + ); + break; + } if (operation.status === "succeeded") { continue; } @@ -2305,6 +2421,9 @@ export class ChatSwarmRuntimeManager { } if (operation.status === "started" && !operation.receipt?.conversationUrl) { + if (createdCarriersThisEnsure > 0 && this.runtimeConfig.provisionStaggerMs > 0) { + await this.sleepFn(this.runtimeConfig.provisionStaggerMs); + } let evidence: ManagedConversationEvidence; try { evidence = await this.adapter.provision({ @@ -2325,6 +2444,7 @@ export class ChatSwarmRuntimeManager { } slot = this.registry.markCarrierCreated(operation.operationId, evidence); operation = this.registry.getProvision(operation.operationId)!; + createdCarriersThisEnsure += 1; } if (slot.state === "SETUP_REQUIRED" || !slot.conversationUrl) continue;