test: reproduce runtime bootstrap lease expiry

src/chat-swarm-runtime.test.ts

@@ -42,6 +42,8 @@ class FakeManagedAdapter implements ChatSwarmManagedCarrierAdapter {
   failProvision = false;
   failStop = false;
   recoverReady = true;
+  provisionDelayMs = 0;
+  bootstrapDelayMs = 0;
   onBootstrap?: (operationId: string, rawIdentity: string) => void;
   readonly rawIdentities = new Map<number, string>();
 
@@ -72,6 +74,9 @@ class FakeManagedAdapter implements ChatSwarmManagedCarrierAdapter {
     deadlineAt: string;
   }): Promise<ManagedConversationEvidence> {
     this.provisionCalls += 1;
+    if (this.provisionDelayMs > 0) {
+      await new Promise((resolvePromise) => setTimeout(resolvePromise, this.provisionDelayMs));
+    }
     if (this.failProvision) throw new Error("response lost after possible create");
     const rawIdentity = `managed-conversation-${input.runtimeSlot}`;
     this.rawIdentities.set(input.runtimeSlot, rawIdentity);
@@ -91,6 +96,9 @@ class FakeManagedAdapter implements ChatSwarmManagedCarrierAdapter {
     deadlineAt: string;
   }) {
     this.bootstrapCalls += 1;
+    if (this.bootstrapDelayMs > 0) {
+      await new Promise((resolvePromise) => setTimeout(resolvePromise, this.bootstrapDelayMs));
+    }
     const rawIdentity = this.rawIdentities.get(input.runtimeSlot)!;
     this.onBootstrap?.(input.operationId, rawIdentity);
     return { disposition: "DELIVERED" as const, remoteMayContinue: true };
@@ -139,7 +147,10 @@ class FakeManagedAdapter implements ChatSwarmManagedCarrierAdapter {
   }
 }
 
-function fixture(workerLimit = 5) {
+function fixture(
+  workerLimit = 5,
+  timing: { operationTimeoutMs?: number; bootstrapWaitMs?: number } = {},
+) {
   const root = mkdtempSync(join(tmpdir(), "devspace-runtime-117-"));
   const store = new ChatSwarmStore(root);
   const coordinator = new ChatSwarmCoordinator(store);
@@ -150,9 +161,9 @@ function fixture(workerLimit = 5) {
   const env: NodeJS.ProcessEnv = {
     DEVSPACE_CHAT_SWARM_RUNTIME: "1",
     DEVSPACE_CHAT_SWARM_PROJECT_URL: "https://chatgpt.com/g/g-p-runtime-test/project",
-    DEVSPACE_CHAT_SWARM_POOL_DEFAULT: "3",
-    DEVSPACE_CHAT_SWARM_RUNTIME_TIMEOUT_MS: "5000",
-    DEVSPACE_CHAT_SWARM_BOOTSTRAP_WAIT_MS: "5000",
+    DEVSPACE_CHAT_SWARM_POOL_DEFAULT: String(Math.min(3, workerLimit)),
+    DEVSPACE_CHAT_SWARM_RUNTIME_TIMEOUT_MS: String(timing.operationTimeoutMs ?? 5_000),
+    DEVSPACE_CHAT_SWARM_BOOTSTRAP_WAIT_MS: String(timing.bootstrapWaitMs ?? 5_000),
   };
   const manager = new ChatSwarmRuntimeManager(
     coordinator,
@@ -404,6 +415,26 @@ test("concurrent runtime ensure creates only missing managed workers and exact r
   }
 });
 
+test("bootstrap authority lease covers sequential bounded provisioning phases", async () => {
+  const f = fixture(1, { operationTimeoutMs: 1_000, bootstrapWaitMs: 1_000 });
+  f.adapter.provisionDelayMs = 800;
+  f.adapter.bootstrapDelayMs = 300;
+  try {
+    const status = await f.manager.ensure(f.owner, f.swarm.id, 1);
+    assert.equal(status.slots[0]?.state, "PARKED");
+    assert.equal(f.adapter.provisionCalls, 1);
+    assert.equal(f.adapter.bootstrapCalls, 1);
+    assert.equal(
+      f.coordinator.store
+        .listWorkers(f.swarm.id)
+        .filter((worker) => worker.lifecycleState !== "DISABLED").length,
+      1,
+    );
+  } finally {
+    cleanup(f);
+  }
+});
+
 test("cold ensure reconciles exact existing carriers before declaring the pool healthy", async () => {
   const f = fixture();
   try {
@@ -439,7 +470,10 @@ test("managed bootstrap binds the authenticated peer separately from the browser
       authenticatedPeerFingerprint: peerFingerprint,
       appBinding: "READY",
     });
-    assert.equal(f.registry.claimBootstrap(prepared.operation!.operationId), true);
+    assert.equal(
+      f.registry.claimBootstrap(prepared.operation!.operationId, 5_000),
+      true,
+    );
     assert.throws(
       () =>
         f.manager.bootstrap(

src/chat-swarm-runtime.ts

@@ -136,6 +136,7 @@ interface ProvisionReceipt {
   conversationUrl?: string;
   conversationFingerprint?: string;
   authenticatedPeerFingerprint?: string;
+  bootstrapExpiresAt?: string;
   workerId?: string;
   remoteMayContinue: boolean;
   observedAt: string;
@@ -736,8 +737,15 @@ export class ChatSwarmRuntimeStore {
     return tx.immediate();
   }
 
-  claimBootstrap(operationId: string): boolean {
+  claimBootstrap(operationId: string, ttlMs: number): boolean {
+    if (!Number.isSafeInteger(ttlMs) || ttlMs <= 0) {
+      throw new ChatSwarmError(
+        "INVALID_STATE",
+        "bootstrap authority lease ttl must be a positive integer",
+      );
+    }
     const observedAt = nowIso();
+    const bootstrapExpiresAt = new Date(Date.parse(observedAt) + ttlMs).toISOString();
     const tx = this.database.sqlite.transaction(() => {
       const operation = this.requireProvision(operationId);
       if (operation.status !== "carrier_created") return false;
@@ -748,6 +756,7 @@ export class ChatSwarmRuntimeStore {
         conversationFingerprint: operation.receipt?.conversationFingerprint,
         authenticatedPeerFingerprint:
           operation.receipt?.authenticatedPeerFingerprint,
+        bootstrapExpiresAt,
         remoteMayContinue: true,
         observedAt,
       };
@@ -796,6 +805,7 @@ export class ChatSwarmRuntimeStore {
         conversationFingerprint: operation.receipt?.conversationFingerprint,
         authenticatedPeerFingerprint:
           operation.receipt?.authenticatedPeerFingerprint,
+        bootstrapExpiresAt: operation.receipt?.bootstrapExpiresAt,
         workerId: operation.receipt?.workerId,
         remoteMayContinue: true,
         observedAt,
@@ -870,6 +880,7 @@ export class ChatSwarmRuntimeStore {
         conversationFingerprint: operation.receipt.conversationFingerprint,
         authenticatedPeerFingerprint:
           operation.receipt.authenticatedPeerFingerprint,
+        bootstrapExpiresAt: operation.receipt.bootstrapExpiresAt,
         workerId: worker.id,
         remoteMayContinue: false,
         observedAt,
@@ -2340,7 +2351,10 @@ export class ChatSwarmRuntimeManager {
       }
 
       if (operation.status === "carrier_created") {
-        if (!this.registry.claimBootstrap(operation.operationId)) {
+        if (!this.registry.claimBootstrap(
+          operation.operationId,
+          this.runtimeConfig.operationTimeoutMs + this.runtimeConfig.bootstrapWaitMs,
+        )) {
           await this.waitForPeerInvocation(slot).catch(() => undefined);
           continue;
         }
@@ -2403,7 +2417,9 @@ export class ChatSwarmRuntimeManager {
         "runtime provision operation not found",
       );
     }
-    if (Date.parse(operation.request.expiresAt) <= Date.now()) {
+    const authorityExpiresAt =
+      operation.receipt?.bootstrapExpiresAt ?? operation.request.expiresAt;
+    if (Date.parse(authorityExpiresAt) <= Date.now()) {
       throw new ChatSwarmError(
         "REQUEST_EXPIRED",
         "runtime provision operation expired",