fix(hub): keep the local sidecar running through a hub restart

.env.hub.example

@@ -98,6 +98,9 @@ PRODUCT_HELLO_WORDING=
 # Agents run in sidecars. By default the hub starts them as local processes.
 # Most local sidecars running at once.
 # SIDECAR_LOCAL_MAX_ACTIVE=16
+# Whether the local sidecar keeps running through a hub restart for the next hub to adopt. Defaults to true unless
+# the hub is a container's first process, where the container stops with it.
+# SIDECAR_LOCAL_OUTLIVES_HUB=true
 # Longest wait, in milliseconds, for a project's coordinator to start. Raise it on a slow machine; it must stay under 300000.
 # COORDINATOR_START_TIMEOUT_MS=60000
 # Optional Docker sidecars. deploy/compose.docker-sidecars.yml sets them for the hub

apps/hub/src/app.ts

@@ -667,15 +667,16 @@ export async function createHubRuntime(
         workflows.start();
         for (const scheduler of projectSchedulers) scheduler.start();
       },
-      async close() {
+      /** `keepSidecar` leaves the local sidecar running for the next hub to adopt; a full stop takes it down. */
+      async close(options?: { readonly keepSidecar?: boolean }) {
         modelLogins.stop();
         for (const scheduler of projectSchedulers) scheduler.stop();
         workflows.stop();
         liveApps.runner.close();
         liveApps.proxy.close();
         agentActivity.dispose();
         sidecarWebSocket.close();
-        await workflows.drain();
+        await workflows.drain(options);
         sessions.dispose();
         authEmail?.close();
         await handle.close();

apps/hub/src/provisioning/workflow-services.ts

@@ -50,7 +50,7 @@ import { createPlacementChooser } from "../sidecar/placement.ts";
 import { restoreLocalSidecars } from "../sidecar/restore.ts";
 import {
   childProcessEnv,
-  createSpawnLocalSidecar,
+  createLocalSidecarLauncher,
   stopStaleLocalSidecar,
 } from "../sidecar/spawn.ts";
 
@@ -123,9 +123,14 @@ export async function createWorkflowServices(options: {
   const dataRoot = localSidecarDataRoot(
     environment["HUB_DATA_DIR"] ?? "data/hub",
   );
+  // A container's runtime stops every process in it once the hub, its first process, exits, so a sidecar left running
+  // there is never adopted; the hub stops it gracefully instead.
+  const sidecarOutlivesHub =
+    (environment["SIDECAR_LOCAL_OUTLIVES_HUB"] ?? String(process.pid !== 1)) ===
+    "true";
   const local = createLocalProcessSidecarProvisioner({
     dataRoot,
-    spawnSidecar: createSpawnLocalSidecar({
+    ...createLocalSidecarLauncher({
       interchangeDir: INTERCHANGE_DIR,
       credentialEncryptionKey,
       maxActiveDeployments,
@@ -288,7 +293,12 @@ export async function createWorkflowServices(options: {
     retireProvisionerSidecar: allocationStore.retireProvisionerSidecar,
     isConnected: router.holdsAllocatedBinding,
   });
-  await restoreLocalSidecars(db, local, adapterManifest);
+  await restoreLocalSidecars(
+    db,
+    local,
+    adapterManifest,
+    router.holdsAllocatedBinding,
+  );
   const pending = new Set<Promise<unknown>>();
   const track = <T>(run: () => Promise<T>): Promise<T> => {
     const task = Promise.resolve().then(run);
@@ -390,9 +400,11 @@ export async function createWorkflowServices(options: {
       for (const scheduler of schedulers) scheduler.stop();
       for (const off of unsubscribers) off();
     },
-    async drain() {
+    async drain(options?: { readonly keepSidecar?: boolean }) {
       await Promise.allSettled(pending);
-      await local.shutdown();
+      await local.shutdown({
+        keepSidecar: sidecarOutlivesHub && options?.keepSidecar === true,
+      });
     },
   };
 }

apps/hub/src/server.ts

@@ -12,10 +12,11 @@ const errorMetadata = (error: unknown) => ({
 if (import.meta.main) {
   await setupHubLogging();
   const log = getLogger(["corbits-builder", "hub"]);
-  let close = async () => {};
+  type Stop = { readonly keepSidecar: boolean };
+  let close: (how: Stop) => Promise<void> = async () => {};
   let stopping: Promise<void> | undefined;
-  const shutDown = async () => {
-    for (const step of [close, dispose]) {
+  const shutDown = async (how: Stop) => {
+    for (const step of [() => close(how), dispose]) {
       try {
         await step();
       } catch (error) {
@@ -24,11 +25,14 @@ if (import.meta.main) {
       }
     }
   };
-  const stop = () => (stopping ??= shutDown());
+  // A restart (SIGTERM from a supervisor, a fatal error or a failed start) leaves the local sidecar running for the next hub to
+  // adopt, so turns in flight carry on; SIGINT, as when the dev script exits, is a full stop that takes it down.
+  const stop = (how: Stop) => (stopping ??= shutDown(how));
+  const restart = { keepSidecar: true };
   const exitOnFatal = (message: string) => async (error: unknown) => {
     try {
       log.error(message, errorMetadata(error));
-      await stop();
+      await stop(restart);
     } finally {
       // Handles a failed shutdown left open would keep a broken hub alive instead of letting its supervisor restart it.
       process.exit(1);
@@ -40,10 +44,10 @@ if (import.meta.main) {
   );
   process.on("uncaughtException", exitOnFatal("Hub hit an uncaught exception"));
   process.once("SIGTERM", () => {
-    void stop();
+    void stop(restart);
   });
   process.once("SIGINT", () => {
-    void stop();
+    void stop({ keepSidecar: false });
   });
   try {
     const runtime = await createHubRuntime();
@@ -55,9 +59,9 @@ if (import.meta.main) {
       fetch: runtime.app.fetch,
       websocket: runtime.websocket,
     });
-    close = async () => {
+    close = async (how) => {
       await server.stop(true);
-      await runtime.close();
+      await runtime.close(how);
     };
     await runtime.start();
     log.info(
@@ -69,6 +73,6 @@ if (import.meta.main) {
       errorMetadata(error),
     );
     process.exitCode = 1;
-    await stop();
+    await stop(restart);
   }
 }