fix(hub): log fatal errors and exit so the supervisor restarts the hub

apps/hub/src/server.ts

@@ -2,10 +2,49 @@ import { getLogger } from "@intx/log";
 import { dispose } from "@logtape/logtape";
 import { createHubRuntime } from "./app.ts";
 import { setupHubLogging } from "./logging.ts";
+import { errorCode, requestErrorMetadata } from "./request-error.ts";
+
+const errorMetadata = (error: unknown) => ({
+  ...errorCode(error),
+  ...requestErrorMetadata(error),
+});
 
 if (import.meta.main) {
   await setupHubLogging();
   const log = getLogger(["corbits-builder", "hub"]);
+  let close = async () => {};
+  let stopping: Promise<void> | undefined;
+  const shutDown = async () => {
+    for (const step of [close, dispose]) {
+      try {
+        await step();
+      } catch (error) {
+        log.error("Hub shutdown failed", errorMetadata(error));
+        process.exitCode = 1;
+      }
+    }
+  };
+  const stop = () => (stopping ??= shutDown());
+  const exitOnFatal = (message: string) => async (error: unknown) => {
+    try {
+      log.error(message, errorMetadata(error));
+      await stop();
+    } finally {
+      // Handles a failed shutdown left open would keep a broken hub alive instead of letting its supervisor restart it.
+      process.exit(1);
+    }
+  };
+  process.on(
+    "unhandledRejection",
+    exitOnFatal("Hub hit an unhandled rejection"),
+  );
+  process.on("uncaughtException", exitOnFatal("Hub hit an uncaught exception"));
+  process.once("SIGTERM", () => {
+    void stop();
+  });
+  process.once("SIGINT", () => {
+    void stop();
+  });
   try {
     const runtime = await createHubRuntime();
     const server = Bun.serve({
@@ -14,28 +53,20 @@ if (import.meta.main) {
       fetch: runtime.app.fetch,
       websocket: runtime.websocket,
     });
+    close = async () => {
+      await server.stop(true);
+      await runtime.close();
+    };
     await runtime.start();
     log.info(
       "Hub ready: Postgres connected; local Interchange sidecar provisioner",
     );
-    let stopping = false;
-    const stop = async () => {
-      if (stopping) return;
-      stopping = true;
-      await server.stop(true);
-      await runtime.close();
-      await dispose();
-    };
-    process.once("SIGTERM", () => {
-      void stop();
-    });
-    process.once("SIGINT", () => {
-      void stop();
-    });
   } catch (error) {
-    log.error("Hub startup failed; check environment and database readiness", {
-      error,
-    });
+    log.error(
+      "Hub startup failed; check environment and database readiness",
+      errorMetadata(error),
+    );
     process.exitCode = 1;
+    await stop();
   }
 }

deploy/compose.yml

@@ -44,6 +44,8 @@ services:
       retries: 30
   hub:
     <<: *hub
+    # The hub exits on a fatal error so it restarts from a clean state.
+    restart: unless-stopped
     # The coder and designer run their commands under bubblewrap, which gives
     # each one its own user, pid and mount namespaces with a fresh /proc.
     # Docker's default seccomp profile refuses the namespaces without