From f5d6e8c0831ecdd36c49b555163f3797ec2c9714 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Wed, 4 Nov 2026 14:39:50 -0800 Subject: [PATCH 1/2] fix(hub): log fatal errors and exit so the supervisor restarts the hub --- apps/hub/src/server.ts | 65 +++++++++++++++++++++++++++++++----------- deploy/compose.yml | 2 ++ 2 files changed, 50 insertions(+), 17 deletions(-) diff --git a/apps/hub/src/server.ts b/apps/hub/src/server.ts index 3e6e60b4..3d2afb2b 100644 --- a/apps/hub/src/server.ts +++ b/apps/hub/src/server.ts @@ -2,10 +2,49 @@ import { getLogger } from "@intx/log"; import { dispose } from "@logtape/logtape"; import { createHubRuntime } from "./app.ts"; import { setupHubLogging } from "./logging.ts"; +import { errorCode, requestErrorMetadata } from "./request-error.ts"; + +const errorMetadata = (error: unknown) => ({ + ...errorCode(error), + ...requestErrorMetadata(error), +}); if (import.meta.main) { await setupHubLogging(); const log = getLogger(["corbits-builder", "hub"]); + let close = async () => {}; + let stopping: Promise | undefined; + const stop = () => + (stopping ??= (async () => { + for (const step of [close, dispose]) { + try { + await step(); + } catch (error) { + log.error("Hub shutdown failed", errorMetadata(error)); + process.exitCode = 1; + } + } + })()); + const exitOnFatal = (message: string) => async (error: unknown) => { + try { + log.error(message, errorMetadata(error)); + await stop(); + } finally { + // Handles a failed shutdown left open would keep a broken hub alive instead of letting its supervisor restart it. + process.exit(1); + } + }; + process.on( + "unhandledRejection", + exitOnFatal("Hub hit an unhandled rejection"), + ); + process.on("uncaughtException", exitOnFatal("Hub hit an uncaught exception")); + process.once("SIGTERM", () => { + void stop(); + }); + process.once("SIGINT", () => { + void stop(); + }); try { const runtime = await createHubRuntime(); const server = Bun.serve({ @@ -14,28 +53,20 @@ if (import.meta.main) { fetch: runtime.app.fetch, websocket: runtime.websocket, }); + close = async () => { + await server.stop(true); + await runtime.close(); + }; await runtime.start(); log.info( "Hub ready: Postgres connected; local Interchange sidecar provisioner", ); - let stopping = false; - const stop = async () => { - if (stopping) return; - stopping = true; - await server.stop(true); - await runtime.close(); - await dispose(); - }; - process.once("SIGTERM", () => { - void stop(); - }); - process.once("SIGINT", () => { - void stop(); - }); } catch (error) { - log.error("Hub startup failed; check environment and database readiness", { - error, - }); + log.error( + "Hub startup failed; check environment and database readiness", + errorMetadata(error), + ); process.exitCode = 1; + await stop(); } } diff --git a/deploy/compose.yml b/deploy/compose.yml index a55745ae..604e2abf 100644 --- a/deploy/compose.yml +++ b/deploy/compose.yml @@ -44,6 +44,8 @@ services: retries: 30 hub: <<: *hub + # The hub exits on a fatal error so it restarts from a clean state. + restart: unless-stopped # The coder and designer run their commands under bubblewrap, which gives # each one its own user, pid and mount namespaces with a fresh /proc. # Docker's default seccomp profile refuses the namespaces without From 3a407b85ac258ecc185a0b5d984f6776755b8bb2 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Wed, 4 Nov 2026 14:53:44 -0800 Subject: [PATCH 2/2] refactor(hub): name the hub shutdown function --- apps/hub/src/server.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/apps/hub/src/server.ts b/apps/hub/src/server.ts index 3d2afb2b..5c03bd2c 100644 --- a/apps/hub/src/server.ts +++ b/apps/hub/src/server.ts @@ -14,17 +14,17 @@ if (import.meta.main) { const log = getLogger(["corbits-builder", "hub"]); let close = async () => {}; let stopping: Promise | undefined; - const stop = () => - (stopping ??= (async () => { - for (const step of [close, dispose]) { - try { - await step(); - } catch (error) { - log.error("Hub shutdown failed", errorMetadata(error)); - process.exitCode = 1; - } + const shutDown = async () => { + for (const step of [close, dispose]) { + try { + await step(); + } catch (error) { + log.error("Hub shutdown failed", errorMetadata(error)); + process.exitCode = 1; } - })()); + } + }; + const stop = () => (stopping ??= shutDown()); const exitOnFatal = (message: string) => async (error: unknown) => { try { log.error(message, errorMetadata(error));