diff --git a/apps/hub/src/server.ts b/apps/hub/src/server.ts index 5df3f1539..95b20dbd2 100644 --- a/apps/hub/src/server.ts +++ b/apps/hub/src/server.ts @@ -211,7 +211,7 @@ export async function createHubServer({ database: process.env["DB_NAME"] ?? "interchange", ...(pgSchema !== undefined && { schema: pgSchema }), }; - const { db } = createDB(dbConfig); + const { db, close: closeDb } = createDB(dbConfig); // Platform schema plus every mounted Corbits package's own migration, // applied once at boot before anything below reads or writes the @@ -852,5 +852,15 @@ export async function createHubServer({ websocket: sidecarWebsocket, port, idleTimeout: 0, + /** Stops every background ticker and the database pool; spawned sidecars are left to their pid files. */ + async shutdown(): Promise { + allocationScheduler.stop(); + probeCleanupScheduler.stop(); + dispatchScheduler.stop(); + connectionRepairScheduler.stop(); + cronTicker?.stop(); + oauthTokenRefresher?.stop(); + await closeDb(); + }, }; } diff --git a/e2e/lib/browser.ts b/e2e/lib/browser.ts index b5c43ac5b..c75feecc1 100644 --- a/e2e/lib/browser.ts +++ b/e2e/lib/browser.ts @@ -4,11 +4,11 @@ // local Chrome. A UI test file calls `bootBrowserApp()` once and gets a // fresh page per test via `newPage()`. import { afterAll, beforeAll, describe } from "bun:test"; -import { existsSync } from "node:fs"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; import path from "node:path"; import puppeteer, { type Browser, type Page } from "puppeteer-core"; import { dbGate } from "./db-gate"; -import { installDisposableHubDataDir } from "./disposable-hub-data-dir"; +import { createDisposableHubDataDir } from "./disposable-hub-data-dir"; import { REPO_ROOT } from "./database-url"; const DEFAULT_CHROME = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"; @@ -54,9 +54,47 @@ async function ensureWebBuild(): Promise { if ((await proc.exited) !== 0) throw new Error("apps/web build failed"); } +// The process provisioner deliberately lets sidecars outlive the hub (they +// are recovered from pid files on the next boot), so a test hub must reap +// its own or they redial a dead port forever. +function killSidecars(hubDataDir: string): void { + const pidFiles: string[] = []; + try { + for (const dir of readdirSync(hubDataDir)) { + if (!dir.startsWith("process-provisioner")) continue; + const root = path.join(hubDataDir, dir, "allocations"); + for (const alloc of readdirSync(root)) { + for (const unit of readdirSync(path.join(root, alloc))) { + pidFiles.push(path.join(root, alloc, unit, "sidecar.pid")); + } + } + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + for (const file of pidFiles) { + let pid: number; + try { + pid = Number(readFileSync(file, "utf8").trim()); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") continue; + throw error; + } + if (!Number.isInteger(pid) || pid <= 0) continue; + try { + process.kill(pid, "SIGKILL"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } + } +} + /** Registers beforeAll/afterAll that boot and tear down hub + web + Chrome. */ export function bootBrowserApp(): () => BrowserApp { - installDisposableHubDataDir(); + let dataDir: ReturnType | undefined; + let hub: + | Awaited> + | undefined; let app: BrowserApp | undefined; let browser: Browser | undefined; let server: ReturnType | undefined; @@ -64,6 +102,8 @@ export function bootBrowserApp(): () => BrowserApp { beforeAll(async () => { const executablePath = chromePath(); if (executablePath === undefined) throw new Error("Chrome not found"); + // Per file, at run time: every file boots against its own data dir. + dataDir = createDisposableHubDataDir(); await ensureWebBuild(); const url = new URL(process.env["DATABASE_URL"] ?? ""); @@ -85,12 +125,13 @@ export function bootBrowserApp(): () => BrowserApp { process.env["PORT"] = String(server.port); const { createHubServer } = await import("../../apps/hub/src/server"); - const hub = await createHubServer(); + hub = await createHubServer(); + const booted = hub; server.reload({ - websocket: hub.websocket as Bun.WebSocketHandler, + websocket: booted.websocket as Bun.WebSocketHandler, fetch: async (req, srv) => { const { pathname } = new URL(req.url); - if (pathname.startsWith("/api") || pathname === "/status") return hub.fetch(req, srv); + if (pathname.startsWith("/api") || pathname === "/status") return booted.fetch(req, srv); const file = Bun.file(path.join(DIST_DIR, pathname)); if (pathname !== "/" && (await file.exists())) return new Response(file); return new Response(Bun.file(path.join(DIST_DIR, "index.html"))); @@ -121,6 +162,9 @@ export function bootBrowserApp(): () => BrowserApp { afterAll(async () => { await browser?.close(); await server?.stop(true); + await hub?.shutdown(); + if (dataDir !== undefined) killSidecars(dataDir.hubDataDir); + dataDir?.restore(); }); return () => {