From c208d7b6d1521ba1e4094de5757aca7782ecb4b8 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Wed, 30 Sep 2026 08:33:50 -0700 Subject: [PATCH] fix(e2e): browser suites tear the hub down and stop colliding on one database (CL-9619) createHubServer returned no way to stop its schedulers, cron ticker, OAuth refresher or DB pool, and the process provisioner's sidecars outlive the hub by design. Each browser file left all of it running against the shared database. The hub now exposes shutdown(); the harness calls it, reaps the file's sidecars from their pid files, and creates the data dir per file at run time. --- apps/hub/src/server.ts | 12 ++++++++- e2e/lib/browser.ts | 56 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/apps/hub/src/server.ts b/apps/hub/src/server.ts index 5df3f1539c..95b20dbd2f 100644 --- a/apps/hub/src/server.ts +++ b/apps/hub/src/server.ts @@ -211,7 +211,7 @@ export async function createHubServer({ database: process.env["DB_NAME"] ?? "interchange", ...(pgSchema !== undefined && { schema: pgSchema }), }; - const { db } = createDB(dbConfig); + const { db, close: closeDb } = createDB(dbConfig); // Platform schema plus every mounted Corbits package's own migration, // applied once at boot before anything below reads or writes the @@ -852,5 +852,15 @@ export async function createHubServer({ websocket: sidecarWebsocket, port, idleTimeout: 0, + /** Stops every background ticker and the database pool; spawned sidecars are left to their pid files. */ + async shutdown(): Promise { + allocationScheduler.stop(); + probeCleanupScheduler.stop(); + dispatchScheduler.stop(); + connectionRepairScheduler.stop(); + cronTicker?.stop(); + oauthTokenRefresher?.stop(); + await closeDb(); + }, }; } diff --git a/e2e/lib/browser.ts b/e2e/lib/browser.ts index b5c43ac5b8..c75feecc13 100644 --- a/e2e/lib/browser.ts +++ b/e2e/lib/browser.ts @@ -4,11 +4,11 @@ // local Chrome. A UI test file calls `bootBrowserApp()` once and gets a // fresh page per test via `newPage()`. import { afterAll, beforeAll, describe } from "bun:test"; -import { existsSync } from "node:fs"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; import path from "node:path"; import puppeteer, { type Browser, type Page } from "puppeteer-core"; import { dbGate } from "./db-gate"; -import { installDisposableHubDataDir } from "./disposable-hub-data-dir"; +import { createDisposableHubDataDir } from "./disposable-hub-data-dir"; import { REPO_ROOT } from "./database-url"; const DEFAULT_CHROME = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"; @@ -54,9 +54,47 @@ async function ensureWebBuild(): Promise { if ((await proc.exited) !== 0) throw new Error("apps/web build failed"); } +// The process provisioner deliberately lets sidecars outlive the hub (they +// are recovered from pid files on the next boot), so a test hub must reap +// its own or they redial a dead port forever. +function killSidecars(hubDataDir: string): void { + const pidFiles: string[] = []; + try { + for (const dir of readdirSync(hubDataDir)) { + if (!dir.startsWith("process-provisioner")) continue; + const root = path.join(hubDataDir, dir, "allocations"); + for (const alloc of readdirSync(root)) { + for (const unit of readdirSync(path.join(root, alloc))) { + pidFiles.push(path.join(root, alloc, unit, "sidecar.pid")); + } + } + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + for (const file of pidFiles) { + let pid: number; + try { + pid = Number(readFileSync(file, "utf8").trim()); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") continue; + throw error; + } + if (!Number.isInteger(pid) || pid <= 0) continue; + try { + process.kill(pid, "SIGKILL"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } + } +} + /** Registers beforeAll/afterAll that boot and tear down hub + web + Chrome. */ export function bootBrowserApp(): () => BrowserApp { - installDisposableHubDataDir(); + let dataDir: ReturnType | undefined; + let hub: + | Awaited> + | undefined; let app: BrowserApp | undefined; let browser: Browser | undefined; let server: ReturnType | undefined; @@ -64,6 +102,8 @@ export function bootBrowserApp(): () => BrowserApp { beforeAll(async () => { const executablePath = chromePath(); if (executablePath === undefined) throw new Error("Chrome not found"); + // Per file, at run time: every file boots against its own data dir. + dataDir = createDisposableHubDataDir(); await ensureWebBuild(); const url = new URL(process.env["DATABASE_URL"] ?? ""); @@ -85,12 +125,13 @@ export function bootBrowserApp(): () => BrowserApp { process.env["PORT"] = String(server.port); const { createHubServer } = await import("../../apps/hub/src/server"); - const hub = await createHubServer(); + hub = await createHubServer(); + const booted = hub; server.reload({ - websocket: hub.websocket as Bun.WebSocketHandler, + websocket: booted.websocket as Bun.WebSocketHandler, fetch: async (req, srv) => { const { pathname } = new URL(req.url); - if (pathname.startsWith("/api") || pathname === "/status") return hub.fetch(req, srv); + if (pathname.startsWith("/api") || pathname === "/status") return booted.fetch(req, srv); const file = Bun.file(path.join(DIST_DIR, pathname)); if (pathname !== "/" && (await file.exists())) return new Response(file); return new Response(Bun.file(path.join(DIST_DIR, "index.html"))); @@ -121,6 +162,9 @@ export function bootBrowserApp(): () => BrowserApp { afterAll(async () => { await browser?.close(); await server?.stop(true); + await hub?.shutdown(); + if (dataDir !== undefined) killSidecars(dataDir.hubDataDir); + dataDir?.restore(); }); return () => {