From 7ef4a80a0863f0973431b18f9dacf713d2b3c126 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Fri, 6 Nov 2026 20:52:33 -0800 Subject: [PATCH] fix(hub): log the cause of failures the hub swallowed Seven places caught every error and carried on. Each now treats only its expected failure as normal and logs anything else with its error code: credential refresh, the pushed ref, the pre-write file check, auth emails, workflow connection sync, migrations and the preview reload. errorCode reads a wrapped error's cause, so Drizzle failures carry their code. --- apps/hub/src/auth-email.ts | 14 ++++++- .../auth/coordinator-credential-refresh.ts | 8 +++- apps/hub/src/builds/app-repository.ts | 16 ++++++-- apps/hub/src/builds/posix-sandbox.ts | 37 +++++++++++++------ apps/hub/src/builds/sidecar-tools.ts | 21 ++++++++--- apps/hub/src/migrate.test.ts | 32 ++++++++++++++++ apps/hub/src/migrate.ts | 8 +++- .../hub/src/provisioning/workflow-services.ts | 15 +++++--- apps/hub/src/request-error.ts | 15 ++++++-- 9 files changed, 132 insertions(+), 34 deletions(-) create mode 100644 apps/hub/src/migrate.test.ts diff --git a/apps/hub/src/auth-email.ts b/apps/hub/src/auth-email.ts index d78b20bb..5e6f0818 100644 --- a/apps/hub/src/auth-email.ts +++ b/apps/hub/src/auth-email.ts @@ -1,4 +1,5 @@ import { createTransport } from "nodemailer"; +import { getLogger } from "@intx/log"; import { verificationEmail, invitationEmail, @@ -7,6 +8,9 @@ import { } from "./config/auth-email.ts"; import type { ProductConfig } from "./config/index.ts"; import type { Environment } from "./environment.ts"; +import { errorCode, requestErrorMetadata } from "./request-error.ts"; + +const log = getLogger(["corbits-builder", "auth-email"]); export function createAuthEmail( environment: Environment, @@ -39,8 +43,14 @@ export function createAuthEmail( ) => { try { await transport.sendMail({ to, ...message }); - } catch { - // Transport errors can contain mailbox addresses, URL credentials or tokens. + } catch (error) { + // Transport errors can contain mailbox addresses, URL credentials or tokens, so only their + // code is kept. + log.warn("Sending an email failed: {failure}", { + failure, + ...errorCode(error), + ...requestErrorMetadata(error), + }); throw new Error(failure); } }; diff --git a/apps/hub/src/auth/coordinator-credential-refresh.ts b/apps/hub/src/auth/coordinator-credential-refresh.ts index 0ebbb07b..719e1c97 100644 --- a/apps/hub/src/auth/coordinator-credential-refresh.ts +++ b/apps/hub/src/auth/coordinator-credential-refresh.ts @@ -12,6 +12,7 @@ import type { MiddlewareHandler } from "hono"; import type { TenantEnv } from "@intx/hub-api"; import { getLogger } from "@intx/log"; import { projectCoordinator } from "../db/schema.ts"; +import { errorCode, requestErrorMetadata } from "../request-error.ts"; const log = getLogger(["corbits-builder", "coordinator-credentials"]); @@ -58,9 +59,12 @@ export function createCoordinatorCredentialRefresh(options: { ), ), ); - } catch { + } catch (error) { // The credential is already stored; the sidecar picks it up when it reconnects. - log.warn("Project workflow credential refresh unavailable"); + log.warn("Project workflow credential refresh unavailable", { + ...errorCode(error), + ...requestErrorMetadata(error), + }); } }; } diff --git a/apps/hub/src/builds/app-repository.ts b/apps/hub/src/builds/app-repository.ts index 2f47f574..bd73894b 100644 --- a/apps/hub/src/builds/app-repository.ts +++ b/apps/hub/src/builds/app-repository.ts @@ -23,7 +23,7 @@ import { HUB_CREDENTIAL_HANDLE } from "@intx/types/credential-delivery"; import { hexDecode } from "@intx/types/hex"; import type { RuntimeCapabilities } from "@intx/types/runtime-capabilities"; import { INPUTS_DIRECTORY } from "../config/inputs.ts"; -import { requestErrorMetadata } from "../request-error.ts"; +import { errorCode, requestErrorMetadata } from "../request-error.ts"; import { excludedFile as excludedSnapshotFile, linkOrGone, @@ -572,9 +572,17 @@ export function createAppRepository(options: { async function push(signal: AbortSignal) { const local = await git.resolveRef({ fs, gitdir, ref: MAIN }); - const pushed = await git - .resolveRef({ fs, gitdir, ref: PUSHED }) - .catch(() => undefined); + let pushed: string | undefined; + try { + pushed = await git.resolveRef({ fs, gitdir, ref: PUSHED }); + } catch (error) { + // Pushing again is harmless and rewrites the ref; a broken repository fails the push itself. + if (!(error instanceof Errors.NotFoundError)) + log.warn("Reading the last pushed version failed", { + ...errorCode(error), + ...requestErrorMetadata(error), + }); + } if (local === pushed) return; const { url, secret } = await pushAccess(signal); try { diff --git a/apps/hub/src/builds/posix-sandbox.ts b/apps/hub/src/builds/posix-sandbox.ts index ea024420..e5ff9afc 100644 --- a/apps/hub/src/builds/posix-sandbox.ts +++ b/apps/hub/src/builds/posix-sandbox.ts @@ -19,7 +19,7 @@ import { type } from "arktype"; import { getLogger } from "@intx/log"; import { hasCode } from "@intx/types/has-code"; import type { ToolCall, ToolResult } from "@intx/types/runtime"; -import { requestErrorMetadata } from "../request-error.ts"; +import { errorCode, requestErrorMetadata } from "../request-error.ts"; import type { buildPackageLayout } from "./package-layout.ts"; import { BROWSER_ARGS } from "./browser.ts"; import { CODER_REGISTRY_HOSTS } from "../config/packages.ts"; @@ -993,6 +993,30 @@ export async function createPosixBuildExecutor( browserStarting ??= trackBrowser(); return browserStarting; }; + const reloadPreview = async () => { + const page = devApp?.page; + if (page?.exitCode !== null) return; + try { + page.stdin.write("reload\n"); + await page.stdin.flush(); + } catch (error) { + log.warn("Reloading the preview after a write failed", { + ...errorCode(error), + ...requestErrorMetadata(error), + }); + } + }; + const runAfter = async ( + previous: Promise, + call: ToolCall, + signal: AbortSignal, + ) => { + await previous; + const result = await execute(call, signal); + if (call.name === "write_file" || call.name === "edit_file") + void reloadPreview(); + return result; + }; return { policy, version: SANDBOX_RUNTIME_VERSION, @@ -1004,17 +1028,8 @@ export async function createPosixBuildExecutor( throw new BuildExecutionError("host_gate_invocation_refused"); if (trustedGate && call.id !== trustedGate.nonce) throw new BuildExecutionError("concurrent_build_tool_refused"); - const turn = queue.then(() => execute(call, signal)); + const turn = runAfter(queue, call, signal); queue = turn.catch(() => undefined); - if (call.name === "write_file" || call.name === "edit_file") - void turn - .then(() => { - const page = devApp?.page; - if (page?.exitCode !== null) return; - page.stdin.write("reload\n"); - return page.stdin.flush(); - }) - .catch(() => undefined); return turn; }, /** diff --git a/apps/hub/src/builds/sidecar-tools.ts b/apps/hub/src/builds/sidecar-tools.ts index 0acad371..fe23031c 100644 --- a/apps/hub/src/builds/sidecar-tools.ts +++ b/apps/hub/src/builds/sidecar-tools.ts @@ -41,7 +41,8 @@ import { type AppRepository, } from "./app-repository.ts"; import { getLogger } from "@intx/log"; -import { requestErrorMetadata } from "../request-error.ts"; +import { hasCode } from "@intx/types/has-code"; +import { errorCode, requestErrorMetadata } from "../request-error.ts"; import { PACKAGE_INSTALL_TOOL, WorkspacePath, @@ -428,10 +429,20 @@ export function createSidecarCodingTools(policy: { typeof path === "string" && typeof content === "string" ) { - const existing = await readFile( - resolve(workspace, path), - "utf8", - ).catch(() => undefined); + let existing: string | undefined; + try { + existing = await readFile(resolve(workspace, path), "utf8"); + } catch (error) { + // The write still runs and reports its own failure, which the model can act on. + if ( + !hasCode(error) || + !["ENOENT", "EISDIR", "ENOTDIR"].includes(error.code) + ) + log.warn("Reading the file a write replaces failed", { + ...errorCode(error), + ...requestErrorMetadata(error), + }); + } if (existing === content) return { callId: call.id, diff --git a/apps/hub/src/migrate.test.ts b/apps/hub/src/migrate.test.ts new file mode 100644 index 00000000..19b82918 --- /dev/null +++ b/apps/hub/src/migrate.test.ts @@ -0,0 +1,32 @@ +import { expect, test } from "bun:test"; +import { fileURLToPath } from "node:url"; + +test("a failed migration prints its error code and never the database URL", async () => { + const child = Bun.spawn( + [ + process.execPath, + "--conditions=intx-src", + fileURLToPath(new URL("./migrate.ts", import.meta.url)), + ], + { + // Nothing listens on port 1, so the connection is refused. + env: { + PATH: process.env["PATH"], + DATABASE_URL: "postgres://builder:db-password@127.0.0.1:1/builder", + }, + stdout: "pipe", + stderr: "pipe", + }, + ); + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]); + expect(exitCode).toBe(1); + expect(JSON.parse(stderr)).toEqual({ + migrationsApplied: false, + errorCode: "ECONNREFUSED", + }); + expect(stdout + stderr).not.toContain("db-password"); +}); diff --git a/apps/hub/src/migrate.ts b/apps/hub/src/migrate.ts index edda4d66..f18d2634 100644 --- a/apps/hub/src/migrate.ts +++ b/apps/hub/src/migrate.ts @@ -9,6 +9,7 @@ import { type Environment, } from "./environment.ts"; import { migrateProductDatabase } from "./db/migrate.ts"; +import { errorCode } from "./request-error.ts"; const migrationsFolder = fileURLToPath( new URL("../migrations/", import.meta.resolve("@intx/db")), @@ -39,8 +40,11 @@ if (import.meta.main) { try { await migrateDatabase(); console.log(JSON.stringify({ migrationsApplied: true })); - } catch { - console.error(JSON.stringify({ migrationsApplied: false })); + } catch (error) { + // The message can carry the database URL; the code, such as a SQLSTATE, cannot. + console.error( + JSON.stringify({ migrationsApplied: false, ...errorCode(error) }), + ); process.exitCode = 1; } } diff --git a/apps/hub/src/provisioning/workflow-services.ts b/apps/hub/src/provisioning/workflow-services.ts index 709a2f38..710a4c03 100644 --- a/apps/hub/src/provisioning/workflow-services.ts +++ b/apps/hub/src/provisioning/workflow-services.ts @@ -39,6 +39,7 @@ import { defaultDockerLimits, } from "./docker.ts"; import { positiveInteger } from "../environment.ts"; +import { errorCode, requestErrorMetadata } from "../request-error.ts"; import { localSidecarDataRoot } from "../sidecar/data-root.ts"; import { createHostCapacity } from "../sidecar/host-capacity.ts"; import { @@ -300,12 +301,16 @@ export async function createWorkflowServices(options: { ); return task; }; - const report = (run: () => Promise) => - track(run) - .then(() => undefined) - .catch(() => { - log.error("Native workflow connection reconciliation failed"); + const report = async (run: () => Promise) => { + try { + await track(run); + } catch (error) { + log.error("Native workflow connection reconciliation failed", { + ...errorCode(error), + ...requestErrorMetadata(error), }); + } + }; const unsubscribers = [ router.events.on("sidecar.disconnect", ({ allocated }) => report(async () => { diff --git a/apps/hub/src/request-error.ts b/apps/hub/src/request-error.ts index bddce222..07ef26ff 100644 --- a/apps/hub/src/request-error.ts +++ b/apps/hub/src/request-error.ts @@ -11,10 +11,19 @@ const repositoryRoot = fileURLToPath(new URL("../../../", import.meta.url)); // A thrown error's code, such as ENOENT, EAUTH, a Postgres SQLSTATE or isomorphic-git's // NotFoundError, names what failed without the message's paths; anything else it carries is left out. -export const errorCode = (error: unknown) => - hasCode(error) && /^[A-Za-z0-9][A-Za-z0-9_]{0,63}$/.test(error.code) - ? { errorCode: error.code } +export const errorCode = (error: unknown) => { + const code = codeOf(error); + return code !== undefined && /^[A-Za-z0-9][A-Za-z0-9_]{0,63}$/.test(code) + ? { errorCode: code } : {}; +}; + +// A wrapper such as Drizzle's query error carries no code of its own, only the driver's as its cause. +function codeOf(error: unknown) { + if (hasCode(error)) return error.code; + if (error instanceof Error && hasCode(error.cause)) return error.cause.code; + return undefined; +} // Never emit error-supplied filenames, messages, causes, or custom names. export function requestErrorMetadata(error: unknown) {