diff --git a/apps/hub/package.json b/apps/hub/package.json index 9257b6c754..e2dd429dfb 100644 --- a/apps/hub/package.json +++ b/apps/hub/package.json @@ -32,7 +32,6 @@ "@corbits/e2b-sandbox-sidecar": "workspace:*", "@corbits/echo-workflow": "workspace:*", "@corbits/error-sink": "workspace:*", - "@corbits/evals": "workspace:*", "@corbits/exa-topic-watch-workflow": "workspace:*", "@corbits/granola-call-workflow": "workspace:*", "@corbits/hub-api-client": "workspace:*", diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index f8ed4dcabe..6670f0f4c0 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -105,11 +105,6 @@ import { createUsageSink, withTurnPartPersistGuard, } from "@corbits/insights"; -import { - applyEvalsMigrations, - createEvalRunRoutes, - createPostgresEvalRunStore, -} from "@corbits/evals"; import { applyInferenceCatalogMigrations, createPostgresBenchModelPolicyStore, @@ -1723,21 +1718,6 @@ export async function createHub(config: HubConfig) { const benchModelPolicy = createPostgresBenchModelPolicyStore( config.databaseUrl, ); - // Eval run history: read-only surface over the package-owned - // `evals.run` table, migrated at hub start like insights. Eval runs - // aren't tenant-owned, so the tenant prefix here is only the grant gate. - await applyEvalsMigrations(config.databaseUrl); - const evalRuns = createPostgresEvalRunStore(config.databaseUrl); - app.route( - `${TENANT_PREFIX}/eval-runs`, - createEvalRunRoutes({ - store: evalRuns.store, - requireGrant: createRequireGrant({ - grantStore: chatGrantStore, - conditionRegistry: grantConditionRegistry, - }), - }), - ); { const mailboxApp = new Hono(); mountMailbox(mailboxApp, { @@ -2367,7 +2347,6 @@ export async function createHub(config: HubConfig) { cronEmitter.stop(); await insightsUsage.close(); await insightsLatency.close(); - await evalRuns.close(); await closeMailbox(); // The pool end waits on in-flight queries; a query whose socket // died with the process must never stall shutdown, so bound it. diff --git a/apps/hub/test/eval-runs-mount.test.ts b/apps/hub/test/eval-runs-mount.test.ts deleted file mode 100644 index 649b4077bf..0000000000 --- a/apps/hub/test/eval-runs-mount.test.ts +++ /dev/null @@ -1,74 +0,0 @@ -// Proves the eval-run read routes (CL-6465) exist inside the native -// tenant middleware — the same shape `presence-mount.test.ts` proves -// for presence. `@corbits/evals`' own request-parsing, grant-gating, -// and error-envelope behavior is covered by that package's own -// `routes.test.ts`; this only proves the composition-root wiring in -// `src/index.ts` actually reaches a live app instead of 404ing. - -import { afterAll, expect, test } from "bun:test"; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import type { HubConfig } from "../src/config.ts"; -import { createHub } from "../src/index.ts"; -import { dbGate } from "../../../scripts/e2e/db-gate"; - -// DB-gated: skipped when DATABASE_URL is unset, matching this repo's -// convention for tests that talk to a real Postgres. -const databaseUrl = process.env["DATABASE_URL"] ?? ""; -const describeIfDb = dbGate(databaseUrl, import.meta.path); - -const root = mkdtempSync(path.join(tmpdir(), "hub-eval-runs-mount-")); -const staticDir = path.join(root, "static"); -mkdirSync(staticDir, { recursive: true }); -writeFileSync(path.join(staticDir, "index.html"), "shell"); -mkdirSync(path.join(root, "data"), { recursive: true }); - -const config: HubConfig = { - databaseUrl, - baseUrl: "http://localhost:3000", - sessionSecret: "insecure-test-only-session-secret-0000", - hubDataDir: path.join(root, "data"), - hubStaticDir: staticDir, - signupRateLimit: { windowSeconds: 60, max: 5 }, - signInRateLimit: { windowSeconds: 60, max: 10 }, - socialProviders: {}, - allowPlaintextSecrets: true, - sidecarProvisioners: [], - chatIdleReapMs: 30 * 60_000, -}; - -const closers: (() => Promise)[] = []; - -afterAll(async () => { - for (const close of closers) await close(); - rmSync(root, { recursive: true, force: true }); -}); - -describeIfDb("eval-runs mount", () => { - test("eval-run routes mount inside the native tenant middleware", async () => { - const hub = await createHub(config); - closers.push(hub.close); - - const gated = await hub.app.request( - "/api/tenants/some-tenant/eval-runs/runs", - ); - // Unauthenticated, not unmounted: a 401 from the tenant auth - // middleware proves the route exists (an unmounted route would - // 404 before ever reaching a grant check). - expect(gated.status).toBe(401); - expect(await gated.json()).toEqual({ - error: { code: "unauthorized", message: "Authentication required" }, - }); - - const gatedDetail = await hub.app.request( - "/api/tenants/some-tenant/eval-runs/runs/evalrun_1", - ); - expect(gatedDetail.status).toBe(401); - - // The route exists only under the tenant scope; outside it the - // path falls through to the interface shell. - const outside = await hub.app.request("/eval-runs/runs"); - expect(await outside.text()).toBe("shell"); - }); -}); diff --git a/apps/web/src/evals-api.test.ts b/apps/web/src/evals-api.test.ts deleted file mode 100644 index b0539d2e4f..0000000000 --- a/apps/web/src/evals-api.test.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { type } from "arktype"; - -import { - evalRunDurationMs, - evalRunOutcome, - evalRunPath, - evalRunsPath, - EvalRunDetailSchema, - EvalRunsResponseSchema, -} from "./evals-api"; - -describe("evalRunOutcome", () => { - test("a run with a failed scorer is failed", () => { - expect(evalRunOutcome({ passed: 2, failed: 1, skipped: 0 })).toBe("failed"); - }); - - test("a run with only passes and skips is passed", () => { - expect(evalRunOutcome({ passed: 3, failed: 0, skipped: 1 })).toBe("passed"); - }); - - test("a run with no scorer calls at all is passed, not falsely failed", () => { - expect(evalRunOutcome({ passed: 0, failed: 0, skipped: 0 })).toBe("passed"); - }); -}); - -describe("evalRunDurationMs", () => { - test("computes the wall-clock span", () => { - expect( - evalRunDurationMs({ - startedAt: "2026-01-01T00:00:00.000Z", - finishedAt: "2026-01-01T00:00:05.000Z", - }), - ).toBe(5000); - }); - - test("returns null for an unparsable timestamp instead of NaN", () => { - expect( - evalRunDurationMs({ startedAt: "not-a-date", finishedAt: "also-not" }), - ).toBeNull(); - }); -}); - -describe("path builders", () => { - test("evalRunsPath carries a limit and an optional eval-name filter", () => { - expect(evalRunsPath("tnt_1", null)).toBe( - "/api/tenants/tnt_1/eval-runs/runs?limit=50", - ); - expect(evalRunsPath("tnt_1", "factory")).toBe( - "/api/tenants/tnt_1/eval-runs/runs?limit=50&evalName=factory", - ); - }); - - test("evalRunPath addresses one run by its opaque store id", () => { - expect(evalRunPath("tnt_1", "evalrun_1")).toBe( - "/api/tenants/tnt_1/eval-runs/runs/evalrun_1", - ); - }); -}); - -describe("wire schemas", () => { - test("parses a real list envelope", () => { - const parsed = EvalRunsResponseSchema({ - runs: [ - { - id: "evalrun_1", - evalName: "factory", - evalDescription: "The factory eval", - configName: "default", - startedAt: "2026-01-01T00:00:00.000Z", - finishedAt: "2026-01-01T00:02:00.000Z", - stepCount: 3, - scorerTally: { passed: 3, failed: 0, skipped: 0 }, - }, - ], - }); - expect(parsed instanceof type.errors).toBe(false); - }); - - test("parses a real run detail envelope with tool calls and scorer reports", () => { - const parsed = EvalRunDetailSchema({ - id: "evalrun_1", - evalName: "factory", - evalDescription: null, - configName: "default", - startedAt: "2026-01-01T00:00:00.000Z", - finishedAt: "2026-01-01T00:02:00.000Z", - steps: [ - { - stepIndex: 0, - turn: { - human: "hello", - replyText: "hi", - toolCalls: [ - { - name: "list_issues", - arguments: { team: "Corbits" }, - isError: false, - result: "[]", - }, - ], - }, - scorerReports: [ - { - name: "mentions-issues", - score: 1, - pass: true, - reason: "found the word", - }, - ], - }, - ], - }); - expect(parsed instanceof type.errors).toBe(false); - }); -}); diff --git a/apps/web/src/evals-api.ts b/apps/web/src/evals-api.ts deleted file mode 100644 index 8757ec3158..0000000000 --- a/apps/web/src/evals-api.ts +++ /dev/null @@ -1,111 +0,0 @@ -// This app's fetch/query plumbing over packages/evals routes (CL-6465): -// arktype schemas that parse every trust boundary, and the path builders -// for this app's tenant-scoped eval-run routes. Same split as -// insights-api.ts — pure wire shapes and paths here, page logic in -// pages/evals-page.tsx. -// GET /eval-runs/runs → { runs: EvalRunSummary[] } -// GET /eval-runs/runs/:runId → EvalRunDetail -// -// Eval runs aren't tenant-owned (packages/evals/src/routes.ts's own -// header comment) — the tenant id in the path is only the grant gate, -// same as insights' non-partitioned reads. - -import { type } from "arktype"; - -const ScorerReportSchema = type({ - name: "string", - score: "number", - pass: "boolean", - reason: "string", - "skipped?": "boolean", -}); - -const ToolCallSchema = type({ - name: "string", - arguments: "object", - isError: "boolean", - result: "string", -}); - -const TurnSchema = type({ - human: "string", - replyText: "string", - toolCalls: ToolCallSchema.array(), -}); - -const EvalStepRecordSchema = type({ - stepIndex: "number", - turn: TurnSchema, - scorerReports: ScorerReportSchema.array(), -}); - -const ScorerTallySchema = type({ - passed: "number", - failed: "number", - skipped: "number", -}); - -export const EvalRunSummarySchema = type({ - id: "string", - evalName: "string", - evalDescription: "string | null", - configName: "string", - startedAt: "string", - finishedAt: "string", - stepCount: "number", - scorerTally: ScorerTallySchema, -}); - -export const EvalRunsResponseSchema = type({ - runs: EvalRunSummarySchema.array(), -}); - -export const EvalRunDetailSchema = type({ - id: "string", - evalName: "string", - evalDescription: "string | null", - configName: "string", - startedAt: "string", - finishedAt: "string", - steps: EvalStepRecordSchema.array(), -}); - -export type ScorerReport = typeof ScorerReportSchema.infer; -export type ToolCall = typeof ToolCallSchema.infer; -export type EvalStepRecord = typeof EvalStepRecordSchema.infer; -export type EvalRunSummary = typeof EvalRunSummarySchema.infer; -export type EvalRunDetail = typeof EvalRunDetailSchema.infer; - -const EVAL_RUNS_LIMIT = 50; - -export function evalRunsPath( - tenantId: string, - evalName: string | null, -): string { - const params = new URLSearchParams({ limit: String(EVAL_RUNS_LIMIT) }); - if (evalName !== null) params.set("evalName", evalName); - return `/api/tenants/${encodeURIComponent(tenantId)}/eval-runs/runs?${params.toString()}`; -} - -export function evalRunPath(tenantId: string, runId: string): string { - return `/api/tenants/${encodeURIComponent(tenantId)}/eval-runs/runs/${encodeURIComponent(runId)}`; -} - -/** Passed only when every scorer that actually ran passed — a run with - * zero recorded scorer calls (a step with no `expect`) is not a failure, - * so this only ever flips to "failed" on a genuine failed report. */ -export function evalRunOutcome( - tally: EvalRunSummary["scorerTally"], -): "passed" | "failed" { - return tally.failed > 0 ? "failed" : "passed"; -} - -export function evalRunDurationMs(run: { - readonly startedAt: string; - readonly finishedAt: string; -}): number | null { - const start = Date.parse(run.startedAt); - const end = Date.parse(run.finishedAt); - if (Number.isNaN(start) || Number.isNaN(end)) return null; - return Math.max(0, end - start); -} diff --git a/apps/web/src/evals-path.test.ts b/apps/web/src/evals-path.test.ts deleted file mode 100644 index 8e81632482..0000000000 --- a/apps/web/src/evals-path.test.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { describe, expect, test } from "bun:test"; - -import { parseEvalsPath } from "./evals-path"; - -describe("parseEvalsPath", () => { - test("resolves the list path", () => { - expect(parseEvalsPath("/evals")).toEqual({ mode: "list", runId: null }); - expect(parseEvalsPath("/evals/")).toEqual({ mode: "list", runId: null }); - }); - - test("resolves a run deep link", () => { - expect(parseEvalsPath("/evals/evalrun_1")).toEqual({ - mode: "run", - runId: "evalrun_1", - }); - }); - - test("a malformed escape on a run deep link falls back to list, not a throw", () => { - expect(() => parseEvalsPath("/evals/%")).not.toThrow(); - expect(parseEvalsPath("/evals/%")).toEqual({ mode: "list", runId: null }); - }); - - test("an unrelated path falls back to list", () => { - expect(parseEvalsPath("/insights")).toEqual({ mode: "list", runId: null }); - }); -}); diff --git a/apps/web/src/evals-path.ts b/apps/web/src/evals-path.ts deleted file mode 100644 index b3f61df99b..0000000000 --- a/apps/web/src/evals-path.ts +++ /dev/null @@ -1,29 +0,0 @@ -// Evals' own route parser (CL-6465) — its own module (not inlined in -// evals-page.tsx), same split as insights-path.ts, so it can be exercised -// directly without dragging in that page's bench/query-client wiring. -// -// An eval run id is an opaque store id (`evalrun_`), never a slug — -// same addressing choice as routines (see path-ids.ts's ROUTINE_SEGMENT -// comment): eval runs have no slug column, so the id is the only stable -// address a link can carry. - -import { decodedOrNull } from "@corbits/url-path"; - -import { EVALS_PATH_PREFIX } from "./path-ids"; - -export function parseEvalsPath(path: string): { - mode: "list" | "run"; - runId: string | null; -} { - if (path === EVALS_PATH_PREFIX || path === `${EVALS_PATH_PREFIX}/`) { - return { mode: "list", runId: null }; - } - const match = /^\/evals\/([^/]+)\/?$/.exec(path); - if (match !== null && match[1] !== undefined) { - const runId = decodedOrNull(match[1]); - if (runId !== null) { - return { mode: "run", runId }; - } - } - return { mode: "list", runId: null }; -} diff --git a/apps/web/src/pages/evals-page.tsx b/apps/web/src/pages/evals-page.tsx deleted file mode 100644 index 7f1e688771..0000000000 --- a/apps/web/src/pages/evals-page.tsx +++ /dev/null @@ -1,445 +0,0 @@ -// Evals (CL-6465): every recorded eval run in `packages/evals`'s own -// store — a run of `bun run eval` against a real Myra deployment, its -// steps, and each step's scorer reports. This is the harness's history, -// not a live production trace — `packages/evals/src/routes.ts`'s own -// header comment says eval runs aren't tenant-owned, so a tenant only -// gates the read (same non-partitioned shape as insights). There is no -// route to start a run from the UI (the harness is a `bun run eval` CLI -// step), so this page is read-only: no "Re-run" action, since a control -// that cannot actually do anything is worse than no control. -// -// "Kind" (agent run / routine / workflow) from the shell mock has no -// backing field on `EvalRunResult` — every recorded run is just "an -// eval" — so the filter row here is the real dimensions instead: which -// eval, and whether any scorer failed. - -import { - Badge, - FilterChip, - PageShell, - RichEmptyState, - RUN_STATUS_TONE, - Skeleton, - StatGrid, - StatGridItem, - Table, - TableBody, - TableCell, - TableHead, - TableHeader, - TableRow, -} from "@corbits/react-ui"; -import { ListBullets } from "@corbits/icons"; -import { useMemo, useState, type KeyboardEvent } from "react"; - -import { durationLabel } from "@corbits/insights/client"; -import { SignedOutNotice } from "@corbits/api-query"; - -import { useAPIQuery } from "../api"; -import { useBench } from "../bench-context"; -import { - evalRunDurationMs, - evalRunOutcome, - evalRunPath, - evalRunsPath, - EvalRunDetailSchema, - EvalRunsResponseSchema, - type EvalRunDetail, - type EvalRunSummary, - type EvalStepRecord, -} from "../evals-api"; -import { EVALS_PATH_PREFIX } from "../path-ids"; -import { parseEvalsPath } from "../evals-path"; -import { useNavigate } from "../navigation"; -import { StageTopBar } from "../shell/stage-top-bar"; - -function formatWhen(iso: string): string { - const date = new Date(iso); - if (Number.isNaN(date.getTime())) return iso; - return date.toLocaleString(undefined, { - month: "short", - day: "numeric", - hour: "2-digit", - minute: "2-digit", - }); -} - -function durationOf(run: { - readonly startedAt: string; - readonly finishedAt: string; -}): string { - const ms = evalRunDurationMs(run); - return ms === null ? "—" : durationLabel(ms); -} - -function outcomeBadge(tally: EvalRunSummary["scorerTally"]) { - const outcome = evalRunOutcome(tally); - const tone = - outcome === "passed" ? RUN_STATUS_TONE.completed : RUN_STATUS_TONE.failed; - const label = outcome === "passed" ? "Passed" : "Failed"; - return {label}; -} - -function onRowActivate(onActivate: () => void) { - return { - role: "button" as const, - tabIndex: 0, - className: "cursor-pointer", - onClick: onActivate, - onKeyDown: (event: KeyboardEvent) => { - if (event.key === "Enter" || event.key === " ") { - event.preventDefault(); - onActivate(); - } - }, - }; -} - -/** Every distinct `evalName` present in the fetched page, in first-seen - * order — the real filter dimension the shell mock's "kind" chips stand - * in for here. */ -function evalNamesInView(runs: readonly EvalRunSummary[]): readonly string[] { - const seen: string[] = []; - for (const run of runs) { - if (!seen.includes(run.evalName)) seen.push(run.evalName); - } - return seen; -} - -function EvalsList({ - runs, - loading, - onOpenRun, -}: { - readonly runs: readonly EvalRunSummary[]; - readonly loading: boolean; - readonly onOpenRun: (id: string) => void; -}) { - const [evalFilter, setEvalFilter] = useState(null); - const [failedOnly, setFailedOnly] = useState(false); - const names = useMemo(() => evalNamesInView(runs), [runs]); - - const filtered = runs.filter((run) => { - if (evalFilter !== null && run.evalName !== evalFilter) return false; - if (failedOnly && evalRunOutcome(run.scorerTally) !== "failed") - return false; - return true; - }); - - const passed = runs.filter( - (r) => evalRunOutcome(r.scorerTally) === "passed", - ).length; - const failed = runs.length - passed; - - if (loading) { - return ( -
- - -
- ); - } - - if (runs.length === 0) { - return ( - } - title="No eval runs yet" - description="When an eval run finishes, it shows up here — a step-by-step transcript with scorer verdicts." - /> - ); - } - - return ( -
- - - - 0} - /> - - -
- setEvalFilter(null)} - > - All evals - - {names.map((name) => ( - setEvalFilter(name)} - > - {name} - - ))} - setFailedOnly((v) => !v)} - > - Failed only - -
- - {filtered.length === 0 ? ( - } - title="No runs match this filter" - description="Clear a filter to see more runs." - /> - ) : ( - - - - Eval - Status - Steps - Duration - Started - - - - {filtered.map((run) => ( - onOpenRun(run.id))} - > - -
- - {run.evalName} - - - {run.configName} - -
- - {outcomeBadge(run.scorerTally)} - {run.stepCount} - - {durationOf(run)} - - - {formatWhen(run.startedAt)} - - - ))} - -
- )} -
- ); -} - -function ScorerReportRow({ - report, -}: { - readonly report: EvalStepRecord["scorerReports"][number]; -}) { - const tone = - report.skipped === true - ? "neutral" - : report.pass - ? RUN_STATUS_TONE.completed - : RUN_STATUS_TONE.failed; - const label = - report.skipped === true ? "Skipped" : report.pass ? "Pass" : "Fail"; - return ( -
-
- {report.name} - {report.reason} -
- {label} -
- ); -} - -function ToolCallRow({ - call, -}: { - readonly call: EvalStepRecord["turn"]["toolCalls"][number]; -}) { - return ( -
- {call.name} - - {call.isError ? "error" : "ok"} - -
- ); -} - -function EvalStepPanel({ step }: { readonly step: EvalStepRecord }) { - return ( -
-
- - Step {step.stepIndex + 1} - -
-
- - Prompt - -

{step.turn.human}

-
-
- - Reply - -

{step.turn.replyText}

-
- {step.turn.toolCalls.length > 0 ? ( -
- - Tool calls - - {step.turn.toolCalls.map((call, i) => ( - - ))} -
- ) : null} - {step.scorerReports.length > 0 ? ( -
- - Scorers - - {step.scorerReports.map((report, i) => ( - - ))} -
- ) : null} -
- ); -} - -function EvalRunDetailView({ run }: { readonly run: EvalRunDetail }) { - const passed = run.steps.reduce( - (n, s) => - n + s.scorerReports.filter((r) => r.pass && r.skipped !== true).length, - 0, - ); - const failed = run.steps.reduce( - (n, s) => - n + s.scorerReports.filter((r) => !r.pass && r.skipped !== true).length, - 0, - ); - - return ( -
- - - - 0} - /> - - - {run.evalDescription !== null ? ( -

{run.evalDescription}

- ) : null} -
- {run.steps.map((step) => ( - - ))} -
-
- ); -} - -export function EvalsRoute({ path }: { readonly path?: string }) { - const { selectedTenantId } = useBench(); - const navigate = useNavigate(); - const currentPath = - path ?? - (typeof window !== "undefined" - ? window.location.pathname - : EVALS_PATH_PREFIX); - const { mode, runId } = parseEvalsPath(currentPath); - - const runs = useAPIQuery( - selectedTenantId === null ? "" : evalRunsPath(selectedTenantId, null), - EvalRunsResponseSchema, - ); - const runDetail = useAPIQuery( - mode !== "run" || selectedTenantId === null || runId === null - ? "" - : evalRunPath(selectedTenantId, runId), - EvalRunDetailSchema, - ); - - if (mode === "run" && runId !== null) { - const summary = - runs.kind === "ready" - ? runs.data.runs.find((r) => r.id === runId) - : undefined; - return ( -
- -
- - {runDetail.kind === "unauthenticated" ? : null} - {runDetail.kind === "loading" ? ( - - ) : null} - {runDetail.kind === "error" ? ( - } - title="Couldn't load this run" - description={runDetail.message} - actions={[{ label: "Retry", onClick: runDetail.retry }]} - /> - ) : null} - {runDetail.kind === "ready" ? ( - - ) : null} - -
-
- ); - } - - return ( -
- -
- - {runs.kind === "unauthenticated" ? : null} - {runs.kind === "error" ? ( - } - title="Couldn't load eval runs" - description={runs.message} - actions={[{ label: "Retry", onClick: runs.retry }]} - /> - ) : null} - {runs.kind === "loading" || runs.kind === "ready" ? ( - - navigate(`${EVALS_PATH_PREFIX}/${encodeURIComponent(id)}`) - } - /> - ) : null} - -
-
- ); -} diff --git a/apps/web/src/path-ids.ts b/apps/web/src/path-ids.ts index 724ed06a7a..feffc5b0ba 100644 --- a/apps/web/src/path-ids.ts +++ b/apps/web/src/path-ids.ts @@ -19,7 +19,6 @@ export const ROUTINES_PATH_PREFIX = "/routines"; export const WORKFLOWS_PATH_PREFIX = "/workflows"; export const INSIGHTS_PATH_PREFIX = "/insights"; export const INSIGHTS_RUNS_PATH = `${INSIGHTS_PATH_PREFIX}/runs`; -export const EVALS_PATH_PREFIX = "/evals"; /** The slug a detail path carries — `null` unless the path is exactly * `/`. Validation reads the raw segment: a slug carries no diff --git a/apps/web/src/routes.tsx b/apps/web/src/routes.tsx index e948be2327..b097ba020d 100644 --- a/apps/web/src/routes.tsx +++ b/apps/web/src/routes.tsx @@ -2,8 +2,8 @@ // (label) and the route switch (render), so navigation and pages cannot // drift apart. The sidebar itself lists workbenches (conversations), not // routes — the first-run footer reaches Routines, Files, Skills, Agents, -// and Plugins; Insights and Evals join that rail only given honest usage. -// Insights, Evals, and Settings stay reachable by deep link and the +// and Plugins; Insights joins that rail only given honest usage. +// Insights and Settings stay reachable by deep link and the // palette even when they are off the rail. Conversation deep links // (`/w/:workbenchId`) stay routable; `/` is the Myra land hop (ensure + // open her conversation) for a bench with a workbench already, or the @@ -25,7 +25,6 @@ import { FlowArrow, FolderOpen, Lightning, - ListBullets, PuzzlePiece, Robot, SlidersHorizontal, @@ -37,7 +36,6 @@ import { lazy, useEffect, type ReactElement, type ReactNode } from "react"; import { AGENTS_PATH_PREFIX, - EVALS_PATH_PREFIX, SKILLS_PATH_PREFIX, ROUTINES_PATH_PREFIX, WORKFLOWS_PATH_PREFIX, @@ -84,9 +82,6 @@ const SkillsRoute = lazy(async () => ({ const InsightsRoute = lazy(async () => ({ default: (await import("./pages/insights-page")).InsightsRoute, })); -const EvalsRoute = lazy(async () => ({ - default: (await import("./pages/evals-page")).EvalsRoute, -})); const PluginsRoute = lazy(async () => ({ default: (await import("./pages/plugins-page")).PluginsRoute, })); @@ -244,7 +239,6 @@ export function matchesRoute(routePath: string, path: string): boolean { routePath === "/library" || routePath === "/files" || routePath === "/insights" || - routePath === EVALS_PATH_PREFIX || routePath === "/agents" || routePath === "/skills" || routePath === "/settings/agents" || @@ -409,12 +403,6 @@ export const APP_ROUTES: readonly AppRoute[] = [ icon: , render: (path: string) => , }, - { - path: EVALS_PATH_PREFIX, - label: "Evals", - icon: , - render: (path: string) => , - }, { // First-run footer rail destination. No `/plugins/:slug` until // CL-6417 (CL-6817 unlinked the stub). @@ -446,8 +434,8 @@ function routesInOrder(paths: readonly string[]): readonly AppRoute[] { /** * Everything the command palette treats as a product destination (its * "Pages" group). The first-run sidebar footer reaches Routines / Files / - * Skills / Agents / Plugins (and Insights / Evals only given honest - * usage); Insights, Evals, and Settings stay palette- and + * Skills / Agents / Plugins (and Insights only given honest + * usage); Insights and Settings stay palette- and * deep-link-reachable even when they are off the rail. */ export const NAV_ROUTES: readonly AppRoute[] = routesInOrder([ @@ -457,6 +445,5 @@ export const NAV_ROUTES: readonly AppRoute[] = routesInOrder([ "/agents", "/plugins", "/insights", - EVALS_PATH_PREFIX, SETTINGS_PATH, ]); diff --git a/apps/web/src/shell/sidebar.tsx b/apps/web/src/shell/sidebar.tsx index a5256c5dfd..fa003bef1f 100644 --- a/apps/web/src/shell/sidebar.tsx +++ b/apps/web/src/shell/sidebar.tsx @@ -1,8 +1,8 @@ // The one sidebar. Header: the brand mark, then create + search. Body: // Agents and Channels — nothing page-scoped ever renders here. Footer: the -// first-run rail is Routines, Files, Skills, Agents, Plugins; Insights and -// Evals join only when the existing usage / eval-run reads return real items -// (never a fabricated row, never a new analytics store). Below the rail: +// first-run rail is Routines, Files, Skills, Agents, Plugins; Insights +// joins only when the existing usage reads return real items (never a +// fabricated row, never a new analytics store). Below the rail: // the account row — avatar + name, the whole row is the trigger for a menu // that pops upward with weekly usage, settings, feedback, and log out. // Always present; there is no collapse affordance and no second nav column. @@ -37,7 +37,6 @@ import { ChatCircleDots, FolderOpen, Lightning, - ListBullets, Plus, PuzzlePiece, Robot, @@ -57,7 +56,6 @@ import { import webPackage from "../../package.json"; import { useAPIQuery } from "../api"; import { useBench } from "../bench-context"; -import { EvalRunsResponseSchema, evalRunsPath } from "../evals-api"; import { OverallUsageSchema, insightsUsagePath } from "../insights-api"; import { matchesRoute, @@ -126,13 +124,7 @@ export function Sidebar({ selectedTenantId === null ? "" : insightsUsagePath(selectedTenantId, range), OverallUsageSchema, ); - const evalsQuery = useAPIQuery( - selectedTenantId === null ? "" : evalRunsPath(selectedTenantId, null), - EvalRunsResponseSchema, - ); const showInsights = usageQuery.kind === "ready" && usageQuery.data.turns > 0; - const showEvals = - evalsQuery.kind === "ready" && evalsQuery.data.runs.length > 0; return ( {/* Footer order: Routines, Files, Skills, Agents, Plugins, then - Insights and Evals only when those existing reads prove real + Insights only when that existing read proves real items, then the account row anchors everything else (weekly usage, Settings, Log out) in its pop-up menu — a single footer, never two stacked rows. Routines (CL-6362) is global-only here @@ -251,18 +243,6 @@ export function Sidebar({ Insights ) : null} - {showEvals ? ( - - ) : null}
diff --git a/apps/web/test/evals-route.test.tsx b/apps/web/test/evals-route.test.tsx deleted file mode 100644 index 4d3acca2f9..0000000000 --- a/apps/web/test/evals-route.test.tsx +++ /dev/null @@ -1,159 +0,0 @@ -// CL-6465: Evals is a brand new surface reading `packages/evals`'s -// real eval-run store through the hub's `/eval-runs` routes (mounted in -// apps/hub/src/index.ts alongside insights and bench-settings). These -// tests exercise the real fetch wiring `EvalsRoute` owns, same shape as -// `insights-route.test.tsx`. -import { afterEach, describe, expect, test } from "bun:test"; -import { act } from "react"; -import { createRoot } from "react-dom/client"; -import type { Root } from "react-dom/client"; - -import { BenchProvider } from "../src/bench-context"; -import { NavigationProvider } from "../src/navigation"; -import { EvalsRoute } from "../src/pages/evals-page"; -import { TestQueryProvider } from "./test-query-provider"; - -const realFetch = globalThis.fetch; - -let container: HTMLDivElement | null = null; -let root: Root | null = null; - -afterEach(() => { - globalThis.fetch = realFetch; - if (root !== null) { - act(() => root?.unmount()); - root = null; - } - if (container !== null) { - container.remove(); - container = null; - } -}); - -const membership = { - data: [ - { - principalId: "prn_1", - tenantId: "tnt_1", - tenantName: "Corbits Bench", - tenantSlug: "corbits-bench", - kind: "user", - status: "active", - roles: [], - }, - ], - nextCursor: null, -}; - -const passedRun = { - id: "evalrun_1", - evalName: "factory", - evalDescription: "The factory eval", - configName: "default", - startedAt: new Date(Date.now() - 120_000).toISOString(), - finishedAt: new Date(Date.now() - 60_000).toISOString(), - stepCount: 3, - scorerTally: { passed: 3, failed: 0, skipped: 0 }, -}; - -const runDetail = { - id: "evalrun_1", - evalName: "factory", - evalDescription: "The factory eval", - configName: "default", - startedAt: passedRun.startedAt, - finishedAt: passedRun.finishedAt, - steps: [ - { - stepIndex: 0, - turn: { - human: "Review the open PR", - replyText: "Here is my review.", - toolCalls: [ - { - name: "list_prs", - arguments: { repo: "workbench" }, - isError: false, - result: "[]", - }, - ], - }, - scorerReports: [ - { - name: "leaves-a-review", - score: 1, - pass: true, - reason: "posted a review comment", - }, - ], - }, - ], -}; - -function json(body: unknown): Response { - return new Response(JSON.stringify(body), { - status: 200, - headers: { "content-type": "application/json" }, - }); -} - -function stubFetch(runs: readonly unknown[]): void { - globalThis.fetch = ((input: RequestInfo | URL) => { - const path = typeof input === "string" ? input : String(input); - if (path.includes("/api/me/principals")) - return Promise.resolve(json(membership)); - if (path.includes("/eval-runs/runs/evalrun_1")) - return Promise.resolve(json(runDetail)); - if (path.includes("/eval-runs/runs")) - return Promise.resolve(json({ runs })); - return Promise.resolve(json({})); - }) as typeof fetch; -} - -async function mount(path = "/evals") { - container = document.createElement("div"); - document.body.appendChild(container); - root = createRoot(container); - await act(async () => { - root?.render( - - undefined}> - - - - - , - ); - }); - for (let i = 0; i < 20; i++) { - await act(async () => { - await new Promise((resolve) => setTimeout(resolve, 0)); - }); - } - return container; -} - -describe("EvalsRoute", () => { - test("a real eval run reaches the list with its outcome", async () => { - stubFetch([passedRun]); - const el = await mount(); - expect(el.textContent).toContain("factory"); - expect(el.textContent).toContain("Passed"); - }); - - test("an empty store shows an honest empty state, not a fabricated row", async () => { - stubFetch([]); - const el = await mount(); - expect(el.textContent).toContain("No eval runs yet"); - expect(el.textContent).not.toContain("bun run eval"); - }); - - test("a run's detail page renders its real steps, tool calls, and scorer reports", async () => { - stubFetch([passedRun]); - const el = await mount("/evals/evalrun_1"); - expect(el.textContent).toContain("Review the open PR"); - expect(el.textContent).toContain("Here is my review."); - expect(el.textContent).toContain("list_prs"); - expect(el.textContent).toContain("leaves-a-review"); - }); -}); diff --git a/apps/web/test/routes.test.tsx b/apps/web/test/routes.test.tsx index d65cfad1fd..5e3c512304 100644 --- a/apps/web/test/routes.test.tsx +++ b/apps/web/test/routes.test.tsx @@ -118,8 +118,8 @@ function stagePageTitle(markup: string): string | undefined { /** The first-run footer rail marks its own destination current: Routines, * Files, Skills, Agents, and Plugins are text rows with `aria-current="page"` - * on the lit one. Insights and Evals join only given honest usage, and - * stay reachable by URL and palette instead until then. Settings lives + * on the lit one. Insights joins only given honest usage, and + * stays reachable by URL and palette instead until then. Settings lives * beside the account row, so its route lights nothing in the chrome - the * stage title carries it. Returns the active row's label so tests confirm * the *right* footer affordance lights, and nothing else does. */ @@ -160,13 +160,12 @@ describe("route table", () => { "/skills", "/settings/skills", "/insights", - "/evals", "/plugins", "/settings", ]); }); - test("palette pages are Routines, Files, Skills, Agents, Plugins, Insights, Evals, Settings", () => { + test("palette pages are Routines, Files, Skills, Agents, Plugins, Insights, Settings", () => { expect(NAV_ROUTES.map((route) => route.label)).toEqual([ "Routines", "Files", @@ -174,7 +173,6 @@ describe("route table", () => { "Agents", "Plugins", "Insights", - "Evals", "Settings", ]); }); diff --git a/apps/web/test/sidebar.test.tsx b/apps/web/test/sidebar.test.tsx index edff62830b..f7ee9184eb 100644 --- a/apps/web/test/sidebar.test.tsx +++ b/apps/web/test/sidebar.test.tsx @@ -72,22 +72,9 @@ function usageBody(turns: number): unknown { }; } -const oneEvalRun = { - id: "evalrun_1", - evalName: "factory", - evalDescription: null, - configName: "default", - startedAt: "2026-01-01T00:00:00.000Z", - finishedAt: "2026-01-01T00:02:00.000Z", - stepCount: 1, - scorerTally: { passed: 1, failed: 0, skipped: 0 }, -}; - function stubFetch(options?: { readonly usageTurns?: number; - readonly evalRuns?: boolean; readonly failUsage?: boolean; - readonly failEvals?: boolean; }): void { globalThis.fetch = ((input: RequestInfo | URL) => { const path = typeof input === "string" ? input : String(input); @@ -100,13 +87,6 @@ function stubFetch(options?: { return Promise.resolve(json({ error: "unavailable" }, 500)); return Promise.resolve(json(usageBody(options?.usageTurns ?? 0))); } - if (path.includes("/eval-runs/runs")) { - if (options?.failEvals === true) - return Promise.resolve(json({ error: "unavailable" }, 500)); - return Promise.resolve( - json({ runs: options?.evalRuns === true ? [oneEvalRun] : [] }), - ); - } return Promise.resolve(json({ items: [] })); }) as typeof fetch; } @@ -217,7 +197,7 @@ describe("Sidebar", () => { expect(markup).not.toContain("shell-rail-item"); }); - test("first-run footer rail is Routines, Files, Skills, Agents, Plugins, then the account row — no Insights, Evals, or Inbox", () => { + test("first-run footer rail is Routines, Files, Skills, Agents, Plugins, then the account row — no Insights or Inbox", () => { const markup = renderSidebar("/w"); expect(footerRowLabelsFromMarkup(markup)).toEqual([ "Routines", @@ -241,8 +221,8 @@ describe("Sidebar", () => { ); }); - test("first-run footer rail does not list Evals or Insights before there is honest usage", async () => { - stubFetch({ usageTurns: 0, evalRuns: false }); + test("first-run footer rail does not list Insights before there is honest usage", async () => { + stubFetch({ usageTurns: 0 }); const { container, root } = await mountSidebar("/w"); expect(footerRowLabelsFromDom(container)).toEqual([ "Routines", @@ -271,13 +251,11 @@ describe("Sidebar", () => { expect(elsewhere).not.toMatch(/>Plugins<[\s\S]{0,80}aria-current="page"/); }); - test("Evals, Insights, and Plugins remain reachable by URL and command palette", () => { + test("Insights and Plugins remain reachable by URL and command palette", () => { const palettePaths = NAV_ROUTES.map((route) => route.path); const routedPaths = APP_ROUTES.map((route) => route.path); - expect(palettePaths).toContain("/evals"); expect(palettePaths).toContain("/insights"); expect(palettePaths).toContain("/plugins"); - expect(routedPaths).toContain("/evals"); expect(routedPaths).toContain("/insights"); expect(routedPaths).toContain("/plugins"); }); @@ -292,7 +270,7 @@ describe("Sidebar", () => { }); test("Insights joins the footer rail only when usage has turns", async () => { - stubFetch({ usageTurns: 4, evalRuns: false }); + stubFetch({ usageTurns: 4 }); const { container, root } = await mountSidebar("/insights"); expect(footerRowLabelsFromDom(container)).toEqual([ "Routines", @@ -310,27 +288,8 @@ describe("Sidebar", () => { container.remove(); }); - test("Evals joins the footer rail only when eval runs exist", async () => { - stubFetch({ usageTurns: 0, evalRuns: true }); - const { container, root } = await mountSidebar("/evals"); - expect(footerRowLabelsFromDom(container)).toEqual([ - "Routines", - "Files", - "Skills", - "Agents", - "Plugins", - "Evals", - ]); - const evals = [ - ...container.querySelectorAll(".shell-sidebar-footer-row"), - ].find((row) => row.textContent?.includes("Evals") === true); - expect(evals?.getAttribute("aria-current")).toBe("page"); - act(() => root.unmount()); - container.remove(); - }); - - test("a failed usage or evals probe omits the row rather than claiming usage", async () => { - stubFetch({ failUsage: true, failEvals: true }); + test("a failed usage probe omits the row rather than claiming usage", async () => { + stubFetch({ failUsage: true }); const { container, root } = await mountSidebar("/w"); expect(footerRowLabelsFromDom(container)).toEqual([ "Routines", diff --git a/bun.lock b/bun.lock index d356ea64a9..396fb739f8 100644 --- a/bun.lock +++ b/bun.lock @@ -57,7 +57,6 @@ "@corbits/e2b-sandbox-sidecar": "workspace:*", "@corbits/echo-workflow": "workspace:*", "@corbits/error-sink": "workspace:*", - "@corbits/evals": "workspace:*", "@corbits/exa-topic-watch-workflow": "workspace:*", "@corbits/granola-call-workflow": "workspace:*", "@corbits/hub-api-client": "workspace:*", @@ -708,7 +707,6 @@ "@corbits/workflows": "workspace:*", "@intx/agent": "workspace:*", "@intx/db": "workspace:*", - "@intx/hub-api": "workspace:*", "@intx/hub-sessions": "workspace:*", "@intx/log": "0.3.0", "@intx/types": "workspace:*", @@ -717,7 +715,6 @@ "@workbench/onboarding": "workspace:*", "arktype": "catalog:", "drizzle-orm": "catalog:", - "hono": "catalog:", "postgres": "catalog:", }, "devDependencies": { @@ -3612,18 +3609,12 @@ "@changesets/write/prettier": ["prettier@2.8.8", "", { "bin": { "prettier": "bin-prettier.js" } }, "sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q=="], - "@corbits/artifacts-hub/@corbits/artifacts": ["@corbits/artifacts@github:corbitsdev/corbits-artifacts#dc435ba", { "dependencies": { "@hono/standard-validator": "^0.2.3" }, "peerDependencies": { "@intx/hub-api": "^0.2.2", "@intx/types": "^0.2.2", "arktype": "^2.1.29", "drizzle-orm": "^0.45.2", "hono": "^4.12.32", "hono-openapi": "^1.2.0", "postgres": "^3.4.9" } }, "corbitsdev-corbits-artifacts-dc435ba", "sha512-VksIyrrJY9nRke0QyjgqGdW3gLD73Bi3+CzpJWG+l8vRJKdUxVi7ms2Mp95NmJXr4TpVX6i1z4odufX/QBxnSw=="], - - "@corbits/chat-ui/@corbits/react-ui": ["@corbits/react-ui@github:corbitsdev/react-ui#3b12281", { "dependencies": { "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "tailwind-merge": "^3.3.1" }, "peerDependencies": { "@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-dropdown-menu": "^2.1.16", "@radix-ui/react-slot": "^1.2.3", "@radix-ui/react-tooltip": "^1.2.8", "@tanstack/react-query": "^5.90.2", "lucide-react": "^0.545.0 || ^1.0.0", "react": "^18.2.0 || ^19.0.0", "react-dom": "^18.2.0 || ^19.0.0", "sonner": "^2.0.7" }, "optionalPeers": ["@tanstack/react-query"] }, "corbitsdev-react-ui-3b12281", "sha512-Abvm/DO0Gqg0ITHGT9355ZxyKRPMVJLSSQSjpd3a8qt4JPrSMOLIOS4sX8ZMNNaArIbnY9F+VKrOWkUJUyO4Nw=="], + "@corbits/artifact-ui/@corbits/react-ui": ["@corbits/react-ui@github:corbitsdev/react-ui#3b12281", { "dependencies": { "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "tailwind-merge": "^3.3.1" }, "peerDependencies": { "@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-dropdown-menu": "^2.1.16", "@radix-ui/react-slot": "^1.2.3", "@radix-ui/react-tooltip": "^1.2.8", "@tanstack/react-query": "^5.90.2", "lucide-react": "^0.545.0 || ^1.0.0", "react": "^18.2.0 || ^19.0.0", "react-dom": "^18.2.0 || ^19.0.0", "sonner": "^2.0.7" }, "optionalPeers": ["@tanstack/react-query"] }, "corbitsdev-react-ui-3b12281", "sha512-Abvm/DO0Gqg0ITHGT9355ZxyKRPMVJLSSQSjpd3a8qt4JPrSMOLIOS4sX8ZMNNaArIbnY9F+VKrOWkUJUyO4Nw=="], - "@corbits/context-menu/@corbits/react-ui": ["@corbits/react-ui@github:corbitsdev/react-ui#3b12281", { "dependencies": { "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "tailwind-merge": "^3.3.1" }, "peerDependencies": { "@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-dropdown-menu": "^2.1.16", "@radix-ui/react-slot": "^1.2.3", "@radix-ui/react-tooltip": "^1.2.8", "@tanstack/react-query": "^5.90.2", "lucide-react": "^0.545.0 || ^1.0.0", "react": "^18.2.0 || ^19.0.0", "react-dom": "^18.2.0 || ^19.0.0", "sonner": "^2.0.7" }, "optionalPeers": ["@tanstack/react-query"] }, "corbitsdev-react-ui-3b12281", "sha512-Abvm/DO0Gqg0ITHGT9355ZxyKRPMVJLSSQSjpd3a8qt4JPrSMOLIOS4sX8ZMNNaArIbnY9F+VKrOWkUJUyO4Nw=="], + "@corbits/artifacts-hub/@corbits/artifacts": ["@corbits/artifacts@github:corbitsdev/corbits-artifacts#dc435ba", { "dependencies": { "@hono/standard-validator": "^0.2.3" }, "peerDependencies": { "@intx/hub-api": "^0.2.2", "@intx/types": "^0.2.2", "arktype": "^2.1.29", "drizzle-orm": "^0.45.2", "hono": "^4.12.32", "hono-openapi": "^1.2.0", "postgres": "^3.4.9" } }, "corbitsdev-corbits-artifacts-dc435ba", "sha512-VksIyrrJY9nRke0QyjgqGdW3gLD73Bi3+CzpJWG+l8vRJKdUxVi7ms2Mp95NmJXr4TpVX6i1z4odufX/QBxnSw=="], "@corbits/mailbox/hono-openapi": ["hono-openapi@1.3.1", "", { "peerDependencies": { "@hono/standard-validator": "^0.2.0", "@standard-community/standard-json": "^0.3.5", "@standard-community/standard-openapi": "^0.2.9", "@types/json-schema": "^7.0.15", "hono": "^4.11.2", "openapi-types": "^12.1.3" }, "optionalPeers": ["@hono/standard-validator", "hono"] }, "sha512-NLVeVkhKZ3drmQNEIPac8HX8Y54uf1hJAgIM/7MfDsaeVVmB+QILWQxx5x3R3NvRHgedcbEbOCGY2uR7WQYyMw=="], - "@corbits/plugins-ui/@corbits/react-ui": ["@corbits/react-ui@github:corbitsdev/react-ui#3b12281", { "dependencies": { "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "tailwind-merge": "^3.3.1" }, "peerDependencies": { "@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-dropdown-menu": "^2.1.16", "@radix-ui/react-slot": "^1.2.3", "@radix-ui/react-tooltip": "^1.2.8", "@tanstack/react-query": "^5.90.2", "lucide-react": "^0.545.0 || ^1.0.0", "react": "^18.2.0 || ^19.0.0", "react-dom": "^18.2.0 || ^19.0.0", "sonner": "^2.0.7" }, "optionalPeers": ["@tanstack/react-query"] }, "corbitsdev-react-ui-3b12281", "sha512-Abvm/DO0Gqg0ITHGT9355ZxyKRPMVJLSSQSjpd3a8qt4JPrSMOLIOS4sX8ZMNNaArIbnY9F+VKrOWkUJUyO4Nw=="], - - "@corbits/settings-ui/@corbits/react-ui": ["@corbits/react-ui@github:corbitsdev/react-ui#3b12281", { "dependencies": { "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "tailwind-merge": "^3.3.1" }, "peerDependencies": { "@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-dropdown-menu": "^2.1.16", "@radix-ui/react-slot": "^1.2.3", "@radix-ui/react-tooltip": "^1.2.8", "@tanstack/react-query": "^5.90.2", "lucide-react": "^0.545.0 || ^1.0.0", "react": "^18.2.0 || ^19.0.0", "react-dom": "^18.2.0 || ^19.0.0", "sonner": "^2.0.7" }, "optionalPeers": ["@tanstack/react-query"] }, "corbitsdev-react-ui-3b12281", "sha512-Abvm/DO0Gqg0ITHGT9355ZxyKRPMVJLSSQSjpd3a8qt4JPrSMOLIOS4sX8ZMNNaArIbnY9F+VKrOWkUJUyO4Nw=="], - "@esbuild-kit/core-utils/esbuild": ["esbuild@0.18.20", "", { "optionalDependencies": { "@esbuild/android-arm": "0.18.20", "@esbuild/android-arm64": "0.18.20", "@esbuild/android-x64": "0.18.20", "@esbuild/darwin-arm64": "0.18.20", "@esbuild/darwin-x64": "0.18.20", "@esbuild/freebsd-arm64": "0.18.20", "@esbuild/freebsd-x64": "0.18.20", "@esbuild/linux-arm": "0.18.20", "@esbuild/linux-arm64": "0.18.20", "@esbuild/linux-ia32": "0.18.20", "@esbuild/linux-loong64": "0.18.20", "@esbuild/linux-mips64el": "0.18.20", "@esbuild/linux-ppc64": "0.18.20", "@esbuild/linux-riscv64": "0.18.20", "@esbuild/linux-s390x": "0.18.20", "@esbuild/linux-x64": "0.18.20", "@esbuild/netbsd-x64": "0.18.20", "@esbuild/openbsd-x64": "0.18.20", "@esbuild/sunos-x64": "0.18.20", "@esbuild/win32-arm64": "0.18.20", "@esbuild/win32-ia32": "0.18.20", "@esbuild/win32-x64": "0.18.20" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA=="], "@eslint-community/eslint-utils/eslint-visitor-keys": ["eslint-visitor-keys@3.4.3", "", {}, "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag=="], @@ -3658,8 +3649,6 @@ "@workbench/sidecar/@corbits/oauth-core": ["@corbits/oauth-core@github:corbitsdev/corbits-oauth-core#e1e69e6", { "dependencies": { "arktype": "2.2.3" } }, "corbitsdev-corbits-oauth-core-e1e69e6", "sha512-wUrD73iVyk/Dtb4yRn3hCh6N8syfsEvAkRz4XYqT0FIB3sYDSMYEWHcDMJx9nq64my/HBOeN1J6PHIOtpjhtPg=="], - "@workbench/web/@corbits/react-ui": ["@corbits/react-ui@github:corbitsdev/react-ui#3b12281", { "dependencies": { "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "tailwind-merge": "^3.3.1" }, "peerDependencies": { "@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-dropdown-menu": "^2.1.16", "@radix-ui/react-slot": "^1.2.3", "@radix-ui/react-tooltip": "^1.2.8", "@tanstack/react-query": "^5.90.2", "lucide-react": "^0.545.0 || ^1.0.0", "react": "^18.2.0 || ^19.0.0", "react-dom": "^18.2.0 || ^19.0.0", "sonner": "^2.0.7" }, "optionalPeers": ["@tanstack/react-query"] }, "corbitsdev-react-ui-3b12281", "sha512-Abvm/DO0Gqg0ITHGT9355ZxyKRPMVJLSSQSjpd3a8qt4JPrSMOLIOS4sX8ZMNNaArIbnY9F+VKrOWkUJUyO4Nw=="], - "ajv-formats/ajv": ["ajv@8.20.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA=="], "better-call/@better-auth/utils": ["@better-auth/utils@0.5.0", "", { "dependencies": { "@noble/hashes": "^2.0.1" } }, "sha512-BL8W4EfIZFwlu0r54m3v1ztjDhu6dDe/amLTm0xybmbZaNgYUqhD3SjpAsnq0q8YD6/ki4iwIgxJNLP/N3TxiA=="], diff --git a/packages/evals/package.json b/packages/evals/package.json index 1425855d82..48715415af 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -20,7 +20,6 @@ "@corbits/webhook-triggers": "workspace:*", "@intx/agent": "workspace:*", "@intx/db": "workspace:*", - "@intx/hub-api": "workspace:*", "@intx/hub-sessions": "workspace:*", "@intx/log": "0.3.0", "@intx/types": "workspace:*", @@ -32,7 +31,6 @@ "@workbench/onboarding": "workspace:*", "arktype": "catalog:", "drizzle-orm": "catalog:", - "hono": "catalog:", "postgres": "catalog:" }, "devDependencies": { diff --git a/packages/evals/src/index.ts b/packages/evals/src/index.ts index 872811fcc6..48464dafe4 100644 --- a/packages/evals/src/index.ts +++ b/packages/evals/src/index.ts @@ -68,7 +68,6 @@ export { export type { EvalRunRecord, EvalRunStore } from "./store/store.ts"; export { createPostgresEvalRunStore } from "./store/pg-store.ts"; export { applyEvalsMigrations } from "./store/migrations.ts"; -export { createEvalRunRoutes, type CreateEvalRunRoutesDeps } from "./routes.ts"; export { bootMyraTarget } from "./targets/real-target.ts"; export type { EvalApiResult, diff --git a/packages/evals/src/routes.test.ts b/packages/evals/src/routes.test.ts deleted file mode 100644 index e570935e14..0000000000 --- a/packages/evals/src/routes.test.ts +++ /dev/null @@ -1,186 +0,0 @@ -// Route-level tests: request parsing, grant gating, and error-envelope -// mapping against an in-memory fake store. Real Postgres persistence is -// covered in store/store.test.ts. -import { expect, test } from "bun:test"; -import { Hono } from "hono"; -import type { MiddlewareHandler } from "hono"; - -import type { TenantEnv } from "@intx/hub-api"; - -import { createEvalRunRoutes } from "./routes.ts"; -import type { EvalRunRecord, EvalRunStore } from "./store/store.ts"; - -const TENANT = { id: "tnt_1" }; - -function fixtureRecord(overrides: Partial = {}): EvalRunRecord { - return { - id: "evalrun_1", - evalName: "ai-daily-research", - configName: "default", - startedAt: "2026-08-16T00:00:00.000Z", - finishedAt: "2026-08-16T00:01:00.000Z", - steps: [ - { - stepIndex: 0, - turn: { human: "hi", replyText: "hello", toolCalls: [] }, - scorerReports: [ - { - name: "asksQuestions", - score: 1, - pass: true, - reason: "ok", - stepIndex: 0, - }, - { - name: "judge", - score: 0, - pass: false, - reason: "missed it", - stepIndex: 0, - }, - ], - }, - ], - ...overrides, - }; -} - -function fakeStore(records: EvalRunRecord[] = []): EvalRunStore { - return { - async save() { - throw new Error("not used by these tests"); - }, - async recent(evalName, limit) { - return records.filter((r) => r.evalName === evalName).slice(0, limit); - }, - async recentAcrossEvals(limit) { - return records.slice(0, limit); - }, - async get(id) { - return records.find((r) => r.id === id) ?? null; - }, - }; -} - -function buildApp( - store: EvalRunStore, - opts?: { forbidden?: boolean }, -): Hono { - const routes = createEvalRunRoutes({ - store, - requireGrant: () => async (c, next) => { - if (opts?.forbidden === true) { - return c.json({ error: { code: "forbidden", message: "no" } }, 403); - } - await next(); - }, - }); - const asPrincipal: MiddlewareHandler = async (c, next) => { - c.set("tenant", TENANT as never); - await next(); - }; - const app = new Hono(); - app.use("*", asPrincipal); - app.route("/", routes); - return app; -} - -test("GET /runs returns an empty list when no runs are recorded", async () => { - const app = buildApp(fakeStore()); - const response = await app.request("/runs"); - expect(response.status).toBe(200); - const body = (await response.json()) as { runs: unknown[] }; - expect(body.runs).toEqual([]); -}); - -test("GET /runs returns shaped summaries across evals, newest first order preserved", async () => { - const record = fixtureRecord(); - const app = buildApp(fakeStore([record])); - const response = await app.request("/runs"); - expect(response.status).toBe(200); - const body = (await response.json()) as { - runs: { - id: string; - evalName: string; - evalDescription: string | null; - configName: string; - stepCount: number; - scorerTally: { passed: number; failed: number; skipped: number }; - }[]; - }; - expect(body.runs).toHaveLength(1); - const run = body.runs[0]; - expect(run?.id).toBe("evalrun_1"); - expect(run?.evalName).toBe("ai-daily-research"); - expect(typeof run?.evalDescription).toBe("string"); - expect(run?.configName).toBe("default"); - expect(run?.stepCount).toBe(1); - expect(run?.scorerTally).toEqual({ passed: 1, failed: 1, skipped: 0 }); -}); - -test("GET /runs?evalName= filters to that eval only", async () => { - const a = fixtureRecord({ id: "evalrun_a", evalName: "ai-daily-research" }); - const b = fixtureRecord({ id: "evalrun_b", evalName: "docs-on-sdk-change" }); - const app = buildApp(fakeStore([a, b])); - const response = await app.request("/runs?evalName=docs-on-sdk-change"); - const body = (await response.json()) as { runs: { id: string }[] }; - expect(body.runs).toHaveLength(1); - expect(body.runs[0]?.id).toBe("evalrun_b"); -}); - -test("GET /runs with an unknown eval name yields a null description, not a crash", async () => { - const record = fixtureRecord({ evalName: "not-a-real-eval" }); - const app = buildApp(fakeStore([record])); - const response = await app.request("/runs"); - const body = (await response.json()) as { - runs: { evalDescription: string | null }[]; - }; - expect(body.runs[0]?.evalDescription).toBeNull(); -}); - -test("GET /runs?limit=0 400s", async () => { - const app = buildApp(fakeStore()); - const response = await app.request("/runs?limit=0"); - expect(response.status).toBe(400); - const body = (await response.json()) as { error: { code: string } }; - expect(body.error.code).toBe("bad_request"); -}); - -test("GET /runs?limit=not-a-number 400s", async () => { - const app = buildApp(fakeStore()); - const response = await app.request("/runs?limit=abc"); - expect(response.status).toBe(400); -}); - -test("GET /runs/:runId returns the run's steps and scorer reports", async () => { - const record = fixtureRecord(); - const app = buildApp(fakeStore([record])); - const response = await app.request("/runs/evalrun_1"); - expect(response.status).toBe(200); - const body = (await response.json()) as { - id: string; - steps: unknown[]; - }; - expect(body.id).toBe("evalrun_1"); - expect(body.steps).toEqual(record.steps); -}); - -test("GET /runs/:runId 404s for an unknown id", async () => { - const app = buildApp(fakeStore()); - const response = await app.request("/runs/evalrun_missing"); - expect(response.status).toBe(404); - const body = (await response.json()) as { error: { code: string } }; - expect(body.error.code).toBe("not_found"); -}); - -test("GET /runs is gated by requireGrant", async () => { - const app = buildApp(fakeStore(), { forbidden: true }); - const response = await app.request("/runs"); - expect(response.status).toBe(403); -}); - -test("GET /runs/:runId is gated by requireGrant", async () => { - const app = buildApp(fakeStore(), { forbidden: true }); - const response = await app.request("/runs/evalrun_1"); - expect(response.status).toBe(403); -}); diff --git a/packages/evals/src/routes.ts b/packages/evals/src/routes.ts deleted file mode 100644 index 54213273bc..0000000000 --- a/packages/evals/src/routes.ts +++ /dev/null @@ -1,150 +0,0 @@ -// Read-only HTTP surface over `EvalRunStore` (CL-6465): recent eval runs -// across every eval, and one run's full step/scorer detail. Mounted the -// same way `@corbits/insights` mounts its own routes.ts — one -// `Hono` app gated by `requireGrant`, no data partition by -// tenant (eval runs aren't tenant-owned; the tenant middleware here is -// only the grant gate). -// -// Every field below is either already human-readable (`evalName`, -// `configName`, scorer `name`/`reason`) or a dedicated `id` a UI never -// prints as text, only follows for `/runs/:runId` — no `run_`/`wfd_`-style -// identifier ever lands in a display field. -import { Hono } from "hono"; -import { type } from "arktype"; - -import type { RequireGrant, TenantEnv } from "@intx/hub-api"; - -import { ALL_EVALS } from "./cases/index.ts"; -import type { EvalRunRecord, EvalRunStore } from "./store/store.ts"; -import { makeErrorEnvelope } from "@corbits/error-sink"; - -const DEFAULT_LIMIT = 20; -const MAX_LIMIT = 100; - -const RunsQuery = type({ - "limit?": "string", - "evalName?": "string", -}); - -function parseLimit(raw: string | undefined): number | undefined { - if (raw === undefined) return DEFAULT_LIMIT; - const parsed = Number(raw); - if (!Number.isInteger(parsed) || parsed < 1 || parsed > MAX_LIMIT) { - return undefined; - } - return parsed; -} - -function evalDescription(evalName: string): string | null { - return ALL_EVALS.find((e) => e.name === evalName)?.description ?? null; -} - -function scorerTally(record: EvalRunRecord): { - passed: number; - failed: number; - skipped: number; -} { - let passed = 0; - let failed = 0; - let skipped = 0; - for (const step of record.steps) { - for (const report of step.scorerReports) { - if (report.skipped) skipped += 1; - else if (report.pass) passed += 1; - else failed += 1; - } - } - return { passed, failed, skipped }; -} - -function toSummary(record: EvalRunRecord) { - return { - id: record.id, - evalName: record.evalName, - evalDescription: evalDescription(record.evalName), - configName: record.configName, - startedAt: record.startedAt, - finishedAt: record.finishedAt, - stepCount: record.steps.length, - scorerTally: scorerTally(record), - }; -} - -function toDetail(record: EvalRunRecord) { - return { - id: record.id, - evalName: record.evalName, - evalDescription: evalDescription(record.evalName), - configName: record.configName, - startedAt: record.startedAt, - finishedAt: record.finishedAt, - steps: record.steps, - }; -} - -export type CreateEvalRunRoutesDeps = { - store: EvalRunStore; - requireGrant: RequireGrant; -}; - -export function createEvalRunRoutes( - deps: CreateEvalRunRoutesDeps, -): Hono { - const app = new Hono(); - - /** - * Recent eval runs across every eval (or one eval, via `?evalName=`), - * newest first — the run history list. - */ - app.get("/runs", deps.requireGrant("eval-run:*", "read"), async (c) => { - const raw = RunsQuery(c.req.query()); - if (raw instanceof type.errors) { - return c.json( - makeErrorEnvelope({ - code: "bad_request", - userMessage: `invalid query: ${raw.summary}`, - }), - 400, - ); - } - const limit = parseLimit(raw.limit); - if (limit === undefined) { - return c.json( - makeErrorEnvelope({ - code: "bad_request", - userMessage: `limit must be an integer between 1 and ${MAX_LIMIT}`, - }), - 400, - ); - } - const records = - raw.evalName === undefined - ? await deps.store.recentAcrossEvals(limit) - : await deps.store.recent(raw.evalName, limit); - return c.json({ runs: records.map(toSummary) }); - }); - - /** - * One run's full step-by-step transcript and scorer reports. - */ - app.get( - "/runs/:runId", - deps.requireGrant("eval-run:*", "read"), - async (c) => { - const runId = c.req.param("runId"); - const record = await deps.store.get(runId); - if (record === null) { - return c.json( - makeErrorEnvelope({ - code: "not_found", - userMessage: "eval run not found", - }), - 404, - ); - } - return c.json(toDetail(record)); - }, - ); - - return app; -}