From b9506bdb9ad9244c4c15776718232c20b1a57284 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Wed, 30 Sep 2026 09:53:25 -0700 Subject: [PATCH] test(e2e): aimock stands in for the inference provider so a worker can reply (CL-9626) --- bun.lock | 27 ++---------------- e2e/browser/reply.test.ts | 59 +++++++++++++++++++++++++++++++++++++++ e2e/lib/aimock.ts | 34 ++++++++++++++++++++++ package.json | 1 + 4 files changed, 97 insertions(+), 24 deletions(-) create mode 100644 e2e/browser/reply.test.ts create mode 100644 e2e/lib/aimock.ts diff --git a/bun.lock b/bun.lock index 9692746c9..bb6f3086d 100644 --- a/bun.lock +++ b/bun.lock @@ -8,6 +8,7 @@ "arktype": "catalog:", }, "devDependencies": { + "@copilotkit/aimock": "^1.43.0", "@corbits/artifacts": "github:corbitsdev/corbits-artifacts#4a2a3c22187991a6ce598faf794908fedd252e98", "@corbits/codex-provider": "github:corbitsdev/corbits-codex-provider#a1f475f341261c05fa8dade0c82f1246c175b5b3", "@corbits/mailbox": "github:corbitsdev/corbits-mailbox#65590a85fa143251b3ac20ba2eca92fc35e70e51", @@ -629,6 +630,8 @@ "@chat-adapter/state-memory": ["@chat-adapter/state-memory@4.40.0", "", { "dependencies": { "chat": "4.40.0" } }, "sha512-NmFFkvUfnDBDS/mkzrtKHdW757EGt0eX3OQH5IPxrKqhjSsNzpqUdkJ/uDB4BP0Rbex4oW/HgwhZ+JGONVbMBQ=="], + "@copilotkit/aimock": ["@copilotkit/aimock@1.43.0", "", { "peerDependencies": { "jest": ">=29", "vitest": ">=3" }, "optionalPeers": ["jest", "vitest"], "bin": { "aimock": "dist/aimock-cli.js", "llmock": "dist/cli.js" } }, "sha512-V8D8cjSMtAR4rHaD2ulp3DR/ZguXQjDNOqwKDNvjb/OJL/KR1I0SDf5xpRCd+TYKjhmf9fM8Xpg3QBKPmV9Kmw=="], + "@corbits/agent-token": ["@corbits/agent-token@github:corbitsdev/corbits-agent-token#ed286f4", { "dependencies": { "@intx/db": "^0.3.0", "arktype": "^2.2.0", "drizzle-orm": "^0.45.2", "hono": "^4.11.9", "postgres": "^3.4.8" } }, "corbitsdev-corbits-agent-token-ed286f4", "sha512-k7puSwtCC2ZF7W0lzswtBoKxu4n+VjWwtzf8oa1OWe72FEI4TDdQ5gybqq/oW9XJAUHeCkjPL9bdwxG6C2TfpQ=="], "@corbits/approvals": ["@corbits/approvals@workspace:packages/approvals"], @@ -1837,12 +1840,8 @@ "@babel/helper-compilation-targets/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], - "@corbits/codex-provider/@corbits/oauth-core": ["@corbits/oauth-core@github:corbitsdev/corbits-oauth-core#3028f86", { "dependencies": { "@intx/db": "^0.3.0", "@intx/hub-api": "^0.3.0", "@intx/hub-common": "^0.3.0", "@intx/types": "^0.3.0", "arktype": "2.2.3", "drizzle-orm": "^0.45.2", "hono": "^4.11.9" } }, "corbitsdev-corbits-oauth-core-3028f86", "sha512-KMxRZgEi7S99tqoWhXqAAm5Bv4ef5tn3eYFcfbiHHX/E+GPnbedF44/ISbGs2gedoR2U9EtA1gnmh+q+Qfg00w=="], - "@corbits/mailbox/hono-openapi": ["hono-openapi@1.3.1", "", { "peerDependencies": { "@hono/standard-validator": "^0.2.0", "@standard-community/standard-json": "^0.3.5", "@standard-community/standard-openapi": "^0.2.9", "@types/json-schema": "^7.0.15", "hono": "^4.11.2", "openapi-types": "^12.1.3" }, "optionalPeers": ["@hono/standard-validator", "hono"] }, "sha512-NLVeVkhKZ3drmQNEIPac8HX8Y54uf1hJAgIM/7MfDsaeVVmB+QILWQxx5x3R3NvRHgedcbEbOCGY2uR7WQYyMw=="], - "@corbits/xai-provider/@corbits/oauth-core": ["@corbits/oauth-core@github:corbitsdev/corbits-oauth-core#3028f86", { "dependencies": { "@intx/db": "^0.3.0", "@intx/hub-api": "^0.3.0", "@intx/hub-common": "^0.3.0", "@intx/types": "^0.3.0", "arktype": "2.2.3", "drizzle-orm": "^0.45.2", "hono": "^4.11.9" } }, "corbitsdev-corbits-oauth-core-3028f86", "sha512-KMxRZgEi7S99tqoWhXqAAm5Bv4ef5tn3eYFcfbiHHX/E+GPnbedF44/ISbGs2gedoR2U9EtA1gnmh+q+Qfg00w=="], - "@esbuild-kit/core-utils/esbuild": ["esbuild@0.18.20", "", { "optionalDependencies": { "@esbuild/android-arm": "0.18.20", "@esbuild/android-arm64": "0.18.20", "@esbuild/android-x64": "0.18.20", "@esbuild/darwin-arm64": "0.18.20", "@esbuild/darwin-x64": "0.18.20", "@esbuild/freebsd-arm64": "0.18.20", "@esbuild/freebsd-x64": "0.18.20", "@esbuild/linux-arm": "0.18.20", "@esbuild/linux-arm64": "0.18.20", "@esbuild/linux-ia32": "0.18.20", "@esbuild/linux-loong64": "0.18.20", "@esbuild/linux-mips64el": "0.18.20", "@esbuild/linux-ppc64": "0.18.20", "@esbuild/linux-riscv64": "0.18.20", "@esbuild/linux-s390x": "0.18.20", "@esbuild/linux-x64": "0.18.20", "@esbuild/netbsd-x64": "0.18.20", "@esbuild/openbsd-x64": "0.18.20", "@esbuild/sunos-x64": "0.18.20", "@esbuild/win32-arm64": "0.18.20", "@esbuild/win32-ia32": "0.18.20", "@esbuild/win32-x64": "0.18.20" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA=="], "@npmcli/agent/lru-cache": ["lru-cache@11.5.2", "", {}, "sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g=="], @@ -1859,18 +1858,6 @@ "@tailwindcss/oxide-wasm32-wasi/tslib": ["tslib@2.8.1", "", { "bundled": true }, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], - "@workbench/hub/@corbits/artifacts": ["@corbits/artifacts@github:corbitsdev/corbits-artifacts#4a2a3c2", { "dependencies": { "@hono/standard-validator": "^0.2.3" }, "peerDependencies": { "@intx/agent": "^0.3.0", "@intx/hub-api": "^0.3.0", "@intx/types": "^0.3.0", "arktype": "^2.1.29", "drizzle-orm": "^0.45.2", "hono": "^4.12.32", "hono-openapi": "^1.2.0", "postgres": "^3.4.9" } }, "corbitsdev-corbits-artifacts-4a2a3c2", "sha512-UY5nkQ31kGmYgqgIOSlbigwvm7c/s2kpqEbF6Oqo0rzoS1RejpPp+w/Yim6h2MRpYCl5CXgAET1MhkDboHuWBA=="], - - "@workbench/hub/@corbits/codex-provider": ["@corbits/codex-provider@github:corbitsdev/corbits-codex-provider#a1f475f", { "dependencies": { "@corbits/oauth-core": "github:corbitsdev/corbits-oauth-core", "@corbits/openai-responses": "github:corbitsdev/corbits-openai-responses", "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": ">=0.3.0", "@intx/types": ">=0.3.0" } }, "corbitsdev-corbits-codex-provider-a1f475f", "sha512-D6d0lblWx/4KudfLwQoQGrLB8GKRLP/HVp1cx08L2/hOGZAgH6txLPpeEtBqjl+R520qXSSPfM0kDC+3XNaAfw=="], - - "@workbench/hub/@corbits/mailbox": ["@corbits/mailbox@github:corbitsdev/corbits-mailbox#65590a8", { "dependencies": { "@hono/standard-validator": "0.2.3", "@intx/crypto": "0.3.0", "@standard-community/standard-json": "0.3.5", "@standard-community/standard-openapi": "0.2.9", "arktype": "2.1.29", "hono-openapi": "1.3.1" }, "peerDependencies": { "@intx/log": "^0.2.2", "drizzle-orm": "^0.45.2", "hono": "^4.12.0", "postgres": "^3.4.0" } }, "corbitsdev-corbits-mailbox-65590a8", "sha512-tK7xXcL780iX3I+5R68gmyqgEA0uwYO1qdTDNn/LF6tOiFIm0lt2szjspJ8JMKMTzKC6SSai03p723rEGM1jOg=="], - - "@workbench/hub/@corbits/mcp": ["@corbits/mcp@github:corbitsdev/corbits-mcp#9df1ab8", { "dependencies": { "@corbits/credential-mcp": "github:corbitsdev/corbits-credential-mcp#fb9ec3a6754348cd33c388c70c0830b8844c83de", "@intx/db": "^0.3.0", "@intx/hub-api": "^0.3.0", "@intx/types": "^0.3.0", "arktype": "2.1.20", "drizzle-orm": "^0.45.2", "hono": "^4.11.9" }, "peerDependencies": { "@intx/agent": ">=0.3.0" } }, "corbitsdev-corbits-mcp-9df1ab8", "sha512-5ILvgNI4J/U2kWitFKZmIJO1qybCdIkKZ73ZELu6PWrbw6tARxO8/ffaGoK0nIMrT4EcBZaMd6YkR0vLdPOWnw=="], - - "@workbench/hub/@corbits/memory": ["@corbits/memory@github:corbitsdev/corbits-memory#8be608b", { "dependencies": { "@intx/agent": "0.2.2", "@intx/authz": "0.2.2", "@intx/hub-api": "0.2.2", "@intx/log": "0.2.2", "@intx/types": "^0.3.0", "@intx/workflow": "0.2.2", "arktype": "^2.1.29", "drizzle-orm": "^0.45.1", "hono": "^4.9.0", "hono-openapi": "^1.3.1", "postgres": "^3.4.7" } }, "corbitsdev-corbits-memory-8be608b", "sha512-7NcVcz3IHFTfx6ICQT+4gjX1PBNLqjlCqoEOPDFVW7wc4e2+Ueg9wOZQ+0MVttpQCaDg6MObl4kie410UQm/sQ=="], - - "@workbench/hub/@corbits/xai-provider": ["@corbits/xai-provider@github:corbitsdev/corbits-xai-provider#9f2d4ba", { "dependencies": { "@corbits/oauth-core": "github:corbitsdev/corbits-oauth-core", "@corbits/openai-responses": "github:corbitsdev/corbits-openai-responses", "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": ">=0.3.0", "@intx/types": ">=0.3.0" } }, "corbitsdev-corbits-xai-provider-9f2d4ba", "sha512-ZjvKxGcp+QzCh8zeXHg7kNbtkpR/itZkmQAcdG1aOAOjHaNwfGVpkPc/c/XtwPNsQyaDjF8W9CHb4rH1xI2uCA=="], - "better-call/@better-auth/utils": ["@better-auth/utils@0.5.0", "", { "dependencies": { "@noble/hashes": "^2.0.1" } }, "sha512-BL8W4EfIZFwlu0r54m3v1ztjDhu6dDe/amLTm0xybmbZaNgYUqhD3SjpAsnq0q8YD6/ki4iwIgxJNLP/N3TxiA=="], "body-parser/content-type": ["content-type@2.1.0", "", {}, "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag=="], @@ -1965,14 +1952,6 @@ "@esbuild-kit/core-utils/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.18.20", "", { "os": "win32", "cpu": "x64" }, "sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ=="], - "@workbench/hub/@corbits/codex-provider/@corbits/openai-responses": ["@corbits/openai-responses@github:corbitsdev/corbits-openai-responses#7efe000", { "dependencies": { "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": ">=0.3.0", "@intx/types": ">=0.3.0" } }, "corbitsdev-corbits-openai-responses-7efe000", "sha512-pH6C8T1WDJjqM+YbnQpofd7HgP5krEj48JCCQK16KI98a6kaz+gffpgdOdQpU3aNVZnFlyADXod1Z4tqUG/3ww=="], - - "@workbench/hub/@corbits/mailbox/hono-openapi": ["hono-openapi@1.3.1", "", { "peerDependencies": { "@hono/standard-validator": "^0.2.0", "@standard-community/standard-json": "^0.3.5", "@standard-community/standard-openapi": "^0.2.9", "@types/json-schema": "^7.0.15", "hono": "^4.11.2", "openapi-types": "^12.1.3" }, "optionalPeers": ["@hono/standard-validator", "hono"] }, "sha512-NLVeVkhKZ3drmQNEIPac8HX8Y54uf1hJAgIM/7MfDsaeVVmB+QILWQxx5x3R3NvRHgedcbEbOCGY2uR7WQYyMw=="], - - "@workbench/hub/@corbits/mcp/@corbits/credential-mcp": ["@corbits/credential-mcp@github:corbitsdev/corbits-credential-mcp#fb9ec3a", { "peerDependencies": { "@intx/harness": ">=0.3.0", "@intx/types": ">=0.3.0" } }, "corbitsdev-corbits-credential-mcp-fb9ec3a", "sha512-Co4jR+L3pfBbVYY7I6jRftoHeSb5+LCxzEEaAhVLYA3aY2PrPHZV37+DWxj3fcK/KdzboqbT3XSnqU8DyHRU9Q=="], - - "@workbench/hub/@corbits/xai-provider/@corbits/openai-responses": ["@corbits/openai-responses@github:corbitsdev/corbits-openai-responses#7efe000", { "dependencies": { "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": ">=0.3.0", "@intx/types": ">=0.3.0" } }, "corbitsdev-corbits-openai-responses-7efe000", "sha512-pH6C8T1WDJjqM+YbnQpofd7HgP5krEj48JCCQK16KI98a6kaz+gffpgdOdQpU3aNVZnFlyADXod1Z4tqUG/3ww=="], - "cross-spawn/which/isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], "minipass-flush/minipass/yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="], diff --git a/e2e/browser/reply.test.ts b/e2e/browser/reply.test.ts new file mode 100644 index 000000000..d5a544327 --- /dev/null +++ b/e2e/browser/reply.test.ts @@ -0,0 +1,59 @@ +// A fresh user connects an inference provider served by aimock, creates a +// bench from /new, and the worker's reply lands in the thread while the +// bench stops showing "working". +import { expect, test } from "bun:test"; +import { bootAimock, MOCK_REPLY } from "../lib/aimock"; +import { bootBrowserApp, browserGate } from "../lib/browser"; +import { clickText, STEP_TIMEOUT, waitForText } from "../lib/first-run"; + +const describeBrowser = browserGate(import.meta.path); + +describeBrowser("worker reply", () => { + const aimock = bootAimock(); + const app = bootBrowserApp(); + + test("the worker's reply lands in the thread", async () => { + const { page, errors } = await app().newPage(); + try { + await page.goto(app().origin, { waitUntil: "networkidle0" }); + await clickText(page, "button", "Create an account"); + await waitForText(page, "Create your account"); + await page.type("input[type=email]", `alice+${String(Date.now())}@example.com`); + await page.type("input[type=password]", "correct-horse-battery-staple"); + await page.click("button[type=submit]"); + + // The Anthropic option pins api.anthropic.com; only Custom takes a base URL. + await waitForText(page, "Connect a brain"); + await clickText(page, "label", "Custom"); + await page.waitForSelector("input[type=password]"); + const inputs = await page.$$("form input:not([type=radio])"); + expect(inputs.length).toBeGreaterThanOrEqual(3); + await page.type("form input:not([type=radio]):not([type=password])", `${aimock().url}/v1`); + await page.type("input[type=password]", "mock"); + const model = await page.$$("form input:not([type=radio]):not([type=password])"); + await model[1]?.type("mock-model"); + await clickText(page, "button", "Connect"); + + await clickText(page, "button", "Start your first workbench"); + await page.waitForSelector("textarea", { timeout: STEP_TIMEOUT }); + await page.type("textarea", "Say hello"); + await page.click("button[aria-label='Start this workbench']"); + await page.waitForFunction(`location.pathname.startsWith("/w/")`, { + timeout: STEP_TIMEOUT, + }); + + await waitForText(page, MOCK_REPLY); + await page.waitForFunction(`!document.body.innerText.toLowerCase().includes("working")`, { + timeout: STEP_TIMEOUT, + }); + expect(aimock().journal().length).toBeGreaterThan(0); + } catch (cause) { + const body = await page.evaluate("document.body.innerText"); + process.stderr.write( + `reply failed at ${page.url()}:\n${String(body)}\nconsole: ${errors.join(" | ")}\njournal: ${JSON.stringify(aimock().journal()).slice(0, 4000)}\n`, + ); + throw cause; + } + await page.close(); + }, 240_000); +}); diff --git a/e2e/lib/aimock.ts b/e2e/lib/aimock.ts new file mode 100644 index 000000000..0de0c7b0c --- /dev/null +++ b/e2e/lib/aimock.ts @@ -0,0 +1,34 @@ +// The e2e stand-in for the inference provider: one aimock server that answers +// every model request with a fixed reply, so a bench's worker can respond with +// no real provider key. Same lifecycle shape as `bootHub`. +import { afterAll, beforeAll } from "bun:test"; +import { LLMock } from "@copilotkit/aimock"; + +export const MOCK_REPLY = "Hi, I'm Myra. I'll go by Ada here."; + +export type BootedAimock = { + /** Origin the mock listens on, e.g. http://127.0.0.1:41234. */ + url: string; + /** Every request the mock has served, oldest first. */ + journal: () => { method: string; path: string; body: unknown }[]; +}; + +/** Call synchronously inside the describe body so teardown registers during collection. */ +export function bootAimock(): () => BootedAimock { + const mock = new LLMock({ port: 0, host: "127.0.0.1" }); + mock.addFixture({ match: { predicate: () => true }, response: { content: MOCK_REPLY } }); + + beforeAll(async () => { + await mock.start(); + }); + + afterAll(async () => { + await mock.stop(); + }); + + return () => ({ + url: mock.url, + journal: () => + mock.getRequests().map((r) => ({ method: r.method, path: r.path, body: r.body as unknown })), + }); +} diff --git a/package.json b/package.json index 99a95dac0..b8aa1ff6c 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "reset": "bun scripts/reset.ts" }, "devDependencies": { + "@copilotkit/aimock": "^1.43.0", "@corbits/artifacts": "github:corbitsdev/corbits-artifacts#4a2a3c22187991a6ce598faf794908fedd252e98", "@corbits/codex-provider": "github:corbitsdev/corbits-codex-provider#a1f475f341261c05fa8dade0c82f1246c175b5b3", "@corbits/ollama-adapter": "github:corbitsdev/corbits-ollama-adapter#0387eb17bee5ac87530e9c48bb3bffd9a6998ded",