test(e2e): aimock stands in for the inference provider so a worker can reply (CL-9626)

e2e/browser/reply.test.ts

@@ -0,0 +1,59 @@
+// A fresh user connects an inference provider served by aimock, creates a
+// bench from /new, and the worker's reply lands in the thread while the
+// bench stops showing "working".
+import { expect, test } from "bun:test";
+import { bootAimock, MOCK_REPLY } from "../lib/aimock";
+import { bootBrowserApp, browserGate } from "../lib/browser";
+import { clickText, STEP_TIMEOUT, waitForText } from "../lib/first-run";
+
+const describeBrowser = browserGate(import.meta.path);
+
+describeBrowser("worker reply", () => {
+  const aimock = bootAimock();
+  const app = bootBrowserApp();
+
+  test("the worker's reply lands in the thread", async () => {
+    const { page, errors } = await app().newPage();
+    try {
+      await page.goto(app().origin, { waitUntil: "networkidle0" });
+      await clickText(page, "button", "Create an account");
+      await waitForText(page, "Create your account");
+      await page.type("input[type=email]", `alice+${String(Date.now())}@example.com`);
+      await page.type("input[type=password]", "correct-horse-battery-staple");
+      await page.click("button[type=submit]");
+
+      // The Anthropic option pins api.anthropic.com; only Custom takes a base URL.
+      await waitForText(page, "Connect a brain");
+      await clickText(page, "label", "Custom");
+      await page.waitForSelector("input[type=password]");
+      const inputs = await page.$$("form input:not([type=radio])");
+      expect(inputs.length).toBeGreaterThanOrEqual(3);
+      await page.type("form input:not([type=radio]):not([type=password])", `${aimock().url}/v1`);
+      await page.type("input[type=password]", "mock");
+      const model = await page.$$("form input:not([type=radio]):not([type=password])");
+      await model[1]?.type("mock-model");
+      await clickText(page, "button", "Connect");
+
+      await clickText(page, "button", "Start your first workbench");
+      await page.waitForSelector("textarea", { timeout: STEP_TIMEOUT });
+      await page.type("textarea", "Say hello");
+      await page.click("button[aria-label='Start this workbench']");
+      await page.waitForFunction(`location.pathname.startsWith("/w/")`, {
+        timeout: STEP_TIMEOUT,
+      });
+
+      await waitForText(page, MOCK_REPLY);
+      await page.waitForFunction(`!document.body.innerText.toLowerCase().includes("working")`, {
+        timeout: STEP_TIMEOUT,
+      });
+      expect(aimock().journal().length).toBeGreaterThan(0);
+    } catch (cause) {
+      const body = await page.evaluate("document.body.innerText");
+      process.stderr.write(
+        `reply failed at ${page.url()}:\n${String(body)}\nconsole: ${errors.join(" | ")}\njournal: ${JSON.stringify(aimock().journal()).slice(0, 4000)}\n`,
+      );
+      throw cause;
+    }
+    await page.close();
+  }, 240_000);
+});

e2e/lib/aimock.ts

@@ -0,0 +1,34 @@
+// The e2e stand-in for the inference provider: one aimock server that answers
+// every model request with a fixed reply, so a bench's worker can respond with
+// no real provider key. Same lifecycle shape as `bootHub`.
+import { afterAll, beforeAll } from "bun:test";
+import { LLMock } from "@copilotkit/aimock";
+
+export const MOCK_REPLY = "Hi, I'm Myra. I'll go by Ada here.";
+
+export type BootedAimock = {
+  /** Origin the mock listens on, e.g. http://127.0.0.1:41234. */
+  url: string;
+  /** Every request the mock has served, oldest first. */
+  journal: () => { method: string; path: string; body: unknown }[];
+};
+
+/** Call synchronously inside the describe body so teardown registers during collection. */
+export function bootAimock(): () => BootedAimock {
+  const mock = new LLMock({ port: 0, host: "127.0.0.1" });
+  mock.addFixture({ match: { predicate: () => true }, response: { content: MOCK_REPLY } });
+
+  beforeAll(async () => {
+    await mock.start();
+  });
+
+  afterAll(async () => {
+    await mock.stop();
+  });
+
+  return () => ({
+    url: mock.url,
+    journal: () =>
+      mock.getRequests().map((r) => ({ method: r.method, path: r.path, body: r.body as unknown })),
+  });
+}

package.json

@@ -25,6 +25,7 @@
     "reset": "bun scripts/reset.ts"
   },
   "devDependencies": {
+    "@copilotkit/aimock": "^1.43.0",
     "@corbits/artifacts": "github:corbitsdev/corbits-artifacts#4a2a3c22187991a6ce598faf794908fedd252e98",
     "@corbits/codex-provider": "github:corbitsdev/corbits-codex-provider#a1f475f341261c05fa8dade0c82f1246c175b5b3",
     "@corbits/ollama-adapter": "github:corbitsdev/corbits-ollama-adapter#0387eb17bee5ac87530e9c48bb3bffd9a6998ded",