diff --git a/AGENTS.md b/AGENTS.md index 50e9cd1..ba7ba73 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -60,7 +60,7 @@ AXIS (Agent Experience Index Score) is a synthetic testing framework for AI agen ### Adapter Pattern -Built-in adapters split into two factories. NDJSON-style adapters (`claude-code`, `codex`) are created via `createAgentAdapter(spec)` from `src/adapters/base/agent-adapter.ts`. ACP-based adapters (`claude-sdk`, `codex-sdk`, `gemini`, `goose`, `opencode`, `qwen-code`, `stakpak`, `blackbox`, `fast-agent`, `mistral-vibe`, `factory-droid`, `poolside`, `vtcode`, `cursor-agent`, `auggie`, `kimi`, `openhands`, `cline`, `kiro-cli`, `kilo`, `qoder`) are created via `createAcpBasedAdapter(spec)` from `src/adapters/base/acp-adapter.ts`. Each adapter is a plain factory function (e.g. `createGeminiAdapter()`) that returns an `AgentAdapter` -no classes, no inheritance. The factory owns the shared plumbing: +Built-in adapters split into two factories. NDJSON-style adapters (`claude-code`, `codex`, `agy`) are created via `createAgentAdapter(spec)` from `src/adapters/base/agent-adapter.ts`. ACP-based adapters (`claude-sdk`, `codex-sdk`, `gemini`, `goose`, `opencode`, `qwen-code`, `stakpak`, `blackbox`, `fast-agent`, `mistral-vibe`, `factory-droid`, `poolside`, `vtcode`, `cursor-agent`, `auggie`, `kimi`, `openhands`, `cline`, `kiro-cli`, `kilo`, `qoder`) are created via `createAcpBasedAdapter(spec)` from `src/adapters/base/acp-adapter.ts`. Each adapter is a plain factory function (e.g. `createGeminiAdapter()`) that returns an `AgentAdapter` -no classes, no inheritance. The factory owns the shared plumbing: - Spawn + cleanup registration (SIGTERM on Ctrl-C); stdin is closed immediately by default, or written with the prompt then closed when `promptVia: "stdin"` - 10-minute timeout → SIGTERM → SIGKILL after 5s grace (timer cleared on clean exit) @@ -128,6 +128,17 @@ Meta's Muse Code has **no native ACP mode** (checked against 1.4.2), so it is no - **No npm package**, so the `npx` fallback cannot install it: `muse` must already be on `PATH` (`curl https://dev.meta.ai/install.sh | bash`) - **Regenerating the wire contract**: `muse schema generate-json-schema --out DIR` / `generate-ts --out DIR` export the exact schema embedded in the installed binary. Use that rather than guessing when MSP changes +#### Antigravity CLI (`agy`) + +NDJSON adapter on `createAgentAdapter`, driving `agy --output-format stream-json` with the prompt on stdin. Events are `init` (conversation id, model), `step_update` and `result`. + +- **Outcome is `result.status`, not the exit code**: agy exits 0 on most failures, including its own print timeout. With no `result` event, the `AGY_ERROR:` JSON line on stderr carries the reason +- **Usage is summed from `step_update`, keyed by `step_index`**: `result.usage` is cumulative over the conversation. `input_tokens` excludes cache reads (a step can report 2,331 input against 17,177 cache reads) and `output_tokens` includes thinking, so they map straight onto AXIS's fields +- **Cost is AXIS's own arithmetic**: `PRICES` in `agy.ts` holds Gemini API rates per model, priced per step so 3.1 Pro's >200k-prompt tier (input + cache reads) applies per call. The model comes from `init.model` with any `-low|-medium|-high` suffix stripped. Unpriced models report no cost, never `$0` +- **Auth and isolation**: `GEMINI_API_KEY` only works with `{"modelProvider":"gemini"}` in `~/.gemini/antigravity-cli/settings.json`. `AGY_CLI_DISABLE_AUTO_UPDATE` must be exactly `true`. All state lives under `HOME/.gemini` +- **Tool steps**: `tool_id` is the `step_index`, so pairing survives interleaved steps. A step ending in `ERROR` becomes an `error` entry (as in the ACP adapter), because scoring counts errors by entry type only +- **No npm package**: `agy` must be on `PATH` + ### Error Handling - `AgentMetadata.error` is the canonical error field for failed runs diff --git a/src/adapters/agy.ts b/src/adapters/agy.ts new file mode 100644 index 0000000..6e8f00e --- /dev/null +++ b/src/adapters/agy.ts @@ -0,0 +1,346 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; +import type { AgentAdapter, TokenUsage } from "../types/agent.js"; +import type { McpServerConfig } from "../types/config.js"; +import { createAgentAdapter, type StreamContext } from "./base/agent-adapter.js"; +import { writeAgySkills } from "./utils/skills.js"; + +interface AgyUsage { + input_tokens?: number; + output_tokens?: number; + cache_read_tokens?: number; +} + +interface AgyStepUpdate { + step_index: number; + state: string; + step_type: string; + text_delta?: string; + tool_name?: string; + tool_info?: { + parameters?: Record; + output?: string; + error?: { message?: string }; + }; + usage?: AgyUsage; +} + +interface AgyResult { + status?: string; + response?: string; + error?: string; + duration_seconds?: number; +} + +interface AgyState { + startTime: Date; + conversationId?: string; + model?: string; + textByStep: Map; + toolSteps: Map; + usageByStep: Map; + lastAgentMessage: string | null; + result?: AgyResult; +} + +/** USD per million tokens. */ +interface Rates { + input: number; + output: number; + cacheRead: number; +} + +const FLASH_PRICE_RISE = new Date("2027-01-01T00:00:00Z"); +const flashRates = (_promptTokens: number, at: Date): Rates => + at < FLASH_PRICE_RISE + ? { input: 0.75, output: 3.75, cacheRead: 0.075 } + : { input: 1.5, output: 7.5, cacheRead: 0.15 }; + +// https://ai.google.dev/gemini-api/docs/pricing as of 2026-10-16; unlisted models (3.7 Flash) report no cost, not a guess. +const PRICES: Record Rates> = { + "gemini-3.8-flash": flashRates, + "gemini-3.6-flash": flashRates, + "gemini-3.1-pro": (promptTokens) => + promptTokens > 200_000 ? { input: 4, output: 18, cacheRead: 0.4 } : { input: 2, output: 12, cacheRead: 0.2 }, +}; + +const TOOL_KINDS: Record = { + view_file: "read", + list_dir: "read", + find_by_name: "search", + grep_search: "search", + write_to_file: "edit", + replace_file_content: "edit", + multi_replace_file_content: "edit", + sed_file: "edit", + notebook_edit: "edit", + run_command: "execute", + command_status: "execute", + send_command_input: "execute", + notebook_execution: "execute", + read_url_content: "fetch", + search_web: "fetch", + manage_task: "think", + define_subagent: "think", + invoke_subagent: "think", + manage_subagents: "think", + ask_question: "think", + ask_permission: "think", + ask_custom_permission: "think", + list_permissions: "think", + finish: "think", + wait: "think", + wait_5_seconds: "think", + schedule: "think", + send_message: "think", + manage_inbox: "think", + delete_knowledge: "think", + run_workflow: "think", +}; + +export function baseModel(model: string): string { + return model.replace(/-(low|medium|high)$/, ""); +} + +export function agyCostUsd(model: string, usages: AgyUsage[], at: Date): number | undefined { + const rates = PRICES[baseModel(model)]; + if (!rates) return undefined; + let usd = 0; + for (const u of usages) { + const input = u.input_tokens ?? 0; + const cached = u.cache_read_tokens ?? 0; + const r = rates(input + cached, at); + usd += (input * r.input + cached * r.cacheRead + (u.output_tokens ?? 0) * r.output) / 1_000_000; + } + return usd; +} + +function writeAgyMcpConfig(configDir: string, servers: Record): void { + const mcpServers: Record = {}; + for (const [name, server] of Object.entries(servers)) { + if (server.type === "stdio") { + mcpServers[name] = { command: server.command, args: server.args ?? [], env: server.env ?? {} }; + } else { + mcpServers[name] = { url: server.url, ...(server.headers ? { headers: server.headers } : {}) }; + } + } + fs.writeFileSync(path.join(configDir, "mcp_config.json"), JSON.stringify({ mcpServers }, null, 2) + "\n"); +} + +// agy exits 0 on most failures: the outcome is the `result` event's status, never the exit code. +export function createAgyAdapter(): AgentAdapter { + return createAgentAdapter({ + name: "agy", + cliCommand: "agy", + promptVia: "stdin", + + requiredEnv: () => ["GEMINI_API_KEY"], + + isolationEnv: () => ({ + // Must be exactly "true", or agy can self-update under running jobs. + AGY_CLI_DISABLE_AUTO_UPDATE: "true", + }), + + prepare: (ctx) => { + const geminiDir = path.join(ctx.homeDirectory, ".gemini"); + const agyHome = path.join(geminiDir, "antigravity-cli"); + const configDir = path.join(geminiDir, "config"); + fs.mkdirSync(agyHome, { recursive: true }); + fs.mkdirSync(configDir, { recursive: true }); + // Without this, agy ignores GEMINI_API_KEY and tries an interactive Google sign-in. + fs.writeFileSync(path.join(agyHome, "settings.json"), JSON.stringify({ modelProvider: "gemini" }) + "\n"); + + if (ctx.input.mcpServers && Object.keys(ctx.input.mcpServers).length > 0) { + writeAgyMcpConfig(configDir, ctx.input.mcpServers); + } + if (ctx.input.resolvedSkills?.length) { + writeAgySkills(configDir, ctx.input.resolvedSkills); + } + }, + + buildArgs: (input) => { + // agy's 5m default would cut runs short of AXIS's own timeout. + const args = ["--output-format", "stream-json", "--print-timeout", "24h"]; + const flags = input.config.flags ?? {}; + if (flags["dangerously-skip-permissions"] ?? true) args.push("--dangerously-skip-permissions"); + if (input.config.model) args.push("--model", input.config.model); + + for (const [key, value] of Object.entries(flags)) { + if (key === "dangerously-skip-permissions") continue; + if (value === true) { + args.push(`--${key}`); + } else if (value !== false) { + args.push(`--${key}`, String(value)); + } + } + return args; + }, + + initialState: () => ({ + startTime: new Date(), + textByStep: new Map(), + toolSteps: new Map(), + usageByStep: new Map(), + lastAgentMessage: null, + }), + + streamConfig: { + mode: "lines", + onLine: (line, ctx) => { + let event: Record; + try { + event = JSON.parse(line) as Record; + } catch { + return; + } + processEvent(event, ctx); + }, + }, + + getResult: ({ state, stderr }) => { + const usages = [...state.usageByStep.values()]; + const tokenUsage = usages.length > 0 ? sumUsage(usages) : undefined; + const totalCostUsd = + state.model && usages.length > 0 ? agyCostUsd(state.model, usages, state.startTime) : undefined; + const metadata = { + tokenUsage, + totalCostUsd, + sessionId: state.conversationId, + ...(state.result?.duration_seconds ? { durationMs: Math.round(state.result.duration_seconds * 1000) } : {}), + }; + + const { result } = state; + if (result?.status === "SUCCESS") { + return { result: result.response?.trim() || state.lastAgentMessage, metadata }; + } + const error = result + ? result.error || `agy finished with status ${result.status ?? "unknown"}` + : (agyErrorFromStderr(stderr) ?? (stderr.trim() || "agy exited without a result")); + return { result: null, metadata: { ...metadata, error } }; + }, + }); +} + +function sumUsage(usages: AgyUsage[]): TokenUsage { + let input = 0; + let output = 0; + let cacheReadInput = 0; + for (const u of usages) { + input += u.input_tokens ?? 0; + output += u.output_tokens ?? 0; + cacheReadInput += u.cache_read_tokens ?? 0; + } + return { input, output, cacheReadInput }; +} + +function agyErrorFromStderr(stderr: string): string | undefined { + const line = stderr.split("\n").find((l) => l.startsWith("AGY_ERROR:")); + if (!line) return undefined; + try { + const parsed = JSON.parse(line.slice(line.indexOf("{"))) as { short_error?: string }; + return parsed.short_error ?? line; + } catch { + return line; + } +} + +function processEvent(event: Record, ctx: StreamContext): void { + switch (event.event) { + case "init": { + ctx.state.conversationId = event.conversation_id as string | undefined; + ctx.state.model = (event.init as { model?: string } | undefined)?.model; + return; + } + case "step_update": { + if (event.step_update) processStep(event.step_update as AgyStepUpdate, ctx); + return; + } + case "result": { + const result = event.result as AgyResult | undefined; + if (!result) return; + ctx.state.result = result; + if (result.status !== "SUCCESS") { + ctx.transcript.push({ type: "error", timestamp: new Date().toISOString(), content: { ...result } }); + } + return; + } + } +} + +function processStep(update: AgyStepUpdate, ctx: StreamContext): void { + const { state } = ctx; + const timestamp = new Date().toISOString(); + // Per call: result.usage is cumulative, and input_tokens already excludes cache reads. + if (update.usage) state.usageByStep.set(update.step_index, update.usage); + + if (update.step_type === "agent_response") { + if (update.text_delta) ctx.feedAssistantText(update.text_delta); + const text = (state.textByStep.get(update.step_index) ?? "") + (update.text_delta ?? ""); + if (update.state !== "DONE") { + state.textByStep.set(update.step_index, text); + return; + } + state.textByStep.delete(update.step_index); + if (text.trim()) { + state.lastAgentMessage = text.trim(); + ctx.transcript.push({ type: "assistant", timestamp, content: { text, step_index: update.step_index } }); + } + return; + } + + if (update.step_type !== "tool") return; + + const phase = state.toolSteps.get(update.step_index); + if (phase === "closed") return; + const content = toolContent(update); + if (!phase) { + state.toolSteps.set(update.step_index, "open"); + ctx.transcript.push({ type: "tool_use", timestamp, content }); + } + if (update.state !== "DONE" && update.state !== "ERROR") return; + + state.toolSteps.set(update.step_index, "closed"); + const output = update.tool_info?.output; + if (output) ctx.feedAssistantText(output); + // Scoring counts errors by entry type, so a failed tool is an `error`, not a `tool_result`. + if (update.state === "ERROR") { + const message = update.tool_info?.error?.message; + ctx.transcript.push({ + type: "error", + timestamp, + content: { + ...content, + error: `Tool failed: ${String(content.tool_name)}${message ? `: ${message}` : ""}`, + output, + }, + }); + return; + } + ctx.transcript.push({ type: "tool_result", timestamp, content: { ...content, output } }); +} + +function toolContent(update: AgyStepUpdate): Record { + const toolName = update.tool_name ?? "tool"; + const parameters = update.tool_info?.parameters ?? {}; + if (toolName === "call_mcp_tool") { + const { ServerName, ToolName, Arguments } = parameters; + return { + tool_name: `mcp__${String(ServerName)}__${String(ToolName)}`, + parameters: parseArguments(Arguments), + tool_id: String(update.step_index), + }; + } + const kind = TOOL_KINDS[toolName]; + return { tool_name: toolName, parameters, ...(kind ? { kind } : {}), tool_id: String(update.step_index) }; +} + +function parseArguments(args: unknown): Record { + if (typeof args === "string") { + try { + return JSON.parse(args) as Record; + } catch { + return { arguments: args }; + } + } + return (args as Record | undefined) ?? {}; +} diff --git a/src/adapters/registry.ts b/src/adapters/registry.ts index cecd520..85a8bf9 100644 --- a/src/adapters/registry.ts +++ b/src/adapters/registry.ts @@ -24,6 +24,7 @@ import { createKiloAdapter } from "./kilo.js"; import { createQoderAdapter } from "./qoder.js"; import { createCopilotAdapter } from "./copilot.js"; import { createMuseAdapter } from "./muse.js"; +import { createAgyAdapter } from "./agy.js"; const BUILTIN_FACTORIES: Record AgentAdapter> = { "claude-sdk": createClaudeSdkAdapter, @@ -51,6 +52,7 @@ const BUILTIN_FACTORIES: Record AgentAdapter> = { qoder: createQoderAdapter, copilot: createCopilotAdapter, muse: createMuseAdapter, + agy: createAgyAdapter, }; const instanceCache = new Map(); diff --git a/src/adapters/utils/skills.ts b/src/adapters/utils/skills.ts index 7f791d2..a778cc5 100644 --- a/src/adapters/utils/skills.ts +++ b/src/adapters/utils/skills.ts @@ -73,3 +73,11 @@ export function writeMuseSkills(museConfigDir: string, skills: ResolvedSkill[]): copyDirRecursive(skill.path, target); } } + +/** Write skills for the Antigravity CLI to `~/.gemini/config/skills/{name}/`; agy ignores `~/.agents/skills`. */ +export function writeAgySkills(configDir: string, skills: ResolvedSkill[]): void { + for (const skill of skills) { + const target = path.join(configDir, "skills", skill.name); + copyDirRecursive(skill.path, target); + } +} diff --git a/src/docs-site/src/pages/agents.astro b/src/docs-site/src/pages/agents.astro index b5bf6c6..576929b 100644 --- a/src/docs-site/src/pages/agents.astro +++ b/src/docs-site/src/pages/agents.astro @@ -76,6 +76,11 @@ import DocsLayout from "../layouts/DocsLayout.astro"; yes GEMINI_API_KEY + + agy + no + GEMINI_API_KEY (see below) + kiro-cli no @@ -293,6 +298,54 @@ import DocsLayout from "../layouts/DocsLayout.astro"; +

Antigravity CLI

+

+ agy runs Google's Antigravity CLI headlessly with + --output-format stream-json, the prompt on stdin, and + --dangerously-skip-permissions (opt out with + {'{ "flags": { "dangerously-skip-permissions": false } }'}). +

+

+ There is no npm package, so AXIS cannot auto-install it: install a release from + GitHub + and put agy on PATH. +

+
    +
  • + Set GEMINI_API_KEY. AXIS writes + {'{ "modelProvider": "gemini" }'} to the isolated + ~/.gemini/antigravity-cli/settings.json; without it agy ignores the key and tries an + interactive Google sign-in. +
  • +
  • + Models and effort. agy only accepts models from its own agy models + list. Pick an effort with {'{ "flags": { "effort": "high" } }'}, or put it in the + model name (gemini-3.8-flash-high) so effort variants get their own + agent|model identity. Gemini 3.1 Pro has no medium effort. +
  • +
  • + Auto-update is pinned off with AGY_CLI_DISABLE_AUTO_UPDATE=true + (agy ignores any other value), so every job in a run grades the same build. +
  • +
  • + The outcome comes from agy's final result event, not the exit + code: agy exits 0 on most failures. +
  • +
  • + Tokens are summed per model call. agy's result.usage is cumulative + over the whole conversation, so AXIS adds up the usage on each step instead. +
  • +
  • + Cost is computed by AXIS. agy reports none, so AXIS prices each model call from + Google's published Gemini API rates. A model without a published rate reports no cost rather + than $0. +
  • +
  • + MCP servers and skills go to ~/.gemini/config/mcp_config.json and + ~/.gemini/config/skills/ under the isolated HOME. +
  • +
+

Custom Agents

Not listed here? You can register any agent via the diff --git a/test/e2e/adapters/agy/axis.config.json b/test/e2e/adapters/agy/axis.config.json new file mode 100644 index 0000000..c75436f --- /dev/null +++ b/test/e2e/adapters/agy/axis.config.json @@ -0,0 +1,4 @@ +{ + "scenarios": "./scenarios", + "agents": ["agy"] +} diff --git a/test/e2e/adapters/agy/scenarios/echo-test.json b/test/e2e/adapters/agy/scenarios/echo-test.json new file mode 100644 index 0000000..592b1f9 --- /dev/null +++ b/test/e2e/adapters/agy/scenarios/echo-test.json @@ -0,0 +1,5 @@ +{ + "name": "agy echo test", + "prompt": "Hello from AXIS agy adapter", + "judge": "The agent should echo back the prompt text." +} diff --git a/test/unit/adapters/agy-e2e.test.ts b/test/unit/adapters/agy-e2e.test.ts new file mode 100644 index 0000000..fa697ae --- /dev/null +++ b/test/unit/adapters/agy-e2e.test.ts @@ -0,0 +1,166 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import { EventEmitter, Readable } from "node:stream"; +import { silentLogger } from "../../../src/types/output.js"; + +const E2E_DIR = path.resolve(import.meta.dirname, "../../e2e/adapters/agy"); + +vi.mock("node:child_process", () => ({ + spawn: vi.fn(), +})); +vi.mock("../../../src/runner/lifecycle.js", () => ({ + executeLifecycleActions: vi.fn().mockResolvedValue([]), +})); +vi.mock("../../../src/adapters/utils/resolve.js", () => ({ + resolveCommand: vi.fn().mockResolvedValue({ command: "agy", prefixArgs: [] }), +})); + +import { spawn } from "node:child_process"; +import { run } from "../../../src/runner/runner.js"; + +const mockSpawn = vi.mocked(spawn); +const CONVERSATION_ID = "36ee5ed1-f410-43ec-961b-fcde2f0c6dcb"; + +const ev = (event: Record) => JSON.stringify(event); +const step = (s: Record) => + ev({ event: "step_update", step_update: { conversation_id: CONVERSATION_ID, ...s } }); + +const SUCCESS = [ + ev({ event: "init", conversation_id: CONVERSATION_ID, init: { model: "gemini-3.8-flash", tools: [] } }), + step({ step_index: 0, state: "DONE", step_type: "user_input" }), + step({ + step_index: 1, + state: "DONE", + step_type: "agent_response", + usage: { input_tokens: 12227, output_tokens: 112, thinking_tokens: 0, cache_read_tokens: 0, total_tokens: 12339 }, + }), + step({ + step_index: 2, + state: "ACTIVE", + step_type: "tool", + tool_name: "view_file", + tool_info: { parameters: { AbsolutePath: "/tmp/README.md" } }, + }), + step({ + step_index: 2, + state: "DONE", + step_type: "tool", + tool_name: "view_file", + tool_info: { parameters: { AbsolutePath: "/tmp/README.md" }, output: "3 lines, 40 bytes" }, + }), + step({ + step_index: 3, + state: "DONE", + step_type: "agent_response", + text_delta: "Hello from AXIS agy adapter\n", + usage: { input_tokens: 12493, output_tokens: 6, thinking_tokens: 0, cache_read_tokens: 3633, total_tokens: 12499 }, + }), + ev({ + event: "result", + result: { + conversation_id: CONVERSATION_ID, + status: "SUCCESS", + response: "Hello from AXIS agy adapter\n", + duration_seconds: 7.628081, + }, + }), +]; + +interface SpawnCapture { + args: string[]; + opts: Record; + stdin: string; + settings: unknown; +} + +let lastSpawn: SpawnCapture; + +function mockAgy(lines: string[]) { + mockSpawn.mockImplementation(((_cmd: string, args: string[], opts: Record) => { + const stdout = new Readable({ read() {} }); + const stderr = new Readable({ read() {} }); + const env = opts.env as Record; + lastSpawn = { + args, + opts, + stdin: "", + settings: JSON.parse( + fs.readFileSync(path.join(env.HOME, ".gemini", "antigravity-cli", "settings.json"), "utf-8"), + ), + }; + const stdin = { + on: vi.fn(), + end: vi.fn((data?: string) => { + lastSpawn.stdin = data ?? ""; + }), + }; + const proc = Object.assign(new EventEmitter(), { stdout, stderr, stdin, kill: vi.fn(), pid: 1 }); + setTimeout(() => { + for (const line of lines) stdout.push(line + "\n"); + stdout.push(null); + stderr.push(null); + setTimeout(() => proc.emit("close", 0), 5); + }, 10); + return proc; + }) as never); +} + +describe("agy adapter e2e", () => { + const origKey = process.env.GEMINI_API_KEY; + + beforeEach(() => { + vi.clearAllMocks(); + process.env.GEMINI_API_KEY = "test-key"; + mockAgy(SUCCESS); + }); + + afterEach(() => { + if (origKey !== undefined) process.env.GEMINI_API_KEY = origKey; + else delete process.env.GEMINI_API_KEY; + }); + + it("runs a scenario end to end", async () => { + const output = await run({ configPath: path.join(E2E_DIR, "axis.config.json"), logger: silentLogger }); + + expect(output.summary).toMatchObject({ total: 1, completed: 1, failed: 0 }); + const result = output.results[0]; + expect(result.agentConfig.agent).toBe("agy"); + expect(result.output.result).toBe("Hello from AXIS agy adapter"); + expect(result.output.metadata.sessionId).toBe(CONVERSATION_ID); + expect(result.output.metadata.durationMs).toBe(7628); + expect(result.output.transcript.map((e) => e.type)).toEqual(["tool_use", "tool_result", "assistant"]); + }); + + it("reports tokens and a cost computed from them", async () => { + const output = await run({ configPath: path.join(E2E_DIR, "axis.config.json"), logger: silentLogger }); + + const meta = output.results[0].output.metadata; + expect(meta.tokenUsage).toEqual({ input: 24720, output: 118, cacheReadInput: 3633 }); + expect(meta.totalCostUsd).toBeCloseTo((24720 * 0.75 + 3633 * 0.075 + 118 * 3.75) / 1_000_000); + }); + + it("isolates agy's state under HOME, signs in with the API key and pins auto-update off", async () => { + await run({ configPath: path.join(E2E_DIR, "axis.config.json"), logger: silentLogger }); + + const env = lastSpawn.opts.env as Record; + expect(env.AGY_CLI_DISABLE_AUTO_UPDATE).toBe("true"); + expect(env.GEMINI_API_KEY).toBe("test-key"); + expect(env.HOME.startsWith(lastSpawn.opts.cwd as string)).toBe(false); + expect(lastSpawn.settings).toEqual({ modelProvider: "gemini" }); + expect(lastSpawn.stdin).toBe("Hello from AXIS agy adapter"); + expect(lastSpawn.args).not.toContain("Hello from AXIS agy adapter"); + }); + + it("marks an ERROR result as a failed run despite exit code 0", async () => { + mockAgy([ + ev({ event: "init", conversation_id: CONVERSATION_ID, init: { model: "gemini-3.8-flash" } }), + ev({ event: "result", result: { status: "ERROR", response: "", error: "interrupted" } }), + ]); + + const output = await run({ configPath: path.join(E2E_DIR, "axis.config.json"), logger: silentLogger }); + + expect(output.summary.failed).toBe(1); + expect(output.results[0].output.metadata.error).toBe("interrupted"); + }); +}); diff --git a/test/unit/adapters/agy.test.ts b/test/unit/adapters/agy.test.ts new file mode 100644 index 0000000..f9d0eda --- /dev/null +++ b/test/unit/adapters/agy.test.ts @@ -0,0 +1,373 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import type * as ChildProcess from "node:child_process"; +import { EventEmitter, Readable } from "node:stream"; +import { createAgyAdapter, agyCostUsd, baseModel } from "../../../src/adapters/agy.js"; +import type { AgentAdapter, AgentInput } from "../../../src/types/agent.js"; +import { normalizeTranscript } from "../../../src/transcript/normalize.js"; + +vi.mock("node:child_process", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, spawn: vi.fn() }; +}); + +vi.mock("../../../src/adapters/utils/resolve.js", () => ({ + resolveCommand: vi.fn().mockResolvedValue({ command: "agy", prefixArgs: [] }), +})); + +import { spawn } from "node:child_process"; +const mockSpawn = vi.mocked(spawn); + +const CONVERSATION_ID = "e32c0a56-6889-4d09-855f-21dcbedce5fa"; + +function createMockProcess(lines: string[], { exitCode = 0, stderr = "" } = {}) { + const stdout = new Readable({ read() {} }); + const stderrStream = new Readable({ read() {} }); + const stdin = { end: vi.fn(), on: vi.fn() }; + const proc = Object.assign(new EventEmitter(), { stdout, stderr: stderrStream, stdin, kill: vi.fn() }); + + setTimeout(() => { + if (stderr) stderrStream.push(stderr); + stderrStream.push(null); + for (const line of lines) stdout.push(line + "\n"); + stdout.push(null); + setTimeout(() => proc.emit("close", exitCode), 5); + }, 10); + + return proc; +} + +const init = (model = "gemini-3.8-flash") => + JSON.stringify({ event: "init", conversation_id: CONVERSATION_ID, init: { model, cwd: "/tmp/work", tools: [] } }); + +const step = (s: Record) => + JSON.stringify({ event: "step_update", step_update: { conversation_id: CONVERSATION_ID, ...s } }); + +const result = (r: Record) => + JSON.stringify({ event: "result", result: { conversation_id: CONVERSATION_ID, ...r } }); + +const usage = (input: number, output: number, cache = 0) => ({ + input_tokens: input, + output_tokens: output, + thinking_tokens: 0, + cache_read_tokens: cache, + total_tokens: input + output, +}); + +let tmpRoot: string; + +function makeInput(overrides: Partial = {}): AgentInput { + const home = path.join(tmpRoot, "home"); + fs.mkdirSync(home, { recursive: true }); + return { + prompt: "test prompt", + config: { agent: "agy" }, + scenario: { key: "test", name: "Test", prompt: "test prompt", judge: [{ check: "test", weight: 1.0 }] }, + workingDirectory: tmpRoot, + homeDirectory: home, + ...overrides, + }; +} + +function mockRun(lines: string[], opts?: { exitCode?: number; stderr?: string }) { + mockSpawn.mockImplementation((() => createMockProcess(lines, opts)) as never); +} + +describe("agy adapter", () => { + let adapter: AgentAdapter; + + beforeEach(() => { + adapter = createAgyAdapter(); + vi.clearAllMocks(); + tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), "agy-test-")); + mockRun([]); + }); + + afterEach(() => { + fs.rmSync(tmpRoot, { recursive: true, force: true }); + }); + + it("requires GEMINI_API_KEY and disables auto-update with the exact value agy accepts", () => { + expect(adapter.requiredEnv!()).toEqual(["GEMINI_API_KEY"]); + expect(adapter.isolationEnv!({ workspace: "/w", home: "/h" })).toEqual({ AGY_CLI_DISABLE_AUTO_UPDATE: "true" }); + }); + + it("runs headless with the prompt on stdin", async () => { + await adapter.run(makeInput({ config: { agent: "agy", model: "gemini-3.1-pro", flags: { effort: "high" } } })); + + const [, args] = mockSpawn.mock.calls[0] as unknown as [string, string[]]; + expect(args).toEqual([ + "--output-format", + "stream-json", + "--print-timeout", + "24h", + "--dangerously-skip-permissions", + "--model", + "gemini-3.1-pro", + "--effort", + "high", + ]); + expect(args).not.toContain("test prompt"); + const proc = mockSpawn.mock.results[0].value as { stdin: { end: ReturnType } }; + expect(proc.stdin.end).toHaveBeenCalledWith("test prompt"); + }); + + it("drops --dangerously-skip-permissions when the flag is false", async () => { + await adapter.run(makeInput({ config: { agent: "agy", flags: { "dangerously-skip-permissions": false } } })); + + const [, args] = mockSpawn.mock.calls[0] as unknown as [string, string[]]; + expect(args).not.toContain("--dangerously-skip-permissions"); + }); + + it("writes the gemini modelProvider setting under the isolated HOME", async () => { + const input = makeInput(); + await adapter.run(input); + + const settings = JSON.parse( + fs.readFileSync(path.join(input.homeDirectory, ".gemini", "antigravity-cli", "settings.json"), "utf-8"), + ); + expect(settings).toEqual({ modelProvider: "gemini" }); + }); + + it("writes MCP servers to ~/.gemini/config/mcp_config.json", async () => { + const input = makeInput({ + mcpServers: { + local: { type: "stdio", command: "node", args: ["server.js"] }, + remote: { type: "http", url: "https://example.com/mcp", headers: { Authorization: "Bearer x" } }, + }, + }); + await adapter.run(input); + + const config = JSON.parse( + fs.readFileSync(path.join(input.homeDirectory, ".gemini", "config", "mcp_config.json"), "utf-8"), + ); + expect(config.mcpServers).toEqual({ + local: { command: "node", args: ["server.js"], env: {} }, + remote: { url: "https://example.com/mcp", headers: { Authorization: "Bearer x" } }, + }); + }); + + it("installs skills under ~/.gemini/config/skills", async () => { + const skillSrc = path.join(tmpRoot, "skill-src"); + fs.mkdirSync(skillSrc); + fs.writeFileSync(path.join(skillSrc, "SKILL.md"), "# greeter\n"); + const input = makeInput({ resolvedSkills: [{ name: "greeter", path: skillSrc }] }); + await adapter.run(input); + + expect(fs.existsSync(path.join(input.homeDirectory, ".gemini", "config", "skills", "greeter", "SKILL.md"))).toBe( + true, + ); + }); + + it("assembles streamed text, tool pairs, usage and the result", async () => { + mockRun([ + init(), + step({ step_index: 0, state: "DONE", step_type: "user_input" }), + step({ step_index: 1, state: "DONE", step_type: "agent_response", usage: usage(12149, 99) }), + step({ + step_index: 2, + state: "ACTIVE", + step_type: "tool", + tool_name: "run_command", + tool_info: { name: "run_command", parameters: { CommandLine: "ls" } }, + }), + step({ + step_index: 2, + state: "DONE", + step_type: "tool", + tool_name: "run_command", + tool_info: { name: "run_command", parameters: { CommandLine: "ls" }, output: "README.md\n" }, + }), + step({ step_index: 3, state: "ACTIVE", step_type: "agent_response", text_delta: "Hello " }), + step({ step_index: 3, state: "DONE", step_type: "agent_response", text_delta: "world", usage: usage(12500, 40) }), + result({ status: "SUCCESS", response: "Hello world\n", duration_seconds: 7.5, usage: usage(24649, 139) }), + ]); + + const output = await adapter.run(makeInput()); + + expect(output.result).toBe("Hello world"); + expect(output.metadata.error).toBeUndefined(); + expect(output.metadata.sessionId).toBe(CONVERSATION_ID); + expect(output.metadata.durationMs).toBe(7500); + expect(output.metadata.tokenUsage).toEqual({ input: 24649, output: 139, cacheReadInput: 0 }); + expect(output.transcript.map((e) => e.type)).toEqual(["tool_use", "tool_result", "assistant"]); + expect(output.transcript[0].content).toMatchObject({ + tool_name: "run_command", + kind: "execute", + parameters: { CommandLine: "ls" }, + }); + expect(output.transcript[1].content).toMatchObject({ tool_name: "run_command", output: "README.md\n" }); + expect(output.transcript[2].content.text).toBe("Hello world"); + }); + + it("sums usage per step rather than trusting the cumulative result.usage", async () => { + mockRun([ + init(), + step({ step_index: 1, state: "DONE", step_type: "agent_response", usage: usage(19235, 122) }), + step({ + step_index: 3, + state: "DONE", + step_type: "agent_response", + text_delta: "ok", + // Real cached step: cache reads exceed input_tokens. + usage: usage(2331, 1566, 17177), + }), + result({ status: "SUCCESS", response: "ok", usage: usage(99999, 999, 0) }), + ]); + + const output = await adapter.run(makeInput()); + + expect(output.metadata.tokenUsage).toEqual({ input: 21566, output: 1688, cacheReadInput: 17177 }); + }); + + it("prices the run from per-step usage and the model init reports", async () => { + mockRun([ + init("gemini-3.8-flash-low"), + step({ + step_index: 1, + state: "DONE", + step_type: "agent_response", + text_delta: "ok", + usage: usage(600_000, 100_000, 400_000), + }), + result({ status: "SUCCESS", response: "ok" }), + ]); + + const output = await adapter.run(makeInput()); + + expect(output.metadata.totalCostUsd).toBeCloseTo(0.45 + 0.03 + 0.375); + }); + + it("fails the run on a non-SUCCESS result even though agy exits 0", async () => { + mockRun([ + init(), + step({ step_index: 1, state: "DONE", step_type: "error_message" }), + result({ status: "ERROR", response: "", error: "API error (attempt 4): request failed (code 500)" }), + ]); + + const output = await adapter.run(makeInput()); + + expect(output.metadata.exitCode).toBe(0); + expect(output.result).toBeNull(); + expect(output.metadata.error).toBe("API error (attempt 4): request failed (code 500)"); + expect(output.transcript.at(-1)?.type).toBe("error"); + }); + + it("reports agy's AGY_ERROR line when no result event arrives", async () => { + mockRun([init()], { + stderr: 'AGY_ERROR: {"short_error":"Invalid Accept header value","status":"INVALID_ARGUMENT","error_code":400}\n', + }); + + const output = await adapter.run(makeInput()); + + expect(output.metadata.error).toBe("Invalid Accept header value"); + }); + + it("records a tool step that ends in ERROR as an error scoring can count", async () => { + mockRun([ + init(), + step({ + step_index: 2, + state: "ERROR", + step_type: "tool", + tool_name: "write_to_file", + tool_info: { parameters: { TargetFile: "/tmp/work/a.txt" }, error: { message: "permission check failed" } }, + }), + result({ status: "SUCCESS", response: "done" }), + ]); + + const output = await adapter.run(makeInput()); + + expect(output.transcript.map((e) => e.type)).toEqual(["tool_use", "error"]); + expect(output.transcript[1].content).toMatchObject({ + tool_name: "write_to_file", + tool_id: "2", + error: "Tool failed: write_to_file: permission check failed", + }); + expect(normalizeTranscript(output.transcript).entries.filter((e) => e.isError)).toHaveLength(1); + }); + + it("pairs interleaved tool steps by step index", async () => { + const tool = (index: number, state: string, output?: string) => + step({ + step_index: index, + state, + step_type: "tool", + tool_name: "run_command", + tool_info: { parameters: { CommandLine: `cmd ${index}` }, ...(output ? { output } : {}) }, + }); + mockRun([ + init(), + tool(2, "ACTIVE"), + tool(3, "ACTIVE"), + tool(2, "DONE", "out 2"), + tool(3, "DONE", "out 3"), + // A repeated DONE must not open a second interaction. + tool(3, "DONE", "out 3"), + result({ status: "SUCCESS", response: "done" }), + ]); + + const output = await adapter.run(makeInput()); + + expect(output.transcript.map((e) => `${e.type}:${String(e.content.tool_id)}`)).toEqual([ + "tool_use:2", + "tool_use:3", + "tool_result:2", + "tool_result:3", + ]); + const entries = normalizeTranscript(output.transcript).entries; + expect(entries[0].pairedIndex).toBe(2); + expect(entries[1].pairedIndex).toBe(3); + }); + + it("names MCP calls mcp____ with their own arguments", async () => { + mockRun([ + init(), + step({ + step_index: 2, + state: "DONE", + step_type: "tool", + tool_name: "call_mcp_tool", + tool_info: { parameters: { ServerName: "echo", ToolName: "say", Arguments: '{"text":"hi"}' } }, + }), + step({ + step_index: 3, + state: "DONE", + step_type: "tool", + tool_name: "call_mcp_tool", + tool_info: { parameters: { ServerName: "echo", ToolName: "say", Arguments: { text: "yo" } } }, + }), + result({ status: "SUCCESS", response: "done" }), + ]); + + const output = await adapter.run(makeInput()); + + expect(output.transcript[0].content).toMatchObject({ tool_name: "mcp__echo__say", parameters: { text: "hi" } }); + expect(output.transcript[2].content).toMatchObject({ tool_name: "mcp__echo__say", parameters: { text: "yo" } }); + }); +}); + +describe("agy pricing", () => { + it("strips the effort suffix agy accepts on model names", () => { + expect(baseModel("gemini-3.8-flash-medium")).toBe("gemini-3.8-flash"); + expect(baseModel("gemini-3.1-pro")).toBe("gemini-3.1-pro"); + }); + + it("uses the >200k prompt tier for Gemini 3.1 Pro per model call", () => { + const at = new Date("2026-10-16"); + expect(agyCostUsd("gemini-3.1-pro-high", [usage(100_000, 0)], at)).toBeCloseTo(0.2); + expect(agyCostUsd("gemini-3.1-pro-high", [usage(300_000, 0)], at)).toBeCloseTo(1.2); + // The tier follows the whole prompt, cache reads included. + expect(agyCostUsd("gemini-3.1-pro-high", [usage(100_000, 0, 150_000)], at)).toBeCloseTo(0.4 + 0.06); + }); + + it("applies the 2027 Flash price rise", () => { + expect(agyCostUsd("gemini-3.6-flash", [usage(1_000_000, 0)], new Date("2027-01-02"))).toBeCloseTo(1.5); + }); + + it("reports no cost for an unpriced model rather than zero", () => { + expect(agyCostUsd("gemini-3.7-flash", [usage(1000, 10)], new Date())).toBeUndefined(); + }); +});