feat: add mode-aware native mouse input

README.md

@@ -65,6 +65,7 @@ Driving an interactive TUI is the same loop, with key chords and a wait for the
 agent-tty run "$SID" 'nvim --clean' --no-wait --json
 agent-tty wait "$SID" --screen-stable-ms 1000 --json
 agent-tty send-keys "$SID" Down Down Enter --json
+agent-tty mouse "$SID" press --button left --row 8 --col 20 --json
 agent-tty screenshot "$SID" --json
 agent-tty record export "$SID" --format webm --json
 ```
@@ -115,13 +116,13 @@ A colleague then used `agent-tty` to build an experimental TUI for Coder agents
 
 Every user-facing command takes `--json` and returns a stable, machine-readable envelope, and exits with a stable code (`0` success, `2` usage error, `3` session not found, `11` wait timeout, …) so scripts can branch without parsing output.
 
-| Group                   | Commands                                                                 |
-| ----------------------- | ------------------------------------------------------------------------ |
-| Session lifecycle       | `create`, `list`, `inspect`, `destroy`, `gc`                             |
-| Input and control       | `run`, `type`, `paste`, `send-keys`, `batch`, `resize`, `signal`, `mark` |
-| Observation and capture | `wait`, `snapshot`, `screenshot`, `record export`                        |
-| Live view               | `dashboard`                                                              |
-| Environment             | `version`, `doctor`, `skills`                                            |
+| Group                   | Commands                                                                          |
+| ----------------------- | --------------------------------------------------------------------------------- |
+| Session lifecycle       | `create`, `list`, `inspect`, `destroy`, `gc`                                      |
+| Input and control       | `run`, `type`, `paste`, `send-keys`, `mouse`, `batch`, `resize`, `signal`, `mark` |
+| Observation and capture | `wait`, `snapshot`, `screenshot`, `record export`                                 |
+| Live view               | `dashboard`                                                                       |
+| Environment             | `version`, `doctor`, `skills`                                                     |
 
 The CLI documents itself: `agent-tty --help` lists every command, and `agent-tty <command> --help` shows its flags. The full reference, including the exit-code table, is in [`docs/USAGE.md`](./docs/USAGE.md); renderer and environment issues are in [`docs/TROUBLESHOOTING.md`](./docs/TROUBLESHOOTING.md).
 

docs/USAGE.md

@@ -50,6 +50,7 @@ agent-tty --home <path> run <session-id> 'command here' --json
 agent-tty --home <path> type <session-id> 'literal text' --json
 agent-tty --home <path> paste <session-id> 'multiline payload' --json
 agent-tty --home <path> send-keys <session-id> Enter Ctrl+C --json
+agent-tty --home <path> mouse <session-id> press --button left --row 8 --col 20 --json
 agent-tty --home <path> resize <session-id> --cols 100 --rows 30 --json
 agent-tty --home <path> signal <session-id> SIGTERM --json
 
@@ -83,6 +84,42 @@ Important flags:
 Use `type` when the target application needs literal interactive typing, `paste` when the target should receive a literal pasted payload, and `send-keys` for discrete control keys such as `Enter`, `Escape`, or `Ctrl+C`.
 `run` is not structured output capture and does not report the child command's exit status.
 
+## `mouse`
+
+`mouse` sends a cell-addressed event through the terminal modes selected by
+the child. The `libghostty-vt` renderer is required because static escape
+sequences cannot account for tracking mode, wire format, pixel mode, viewport
+clamping, or motion deduplication.
+
+```bash
+agent-tty mouse <session-id> press --button left --row 8 --col 20 --json
+agent-tty mouse <session-id> move --row 9 --col 24 --json
+agent-tty mouse <session-id> release --button left --row 9 --col 24 --json
+agent-tty mouse <session-id> press --button wheel-down --row 9 --col 24 --json
+```
+
+Rows and columns are zero-based. `press` and `release` require `--button`;
+`move` uses the host's held-button state so press, move, and release form a real
+drag. Buttons are `left`, `middle`, `right`, `wheel-up`, `wheel-down`,
+`wheel-left`, and `wheel-right`. `--shift`, `--alt`, and `--ctrl` add modifiers.
+An event suppressed by the child's tracking mode succeeds with
+`reported: false` and `bytesWritten: 0`. Missing renderer support exits `12`
+with `CAPABILITY_UNAVAILABLE`.
+
+The default surface uses virtual one-pixel cells. For SGR-pixels with a known
+surface, pass `--cell-width` and `--cell-height` in pixels; the requested cell
+maps to its top-left pixel. For example, row 2, column 3 with a 10x20 cell maps
+to pixel 30,40. Screen dimensions follow the session's current rows and columns.
+The same optional `cellWidth` and `cellHeight` fields are accepted in batch steps.
+
+The host writes the native buffer directly to the PTY, including non-UTF-8 X10
+bytes. Each `input_mouse` event stores those exact bytes as `dataBase64` alongside
+the action, cell position, modifiers, renderer, and any explicit cell metrics.
+Suppressed events still receive a sequence number and record an empty payload.
+Older native packages remain usable for other operations; see the
+[native mouse development setup](mouse-native-development.md) until a
+mouse-capable package is published.
+
 ## `wait`
 
 Use `wait` to synchronize on terminal state:
@@ -153,6 +190,7 @@ Steps are a JSON array; each step is exactly one verb. The shape mirrors the res
   { "run": "nvim --clean", "noWait": true },
   { "wait": { "screenStableMs": 1000 } },
   { "sendKeys": ["i"] },
+  { "mouse": { "action": "press", "button": "left", "row": 8, "col": 20 } },
   { "type": "hello" },
   { "sendKeys": ["Escape"] },
   { "type": ":wq" },
@@ -163,6 +201,7 @@ Steps are a JSON array; each step is exactly one verb. The shape mirrors the res
 
 - `type` / `paste`: a string of literal text.
 - `sendKeys`: a non-empty array of key names — individual named keys or single characters (e.g. `["Enter"]`, `["Ctrl+C"]`, `["Escape", "Enter"]`). Multi-character literal text such as `:wq` is not a key name; send it with a `type` step.
+- `mouse`: the same `action`, `button`, `row`, `col`, and optional `modifiers` fields as the `mouse` command. It may also carry `rendererName` for an explicit backend override.
 - `run`: a command string, with optional `noWait` (fire-and-forget) and `timeout` (ms). A `run` step is a waited run by default.
 - `wait`: the same conditions as the `wait` command — `text`, `regex`, `scope`, `screenStableMs`, `cursorRow`, `cursorCol`, and `timeout` (ms).
 
@@ -297,6 +336,7 @@ Every command exits with a stable code, so scripts can branch without parsing ou
 | `9`       | Protocol or RPC error.                                                                                                      |
 | `10`      | Replay failed.                                                                                                              |
 | `11`      | A standalone `wait` timed out, or a `wait` step inside a fail-fast `batch` timed out (`WAIT_TIMEOUT`; see [`wait`](#wait)). |
+| `12`      | A requested runtime capability is unavailable (`CAPABILITY_UNAVAILABLE`).                                                   |
 
 A fail-fast `batch` exits with the failed step's code (for example `11` for a wait timeout); `--keep-going` exits `1` if any step failed.
 

docs/mouse-native-development.md

@@ -0,0 +1,39 @@
+# Testing native mouse input
+
+Mouse input depends on [libghostty-vt-node PR #12](https://github.com/coder/libghostty-vt-node/pull/12).
+The existing optional dependency remains compatible for semantic operations;
+its published version does not yet encode mouse input. The adapter checks the
+terminal method at runtime and reports `CAPABILITY_UNAVAILABLE` when absent.
+
+To test the binding before publication, build and pack its exact revision with
+the pinned Zig 0.15.2 toolchain, Node, and node-gyp's platform prerequisites:
+
+```bash
+git clone https://github.com/pmarreck/libghostty-vt-node.git
+cd libghostty-vt-node
+git checkout 8318d03cc4c5d417d988548f920a784abfde33af
+npm ci
+npm run build:libghostty
+npm run build
+npm run build:prebuild
+npm run verify
+npm pack --ignore-scripts
+```
+
+From the `agent-tty` checkout, install that tarball locally without changing the
+published dependency or lockfile:
+
+```bash
+npm install --no-save --package-lock=false ../libghostty-vt-node/coder-libghostty-vt-node-0.1.0-beta.1.tgz
+AGENT_TTY_REQUIRE_MOUSE_NATIVE=1 npm test -- test/integration/mouse.test.ts
+```
+
+The integration tests use isolated session homes and a raw-mode PTY child to
+check the bytes it actually receives, including non-UTF-8 X10 bytes. They also
+check event-log preservation and replay, disabled reporting, and unsupported
+renderers. They run automatically when mouse support is installed; the
+environment flag makes missing support a failure instead of a skip.
+
+After the upstream package is released with this API, update the optional
+dependency and lockfile together. A Git source dependency alone is insufficient:
+the binding needs built TypeScript and a platform-specific native addon.

skill-data/agent-tty/SKILL.md

@@ -49,6 +49,7 @@ agent-tty --home <path> run <session-id> 'command here' --json
 agent-tty --home <path> type <session-id> 'literal text' --json
 agent-tty --home <path> paste <session-id> 'multiline payload' --json
 agent-tty --home <path> send-keys <session-id> Enter Ctrl+C --json
+agent-tty --home <path> mouse <session-id> press --button left --row 8 --col 20 --json
 agent-tty --home <path> batch <session-id> '[{"run":"htop","noWait":true},{"wait":{"screenStableMs":1000}}]' --json
 
 # Observation and proof
@@ -77,6 +78,13 @@ agent-tty --home "$AGENT_HOME" snapshot "$SESSION_ID" --format text --json
 
 Use `batch` to run an ordered sequence of input-and-`wait` steps in one call instead of separate `run`/`wait`/`send-keys` invocations. Each `wait` step is anchored to a Wait Baseline — it only observes screen state produced _after_ the preceding input step, so the sequence cannot race ahead and match a stale screen. A batch stops at the first failed step by default (`--keep-going` attempts every step).
 
+Use `mouse` for TUI hit-testing, scrolling, and drag behavior. Coordinates are
+zero-based cells. A drag is three actions: `press --button left`, one or more
+button-free `move` actions, then `release --button left`. The child must enable
+mouse tracking; a suppressed event returns `reported: false`. Mouse input
+requires the `libghostty-vt` capability and must never be replaced with a
+hard-coded SGR sequence.
+
 ```bash
 AGENT_HOME="$(mktemp -d)"
 SESSION_ID=$(agent-tty --home "$AGENT_HOME" create --json -- /bin/bash | jq -r '.result.sessionId')

src/batch/executor.ts

@@ -170,6 +170,16 @@ async function runStep(
         seq,
       };
     }
+    case 'mouse': {
+      const seq = await driver.mouse(step.input);
+      return {
+        index,
+        durationMs: Date.now() - startedAt,
+        kind: 'mouse',
+        status: 'completed',
+        seq,
+      };
+    }
     case 'run':
       return runRunStep(step, index, driver, startedAt);
     case 'wait':
@@ -324,6 +334,7 @@ function failedRecord(
     case 'type':
     case 'paste':
     case 'sendKeys':
+    case 'mouse':
       return { ...base, kind: step.kind, error: stepError };
     default:
       return unreachable(step, `batch failed-step kind at index ${index}`);

src/batch/plan.ts

@@ -1,16 +1,19 @@
 import { z } from 'zod';
 
 import type { PreparedRenderWaitCondition } from '../renderWait/matcher.js';
+import type { MouseParams } from '../protocol/messages.js';
 
 import { assertValidKeyName } from '../pty/keyEncoder.js';
 import { ERROR_CODES, makeCliError } from '../protocol/errors.js';
+import { MouseParamsSchema } from '../protocol/messages.js';
 import { prepareRenderWaitCondition } from '../renderWait/matcher.js';
 import { invariant, unreachable } from '../util/assert.js';
 
 export type BatchStep =
   | { kind: 'type'; text: string }
   | { kind: 'paste'; text: string }
   | { kind: 'sendKeys'; keys: string[] }
+  | { kind: 'mouse'; input: MouseParams }
   | {
       kind: 'run';
       command: string;
@@ -27,7 +30,14 @@ export interface BatchPlan {
   steps: BatchStep[];
 }
 
-const VERB_KEYS = ['type', 'paste', 'sendKeys', 'run', 'wait'] as const;
+const VERB_KEYS = [
+  'type',
+  'paste',
+  'sendKeys',
+  'mouse',
+  'run',
+  'wait',
+] as const;
 
 type VerbKey = (typeof VERB_KEYS)[number];
 
@@ -41,6 +51,7 @@ const PasteStepSchema = z.object({ paste: z.string().min(1) }).strict();
 const SendKeysStepSchema = z
   .object({ sendKeys: z.array(z.string().min(1)).min(1) })
   .strict();
+const MouseStepSchema = z.object({ mouse: z.unknown() }).strict();
 const RunStepSchema = z
   .object({
     run: z.string().min(1),
@@ -90,13 +101,13 @@ function parseStep(rawStep: unknown, index: number): BatchStep {
   const verbs = presentVerbKeys(rawStep);
   if (verbs.length === 0) {
     return invalidInput(
-      `Batch step ${String(index)} must have exactly one of type|paste|sendKeys|run|wait; found none`,
+      `Batch step ${String(index)} must have exactly one of type|paste|sendKeys|mouse|run|wait; found none`,
       index,
     );
   }
   if (verbs.length > 1) {
     return invalidInput(
-      `Batch step ${String(index)} must have exactly one of type|paste|sendKeys|run|wait; found ${verbs.join(', ')}`,
+      `Batch step ${String(index)} must have exactly one of type|paste|sendKeys|mouse|run|wait; found ${verbs.join(', ')}`,
       index,
     );
   }
@@ -111,6 +122,8 @@ function parseStep(rawStep: unknown, index: number): BatchStep {
       return parsePasteStep(rawStep, index);
     case 'sendKeys':
       return parseSendKeysStep(rawStep, index);
+    case 'mouse':
+      return parseMouseStep(rawStep, index);
     case 'run':
       return parseRunStep(rawStep, index);
     case 'wait':
@@ -164,6 +177,21 @@ function parseSendKeysStep(
   return { kind: 'sendKeys', keys };
 }
 
+function parseMouseStep(
+  rawStep: Record<string, unknown>,
+  index: number,
+): BatchStep {
+  const { mouse } = unwrapStep(MouseStepSchema, rawStep, index);
+  const result = MouseParamsSchema.safeParse(mouse);
+  if (!result.success) {
+    throw makeCliError(ERROR_CODES.INVALID_INPUT, {
+      message: `Batch step ${String(index)} is invalid`,
+      details: { stepIndex: index, issues: result.error.issues },
+    });
+  }
+  return { kind: 'mouse', input: result.data };
+}
+
 function parseRunStep(
   rawStep: Record<string, unknown>,
   index: number,

src/batch/result.ts

@@ -30,7 +30,7 @@ export const InputStepRecordSchema = z
   .object({
     index: NonNegativeIntSchema,
     durationMs: NonNegativeIntSchema,
-    kind: z.enum(['type', 'paste', 'sendKeys']),
+    kind: z.enum(['type', 'paste', 'sendKeys', 'mouse']),
     status: StepStatusSchema,
     seq: NonNegativeIntSchema.optional(),
     error: BatchStepErrorSchema.optional(),
@@ -105,6 +105,7 @@ export function unreachedStepRecord(
     case 'type':
     case 'paste':
     case 'sendKeys':
+    case 'mouse':
       return { ...base, kind: step.kind };
     default:
       return unreachable(step, `batch unreached-step kind at index ${index}`);

src/batch/stepDriver.ts

@@ -1,11 +1,16 @@
 import type { z } from 'zod';
 
-import type { RunResult, WaitForRenderResult } from '../protocol/messages.js';
+import type {
+  MouseParams,
+  RunResult,
+  WaitForRenderResult,
+} from '../protocol/messages.js';
 import type { PreparedRenderWaitCondition } from '../renderWait/matcher.js';
 
 import { sendRpc } from '../host/rpcClient.js';
 import {
   PasteResultSchema,
+  MouseResultSchema,
   RunResultSchema,
   SendKeysResultSchema,
   TypeResultSchema,
@@ -23,6 +28,7 @@ export interface StepDriver {
   type(text: string): Promise<number>;
   paste(text: string): Promise<number>;
   sendKeys(keys: string[]): Promise<number>;
+  mouse(input: MouseParams): Promise<number>;
   run(
     command: string,
     noWait: boolean,
@@ -107,6 +113,17 @@ export function createRpcStepDriver(
       return parseOrThrow(SendKeysResultSchema, raw, 'sendKeys').seq;
     },
 
+    async mouse(input: MouseParams): Promise<number> {
+      const params = {
+        ...input,
+        ...(input.rendererName === undefined && rendererName !== undefined
+          ? { rendererName }
+          : {}),
+      };
+      const raw = await sendRpc(socketPath, 'mouse', params);
+      return parseOrThrow(MouseResultSchema, raw, 'mouse').seq;
+    },
+
     async run(
       command: string,
       noWait: boolean,

src/cli/commands/batch.ts

@@ -177,6 +177,7 @@ function stepOutcome(record: BatchStepRecord): string {
     case 'type':
     case 'paste':
     case 'sendKeys':
+    case 'mouse':
       return 'completed';
   }
 }