Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions apps/connector/src/ending.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
/** How a build ended, as Chopin says in its answer to the build's last report. */

export type BuildEnding =
| { kind: "stopped"; by: string }
| { kind: "blocked"; task?: string; reason: string }
| { kind: "failed"; error: string }
| { kind: "finished"; pullRequests: string[]; tasks: number }
| { kind: "returned"; reason: string }
| { kind: "unfinished" };

/** The ending in Chopin's answer, when a server that says one sent it. */
export function endingOf(answer: unknown): BuildEnding | undefined {
let ending = (answer as { ending?: unknown } | undefined)?.ending as BuildEnding | undefined;
if (!ending || typeof ending !== "object" || typeof ending.kind !== "string") return;
return ending;
}

/** A blocked build's reason, naming its task when Chopin did. */
export function blockedText(ending: { task?: string; reason: string }): string {
return `${ending.task ? `“${ending.task}” is blocked: ` : ""}${ending.reason}`.replace(
/[.\s]+$/,
"",
);
}
35 changes: 33 additions & 2 deletions apps/connector/src/output.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
* The connector's human-facing terminal lines. Messages go to stdout so terminals don't paint them
* as errors; failures go to stderr. ANSI styling only reaches a TTY and honours `NO_COLOR`.
*/
import { blockedText, type BuildEnding } from "./ending";

type Stream = { write(text: string): unknown; isTTY?: boolean };

const CODES = {
Expand Down Expand Up @@ -147,8 +149,37 @@ export function createOutput(
finished() {
say(` ${s.green("✓")} Finished`, "");
},
stopped() {
say(` ${s.dim("✗ Stopped from the connector terminal.")}`, "");
/**
* A build's agent ended its turn, and Chopin said how the build ended: blocked on a task and
* why, finished with the pull requests it opened, or with tasks left. A server that says
* nothing gets plain Finished.
*/
ended(ending?: BuildEnding) {
if (ending?.kind === "blocked") {
return say(` ${s.dim(`✗ Blocked: ${blockedText(ending)}.`)}`, "");
}
if (ending?.kind === "unfinished") {
return say(
` ${s.dim("✗ Ended with tasks left; Build plan in Chopin builds them again.")}`,
"",
);
}
if (ending?.kind === "returned") {
return say(
` ${s.dim(`✗ Returned for changes: ${ending.reason.replace(/[.\s]+$/, "")}.`)}`,
"",
);
}
if (ending?.kind === "finished") {
let tasks = `${ending.tasks} ${ending.tasks === 1 ? "task" : "tasks"}`;
let opened = ending.pullRequests.length ? `, ${ending.pullRequests.join(", ")}` : "";
return say(` ${s.green("✓")} Finished: ${tasks}${opened}`, "");
}
say(` ${s.green("✓")} Finished`, "");
},
/** Ctrl-C here, or `why`: a person's Stop in Chopin, which names them. */
stopped(why = "Stopped from the connector terminal.") {
say(` ${s.dim(`✗ ${why}`)}`, "");
},
/** `agent` is the agent's last stderr lines, shown when debugging didn't already show them. */
failed(message: string, agent: string[] = []) {
Expand Down
75 changes: 75 additions & 0 deletions apps/connector/src/run.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -981,6 +981,81 @@ test("a run Chopin has ended stops its agent at once", async () => {
}]);
});

/** The run's journal, read before the run's folders are removed. */
function journalOf(into: { value?: Record<string, unknown> }) {
return async ({ state, id }: { state: string; id: string }) => {
into.value = JSON.parse(await readFile(join(state, "runs", `${id}.json`), "utf8"));
};
}

test("a build a person stopped in Chopin says who stopped it, as a stop and not a failure", async () => {
let terminal = captured();
let journal: { value?: Record<string, unknown> } = {};
let { calls } = await run(
{},
["--hang"],
name => {
if (name === "renew_implementation_build") {
throw new ConnectorError("build-stopped", "build-stopped: Stopped in Chopin by @ana.");
}
return {};
},
undefined,
{ output: terminal.output },
journalOf(journal),
);
expect(terminal.out()).toContain(" ✗ Stopped in Chopin by @ana.\n");
expect(terminal.err()).toBe("");
// Chopin ended the build itself, so there is nothing to report back.
expect(calls.at(-1)?.[0]).toBe("renew_implementation_build");
expect(journal.value).toMatchObject({
state: "finished",
outcome: { state: "stopped", reason: "Stopped in Chopin by @ana." },
});
});

test("a build whose agent blocked a task says it is blocked and why, not Finished", async () => {
let terminal = captured();
let journal: { value?: Record<string, unknown> } = {};
await run(
{ live: true },
["--live"],
name =>
name === "report_implementation_build"
? {
accepted: true,
ending: { kind: "blocked", task: "Prove it", reason: "Missing source: the files" },
}
: {},
undefined,
{ output: terminal.output },
journalOf(journal),
);
expect(terminal.out()).toContain(
" ✗ Blocked: “Prove it” is blocked: Missing source: the files.\n",
);
expect(terminal.out()).not.toContain("Finished");
expect(journal.value).toMatchObject({
outcome: { state: "blocked", reason: "“Prove it” is blocked: Missing source: the files" },
});
});

test("a build that delivered names its pull requests when it finishes", async () => {
let terminal = captured();
let url = "https://github.com/org/repo/pull/7";
await run(
{ live: true },
["--live"],
name =>
name === "report_implementation_build"
? { accepted: true, ending: { kind: "finished", pullRequests: [url], tasks: 1 } }
: {},
undefined,
{ output: terminal.output },
);
expect(terminal.out()).toContain(` ✓ Finished: 1 task, ${url}\n`);
});

test("a build Chopin forgot by restarting stops saying so", async () => {
let { calls } = await run({}, ["--hang"], name => {
if (name === "renew_implementation_build") {
Expand Down
54 changes: 41 additions & 13 deletions apps/connector/src/run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import { AgentTimedOut, runAgent, timedOut } from "./acp";
import type { AgentInfo } from "./acp";
import { serveLocalBridge } from "./local-bridge";
import { createOutput } from "./output";
import { blockedText, endingOf } from "./ending";
import type { Output } from "./output";
import { ask, describePermission, permissionMemory } from "./permission";
import type { Sandbox } from "./sandbox";
Expand Down Expand Up @@ -86,6 +87,12 @@ export const STOPPED = "Stopped from the connector terminal.";
/** A run out of reach says it is still trying at most this often. */
const RETRY_SAID_MS = 10_000;

/** A refusal's message without the code Chopin put in front of it. */
function refusalMessage(error: unknown): string | undefined {
let message = error instanceof Error ? error.message.replace(/^[a-z][a-z0-9-]*:\s*/, "") : "";
return message.trim() || undefined;
}

function minutes(ms: number) {
let value = Math.round(ms / 60_000);
return value >= 1
Expand Down Expand Up @@ -142,6 +149,8 @@ export async function runWork(
let ended: string | undefined;
/** Chopin forgot this connection, so the connector must stop once the run has reported. */
let forgotten = false;
/** A person stopped the build in Chopin, which says who: a stop, not a failure. */
let stoppedInChopin = false;
let now = options.now ?? Date.now;
/** When the terminal last said Chopin is out of reach, so retries say so at most every 10 s. */
let saidAt = 0;
Expand Down Expand Up @@ -211,6 +220,10 @@ export async function runWork(
continue;
}
let code = errorCode(error);
if (code === "build-stopped" && ended === undefined) {
stoppedInChopin = true;
ended = refusalMessage(error) ?? "Stopped in Chopin.";
}
// A build claimed before Chopin restarted is one it no longer knows.
ended ??= code === "server-restarted"
? "Chopin restarted, so it ended this run."
Expand Down Expand Up @@ -444,12 +457,19 @@ export async function runWork(
agentStop = stop;
throw new Error(`Agent stopped: ${stop}`);
}
await reports.deliver(implementation ? "report_implementation_build" : "complete_experiment", {
...identity,
...(implementation ? { state: "stopped", idempotencyKey: crypto.randomUUID() } : {}),
});
output.finished();
outcome = { state: "finished" };
let answer = await reports.deliver(
implementation ? "report_implementation_build" : "complete_experiment",
{
...identity,
...(implementation ? { state: "stopped", idempotencyKey: crypto.randomUUID() } : {}),
},
);
// Chopin says how the build ended: blocked on a task, or the pull requests it opened.
let ending = endingOf(answer);
output.ended(ending);
outcome = ending?.kind === "blocked"
? { state: "blocked", reason: blockedText(ending) }
: { state: "finished" };
} catch (error) {
// Ctrl-C reports why before the connector disconnects, so Chopin knows at once. An agent
// that cancelled its own turn was interrupted too, unless the connector cancelled it.
Expand All @@ -469,10 +489,16 @@ export async function runWork(
: error instanceof Error
? error.message
: "Local work failed";
// A person's Stop in Chopin ended the build there already: nothing failed, and nothing is
// left to report.
let chopinStop = !stopped && stoppedInChopin && ended === message;
outcome = stopped
? { state: "stopped" }
: chopinStop
? { state: "stopped", reason: message }
: { state: "failed", error: redacted(message, secrets()).slice(0, 2000) };
if (stopped) output.stopped();
else if (chopinStop) output.stopped(message);
else {
output.failed(
redacted(message, secrets()),
Expand All @@ -481,13 +507,15 @@ export async function runWork(
}
// After Ctrl-C, or out of reach for the whole grace period, the failure gets one bounded attempt.
let send = stopped || reports.expired() ? reports.final : reports.deliver;
await send(implementation ? "report_implementation_build" : "fail_experiment", {
...identity,
...(implementation ? { state: "failed", idempotencyKey: crypto.randomUUID() } : {}),
error: stopped || ended || late || !implementation
? message.slice(0, 2000)
: "Local implementation failed. Inspect the connector terminal.",
}).catch(() => {});
if (!chopinStop) {
await send(implementation ? "report_implementation_build" : "fail_experiment", {
...identity,
...(implementation ? { state: "failed", idempotencyKey: crypto.randomUUID() } : {}),
error: stopped || ended || late || !implementation
? message.slice(0, 2000)
: "Local implementation failed. Inspect the connector terminal.",
}).catch(() => {});
}
// Retrying cannot reach a server that forgot this connection, so the connector stops too.
if (forgotten && !stopped) throw new ConnectorStopped("forgotten", message);
} finally {
Expand Down
5 changes: 4 additions & 1 deletion apps/connector/src/workspace.ts
Original file line number Diff line number Diff line change
Expand Up @@ -694,7 +694,10 @@ function extraWorktrees(path: string) {
/** How a run ended, as its journal records it: its agent finished, failed and why, or was stopped. */
export type RunOutcome =
| { state: "finished" }
| { state: "stopped" }
/** `reason` names who stopped it in Chopin; Ctrl-C here has none. */
| { state: "stopped"; reason?: string }
/** Its agent blocked a task: which one and why. */
| { state: "blocked"; reason: string }
| { state: "failed"; error: string };

/**
Expand Down
4 changes: 2 additions & 2 deletions apps/server/src/agent/scoped-followup-tools.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { jsonSchema, tool } from "ai";
import { answer, cardToolSchema } from "./scoped-tool-result";
import { runJobTool } from "./job-scope";
import { followupIntent, scopeSnapshot } from "../questions/decision-followup";
import { followupIntent, readScope } from "../questions/decision-followup";
import { type ScopeWrite, writeScope } from "../questions/decision-followup-write";
import type { DocumentRoom } from "./tools";

Expand All @@ -27,7 +27,7 @@ export function createFollowupTools() {
answer("read_decision_scope", () =>
room.exclusive(async () => {
let job = current(room);
let { indices: _indices, ...snapshot } = scopeSnapshot(room.plan, job.target);
let { indices: _indices, ...snapshot } = readScope(room.plan, job);
return snapshot;
})),
}),
Expand Down
64 changes: 64 additions & 0 deletions apps/server/src/chat/archive-runs.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
import { expect, test } from "bun:test";
import * as Chat from "./service";
import * as Plan from "../plan/service";
import { hostInputRoom } from "../testing/decisions";

import type { Chat as Wire } from "@chopin/protocol";
import type { Server } from "bun";
import type { SocketData } from "../wire";

let server = { publish() {} } as unknown as Server<SocketData>;

function card(id: string, status: Wire.Run["status"], waiting = 0): Wire.Run {
return { id, name: id, status, started: 1, updated: 2, stages: [], waiting };
}

test("archiving ends every live run card, one waiting on a question included, and saves it", async () => {
// A disconnected Planner's cards come back as it last reported them, live.
let chat = Chat.restore(
[],
[card("plan", "waiting", 1), card("verify", "running"), card("earlier", "finished")],
false,
"down",
);
expect(chat.runs?.map(run => run.status)).toEqual(["waiting", "running", "finished"]);
let saved: Wire.Run[] | undefined;
await Chat.endRuns({
chat,
room: "document",
server,
persist: async change => {
saved = change?.().runs;
},
});
expect(saved?.map(run => [run.status, run.waiting])).toEqual([
["stopped", 0],
["stopped", 0],
["finished", 0],
]);
expect(chat.runs).toEqual(saved);
expect(chat.runs?.some(run => run.disconnected)).toBe(false);
});

test("an archived document opens with its runs stopped, even if a Planner hold still claims them", async () => {
let first = await hostInputRoom();
first.plan.chat.runs = [card("plan", "waiting", 1)];
await Plan.persist(first.plan, () => first.plan.chat);
await first.close();
let reopen = { storage: first.storage, channelId: first.channelId, lease: first.lease };
// A hold the document's archive could not end, as when it came while a deploy handed over.
let backend = { plannerKeepsRuns: () => "down" as const };
let live = await hostInputRoom(undefined, { ...reopen, backend });
expect(live.plan.chat.runs?.map(run => [run.status, run.waiting])).toEqual([["waiting", 1]]);
await live.close();
await first.storage.channels.archive({ id: first.channelId, now: new Date() });
let archived = await hostInputRoom(undefined, { ...reopen, backend });
try {
expect(archived.plan.chat.runs?.map(run => [run.status, run.waiting])).toEqual([[
"stopped",
0,
]]);
} finally {
await archived.close();
}
});
4 changes: 3 additions & 1 deletion apps/server/src/chat/job-tools.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,9 @@ test("two failed own-tool calls end the turn after one possible re-read", async
let outcome = await running;
expect(outcome.status).toBe("failed");
if (outcome.status === "done") throw new Error("two tool failures completed the job");
expect(outcome.reason).toContain("failed twice");
expect(outcome.reason).toBe(
"The Planner's refine_decision failed twice: the document changed while it was being written.",
);
expect(state.aborts()).toBe(1);
expect(state.chat.job).toBeUndefined();
});
Expand Down
Loading
Loading