From e6a515e8e30b0fa220d3e1e5ec66209995ab9f5c Mon Sep 17 00:00:00 2001 From: phroi <90913182+phroi@users.noreply.github.com> Date: Wed, 20 May 2026 05:32:23 +0000 Subject: [PATCH 01/11] feat(supervisor): add deterministic live supervisor --- apps/supervisor/package.json | 51 + apps/supervisor/src/index.test.ts | 1770 +++++++++++++++++++++++ apps/supervisor/src/index.ts | 2047 +++++++++++++++++++++++++++ apps/supervisor/tsconfig.build.json | 9 + apps/supervisor/tsconfig.json | 11 + apps/supervisor/vitest.config.mts | 10 + 6 files changed, 3898 insertions(+) create mode 100644 apps/supervisor/package.json create mode 100644 apps/supervisor/src/index.test.ts create mode 100644 apps/supervisor/src/index.ts create mode 100644 apps/supervisor/tsconfig.build.json create mode 100644 apps/supervisor/tsconfig.json create mode 100644 apps/supervisor/vitest.config.mts diff --git a/apps/supervisor/package.json b/apps/supervisor/package.json new file mode 100644 index 0000000..a4712a8 --- /dev/null +++ b/apps/supervisor/package.json @@ -0,0 +1,51 @@ +{ + "name": "@ickb/supervisor", + "version": "1001.0.0", + "description": "Deterministic iCKB testnet live supervisor", + "keywords": [ + "ickb", + "ckb", + "testnet", + "supervisor" + ], + "author": "phroi", + "license": "MIT", + "homepage": "https://ickb.org", + "repository": { + "type": "git", + "url": "https://github.com/ickb/stack" + }, + "bugs": { + "url": "https://github.com/ickb/stack/issues" + }, + "sideEffects": false, + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": { + "import": "./dist/index.js", + "types": "./dist/index.d.ts" + } + }, + "scripts": { + "test": "vitest", + "test:ci": "vitest run", + "build": "pnpm clean && tsc -p tsconfig.build.json", + "lint": "eslint ./src", + "clean": "rm -fr dist", + "clean:deep": "rm -fr dist node_modules", + "start": "node dist/index.js" + }, + "files": [ + "dist", + "src" + ], + "publishConfig": { + "access": "public", + "provenance": true + }, + "devDependencies": { + "@types/node": "catalog:" + } +} diff --git a/apps/supervisor/src/index.test.ts b/apps/supervisor/src/index.test.ts new file mode 100644 index 0000000..3d6beeb --- /dev/null +++ b/apps/supervisor/src/index.test.ts @@ -0,0 +1,1770 @@ +import { EventEmitter } from "node:events"; +import { describe, expect, it, vi } from "vitest"; +import { + appendBoundedOutput, + boundedOutputText, + chooseScenario, + classifyActorResult, + createBoundedOutputCapture, + createCoverageLedger, + parseArgs, + parseJsonEvidence, + parsePreflightEvidence, + recordScenarioAttempt, + recordOutcome, + resolvePlan, + safeArtifactText, + supervise, + usage, + type CommandResult, +} from "./index.js"; + +describe("supervisor CLI", () => { + it("parses bounded live supervisor arguments", () => { + const args = parseArgs([ + "--bot-config", "config/bot-testnet.json", + "--tester-config", "config/tester-testnet.json", + "--max-cycles", "3", + "--stop-after-tx-count", "2", + "--scenario", "tester-fresh-skip-two-pass", + "--tester-scenario", "all-ckb-limit-order", + "--tester-fee", "1000", + "--tester-fee-base", "100000", + "--target-outcome", "bot_match_committed", + "--target-outcome", "bot_no_action_skip", + ]); + + expect(args.botConfigPath).toBe("config/bot-testnet.json"); + expect(args.testerConfigPath).toBe("config/tester-testnet.json"); + expect(args.maxCycles).toBe(3); + expect(args.stopAfterTxCount).toBe(2); + expect(args.scenario).toBe("tester-fresh-skip-two-pass"); + expect(args.testerScenario).toBe("all-ckb-limit-order"); + expect(args.testerScenarioExplicit).toBe(true); + expect(args.testerFee).toBe("1000"); + expect(args.testerFeeBase).toBe("100000"); + expect(args.testerFeeExplicit).toBe(true); + expect(args.testerFeeBaseExplicit).toBe(true); + expect(args.targetOutcomes).toEqual(["bot_match_committed", "bot_no_action_skip"]); + expect(usage()).toContain("--bot-config"); + expect(usage()).toContain("sdk-conversion"); + expect(usage()).toContain("tester-fresh-skip-two-pass"); + }); + + it("parses the SDK conversion tester scenario", () => { + const args = parseArgs(["--tester-scenario", "sdk-conversion"]); + + expect(args.testerScenario).toBe("sdk-conversion"); + expect(args.testerScenarioExplicit).toBe(true); + expect(parseArgs(["--tester-scenario", "two-ckb-to-ickb-limit-orders"]).testerScenario).toBe( + "two-ckb-to-ickb-limit-orders", + ); + expect(parseArgs(["--tester-scenario", "two-ickb-to-ckb-limit-orders"]).testerScenario).toBe( + "two-ickb-to-ckb-limit-orders", + ); + expect(parseArgs(["--tester-scenario", "mixed-direction-limit-orders"]).testerScenario).toBe( + "mixed-direction-limit-orders", + ); + expect(parseArgs(["--tester-scenario", "multi-order-limit-orders"]).testerScenario).toBe( + "multi-order-limit-orders", + ); + expect(() => parseArgs(["--tester-scenario", "interface-like"])).toThrow("Invalid --tester-scenario"); + }); + + it("rejects malformed tester fee controls", () => { + expect(() => parseArgs(["--tester-fee", "1.5"])).toThrow("Invalid --tester-fee"); + expect(() => parseArgs(["--tester-fee-base", "-1"])).toThrow("Invalid --tester-fee-base"); + }); + + it("defaults bare supervisor runs to deterministic live configs", () => { + const args = parseArgs([]); + + expect(args.botConfigPath).toBe("config/bot-testnet.json"); + expect(args.testerConfigPath).toBe("config/tester-testnet.json"); + expect(args.testerScenario).toBe("auto"); + expect(args.testerScenarioExplicit).toBe(false); + expect(args.testerFee).toBe("1"); + expect(args.testerFeeBase).toBe("100000"); + expect(args.testerFeeExplicit).toBe(false); + expect(args.testerFeeBaseExplicit).toBe(false); + }); + + it("rejects old LLM command-shape and repair mode flags", () => { + expect(() => parseArgs(["--llm-bin", "custom-llm"])).toThrow("Unknown argument: --llm-bin"); + expect(() => parseArgs(["--llm-command", "custom-llm"])).toThrow("Unknown argument: --llm-command"); + expect(() => parseArgs(["--llm-arg", "repair"])).toThrow("Unknown argument: --llm-arg"); + expect(() => parseArgs(["--llm-on-incident"])).toThrow("Unknown argument: --llm-on-incident"); + expect(() => parseArgs(["--no-llm-on-incident"])).toThrow("Unknown argument: --no-llm-on-incident"); + expect(() => parseArgs(["--llm-timeout-seconds", "60"])).toThrow("Unknown argument: --llm-timeout-seconds"); + expect(() => parseArgs(["--llm-max-attempts", "2"])).toThrow("Unknown argument: --llm-max-attempts"); + expect(() => parseArgs(["--max-repair-rounds", "1"])).toThrow("Unknown argument: --max-repair-rounds"); + expect(() => parseArgs(["--autonomous-repair"])).toThrow("Unknown argument: --autonomous-repair"); + expect(() => parseArgs(["--repair-commit-message", "repair test"])).toThrow("Unknown argument: --repair-commit-message"); + expect(() => parseArgs(["--coverage-goal", "bot_match_committed"])).toThrow("Unknown argument: --coverage-goal"); + expect(() => parseArgs(["--stop-on", "unmet_coverage_goal"])).toThrow("Unknown argument: --stop-on"); + expect(() => parseArgs(["--force"])).toThrow("Unknown argument: --force"); + expect(() => parseArgs(["--verify-command", "pnpm test"])).toThrow("Unknown argument: --verify-command"); + expect(() => parseArgs(["--expected-chain", "mainnet"])).toThrow("Unknown argument: --expected-chain"); + expect(() => parseArgs(["--no-preflight"])).toThrow("Unknown argument: --no-preflight"); + }); + + it("refuses non-ignored output paths", () => { + const args = parseArgs(["--dry-run", "--out-dir", "not-ignored"]); + + expect(() => resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(false) })).toThrow( + "Supervisor output directory must be under logs/live-supervisor/", + ); + }); + + it("refuses ignored output paths outside the supervisor artifact root", () => { + const args = parseArgs(["--dry-run", "--out-dir", "config/supervisor"]); + + expect(() => resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) })).toThrow( + "Supervisor output directory must be under logs/live-supervisor/", + ); + }); + + it("resolves ignored dry-run artifact paths without configs", () => { + const args = parseArgs(["--dry-run", "--out-dir", "logs/live-supervisor/test"]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) }); + + expect(plan.relativeOutDir).toBe("logs/live-supervisor/test"); + expect(plan.botConfigPath).toBeUndefined(); + expect(plan.testerConfigPath).toBeUndefined(); + }); + + it("resolves default ignored live config paths", () => { + const plan = resolvePlan(parseArgs(["--out-dir", "logs/live-supervisor/test"]), "/repo", { spawnSyncCommand: selectiveIgnoredChecker(new Set([ + "logs/live-supervisor/test", + "config/bot-testnet.json", + "config/tester-testnet.json", + ])) }); + + expect(plan.botConfigPath).toBe("/repo/config/bot-testnet.json"); + expect(plan.testerConfigPath).toBe("/repo/config/tester-testnet.json"); + }); + + it("refuses to reuse an existing output directory", async () => { + const args = parseArgs(["--dry-run", "--out-dir", "logs/live-supervisor/existing"]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) }); + + await expect(supervise(args, plan, { + stat: () => Promise.resolve({} as never), + mkdir: () => Promise.resolve(undefined), + })).rejects.toThrow("Output directory already exists: logs/live-supervisor/existing"); + }); + + it("refuses symlinked supervisor artifact parents", async () => { + const args = parseArgs(["--dry-run", "--out-dir", "logs/live-supervisor/symlink-parent"]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) }); + + await expect(supervise(args, plan, { + lstat: (path) => { + if (pathToString(path) === "/repo/logs") { + return Promise.resolve({ isSymbolicLink: () => true } as never); + } + return Promise.resolve({ isSymbolicLink: () => false } as never); + }, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + })).rejects.toThrow("Refusing to write supervisor artifacts through symlinked path: logs"); + }); + + it("refuses real supervisor artifact paths outside the repo", async () => { + const args = parseArgs(["--dry-run", "--out-dir", "logs/live-supervisor/escaped"]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) }); + + await expect(supervise(args, plan, { + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + realpath: (path) => Promise.resolve(pathToString(path) === "/repo" ? "/repo" : "/tmp/escaped"), + })).rejects.toThrow("Supervisor output directory must stay inside the repo"); + }); + + it("spawns live actors with an allowlisted environment", async () => { + const originalPrivateKey = process.env.PRIVATE_KEY; + process.env.PRIVATE_KEY = "operator-secret"; + try { + const spawned: Array<{ args: string[]; env?: NodeJS.ProcessEnv }> = []; + const args = parseArgs([ + "--out-dir", "logs/live-supervisor/env-test", + "--scenario", "bot-only", + "--max-cycles", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) }); + + await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + spawnCommand: ((_command: string, commandArgs: string[], options: { env?: NodeJS.ProcessEnv }) => { + spawned.push({ args: commandArgs, env: options.env }); + return isPreflightCommand(commandArgs) + ? fakeSuccessfulPreflightChild() + : fakeChild(JSON.stringify(botEvent("bot.decision.skipped", { + reason: "no_actions", + actions: emptyActions(), + }))); + }) as never, + spawnSyncCommand: ignoredChecker(true) as never, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + appendFile: () => Promise.resolve(), + writeFile: () => Promise.resolve(), + }); + + const preflight = spawned.find((item) => isPreflightCommand(item.args)); + const actor = spawned.find((item) => item.args[0] === "apps/bot/dist/index.js"); + expect(preflight?.env).not.toHaveProperty("PRIVATE_KEY"); + expect(preflight?.env).not.toHaveProperty("COWORKER_BUILD"); + expect(actor?.env).toMatchObject({ BOT_CONFIG_FILE: "/repo/config/bot-testnet.json", INIT_CWD: "/repo" }); + expect(actor?.env).not.toHaveProperty("PRIVATE_KEY"); + expect(actor?.env).not.toHaveProperty("COWORKER_BUILD"); + } finally { + if (originalPrivateKey === undefined) { + delete process.env.PRIVATE_KEY; + } else { + process.env.PRIVATE_KEY = originalPrivateKey; + } + } + }); + + it("steers tester conversion coverage to the SDK conversion builder", async () => { + const spawned: Array<{ args: string[]; env?: NodeJS.ProcessEnv }> = []; + const args = parseArgs([ + "--out-dir", "logs/live-supervisor/conversion-env-test", + "--target-outcome", "tester_conversion_created", + "--max-cycles", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) }); + + const exitCode = await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + spawnCommand: ((_command: string, commandArgs: string[], options: { env?: NodeJS.ProcessEnv }) => { + spawned.push({ args: commandArgs, env: options.env }); + return isPreflightCommand(commandArgs) + ? fakeSuccessfulPreflightChild() + : fakeChild(JSON.stringify({ + startTime: "now", + actions: { conversion: { kind: "direct" }, cancelledOrders: 0 }, + txHash: txHash("15"), + ElapsedSeconds: 1, + })); + }) as never, + spawnSyncCommand: ignoredChecker(true) as never, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + appendFile: () => Promise.resolve(), + writeFile: () => Promise.resolve(), + }); + + const tester = spawned.find((item) => item.args[0] === "apps/tester/dist/index.js"); + expect(exitCode).toBe(0); + expect(tester?.env).toMatchObject({ TESTER_SCENARIO: "sdk-conversion" }); + }); + + it("preserves explicit tester scenario over conversion target steering", async () => { + const spawned: Array<{ args: string[]; env?: NodeJS.ProcessEnv }> = []; + const args = parseArgs([ + "--out-dir", "logs/live-supervisor/explicit-tester-env-test", + "--target-outcome", "tester_conversion_created", + "--tester-scenario", "ickb-to-ckb-limit-order", + "--stop-after-tx-count", "1", + "--max-cycles", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) }); + + const exitCode = await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + spawnCommand: ((_command: string, commandArgs: string[], options: { env?: NodeJS.ProcessEnv }) => { + spawned.push({ args: commandArgs, env: options.env }); + return isPreflightCommand(commandArgs) + ? fakeSuccessfulPreflightChild() + : fakeChild(JSON.stringify({ + startTime: "now", + actions: { + testerScenario: "ickb-to-ckb-limit-order", + newOrder: { giveIckb: "10", takeCkb: "9", fee: "0.1" }, + cancelledOrders: 0, + }, + txHash: txHash("16"), + ElapsedSeconds: 1, + })); + }) as never, + spawnSyncCommand: ignoredChecker(true) as never, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + appendFile: () => Promise.resolve(), + writeFile: () => Promise.resolve(), + }); + + const tester = spawned.find((item) => item.args[0] === "apps/tester/dist/index.js"); + expect(exitCode).toBe(0); + expect(tester?.env).toMatchObject({ TESTER_SCENARIO: "ickb-to-ckb-limit-order" }); + }); + + it("preserves explicit tester auto scenario over conversion target steering", async () => { + const spawned: Array<{ args: string[]; env?: NodeJS.ProcessEnv }> = []; + const args = parseArgs([ + "--out-dir", "logs/live-supervisor/explicit-auto-env-test", + "--target-outcome", "tester_conversion_created", + "--tester-scenario", "auto", + "--max-cycles", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) }); + + const exitCode = await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + spawnCommand: ((_command: string, commandArgs: string[], options: { env?: NodeJS.ProcessEnv }) => { + spawned.push({ args: commandArgs, env: options.env }); + return isPreflightCommand(commandArgs) + ? fakeSuccessfulPreflightChild() + : fakeChild(JSON.stringify({ + startTime: "now", + actions: { conversion: { kind: "direct" }, cancelledOrders: 0 }, + txHash: txHash("17"), + ElapsedSeconds: 1, + })); + }) as never, + spawnSyncCommand: ignoredChecker(true) as never, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + appendFile: () => Promise.resolve(), + writeFile: () => Promise.resolve(), + }); + + const tester = spawned.find((item) => item.args[0] === "apps/tester/dist/index.js"); + expect(exitCode).toBe(0); + expect(tester?.env).toMatchObject({ TESTER_SCENARIO: "auto" }); + }); + + it("passes tester fee controls only to the tester actor", async () => { + const spawned: Array<{ args: string[]; env?: NodeJS.ProcessEnv }> = []; + const args = parseArgs([ + "--out-dir", "logs/live-supervisor/tester-fee-env-test", + "--scenario", "standard-cycle", + "--tester-fee", "1000", + "--tester-fee-base", "100000", + "--max-cycles", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) }); + + const exitCode = await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + spawnCommand: ((_command: string, commandArgs: string[], options: { env?: NodeJS.ProcessEnv }) => { + spawned.push({ args: commandArgs, env: options.env }); + if (isPreflightCommand(commandArgs)) { + return fakeSuccessfulPreflightChild(); + } + return commandArgs[0] === "apps/tester/dist/index.js" + ? fakeChild(JSON.stringify({ + startTime: "now", + actions: { newOrder: { giveCkb: "10", takeIckb: "9", fee: "0.1" }, cancelledOrders: 0 }, + txHash: txHash("18"), + ElapsedSeconds: 1, + })) + : fakeChild(JSON.stringify(botEvent("bot.decision.skipped", { + reason: "no_actions", + actions: emptyActions(), + }))); + }) as never, + spawnSyncCommand: ignoredChecker(true) as never, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + appendFile: () => Promise.resolve(), + writeFile: () => Promise.resolve(), + }); + + const tester = spawned.find((item) => item.args[0] === "apps/tester/dist/index.js"); + const bot = spawned.find((item) => item.args[0] === "apps/bot/dist/index.js"); + expect(exitCode).toBe(0); + expect(tester?.env).toMatchObject({ TESTER_FEE: "1000", TESTER_FEE_BASE: "100000" }); + expect(bot?.env).not.toHaveProperty("TESTER_FEE"); + expect(bot?.env).not.toHaveProperty("TESTER_FEE_BASE"); + }); + + it("runs the same tester twice for fresh-skip multi-order coverage", async () => { + const spawned: Array<{ args: string[]; env?: NodeJS.ProcessEnv }> = []; + const writes = new Map(); + let testerRuns = 0; + const args = parseArgs([ + "--out-dir", "logs/live-supervisor/two-pass-test", + "--scenario", "tester-fresh-skip-two-pass", + "--target-outcome", "tester_order_created", + "--target-outcome", "tester_fresh_order_skip", + "--max-cycles", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) }); + + const exitCode = await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + spawnCommand: ((_command: string, commandArgs: string[], options: { env?: NodeJS.ProcessEnv }) => { + spawned.push({ args: commandArgs, env: options.env }); + if (isPreflightCommand(commandArgs)) { + return fakeSuccessfulPreflightChild(); + } + testerRuns += 1; + return fakeChild(JSON.stringify(testerRuns === 1 + ? { + startTime: "now", + actions: { + requestedTesterScenario: "multi-order-limit-orders", + testerScenario: "mixed-direction-limit-orders", + newOrders: [ + { giveCkb: "10", takeIckb: "9", fee: "0.1" }, + { giveIckb: "20", takeCkb: "18", fee: "0.2" }, + ], + orderCount: 2, + cancelledOrders: 0, + }, + txHash: txHash("77"), + ElapsedSeconds: 1, + } + : { skip: { reason: "fresh-matchable-order", txHash: txHash("77") } })); + }) as never, + spawnSyncCommand: ignoredChecker(true) as never, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + appendFile: () => Promise.resolve(), + writeFile: (path, text) => { + writes.set(pathToString(path), textToString(text)); + return Promise.resolve(); + }, + }); + + const testerSpawns = spawned.filter((item) => item.args[0] === "apps/tester/dist/index.js"); + expect(exitCode).toBe(0); + expect(testerSpawns).toHaveLength(2); + expect(testerSpawns[0]?.env).toMatchObject({ TESTER_SCENARIO: "multi-order-limit-orders" }); + expect(testerSpawns[1]?.env).toMatchObject({ TESTER_SCENARIO: "auto" }); + expect(writes.has("/repo/logs/live-supervisor/two-pass-test/cycle-0001-tester-pass-1.stdout.ndjson")).toBe(true); + expect(writes.has("/repo/logs/live-supervisor/two-pass-test/cycle-0001-tester-pass-2.stdout.ndjson")).toBe(true); + const summary = jsonArtifact(writes, "/repo/logs/live-supervisor/two-pass-test/summary.json"); + expect(recordAt(summary["aggregateCounts"], "summary aggregate counts")).toMatchObject({ + tester_order_created: 1, + tester_fresh_order_skip: 1, + }); + }); + + it("refuses live config paths through symlinked parents", async () => { + const args = parseArgs([ + "--out-dir", "logs/live-supervisor/config-symlink-test", + "--scenario", "bot-only", + "--max-cycles", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) }); + + await expect(supervise(args, plan, { + skipBuiltRuntimeCheck: true, + lstat: (path) => { + if (pathToString(path) === "/repo/config") { + return Promise.resolve({ isSymbolicLink: () => true } as never); + } + return Promise.resolve({ isSymbolicLink: () => false } as never); + }, + spawnSyncCommand: ignoredChecker(true) as never, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + appendFile: () => Promise.resolve(), + writeFile: () => Promise.resolve(), + })).rejects.toThrow("Refusing to use bot config path through symlinked path: config"); + }); + + it("refuses non-ignored config paths", () => { + const args = parseArgs([ + "--bot-config", "tracked-bot.json", + "--tester-config", "config/tester-testnet.json", + "--out-dir", "logs/live-supervisor/test", + ]); + + expect(() => resolvePlan(args, "/repo", { spawnSyncCommand: selectiveIgnoredChecker(new Set([ + "logs/live-supervisor/test", + "config/tester-testnet.json", + ])) })).toThrow("Refusing to use non-ignored Bot config path: tracked-bot.json"); + }); + + it("refuses non-ignored default config paths", () => { + const args = parseArgs(["--out-dir", "logs/live-supervisor/test"]); + + expect(() => resolvePlan(args, "/repo", { spawnSyncCommand: selectiveIgnoredChecker(new Set([ + "logs/live-supervisor/test", + "config/tester-testnet.json", + ])) })).toThrow("Refusing to use non-ignored Bot config path: config/bot-testnet.json"); + }); + + it("requires built runtime outputs before live actor spawn", async () => { + const args = parseArgs([ + "--out-dir", "logs/live-supervisor/missing-build-test", + "--scenario", "bot-only", + "--target-outcome", "bot_match_committed", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: selectiveIgnoredChecker(new Set([ + "logs/live-supervisor/missing-build-test", + "config/bot-testnet.json", + "config/tester-testnet.json", + ])) }); + let spawned = false; + let createdOutputDirectory = false; + + await expect(supervise(args, plan, { + existsSync: (path) => !pathToString(path).endsWith("packages/core/node_modules/@ckb-ccc/udt/dist/index.js"), + spawnCommand: (() => { + spawned = true; + return fakeChild(""); + }) as never, + stat: missingStat, + mkdir: () => { + createdOutputDirectory = true; + return Promise.resolve(undefined); + }, + })).rejects.toThrow("Missing built CCC UDT: packages/core/node_modules/@ckb-ccc/udt/dist/index.js"); + expect(spawned).toBe(false); + expect(createdOutputDirectory).toBe(false); + }); +}); + +describe("evidence parsing", () => { + it("keeps JSON records, discards banners, and flags malformed JSON", () => { + const evidence = parseJsonEvidence([ + "> package banner", + JSON.stringify({ app: "bot", type: "bot.run.started" }), + "{not-json}", + "", + ].join("\n")); + + expect(evidence.records).toHaveLength(1); + expect(evidence.ignoredLines).toEqual(["> package banner"]); + expect(evidence.malformedLines).toEqual(["{not-json}"]); + }); + + it("parses pretty preflight JSON as one report", () => { + const evidence = parsePreflightEvidence(JSON.stringify({ chain: "testnet" }, null, 2)); + + expect(evidence.records).toEqual([{ chain: "testnet" }]); + expect(evidence.malformedLines).toEqual([]); + }); +}); + +describe("classification", () => { + it("classifies tester order creation", () => { + const result = commandResult("tester", JSON.stringify({ + startTime: "now", + actions: { newOrder: { giveCkb: "10", takeIckb: "9", fee: "0.1" }, cancelledOrders: 0 }, + txHash: txHash("11"), + ElapsedSeconds: 1, + })); + + expect(classifyActorResult("tester", result).outcome).toBe("tester_order_created"); + }); + + it("classifies tester direct conversions separately from order creation", () => { + const result = commandResult("tester", JSON.stringify({ + startTime: "now", + actions: { conversion: { kind: "direct" }, cancelledOrders: 0 }, + txHash: txHash("12"), + ElapsedSeconds: 1, + })); + + expect(classifyActorResult("tester", result).outcome).toBe("tester_conversion_created"); + }); + + it("classifies tester hybrid direct-plus-order conversions as conversion coverage", () => { + const result = commandResult("tester", JSON.stringify({ + startTime: "now", + actions: { + conversion: { kind: "direct-plus-order" }, + newOrder: { giveCkb: "10", takeIckb: "9", fee: "0.1" }, + cancelledOrders: 0, + }, + txHash: txHash("13"), + ElapsedSeconds: 1, + })); + + expect(classifyActorResult("tester", result).outcome).toBe("tester_conversion_created"); + }); + + it("classifies tester SDK order conversions as conversion coverage", () => { + const result = commandResult("tester", JSON.stringify({ + startTime: "now", + actions: { + conversion: { kind: "order" }, + newOrder: { giveCkb: "10", takeIckb: "9", fee: "0.1" }, + cancelledOrders: 0, + }, + txHash: txHash("14"), + ElapsedSeconds: 1, + })); + + expect(classifyActorResult("tester", result).outcome).toBe("tester_conversion_created"); + }); + + it("accepts explicit iCKB-to-CKB tester scenario evidence", () => { + const result = commandResult("tester", JSON.stringify({ + startTime: "now", + actions: { + testerScenario: "ickb-to-ckb-limit-order", + newOrder: { giveIckb: "10", takeCkb: "9", fee: "0.1" }, + cancelledOrders: 0, + }, + txHash: txHash("15"), + ElapsedSeconds: 1, + })); + + expect(classifyActorResult("tester", result, { scenario: "ickb-to-ckb-limit-order" })).toMatchObject({ + outcome: "tester_order_created", + terminal: false, + }); + }); + + it("accepts either order direction for explicit random-order evidence", () => { + const result = commandResult("tester", JSON.stringify({ + startTime: "now", + actions: { + testerScenario: "random-order", + newOrder: { giveIckb: "10", takeCkb: "9", fee: "0.1" }, + cancelledOrders: 0, + }, + txHash: txHash("18"), + ElapsedSeconds: 1, + })); + + expect(classifyActorResult("tester", result, { scenario: "random-order" })).toMatchObject({ + outcome: "tester_order_created", + terminal: false, + }); + }); + + it("accepts explicit two-order CKB-to-iCKB tester scenario evidence", () => { + const result = commandResult("tester", JSON.stringify({ + startTime: "now", + actions: { + testerScenario: "two-ckb-to-ickb-limit-orders", + newOrders: [ + { giveCkb: "10", takeIckb: "9", fee: "0.1" }, + { giveCkb: "20", takeIckb: "18", fee: "0.2" }, + ], + orderCount: 2, + cancelledOrders: 0, + }, + txHash: txHash("19"), + ElapsedSeconds: 1, + })); + + expect(classifyActorResult("tester", result, { scenario: "two-ckb-to-ickb-limit-orders" })).toMatchObject({ + outcome: "tester_order_created", + terminal: false, + }); + }); + + it("accepts any concrete multi-order evidence for generic multi-order tester expectations", () => { + const mixedResult = commandResult("tester", JSON.stringify({ + startTime: "now", + actions: { + requestedTesterScenario: "multi-order-limit-orders", + testerScenario: "mixed-direction-limit-orders", + newOrders: [ + { giveCkb: "10", takeIckb: "9", fee: "0.1" }, + { giveIckb: "20", takeCkb: "18", fee: "0.2" }, + ], + orderCount: 2, + cancelledOrders: 0, + }, + txHash: txHash("20"), + ElapsedSeconds: 1, + })); + const ckbResult = commandResult("tester", JSON.stringify({ + startTime: "now", + actions: { + requestedTesterScenario: "multi-order-limit-orders", + testerScenario: "two-ckb-to-ickb-limit-orders", + newOrders: [ + { giveCkb: "10", takeIckb: "9", fee: "0.1" }, + { giveCkb: "20", takeIckb: "18", fee: "0.2" }, + ], + orderCount: 2, + cancelledOrders: 0, + }, + txHash: txHash("21"), + ElapsedSeconds: 1, + })); + const ickbResult = commandResult("tester", JSON.stringify({ + startTime: "now", + actions: { + requestedTesterScenario: "multi-order-limit-orders", + testerScenario: "two-ickb-to-ckb-limit-orders", + newOrders: [ + { giveIckb: "10", takeCkb: "9", fee: "0.1" }, + { giveIckb: "20", takeCkb: "18", fee: "0.2" }, + ], + orderCount: 2, + cancelledOrders: 0, + }, + txHash: txHash("22"), + ElapsedSeconds: 1, + })); + + expect(classifyActorResult("tester", mixedResult, { scenario: "multi-order-limit-orders" })).toMatchObject({ + outcome: "tester_order_created", + terminal: false, + }); + expect(classifyActorResult("tester", ckbResult, { scenario: "multi-order-limit-orders" })).toMatchObject({ + outcome: "tester_order_created", + terminal: false, + }); + expect(classifyActorResult("tester", ickbResult, { scenario: "multi-order-limit-orders" })).toMatchObject({ + outcome: "tester_order_created", + terminal: false, + }); + }); + + it("rejects non-multi-order evidence for generic multi-order tester expectations", () => { + const result = commandResult("tester", JSON.stringify({ + startTime: "now", + actions: { + testerScenario: "random-order", + newOrders: [ + { giveCkb: "10", takeIckb: "9", fee: "0.1" }, + { giveIckb: "20", takeCkb: "18", fee: "0.2" }, + ], + orderCount: 2, + cancelledOrders: 0, + }, + txHash: txHash("23"), + ElapsedSeconds: 1, + })); + + expect(classifyActorResult("tester", result, { scenario: "multi-order-limit-orders" })).toMatchObject({ + outcome: "tester_deterministic_pre_broadcast_error", + terminal: true, + reason: "tester scenario multi-order-limit-orders committed with non-multi-order selected scenario evidence", + }); + }); + + it("rejects concrete direction mismatches for generic multi-order tester expectations", () => { + const result = commandResult("tester", JSON.stringify({ + startTime: "now", + actions: { + requestedTesterScenario: "multi-order-limit-orders", + testerScenario: "mixed-direction-limit-orders", + newOrders: [ + { giveCkb: "10", takeIckb: "9", fee: "0.1" }, + { giveCkb: "20", takeIckb: "18", fee: "0.2" }, + ], + orderCount: 2, + cancelledOrders: 0, + }, + txHash: txHash("24"), + ElapsedSeconds: 1, + })); + + expect(classifyActorResult("tester", result, { scenario: "multi-order-limit-orders" })).toMatchObject({ + outcome: "tester_deterministic_pre_broadcast_error", + terminal: true, + reason: "tester scenario multi-order-limit-orders committed without mixed order direction evidence", + }); + }); + + it("rejects wrong order count evidence for explicit two-order scenarios", () => { + const result = commandResult("tester", JSON.stringify({ + startTime: "now", + actions: { + testerScenario: "two-ckb-to-ickb-limit-orders", + newOrders: [{ giveCkb: "10", takeIckb: "9", fee: "0.1" }], + orderCount: 1, + cancelledOrders: 0, + }, + txHash: txHash("1a"), + ElapsedSeconds: 1, + })); + + expect(classifyActorResult("tester", result, { scenario: "two-ckb-to-ickb-limit-orders" })).toMatchObject({ + outcome: "tester_deterministic_pre_broadcast_error", + terminal: true, + reason: "tester scenario two-ckb-to-ickb-limit-orders committed without 2 new order evidence entries", + }); + }); + + it("accepts explicit two-order iCKB-to-CKB tester scenario evidence", () => { + const result = commandResult("tester", JSON.stringify({ + startTime: "now", + actions: { + testerScenario: "two-ickb-to-ckb-limit-orders", + newOrders: [ + { giveIckb: "10", takeCkb: "9", fee: "0.1" }, + { giveIckb: "20", takeCkb: "18", fee: "0.2" }, + ], + orderCount: 2, + cancelledOrders: 0, + }, + txHash: txHash("1b"), + ElapsedSeconds: 1, + })); + + expect(classifyActorResult("tester", result, { scenario: "two-ickb-to-ckb-limit-orders" })).toMatchObject({ + outcome: "tester_order_created", + terminal: false, + }); + }); + + it("rejects wrong order direction evidence for explicit two-order iCKB-to-CKB scenarios", () => { + const result = commandResult("tester", JSON.stringify({ + startTime: "now", + actions: { + testerScenario: "two-ickb-to-ckb-limit-orders", + newOrders: [ + { giveCkb: "10", takeIckb: "9", fee: "0.1" }, + { giveCkb: "20", takeIckb: "18", fee: "0.2" }, + ], + orderCount: 2, + cancelledOrders: 0, + }, + txHash: txHash("1c"), + ElapsedSeconds: 1, + })); + + expect(classifyActorResult("tester", result, { scenario: "two-ickb-to-ckb-limit-orders" })).toMatchObject({ + outcome: "tester_deterministic_pre_broadcast_error", + terminal: true, + reason: "tester scenario two-ickb-to-ckb-limit-orders committed with wrong order direction evidence", + }); + }); + + it("accepts explicit mixed-direction tester scenario evidence", () => { + const result = commandResult("tester", JSON.stringify({ + startTime: "now", + actions: { + testerScenario: "mixed-direction-limit-orders", + newOrders: [ + { giveCkb: "10", takeIckb: "9", fee: "0.1" }, + { giveIckb: "20", takeCkb: "18", fee: "0.2" }, + ], + orderCount: 2, + cancelledOrders: 0, + }, + txHash: txHash("1d"), + ElapsedSeconds: 1, + })); + + expect(classifyActorResult("tester", result, { scenario: "mixed-direction-limit-orders" })).toMatchObject({ + outcome: "tester_order_created", + terminal: false, + }); + }); + + it("rejects same-direction evidence for explicit mixed-direction scenarios", () => { + const result = commandResult("tester", JSON.stringify({ + startTime: "now", + actions: { + testerScenario: "mixed-direction-limit-orders", + newOrders: [ + { giveCkb: "10", takeIckb: "9", fee: "0.1" }, + { giveCkb: "20", takeIckb: "18", fee: "0.2" }, + ], + orderCount: 2, + cancelledOrders: 0, + }, + txHash: txHash("1e"), + ElapsedSeconds: 1, + })); + + expect(classifyActorResult("tester", result, { scenario: "mixed-direction-limit-orders" })).toMatchObject({ + outcome: "tester_deterministic_pre_broadcast_error", + terminal: true, + reason: "tester scenario mixed-direction-limit-orders committed without mixed order direction evidence", + }); + }); + + it("rejects ambiguous mixed-direction order evidence", () => { + const result = commandResult("tester", JSON.stringify({ + startTime: "now", + actions: { + testerScenario: "mixed-direction-limit-orders", + newOrders: [ + { giveCkb: "10", takeIckb: "9", giveIckb: "20", takeCkb: "18", fee: "0.1" }, + "not-an-order", + ], + orderCount: 2, + cancelledOrders: 0, + }, + txHash: txHash("1f"), + ElapsedSeconds: 1, + })); + + expect(classifyActorResult("tester", result, { scenario: "mixed-direction-limit-orders" })).toMatchObject({ + outcome: "tester_deterministic_pre_broadcast_error", + terminal: true, + reason: "tester scenario mixed-direction-limit-orders committed without mixed order direction evidence", + }); + }); + + it("rejects committed tester evidence that does not match the explicit scenario", () => { + const result = commandResult("tester", JSON.stringify({ + startTime: "now", + actions: { + testerScenario: "ickb-to-ckb-limit-order", + newOrder: { giveCkb: "10", takeIckb: "9", fee: "0.1" }, + cancelledOrders: 0, + }, + txHash: txHash("16"), + ElapsedSeconds: 1, + })); + + expect(classifyActorResult("tester", result, { scenario: "ickb-to-ckb-limit-order" })).toMatchObject({ + outcome: "tester_deterministic_pre_broadcast_error", + terminal: true, + reason: "tester scenario ickb-to-ckb-limit-order committed with wrong order direction evidence", + }); + }); + + it("requires conversion evidence for explicit SDK conversion scenarios", () => { + const result = commandResult("tester", JSON.stringify({ + startTime: "now", + actions: { + testerScenario: "sdk-conversion", + newOrder: { giveCkb: "10", takeIckb: "9", fee: "0.1" }, + cancelledOrders: 0, + }, + txHash: txHash("17"), + ElapsedSeconds: 1, + })); + + expect(classifyActorResult("tester", result, { scenario: "sdk-conversion" })).toMatchObject({ + outcome: "tester_deterministic_pre_broadcast_error", + terminal: true, + reason: "tester scenario sdk-conversion committed without conversion evidence", + }); + }); + + it("classifies tester fresh-order and sampled-too-small skips", () => { + expect(classifyActorResult("tester", commandResult("tester", JSON.stringify({ + skip: { reason: "fresh-matchable-order", txHash: txHash("22") }, + }))).outcome).toBe("tester_fresh_order_skip"); + expect(classifyActorResult("tester", commandResult("tester", JSON.stringify({ + skip: { reason: "sampled-amount-too-small" }, + }))).outcome).toBe("tester_sampled_too_small_skip"); + expect(classifyActorResult("tester", commandResult("tester", JSON.stringify({ + skip: { reason: "post-tx-ckb-reserve" }, + })))).toMatchObject({ + outcome: "tester_reserve_skip", + terminal: false, + skipReason: "post-tx-ckb-reserve", + }); + }); + + it("classifies bot committed actions", () => { + const stdout = [ + botEvent("bot.state.read", { + orders: { marketCount: 4, userCount: 0, receiptCount: 1 }, + poolDeposits: { readyCount: 2, nearReadyCount: 1, futureCount: 3 }, + }), + botEvent("bot.transaction.built", { + actions: { collectedOrders: 0, completedDeposits: 0, matchedOrders: 1, deposits: 1, withdrawalRequests: 0, withdrawals: 0 }, + }), + botEvent("bot.transaction.committed", { txHash: txHash("33"), status: "committed" }), + ].map(JSON.stringify).join("\n"); + const classification = classifyActorResult("bot", commandResult("bot", stdout)); + + expect(classification.outcome).toBe("bot_match_plus_deposit_committed"); + expect(classification.txHashes).toEqual([txHash("33")]); + expect(classification.publicState).toEqual({ + marketOrderCount: 4, + userOrderCount: 0, + receiptCount: 1, + readyPoolDepositCount: 2, + nearReadyPoolDepositCount: 1, + futurePoolDepositCount: 3, + }); + }); + + it("classifies bot no-action and low-capital skips", () => { + expect(classifyActorResult("bot", commandResult("bot", JSON.stringify(botEvent("bot.decision.skipped", { + reason: "no_actions", + actions: emptyActions(), + })))).outcome).toBe("bot_no_action_skip"); + expect(classifyActorResult("bot", commandResult("bot", JSON.stringify(botEvent("bot.decision.skipped", { + reason: "capital_below_minimum", + actions: emptyActions(), + })))).outcome).toBe("low_capital_stop"); + }); + + it("keeps bot low-capital safety stops classified despite exit code 2", () => { + const result = { + ...commandResult("bot", JSON.stringify(botEvent("bot.decision.skipped", { + reason: "capital_below_minimum", + actions: emptyActions(), + }))), + status: 2, + }; + + expect(classifyActorResult("bot", result)).toMatchObject({ + outcome: "low_capital_stop", + terminal: true, + skipReason: "capital_below_minimum", + }); + }); + + it("treats nonzero actor exits as terminal even when stdout has success evidence", () => { + const botResult = { + ...commandResult("bot", JSON.stringify(botEvent("bot.decision.skipped", { + reason: "no_actions", + actions: emptyActions(), + }))), + status: 1, + }; + const testerResult = { + ...commandResult("tester", JSON.stringify({ txHash: txHash("99") })), + status: 1, + }; + + expect(classifyActorResult("bot", botResult)).toMatchObject({ + outcome: "nonzero_exit", + terminal: true, + }); + expect(classifyActorResult("tester", testerResult)).toMatchObject({ + outcome: "nonzero_exit", + terminal: true, + }); + }); + + it("keeps tester confirmation timeouts classified by safety evidence despite exit code 2", () => { + const result = { + ...commandResult("tester", JSON.stringify({ + txHash: txHash("aa"), + error: { + name: "TransactionConfirmationError", + message: "Transaction confirmation timed out", + txHash: txHash("aa"), + status: "sent", + }, + })), + status: 2, + }; + + expect(classifyActorResult("tester", result)).toMatchObject({ + outcome: "confirmation_timeout", + terminal: true, + }); + }); + + it("classifies serialized tester funding failures as low-capital stops", () => { + const result = { + ...commandResult("tester", JSON.stringify({ + error: { + name: "TesterTerminalError", + message: "Not enough CKB for all-CKB limit order scenario", + }, + })), + status: 1, + }; + + expect(classifyActorResult("tester", result)).toMatchObject({ + outcome: "low_capital_stop", + terminal: true, + }); + }); + + it("safety classifications override ordinary exits", () => { + expect(classifyActorResult("bot", commandResult("bot", "{not-json}"))).toMatchObject({ + outcome: "malformed_evidence", + terminal: true, + }); + expect(classifyActorResult("bot", commandResult("bot", JSON.stringify({ privateKey: "0xsecret" })))).toMatchObject({ + outcome: "secret_leak_sentinel", + terminal: true, + }); + expect(classifyActorResult("bot", { ...commandResult("bot", ""), timedOut: true })).toMatchObject({ + outcome: "command_timeout", + terminal: true, + }); + }); + + it("classifies terminal preflight safety failures before launch", () => { + expect(classifyActorResult("preflight", { ...commandResult("preflight", ""), timedOut: true })).toMatchObject({ + outcome: "command_timeout", + terminal: true, + }); + expect(classifyActorResult("preflight", commandResult("preflight", JSON.stringify({ + privateKey: "0xsecret", + })))).toMatchObject({ + outcome: "secret_leak_sentinel", + terminal: true, + }); + }); + + it("sanitizes transaction-shaped preflight errors before classification reaches artifacts", () => { + const classification = classifyActorResult("preflight", { + ...commandResult("preflight", "{}"), + status: 1, + stderr: JSON.stringify({ witnesses: ["0xsignature"], inputs: [] }), + }); + + expect(classification.reason).toBe("\n"); + }); + + it("withholds secret-shaped raw artifacts", () => { + expect(safeArtifactText(JSON.stringify({ privateKey: "0xsecret" }))).toBe( + "\n", + ); + expect(safeArtifactText(`PRIVATE_KEY=0x${"11".repeat(32)}`)).toBe( + "\n", + ); + expect(safeArtifactText("SEED_PHRASE=alpha beta gamma")).toBe( + "\n", + ); + expect(safeArtifactText("RPC_URL=https://user:pass@testnet.example/path?token=secret")).toBe( + "\n", + ); + expect(safeArtifactText(JSON.stringify({ witnesses: ["0xsignature"], inputs: [] }))).toBe( + "\n", + ); + expect(safeArtifactText(JSON.stringify({ system: { tip: { hash: txHash("aa") } } }))).toBe( + JSON.stringify({ system: { tip: { hash: txHash("aa") } } }), + ); + expect(safeArtifactText(JSON.stringify({ app: "bot" }))).toBe(JSON.stringify({ app: "bot" })); + }); + + it("caps captured command output", () => { + const capture = createBoundedOutputCapture(); + appendBoundedOutput(capture, Buffer.from("abcdef"), 4); + appendBoundedOutput(capture, Buffer.from("gh"), 4); + + expect(boundedOutputText(capture)).toBe("abcd\n"); + }); + + it("kills timed-out actor commands after the grace period", async () => { + vi.useFakeTimers(); + try { + const writes = new Map(); + const kills: Array<{ pid: number; signal: NodeJS.Signals }> = []; + const args = parseArgs([ + "--out-dir", "logs/live-supervisor/timeout-kill-test", + "--scenario", "bot-only", + "--target-outcome", "bot_match_committed", + "--max-cycles", "1", + "--command-timeout-seconds", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: selectiveIgnoredChecker(new Set([ + "logs/live-supervisor/timeout-kill-test", + "config/bot-testnet.json", + "config/tester-testnet.json", + ])) }); + const child = fakeHangingChild(); + + const run = supervise(args, plan, { + skipBuiltRuntimeCheck: true, + commandKillGraceMs: 10, + killProcess: (pid, signal) => { + kills.push({ pid, signal }); + if (signal === "SIGKILL") { + queueMicrotask(() => child.emit("close", null, "SIGKILL")); + } + }, + spawnCommand: ((_command: string, commandArgs: string[]) => isPreflightCommand(commandArgs) ? fakeSuccessfulPreflightChild() : child) as never, + spawnSyncCommand: ignoredChecker(true) as never, + lstat: missingStat, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + realpath: (path) => Promise.resolve(pathToString(path)), + appendFile: (path, text) => { + const key = pathToString(path); + writes.set(key, `${writes.get(key) ?? ""}${textToString(text)}`); + return Promise.resolve(); + }, + writeFile: (path, text) => { + writes.set(pathToString(path), textToString(text)); + return Promise.resolve(); + }, + }); + + await vi.advanceTimersByTimeAsync(1010); + + await expect(run).resolves.toBe(2); + expect(kills).toEqual([ + { pid: -1234, signal: "SIGTERM" }, + { pid: -1234, signal: "SIGKILL" }, + ]); + expect(writes.get("/repo/logs/live-supervisor/timeout-kill-test/cycle-0001-incident.json")).toContain( + "command_timeout", + ); + } finally { + vi.useRealTimers(); + } + }); +}); + +describe("scenario planning", () => { + it("prefers under-covered safe outcomes", () => { + const ledger = createCoverageLedger(["tester_order_created", "bot_match_committed"]); + recordOutcome(ledger, "tester_order_created"); + + const choice = chooseScenario({ scenario: "auto", targetOutcomes: [] }, ledger); + + expect(choice).toMatchObject({ + kind: "scenario", + scenario: { name: "bot-only" }, + targetOutcomes: ["bot_match_committed"], + }); + }); + + it("rotates past attempted uncovered outcomes before retrying them", () => { + const ledger = createCoverageLedger(["tester_order_created", "bot_no_action_skip"]); + const first = chooseScenario({ scenario: "auto", targetOutcomes: [] }, ledger); + recordScenarioAttempt(ledger, 1, first); + + const second = chooseScenario({ scenario: "auto", targetOutcomes: [] }, ledger); + + expect(first).toMatchObject({ + kind: "scenario", + targetOutcomes: ["tester_order_created"], + }); + expect(second).toMatchObject({ + kind: "scenario", + scenario: { name: "bot-only" }, + targetOutcomes: ["bot_no_action_skip"], + }); + }); + + it("prefers unattempted scenarios before retrying the same scenario for a new target", () => { + const ledger = createCoverageLedger(["tester_order_created", "tester_sampled_too_small_skip"]); + const first = chooseScenario({ scenario: "auto", targetOutcomes: [] }, ledger); + recordScenarioAttempt(ledger, 1, first); + + const second = chooseScenario({ scenario: "auto", targetOutcomes: [] }, ledger); + + expect(second).toMatchObject({ + kind: "scenario", + scenario: { name: "standard-cycle" }, + targetOutcomes: ["tester_sampled_too_small_skip"], + }); + }); + + it("reports unsupported explicit scenario goals", () => { + const ledger = createCoverageLedger(["wrong_chain"]); + const choice = chooseScenario({ scenario: "bot-only", targetOutcomes: ["wrong_chain"] }, ledger); + + expect(choice).toMatchObject({ + kind: "unsupported", + requested: "wrong_chain", + }); + }); + + it("uses explicit scenario targets when no target outcome is supplied", () => { + const ledger = createCoverageLedger(["tester_order_created", "bot_match_committed"]); + const choice = chooseScenario({ scenario: "bot-only", targetOutcomes: [] }, ledger); + + expect(choice).toMatchObject({ + kind: "scenario", + scenario: { name: "bot-only" }, + targetOutcomes: ["bot_no_action_skip"], + }); + }); + + it("records unsupported explicit goals as full terminal classifications", async () => { + const writes = new Map(); + const args = parseArgs([ + "--out-dir", "logs/live-supervisor/unsupported-test", + "--scenario", "bot-only", + "--target-outcome", "wrong_chain", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: selectiveIgnoredChecker(new Set([ + "logs/live-supervisor/unsupported-test", + "config/bot-testnet.json", + "config/tester-testnet.json", + ])) }); + + const exitCode = await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + writeFile: (path, text) => { + writes.set(pathToString(path), textToString(text)); + return Promise.resolve(); + }, + }); + + expect(exitCode).toBe(2); + const incident = jsonArtifact(writes, "/repo/logs/live-supervisor/unsupported-test/cycle-0001-incident.json"); + const classification = recordAt(incident["classification"], "incident classification"); + expect(classification).toMatchObject({ actor: "preflight", outcome: "unknown", terminal: true }); + expect(classification["txHashes"]).toEqual([]); + expect(recordAt(classification["evidence"], "incident classification evidence")).toMatchObject({ + recordsAccepted: 0, + ignoredLineCount: 0, + malformedLineCount: 0, + exitStatus: null, + signal: null, + timedOut: false, + }); + const summary = jsonArtifact(writes, "/repo/logs/live-supervisor/unsupported-test/summary.json"); + expect(recordAt(summary["aggregateCounts"], "summary aggregate counts")).toMatchObject({ unknown: 1 }); + }); + + it("plans remaining target outcomes after earlier targets are covered", () => { + const ledger = createCoverageLedger(["bot_no_action_skip", "bot_match_committed"]); + recordOutcome(ledger, "bot_no_action_skip"); + const choice = chooseScenario({ + scenario: "auto", + targetOutcomes: ["bot_no_action_skip", "bot_match_committed"], + }, ledger); + + expect(choice).toMatchObject({ + kind: "scenario", + targetOutcomes: ["bot_match_committed"], + }); + }); +}); + +describe("deterministic incident handling", () => { + it("treats unmet explicit target outcomes at max cycles as logical incidents", async () => { + const writes = new Map(); + const spawned: Array<{ command: string; args: string[] }> = []; + const args = parseArgs([ + "--bot-config", "config/bot-testnet.json", + "--tester-config", "config/tester-testnet.json", + "--out-dir", "logs/live-supervisor/unmet-coverage-test", + "--scenario", "bot-only", + "--target-outcome", "bot_match_committed", + "--max-cycles", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) }); + + const exitCode = await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + spawnCommand: ((command: string, commandArgs: string[]) => { + spawned.push({ command, args: commandArgs }); + if (isPreflightCommand(commandArgs)) { + return fakeSuccessfulPreflightChild(); + } + return fakeChild(JSON.stringify(botEvent("bot.decision.skipped", { + reason: "no_actions", + actions: emptyActions(), + }))); + }) as never, + spawnSyncCommand: ignoredChecker(true) as never, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + appendFile: (path, text) => { + const key = pathToString(path); + writes.set(key, `${writes.get(key) ?? ""}${textToString(text)}`); + return Promise.resolve(); + }, + writeFile: (path, text) => { + writes.set(pathToString(path), textToString(text)); + return Promise.resolve(); + }, + }); + + expect(exitCode).toBe(2); + expect(spawned.filter((item) => !isPreflightCommand(item.args)).map((item) => item.command)).toEqual([process.execPath]); + const incident = jsonArtifact(writes, "/repo/logs/live-supervisor/unmet-coverage-test/cycle-0001-incident.json"); + expect(incident).toMatchObject({ unmetGoals: ["bot_match_committed"] }); + const incidentArtifacts = stringArrayAt(incident["artifacts"], "incident artifacts"); + expect(incidentArtifacts).toContain("logs/live-supervisor/unmet-coverage-test/cycle-0001-bot.stdout.ndjson"); + expect(incidentArtifacts).toContain("logs/live-supervisor/unmet-coverage-test/cycle-0001-bot.stderr.log"); + expect(incidentArtifacts).toContain("logs/live-supervisor/unmet-coverage-test/cycle-0001-bot.command.json"); + const summary = jsonArtifact(writes, "/repo/logs/live-supervisor/unmet-coverage-test/summary.json"); + expect(summary).toMatchObject({ stopped: "unmet_coverage_goal" }); + const summaryArtifacts = stringArrayAt(summary["artifacts"], "summary artifacts"); + expect(summaryArtifacts).toContain("logs/live-supervisor/unmet-coverage-test/cycle-0001-bot.stdout.ndjson"); + expect(summaryArtifacts).toContain("logs/live-supervisor/unmet-coverage-test/cycle-0001-bot.stderr.log"); + expect(summaryArtifacts).toContain("logs/live-supervisor/unmet-coverage-test/cycle-0001-bot.command.json"); + expect([...writes.keys()].some((path) => path.includes(".llm"))).toBe(false); + expect(summary).not.toHaveProperty("llmWorker"); + }); + + it("treats repeated target outcomes as one explicit contract", async () => { + const writes = new Map(); + const args = parseArgs([ + "--bot-config", "config/bot-testnet.json", + "--tester-config", "config/tester-testnet.json", + "--out-dir", "logs/live-supervisor/mixed-contract-test", + "--scenario", "bot-only", + "--target-outcome", "bot_no_action_skip", + "--target-outcome", "bot_match_committed", + "--max-cycles", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) }); + + const exitCode = await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + spawnCommand: ((_command: string, commandArgs: string[]) => isPreflightCommand(commandArgs) ? fakeSuccessfulPreflightChild() : fakeChild(JSON.stringify(botEvent("bot.decision.skipped", { + reason: "no_actions", + actions: emptyActions(), + })))) as never, + spawnSyncCommand: ignoredChecker(true) as never, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + appendFile: (path, text) => { + const key = pathToString(path); + writes.set(key, `${writes.get(key) ?? ""}${textToString(text)}`); + return Promise.resolve(); + }, + writeFile: (path, text) => { + writes.set(pathToString(path), textToString(text)); + return Promise.resolve(); + }, + }); + + expect(exitCode).toBe(2); + expect(writes.get("/repo/logs/live-supervisor/mixed-contract-test/cycle-0001-incident.json")).toContain("bot_match_committed"); + expect(writes.get("/repo/logs/live-supervisor/mixed-contract-test/summary.json")).toContain("unmet_coverage_goal"); + }); + + it("uses explicit target outcomes as coverage ledger goals", async () => { + const writes = new Map(); + const args = parseArgs([ + "--bot-config", "config/bot-testnet.json", + "--tester-config", "config/tester-testnet.json", + "--out-dir", "logs/live-supervisor/target-ledger-test", + "--scenario", "tester-only", + "--target-outcome", "tester_estimated_too_small_skip", + "--max-cycles", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) }); + + const exitCode = await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + spawnCommand: ((_command: string, commandArgs: string[]) => isPreflightCommand(commandArgs) ? fakeSuccessfulPreflightChild() : fakeChild(JSON.stringify({ + startTime: "now", + actions: { newOrder: { giveCkb: "10", takeIckb: "9", fee: "0.1" }, cancelledOrders: 0 }, + txHash: txHash("66"), + ElapsedSeconds: 1, + }))) as never, + spawnSyncCommand: ignoredChecker(true) as never, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + appendFile: (path, text) => { + const key = pathToString(path); + writes.set(key, `${writes.get(key) ?? ""}${textToString(text)}`); + return Promise.resolve(); + }, + writeFile: (path, text) => { + writes.set(pathToString(path), textToString(text)); + return Promise.resolve(); + }, + }); + + expect(exitCode).toBe(2); + const summary = jsonArtifact(writes, "/repo/logs/live-supervisor/target-ledger-test/summary.json"); + expect(recordAt(summary["coverage"], "coverage")["goals"]).toEqual(["tester_estimated_too_small_skip"]); + }); + + it("keeps successful preflight probes out of aggregate outcome counts", async () => { + const writes = new Map(); + const args = parseArgs([ + "--out-dir", "logs/live-supervisor/preflight-summary-test", + "--scenario", "bot-only", + "--target-outcome", "bot_no_action_skip", + "--max-cycles", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: selectiveIgnoredChecker(new Set([ + "logs/live-supervisor/preflight-summary-test", + "config/bot-testnet.json", + "config/tester-testnet.json", + ])) }); + + const exitCode = await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + spawnCommand: ((_command: string, commandArgs: string[]) => isPreflightCommand(commandArgs) ? fakeSuccessfulPreflightChild() : fakeChild(JSON.stringify(botEvent("bot.decision.skipped", { + reason: "no_actions", + actions: emptyActions(), + })))) as never, + spawnSyncCommand: ignoredChecker(true) as never, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + appendFile: () => Promise.resolve(), + writeFile: (path, text) => { + writes.set(pathToString(path), textToString(text)); + return Promise.resolve(); + }, + }); + + expect(exitCode).toBe(0); + const summary = jsonArtifact(writes, "/repo/logs/live-supervisor/preflight-summary-test/summary.json"); + expect(recordAt(summary["aggregateCounts"], "summary aggregate counts")).toEqual({ bot_no_action_skip: 1 }); + }); + + it("keeps default coverage goals best-effort at max cycles", async () => { + let spawnCount = 0; + const writes = new Map(); + const args = parseArgs([ + "--out-dir", "logs/live-supervisor/default-coverage-test", + "--max-cycles", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: selectiveIgnoredChecker(new Set([ + "logs/live-supervisor/default-coverage-test", + "config/bot-testnet.json", + "config/tester-testnet.json", + ])) }); + + const exitCode = await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + spawnCommand: ((_command: string, commandArgs: string[]) => { + if (isPreflightCommand(commandArgs)) { + return fakeSuccessfulPreflightChild(); + } + spawnCount += 1; + return fakeChild(spawnCount === 1 + ? JSON.stringify({ + startTime: "now", + actions: { newOrder: { giveCkb: "10", takeIckb: "9", fee: "0.1" }, cancelledOrders: 0 }, + txHash: txHash("55"), + ElapsedSeconds: 1, + }) + : JSON.stringify(botEvent("bot.decision.skipped", { + reason: "no_actions", + actions: emptyActions(), + }))); + }) as never, + spawnSyncCommand: ignoredChecker(true) as never, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + appendFile: (path, text) => { + const key = pathToString(path); + writes.set(key, `${writes.get(key) ?? ""}${textToString(text)}`); + return Promise.resolve(); + }, + writeFile: (path, text) => { + writes.set(pathToString(path), textToString(text)); + return Promise.resolve(); + }, + }); + + expect(exitCode).toBe(0); + }); + + it("treats stop-after-tx-count as a successful operator stop", async () => { + const writes = new Map(); + const args = parseArgs([ + "--bot-config", "config/bot-testnet.json", + "--tester-config", "config/tester-testnet.json", + "--out-dir", "logs/live-supervisor/stop-after-tx-test", + "--scenario", "tester-only", + "--target-outcome", "tester_fresh_order_skip", + "--stop-after-tx-count", "1", + "--max-cycles", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) }); + + const exitCode = await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + spawnCommand: ((_command: string, commandArgs: string[]) => isPreflightCommand(commandArgs) ? fakeSuccessfulPreflightChild() : fakeChild(JSON.stringify({ + startTime: "now", + actions: { newOrder: { giveCkb: "10", takeIckb: "9", fee: "0.1" }, cancelledOrders: 0 }, + txHash: txHash("44"), + ElapsedSeconds: 1, + }))) as never, + spawnSyncCommand: ignoredChecker(true) as never, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + appendFile: (path, text) => { + const key = pathToString(path); + writes.set(key, `${writes.get(key) ?? ""}${textToString(text)}`); + return Promise.resolve(); + }, + writeFile: (path, text) => { + writes.set(pathToString(path), textToString(text)); + return Promise.resolve(); + }, + }); + + expect(exitCode).toBe(0); + expect([...writes.keys()].some((path) => path.endsWith("incident.json"))).toBe(false); + expect(writes.get("/repo/logs/live-supervisor/stop-after-tx-test/summary.json")).toContain("stop_after_tx_count"); + }); + + it("does not count skip reference hashes toward stop-after-tx-count", async () => { + const writes = new Map(); + const args = parseArgs([ + "--bot-config", "config/bot-testnet.json", + "--tester-config", "config/tester-testnet.json", + "--out-dir", "logs/live-supervisor/skip-hash-stop-test", + "--scenario", "tester-only", + "--target-outcome", "tester_fresh_order_skip", + "--stop-after-tx-count", "1", + "--max-cycles", "1", + ]); + const plan = resolvePlan(args, "/repo", { spawnSyncCommand: ignoredChecker(true) }); + + const exitCode = await supervise(args, plan, { + skipBuiltRuntimeCheck: true, + spawnCommand: ((_command: string, commandArgs: string[]) => isPreflightCommand(commandArgs) ? fakeSuccessfulPreflightChild() : fakeChild(JSON.stringify({ + skip: { reason: "fresh-matchable-order", txHash: txHash("44") }, + }))) as never, + spawnSyncCommand: ignoredChecker(true) as never, + stat: missingStat, + mkdir: () => Promise.resolve(undefined), + appendFile: (path, text) => { + const key = pathToString(path); + writes.set(key, `${writes.get(key) ?? ""}${textToString(text)}`); + return Promise.resolve(); + }, + writeFile: (path, text) => { + writes.set(pathToString(path), textToString(text)); + return Promise.resolve(); + }, + }); + + expect(exitCode).toBe(0); + const summary = jsonArtifact(writes, "/repo/logs/live-supervisor/skip-hash-stop-test/summary.json"); + expect(summary).toMatchObject({ stopped: "max_cycles" }); + expect(recordAt(summary["txHashesByOutcome"], "summary tx hashes")).toEqual({ + tester_fresh_order_skip: [txHash("44")], + }); + }); + +}); + +function commandResult(actor: "bot" | "tester" | "preflight", stdout: string): CommandResult { + return { + actor, + command: "fixture", + args: [], + status: 0, + signal: null, + timedOut: false, + stdout: `${stdout}\n`, + stderr: "", + stdoutTruncated: false, + stderrTruncated: false, + elapsedMs: 1, + }; +} + +function botEvent(type: string, fields: Record): Record { + return { + version: 1, + app: "bot", + chain: "testnet", + runId: "test", + iterationId: 1, + timestamp: "2026-01-01T00:00:00.000Z", + type, + ...fields, + }; +} + +function emptyActions(): Record { + return { + collectedOrders: 0, + completedDeposits: 0, + matchedOrders: 0, + deposits: 0, + withdrawalRequests: 0, + withdrawals: 0, + }; +} + +function txHash(byte: string): string { + return `0x${byte.repeat(32)}`; +} + +function ignoredChecker(ignored: boolean): () => { status: number } { + return () => ({ status: ignored ? 0 : 1 }); +} + +function selectiveIgnoredChecker(ignoredPaths: Set): (_command: string, args: string[]) => { status: number } { + return (_command, args) => ({ status: ignoredPaths.has(args.at(-1) ?? "") ? 0 : 1 }); +} + +function isPreflightCommand(args: string[]): boolean { + return args[0] === "scripts/ickb-live-preflight.mjs"; +} + +function fakeSuccessfulPreflightChild(): ReturnType { + return fakeChild(JSON.stringify({ chain: "testnet" })); +} + +function fakeChild(stdout: string, status = 0): EventEmitter & { + stdout: EventEmitter; + stderr: EventEmitter; + kill: () => boolean; +} { + const child = new EventEmitter() as EventEmitter & { + stdout: EventEmitter; + stderr: EventEmitter; + kill: () => boolean; + }; + child.stdout = new EventEmitter(); + child.stderr = new EventEmitter(); + child.kill = (): boolean => true; + queueMicrotask(() => { + child.stdout.emit("data", Buffer.from(`${stdout}\n`)); + child.emit("close", status, null); + }); + return child; +} + +function fakeHangingChild(): EventEmitter & { + stdout: EventEmitter; + stderr: EventEmitter; + pid: number; + kill: (signal?: NodeJS.Signals) => boolean; +} { + const child = new EventEmitter() as EventEmitter & { + stdout: EventEmitter; + stderr: EventEmitter; + pid: number; + kill: (signal?: NodeJS.Signals) => boolean; + }; + child.stdout = new EventEmitter(); + child.stderr = new EventEmitter(); + child.pid = 1234; + child.kill = (signal = "SIGTERM"): boolean => { + if (signal === "SIGKILL") { + queueMicrotask(() => child.emit("close", null, "SIGKILL")); + } + return true; + }; + return child; +} + +function missingStat(): never { + const error = new Error("missing") as NodeJS.ErrnoException; + error.code = "ENOENT"; + throw error; +} + +function jsonArtifact(writes: Map, path: string): Record { + const text = writes.get(path); + if (text === undefined) { + throw new Error(`Missing artifact: ${path}`); + } + const parsed: unknown = JSON.parse(text); + return recordAt(parsed, path); +} + +function recordAt(value: unknown, label: string): Record { + if (typeof value === "object" && value !== null && !Array.isArray(value)) { + return value as Record; + } + throw new Error(`Expected record: ${label}`); +} + +function stringArrayAt(value: unknown, label: string): string[] { + if (Array.isArray(value) && value.every((item) => typeof item === "string")) { + return value; + } + throw new Error(`Expected string array: ${label}`); +} + +function pathToString(path: unknown): string { + if (typeof path === "string") { + return path; + } + if (Buffer.isBuffer(path)) { + return path.toString("utf8"); + } + if (path instanceof URL) { + return path.toString(); + } + throw new TypeError("Unexpected artifact path type"); +} + +function textToString(text: unknown): string { + if (typeof text === "string") { + return text; + } + if (ArrayBuffer.isView(text)) { + return Buffer.from(text.buffer, text.byteOffset, text.byteLength).toString("utf8"); + } + throw new TypeError("Unexpected artifact text type"); +} diff --git a/apps/supervisor/src/index.ts b/apps/supervisor/src/index.ts new file mode 100644 index 0000000..33e6bbd --- /dev/null +++ b/apps/supervisor/src/index.ts @@ -0,0 +1,2047 @@ +import { spawn, spawnSync, type SpawnSyncReturns } from "node:child_process"; +import { existsSync } from "node:fs"; +import { appendFile, lstat, mkdir, realpath, stat, writeFile } from "node:fs/promises"; +import { isAbsolute, join, relative, resolve } from "node:path"; +import process from "node:process"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const repoRoot = fileURLToPath(new URL("../../..", import.meta.url)); +const DEFAULT_COMMAND_TIMEOUT_SECONDS = 15 * 60; +const DEFAULT_COMMAND_KILL_GRACE_MS = 5 * 1000; +const DEFAULT_MAX_OUTPUT_BYTES = 1024 * 1024; +const DEFAULT_BOT_CONFIG_PATH = "config/bot-testnet.json"; +const DEFAULT_TESTER_CONFIG_PATH = "config/tester-testnet.json"; +const STOP_EXIT_CODE = 2; +const TX_HASH_PATTERN = /^0x[0-9a-fA-F]{64}$/u; +const SUPERVISOR_OUTPUT_ROOT = "logs/live-supervisor/"; + +export const OUTCOME_KINDS = [ + "tester_order_created", + "tester_conversion_created", + "tester_fresh_order_skip", + "tester_sampled_too_small_skip", + "tester_estimated_too_small_skip", + "tester_reserve_skip", + "tester_deterministic_pre_broadcast_error", + "bot_no_action_skip", + "bot_reserve_skip", + "bot_match_committed", + "bot_match_plus_deposit_committed", + "bot_receipt_completion_committed", + "bot_deposit_only_committed", + "bot_withdrawal_request_committed", + "bot_withdrawal_completion_committed", + "low_capital_stop", + "confirmation_timeout", + "terminal_chain_rejection", + "post_broadcast_unresolved", + "wrong_chain", + "malformed_evidence", + "secret_leak_sentinel", + "command_timeout", + "nonzero_exit", + "unmet_coverage_goal", + "unknown", +] as const; + +export type OutcomeKind = typeof OUTCOME_KINDS[number]; +export type Actor = "bot" | "tester"; +export type ScenarioName = "auto" | "standard-cycle" | "tester-only" | "bot-only" | "tester-fresh-skip-two-pass"; +export type TesterScenario = "auto" | "random-order" | "sdk-conversion" | "extra-large-limit-order" | "multi-order-limit-orders" | "two-ckb-to-ickb-limit-orders" | "all-ckb-limit-order" | "all-ickb-limit-order" | "ickb-to-ckb-limit-order" | "two-ickb-to-ckb-limit-orders" | "mixed-direction-limit-orders" | "dust-ckb-conversion" | "dust-ickb-conversion"; +type TesterDirection = "ckb-to-ickb" | "ickb-to-ckb"; + +interface ScenarioStep { + actor: Actor; + label?: string; + testerScenario?: TesterScenario; +} + +interface ScenarioDefinition { + name: Exclude; + steps: ScenarioStep[]; + targetOutcomes: OutcomeKind[]; + reason: string; +} + +export interface ParsedArgs { + help: boolean; + dryRun: boolean; + botConfigPath?: string; + testerConfigPath?: string; + outDir?: string; + maxCycles: number; + maxWallClockSeconds?: number; + stopAfterTxCount?: number; + scenario: ScenarioName; + targetOutcomes: OutcomeKind[]; + testerScenario: TesterScenario; + testerScenarioExplicit: boolean; + testerFee: string; + testerFeeBase: string; + testerFeeExplicit: boolean; + testerFeeBaseExplicit: boolean; + commandTimeoutSeconds: number; +} + +export interface SupervisorPlan extends ParsedArgs { + runId: string; + rootDir: string; + botConfigPath?: string; + testerConfigPath?: string; + outDir: string; + relativeOutDir: string; +} + +export interface CommandResult { + actor: Actor | "preflight"; + command: string; + args: string[]; + spawnError?: string; + status: number | null; + signal: NodeJS.Signals | null; + timedOut: boolean; + stdout: string; + stderr: string; + stdoutTruncated: boolean; + stderrTruncated: boolean; + elapsedMs: number; +} + +export interface ParsedEvidence { + records: Record[]; + ignoredLines: string[]; + malformedLines: string[]; +} + +export interface Classification { + actor: Actor | "preflight"; + outcome: OutcomeKind; + terminal: boolean; + reason: string; + txHashes: string[]; + evidence: { + recordsAccepted: number; + ignoredLineCount: number; + malformedLineCount: number; + exitStatus: number | null; + signal: NodeJS.Signals | null; + timedOut: boolean; + }; + actions?: ActionCounts; + skipReason?: string; + publicState?: PublicStateAssumption; +} + +interface TesterEvidenceExpectation { + scenario: TesterScenario; +} + +export interface ActionCounts { + collectedOrders: number; + completedDeposits: number; + matchedOrders: number; + deposits: number; + withdrawalRequests: number; + withdrawals: number; +} + +export interface PublicStateAssumption { + marketOrderCount?: number; + userOrderCount?: number; + receiptCount?: number; + readyPoolDepositCount?: number; + nearReadyPoolDepositCount?: number; + futurePoolDepositCount?: number; +} + +export interface CoverageLedger { + goals: OutcomeKind[]; + counts: Record; + attempts: Array<{ + cycleIndex: number; + scenario: Exclude; + targetOutcomes: OutcomeKind[]; + reason: string; + }>; + unsupported: Array<{ + cycleIndex: number; + requested: OutcomeKind; + reason: string; + }>; +} + +export interface ScenarioChoice { + kind: "scenario"; + scenario: ScenarioDefinition; + targetOutcomes: OutcomeKind[]; + reason: string; +} + +export interface UnsupportedScenarioChoice { + kind: "unsupported"; + requested: OutcomeKind; + reason: string; +} + +type ScenarioChoiceResult = ScenarioChoice | UnsupportedScenarioChoice; + +interface Dependencies { + spawnCommand?: typeof spawn; + spawnSyncCommand?: typeof spawnSync; + now?: () => number; + writeFile?: typeof writeFile; + appendFile?: typeof appendFile; + mkdir?: typeof mkdir; + lstat?: typeof lstat; + realpath?: typeof realpath; + stat?: typeof stat; + existsSync?: typeof existsSync; + skipBuiltRuntimeCheck?: boolean; + maxOutputBytes?: number; + commandKillGraceMs?: number; + killProcess?: (pid: number, signal: NodeJS.Signals) => void; +} + +interface IncidentArtifact { + relativePath: string; + classification: Classification; +} + +export interface BoundedOutputCapture { + chunks: Buffer[]; + byteLength: number; + truncatedBytes: number; +} + +const SCENARIOS: ScenarioDefinition[] = [ + { + name: "standard-cycle", + steps: [{ actor: "tester" }, { actor: "bot" }], + targetOutcomes: [ + "tester_order_created", + "tester_conversion_created", + "tester_fresh_order_skip", + "tester_sampled_too_small_skip", + "tester_estimated_too_small_skip", + "bot_no_action_skip", + "bot_match_committed", + "bot_match_plus_deposit_committed", + "bot_receipt_completion_committed", + "bot_deposit_only_committed", + "bot_withdrawal_request_committed", + "bot_withdrawal_completion_committed", + ], + reason: "run tester then bot so new and existing public state can exercise bot branches", + }, + { + name: "tester-only", + steps: [{ actor: "tester" }], + targetOutcomes: [ + "tester_order_created", + "tester_conversion_created", + "tester_fresh_order_skip", + "tester_sampled_too_small_skip", + "tester_estimated_too_small_skip", + ], + reason: "focus on tester order creation and skip branches without adding an immediate bot mutation", + }, + { + name: "bot-only", + steps: [{ actor: "bot" }], + targetOutcomes: [ + "bot_no_action_skip", + "bot_match_committed", + "bot_match_plus_deposit_committed", + "bot_receipt_completion_committed", + "bot_deposit_only_committed", + "bot_withdrawal_request_committed", + "bot_withdrawal_completion_committed", + ], + reason: "focus on bot behavior against current public iCKB pool and order state", + }, + { + name: "tester-fresh-skip-two-pass", + steps: [ + { actor: "tester", label: "tester-pass-1", testerScenario: "multi-order-limit-orders" }, + { actor: "tester", label: "tester-pass-2" }, + ], + targetOutcomes: [ + "tester_order_created", + "tester_fresh_order_skip", + ], + reason: "run the same tester twice so a multi-order first pass can leave a fresh owned order for skip coverage", + }, +]; + +const DEFAULT_COVERAGE_GOALS: OutcomeKind[] = [ + "tester_order_created", + "tester_fresh_order_skip", + "tester_sampled_too_small_skip", + "bot_no_action_skip", + "bot_match_committed", + "bot_match_plus_deposit_committed", + "bot_receipt_completion_committed", + "bot_deposit_only_committed", + "bot_withdrawal_request_committed", + "bot_withdrawal_completion_committed", +]; + +const OUTCOME_SET = new Set(OUTCOME_KINDS); +const TX_CREATING_OUTCOMES = new Set([ + "tester_order_created", + "tester_conversion_created", + "bot_match_committed", + "bot_match_plus_deposit_committed", + "bot_receipt_completion_committed", + "bot_deposit_only_committed", + "bot_withdrawal_request_committed", + "bot_withdrawal_completion_committed", + "confirmation_timeout", + "terminal_chain_rejection", + "post_broadcast_unresolved", +]); + +export function usage(): string { + return [ + "Usage: node apps/supervisor/dist/index.js [options]", + "Options:", + ` --bot-config Default: ${DEFAULT_BOT_CONFIG_PATH}`, + ` --tester-config Default: ${DEFAULT_TESTER_CONFIG_PATH}`, + " --out-dir Default: logs/live-supervisor/", + " --max-cycles Default: 1", + " --max-wall-clock-seconds ", + " --stop-after-tx-count ", + " --scenario auto|standard-cycle|tester-only|bot-only|tester-fresh-skip-two-pass", + " --tester-scenario auto|random-order|sdk-conversion|extra-large-limit-order|multi-order-limit-orders|two-ckb-to-ickb-limit-orders|all-ckb-limit-order|all-ickb-limit-order|ickb-to-ckb-limit-order|two-ickb-to-ckb-limit-orders|mixed-direction-limit-orders|dust-ckb-conversion|dust-ickb-conversion", + " --tester-fee Default: 1", + " --tester-fee-base Default: 100000", + " --target-outcome Repeatable; planner prefers these first", + " --command-timeout-seconds Default: 900", + " --dry-run Fixture-only run; no live configs required", + " -h, --help", + ].join("\n"); +} + +export function parseArgs(argv: string[]): ParsedArgs { + const args: ParsedArgs = { + help: false, + dryRun: false, + botConfigPath: DEFAULT_BOT_CONFIG_PATH, + testerConfigPath: DEFAULT_TESTER_CONFIG_PATH, + maxCycles: 1, + scenario: "auto", + targetOutcomes: [], + testerScenario: "auto", + testerScenarioExplicit: false, + testerFee: "1", + testerFeeBase: "100000", + testerFeeExplicit: false, + testerFeeBaseExplicit: false, + commandTimeoutSeconds: DEFAULT_COMMAND_TIMEOUT_SECONDS, + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === undefined) { + throw new Error(`Missing argument at index ${String(index)}`); + } + if (arg === "--") { + continue; + } + if (arg === "-h" || arg === "--help") { + args.help = true; + continue; + } + if (arg === "--dry-run") { + args.dryRun = true; + continue; + } + if (arg === "--bot-config") { + args.botConfigPath = valueAfter(argv, ++index, arg); + continue; + } + if (arg === "--tester-config") { + args.testerConfigPath = valueAfter(argv, ++index, arg); + continue; + } + if (arg === "--out-dir") { + args.outDir = valueAfter(argv, ++index, arg); + continue; + } + if (arg === "--max-cycles") { + args.maxCycles = parsePositiveInteger(valueAfter(argv, ++index, arg), arg); + continue; + } + if (arg === "--max-wall-clock-seconds") { + args.maxWallClockSeconds = parsePositiveInteger(valueAfter(argv, ++index, arg), arg); + continue; + } + if (arg === "--stop-after-tx-count") { + args.stopAfterTxCount = parsePositiveInteger(valueAfter(argv, ++index, arg), arg); + continue; + } + if (arg === "--scenario") { + args.scenario = parseScenarioName(valueAfter(argv, ++index, arg)); + continue; + } + if (arg === "--target-outcome") { + args.targetOutcomes.push(parseOutcome(valueAfter(argv, ++index, arg), arg)); + continue; + } + if (arg === "--tester-scenario") { + args.testerScenario = parseTesterScenario(valueAfter(argv, ++index, arg)); + args.testerScenarioExplicit = true; + continue; + } + if (arg === "--tester-fee") { + args.testerFee = parseTesterFeeValue(valueAfter(argv, ++index, arg), arg); + args.testerFeeExplicit = true; + continue; + } + if (arg === "--tester-fee-base") { + args.testerFeeBase = parseTesterFeeValue(valueAfter(argv, ++index, arg), arg); + args.testerFeeBaseExplicit = true; + continue; + } + if (arg === "--command-timeout-seconds") { + args.commandTimeoutSeconds = parsePositiveInteger(valueAfter(argv, ++index, arg), arg); + continue; + } + throw new Error(`Unknown argument: ${arg}`); + } + + if (args.help || args.dryRun) { + args.botConfigPath = undefined; + args.testerConfigPath = undefined; + } + return args; +} + +export function resolvePlan(args: ParsedArgs, rootDir = repoRoot, dependencies: Dependencies = {}): SupervisorPlan { + const runId = createRunId(); + const outDir = insideRepoPath(rootDir, args.outDir ?? `logs/live-supervisor/${runId}`, "Output directory"); + const relativeOutDir = relative(rootDir, outDir); + const botConfigPath = args.botConfigPath === undefined + ? undefined + : insideRepoPath(rootDir, args.botConfigPath, "Bot config path"); + const testerConfigPath = args.testerConfigPath === undefined + ? undefined + : insideRepoPath(rootDir, args.testerConfigPath, "Tester config path"); + assertSupervisorOutputDirectory(relativeOutDir); + if (!isIgnoredPath(rootDir, relativeOutDir, dependencies)) { + throw new Error(`Refusing to write non-ignored supervisor output directory: ${relativeOutDir}`); + } + if (botConfigPath !== undefined) { + assertIgnoredConfigPath(rootDir, botConfigPath, "Bot config path", dependencies); + } + if (testerConfigPath !== undefined) { + assertIgnoredConfigPath(rootDir, testerConfigPath, "Tester config path", dependencies); + } + + return { + ...args, + runId, + rootDir, + botConfigPath, + testerConfigPath, + outDir, + relativeOutDir, + }; +} + +export function parseJsonEvidence(stdout: string): ParsedEvidence { + const records = new Array>(); + const ignoredLines = new Array(); + const malformedLines = new Array(); + for (const line of stdout.split(/\r?\n/u)) { + const trimmed = line.trim(); + if (trimmed === "") { + continue; + } + if (!trimmed.startsWith("{")) { + ignoredLines.push(trimmed); + continue; + } + try { + const parsed: unknown = JSON.parse(trimmed); + if (isRecord(parsed)) { + records.push(parsed); + } else { + malformedLines.push(trimmed); + } + } catch { + malformedLines.push(trimmed); + } + } + return { records, ignoredLines, malformedLines }; +} + +export function parsePreflightEvidence(stdout: string): ParsedEvidence { + const trimmed = stdout.trim(); + if (trimmed === "") { + return { records: [], ignoredLines: [], malformedLines: [] }; + } + try { + const parsed: unknown = JSON.parse(trimmed); + return isRecord(parsed) + ? { records: [parsed], ignoredLines: [], malformedLines: [] } + : { records: [], ignoredLines: [], malformedLines: [trimmed] }; + } catch { + return parseJsonEvidence(stdout); + } +} + +export function classifyActorResult( + actor: Actor | "preflight", + result: CommandResult, + expectation?: TesterEvidenceExpectation, +): Classification { + const evidence = actor === "preflight" + ? parsePreflightEvidence(result.stdout) + : parseJsonEvidence(result.stdout); + const txHashes = extractTxHashes(evidence.records); + const base = { + actor, + txHashes, + evidence: { + recordsAccepted: evidence.records.length, + ignoredLineCount: evidence.ignoredLines.length, + malformedLineCount: evidence.malformedLines.length, + exitStatus: result.status, + signal: result.signal, + timedOut: result.timedOut, + }, + }; + + if (containsSecretLeak(result.stdout) || containsSecretLeak(result.stderr)) { + return { + ...base, + outcome: "secret_leak_sentinel", + terminal: true, + reason: "stdout or stderr contained a secret-shaped field", + }; + } + if (result.timedOut) { + return { + ...base, + outcome: "command_timeout", + terminal: true, + reason: "supervisor command timeout expired", + }; + } + if (evidence.malformedLines.length > 0) { + return { + ...base, + outcome: "malformed_evidence", + terminal: true, + reason: "stdout contained malformed JSON evidence", + }; + } + if (actor === "preflight") { + return classifyPreflightResult(result, evidence, base); + } + if (actor === "bot") { + return classifyBotResult(result, evidence, base); + } + return classifyTesterResult(result, evidence, base, expectation); +} + +export function createCoverageLedger(goals: OutcomeKind[] = DEFAULT_COVERAGE_GOALS): CoverageLedger { + return { + goals, + counts: Object.fromEntries(OUTCOME_KINDS.map((outcome) => [outcome, 0])) as Record, + attempts: [], + unsupported: [], + }; +} + +export function chooseScenario(args: Pick, ledger: CoverageLedger): ScenarioChoiceResult { + const explicitGoals = explicitCoverageGoals(args); + if (args.scenario !== "auto") { + const explicit = scenarioByName(args.scenario); + const requestedGoals = explicitGoals.length > 0 ? explicitGoals : explicit.targetOutcomes; + const underCovered = requestedGoals.find((outcome) => ledger.counts[outcome] === 0) ?? requestedGoals[0]; + if (underCovered === undefined) { + throw new Error(`Scenario ${explicit.name} has no target outcomes configured`); + } + if (!explicit.targetOutcomes.includes(underCovered)) { + return { + kind: "unsupported", + requested: underCovered, + reason: `scenario ${explicit.name} does not safely target ${underCovered}`, + }; + } + return { + kind: "scenario", + scenario: explicit, + targetOutcomes: [underCovered], + reason: `explicit scenario ${explicit.name} selected for ${underCovered}`, + }; + } + + const requestedGoals = explicitGoals.length > 0 ? explicitGoals : ledger.goals; + const underCovered = nextUncoveredGoal(requestedGoals, ledger); + if (underCovered === undefined) { + const defaultScenario = SCENARIOS[0]; + if (defaultScenario === undefined) { + throw new Error("No supervisor scenarios configured"); + } + return { + kind: "scenario", + scenario: defaultScenario, + targetOutcomes: defaultScenario.targetOutcomes, + reason: "default standard cycle because no target outcomes were configured", + }; + } + + const candidates = SCENARIOS + .filter((scenario) => scenario.targetOutcomes.includes(underCovered)) + .sort((left, right) => left.steps.length - right.steps.length); + const scenario = candidates.find((candidate) => !ledger.attempts.some((attempt) => attempt.scenario === candidate.name)) + ?? candidates[0]; + if (scenario === undefined) { + return { + kind: "unsupported", + requested: underCovered, + reason: `${underCovered} is not reachable through safe supervisor/test-harness controls`, + }; + } + return { + kind: "scenario", + scenario, + targetOutcomes: [underCovered], + reason: `selected under-covered safe outcome ${underCovered}`, + }; +} + +function nextUncoveredGoal(requestedGoals: OutcomeKind[], ledger: CoverageLedger): OutcomeKind | undefined { + const uncovered = requestedGoals.filter((outcome) => ledger.counts[outcome] === 0); + return uncovered.find((outcome) => !ledger.attempts.some((attempt) => attempt.targetOutcomes.includes(outcome))) + ?? uncovered[0] + ?? requestedGoals[0]; +} + +export function recordScenarioAttempt( + ledger: CoverageLedger, + cycleIndex: number, + choice: ScenarioChoiceResult, +): void { + if (choice.kind === "unsupported") { + ledger.unsupported.push({ cycleIndex, requested: choice.requested, reason: choice.reason }); + return; + } + ledger.attempts.push({ + cycleIndex, + scenario: choice.scenario.name, + targetOutcomes: choice.targetOutcomes, + reason: choice.reason, + }); +} + +export function recordOutcome(ledger: CoverageLedger, outcome: OutcomeKind): void { + ledger.counts[outcome] += 1; +} + +export async function supervise(args: ParsedArgs, plan: SupervisorPlan, dependencies: Dependencies = {}): Promise { + return superviseOnce(args, plan, dependencies); +} + +async function superviseOnce(args: ParsedArgs, plan: SupervisorPlan, dependencies: Dependencies = {}): Promise { + const startedAt = now(dependencies); + if (!args.dryRun && dependencies.skipBuiltRuntimeCheck !== true) { + assertBuiltRuntime(plan, dependencies); + } + await prepareOutputDirectory(plan, dependencies); + const ledger = createCoverageLedger(explicitCoverageGoals(args).length > 0 ? explicitCoverageGoals(args) : DEFAULT_COVERAGE_GOALS); + const classifications = new Array(); + const artifacts = new Array(); + let txCount = 0; + let latestPublicState: PublicStateAssumption | undefined; + + if (args.dryRun) { + const dryRunResult = await runDryRun(plan, ledger, dependencies); + return dryRunResult; + } + + for (let cycleIndex = 1; cycleIndex <= args.maxCycles; cycleIndex += 1) { + if (args.maxWallClockSeconds !== undefined && now(dependencies) - startedAt >= args.maxWallClockSeconds * 1000) { + await writeSummary(plan, ledger, classifications, artifacts, latestPublicState, "max_wall_clock_seconds", dependencies); + return 0; + } + + const choice = chooseScenario(args, ledger); + recordScenarioAttempt(ledger, cycleIndex, choice); + if (choice.kind === "unsupported") { + const classification = unsupportedClassification(choice); + classifications.push(classification); + const incident = unsupportedIncident(plan, cycleIndex, choice, ledger, classification); + await writeJsonArtifact(plan, `cycle-${padCycle(cycleIndex)}-incident.json`, incident, artifacts, dependencies); + await writeSummary(plan, ledger, classifications, artifacts, latestPublicState, "unsupported_scenario", dependencies); + return STOP_EXIT_CODE; + } + + await appendSupervisorEvent(plan, { + type: "cycle.started", + cycleIndex, + scenario: choice.scenario.name, + targetOutcomes: choice.targetOutcomes, + reason: choice.reason, + }, dependencies); + + for (const step of choice.scenario.steps) { + const result = await runPreflight(step.actor, plan, cycleIndex, stepLabel(step), dependencies); + artifacts.push(...await writeCommandArtifacts(plan, cycleIndex, `${stepLabel(step)}-preflight`, result, dependencies)); + const classification = classifyActorResult("preflight", result); + if (classification.terminal) { + classifications.push(classification); + await writeIncident(plan, cycleIndex, step.actor, choice, classification, result, ledger, artifacts, dependencies); + await writeSummary(plan, ledger, classifications, artifacts, latestPublicState, classification.outcome, dependencies); + return STOP_EXIT_CODE; + } + } + + for (const step of choice.scenario.steps) { + const result = await runActor(step, plan, choice.targetOutcomes, dependencies); + artifacts.push(...await writeCommandArtifacts(plan, cycleIndex, stepLabel(step), result, dependencies)); + const classification = classifyActorResult(step.actor, result, step.actor === "tester" ? testerEvidenceExpectation(plan, step) : undefined); + classifications.push(classification); + recordOutcome(ledger, classification.outcome); + txCount += txCreatingHashCount(classification); + latestPublicState = classification.publicState ?? latestPublicState; + await appendSupervisorEvent(plan, { + type: "actor.classified", + cycleIndex, + actor: step.actor, + step: stepLabel(step), + outcome: classification.outcome, + terminal: classification.terminal, + reason: classification.reason, + txHashes: classification.txHashes, + }, dependencies); + + if (classification.terminal) { + const incident = await writeIncident(plan, cycleIndex, step.actor, choice, classification, result, ledger, artifacts, dependencies); + await writeSummary(plan, ledger, classifications, artifacts, latestPublicState, incident.classification.outcome, dependencies); + return classification.outcome === "nonzero_exit" ? 1 : STOP_EXIT_CODE; + } + if (args.stopAfterTxCount !== undefined && txCount >= args.stopAfterTxCount) { + await writeSummary(plan, ledger, classifications, artifacts, latestPublicState, "stop_after_tx_count", dependencies); + return 0; + } + } + } + + const unmetGoals = unmetExplicitGoals(args, ledger); + if (unmetGoals.length > 0) { + const incident = await writeUnmetCoverageIncident(plan, args.maxCycles, unmetGoals, ledger, artifacts, dependencies); + classifications.push(incident.classification); + await writeSummary(plan, ledger, classifications, artifacts, latestPublicState, incident.classification.outcome, dependencies); + return STOP_EXIT_CODE; + } + + await writeSummary(plan, ledger, classifications, artifacts, latestPublicState, "max_cycles", dependencies); + return 0; +} + +export async function main(argv: string[], io: { stdout?: NodeJS.WritableStream; stderr?: NodeJS.WritableStream } = {}): Promise { + const stdout = io.stdout ?? process.stdout; + const stderr = io.stderr ?? process.stderr; + let parsed: ParsedArgs; + try { + parsed = parseArgs(argv); + } catch (error) { + stderr.write(`${errorMessage(error)}\n${usage()}\n`); + return 1; + } + if (parsed.help) { + stdout.write(`${usage()}\n`); + return 0; + } + + try { + const plan = resolvePlan(parsed); + const exitCode = await supervise(parsed, plan); + stdout.write(`live supervisor artifacts: ${plan.relativeOutDir}\n`); + return exitCode; + } catch (error) { + stderr.write(`Live supervisor failed: ${errorMessage(error)}\n`); + return 1; + } +} + +async function runDryRun(plan: SupervisorPlan, ledger: CoverageLedger, dependencies: Dependencies): Promise { + const artifacts = new Array(); + const classifications = new Array(); + const choice = chooseScenario(plan, ledger); + recordScenarioAttempt(ledger, 1, choice); + if (choice.kind === "unsupported") { + const classification = unsupportedClassification(choice); + classifications.push(classification); + const incident = unsupportedIncident(plan, 1, choice, ledger, classification); + await writeJsonArtifact(plan, "dry-run-incident.json", incident, artifacts, dependencies); + await writeSummary(plan, ledger, classifications, artifacts, undefined, "unsupported_scenario", dependencies); + return STOP_EXIT_CODE; + } + + const samples: Array<{ actor: Actor; result: CommandResult }> = [ + { + actor: "tester", + result: fixtureResult("tester", JSON.stringify({ + startTime: "dry-run", + actions: { newOrder: { giveCkb: "100", takeIckb: "99", fee: "0.001" }, cancelledOrders: 0 }, + txHash: sampleTxHash("11"), + ElapsedSeconds: 1, + })), + }, + { + actor: "bot", + result: fixtureResult("bot", [ + JSON.stringify(botEvent("bot.state.read", { + orders: { marketCount: 3, userCount: 0, receiptCount: 1 }, + poolDeposits: { readyCount: 2, nearReadyCount: 1, futureCount: 5 }, + })), + JSON.stringify(botEvent("bot.decision.skipped", { + reason: "no_actions", + actions: emptyActions(), + })), + ].join("\n")), + }, + ]; + + let latestPublicState: PublicStateAssumption | undefined; + for (const sample of samples.filter((item) => scenarioActors(choice.scenario).includes(item.actor))) { + const classification = classifyActorResult(sample.actor, sample.result); + classifications.push(classification); + recordOutcome(ledger, classification.outcome); + latestPublicState = classification.publicState ?? latestPublicState; + artifacts.push(...await writeCommandArtifacts(plan, 1, `dry-run-${sample.actor}`, sample.result, dependencies)); + } + + await writeSummary(plan, ledger, classifications, artifacts, latestPublicState, "dry_run", dependencies); + return 0; +} + +async function prepareOutputDirectory(plan: SupervisorPlan, dependencies: Dependencies): Promise { + const statFn = dependencies.stat ?? stat; + const mkdirFn = dependencies.mkdir ?? mkdir; + await assertNoSymlinkedOutputAncestors(plan, dependencies); + try { + await statFn(plan.outDir); + throw new Error(`Output directory already exists: ${plan.relativeOutDir}`); + } catch (error) { + if (!isNotFoundError(error)) { + throw error; + } + } + await mkdirFn(plan.outDir, { recursive: true }); + await assertRealOutputDirectory(plan, dependencies); +} + +async function assertNoSymlinkedOutputAncestors(plan: SupervisorPlan, dependencies: Dependencies): Promise { + const lstatFn = dependencies.lstat ?? lstat; + const parts = plan.relativeOutDir.split("/").filter((part) => part !== ""); + let current = plan.rootDir; + for (const part of parts) { + current = join(current, part); + try { + const stats = await lstatFn(current); + if (stats.isSymbolicLink()) { + throw new Error(`Refusing to write supervisor artifacts through symlinked path: ${relative(plan.rootDir, current)}`); + } + } catch (error) { + if (isNotFoundError(error)) { + return; + } + throw error; + } + } +} + +async function assertRealOutputDirectory(plan: SupervisorPlan, dependencies: Dependencies): Promise { + const realpathFn = dependencies.realpath ?? realpath; + let realRoot: string; + let realOutDir: string; + try { + [realRoot, realOutDir] = await Promise.all([ + realpathFn(plan.rootDir), + realpathFn(plan.outDir), + ]); + } catch (error) { + if (dependencies.mkdir !== undefined && isNotFoundError(error)) { + return; + } + throw error; + } + const relativeOutDir = relative(realRoot, realOutDir); + if (relativeOutDir.startsWith("..") || isAbsolute(relativeOutDir)) { + throw new Error("Supervisor output directory must stay inside the repo"); + } +} + +function assertBuiltRuntime(plan: SupervisorPlan, dependencies: Dependencies): void { + const required = [ + ["CCC core", join(plan.rootDir, "packages/node-utils/node_modules/@ckb-ccc/core/dist/index.js")], + ["CCC UDT", join(plan.rootDir, "packages/core/node_modules/@ckb-ccc/udt/dist/index.js")], + ["utils package", join(plan.rootDir, "packages/utils/dist/index.js")], + ["DAO package", join(plan.rootDir, "packages/dao/dist/index.js")], + ["core package", join(plan.rootDir, "packages/core/dist/index.js")], + ["order package", join(plan.rootDir, "packages/order/dist/index.js")], + ["SDK package", join(plan.rootDir, "packages/sdk/dist/index.js")], + ["node-utils package", join(plan.rootDir, "packages/node-utils/dist/index.js")], + ["bot app", join(plan.rootDir, "apps/bot/dist/index.js")], + ["tester app", join(plan.rootDir, "apps/tester/dist/index.js")], + ["live preflight script", join(plan.rootDir, "scripts/ickb-live-preflight.mjs")], + ] as const; + const exists = dependencies.existsSync ?? existsSync; + for (const [label, path] of required) { + if (!exists(path)) { + throw new Error(`Missing built ${label}: ${relative(plan.rootDir, path)}`); + } + } +} + +async function runPreflight(actor: Actor, plan: SupervisorPlan, cycleIndex: number, role: string, dependencies: Dependencies): Promise { + const configPath = actor === "bot" ? plan.botConfigPath : plan.testerConfigPath; + if (configPath === undefined) { + throw new Error(`Missing ${actor} config path`); + } + await assertNoSymlinkedConfigPath(plan.rootDir, configPath, `${actor} config path`, dependencies); + return runCommand({ + actor: "preflight", + command: process.execPath, + args: ["scripts/ickb-live-preflight.mjs", "--config", configPath, "--role", `${role}-${String(cycleIndex)}`], + cwd: plan.rootDir, + env: liveActorEnv({ INIT_CWD: plan.rootDir }), + inheritEnv: false, + timeoutMs: plan.commandTimeoutSeconds * 1000, + }, dependencies); +} + +async function runActor(step: ScenarioStep, plan: SupervisorPlan, targetOutcomes: OutcomeKind[], dependencies: Dependencies): Promise { + const actor = step.actor; + const configPath = actor === "bot" ? plan.botConfigPath : plan.testerConfigPath; + if (configPath === undefined) { + throw new Error(`Missing ${actor} config path`); + } + await assertNoSymlinkedConfigPath(plan.rootDir, configPath, `${actor} config path`, dependencies); + const entrypoint = actor === "bot" ? "apps/bot/dist/index.js" : "apps/tester/dist/index.js"; + const configEnvName = actor === "bot" ? "BOT_CONFIG_FILE" : "TESTER_CONFIG_FILE"; + return runCommand({ + actor, + command: process.execPath, + args: [entrypoint], + cwd: plan.rootDir, + env: liveActorEnv({ + [configEnvName]: configPath, + INIT_CWD: plan.rootDir, + ...(actor === "tester" ? testerEnv(plan, targetOutcomes, step) : {}), + }), + inheritEnv: false, + timeoutMs: plan.commandTimeoutSeconds * 1000, + }, dependencies); +} + +function testerEnv(plan: SupervisorPlan, targetOutcomes: OutcomeKind[], step: ScenarioStep): Record { + return { + TESTER_SCENARIO: testerScenarioForTargets(plan.testerScenario, plan.testerScenarioExplicit, targetOutcomes, step.testerScenario), + ...(plan.testerFeeExplicit ? { TESTER_FEE: plan.testerFee } : {}), + ...(plan.testerFeeBaseExplicit ? { TESTER_FEE_BASE: plan.testerFeeBase } : {}), + }; +} + +function testerEvidenceExpectation(plan: SupervisorPlan, step: ScenarioStep): TesterEvidenceExpectation | undefined { + const scenario = testerScenarioForTargets(plan.testerScenario, plan.testerScenarioExplicit, [], step.testerScenario); + return scenario !== "auto" ? { scenario } : undefined; +} + +function testerScenarioForTargets( + configuredScenario: TesterScenario, + testerScenarioExplicit: boolean, + targetOutcomes: OutcomeKind[], + stepScenario: TesterScenario | undefined, +): TesterScenario { + if (testerScenarioExplicit || configuredScenario !== "auto") { + return configuredScenario; + } + if (stepScenario !== undefined) { + return stepScenario; + } + if (targetOutcomes.includes("tester_conversion_created")) { + return "sdk-conversion"; + } + return configuredScenario; +} + +function stepLabel(step: ScenarioStep): string { + return step.label ?? step.actor; +} + +function scenarioActors(scenario: ScenarioDefinition): Actor[] { + return [...new Set(scenario.steps.map((step) => step.actor))]; +} + +async function runCommand(spec: { + actor: CommandResult["actor"]; + command: string; + args: string[]; + cwd: string; + env?: Record; + inheritEnv?: boolean; + timeoutMs: number; +}, dependencies: Dependencies): Promise { + const spawnCommand = dependencies.spawnCommand ?? spawn; + const start = now(dependencies); + return new Promise((resolve) => { + const child = spawnCommand(spec.command, spec.args, { + cwd: spec.cwd, + env: { ...(spec.inheritEnv === false ? {} : process.env), ...spec.env }, + stdio: ["ignore", "pipe", "pipe"], + detached: process.platform !== "win32", + }); + const maxOutputBytes = dependencies.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES; + const stdoutCapture = createBoundedOutputCapture(); + const stderrCapture = createBoundedOutputCapture(); + let timedOut = false; + let settled = false; + let killTimeout: NodeJS.Timeout | undefined; + const timeout = setTimeout(() => { + timedOut = true; + signalChild(child, "SIGTERM", dependencies); + killTimeout = setTimeout(() => { + signalChild(child, "SIGKILL", dependencies); + }, dependencies.commandKillGraceMs ?? DEFAULT_COMMAND_KILL_GRACE_MS); + }, spec.timeoutMs); + + const finish = (result: Omit): void => { + if (settled) { + return; + } + settled = true; + clearTimeout(timeout); + if (killTimeout !== undefined) { + clearTimeout(killTimeout); + } + resolve({ + actor: spec.actor, + command: spec.command, + args: spec.args, + elapsedMs: now(dependencies) - start, + ...result, + }); + }; + + child.stdout.on("data", (chunk: Buffer) => { + appendBoundedOutput(stdoutCapture, chunk, maxOutputBytes); + }); + child.stderr.on("data", (chunk: Buffer) => { + appendBoundedOutput(stderrCapture, chunk, maxOutputBytes); + }); + child.on("error", (error) => { + finish({ + spawnError: errorMessage(error), + status: null, + signal: null, + timedOut, + stdout: boundedOutputText(stdoutCapture), + stderr: boundedOutputText(stderrCapture), + stdoutTruncated: stdoutCapture.truncatedBytes > 0, + stderrTruncated: stderrCapture.truncatedBytes > 0, + }); + }); + child.on("close", (status, signal) => { + const stdout = boundedOutputText(stdoutCapture); + const stderr = boundedOutputText(stderrCapture); + finish({ + status, + signal, + timedOut, + stdout, + stderr, + stdoutTruncated: stdoutCapture.truncatedBytes > 0, + stderrTruncated: stderrCapture.truncatedBytes > 0, + }); + }); + }); +} + +function signalChild( + child: { pid?: number; kill: (signal?: NodeJS.Signals) => boolean }, + signal: NodeJS.Signals, + dependencies: Dependencies, +): void { + if (process.platform !== "win32" && child.pid !== undefined) { + try { + (dependencies.killProcess ?? process.kill)(-child.pid, signal); + return; + } catch { + // Fall back to the direct child for fakes or platforms without process groups. + } + } + child.kill(signal); +} + +function classifyPreflightResult( + result: CommandResult, + evidence: ParsedEvidence, + base: Omit, +): Classification { + if (result.status !== 0) { + return { + ...base, + outcome: result.stderr.includes("Invalid") && result.stderr.includes("chain identity") ? "wrong_chain" : "nonzero_exit", + terminal: true, + reason: boundedText(safeArtifactText(result.stderr), 240) || "preflight command exited nonzero", + }; + } + const report = evidence.records[0]; + if (report === undefined) { + return { + ...base, + outcome: "malformed_evidence", + terminal: true, + reason: "preflight did not return a JSON report", + }; + } + return { + ...base, + outcome: "bot_no_action_skip", + terminal: false, + reason: "preflight succeeded", + }; +} + +function classifyBotResult( + result: CommandResult, + evidence: ParsedEvidence, + base: Omit, +): Classification { + const botRecords = evidence.records.filter(isBotRecord); + const publicState = latestPublicState(botRecords); + const failed = lastRecordOfType(botRecords, "bot.transaction.failed"); + if (failed !== undefined) { + const outcome = stringField(failed, "outcome"); + const phase = stringField(failed, "phase"); + if (outcome === "timeout_after_broadcast") { + return { ...base, outcome: "confirmation_timeout", terminal: true, reason: "bot tx confirmation timed out", publicState }; + } + if (outcome === "post_broadcast_unresolved") { + return { ...base, outcome: "post_broadcast_unresolved", terminal: true, reason: "bot tx remained unresolved after broadcast", publicState }; + } + if (outcome === "terminal_rejection") { + return { ...base, outcome: "terminal_chain_rejection", terminal: true, reason: "bot tx reached terminal chain rejection", publicState }; + } + if (phase === "pre_broadcast") { + return { ...base, outcome: "unknown", terminal: true, reason: "bot pre-broadcast transaction failure", publicState }; + } + } + + const skip = lastRecordOfType(botRecords, "bot.decision.skipped"); + if (skip !== undefined && stringField(skip, "reason") === "capital_below_minimum") { + return { + ...base, + outcome: "low_capital_stop", + terminal: true, + reason: "bot reported capital_below_minimum", + actions: actionCounts(skip["actions"]), + skipReason: "capital_below_minimum", + publicState, + }; + } + + if (result.status !== 0) { + return { + ...base, + outcome: "nonzero_exit", + terminal: true, + reason: `bot exited with status ${String(result.status)}`, + publicState, + }; + } + + const committed = lastRecordOfType(botRecords, "bot.transaction.committed"); + if (committed !== undefined) { + const actions = latestBotActions(botRecords) ?? emptyActions(); + return { + ...base, + outcome: classifyBotCommittedActions(actions), + terminal: false, + reason: "bot transaction committed according to app evidence", + actions, + publicState, + }; + } + + if (skip !== undefined) { + const reason = stringField(skip, "reason") ?? "unknown"; + const actions = actionCounts(skip["actions"]); + if (reason === "post_tx_ckb_reserve") { + return { + ...base, + outcome: "bot_reserve_skip", + terminal: false, + reason: "bot skipped to preserve CKB reserve", + actions, + skipReason: reason, + publicState, + }; + } + return { + ...base, + outcome: "bot_no_action_skip", + terminal: false, + reason: `bot skipped: ${reason}`, + actions, + skipReason: reason, + publicState, + }; + } + return { ...base, outcome: "unknown", terminal: true, reason: "bot produced no classifiable terminal evidence", publicState }; +} + +function classifyTesterResult( + result: CommandResult, + evidence: ParsedEvidence, + base: Omit, + expectation?: TesterEvidenceExpectation, +): Classification { + const testerLogs = evidence.records.filter((record) => !isBotRecord(record)); + const latest = testerLogs.at(-1); + if (latest !== undefined) { + const error = latest["error"]; + if (error !== undefined) { + if (hasTimeoutError(error)) { + return { ...base, outcome: "confirmation_timeout", terminal: true, reason: "tester transaction confirmation timed out" }; + } + if (isTesterFundingError(error)) { + return { ...base, outcome: "low_capital_stop", terminal: true, reason: "tester reported insufficient funds" }; + } + return { + ...base, + outcome: "tester_deterministic_pre_broadcast_error", + terminal: true, + reason: "tester recorded an error before a committed transaction was proven", + }; + } + } + if (result.status !== 0) { + return { ...base, outcome: "nonzero_exit", terminal: true, reason: `tester exited with status ${String(result.status)}` }; + } + if (latest !== undefined) { + const skip = recordField(latest, "skip"); + if (skip !== undefined) { + const reason = stringField(skip, "reason") ?? "unknown"; + if (reason === "fresh-matchable-order") { + return { ...base, outcome: "tester_fresh_order_skip", terminal: false, reason: "tester skipped fresh matchable order", skipReason: reason }; + } + if (reason === "sampled-amount-too-small") { + return { ...base, outcome: "tester_sampled_too_small_skip", terminal: false, reason: "tester sampled amount too small", skipReason: reason }; + } + if (reason === "estimated-conversion-too-small") { + return { ...base, outcome: "tester_estimated_too_small_skip", terminal: false, reason: "tester estimate converted amount too small", skipReason: reason }; + } + if (reason === "post-tx-ckb-reserve") { + return { ...base, outcome: "tester_reserve_skip", terminal: false, reason: "tester skipped to preserve CKB reserve", skipReason: reason }; + } + return { ...base, outcome: "unknown", terminal: true, reason: `tester skip reason is not classified: ${reason}`, skipReason: reason }; + } + + if (typeof latest["txHash"] === "string") { + const actions = recordField(latest, "actions"); + const expectationFailure = validateTesterEvidenceExpectation(actions, expectation); + if (expectationFailure !== undefined) { + return { ...base, outcome: "tester_deterministic_pre_broadcast_error", terminal: true, reason: expectationFailure }; + } + const conversionKind = stringField(recordField(actions ?? {}, "conversion"), "kind"); + if (conversionKind !== undefined) { + return { ...base, outcome: "tester_conversion_created", terminal: false, reason: "tester created a direct conversion transaction" }; + } + return { ...base, outcome: "tester_order_created", terminal: false, reason: "tester created an order transaction" }; + } + } + return { ...base, outcome: "unknown", terminal: true, reason: "tester produced no classifiable terminal evidence" }; +} + +function validateTesterEvidenceExpectation(actions: Record | undefined, expectation: TesterEvidenceExpectation | undefined): string | undefined { + if (expectation === undefined) { + return undefined; + } + if (actions === undefined) { + return `tester committed tx without actions for expected scenario ${expectation.scenario}`; + } + const loggedScenario = stringField(actions, "testerScenario"); + if (expectation.scenario === "multi-order-limit-orders") { + return validateAnyMultiOrders(actions, expectation.scenario); + } + if (loggedScenario !== expectation.scenario) { + return `tester committed tx for scenario ${loggedScenario ?? "unknown"}, expected ${expectation.scenario}`; + } + if (isSdkConversionTesterScenario(expectation.scenario)) { + return recordField(actions, "conversion") === undefined + ? `tester scenario ${expectation.scenario} committed without conversion evidence` + : undefined; + } + if (expectation.scenario === "two-ckb-to-ickb-limit-orders") { + return validateMultiOrders(actions, 2, expectation.scenario, "giveCkb", "takeIckb"); + } + if (expectation.scenario === "two-ickb-to-ckb-limit-orders") { + return validateMultiOrders(actions, 2, expectation.scenario, "giveIckb", "takeCkb"); + } + if (expectation.scenario === "mixed-direction-limit-orders") { + return validateMixedDirectionOrders(actions, expectation.scenario); + } + const newOrder = recordField(actions, "newOrder"); + if (newOrder === undefined) { + return `tester scenario ${expectation.scenario} committed without new order evidence`; + } + if (expectation.scenario === "random-order") { + return hasOrderFields(newOrder, "giveCkb", "takeIckb") || hasOrderFields(newOrder, "giveIckb", "takeCkb") + ? undefined + : "tester scenario random-order committed with wrong order direction evidence"; + } + return isIckbToCkbTesterScenario(expectation.scenario) + ? requireOrderFields(expectation.scenario, newOrder, "giveIckb", "takeCkb") + : requireOrderFields(expectation.scenario, newOrder, "giveCkb", "takeIckb"); +} + +function validateAnyMultiOrders(actions: Record, scenario: TesterScenario): string | undefined { + const loggedScenario = stringField(actions, "testerScenario"); + if (loggedScenario === "two-ckb-to-ickb-limit-orders") { + return validateMultiOrders(actions, 2, scenario, "giveCkb", "takeIckb"); + } + if (loggedScenario === "two-ickb-to-ckb-limit-orders") { + return validateMultiOrders(actions, 2, scenario, "giveIckb", "takeCkb"); + } + if (loggedScenario === "mixed-direction-limit-orders") { + return validateMixedDirectionOrders(actions, scenario); + } + return `tester scenario ${scenario} committed with non-multi-order selected scenario evidence`; +} + +function validateMultiOrders(actions: Record, expectedCount: number, scenario: TesterScenario, giveField: string, takeField: string): string | undefined { + const newOrders = arrayField(actions, "newOrders"); + if (newOrders === undefined || newOrders.length !== expectedCount) { + return `tester scenario ${scenario} committed without ${String(expectedCount)} new order evidence entries`; + } + if (numberField(actions, "orderCount") !== expectedCount) { + return `tester scenario ${scenario} committed with wrong order count evidence`; + } + return newOrders.every((order) => isRecord(order) && hasOrderFields(order, giveField, takeField)) + ? undefined + : `tester scenario ${scenario} committed with wrong order direction evidence`; +} + +function validateMixedDirectionOrders(actions: Record, scenario: TesterScenario): string | undefined { + const newOrders = arrayField(actions, "newOrders"); + if (newOrders === undefined || newOrders.length !== 2) { + return `tester scenario ${scenario} committed without 2 new order evidence entries`; + } + if (numberField(actions, "orderCount") !== 2) { + return `tester scenario ${scenario} committed with wrong order count evidence`; + } + const directions = newOrders.map(orderDirection); + return directions.includes("ckb-to-ickb") && directions.includes("ickb-to-ckb") + ? undefined + : `tester scenario ${scenario} committed without mixed order direction evidence`; +} + +function orderDirection(order: unknown): TesterDirection | undefined { + if (!isRecord(order)) { + return undefined; + } + const ckbToIckb = hasOrderFields(order, "giveCkb", "takeIckb"); + const ickbToCkb = hasOrderFields(order, "giveIckb", "takeCkb"); + if (ckbToIckb === ickbToCkb) { + return undefined; + } + return ckbToIckb ? "ckb-to-ickb" : "ickb-to-ckb"; +} + +function requireOrderFields(scenario: TesterScenario, newOrder: Record, giveField: string, takeField: string): string | undefined { + if (hasOrderFields(newOrder, giveField, takeField)) { + return undefined; + } + return `tester scenario ${scenario} committed with wrong order direction evidence`; +} + +function hasOrderFields(newOrder: Record, giveField: string, takeField: string): boolean { + return typeof newOrder[giveField] === "string" && typeof newOrder[takeField] === "string"; +} + +function arrayField(record: Record, key: string): unknown[] | undefined { + const value = record[key]; + return Array.isArray(value) ? value : undefined; +} + +function isSdkConversionTesterScenario(scenario: TesterScenario): boolean { + return scenario === "sdk-conversion"; +} + +function isIckbToCkbTesterScenario(scenario: TesterScenario): boolean { + return scenario === "all-ickb-limit-order" || scenario === "ickb-to-ckb-limit-order" || scenario === "two-ickb-to-ckb-limit-orders" || scenario === "dust-ickb-conversion"; +} + +function classifyBotCommittedActions(actions: ActionCounts): OutcomeKind { + if (actions.matchedOrders > 0 && actions.deposits > 0) { + return "bot_match_plus_deposit_committed"; + } + if (actions.matchedOrders > 0) { + return "bot_match_committed"; + } + if (actions.completedDeposits > 0) { + return "bot_receipt_completion_committed"; + } + if (actions.deposits > 0) { + return "bot_deposit_only_committed"; + } + if (actions.withdrawalRequests > 0) { + return "bot_withdrawal_request_committed"; + } + if (actions.withdrawals > 0) { + return "bot_withdrawal_completion_committed"; + } + return "unknown"; +} + +function unsupportedClassification(choice: UnsupportedScenarioChoice): Classification { + return { + actor: "preflight", + outcome: "unknown", + terminal: true, + reason: choice.reason, + txHashes: [], + evidence: { + recordsAccepted: 0, + ignoredLineCount: 0, + malformedLineCount: 0, + exitStatus: null, + signal: null, + timedOut: false, + }, + }; +} + +function unmetExplicitGoals(args: Pick, ledger: CoverageLedger): OutcomeKind[] { + return explicitCoverageGoals(args).filter((outcome) => ledger.counts[outcome] === 0); +} + +function explicitCoverageGoals(args: Pick): OutcomeKind[] { + return [...new Set(args.targetOutcomes)]; +} + +async function writeUnmetCoverageIncident( + plan: SupervisorPlan, + cycleIndex: number, + unmetGoals: OutcomeKind[], + ledger: CoverageLedger, + artifacts: string[], + dependencies: Dependencies, +): Promise { + const classification: Classification = { + actor: "preflight", + outcome: "unmet_coverage_goal", + terminal: true, + reason: `bounded cycle budget ended before observing requested outcomes: ${unmetGoals.join(", ")}`, + txHashes: [], + evidence: { + recordsAccepted: 0, + ignoredLineCount: 0, + malformedLineCount: 0, + exitStatus: null, + signal: null, + timedOut: false, + }, + }; + const relativePath = await writeJsonArtifact(plan, `cycle-${padCycle(cycleIndex)}-incident.json`, { + runId: plan.runId, + cycleIndex, + actor: "supervisor", + classification, + unmetGoals, + coverage: coverageSummary(ledger), + artifacts, + suggestedNextAction: suggestedNextAction(classification), + }, artifacts, dependencies); + return { relativePath, classification }; +} + +async function writeIncident( + plan: SupervisorPlan, + cycleIndex: number, + actor: Actor, + choice: ScenarioChoice, + classification: Classification, + result: CommandResult, + ledger: CoverageLedger, + artifacts: string[], + dependencies: Dependencies, +): Promise { + const relativePath = await writeJsonArtifact(plan, `cycle-${padCycle(cycleIndex)}-incident.json`, { + runId: plan.runId, + cycleIndex, + actor, + scenario: choice.scenario.name, + targetOutcomes: choice.targetOutcomes, + command: redactedCommandShape(plan, result), + exit: { + spawnError: result.spawnError, + status: result.status, + signal: result.signal, + timedOut: result.timedOut, + stdoutTruncated: result.stdoutTruncated, + stderrTruncated: result.stderrTruncated, + elapsedMs: result.elapsedMs, + }, + classification, + coverage: coverageSummary(ledger), + stdoutExcerpt: boundedText(safeArtifactText(result.stdout), 4000), + stderrExcerpt: boundedText(safeArtifactText(result.stderr), 4000), + artifacts, + suggestedNextAction: suggestedNextAction(classification), + }, artifacts, dependencies); + return { relativePath, classification }; +} + +function unsupportedIncident( + plan: SupervisorPlan, + cycleIndex: number, + choice: UnsupportedScenarioChoice, + ledger: CoverageLedger, + classification: Classification, +): Record { + return { + runId: plan.runId, + cycleIndex, + actor: "supervisor", + classification, + requested: choice.requested, + coverage: coverageSummary(ledger), + suggestedNextAction: "provide an alternate ignored config or add a tested supervisor/test-harness control surface; do not mutate funded configs in place", + }; +} + +async function writeSummary( + plan: SupervisorPlan, + ledger: CoverageLedger, + classifications: Classification[], + artifacts: string[], + latestPublicState: PublicStateAssumption | undefined, + stopReason: string, + dependencies: Dependencies, +): Promise { + return await writeJsonArtifact(plan, "summary.json", { + runId: plan.runId, + stopped: stopReason, + artifacts, + aggregateCounts: aggregateClassifications(classifications), + txHashesByOutcome: txHashesByOutcome(classifications), + skipReasons: classifications.map((item) => item.skipReason).filter((item) => item !== undefined), + scenarioAttempts: ledger.attempts, + coverage: coverageSummary(ledger), + publicVsOwnedStateAssumptions: latestPublicState ?? null, + }, artifacts, dependencies); +} + +async function writeCommandArtifacts( + plan: SupervisorPlan, + cycleIndex: number, + label: string, + result: CommandResult, + dependencies: Dependencies, +): Promise { + const base = `cycle-${padCycle(cycleIndex)}-${label}`; + const stdoutPath = await writeTextArtifact(plan, `${base}.stdout.ndjson`, safeArtifactText(result.stdout), dependencies); + const stderrPath = await writeTextArtifact(plan, `${base}.stderr.log`, safeArtifactText(result.stderr), dependencies); + const commandPath = await writeJsonArtifact(plan, `${base}.command.json`, { + command: redactedCommandShape(plan, result), + exit: { + spawnError: result.spawnError, + status: result.status, + signal: result.signal, + timedOut: result.timedOut, + stdoutTruncated: result.stdoutTruncated, + stderrTruncated: result.stderrTruncated, + elapsedMs: result.elapsedMs, + }, + }, [], dependencies); + return [stdoutPath, stderrPath, commandPath]; +} + +async function writeTextArtifact(plan: SupervisorPlan, fileName: string, text: string, dependencies: Dependencies): Promise { + const writeFileFn = dependencies.writeFile ?? writeFile; + const artifactPath = join(plan.outDir, fileName); + await writeFileFn(artifactPath, text); + return relative(plan.rootDir, artifactPath); +} + +async function writeJsonArtifact( + plan: SupervisorPlan, + fileName: string, + value: unknown, + artifacts: string[], + dependencies: Dependencies, +): Promise { + const writeFileFn = dependencies.writeFile ?? writeFile; + const artifactPath = join(plan.outDir, fileName); + await writeFileFn(artifactPath, `${JSON.stringify(value, jsonReplacer, 2)}\n`); + const relativePath = relative(plan.rootDir, artifactPath); + if (!artifacts.includes(relativePath)) { + artifacts.push(relativePath); + } + return relativePath; +} + +async function appendSupervisorEvent(plan: SupervisorPlan, fields: Record, dependencies: Dependencies): Promise { + const appendFileFn = dependencies.appendFile ?? appendFile; + await appendFileFn(join(plan.outDir, "supervisor.ndjson"), `${JSON.stringify({ + version: 1, + app: "supervisor", + runId: plan.runId, + timestamp: new Date().toISOString(), + ...fields, + }, jsonReplacer)}\n`); +} + +function redactedCommandShape(plan: SupervisorPlan, result: CommandResult): Record { + return { + command: result.command === process.execPath ? "node" : result.command, + args: result.args.map((arg) => arg === plan.botConfigPath + ? "" + : arg === plan.testerConfigPath + ? "" + : arg), + }; +} + +function aggregateClassifications(classifications: Classification[]): Record { + const counts = new Map(); + for (const classification of classifications) { + counts.set(classification.outcome, (counts.get(classification.outcome) ?? 0) + 1); + } + return Object.fromEntries(counts); +} + +function txHashesByOutcome(classifications: Classification[]): Record { + const hashes = new Map(); + for (const classification of classifications) { + if (classification.txHashes.length === 0) { + continue; + } + hashes.set(classification.outcome, [ + ...(hashes.get(classification.outcome) ?? []), + ...classification.txHashes, + ]); + } + return Object.fromEntries(hashes); +} + +function txCreatingHashCount(classification: Classification): number { + return TX_CREATING_OUTCOMES.has(classification.outcome) ? classification.txHashes.length : 0; +} + +function coverageSummary(ledger: CoverageLedger): Record { + return { + goals: ledger.goals, + covered: ledger.goals.filter((goal) => ledger.counts[goal] > 0), + uncovered: ledger.goals.filter((goal) => ledger.counts[goal] === 0), + counts: ledger.counts, + attempts: ledger.attempts, + unsupported: ledger.unsupported, + }; +} + +function suggestedNextAction(classification: Classification): string { + if (classification.outcome === "confirmation_timeout" || classification.outcome === "post_broadcast_unresolved") { + return "confirm the tx hash with a read-only chain query before sending any follow-up work"; + } + if (classification.outcome === "secret_leak_sentinel") { + return "stop, inspect artifacts for leakage, and rotate any exposed disposable key before relaunch"; + } + if (classification.outcome === "low_capital_stop") { + return "fund the supervised account or provide an alternate ignored config, then rerun a bounded smoke"; + } + return "inspect the incident bundle and run a review pass for material code changes before extended relaunch"; +} + +function latestBotActions(records: Record[]): ActionCounts | undefined { + for (let index = records.length - 1; index >= 0; index -= 1) { + const record = records[index]; + if (record === undefined) { + continue; + } + if (stringField(record, "type") === "bot.transaction.built") { + return actionCounts(record["actions"]); + } + } + return undefined; +} + +function latestPublicState(records: Record[]): PublicStateAssumption | undefined { + for (let index = records.length - 1; index >= 0; index -= 1) { + const record = records[index]; + if (record === undefined) { + continue; + } + if (stringField(record, "type") !== "bot.state.read") { + continue; + } + const orders = recordField(record, "orders"); + const poolDeposits = recordField(record, "poolDeposits"); + return { + marketOrderCount: numberField(orders, "marketCount"), + userOrderCount: numberField(orders, "userCount"), + receiptCount: numberField(orders, "receiptCount"), + readyPoolDepositCount: numberField(poolDeposits, "readyCount"), + nearReadyPoolDepositCount: numberField(poolDeposits, "nearReadyCount"), + futurePoolDepositCount: numberField(poolDeposits, "futureCount"), + }; + } + return undefined; +} + +function actionCounts(value: unknown): ActionCounts { + const record = isRecord(value) ? value : {}; + return { + collectedOrders: numberField(record, "collectedOrders") ?? 0, + completedDeposits: numberField(record, "completedDeposits") ?? 0, + matchedOrders: numberField(record, "matchedOrders") ?? 0, + deposits: numberField(record, "deposits") ?? 0, + withdrawalRequests: numberField(record, "withdrawalRequests") ?? 0, + withdrawals: numberField(record, "withdrawals") ?? 0, + }; +} + +function emptyActions(): ActionCounts { + return { + collectedOrders: 0, + completedDeposits: 0, + matchedOrders: 0, + deposits: 0, + withdrawalRequests: 0, + withdrawals: 0, + }; +} + +function extractTxHashes(records: Record[]): string[] { + const hashes = new Set(); + for (const record of records) { + const txHash = record["txHash"]; + if (typeof txHash === "string" && TX_HASH_PATTERN.test(txHash)) { + hashes.add(txHash); + } + const skip = recordField(record, "skip"); + const skipTxHash = skip?.["txHash"]; + if (typeof skipTxHash === "string" && TX_HASH_PATTERN.test(skipTxHash)) { + hashes.add(skipTxHash); + } + } + return [...hashes]; +} + +function containsSecretLeak(text: string): boolean { + return /["']?(private[-_]?key|mnemonic|seed[-_]?phrase)["']?\s*[:=]/iu.test(text) || + /["']?rpc[-_]?url["']?\s*[:=]\s*["']?https?:\/\/[^\s"']*(?:@|[?&][^\s"']*=)/iu.test(text); +} + +function containsTransactionLeak(text: string): boolean { + return /["']?(witnesses|cellDeps|headerDeps|inputs|outputs|outputsData)["']?\s*:/iu.test(text); +} + +export function safeArtifactText(text: string): string { + if (containsSecretLeak(text)) { + return "\n"; + } + if (containsTransactionLeak(text)) { + return "\n"; + } + return text; +} + +function minimalProcessEnv(env: NodeJS.ProcessEnv): Record { + return Object.fromEntries(["PATH", "HOME", "LANG", "LC_ALL", "TERM"].flatMap((key) => { + const value = env[key]; + return value === undefined ? [] : [[key, value]]; + })); +} + +function liveActorEnv(extra: Record): Record { + return { + ...minimalProcessEnv(process.env), + ...extra, + }; +} + +function hasTimeoutError(value: unknown): boolean { + if (!isRecord(value)) { + return false; + } + return value["isTimeout"] === true || stringField(value, "name") === "TransactionConfirmationError"; +} + +function isTesterFundingError(value: unknown): boolean { + const message = typeof value === "string" ? value : stringField(isRecord(value) ? value : undefined, "message"); + return message !== undefined && /Not enough (?:funds|CKB|iCKB)/u.test(message); +} + +function isBotRecord(record: Record): boolean { + return record["app"] === "bot"; +} + +function lastRecordOfType(records: Record[], type: string): Record | undefined { + for (let index = records.length - 1; index >= 0; index -= 1) { + const record = records[index]; + if (record === undefined) { + continue; + } + if (stringField(record, "type") === type) { + return record; + } + } + return undefined; +} + +function recordField(record: Record, key: string): Record | undefined { + const value = record[key]; + return isRecord(value) ? value : undefined; +} + +function stringField(record: Record | undefined, key: string): string | undefined { + const value = record?.[key]; + return typeof value === "string" ? value : undefined; +} + +function numberField(record: Record | undefined, key: string): number | undefined { + const value = record?.[key]; + return typeof value === "number" ? value : undefined; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function scenarioByName(name: Exclude): ScenarioDefinition { + const scenario = SCENARIOS.find((candidate) => candidate.name === name); + if (scenario === undefined) { + throw new Error(`Unknown scenario: ${name}`); + } + return scenario; +} + +function parseOutcome(value: string, flag: string): OutcomeKind { + if (OUTCOME_SET.has(value as OutcomeKind)) { + return value as OutcomeKind; + } + throw new Error(`Invalid ${flag}: unknown outcome ${value}`); +} + +function parseScenarioName(value: string): ScenarioName { + if (value === "auto" || value === "standard-cycle" || value === "tester-only" || value === "bot-only" || value === "tester-fresh-skip-two-pass") { + return value; + } + throw new Error("Invalid --scenario: expected auto, standard-cycle, tester-only, bot-only, or tester-fresh-skip-two-pass"); +} + +function parseTesterScenario(value: string): TesterScenario { + if ( + value === "auto" || + value === "random-order" || + value === "sdk-conversion" || + value === "extra-large-limit-order" || + value === "multi-order-limit-orders" || + value === "two-ckb-to-ickb-limit-orders" || + value === "all-ckb-limit-order" || + value === "all-ickb-limit-order" || + value === "ickb-to-ckb-limit-order" || + value === "two-ickb-to-ckb-limit-orders" || + value === "mixed-direction-limit-orders" || + value === "dust-ckb-conversion" || + value === "dust-ickb-conversion" + ) { + return value; + } + throw new Error("Invalid --tester-scenario: expected auto, random-order, sdk-conversion, extra-large-limit-order, multi-order-limit-orders, two-ckb-to-ickb-limit-orders, all-ckb-limit-order, all-ickb-limit-order, ickb-to-ckb-limit-order, two-ickb-to-ckb-limit-orders, mixed-direction-limit-orders, dust-ckb-conversion, or dust-ickb-conversion"); +} + +function valueAfter(argv: string[], index: number, option: string): string { + const value = argv[index]; + if (value === undefined || value.startsWith("--")) { + throw new Error(`Missing value for ${option}`); + } + return value; +} + +function parsePositiveInteger(value: string, flag: string): number { + if (!/^[1-9][0-9]*$/u.test(value)) { + throw new Error(`Invalid ${flag}: expected a positive integer`); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) { + throw new Error(`Invalid ${flag}: expected a safe integer`); + } + return parsed; +} + +function parseTesterFeeValue(value: string, flag: string): string { + if (!/^(?:0|[1-9][0-9]*)$/u.test(value)) { + throw new Error(`Invalid ${flag}: expected an unsigned integer`); + } + return value; +} + +function insideRepoPath(rootDir: string, path: string, label: string): string { + if (path === "") { + throw new Error(`${label} must not be empty`); + } + const absolutePath = isAbsolute(path) ? path : resolve(rootDir, path); + const relativePath = relative(rootDir, absolutePath); + if (relativePath.startsWith("..") || isAbsolute(relativePath)) { + throw new Error(`${label} must stay inside the repo`); + } + return absolutePath; +} + +function assertSupervisorOutputDirectory(relativeOutDir: string): void { + if (relativeOutDir === "logs/live-supervisor" || relativeOutDir.startsWith(SUPERVISOR_OUTPUT_ROOT)) { + return; + } + throw new Error(`Supervisor output directory must be under ${SUPERVISOR_OUTPUT_ROOT}`); +} + +function assertIgnoredConfigPath( + rootDir: string, + absolutePath: string, + label: string, + dependencies: Dependencies, +): void { + const relativePath = relative(rootDir, absolutePath); + if (!isIgnoredPath(rootDir, relativePath, dependencies)) { + throw new Error(`Refusing to use non-ignored ${label}: ${relativePath}`); + } +} + +async function assertNoSymlinkedConfigPath( + rootDir: string, + absolutePath: string, + label: string, + dependencies: Dependencies, +): Promise { + const lstatFn = dependencies.lstat ?? lstat; + const parts = relative(rootDir, absolutePath).split("/").filter((part) => part !== ""); + let current = rootDir; + for (const part of parts) { + current = join(current, part); + try { + const stats = await lstatFn(current); + if (stats.isSymbolicLink()) { + throw new Error(`Refusing to use ${label} through symlinked path: ${relative(rootDir, current)}`); + } + } catch (error) { + if (isNotFoundError(error)) { + return; + } + throw error; + } + } +} + +function isIgnoredPath(rootDir: string, relativePath: string, dependencies: Dependencies): boolean { + const spawnSyncCommand = dependencies.spawnSyncCommand ?? spawnSync; + const result: SpawnSyncReturns = spawnSyncCommand("git", ["-C", rootDir, "check-ignore", "--", relativePath], { + encoding: "utf8", + }); + return result.status === 0; +} + +export function createBoundedOutputCapture(): BoundedOutputCapture { + return { chunks: [], byteLength: 0, truncatedBytes: 0 }; +} + +export function appendBoundedOutput(capture: BoundedOutputCapture, chunk: Buffer, maxBytes: number): void { + if (maxBytes < 1) { + capture.truncatedBytes += chunk.length; + return; + } + const remaining = maxBytes - capture.byteLength; + if (remaining <= 0) { + capture.truncatedBytes += chunk.length; + return; + } + const kept = chunk.subarray(0, remaining); + capture.chunks.push(kept); + capture.byteLength += kept.length; + capture.truncatedBytes += chunk.length - kept.length; +} + +export function boundedOutputText(capture: BoundedOutputCapture): string { + const text = Buffer.concat(capture.chunks).toString("utf8"); + return capture.truncatedBytes === 0 + ? text + : `${text}\n`; +} + +function createRunId(): string { + return new Date().toISOString().replace(/[:.]/gu, "-"); +} + +function now(dependencies: Dependencies): number { + return dependencies.now?.() ?? Date.now(); +} + +function padCycle(cycleIndex: number): string { + return cycleIndex.toString().padStart(4, "0"); +} + +function boundedText(text: string, limit: number): string { + if (text.length <= limit) { + return text; + } + return `${text.slice(0, limit)}\n`; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : typeof error === "string" ? error : "Unknown error"; +} + +function isNotFoundError(error: unknown): boolean { + return isRecord(error) && error["code"] === "ENOENT"; +} + +function jsonReplacer(_key: string, value: unknown): unknown { + return typeof value === "bigint" ? value.toString() : value; +} + +function fixtureResult(actor: Actor, stdout: string): CommandResult { + return { + actor, + command: "fixture", + args: [], + status: 0, + signal: null, + timedOut: false, + stdout: `${stdout}\n`, + stderr: "", + stdoutTruncated: false, + stderrTruncated: false, + elapsedMs: 1, + }; +} + +function botEvent(type: string, fields: Record): Record { + return { + version: 1, + app: "bot", + chain: "testnet", + runId: "dry-run", + iterationId: 1, + timestamp: "2026-01-01T00:00:00.000Z", + type, + ...fields, + }; +} + +function sampleTxHash(byte: string): string { + return `0x${byte.repeat(32)}`; +} + +if (process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href) { + process.exit(await main(process.argv.slice(2))); +} diff --git a/apps/supervisor/tsconfig.build.json b/apps/supervisor/tsconfig.build.json new file mode 100644 index 0000000..998f832 --- /dev/null +++ b/apps/supervisor/tsconfig.build.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declarationMap": false, + "sourceRoot": "", + "sourceMap": false + }, + "exclude": ["src/**/*.test.ts"] +} diff --git a/apps/supervisor/tsconfig.json b/apps/supervisor/tsconfig.json new file mode 100644 index 0000000..1d7ed28 --- /dev/null +++ b/apps/supervisor/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "noEmit": false, + "rootDir": "src", + "outDir": "dist", + "sourceRoot": "../src", + "types": ["node"] + }, + "include": ["src"] +} diff --git a/apps/supervisor/vitest.config.mts b/apps/supervisor/vitest.config.mts new file mode 100644 index 0000000..dc6a587 --- /dev/null +++ b/apps/supervisor/vitest.config.mts @@ -0,0 +1,10 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["src/**/*.test.ts"], + coverage: { + include: ["src/**/*.ts"], + }, + }, +}); From 8cb4f6e6b114e46be7d31e8cb27cf364a16e1b5c Mon Sep 17 00:00:00 2001 From: phroi <90913182+phroi@users.noreply.github.com> Date: Wed, 20 May 2026 05:32:29 +0000 Subject: [PATCH 02/11] feat(supervisor): add bounded external loop --- scripts/ickb-supervisor-loop.mjs | 433 ++++++++++++++++++++++++++ scripts/ickb-supervisor-loop.test.mjs | 227 ++++++++++++++ 2 files changed, 660 insertions(+) create mode 100644 scripts/ickb-supervisor-loop.mjs create mode 100644 scripts/ickb-supervisor-loop.test.mjs diff --git a/scripts/ickb-supervisor-loop.mjs b/scripts/ickb-supervisor-loop.mjs new file mode 100644 index 0000000..732c3bc --- /dev/null +++ b/scripts/ickb-supervisor-loop.mjs @@ -0,0 +1,433 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import { readFile } from "node:fs/promises"; +import { isAbsolute, join, relative, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const rootDir = fileURLToPath(new URL("..", import.meta.url)); +const DEFAULT_MAX_RUNS = 10; +const DEFAULT_STABLE_LIMIT = 3; +const DEFAULT_BACKOFF_SECONDS = 30; +const DEFAULT_SUPERVISOR_SCRIPT = "apps/supervisor/dist/index.js"; +const SUPERVISOR_OUTPUT_ROOT = "logs/live-supervisor"; +const TX_CREATING_OUTCOMES = new Set([ + "tester_order_created", + "tester_conversion_created", + "bot_match_committed", + "bot_match_plus_deposit_committed", + "bot_receipt_completion_committed", + "bot_deposit_only_committed", + "bot_withdrawal_request_committed", + "bot_withdrawal_completion_committed", + "confirmation_timeout", + "terminal_chain_rejection", + "post_broadcast_unresolved", +]); + +export function parseArgs(argv) { + if (argv.length === 2 && argv[0] === "--" && (argv[1] === "-h" || argv[1] === "--help")) { + return { + help: true, + maxRuns: DEFAULT_MAX_RUNS, + stableLimit: DEFAULT_STABLE_LIMIT, + backoffSeconds: DEFAULT_BACKOFF_SECONDS, + supervisorScript: DEFAULT_SUPERVISOR_SCRIPT, + supervisorArgs: [], + }; + } + const args = { + help: false, + maxRuns: DEFAULT_MAX_RUNS, + stableLimit: DEFAULT_STABLE_LIMIT, + backoffSeconds: DEFAULT_BACKOFF_SECONDS, + supervisorScript: DEFAULT_SUPERVISOR_SCRIPT, + supervisorArgs: [], + }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--") { + args.supervisorArgs = argv.slice(index + 1); + break; + } + if (arg === "-h" || arg === "--help") { + args.help = true; + continue; + } + if (arg === "--out-root") { + args.outRoot = valueAfter(argv, ++index, arg); + continue; + } + if (arg === "--max-runs") { + args.maxRuns = parsePositiveInteger(valueAfter(argv, ++index, arg), arg); + continue; + } + if (arg === "--stable-limit") { + args.stableLimit = parsePositiveInteger(valueAfter(argv, ++index, arg), arg); + continue; + } + if (arg === "--backoff-seconds") { + args.backoffSeconds = parseNonNegativeInteger(valueAfter(argv, ++index, arg), arg); + continue; + } + if (arg === "--supervisor-script") { + args.supervisorScript = valueAfter(argv, ++index, arg); + continue; + } + throw new Error(`Unknown argument before --: ${arg}`); + } + if (args.supervisorArgs.includes("--out-dir")) { + throw new Error("Do not pass supervisor --out-dir; use loop --out-root instead"); + } + return args; +} + +export function usage() { + return [ + "Usage: node scripts/ickb-supervisor-loop.mjs [loop-options] -- [supervisor-options]", + "Loop options:", + ` --out-root Default: ${SUPERVISOR_OUTPUT_ROOT}/loop-