|
| 1 | +import { |
| 2 | + type AgentCliCommand, |
| 3 | + type AgentCliRuntime, |
| 4 | + agentCliFail, |
| 5 | + agentCliOk, |
| 6 | +} from '@/lib/mothership/tools/handlers/agent-cli/types' |
| 7 | + |
| 8 | +/** |
| 9 | + * `workflow trace <runId>` — a run's trace rolled up for diagnosis, replacing |
| 10 | + * the hand-written span filters agents otherwise improvise (a field audit found |
| 11 | + * four divergent jq programs over the same trace, two of which disagreed 7 vs |
| 12 | + * 21 blocks because a shallow walk silently drops every block inside a child |
| 13 | + * workflow). The walk here is always recursive, errors come only from status |
| 14 | + * and error FIELDS (never from schema literals that merely contain the word |
| 15 | + * "error"), and subworkflow spans keep their nesting depth visible. |
| 16 | + */ |
| 17 | + |
| 18 | +interface TraceSpan { |
| 19 | + id?: string |
| 20 | + name?: string |
| 21 | + type?: string |
| 22 | + duration?: number |
| 23 | + durationMs?: number |
| 24 | + status?: string |
| 25 | + errorHandled?: boolean |
| 26 | + errorType?: string |
| 27 | + errorMessage?: string |
| 28 | + blockId?: string |
| 29 | + children?: TraceSpan[] |
| 30 | +} |
| 31 | + |
| 32 | +interface FlatSpan { |
| 33 | + name: string |
| 34 | + type: string |
| 35 | + durationMs: number |
| 36 | + depth: number |
| 37 | + status?: string |
| 38 | + errorMessage?: string |
| 39 | + errorHandled?: boolean |
| 40 | +} |
| 41 | + |
| 42 | +function flattenSpans(spans: TraceSpan[], depth: number, out: FlatSpan[]): void { |
| 43 | + for (const span of spans) { |
| 44 | + out.push({ |
| 45 | + name: span.name ?? span.blockId ?? 'unnamed', |
| 46 | + type: span.type ?? 'unknown', |
| 47 | + durationMs: span.durationMs ?? span.duration ?? 0, |
| 48 | + depth, |
| 49 | + ...(span.status !== undefined ? { status: span.status } : {}), |
| 50 | + ...(span.errorMessage !== undefined ? { errorMessage: span.errorMessage } : {}), |
| 51 | + ...(span.errorHandled !== undefined ? { errorHandled: span.errorHandled } : {}), |
| 52 | + }) |
| 53 | + if (span.children?.length) flattenSpans(span.children, depth + 1, out) |
| 54 | + } |
| 55 | +} |
| 56 | + |
| 57 | +function percentile(sorted: number[], p: number): number { |
| 58 | + if (sorted.length === 0) return 0 |
| 59 | + const index = Math.min(sorted.length - 1, Math.floor(p * sorted.length)) |
| 60 | + return sorted[index] ?? 0 |
| 61 | +} |
| 62 | + |
| 63 | +export const workflowTraceCommand: AgentCliCommand = { |
| 64 | + path: ['workflow', 'trace'], |
| 65 | + summary: 'Roll up one run trace: per-block timings, per-type stats, real errors, slowest path', |
| 66 | + usage: 'workflow trace <runId>', |
| 67 | + async execute(rest, runtime: AgentCliRuntime) { |
| 68 | + const runId = rest[0] |
| 69 | + if (!runId) return agentCliFail('Usage: sim workflow trace <runId>') |
| 70 | + const run = await runtime.client.request<{ |
| 71 | + data: { |
| 72 | + status?: string |
| 73 | + totalDurationMs?: number |
| 74 | + trigger?: string |
| 75 | + workflow?: { id?: string; name?: string } |
| 76 | + traceSpans?: TraceSpan[] |
| 77 | + } |
| 78 | + }>(`/api/v2/logs/${encodeURIComponent(runId)}`) |
| 79 | + const record = run.data |
| 80 | + const spans: FlatSpan[] = [] |
| 81 | + flattenSpans(record.traceSpans ?? [], 0, spans) |
| 82 | + if (spans.length === 0) { |
| 83 | + return agentCliOk( |
| 84 | + JSON.stringify( |
| 85 | + { |
| 86 | + runId, |
| 87 | + status: record.status, |
| 88 | + workflow: record.workflow?.name, |
| 89 | + note: 'No trace spans (spans age out on their own retention schedule).', |
| 90 | + }, |
| 91 | + null, |
| 92 | + 2 |
| 93 | + ) |
| 94 | + ) |
| 95 | + } |
| 96 | + |
| 97 | + const byType = new Map<string, number[]>() |
| 98 | + for (const span of spans) { |
| 99 | + const durations = byType.get(span.type) ?? [] |
| 100 | + durations.push(span.durationMs) |
| 101 | + byType.set(span.type, durations) |
| 102 | + } |
| 103 | + const typeStats = [...byType.entries()] |
| 104 | + .map(([type, durations]) => { |
| 105 | + const sorted = [...durations].sort((a, b) => a - b) |
| 106 | + return { |
| 107 | + type, |
| 108 | + count: sorted.length, |
| 109 | + totalMs: sorted.reduce((a, b) => a + b, 0), |
| 110 | + p50Ms: percentile(sorted, 0.5), |
| 111 | + maxMs: sorted[sorted.length - 1] ?? 0, |
| 112 | + } |
| 113 | + }) |
| 114 | + .sort((a, b) => b.totalMs - a.totalMs) |
| 115 | + |
| 116 | + const errors = spans |
| 117 | + .filter((span) => span.errorMessage || (span.status && /^(error|failed)$/i.test(span.status))) |
| 118 | + .map((span) => ({ |
| 119 | + block: span.name, |
| 120 | + type: span.type, |
| 121 | + depth: span.depth, |
| 122 | + ...(span.status ? { status: span.status } : {}), |
| 123 | + ...(span.errorMessage ? { message: span.errorMessage.slice(0, 400) } : {}), |
| 124 | + ...(span.errorHandled !== undefined ? { handled: span.errorHandled } : {}), |
| 125 | + })) |
| 126 | + |
| 127 | + const slowest = [...spans] |
| 128 | + .sort((a, b) => b.durationMs - a.durationMs) |
| 129 | + .slice(0, 10) |
| 130 | + .map((span) => ({ |
| 131 | + block: span.name, |
| 132 | + type: span.type, |
| 133 | + durationMs: span.durationMs, |
| 134 | + depth: span.depth, |
| 135 | + })) |
| 136 | + |
| 137 | + return agentCliOk( |
| 138 | + JSON.stringify( |
| 139 | + { |
| 140 | + runId, |
| 141 | + workflow: record.workflow?.name, |
| 142 | + status: record.status, |
| 143 | + trigger: record.trigger, |
| 144 | + totalDurationMs: record.totalDurationMs, |
| 145 | + blockCount: spans.length, |
| 146 | + maxDepth: Math.max(...spans.map((span) => span.depth)), |
| 147 | + errors, |
| 148 | + typeStats, |
| 149 | + slowestBlocks: slowest, |
| 150 | + }, |
| 151 | + null, |
| 152 | + 2 |
| 153 | + ) |
| 154 | + ) |
| 155 | + }, |
| 156 | +} |
0 commit comments