feat(cli): add junit reporter and output support

.changeset/cli-junit-output.md

@@ -0,0 +1,5 @@
+---
+"@chatcops/cli": minor
+---
+
+Add a JUnit XML reporter and output-file support to chatcops test.

packages/cli/src/main.ts

@@ -58,7 +58,10 @@ Test options:
   --repeat <n>          Run each test n times
   --concurrency <n>     Tests run at once (default 4)
   --verbose             Show assertion details for every test
-  --reporter <name>     pretty (default) or json. Repeatable.
+  --reporter <name[=file]>  pretty (default), json or junit. Repeatable; each name once.
+  --output <file>       Write a single reporter to a file instead of stdout (replaces the file).
+                        With several reporters, use name=file; file destinations must differ.
+                        Do not combine --output with name=file. Parent folders are created.
   --json                Same as --reporter json
   --record              Call the APIs and save what they answer to __cassettes__
   --replay              Answer every API call from __cassettes__ (the default when CI is set)
@@ -139,6 +142,7 @@ function parse(argv: string[]) {
       repeat: { type: 'string' },
       verbose: { type: 'boolean' },
       reporter: { type: 'string', multiple: true },
+      output: { type: 'string' },
       record: { type: 'boolean' },
       live: { type: 'boolean' },
       'judge-model': { type: 'string' },

packages/cli/src/test/command.ts

@@ -1,5 +1,6 @@
 import { performance } from 'node:perf_hooks';
-import { relative } from 'node:path';
+import { dirname, relative, resolve } from 'node:path';
+import { mkdir, writeFile } from 'node:fs/promises';
 import { createProvider, isAIProvider, type AIProvider, type ProviderConfig } from '@chatcops/core';
 import type { ChatCopsServerConfig } from '@chatcops/server';
 import { ConfigError, type ChatCopsCliConfig } from '../config.js';
@@ -47,12 +48,32 @@ export async function runTests(
   }
   if (!fixtures.length) throw new ConfigError(`No tests match --filter ${values.filter}.`);
 
-  const reporters = (values.reporter?.length ? values.reporter : [values.json ? 'json' : 'pretty']).map((name) => {
-    const reporter = REPORTERS.get(name);
-    if (!reporter) throw new ConfigError(`--reporter ${name} doesn't exist. Use one of: ${[...REPORTERS.keys()].join(', ')}.`);
-    return reporter(io.stdout, { verbose: values.verbose === true, env: io.env ?? process.env });
-  });
   if (values.json && values.reporter?.length) throw new ConfigError('Use --reporter json, or --json, not both.');
+  const selected = values.reporter?.length ? values.reporter : [values.json ? 'json' : 'pretty'];
+  if (values.output !== undefined && selected.length !== 1) throw new ConfigError('--output needs a single reporter. Use --reporter name=file for multiple reporters.');
+  const names = new Set<string>();
+  const destinations = new Set<string>();
+  const reporters = selected.map((selection) => {
+    const separator = selection.indexOf('=');
+    const name = separator === -1 ? selection : selection.slice(0, separator);
+    if (!name.trim()) throw new ConfigError('--reporter needs a name.');
+    const factory = REPORTERS.get(name);
+    if (!factory) throw new ConfigError(`--reporter ${name} doesn't exist. Use one of: ${[...REPORTERS.keys()].join(', ')}.`);
+    if (names.has(name)) throw new ConfigError(`--reporter ${name} was given more than once.`);
+    names.add(name);
+    if (separator !== -1 && values.output !== undefined) throw new ConfigError('Use --output or --reporter name=file, not both.');
+    const destination = separator === -1 ? values.output : selection.slice(separator + 1);
+    if (destination !== undefined && !destination.trim()) throw new ConfigError('The reporter output path must not be empty.');
+    const file = destination === undefined ? undefined : resolve(io.cwd, destination);
+    if (file !== undefined) {
+      const key = process.platform === 'win32' ? file.toLowerCase() : file;
+      if (destinations.has(key)) throw new ConfigError(`Reporters must use different output files: ${destination}.`);
+      destinations.add(key);
+    }
+    const chunks: string[] = [];
+    const output = file === undefined ? io.stdout : { write: (text: string) => chunks.push(text) };
+    return { reporter: factory(output, { verbose: values.verbose === true, env: io.env ?? process.env }), file, chunks };
+  });
 
   // Started here so a missing key fails now, not as provider_unavailable on
   // every turn, and so turns can be priced by the model id.
@@ -103,7 +124,7 @@ export async function runTests(
     ...(concurrency !== undefined ? { concurrency } : {}),
     ...(repeat !== undefined ? { repeat } : {}),
     onResult: (result) => {
-      for (const reporter of reporters) reporter.onResult?.(result);
+      for (const { reporter } of reporters) reporter.onResult?.(result);
     },
   });
   const elapsedMs = performance.now() - started;
@@ -125,13 +146,24 @@ export async function runTests(
     ...(judge ? { judge: judge.id } : {}),
     ...(priced.length ? { costUsd: priced.reduce((a, b) => a + b, 0) } : {}),
   };
-  for (const reporter of reporters) await reporter.onEnd?.(results, summary);
+  for (const { reporter } of reporters) await reporter.onEnd?.(results, summary);
   if (cassettes?.mode === 'record') {
     const written = await cassettes.save(
       fixtures.map((job) => ({ path: job.path ?? job.file, ran: job.tests.map((t) => t.name), all: job.fixture.tests.map((t) => t.name) }))
     );
     for (const file of written) io.stderr.write(`Saved ${relative(io.cwd, file)}\n`);
   }
+  // Preserve terminal summaries and recordings even if a report file cannot be written.
+  for (const { file, chunks } of reporters) {
+    if (file !== undefined) {
+      try {
+        await mkdir(dirname(file), { recursive: true });
+        await writeFile(file, chunks.join(''), 'utf8');
+      } catch (error) {
+        throw new ConfigError(`Cannot write report to "${file}": ${(error as Error).message}`);
+      }
+    }
+  }
   return results.some((result) => !result.pass) ? EXIT.problems : EXIT.ok;
 }
 

packages/cli/src/test/reporters/index.ts

@@ -2,6 +2,7 @@ import type { CassetteMode } from '../cassette.js';
 import type { TestResult } from '../types.js';
 import { jsonReporter } from './json.js';
 import { prettyReporter } from './pretty.js';
+import { junitReporter } from './junit.js';
 
 /** Where a reporter writes: stdout, unless the reporter is given a file. */
 export interface ReporterOutput {
@@ -38,4 +39,5 @@ export type ReporterFactory = (output: ReporterOutput, options: { verbose?: bool
 export const REPORTERS = new Map<string, ReporterFactory>([
   ['pretty', prettyReporter],
   ['json', jsonReporter],
+  ['junit', junitReporter],
 ]);

packages/cli/src/test/reporters/junit.ts

@@ -0,0 +1,66 @@
+import type { TestResult } from '../types.js';
+import type { ReporterFactory } from './index.js';
+
+/** One testcase per aggregate result, including when a test repeats. */
+export const junitReporter: ReporterFactory = (output) => ({
+  onEnd(results) {
+    const groups = new Map<string, TestResult[]>();
+    for (const result of results) {
+      const group = groups.get(result.fixture) ?? [];
+      group.push(result);
+      groups.set(result.fixture, group);
+    }
+    const lines = ['<?xml version="1.0" encoding="UTF-8"?>', `<testsuites ${statistics(results)}>`];
+    for (const [fixture, tests] of groups) {
+      lines.push(`  <testsuite name="${xml(fixture)}" ${statistics(tests)}>`);
+      for (const result of tests) {
+        lines.push(`    <testcase classname="${xml(fixture)}" name="${xml(result.test)}" time="${seconds(latency(result))}">`);
+        const outcome = problem(result);
+        if (outcome) {
+          const details: string[] = [];
+          for (const run of result.runs) {
+            if (run.error) details.push(`Run ${run.run}: ${run.error}`);
+            for (const [index, { turn, results: checks }] of run.turns.entries()) {
+              const failed = checks.filter((check) => !check.pass);
+              if (!failed.length) continue;
+              for (const check of failed) {
+                details.push(`Run ${run.run}, turn ${index + 1}: ${check.reason ?? Object.keys(check.assertion)[0]}`);
+              }
+              details.push(`Reply: ${turn.text}`);
+            }
+          }
+          lines.push(`      <${outcome} message="${outcome === 'error' ? 'Execution error' : 'Assertion failed'}">${xml(details.join('\n'))}</${outcome}>`);
+        }
+        lines.push('    </testcase>');
+      }
+      lines.push('  </testsuite>');
+    }
+    lines.push('</testsuites>');
+    output.write(`${lines.join('\n')}\n`);
+  },
+});
+
+function problem(result: TestResult): 'error' | 'failure' | undefined {
+  // The aggregate pass-rate verdict wins. For a failed aggregate, any execution
+  // error takes precedence over assertion failures, so each testcase counts once.
+  if (result.pass) return undefined;
+  return result.runs.some((run) => run.error !== undefined) ? 'error' : 'failure';
+}
+
+function latency(result: TestResult): number {
+  return result.runs.reduce((total, run) => total + run.turns.reduce((sum, { turn }) => sum + turn.latencyMs, 0), 0);
+}
+
+function seconds(milliseconds: number): string {
+  return (milliseconds / 1000).toFixed(3);
+}
+
+function statistics(results: TestResult[]): string {
+  return `tests="${results.length}" failures="${results.filter((r) => problem(r) === 'failure').length}" errors="${results.filter((r) => problem(r) === 'error').length}" time="${seconds(results.reduce((sum, r) => sum + latency(r), 0))}"`;
+}
+
+/** XML 1.0 characters and entities, including whitespace in attribute values. */
+function xml(text: string): string {
+  const entities: Record<string, string> = { '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&apos;', '\r': '&#13;', '\n': '&#10;', '\t': '&#9;' };
+  return text.replace(/[^\t\n\r\u0020-\uD7FF\uE000-\uFFFD\u{10000}-\u{10FFFF}]/gu, '').replace(/[&<>"'\r\n\t]/g, (character) => entities[character]);
+}

packages/cli/tests/cassette.test.ts

@@ -108,6 +108,22 @@ const entries = (dir: string) =>
     .map((line) => JSON.parse(line));
 
 describe('chatcops test --record and --replay', () => {
+  it('saves recordings even when writing the report fails', async () => {
+    const dir = project({ 'evals/pricing.chat.yaml': PRICING });
+    const recorded = await run(['test', '--record', '--output', 'evals'], dir);
+    expect(recorded.code).toBe(EXIT.unusable);
+    expect(recorded.stderr).toContain('Cannot write report to');
+    expect(recorded.stderr).toContain(`Saved ${join('evals', '__cassettes__', 'pricing.chat.jsonl')}`);
+    expect(existsSync(cassetteOf(dir))).toBe(true);
+    expect(entries(dir)).toHaveLength(3);
+    expect(hits).toHaveLength(3);
+
+    hits = [];
+    const replayed = await run(['test', '--replay'], dir);
+    expect(replayed.code).toBe(EXIT.ok);
+    expect(hits).toHaveLength(0);
+  });
+
   it('records each call, then replays them without the network', async () => {
     const dir = project({ 'evals/pricing.chat.yaml': PRICING });
     const recorded = await run(['test', '--record'], dir);

packages/cli/tests/junit.test.ts

@@ -0,0 +1,83 @@
+import { describe, expect, it } from 'vitest';
+import { junitReporter } from '../src/test/reporters/junit.js';
+import type { TestResult, TestRunResult } from '../src/test/types.js';
+
+function run(overrides: Partial<TestRunResult> = {}): TestRunResult {
+  return {
+    fixture: 'evals/pricing.chat.yaml', test: 'quotes the price', run: 1, pass: true,
+    turns: [{ turn: { user: 'price?', text: '$19', latencyMs: 1200, toolCalls: [], sources: [], leads: [] }, results: [] }],
+    ...overrides,
+  };
+}
+
+function result(overrides: Partial<TestResult> = {}): TestResult {
+  return { fixture: 'evals/pricing.chat.yaml', test: 'quotes the price', pass: true, passRate: 1, runs: [run()], durationMs: 99999, ...overrides };
+}
+
+async function report(results: TestResult[]): Promise<string> {
+  let text = '';
+  await junitReporter({ write: (chunk) => { text += chunk; } }, {}).onEnd?.(results, { mode: 'live', elapsedMs: 99999, costIncomplete: false });
+  return text;
+}
+
+describe('JUnit reporter', () => {
+  it('emits a complete XML document for a passing test, using turn latency rather than wall time', async () => {
+    expect(await report([result()])).toBe(`<?xml version="1.0" encoding="UTF-8"?>
+<testsuites tests="1" failures="0" errors="0" time="1.200">
+  <testsuite name="evals/pricing.chat.yaml" tests="1" failures="0" errors="0" time="1.200">
+    <testcase classname="evals/pricing.chat.yaml" name="quotes the price" time="1.200">
+    </testcase>
+  </testsuite>
+</testsuites>
+`);
+  });
+
+  it('escapes attributes and failure details, with run, turn, reason and reply', async () => {
+    const special = `<tag> & "quoted" 'single'`;
+    const escaped = '&lt;tag&gt; &amp; &quot;quoted&quot; &apos;single&apos;';
+    const failed = run({ pass: false });
+    failed.turns.push({ turn: { ...failed.turns[0].turn, text: special, latencyMs: 300 }, results: [
+      { assertion: { contains: 'x' }, pass: false, reason: special },
+      { assertion: { equals: 'x' }, pass: false },
+    ] });
+    const xml = await report([result({ fixture: special, test: `${special}\u0000`, pass: false, runs: [failed] })]);
+    expect(xml).toContain(`classname="${escaped}" name="${escaped}" time="1.500"`);
+    expect(xml).toContain('<failure message="Assertion failed">');
+    expect(xml).toContain(`Run 1, turn 2: ${escaped}&#10;Run 1, turn 2: equals&#10;Reply: ${escaped}</failure>`);
+    expect(xml).not.toContain('\u0000');
+    expect(xml).not.toContain('<error');
+  });
+
+  it('groups eval files and counts passing, assertion failures and execution errors once each', async () => {
+    const failure = result({ test: 'fails', pass: false, runs: [run({ pass: false })] });
+    const error = result({ fixture: 'evals/other.chat.yaml', test: 'throws', pass: false, runs: [run({ pass: false, error: 'provider <down> & unavailable' })] });
+    const xml = await report([result(), failure, error]);
+    expect(xml).toContain('<testsuites tests="3" failures="1" errors="1" time="3.600">');
+    expect(xml).toContain('<testsuite name="evals/pricing.chat.yaml" tests="2" failures="1" errors="0" time="2.400">');
+    expect(xml).toContain('<testsuite name="evals/other.chat.yaml" tests="1" failures="0" errors="1" time="1.200">');
+    expect(xml.match(/<testcase /g)).toHaveLength(3);
+    expect(xml).toContain('<error message="Execution error">Run 1: provider &lt;down&gt; &amp; unavailable</error>');
+  });
+
+  it('respects aggregate pass-rate verdicts and sums latency across all repeats', async () => {
+    const runs = [run(), run({ run: 2, pass: false, error: 'failed repeat' })];
+    const xml = await report([result({ runs, passRate: 0.5 })]);
+    expect(xml).toContain('tests="1" failures="0" errors="0" time="2.400"');
+    expect(xml).not.toMatch(/<(failure|error) /);
+    const failed = await report([result({ runs, passRate: 0.5, pass: false })]);
+    expect(failed).toContain('failures="0" errors="1"');
+    expect(failed).toContain('Run 2: failed repeat');
+  });
+
+  it('gives execution errors precedence in a failed result with mixed runs', async () => {
+    const failed = run({ pass: false });
+    failed.turns[0].results.push({ assertion: { contains: 'x' }, pass: false, reason: 'missing x' });
+    const xml = await report([result({ pass: false, runs: [failed, run({ run: 2, pass: false, error: 'model down' })] })]);
+    expect(xml).toContain('<error message="Execution error">Run 1, turn 1: missing x&#10;Reply: $19&#10;Run 2: model down</error>');
+    expect(xml).not.toContain('<failure');
+  });
+
+  it('emits a valid empty report', async () => {
+    expect(await report([])).toBe('<?xml version="1.0" encoding="UTF-8"?>\n<testsuites tests="0" failures="0" errors="0" time="0.000">\n</testsuites>\n');
+  });
+});