diff --git a/.changeset/cli-junit-output.md b/.changeset/cli-junit-output.md new file mode 100644 index 00000000..873cbe8f --- /dev/null +++ b/.changeset/cli-junit-output.md @@ -0,0 +1,5 @@ +--- +"@chatcops/cli": minor +--- + +Add a JUnit XML reporter and output-file support to chatcops test. diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index dd7a7f36..a77eb718 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -58,7 +58,10 @@ Test options: --repeat Run each test n times --concurrency Tests run at once (default 4) --verbose Show assertion details for every test - --reporter pretty (default) or json. Repeatable. + --reporter pretty (default), json or junit. Repeatable; each name once. + --output Write a single reporter to a file instead of stdout (replaces the file). + With several reporters, use name=file; file destinations must differ. + Do not combine --output with name=file. Parent folders are created. --json Same as --reporter json --record Call the APIs and save what they answer to __cassettes__ --replay Answer every API call from __cassettes__ (the default when CI is set) @@ -139,6 +142,7 @@ function parse(argv: string[]) { repeat: { type: 'string' }, verbose: { type: 'boolean' }, reporter: { type: 'string', multiple: true }, + output: { type: 'string' }, record: { type: 'boolean' }, live: { type: 'boolean' }, 'judge-model': { type: 'string' }, diff --git a/packages/cli/src/test/command.ts b/packages/cli/src/test/command.ts index 29ab8e45..c33f9ba5 100644 --- a/packages/cli/src/test/command.ts +++ b/packages/cli/src/test/command.ts @@ -1,5 +1,6 @@ import { performance } from 'node:perf_hooks'; -import { relative } from 'node:path'; +import { dirname, relative, resolve } from 'node:path'; +import { mkdir, writeFile } from 'node:fs/promises'; import { createProvider, isAIProvider, type AIProvider, type ProviderConfig } from '@chatcops/core'; import type { ChatCopsServerConfig } from '@chatcops/server'; import { ConfigError, type ChatCopsCliConfig } from '../config.js'; @@ -47,12 +48,32 @@ export async function runTests( } if (!fixtures.length) throw new ConfigError(`No tests match --filter ${values.filter}.`); - const reporters = (values.reporter?.length ? values.reporter : [values.json ? 'json' : 'pretty']).map((name) => { - const reporter = REPORTERS.get(name); - if (!reporter) throw new ConfigError(`--reporter ${name} doesn't exist. Use one of: ${[...REPORTERS.keys()].join(', ')}.`); - return reporter(io.stdout, { verbose: values.verbose === true, env: io.env ?? process.env }); - }); if (values.json && values.reporter?.length) throw new ConfigError('Use --reporter json, or --json, not both.'); + const selected = values.reporter?.length ? values.reporter : [values.json ? 'json' : 'pretty']; + if (values.output !== undefined && selected.length !== 1) throw new ConfigError('--output needs a single reporter. Use --reporter name=file for multiple reporters.'); + const names = new Set(); + const destinations = new Set(); + const reporters = selected.map((selection) => { + const separator = selection.indexOf('='); + const name = separator === -1 ? selection : selection.slice(0, separator); + if (!name.trim()) throw new ConfigError('--reporter needs a name.'); + const factory = REPORTERS.get(name); + if (!factory) throw new ConfigError(`--reporter ${name} doesn't exist. Use one of: ${[...REPORTERS.keys()].join(', ')}.`); + if (names.has(name)) throw new ConfigError(`--reporter ${name} was given more than once.`); + names.add(name); + if (separator !== -1 && values.output !== undefined) throw new ConfigError('Use --output or --reporter name=file, not both.'); + const destination = separator === -1 ? values.output : selection.slice(separator + 1); + if (destination !== undefined && !destination.trim()) throw new ConfigError('The reporter output path must not be empty.'); + const file = destination === undefined ? undefined : resolve(io.cwd, destination); + if (file !== undefined) { + const key = process.platform === 'win32' ? file.toLowerCase() : file; + if (destinations.has(key)) throw new ConfigError(`Reporters must use different output files: ${destination}.`); + destinations.add(key); + } + const chunks: string[] = []; + const output = file === undefined ? io.stdout : { write: (text: string) => chunks.push(text) }; + return { reporter: factory(output, { verbose: values.verbose === true, env: io.env ?? process.env }), file, chunks }; + }); // Started here so a missing key fails now, not as provider_unavailable on // every turn, and so turns can be priced by the model id. @@ -103,7 +124,7 @@ export async function runTests( ...(concurrency !== undefined ? { concurrency } : {}), ...(repeat !== undefined ? { repeat } : {}), onResult: (result) => { - for (const reporter of reporters) reporter.onResult?.(result); + for (const { reporter } of reporters) reporter.onResult?.(result); }, }); const elapsedMs = performance.now() - started; @@ -125,13 +146,24 @@ export async function runTests( ...(judge ? { judge: judge.id } : {}), ...(priced.length ? { costUsd: priced.reduce((a, b) => a + b, 0) } : {}), }; - for (const reporter of reporters) await reporter.onEnd?.(results, summary); + for (const { reporter } of reporters) await reporter.onEnd?.(results, summary); if (cassettes?.mode === 'record') { const written = await cassettes.save( fixtures.map((job) => ({ path: job.path ?? job.file, ran: job.tests.map((t) => t.name), all: job.fixture.tests.map((t) => t.name) })) ); for (const file of written) io.stderr.write(`Saved ${relative(io.cwd, file)}\n`); } + // Preserve terminal summaries and recordings even if a report file cannot be written. + for (const { file, chunks } of reporters) { + if (file !== undefined) { + try { + await mkdir(dirname(file), { recursive: true }); + await writeFile(file, chunks.join(''), 'utf8'); + } catch (error) { + throw new ConfigError(`Cannot write report to "${file}": ${(error as Error).message}`); + } + } + } return results.some((result) => !result.pass) ? EXIT.problems : EXIT.ok; } diff --git a/packages/cli/src/test/reporters/index.ts b/packages/cli/src/test/reporters/index.ts index 08077e4e..935895ea 100644 --- a/packages/cli/src/test/reporters/index.ts +++ b/packages/cli/src/test/reporters/index.ts @@ -2,6 +2,7 @@ import type { CassetteMode } from '../cassette.js'; import type { TestResult } from '../types.js'; import { jsonReporter } from './json.js'; import { prettyReporter } from './pretty.js'; +import { junitReporter } from './junit.js'; /** Where a reporter writes: stdout, unless the reporter is given a file. */ export interface ReporterOutput { @@ -38,4 +39,5 @@ export type ReporterFactory = (output: ReporterOutput, options: { verbose?: bool export const REPORTERS = new Map([ ['pretty', prettyReporter], ['json', jsonReporter], + ['junit', junitReporter], ]); diff --git a/packages/cli/src/test/reporters/junit.ts b/packages/cli/src/test/reporters/junit.ts new file mode 100644 index 00000000..56d3c612 --- /dev/null +++ b/packages/cli/src/test/reporters/junit.ts @@ -0,0 +1,66 @@ +import type { TestResult } from '../types.js'; +import type { ReporterFactory } from './index.js'; + +/** One testcase per aggregate result, including when a test repeats. */ +export const junitReporter: ReporterFactory = (output) => ({ + onEnd(results) { + const groups = new Map(); + for (const result of results) { + const group = groups.get(result.fixture) ?? []; + group.push(result); + groups.set(result.fixture, group); + } + const lines = ['', ``]; + for (const [fixture, tests] of groups) { + lines.push(` `); + for (const result of tests) { + lines.push(` `); + const outcome = problem(result); + if (outcome) { + const details: string[] = []; + for (const run of result.runs) { + if (run.error) details.push(`Run ${run.run}: ${run.error}`); + for (const [index, { turn, results: checks }] of run.turns.entries()) { + const failed = checks.filter((check) => !check.pass); + if (!failed.length) continue; + for (const check of failed) { + details.push(`Run ${run.run}, turn ${index + 1}: ${check.reason ?? Object.keys(check.assertion)[0]}`); + } + details.push(`Reply: ${turn.text}`); + } + } + lines.push(` <${outcome} message="${outcome === 'error' ? 'Execution error' : 'Assertion failed'}">${xml(details.join('\n'))}`); + } + lines.push(' '); + } + lines.push(' '); + } + lines.push(''); + output.write(`${lines.join('\n')}\n`); + }, +}); + +function problem(result: TestResult): 'error' | 'failure' | undefined { + // The aggregate pass-rate verdict wins. For a failed aggregate, any execution + // error takes precedence over assertion failures, so each testcase counts once. + if (result.pass) return undefined; + return result.runs.some((run) => run.error !== undefined) ? 'error' : 'failure'; +} + +function latency(result: TestResult): number { + return result.runs.reduce((total, run) => total + run.turns.reduce((sum, { turn }) => sum + turn.latencyMs, 0), 0); +} + +function seconds(milliseconds: number): string { + return (milliseconds / 1000).toFixed(3); +} + +function statistics(results: TestResult[]): string { + return `tests="${results.length}" failures="${results.filter((r) => problem(r) === 'failure').length}" errors="${results.filter((r) => problem(r) === 'error').length}" time="${seconds(results.reduce((sum, r) => sum + latency(r), 0))}"`; +} + +/** XML 1.0 characters and entities, including whitespace in attribute values. */ +function xml(text: string): string { + const entities: Record = { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''', '\r': ' ', '\n': ' ', '\t': ' ' }; + return text.replace(/[^\t\n\r\u0020-\uD7FF\uE000-\uFFFD\u{10000}-\u{10FFFF}]/gu, '').replace(/[&<>"'\r\n\t]/g, (character) => entities[character]); +} diff --git a/packages/cli/tests/cassette.test.ts b/packages/cli/tests/cassette.test.ts index 6a4780f6..0872c1fd 100644 --- a/packages/cli/tests/cassette.test.ts +++ b/packages/cli/tests/cassette.test.ts @@ -108,6 +108,22 @@ const entries = (dir: string) => .map((line) => JSON.parse(line)); describe('chatcops test --record and --replay', () => { + it('saves recordings even when writing the report fails', async () => { + const dir = project({ 'evals/pricing.chat.yaml': PRICING }); + const recorded = await run(['test', '--record', '--output', 'evals'], dir); + expect(recorded.code).toBe(EXIT.unusable); + expect(recorded.stderr).toContain('Cannot write report to'); + expect(recorded.stderr).toContain(`Saved ${join('evals', '__cassettes__', 'pricing.chat.jsonl')}`); + expect(existsSync(cassetteOf(dir))).toBe(true); + expect(entries(dir)).toHaveLength(3); + expect(hits).toHaveLength(3); + + hits = []; + const replayed = await run(['test', '--replay'], dir); + expect(replayed.code).toBe(EXIT.ok); + expect(hits).toHaveLength(0); + }); + it('records each call, then replays them without the network', async () => { const dir = project({ 'evals/pricing.chat.yaml': PRICING }); const recorded = await run(['test', '--record'], dir); diff --git a/packages/cli/tests/junit.test.ts b/packages/cli/tests/junit.test.ts new file mode 100644 index 00000000..b05a2eb2 --- /dev/null +++ b/packages/cli/tests/junit.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from 'vitest'; +import { junitReporter } from '../src/test/reporters/junit.js'; +import type { TestResult, TestRunResult } from '../src/test/types.js'; + +function run(overrides: Partial = {}): TestRunResult { + return { + fixture: 'evals/pricing.chat.yaml', test: 'quotes the price', run: 1, pass: true, + turns: [{ turn: { user: 'price?', text: '$19', latencyMs: 1200, toolCalls: [], sources: [], leads: [] }, results: [] }], + ...overrides, + }; +} + +function result(overrides: Partial = {}): TestResult { + return { fixture: 'evals/pricing.chat.yaml', test: 'quotes the price', pass: true, passRate: 1, runs: [run()], durationMs: 99999, ...overrides }; +} + +async function report(results: TestResult[]): Promise { + let text = ''; + await junitReporter({ write: (chunk) => { text += chunk; } }, {}).onEnd?.(results, { mode: 'live', elapsedMs: 99999, costIncomplete: false }); + return text; +} + +describe('JUnit reporter', () => { + it('emits a complete XML document for a passing test, using turn latency rather than wall time', async () => { + expect(await report([result()])).toBe(` + + + + + + +`); + }); + + it('escapes attributes and failure details, with run, turn, reason and reply', async () => { + const special = ` & "quoted" 'single'`; + const escaped = '<tag> & "quoted" 'single''; + const failed = run({ pass: false }); + failed.turns.push({ turn: { ...failed.turns[0].turn, text: special, latencyMs: 300 }, results: [ + { assertion: { contains: 'x' }, pass: false, reason: special }, + { assertion: { equals: 'x' }, pass: false }, + ] }); + const xml = await report([result({ fixture: special, test: `${special}\u0000`, pass: false, runs: [failed] })]); + expect(xml).toContain(`classname="${escaped}" name="${escaped}" time="1.500"`); + expect(xml).toContain(''); + expect(xml).toContain(`Run 1, turn 2: ${escaped} Run 1, turn 2: equals Reply: ${escaped}`); + expect(xml).not.toContain('\u0000'); + expect(xml).not.toContain(' { + const failure = result({ test: 'fails', pass: false, runs: [run({ pass: false })] }); + const error = result({ fixture: 'evals/other.chat.yaml', test: 'throws', pass: false, runs: [run({ pass: false, error: 'provider & unavailable' })] }); + const xml = await report([result(), failure, error]); + expect(xml).toContain(''); + expect(xml).toContain(''); + expect(xml).toContain(''); + expect(xml.match(/Run 1: provider <down> & unavailable'); + }); + + it('respects aggregate pass-rate verdicts and sums latency across all repeats', async () => { + const runs = [run(), run({ run: 2, pass: false, error: 'failed repeat' })]; + const xml = await report([result({ runs, passRate: 0.5 })]); + expect(xml).toContain('tests="1" failures="0" errors="0" time="2.400"'); + expect(xml).not.toMatch(/<(failure|error) /); + const failed = await report([result({ runs, passRate: 0.5, pass: false })]); + expect(failed).toContain('failures="0" errors="1"'); + expect(failed).toContain('Run 2: failed repeat'); + }); + + it('gives execution errors precedence in a failed result with mixed runs', async () => { + const failed = run({ pass: false }); + failed.turns[0].results.push({ assertion: { contains: 'x' }, pass: false, reason: 'missing x' }); + const xml = await report([result({ pass: false, runs: [failed, run({ run: 2, pass: false, error: 'model down' })] })]); + expect(xml).toContain('Run 1, turn 1: missing x Reply: $19 Run 2: model down'); + expect(xml).not.toContain(' { + expect(await report([])).toBe('\n\n\n'); + }); +}); diff --git a/packages/cli/tests/test-command.test.ts b/packages/cli/tests/test-command.test.ts index 057290d0..c567df3d 100644 --- a/packages/cli/tests/test-command.test.ts +++ b/packages/cli/tests/test-command.test.ts @@ -1,10 +1,11 @@ -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { dirname, join, resolve } from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { EXIT, main } from '../src/index.js'; import { check, unknownAssertions } from '../src/test/assertions/index.js'; import { matchesFilter } from '../src/test/load.js'; import type { TurnResult } from '../src/test/types.js'; +import { REPORTERS } from '../src/test/reporters/index.js'; const TMP = resolve(__dirname, '.tmp-test'); @@ -322,11 +323,108 @@ tests: expect(both.stdout).toContain(' ✓ says hello'); expect(JSON.parse(both.stdout.trim().split('\n').at(-1) ?? '')).toMatchObject({ passed: 2 }); const unknown = await run(['test', '--reporter', 'html'], dir); - expect(unknown.stderr).toContain("--reporter html doesn't exist. Use one of: pretty, json."); + expect(unknown.stderr).toContain("--reporter html doesn't exist. Use one of: pretty, json, junit."); expect(unknown.code).toBe(EXIT.unusable); expect((await run(['test', '--json', '--reporter', 'json'], dir)).stderr).toContain('not both'); }); + it('writes JUnit to --output relative to io.cwd and replaces existing files without stdout duplication', async () => { + const dir = project({ 'evals/pricing.chat.yaml': PRICING, 'evals.xml': 'old contents' }); + const result = await run(['test', '--reporter', 'junit', '--output', 'evals.xml'], dir); + expect(result).toEqual({ code: EXIT.ok, stdout: '', stderr: '' }); + const xml = readFileSync(join(dir, 'evals.xml'), 'utf8'); + expect(xml).toMatch(/^<\?xml version="1.0" encoding="UTF-8"\?>/); + expect(xml).toContain('\n$/); + }); + + it('routes reporters independently and splits destinations on the first equals sign', async () => { + const dir = project({ 'evals/pricing.chat.yaml': PRICING }); + const mixed = await run(['test', '--reporter', 'pretty', '--reporter', 'junit=report=evals.xml'], dir); + expect(mixed.code).toBe(EXIT.ok); + expect(mixed.stdout).toContain('2 passed, 0 failed.'); + expect(mixed.stdout).not.toContain(''); + }); + + it('supports --output with the default reporter and --json', async () => { + const dir = project({ 'evals/pricing.chat.yaml': PRICING }); + expect((await run(['test', '--output', 'pretty.txt'], dir)).stdout).toBe(''); + expect(readFileSync(join(dir, 'pretty.txt'), 'utf8')).toContain('2 passed, 0 failed.'); + expect((await run(['test', '--json', '--output', 'evals.json'], dir)).code).toBe(EXIT.ok); + expect(JSON.parse(readFileSync(join(dir, 'evals.json'), 'utf8')).passed).toBe(2); + }); + + it('buffers onResult and asynchronous onEnd output before writing the file', async () => { + const dir = project({ 'evals/pricing.chat.yaml': PRICING }); + REPORTERS.set('buffer-test', (output) => ({ + onResult(result) { output.write(`${result.test}\n`); }, + async onEnd() { await Promise.resolve(); output.write('done\n'); }, + })); + try { + expect((await run(['test', '--reporter', 'buffer-test=buffer.txt'], dir)).stdout).toBe(''); + const text = readFileSync(join(dir, 'buffer.txt'), 'utf8'); + expect(text).toContain('quotes the team price\n'); + expect(text).toContain('says hello\n'); + expect(text).toMatch(/done\n$/); + } finally { + REPORTERS.delete('buffer-test'); + } + }); + + it('writes reports for failed evals and creates missing parent folders', async () => { + const dir = project({ 'evals/pricing.chat.yaml': PRICING.replace('contains: "$19"', 'contains: "$29"') }); + const failed = await run(['test', '--reporter', 'junit=failed.xml'], dir); + expect(failed.code).toBe(EXIT.problems); + expect(failed.stdout).toBe(''); + expect(readFileSync(join(dir, 'failed.xml'), 'utf8')).toContain(''); + const nested = await run(['test', '--reporter', 'junit', '--output', 'missing/report.xml'], dir); + expect(nested.code).toBe(EXIT.problems); + expect(nested.stdout).toBe(''); + expect(nested.stderr).toBe(''); + expect(readFileSync(join(dir, 'missing/report.xml'), 'utf8')).toContain(''); + }); + + it('finishes later reporters before reporting a file write error', async () => { + const dir = project({ 'evals/pricing.chat.yaml': PRICING }); + const unwritable = await run(['test', '--reporter', 'junit=evals', '--reporter', 'pretty'], dir); + expect(unwritable.code).toBe(EXIT.unusable); + expect(unwritable.stdout).toContain('2 passed, 0 failed.'); + expect(unwritable.stderr).toContain('Cannot write report to'); + }); + + it.each([ + [['--reporter', ''], 'needs a name'], + [['--reporter', '=report.xml'], 'needs a name'], + [['--reporter', 'junit='], 'must not be empty'], + [['--output', ''], 'must not be empty'], + [['--reporter', 'html=report.xml'], "doesn't exist"], + [['--reporter', 'junit', '--reporter', 'json', '--output', 'report.xml'], 'single reporter'], + [['--reporter', 'junit=a.xml', '--output', 'b.xml'], 'not both'], + [['--reporter', 'junit', '--reporter', 'junit=a.xml'], 'more than once'], + [['--reporter', 'junit=a.xml', '--reporter', 'json=./a.xml'], 'different output files'], + ])('rejects invalid reporter destinations %j', async (args, message) => { + const dir = project({ 'evals/pricing.chat.yaml': PRICING }); + const result = await run(['test', ...args], dir); + expect(result.code).toBe(EXIT.unusable); + expect(result.stderr).toContain(message); + expect(result.stdout).toBe(''); + }); + + it('lists JUnit and output destinations in help', async () => { + const help = await run(['test', '--help'], TMP); + expect(help.stdout).toContain('junit'); + expect(help.stdout).toContain('--output '); + expect(help.stdout).toContain('name=file'); + }); + it('loads module eval files', async () => { const dir = project({ 'evals/pricing.chat.mjs': `export default { version: 1, name: 'Module', tests: [{ name: 'js', turns: [{ user: 'price', expect: [{ contains: '$19' }] }] }] };`, diff --git a/website/src/content/docs/core/testing.mdx b/website/src/content/docs/core/testing.mdx index a533c68b..c4034809 100644 --- a/website/src/content/docs/core/testing.mdx +++ b/website/src/content/docs/core/testing.mdx @@ -233,7 +233,8 @@ export default defineConfig({ | `--filter ` | Only tests whose name contains the text, or `tag:` for a tag | | `--repeat ` | Run every test n times, instead of each test's own `repeat` | | `--concurrency ` | Tests run at once. Default 4. | -| `--reporter ` | `pretty` (the default) or `json`, which prints one JSON object with every turn's reply, tool calls, sources and token usage. Repeatable. | +| `--reporter ` | `pretty` (the default), `json` (every turn's reply, tool calls, sources and token usage), or `junit` (JUnit XML for CI). Repeatable, with each name used once. Add `=file` to write that reporter to a file. | +| `--output ` | Write a single reporter's complete output to a file instead of stdout. Also works with the default reporter. | | `--json` | The same as `--reporter json` | | `--judge-model ` | The judge for `judge` assertions: `gemini`, `openai` or `claude`, with a model id or without one for the default | | `--max-cost ` | Stop starting turns once the run has spent this much. Needs `prices`. | @@ -284,6 +285,38 @@ A few things to know about recordings: - `repeat` records every run, and replay plays them back in the same order. - Replays answer at once, so `latency` in a replayed run is not the real latency. +### JUnit reports for CI + +Write a JUnit report for a CI test report integration: + +```bash +npx chatcops test --reporter junit --output evals.xml +``` + +To keep the readable summary in the terminal and save XML separately, give the JUnit reporter a destination: + +```bash +npx chatcops test --reporter pretty --reporter junit=evals.xml +``` + +File paths are relative to the current folder. Parent folders are created as needed, and an existing report is replaced after reporting finishes. Paths containing `=` work: only the first `=` separates the reporter name from its destination. Each reporter name can be used once, and file destinations must differ. `--output` is a shortcut for one reporter; it cannot be combined with multiple reporters or `name=file`. A file write error exits 2, after all reporters have finished and any recordings have been saved. + +JUnit groups tests by eval file. Each test's time is the sum of its turns' latencies across all repeats, in seconds. The aggregate pass-rate verdict decides whether the testcase passed. A failed test with any execution error gets an ``; other failed tests get a `` with the assertion reasons, turn numbers and replies. + +For example, GitLab CI can display the report even when an eval fails (record the API calls first so CI can replay them): + +```yaml +evals: + image: node:22 + script: + - npm ci + - npx chatcops test --reporter junit --output evals.xml + artifacts: + when: always + reports: + junit: evals.xml +``` + ### GitHub Actions ```yaml