From 1bc53ac92c639ff9e0a92890a2091ba47bccab4c Mon Sep 17 00:00:00 2001 From: Siddarth Date: Fri, 6 Nov 2026 19:23:59 +0530 Subject: [PATCH 1/2] feat(cli): add junit reporter and output support --- .changeset/cli-junit-output.md | 5 ++ packages/cli/src/main.ts | 6 +- packages/cli/src/test/command.ts | 46 +++++++++-- packages/cli/src/test/reporters/index.ts | 2 + packages/cli/src/test/reporters/junit.ts | 66 ++++++++++++++++ packages/cli/tests/junit.test.ts | 83 ++++++++++++++++++++ packages/cli/tests/test-command.test.ts | 94 ++++++++++++++++++++++- website/src/content/docs/core/testing.mdx | 35 ++++++++- 8 files changed, 325 insertions(+), 12 deletions(-) create mode 100644 .changeset/cli-junit-output.md create mode 100644 packages/cli/src/test/reporters/junit.ts create mode 100644 packages/cli/tests/junit.test.ts diff --git a/.changeset/cli-junit-output.md b/.changeset/cli-junit-output.md new file mode 100644 index 00000000..873cbe8f --- /dev/null +++ b/.changeset/cli-junit-output.md @@ -0,0 +1,5 @@ +--- +"@chatcops/cli": minor +--- + +Add a JUnit XML reporter and output-file support to chatcops test. diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index dd7a7f36..7963a448 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -58,7 +58,10 @@ Test options: --repeat Run each test n times --concurrency Tests run at once (default 4) --verbose Show assertion details for every test - --reporter pretty (default) or json. Repeatable. + --reporter pretty (default), json or junit. Repeatable; each name once. + --output Write a single reporter to a file instead of stdout (replaces the file). + With several reporters, use name=file; file destinations must differ. + Do not combine --output with name=file. Parent folders must exist. --json Same as --reporter json --record Call the APIs and save what they answer to __cassettes__ --replay Answer every API call from __cassettes__ (the default when CI is set) @@ -139,6 +142,7 @@ function parse(argv: string[]) { repeat: { type: 'string' }, verbose: { type: 'boolean' }, reporter: { type: 'string', multiple: true }, + output: { type: 'string' }, record: { type: 'boolean' }, live: { type: 'boolean' }, 'judge-model': { type: 'string' }, diff --git a/packages/cli/src/test/command.ts b/packages/cli/src/test/command.ts index 29ab8e45..9cef474c 100644 --- a/packages/cli/src/test/command.ts +++ b/packages/cli/src/test/command.ts @@ -1,5 +1,6 @@ import { performance } from 'node:perf_hooks'; -import { relative } from 'node:path'; +import { relative, resolve } from 'node:path'; +import { writeFile } from 'node:fs/promises'; import { createProvider, isAIProvider, type AIProvider, type ProviderConfig } from '@chatcops/core'; import type { ChatCopsServerConfig } from '@chatcops/server'; import { ConfigError, type ChatCopsCliConfig } from '../config.js'; @@ -47,12 +48,32 @@ export async function runTests( } if (!fixtures.length) throw new ConfigError(`No tests match --filter ${values.filter}.`); - const reporters = (values.reporter?.length ? values.reporter : [values.json ? 'json' : 'pretty']).map((name) => { - const reporter = REPORTERS.get(name); - if (!reporter) throw new ConfigError(`--reporter ${name} doesn't exist. Use one of: ${[...REPORTERS.keys()].join(', ')}.`); - return reporter(io.stdout, { verbose: values.verbose === true, env: io.env ?? process.env }); - }); if (values.json && values.reporter?.length) throw new ConfigError('Use --reporter json, or --json, not both.'); + const selected = values.reporter?.length ? values.reporter : [values.json ? 'json' : 'pretty']; + if (values.output !== undefined && selected.length !== 1) throw new ConfigError('--output needs a single reporter. Use --reporter name=file for multiple reporters.'); + const names = new Set(); + const destinations = new Set(); + const reporters = selected.map((selection) => { + const separator = selection.indexOf('='); + const name = separator === -1 ? selection : selection.slice(0, separator); + if (!name.trim()) throw new ConfigError('--reporter needs a name.'); + const factory = REPORTERS.get(name); + if (!factory) throw new ConfigError(`--reporter ${name} doesn't exist. Use one of: ${[...REPORTERS.keys()].join(', ')}.`); + if (names.has(name)) throw new ConfigError(`--reporter ${name} was given more than once.`); + names.add(name); + if (separator !== -1 && values.output !== undefined) throw new ConfigError('Use --output or --reporter name=file, not both.'); + const destination = separator === -1 ? values.output : selection.slice(separator + 1); + if (destination !== undefined && !destination.trim()) throw new ConfigError('The reporter output path must not be empty.'); + const file = destination === undefined ? undefined : resolve(io.cwd, destination); + if (file !== undefined) { + const key = process.platform === 'win32' ? file.toLowerCase() : file; + if (destinations.has(key)) throw new ConfigError(`Reporters must use different output files: ${destination}.`); + destinations.add(key); + } + const chunks: string[] = []; + const output = file === undefined ? io.stdout : { write: (text: string) => chunks.push(text) }; + return { reporter: factory(output, { verbose: values.verbose === true, env: io.env ?? process.env }), file, chunks }; + }); // Started here so a missing key fails now, not as provider_unavailable on // every turn, and so turns can be priced by the model id. @@ -103,7 +124,7 @@ export async function runTests( ...(concurrency !== undefined ? { concurrency } : {}), ...(repeat !== undefined ? { repeat } : {}), onResult: (result) => { - for (const reporter of reporters) reporter.onResult?.(result); + for (const { reporter } of reporters) reporter.onResult?.(result); }, }); const elapsedMs = performance.now() - started; @@ -125,7 +146,16 @@ export async function runTests( ...(judge ? { judge: judge.id } : {}), ...(priced.length ? { costUsd: priced.reduce((a, b) => a + b, 0) } : {}), }; - for (const reporter of reporters) await reporter.onEnd?.(results, summary); + for (const { reporter, file, chunks } of reporters) { + await reporter.onEnd?.(results, summary); + if (file !== undefined) { + try { + await writeFile(file, chunks.join(''), 'utf8'); + } catch (error) { + throw new ConfigError(`Cannot write report to "${file}": ${(error as Error).message}`); + } + } + } if (cassettes?.mode === 'record') { const written = await cassettes.save( fixtures.map((job) => ({ path: job.path ?? job.file, ran: job.tests.map((t) => t.name), all: job.fixture.tests.map((t) => t.name) })) diff --git a/packages/cli/src/test/reporters/index.ts b/packages/cli/src/test/reporters/index.ts index 08077e4e..935895ea 100644 --- a/packages/cli/src/test/reporters/index.ts +++ b/packages/cli/src/test/reporters/index.ts @@ -2,6 +2,7 @@ import type { CassetteMode } from '../cassette.js'; import type { TestResult } from '../types.js'; import { jsonReporter } from './json.js'; import { prettyReporter } from './pretty.js'; +import { junitReporter } from './junit.js'; /** Where a reporter writes: stdout, unless the reporter is given a file. */ export interface ReporterOutput { @@ -38,4 +39,5 @@ export type ReporterFactory = (output: ReporterOutput, options: { verbose?: bool export const REPORTERS = new Map([ ['pretty', prettyReporter], ['json', jsonReporter], + ['junit', junitReporter], ]); diff --git a/packages/cli/src/test/reporters/junit.ts b/packages/cli/src/test/reporters/junit.ts new file mode 100644 index 00000000..56d3c612 --- /dev/null +++ b/packages/cli/src/test/reporters/junit.ts @@ -0,0 +1,66 @@ +import type { TestResult } from '../types.js'; +import type { ReporterFactory } from './index.js'; + +/** One testcase per aggregate result, including when a test repeats. */ +export const junitReporter: ReporterFactory = (output) => ({ + onEnd(results) { + const groups = new Map(); + for (const result of results) { + const group = groups.get(result.fixture) ?? []; + group.push(result); + groups.set(result.fixture, group); + } + const lines = ['', ``]; + for (const [fixture, tests] of groups) { + lines.push(` `); + for (const result of tests) { + lines.push(` `); + const outcome = problem(result); + if (outcome) { + const details: string[] = []; + for (const run of result.runs) { + if (run.error) details.push(`Run ${run.run}: ${run.error}`); + for (const [index, { turn, results: checks }] of run.turns.entries()) { + const failed = checks.filter((check) => !check.pass); + if (!failed.length) continue; + for (const check of failed) { + details.push(`Run ${run.run}, turn ${index + 1}: ${check.reason ?? Object.keys(check.assertion)[0]}`); + } + details.push(`Reply: ${turn.text}`); + } + } + lines.push(` <${outcome} message="${outcome === 'error' ? 'Execution error' : 'Assertion failed'}">${xml(details.join('\n'))}`); + } + lines.push(' '); + } + lines.push(' '); + } + lines.push(''); + output.write(`${lines.join('\n')}\n`); + }, +}); + +function problem(result: TestResult): 'error' | 'failure' | undefined { + // The aggregate pass-rate verdict wins. For a failed aggregate, any execution + // error takes precedence over assertion failures, so each testcase counts once. + if (result.pass) return undefined; + return result.runs.some((run) => run.error !== undefined) ? 'error' : 'failure'; +} + +function latency(result: TestResult): number { + return result.runs.reduce((total, run) => total + run.turns.reduce((sum, { turn }) => sum + turn.latencyMs, 0), 0); +} + +function seconds(milliseconds: number): string { + return (milliseconds / 1000).toFixed(3); +} + +function statistics(results: TestResult[]): string { + return `tests="${results.length}" failures="${results.filter((r) => problem(r) === 'failure').length}" errors="${results.filter((r) => problem(r) === 'error').length}" time="${seconds(results.reduce((sum, r) => sum + latency(r), 0))}"`; +} + +/** XML 1.0 characters and entities, including whitespace in attribute values. */ +function xml(text: string): string { + const entities: Record = { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''', '\r': ' ', '\n': ' ', '\t': ' ' }; + return text.replace(/[^\t\n\r\u0020-\uD7FF\uE000-\uFFFD\u{10000}-\u{10FFFF}]/gu, '').replace(/[&<>"'\r\n\t]/g, (character) => entities[character]); +} diff --git a/packages/cli/tests/junit.test.ts b/packages/cli/tests/junit.test.ts new file mode 100644 index 00000000..b05a2eb2 --- /dev/null +++ b/packages/cli/tests/junit.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from 'vitest'; +import { junitReporter } from '../src/test/reporters/junit.js'; +import type { TestResult, TestRunResult } from '../src/test/types.js'; + +function run(overrides: Partial = {}): TestRunResult { + return { + fixture: 'evals/pricing.chat.yaml', test: 'quotes the price', run: 1, pass: true, + turns: [{ turn: { user: 'price?', text: '$19', latencyMs: 1200, toolCalls: [], sources: [], leads: [] }, results: [] }], + ...overrides, + }; +} + +function result(overrides: Partial = {}): TestResult { + return { fixture: 'evals/pricing.chat.yaml', test: 'quotes the price', pass: true, passRate: 1, runs: [run()], durationMs: 99999, ...overrides }; +} + +async function report(results: TestResult[]): Promise { + let text = ''; + await junitReporter({ write: (chunk) => { text += chunk; } }, {}).onEnd?.(results, { mode: 'live', elapsedMs: 99999, costIncomplete: false }); + return text; +} + +describe('JUnit reporter', () => { + it('emits a complete XML document for a passing test, using turn latency rather than wall time', async () => { + expect(await report([result()])).toBe(` + + + + + + +`); + }); + + it('escapes attributes and failure details, with run, turn, reason and reply', async () => { + const special = ` & "quoted" 'single'`; + const escaped = '<tag> & "quoted" 'single''; + const failed = run({ pass: false }); + failed.turns.push({ turn: { ...failed.turns[0].turn, text: special, latencyMs: 300 }, results: [ + { assertion: { contains: 'x' }, pass: false, reason: special }, + { assertion: { equals: 'x' }, pass: false }, + ] }); + const xml = await report([result({ fixture: special, test: `${special}\u0000`, pass: false, runs: [failed] })]); + expect(xml).toContain(`classname="${escaped}" name="${escaped}" time="1.500"`); + expect(xml).toContain(''); + expect(xml).toContain(`Run 1, turn 2: ${escaped} Run 1, turn 2: equals Reply: ${escaped}`); + expect(xml).not.toContain('\u0000'); + expect(xml).not.toContain(' { + const failure = result({ test: 'fails', pass: false, runs: [run({ pass: false })] }); + const error = result({ fixture: 'evals/other.chat.yaml', test: 'throws', pass: false, runs: [run({ pass: false, error: 'provider & unavailable' })] }); + const xml = await report([result(), failure, error]); + expect(xml).toContain(''); + expect(xml).toContain(''); + expect(xml).toContain(''); + expect(xml.match(/Run 1: provider <down> & unavailable'); + }); + + it('respects aggregate pass-rate verdicts and sums latency across all repeats', async () => { + const runs = [run(), run({ run: 2, pass: false, error: 'failed repeat' })]; + const xml = await report([result({ runs, passRate: 0.5 })]); + expect(xml).toContain('tests="1" failures="0" errors="0" time="2.400"'); + expect(xml).not.toMatch(/<(failure|error) /); + const failed = await report([result({ runs, passRate: 0.5, pass: false })]); + expect(failed).toContain('failures="0" errors="1"'); + expect(failed).toContain('Run 2: failed repeat'); + }); + + it('gives execution errors precedence in a failed result with mixed runs', async () => { + const failed = run({ pass: false }); + failed.turns[0].results.push({ assertion: { contains: 'x' }, pass: false, reason: 'missing x' }); + const xml = await report([result({ pass: false, runs: [failed, run({ run: 2, pass: false, error: 'model down' })] })]); + expect(xml).toContain('Run 1, turn 1: missing x Reply: $19 Run 2: model down'); + expect(xml).not.toContain(' { + expect(await report([])).toBe('\n\n\n'); + }); +}); diff --git a/packages/cli/tests/test-command.test.ts b/packages/cli/tests/test-command.test.ts index 057290d0..2c443fd6 100644 --- a/packages/cli/tests/test-command.test.ts +++ b/packages/cli/tests/test-command.test.ts @@ -1,10 +1,11 @@ -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { dirname, join, resolve } from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { EXIT, main } from '../src/index.js'; import { check, unknownAssertions } from '../src/test/assertions/index.js'; import { matchesFilter } from '../src/test/load.js'; import type { TurnResult } from '../src/test/types.js'; +import { REPORTERS } from '../src/test/reporters/index.js'; const TMP = resolve(__dirname, '.tmp-test'); @@ -322,11 +323,100 @@ tests: expect(both.stdout).toContain(' ✓ says hello'); expect(JSON.parse(both.stdout.trim().split('\n').at(-1) ?? '')).toMatchObject({ passed: 2 }); const unknown = await run(['test', '--reporter', 'html'], dir); - expect(unknown.stderr).toContain("--reporter html doesn't exist. Use one of: pretty, json."); + expect(unknown.stderr).toContain("--reporter html doesn't exist. Use one of: pretty, json, junit."); expect(unknown.code).toBe(EXIT.unusable); expect((await run(['test', '--json', '--reporter', 'json'], dir)).stderr).toContain('not both'); }); + it('writes JUnit to --output relative to io.cwd and replaces existing files without stdout duplication', async () => { + const dir = project({ 'evals/pricing.chat.yaml': PRICING, 'evals.xml': 'old contents' }); + const result = await run(['test', '--reporter', 'junit', '--output', 'evals.xml'], dir); + expect(result).toEqual({ code: EXIT.ok, stdout: '', stderr: '' }); + const xml = readFileSync(join(dir, 'evals.xml'), 'utf8'); + expect(xml).toMatch(/^<\?xml version="1.0" encoding="UTF-8"\?>/); + expect(xml).toContain('\n$/); + }); + + it('routes reporters independently and splits destinations on the first equals sign', async () => { + const dir = project({ 'evals/pricing.chat.yaml': PRICING }); + const mixed = await run(['test', '--reporter', 'pretty', '--reporter', 'junit=report=evals.xml'], dir); + expect(mixed.code).toBe(EXIT.ok); + expect(mixed.stdout).toContain('2 passed, 0 failed.'); + expect(mixed.stdout).not.toContain(''); + }); + + it('supports --output with the default reporter and --json', async () => { + const dir = project({ 'evals/pricing.chat.yaml': PRICING }); + expect((await run(['test', '--output', 'pretty.txt'], dir)).stdout).toBe(''); + expect(readFileSync(join(dir, 'pretty.txt'), 'utf8')).toContain('2 passed, 0 failed.'); + expect((await run(['test', '--json', '--output', 'evals.json'], dir)).code).toBe(EXIT.ok); + expect(JSON.parse(readFileSync(join(dir, 'evals.json'), 'utf8')).passed).toBe(2); + }); + + it('buffers onResult and asynchronous onEnd output before writing the file', async () => { + const dir = project({ 'evals/pricing.chat.yaml': PRICING }); + REPORTERS.set('buffer-test', (output) => ({ + onResult(result) { output.write(`${result.test}\n`); }, + async onEnd() { await Promise.resolve(); output.write('done\n'); }, + })); + try { + expect((await run(['test', '--reporter', 'buffer-test=buffer.txt'], dir)).stdout).toBe(''); + const text = readFileSync(join(dir, 'buffer.txt'), 'utf8'); + expect(text).toContain('quotes the team price\n'); + expect(text).toContain('says hello\n'); + expect(text).toMatch(/done\n$/); + } finally { + REPORTERS.delete('buffer-test'); + } + }); + + it('writes reports for failed evals and reports output errors with exit 2', async () => { + const dir = project({ 'evals/pricing.chat.yaml': PRICING.replace('contains: "$19"', 'contains: "$29"') }); + const failed = await run(['test', '--reporter', 'junit=failed.xml'], dir); + expect(failed.code).toBe(EXIT.problems); + expect(failed.stdout).toBe(''); + expect(readFileSync(join(dir, 'failed.xml'), 'utf8')).toContain(''); + const unwritable = await run(['test', '--reporter', 'junit', '--output', 'missing/report.xml'], dir); + expect(unwritable.code).toBe(EXIT.unusable); + expect(unwritable.stdout).toBe(''); + expect(unwritable.stderr).toContain('Cannot write report to'); + expect(unwritable.stderr).toContain('report.xml'); + }); + + it.each([ + [['--reporter', ''], 'needs a name'], + [['--reporter', '=report.xml'], 'needs a name'], + [['--reporter', 'junit='], 'must not be empty'], + [['--output', ''], 'must not be empty'], + [['--reporter', 'html=report.xml'], "doesn't exist"], + [['--reporter', 'junit', '--reporter', 'json', '--output', 'report.xml'], 'single reporter'], + [['--reporter', 'junit=a.xml', '--output', 'b.xml'], 'not both'], + [['--reporter', 'junit', '--reporter', 'junit=a.xml'], 'more than once'], + [['--reporter', 'junit=a.xml', '--reporter', 'json=./a.xml'], 'different output files'], + ])('rejects invalid reporter destinations %j', async (args, message) => { + const dir = project({ 'evals/pricing.chat.yaml': PRICING }); + const result = await run(['test', ...args], dir); + expect(result.code).toBe(EXIT.unusable); + expect(result.stderr).toContain(message); + expect(result.stdout).toBe(''); + }); + + it('lists JUnit and output destinations in help', async () => { + const help = await run(['test', '--help'], TMP); + expect(help.stdout).toContain('junit'); + expect(help.stdout).toContain('--output '); + expect(help.stdout).toContain('name=file'); + }); + it('loads module eval files', async () => { const dir = project({ 'evals/pricing.chat.mjs': `export default { version: 1, name: 'Module', tests: [{ name: 'js', turns: [{ user: 'price', expect: [{ contains: '$19' }] }] }] };`, diff --git a/website/src/content/docs/core/testing.mdx b/website/src/content/docs/core/testing.mdx index a533c68b..f00697e6 100644 --- a/website/src/content/docs/core/testing.mdx +++ b/website/src/content/docs/core/testing.mdx @@ -233,7 +233,8 @@ export default defineConfig({ | `--filter ` | Only tests whose name contains the text, or `tag:` for a tag | | `--repeat ` | Run every test n times, instead of each test's own `repeat` | | `--concurrency ` | Tests run at once. Default 4. | -| `--reporter ` | `pretty` (the default) or `json`, which prints one JSON object with every turn's reply, tool calls, sources and token usage. Repeatable. | +| `--reporter ` | `pretty` (the default), `json` (every turn's reply, tool calls, sources and token usage), or `junit` (JUnit XML for CI). Repeatable, with each name used once. Add `=file` to write that reporter to a file. | +| `--output ` | Write a single reporter's complete output to a file instead of stdout. Also works with the default reporter. | | `--json` | The same as `--reporter json` | | `--judge-model ` | The judge for `judge` assertions: `gemini`, `openai` or `claude`, with a model id or without one for the default | | `--max-cost ` | Stop starting turns once the run has spent this much. Needs `prices`. | @@ -284,6 +285,38 @@ A few things to know about recordings: - `repeat` records every run, and replay plays them back in the same order. - Replays answer at once, so `latency` in a replayed run is not the real latency. +### JUnit reports for CI + +Write a JUnit report for a CI test report integration: + +```bash +npx chatcops test --reporter junit --output evals.xml +``` + +To keep the readable summary in the terminal and save XML separately, give the JUnit reporter a destination: + +```bash +npx chatcops test --reporter pretty --reporter junit=evals.xml +``` + +File paths are relative to the current folder. The parent folder must exist, and an existing report is replaced after reporting finishes. Paths containing `=` work: only the first `=` separates the reporter name from its destination. Each reporter name can be used once, and file destinations must differ. `--output` is a shortcut for one reporter; it cannot be combined with multiple reporters or `name=file`. A file write error exits 2. + +JUnit groups tests by eval file. Each test's time is the sum of its turns' latencies across all repeats, in seconds. The aggregate pass-rate verdict decides whether the testcase passed. A failed test with any execution error gets an ``; other failed tests get a `` with the assertion reasons, turn numbers and replies. + +For example, GitLab CI can display the report even when an eval fails (record the API calls first so CI can replay them): + +```yaml +evals: + image: node:22 + script: + - npm ci + - npx chatcops test --reporter junit --output evals.xml + artifacts: + when: always + reports: + junit: evals.xml +``` + ### GitHub Actions ```yaml From 87af76dfc34e55d996d707106ff3d57ff3f311d5 Mon Sep 17 00:00:00 2001 From: Siddarth Date: Fri, 6 Nov 2026 21:30:40 +0530 Subject: [PATCH 2/2] fix(cli): handle report output failures safely --- packages/cli/src/main.ts | 2 +- packages/cli/src/test/command.ts | 22 ++++++++++++---------- packages/cli/tests/cassette.test.ts | 16 ++++++++++++++++ packages/cli/tests/test-command.test.ts | 16 ++++++++++++---- website/src/content/docs/core/testing.mdx | 2 +- 5 files changed, 42 insertions(+), 16 deletions(-) diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index 7963a448..a77eb718 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -61,7 +61,7 @@ Test options: --reporter pretty (default), json or junit. Repeatable; each name once. --output Write a single reporter to a file instead of stdout (replaces the file). With several reporters, use name=file; file destinations must differ. - Do not combine --output with name=file. Parent folders must exist. + Do not combine --output with name=file. Parent folders are created. --json Same as --reporter json --record Call the APIs and save what they answer to __cassettes__ --replay Answer every API call from __cassettes__ (the default when CI is set) diff --git a/packages/cli/src/test/command.ts b/packages/cli/src/test/command.ts index 9cef474c..c33f9ba5 100644 --- a/packages/cli/src/test/command.ts +++ b/packages/cli/src/test/command.ts @@ -1,6 +1,6 @@ import { performance } from 'node:perf_hooks'; -import { relative, resolve } from 'node:path'; -import { writeFile } from 'node:fs/promises'; +import { dirname, relative, resolve } from 'node:path'; +import { mkdir, writeFile } from 'node:fs/promises'; import { createProvider, isAIProvider, type AIProvider, type ProviderConfig } from '@chatcops/core'; import type { ChatCopsServerConfig } from '@chatcops/server'; import { ConfigError, type ChatCopsCliConfig } from '../config.js'; @@ -146,22 +146,24 @@ export async function runTests( ...(judge ? { judge: judge.id } : {}), ...(priced.length ? { costUsd: priced.reduce((a, b) => a + b, 0) } : {}), }; - for (const { reporter, file, chunks } of reporters) { - await reporter.onEnd?.(results, summary); + for (const { reporter } of reporters) await reporter.onEnd?.(results, summary); + if (cassettes?.mode === 'record') { + const written = await cassettes.save( + fixtures.map((job) => ({ path: job.path ?? job.file, ran: job.tests.map((t) => t.name), all: job.fixture.tests.map((t) => t.name) })) + ); + for (const file of written) io.stderr.write(`Saved ${relative(io.cwd, file)}\n`); + } + // Preserve terminal summaries and recordings even if a report file cannot be written. + for (const { file, chunks } of reporters) { if (file !== undefined) { try { + await mkdir(dirname(file), { recursive: true }); await writeFile(file, chunks.join(''), 'utf8'); } catch (error) { throw new ConfigError(`Cannot write report to "${file}": ${(error as Error).message}`); } } } - if (cassettes?.mode === 'record') { - const written = await cassettes.save( - fixtures.map((job) => ({ path: job.path ?? job.file, ran: job.tests.map((t) => t.name), all: job.fixture.tests.map((t) => t.name) })) - ); - for (const file of written) io.stderr.write(`Saved ${relative(io.cwd, file)}\n`); - } return results.some((result) => !result.pass) ? EXIT.problems : EXIT.ok; } diff --git a/packages/cli/tests/cassette.test.ts b/packages/cli/tests/cassette.test.ts index 6a4780f6..0872c1fd 100644 --- a/packages/cli/tests/cassette.test.ts +++ b/packages/cli/tests/cassette.test.ts @@ -108,6 +108,22 @@ const entries = (dir: string) => .map((line) => JSON.parse(line)); describe('chatcops test --record and --replay', () => { + it('saves recordings even when writing the report fails', async () => { + const dir = project({ 'evals/pricing.chat.yaml': PRICING }); + const recorded = await run(['test', '--record', '--output', 'evals'], dir); + expect(recorded.code).toBe(EXIT.unusable); + expect(recorded.stderr).toContain('Cannot write report to'); + expect(recorded.stderr).toContain(`Saved ${join('evals', '__cassettes__', 'pricing.chat.jsonl')}`); + expect(existsSync(cassetteOf(dir))).toBe(true); + expect(entries(dir)).toHaveLength(3); + expect(hits).toHaveLength(3); + + hits = []; + const replayed = await run(['test', '--replay'], dir); + expect(replayed.code).toBe(EXIT.ok); + expect(hits).toHaveLength(0); + }); + it('records each call, then replays them without the network', async () => { const dir = project({ 'evals/pricing.chat.yaml': PRICING }); const recorded = await run(['test', '--record'], dir); diff --git a/packages/cli/tests/test-command.test.ts b/packages/cli/tests/test-command.test.ts index 2c443fd6..c567df3d 100644 --- a/packages/cli/tests/test-command.test.ts +++ b/packages/cli/tests/test-command.test.ts @@ -379,17 +379,25 @@ tests: } }); - it('writes reports for failed evals and reports output errors with exit 2', async () => { + it('writes reports for failed evals and creates missing parent folders', async () => { const dir = project({ 'evals/pricing.chat.yaml': PRICING.replace('contains: "$19"', 'contains: "$29"') }); const failed = await run(['test', '--reporter', 'junit=failed.xml'], dir); expect(failed.code).toBe(EXIT.problems); expect(failed.stdout).toBe(''); expect(readFileSync(join(dir, 'failed.xml'), 'utf8')).toContain(''); - const unwritable = await run(['test', '--reporter', 'junit', '--output', 'missing/report.xml'], dir); + const nested = await run(['test', '--reporter', 'junit', '--output', 'missing/report.xml'], dir); + expect(nested.code).toBe(EXIT.problems); + expect(nested.stdout).toBe(''); + expect(nested.stderr).toBe(''); + expect(readFileSync(join(dir, 'missing/report.xml'), 'utf8')).toContain(''); + }); + + it('finishes later reporters before reporting a file write error', async () => { + const dir = project({ 'evals/pricing.chat.yaml': PRICING }); + const unwritable = await run(['test', '--reporter', 'junit=evals', '--reporter', 'pretty'], dir); expect(unwritable.code).toBe(EXIT.unusable); - expect(unwritable.stdout).toBe(''); + expect(unwritable.stdout).toContain('2 passed, 0 failed.'); expect(unwritable.stderr).toContain('Cannot write report to'); - expect(unwritable.stderr).toContain('report.xml'); }); it.each([ diff --git a/website/src/content/docs/core/testing.mdx b/website/src/content/docs/core/testing.mdx index f00697e6..c4034809 100644 --- a/website/src/content/docs/core/testing.mdx +++ b/website/src/content/docs/core/testing.mdx @@ -299,7 +299,7 @@ To keep the readable summary in the terminal and save XML separately, give the J npx chatcops test --reporter pretty --reporter junit=evals.xml ``` -File paths are relative to the current folder. The parent folder must exist, and an existing report is replaced after reporting finishes. Paths containing `=` work: only the first `=` separates the reporter name from its destination. Each reporter name can be used once, and file destinations must differ. `--output` is a shortcut for one reporter; it cannot be combined with multiple reporters or `name=file`. A file write error exits 2. +File paths are relative to the current folder. Parent folders are created as needed, and an existing report is replaced after reporting finishes. Paths containing `=` work: only the first `=` separates the reporter name from its destination. Each reporter name can be used once, and file destinations must differ. `--output` is a shortcut for one reporter; it cannot be combined with multiple reporters or `name=file`. A file write error exits 2, after all reporters have finished and any recordings have been saved. JUnit groups tests by eval file. Each test's time is the sum of its turns' latencies across all repeats, in seconds. The aggregate pass-rate verdict decides whether the testcase passed. A failed test with any execution error gets an ``; other failed tests get a `` with the assertion reasons, turn numbers and replies.