feat(verify): report every fixture case in verify --json

.changeset/verify-json-cases.md

@@ -0,0 +1,5 @@
+---
+"@taskless/cli": patch
+---
+
+`taskless verify --json` now reports every fixture case under each rule's new `cases` array, for `sg`, `vale` and `runtime` alike: `{ name, expected, ok, message?, files? }`, where `name` is the case's path under `tests/` (such as `fail/cast`). A failing case carries the same line `errors` prints for it as `message`, and its files as `files` (project-relative paths, the spelling `findings[].file` uses; a file over 64 KiB or one that is not text is left out). The entries are built from each runner's own per-case verdict, so a consumer no longer has to parse `errors` to learn which case failed. The field is additive: `errors` is unchanged, and `cases` is always present, empty when no case ran (for `validate`, a refused runtime run, or a rule that failed before its fixtures ran). It is also declared in the schema `@taskless/cli/schemas` publishes.

packages/cli/src/rules/fixtures.ts

@@ -1,5 +1,5 @@
 import type { Dirent } from "node:fs";
-import { readdir } from "node:fs/promises";
+import { readdir, readFile, stat } from "node:fs/promises";
 import { join, posix, relative, sep } from "node:path";
 
 import { ruleDirectory, ruleTestsDirectory } from "./engines";
@@ -291,48 +291,169 @@ export function casePassed(outcome: CaseOutcome): boolean {
 }
 
 /**
- * One error line per case that did not come out as its bucket asserts.
+ * Why one case did not come out as its bucket asserts, or `undefined` when it
+ * did.
  *
  * `scope` names what was in effect, in the rule's own terms (the `files:`
  * globs, or the Vale matchers), so a "scope excluded" message points at the
  * exact text an author would edit.
+ *
+ * The single source of a case's failure wording: `errors` and `cases[].message`
+ * are both built from it, so the two can never describe one case differently.
  */
+export function describeCaseFailure(
+  cwd: string,
+  outcome: CaseOutcome,
+  scope: string | undefined
+): string | undefined {
+  if (casePassed(outcome)) return undefined;
+  const path = projectRelative(cwd, outcome.fixture.root);
+  if (outcome.unread === "scope-excluded") {
+    return (
+      `${outcome.fixture.bucket} case was never read, because the rule's ` +
+      `scope excluded every file in it: ${path}/. No file sits at a path ` +
+      `admitted by ${scope ?? "the scope"}. Move the fixture to such a path.`
+    );
+  }
+  if (outcome.unread === "not-walked") {
+    return (
+      `${outcome.fixture.bucket} case was never read: ${path}/. The engine ` +
+      `walked no file in it, even with the scope removed: the project's ` +
+      `.gitignore or .ignore covers them, they are empty, or none is in a ` +
+      `language or format the engine reads. Rename the path or the file.`
+    );
+  }
+  if (outcome.fixture.bucket === "pass") {
+    return `pass case wrongly fired: ${path}/`;
+  }
+  if (outcome.quiet === "scope-excluded") {
+    return (
+      `fail case did not fire because the rule's scope excluded it: ${path}/. ` +
+      `It fires with the scope removed, so the pattern matches, and no ` +
+      `file in the case sits at a path admitted by ${scope ?? "the scope"}. ` +
+      `Move the fixture to such a path, or widen the scope.`
+    );
+  }
+  return `fail case did not fire: ${path}/ (the rule did not match it)`;
+}
+
+/** One error line per case that did not come out as its bucket asserts. */
 export function describeCaseFailures(
   cwd: string,
   outcomes: readonly CaseOutcome[],
   scope: string | undefined
 ): string[] {
-  const errors: string[] = [];
-  for (const outcome of outcomes) {
-    if (casePassed(outcome)) continue;
-    const path = projectRelative(cwd, outcome.fixture.root);
-    if (outcome.unread === "scope-excluded") {
-      errors.push(
-        `${outcome.fixture.bucket} case was never read, because the rule's ` +
-          `scope excluded every file in it: ${path}/. No file sits at a path ` +
-          `admitted by ${scope ?? "the scope"}. Move the fixture to such a path.`
-      );
-    } else if (outcome.unread === "not-walked") {
-      errors.push(
-        `${outcome.fixture.bucket} case was never read: ${path}/. The engine ` +
-          `walked no file in it, even with the scope removed: the project's ` +
-          `.gitignore or .ignore covers them, they are empty, or none is in a ` +
-          `language or format the engine reads. Rename the path or the file.`
-      );
-    } else if (outcome.fixture.bucket === "pass") {
-      errors.push(`pass case wrongly fired: ${path}/`);
-    } else if (outcome.quiet === "scope-excluded") {
-      errors.push(
-        `fail case did not fire because the rule's scope excluded it: ${path}/. ` +
-          `It fires with the scope removed, so the pattern matches, and no ` +
-          `file in the case sits at a path admitted by ${scope ?? "the scope"}. ` +
-          `Move the fixture to such a path, or widen the scope.`
-      );
-    } else {
-      errors.push(
-        `fail case did not fire: ${path}/ (the rule did not match it)`
-      );
+  return outcomes.flatMap((outcome) => {
+    const message = describeCaseFailure(cwd, outcome, scope);
+    return message === undefined ? [] : [message];
+  });
+}
+
+/** One file of a fixture case, as `verify --json` reports it. */
+export interface FixtureCaseFile {
+  /** Project-relative, `/`-separated: the same spelling as `findings[].file`. */
+  path: string;
+  content: string;
+}
+
+/**
+ * One fixture case's verdict, as `verify --json` reports it under `cases`.
+ *
+ * Structured so a consumer never has to recover a case from `errors`, whose
+ * wording is free to change. Built from the outcome each runner already
+ * judged, never from the error text.
+ */
+export interface FixtureCaseResult {
+  /** `<bucket>/<case>`, the case's path under `tests/`, e.g. `fail/cast`. */
+  name: string;
+  /** The bucket's assertion: `fail` should make the rule fire, `pass` should not. */
+  expected: FixtureBucket;
+  /** Whether the case came out as its bucket asserts. */
+  ok: boolean;
+  /** Why it did not, when `ok` is false: the same line `errors` carries. */
+  message?: string;
+  /** The case's files, on a failing case only. See {@link readCaseFiles}. */
+  files?: FixtureCaseFile[];
+}
+
+/** Larger than any fixture worth reading in a report; such a file is left out. */
+const CASE_FILE_LIMIT_BYTES = 64 * 1024;
+
+/**
+ * Strict, so bytes that are not UTF-8 throw instead of decoding to U+FFFD, and
+ * BOM-preserving, so a file that decodes comes back exactly as it is on disk.
+ */
+const CASE_FILE_DECODER = new TextDecoder("utf-8", {
+  fatal: true,
+  ignoreBOM: true,
+});
+
+/**
+ * A case's files, for a report reader who wants to see what failed.
+ *
+ * Text only and bounded: a file over {@link CASE_FILE_LIMIT_BYTES}, holding
+ * a NUL byte, or not valid UTF-8 is left out rather than truncated or decoded
+ * with replacement characters, since either misrepresents the case it came
+ * from. Operating-system metadata is skipped
+ * for the reason {@link NOT_FIXTURES} gives. Never rejects: the files are
+ * context for a verdict already reached, so a file that cannot be read is
+ * left out rather than turning a report into a failure.
+ */
+export async function readCaseFiles(
+  cwd: string,
+  root: string
+): Promise<FixtureCaseFile[]> {
+  let entries: Dirent[];
+  try {
+    entries = await readdir(root, { recursive: true, withFileTypes: true });
+  } catch {
+    return [];
+  }
+  const files: FixtureCaseFile[] = [];
+  for (const entry of entries) {
+    if (entry.isDirectory() || NOT_FIXTURES.has(entry.name)) continue;
+    const absolute = join(entry.parentPath, entry.name);
+    try {
+      const { size } = await stat(absolute);
+      if (size > CASE_FILE_LIMIT_BYTES) continue;
+      const bytes = await readFile(absolute);
+      if (bytes.includes(0)) continue;
+      const content = CASE_FILE_DECODER.decode(bytes);
+      files.push({ path: projectRelative(cwd, absolute), content });
+    } catch {
+      continue;
     }
   }
-  return errors;
+  return files.toSorted((a, b) => a.path.localeCompare(b.path));
+}
+
+/** A case's verdict, with its files attached when it failed. */
+export async function caseResult(
+  cwd: string,
+  fixture: FixtureCase,
+  message: string | undefined
+): Promise<FixtureCaseResult> {
+  const result: FixtureCaseResult = {
+    name: `${fixture.bucket}/${fixture.name}`,
+    expected: fixture.bucket,
+    ok: message === undefined,
+  };
+  if (message !== undefined) {
+    result.message = message;
+    result.files = await readCaseFiles(cwd, fixture.root);
+  }
+  return result;
+}
+
+/** Every case's verdict, for the two engines whose check is a scan. */
+export function describeCaseResults(
+  cwd: string,
+  outcomes: readonly CaseOutcome[],
+  scope: string | undefined
+): Promise<FixtureCaseResult[]> {
+  return Promise.all(
+    outcomes.map((outcome) =>
+      caseResult(cwd, outcome.fixture, describeCaseFailure(cwd, outcome, scope))
+    )
+  );
 }

packages/cli/src/rules/inspect.ts

@@ -8,7 +8,11 @@ import {
   ruleDirectory,
   ruleFilePath,
 } from "./engines";
-import { describeCoverageShortfall, type FixtureFinding } from "./fixtures";
+import {
+  describeCoverageShortfall,
+  type FixtureCaseResult,
+  type FixtureFinding,
+} from "./fixtures";
 import { describeEmptySgScope } from "./sg-scope";
 import { type EngineName } from "./layout";
 import {
@@ -132,6 +136,14 @@ export interface RuleVerification {
    * only ever produced by the runs that passed.
    */
   findings: FixtureFinding[];
+  /**
+   * Every fixture case's verdict, one entry per case.
+   *
+   * Empty, never absent, whenever the cases did not run, for `findings`' reason.
+   * Built from the outcome each runner judged rather than from `errors`, whose
+   * wording a consumer must not have to parse to learn which case failed.
+   */
+  cases: FixtureCaseResult[];
 }
 
 /** What `verify` needs beyond a rule, all of it about the runtime engine. */
@@ -441,7 +453,7 @@ export async function verifyOneRule(
     // call, which would run every case twice for one answer.
     const validation = await withIdCollision(cwd, ruleId, verdict);
     if (!validation.ok) {
-      return { ...validation, ran: false, findings: [] };
+      return { ...validation, ran: false, findings: [], cases: [] };
     }
     // A rule that populated only one bucket has proved only half of what a
     // rule claims, and the CLI says so: coverage is counted over case
@@ -468,12 +480,13 @@ export async function verifyOneRule(
       // The findings that decided the verdict, as the scan reported them:
       // rendered messages, and `file` naming the fixture in its case.
       findings: result.tests.findings,
+      cases: result.tests.cases,
     };
   }
 
   const validation = await validateOneRule(cwd, rule);
   if (!validation.ok) {
-    return { ...validation, ran: false, findings: [] };
+    return { ...validation, ran: false, findings: [], cases: [] };
   }
 
   if (engine === "vale") {
@@ -488,6 +501,7 @@ export async function verifyOneRule(
         ran: false,
         notices: [],
         findings: [],
+        cases: [],
       };
     }
     const errors: string[] = [];
@@ -503,6 +517,7 @@ export async function verifyOneRule(
       ran: true,
       notices: result.notices,
       findings: result.findings,
+      cases: result.cases,
     };
   }
 
@@ -542,6 +557,7 @@ export async function verifyOneRule(
       violations: [],
       ran: false,
       findings: [],
+      cases: [],
       refused: reason,
       notices: [],
     };
@@ -567,6 +583,7 @@ export async function verifyOneRule(
       violations: [],
       ran: false,
       findings: [],
+      cases: [],
       refused: reason,
       notices: [],
     };
@@ -589,14 +606,15 @@ export async function verifyOneRule(
       ran: false,
       notices: [],
       findings: [],
+      cases: [],
     };
   }
 
   // Warned at the last possible moment: everything above can still decline to
   // run anything, and a warning about code that never executed is noise.
   options.onRuntimeWarning?.(RUN_SCRIPTS_WARNING);
 
-  const report = await runRuntimeFixtures(runtimeRule, fixtures);
+  const report = await runRuntimeFixtures(cwd, runtimeRule, fixtures);
   return {
     engine,
     ruleId,
@@ -606,5 +624,6 @@ export async function verifyOneRule(
     ran: true,
     notices: [],
     findings: report.findings,
+    cases: report.cases,
   };
 }