From b947be992c7fb62bdb034df9b86684af3e7fee13 Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Fri, 30 Oct 2026 08:36:47 -0700 Subject: [PATCH] fix(reference): cover utilized and utilization in prefer-use-over-utilize The prompt asks for the inflected forms, but the swap lacked utilized and the noun utilization, so downstream evals graded correct generations as wrong. Add both to the swap with a fail case each, name the forms in the prompt, and verify the published vale reference against its own cases. --- .changeset/reference-utilize-inflections.md | 5 ++ .../prefer-use-over-utilize.yml | 2 + packages/cli/assets/reference-vale/prompt.md | 3 +- .../tests/fail/utilization-noun/README.md | 3 + .../tests/fail/utilized-past/README.md | 3 + packages/cli/assets/reference.json | 24 ++++++- packages/cli/src/rules/corpus/manifest.ts | 2 + packages/cli/test/reference-verify.test.ts | 69 +++++++++++++++++++ 8 files changed, 108 insertions(+), 3 deletions(-) create mode 100644 .changeset/reference-utilize-inflections.md create mode 100644 packages/cli/assets/reference-vale/tests/fail/utilization-noun/README.md create mode 100644 packages/cli/assets/reference-vale/tests/fail/utilized-past/README.md create mode 100644 packages/cli/test/reference-verify.test.ts diff --git a/.changeset/reference-utilize-inflections.md b/.changeset/reference-utilize-inflections.md new file mode 100644 index 00000000..79a48845 --- /dev/null +++ b/.changeset/reference-utilize-inflections.md @@ -0,0 +1,5 @@ +--- +"@taskless/cli": patch +--- + +The `prefer-use-over-utilize` entry in `@taskless/cli/reference.json` now flags `utilized` and `utilization`, as its prompt asks, and carries a fail case for each. Previously the reference rule missed both, so a rule generated correctly from the prompt was graded against a reference that did not fire on those cases. The prompt now names the forms it expects. The corpus `version` is unchanged: the shape is the same, only the content of one entry moved. diff --git a/packages/cli/assets/reference-vale/prefer-use-over-utilize.yml b/packages/cli/assets/reference-vale/prefer-use-over-utilize.yml index b7206b10..28a46e5c 100644 --- a/packages/cli/assets/reference-vale/prefer-use-over-utilize.yml +++ b/packages/cli/assets/reference-vale/prefer-use-over-utilize.yml @@ -5,4 +5,6 @@ ignorecase: true swap: utilize: use utilizes: uses + utilized: used utilizing: using + utilization: use diff --git a/packages/cli/assets/reference-vale/prompt.md b/packages/cli/assets/reference-vale/prompt.md index 6ad7be39..a288acef 100644 --- a/packages/cli/assets/reference-vale/prompt.md +++ b/packages/cli/assets/reference-vale/prompt.md @@ -3,7 +3,8 @@ suggests "use" instead. "Utilize" is longer than "use" and means the same thing in nearly every sentence a reader will meet. It should match case-insensitively and cover the -inflected forms. +inflected forms ("utilizes", "utilized", "utilizing") as well as the noun +"utilization". This is a prose rule. It applies to markdown, and it must not look at source code. diff --git a/packages/cli/assets/reference-vale/tests/fail/utilization-noun/README.md b/packages/cli/assets/reference-vale/tests/fail/utilization-noun/README.md new file mode 100644 index 00000000..3b0886d5 --- /dev/null +++ b/packages/cli/assets/reference-vale/tests/fail/utilization-noun/README.md @@ -0,0 +1,3 @@ +# Limits + +Memory utilization grows with the size of the batch. diff --git a/packages/cli/assets/reference-vale/tests/fail/utilized-past/README.md b/packages/cli/assets/reference-vale/tests/fail/utilized-past/README.md new file mode 100644 index 00000000..43d11d66 --- /dev/null +++ b/packages/cli/assets/reference-vale/tests/fail/utilized-past/README.md @@ -0,0 +1,3 @@ +# Migration + +We utilized the API to export the records. diff --git a/packages/cli/assets/reference.json b/packages/cli/assets/reference.json index 1b9ad03f..86b5c902 100644 --- a/packages/cli/assets/reference.json +++ b/packages/cli/assets/reference.json @@ -185,11 +185,11 @@ "id": "prefer-use-over-utilize", "directory": ".taskless/rules/vale/prefer-use-over-utilize", "ruleFile": "prefer-use-over-utilize.yml", - "prompt": "Create a rule that flags the word \"utilize\" in markdown documentation and\nsuggests \"use\" instead.\n\n\"Utilize\" is longer than \"use\" and means the same thing in nearly every\nsentence a reader will meet. It should match case-insensitively and cover the\ninflected forms.\n\nThis is a prose rule. It applies to markdown, and it must not look at source\ncode.\n\nProse that should NOT be flagged:\n\n```md\nUse the installer to write the config file.\n```\n\nProse that SHOULD be flagged:\n\n```md\nUtilize the installer to write the config file.\n```\n", + "prompt": "Create a rule that flags the word \"utilize\" in markdown documentation and\nsuggests \"use\" instead.\n\n\"Utilize\" is longer than \"use\" and means the same thing in nearly every\nsentence a reader will meet. It should match case-insensitively and cover the\ninflected forms (\"utilizes\", \"utilized\", \"utilizing\") as well as the noun\n\"utilization\".\n\nThis is a prose rule. It applies to markdown, and it must not look at source\ncode.\n\nProse that should NOT be flagged:\n\n```md\nUse the installer to write the config file.\n```\n\nProse that SHOULD be flagged:\n\n```md\nUtilize the installer to write the config file.\n```\n", "rule": [ { "path": "prefer-use-over-utilize.yml", - "content": "extends: substitution\nmessage: \"Prefer '%s' over '%s'.\"\nlevel: warning\nignorecase: true\nswap:\n utilize: use\n utilizes: uses\n utilizing: using\n" + "content": "extends: substitution\nmessage: \"Prefer '%s' over '%s'.\"\nlevel: warning\nignorecase: true\nswap:\n utilize: use\n utilizes: uses\n utilized: used\n utilizing: using\n utilization: use\n" }, { "path": ".vale.ini", @@ -206,6 +206,14 @@ { "path": "tests/fail/utilize-in-prose/README.md", "content": "# Setup\n\nUtilize the installer to write the config file.\n" + }, + { + "path": "tests/fail/utilized-past/README.md", + "content": "# Migration\n\nWe utilized the API to export the records.\n" + }, + { + "path": "tests/fail/utilization-noun/README.md", + "content": "# Limits\n\nMemory utilization grows with the size of the batch.\n" } ], "cases": [ @@ -220,6 +228,18 @@ "name": "utilize-in-prose", "path": "tests/fail/utilize-in-prose", "files": ["tests/fail/utilize-in-prose/README.md"] + }, + { + "bucket": "fail", + "name": "utilized-past", + "path": "tests/fail/utilized-past", + "files": ["tests/fail/utilized-past/README.md"] + }, + { + "bucket": "fail", + "name": "utilization-noun", + "path": "tests/fail/utilization-noun", + "files": ["tests/fail/utilization-noun/README.md"] } ] } diff --git a/packages/cli/src/rules/corpus/manifest.ts b/packages/cli/src/rules/corpus/manifest.ts index 7120270f..0a4b6a52 100644 --- a/packages/cli/src/rules/corpus/manifest.ts +++ b/packages/cli/src/rules/corpus/manifest.ts @@ -95,6 +95,8 @@ export const CORPUS_MANIFESTS: readonly CorpusManifest[] = [ testPaths: [ "tests/pass/use-in-prose/README.md", "tests/fail/utilize-in-prose/README.md", + "tests/fail/utilized-past/README.md", + "tests/fail/utilization-noun/README.md", ], }, { diff --git a/packages/cli/test/reference-verify.test.ts b/packages/cli/test/reference-verify.test.ts new file mode 100644 index 00000000..029ead53 --- /dev/null +++ b/packages/cli/test/reference-verify.test.ts @@ -0,0 +1,69 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { readFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import type { Reference } from "../src/rules/reference"; +import { findValeBinary } from "../src/rules/vale/binary"; +import { verifyValeRule } from "../src/rules/vale/verify"; + +/** + * A reference rule passes its own held-out cases. + * + * Other teams grade a generated rule by running OUR rule against THEIR cases, + * so a reference narrower than its prompt marks a correct generation wrong. + * `prefer-use-over-utilize` shipped with a prompt asking for the inflected + * forms and a swap that lacked `utilized` (#591). Nothing ran the reference + * against its own cases, so nothing noticed. This stages the published entry + * exactly as a consumer would and verifies it. + */ + +const referencePath = join( + import.meta.dirname, + "..", + "assets", + "reference.json" +); + +const withVale = findValeBinary().path === undefined ? describe.skip : describe; + +const workspaces: string[] = []; +afterEach(() => { + for (const workspace of workspaces.splice(0)) { + rmSync(workspace, { recursive: true, force: true }); + } +}); + +/** Write a reference entry's rule and cases into a fresh project. */ +function stage(rule: Reference["rules"][number]): string { + const cwd = mkdtempSync(join(tmpdir(), "taskless-reference-")); + workspaces.push(cwd); + for (const file of [...rule.rule, ...rule.tests.files]) { + const target = join(cwd, rule.directory, file.path); + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, file.content); + } + return cwd; +} + +withVale("the vale reference rule", () => { + it("fires on every fail case and stays quiet on every pass case", async () => { + const reference = JSON.parse( + await readFile(referencePath, "utf8") + ) as Reference; + const rule = reference.rules.find((entry) => entry.engine === "vale"); + if (rule === undefined) throw new Error("reference has no vale rule"); + + const result = await verifyValeRule(stage(rule), rule.id); + if ("outcome" in result) { + throw new Error( + `expected a verification, got Vale ${result.outcome.status}` + ); + } + + expect(result.errors).toEqual([]); + expect(result.passed).toBe(true); + }); +});