From 4bc7c24792dcd61b014485a619de37cadaeb4fe4 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 4 Oct 2026 12:45:40 -0400 Subject: [PATCH 1/2] test(architecture): add no-findings eval suite for improve (row 19) Eval-first gate for the Pocock-sync row 19 candidate: does improve already give a no-findings verdict on a codebase with nothing to deepen? Four claude plugin eval cases tagged row19: the converted deep-module-not-over-flagged case, two scaffolded repositories whose modules are already deep, and a control with a shallow pass-through chain so an always-empty verdict cannot pass. No skill text changed. Co-authored-by: Claude Opus 5.5 --- .../architecture/.claude-plugin/plugin.json | 2 +- plugins/architecture/CHANGELOG.md | 10 + .../control-shallow-chain-is-named/case.yaml | 4 + .../graders/candidate-judge.md | 9 + .../graders/names-the-chain.md | 4 + .../graders/no-empty-verdict.md | 6 + .../control-shallow-chain-is-named/prompt.md | 11 ++ .../samples/candidate-judge.json | 14 ++ .../samples/names-the-chain.json | 18 ++ .../samples/no-empty-verdict.json | 18 ++ .../scaffold.sh | 95 +++++++++ .../graders/no-promoted-badge.md | 6 + .../graders/not-flagged-judge.md | 11 ++ .../graders/says-already-deep.md | 5 + .../deep-module-not-over-flagged/prompt.md | 11 ++ .../samples/no-promoted-badge.json | 22 +++ .../samples/not-flagged-judge.json | 14 ++ .../samples/says-already-deep.json | 18 ++ .../no-findings-deep-tax-package/case.yaml | 4 + .../graders/no-findings-judge.md | 13 ++ .../graders/no-promoted-badge.md | 6 + .../graders/skill-fired.md | 7 + .../graders/states-no-candidates.md | 5 + .../no-findings-deep-tax-package/prompt.md | 11 ++ .../samples/no-findings-judge.json | 14 ++ .../samples/no-promoted-badge.json | 18 ++ .../samples/skill-fired.json | 33 ++++ .../samples/states-no-candidates.json | 22 +++ .../no-findings-deep-tax-package/scaffold.sh | 182 ++++++++++++++++++ .../no-findings-deep-toc-library/case.yaml | 4 + .../graders/no-findings-judge.md | 13 ++ .../graders/no-promoted-badge.md | 6 + .../graders/skill-fired.md | 7 + .../graders/states-no-candidates.md | 5 + .../no-findings-deep-toc-library/prompt.md | 11 ++ .../samples/no-findings-judge.json | 14 ++ .../samples/no-promoted-badge.json | 18 ++ .../samples/skill-fired.json | 33 ++++ .../samples/states-no-candidates.json | 22 +++ .../no-findings-deep-toc-library/scaffold.sh | 158 +++++++++++++++ 40 files changed, 883 insertions(+), 1 deletion(-) create mode 100644 plugins/architecture/evals/control-shallow-chain-is-named/case.yaml create mode 100644 plugins/architecture/evals/control-shallow-chain-is-named/graders/candidate-judge.md create mode 100644 plugins/architecture/evals/control-shallow-chain-is-named/graders/names-the-chain.md create mode 100644 plugins/architecture/evals/control-shallow-chain-is-named/graders/no-empty-verdict.md create mode 100644 plugins/architecture/evals/control-shallow-chain-is-named/prompt.md create mode 100644 plugins/architecture/evals/control-shallow-chain-is-named/samples/candidate-judge.json create mode 100644 plugins/architecture/evals/control-shallow-chain-is-named/samples/names-the-chain.json create mode 100644 plugins/architecture/evals/control-shallow-chain-is-named/samples/no-empty-verdict.json create mode 100755 plugins/architecture/evals/control-shallow-chain-is-named/scaffold.sh create mode 100644 plugins/architecture/evals/deep-module-not-over-flagged/graders/no-promoted-badge.md create mode 100644 plugins/architecture/evals/deep-module-not-over-flagged/graders/not-flagged-judge.md create mode 100644 plugins/architecture/evals/deep-module-not-over-flagged/graders/says-already-deep.md create mode 100644 plugins/architecture/evals/deep-module-not-over-flagged/prompt.md create mode 100644 plugins/architecture/evals/deep-module-not-over-flagged/samples/no-promoted-badge.json create mode 100644 plugins/architecture/evals/deep-module-not-over-flagged/samples/not-flagged-judge.json create mode 100644 plugins/architecture/evals/deep-module-not-over-flagged/samples/says-already-deep.json create mode 100644 plugins/architecture/evals/no-findings-deep-tax-package/case.yaml create mode 100644 plugins/architecture/evals/no-findings-deep-tax-package/graders/no-findings-judge.md create mode 100644 plugins/architecture/evals/no-findings-deep-tax-package/graders/no-promoted-badge.md create mode 100644 plugins/architecture/evals/no-findings-deep-tax-package/graders/skill-fired.md create mode 100644 plugins/architecture/evals/no-findings-deep-tax-package/graders/states-no-candidates.md create mode 100644 plugins/architecture/evals/no-findings-deep-tax-package/prompt.md create mode 100644 plugins/architecture/evals/no-findings-deep-tax-package/samples/no-findings-judge.json create mode 100644 plugins/architecture/evals/no-findings-deep-tax-package/samples/no-promoted-badge.json create mode 100644 plugins/architecture/evals/no-findings-deep-tax-package/samples/skill-fired.json create mode 100644 plugins/architecture/evals/no-findings-deep-tax-package/samples/states-no-candidates.json create mode 100755 plugins/architecture/evals/no-findings-deep-tax-package/scaffold.sh create mode 100644 plugins/architecture/evals/no-findings-deep-toc-library/case.yaml create mode 100644 plugins/architecture/evals/no-findings-deep-toc-library/graders/no-findings-judge.md create mode 100644 plugins/architecture/evals/no-findings-deep-toc-library/graders/no-promoted-badge.md create mode 100644 plugins/architecture/evals/no-findings-deep-toc-library/graders/skill-fired.md create mode 100644 plugins/architecture/evals/no-findings-deep-toc-library/graders/states-no-candidates.md create mode 100644 plugins/architecture/evals/no-findings-deep-toc-library/prompt.md create mode 100644 plugins/architecture/evals/no-findings-deep-toc-library/samples/no-findings-judge.json create mode 100644 plugins/architecture/evals/no-findings-deep-toc-library/samples/no-promoted-badge.json create mode 100644 plugins/architecture/evals/no-findings-deep-toc-library/samples/skill-fired.json create mode 100644 plugins/architecture/evals/no-findings-deep-toc-library/samples/states-no-candidates.json create mode 100755 plugins/architecture/evals/no-findings-deep-toc-library/scaffold.sh diff --git a/plugins/architecture/.claude-plugin/plugin.json b/plugins/architecture/.claude-plugin/plugin.json index b8ebde40b9..468de71805 100644 --- a/plugins/architecture/.claude-plugin/plugin.json +++ b/plugins/architecture/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "architecture", - "version": "0.22.1", + "version": "0.22.2", "description": "improve uses Ousterhout's deep-module lens to find shallow modules, seam leaks, and locality gaps, reports them as HTML, and interviews the chosen one. map-landscape charts a C4 system landscape and portfolio table with drift checks. map-dependencies, map-components, map-containers, map-context, map-flow, map-events, map-data, and map-deployment draw dependency, C4, sequence, event, data, and deployment views from committed files. record-decision writes an ADR in the repo's convention.", "author": { "name": "Melodic Software", diff --git a/plugins/architecture/CHANGELOG.md b/plugins/architecture/CHANGELOG.md index 31d6b7a909..8243023071 100644 --- a/plugins/architecture/CHANGELOG.md +++ b/plugins/architecture/CHANGELOG.md @@ -3,6 +3,16 @@ All notable changes to the `architecture` plugin are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); this plugin uses semantic versioning. +## [0.22.2] - 2026-10-04 + +### Added + +- **A `claude plugin eval` suite for `improve` on codebases with nothing to deepen.** Four cases under + `evals/`, tagged `row19`: the `deep-module-not-over-flagged` case converted from `evals.json`, two + scaffolded repositories whose modules are already deep (the reply must say there is no candidate + and show no Strong or Worth exploring card), and a control repository with a shallow pass-through + chain that the reply must name. No skill text changed. + ## [0.22.1] - 2026-10-04 ### Changed diff --git a/plugins/architecture/evals/control-shallow-chain-is-named/case.yaml b/plugins/architecture/evals/control-shallow-chain-is-named/case.yaml new file mode 100644 index 0000000000..19e26b581e --- /dev/null +++ b/plugins/architecture/evals/control-shallow-chain-is-named/case.yaml @@ -0,0 +1,4 @@ +schema_version: "1.1" +name: control-shallow-chain-is-named +context: + scaffold_script: scaffold.sh diff --git a/plugins/architecture/evals/control-shallow-chain-is-named/graders/candidate-judge.md b/plugins/architecture/evals/control-shallow-chain-is-named/graders/candidate-judge.md new file mode 100644 index 0000000000..bc390872ac --- /dev/null +++ b/plugins/architecture/evals/control-shallow-chain-is-named/graders/candidate-judge.md @@ -0,0 +1,9 @@ +--- +type: llm +--- + +The repository's order intake is a chain of three classes: `OrderHandler.handle` applies a coupon and calls `OrderValidator.validate`, which checks lines, computes the subtotal and calls `OrderRepo.save`, which applies the discount and stores the order. Each class has one method, the pricing rules are split across all three, and the tests mock each next hop. + +PASS if the reply presents at least one deepening candidate that covers this chain (all or part of it), describes it as shallow or as rules spread across pass-through hops, and does not claim the repository has no candidates. + +FAIL if the reply reports no candidates, names only unrelated modules, or proposes code changes it has applied instead of reporting candidates. diff --git a/plugins/architecture/evals/control-shallow-chain-is-named/graders/names-the-chain.md b/plugins/architecture/evals/control-shallow-chain-is-named/graders/names-the-chain.md new file mode 100644 index 0000000000..926e480c6c --- /dev/null +++ b/plugins/architecture/evals/control-shallow-chain-is-named/graders/names-the-chain.md @@ -0,0 +1,4 @@ +--- +type: regex +pattern: 'Order(Handler|Validator|Repo)|(handler|validator|repo)\.py' +--- diff --git a/plugins/architecture/evals/control-shallow-chain-is-named/graders/no-empty-verdict.md b/plugins/architecture/evals/control-shallow-chain-is-named/graders/no-empty-verdict.md new file mode 100644 index 0000000000..9f3f219299 --- /dev/null +++ b/plugins/architecture/evals/control-shallow-chain-is-named/graders/no-empty-verdict.md @@ -0,0 +1,6 @@ +--- +type: regex +pattern: '\bno\s+(deepening\s+)?(candidates|opportunities)\b|\bnothing\s+(to\s+deepen|worth\s+deepening)' +flags: i +match: not_contains +--- diff --git a/plugins/architecture/evals/control-shallow-chain-is-named/prompt.md b/plugins/architecture/evals/control-shallow-chain-is-named/prompt.md new file mode 100644 index 0000000000..dea0678e35 --- /dev/null +++ b/plugins/architecture/evals/control-shallow-chain-is-named/prompt.md @@ -0,0 +1,11 @@ +--- +description: "Control: a real shallow pass-through chain is named as a candidate, so an always-empty verdict cannot pass the row 19 cases" +tags: [architecture, improve, row19, scaffold, control] +runs: 3 +max_turns: 40 +timeout_seconds: 900 +allowed_tools: [Read, Glob, Grep, Skill, Agent] +expected_outcome: "Names the OrderHandler -> OrderValidator -> OrderRepo chain as a deepening candidate (shallow pass-through hops with the pricing rules spread across them) and does not report that the repository has no candidates" +--- + +/architecture:improve deepening — scan this repository for deepening opportunities. diff --git a/plugins/architecture/evals/control-shallow-chain-is-named/samples/candidate-judge.json b/plugins/architecture/evals/control-shallow-chain-is-named/samples/candidate-judge.json new file mode 100644 index 0000000000..24e7fe22ce --- /dev/null +++ b/plugins/architecture/evals/control-shallow-chain-is-named/samples/candidate-judge.json @@ -0,0 +1,14 @@ +{ + "pass": [ + { + "answer": "Candidate 1 (Strong): order intake. OrderHandler, OrderValidator and OrderRepo each have one method that forwards to the next, and the coupon, subtotal and discount rules are split across them, so the tests mock every hop. Deleting the middle class would concentrate the rules rather than move them. Which candidate would you like to explore?", + "why": "names the chain as shallow and stops at the report" + } + ], + "fail": [ + { + "answer": "No deepening candidates. The orders package is small and each class has a clear job.", + "why": "misses a real shallow chain" + } + ] +} diff --git a/plugins/architecture/evals/control-shallow-chain-is-named/samples/names-the-chain.json b/plugins/architecture/evals/control-shallow-chain-is-named/samples/names-the-chain.json new file mode 100644 index 0000000000..444a15ec53 --- /dev/null +++ b/plugins/architecture/evals/control-shallow-chain-is-named/samples/names-the-chain.json @@ -0,0 +1,18 @@ +{ + "pass": [ + { + "answer": "Candidate 1 (Strong): OrderHandler -> OrderValidator -> OrderRepo is a shallow chain; the pricing rules are split across all three.", + "why": "names the chain" + } + ], + "fail": [ + { + "answer": "", + "why": "empty reply" + }, + { + "answer": "No deepening candidates found in this repository.", + "why": "names nothing" + } + ] +} diff --git a/plugins/architecture/evals/control-shallow-chain-is-named/samples/no-empty-verdict.json b/plugins/architecture/evals/control-shallow-chain-is-named/samples/no-empty-verdict.json new file mode 100644 index 0000000000..0863b35a8e --- /dev/null +++ b/plugins/architecture/evals/control-shallow-chain-is-named/samples/no-empty-verdict.json @@ -0,0 +1,18 @@ +{ + "pass": [ + { + "answer": "One candidate: the order intake chain. No other modules in the repository.", + "why": "reports a candidate" + } + ], + "fail": [ + { + "answer": "No deepening candidates: every module here earns its keep.", + "why": "empty verdict on a shallow chain" + }, + { + "answer": "I found nothing worth deepening.", + "why": "empty verdict, other wording" + } + ] +} diff --git a/plugins/architecture/evals/control-shallow-chain-is-named/scaffold.sh b/plugins/architecture/evals/control-shallow-chain-is-named/scaffold.sh new file mode 100755 index 0000000000..f0bdac0486 --- /dev/null +++ b/plugins/architecture/evals/control-shallow-chain-is-named/scaffold.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# Control for the no-findings cases: an order-intake path of three one-method pass-through classes, +# with the pricing and validation rules spread across them and tests that mock each hop. A correct +# scan names this chain as a candidate, so a skill that always answers "no candidates" fails here. +set -euo pipefail + +git init -q -b main +git config user.name eval +git config user.email eval@example.invalid +mkdir -p orders tests +: > tests/__init__.py + +cat > orders/__init__.py <<'PY' +PY + +cat > orders/handler.py <<'PY' +from orders.validator import OrderValidator + + +class OrderHandler: + def __init__(self, validator=None): + self.validator = validator or OrderValidator() + + def handle(self, order): + if order.get("coupon") == "WELCOME10": + order["discount"] = 0.10 + return self.validator.validate(order) +PY + +cat > orders/validator.py <<'PY' +from orders.repo import OrderRepo + + +class OrderValidator: + def __init__(self, repo=None): + self.repo = repo or OrderRepo() + + def validate(self, order): + if not order.get("lines"): + raise ValueError("order has no lines") + order["subtotal"] = sum(l["price"] * l["qty"] for l in order["lines"]) + return self.repo.save(order) +PY + +cat > orders/repo.py <<'PY' +class OrderRepo: + def __init__(self): + self.rows = [] + + def save(self, order): + order["total"] = round(order["subtotal"] * (1 - order.get("discount", 0)), 2) + self.rows.append(order) + return len(self.rows) +PY + +cat > tests/test_handler.py <<'PY' +import unittest +from unittest.mock import MagicMock + +from orders.handler import OrderHandler + + +class HandlerTest(unittest.TestCase): + def test_forwards_to_validator(self): + validator = MagicMock() + validator.validate.return_value = 7 + self.assertEqual(OrderHandler(validator).handle({"lines": []}), 7) + validator.validate.assert_called_once() + + +if __name__ == "__main__": + unittest.main() +PY + +cat > tests/test_validator.py <<'PY' +import unittest +from unittest.mock import MagicMock + +from orders.validator import OrderValidator + + +class ValidatorTest(unittest.TestCase): + def test_forwards_to_repo(self): + repo = MagicMock() + repo.save.return_value = 1 + self.assertEqual(OrderValidator(repo).validate({"lines": [{"price": 2, "qty": 1}]}), 1) + repo.save.assert_called_once() + + +if __name__ == "__main__": + unittest.main() +PY + +git add . +git commit -q -m "feat: order intake" diff --git a/plugins/architecture/evals/deep-module-not-over-flagged/graders/no-promoted-badge.md b/plugins/architecture/evals/deep-module-not-over-flagged/graders/no-promoted-badge.md new file mode 100644 index 0000000000..cd02090f94 --- /dev/null +++ b/plugins/architecture/evals/deep-module-not-over-flagged/graders/no-promoted-badge.md @@ -0,0 +1,6 @@ +--- +type: regex +pattern: '(badge|recommendation|rating)\W{0,4}(Strong|Worth exploring)\b|\[(Strong|Worth exploring)\]|\*\*(Strong|Worth exploring)\*\*' +flags: i +match: not_contains +--- diff --git a/plugins/architecture/evals/deep-module-not-over-flagged/graders/not-flagged-judge.md b/plugins/architecture/evals/deep-module-not-over-flagged/graders/not-flagged-judge.md new file mode 100644 index 0000000000..4a3876cc4c --- /dev/null +++ b/plugins/architecture/evals/deep-module-not-over-flagged/graders/not-flagged-judge.md @@ -0,0 +1,11 @@ +--- +type: llm +--- + +PASS if every one of these holds: + +1. The reply does not recommend deepening the pricing engine, and treats it as already deep (a small interface over a large, tested implementation). +2. If the pricing engine appears as a candidate at all, its badge is Speculative and the reply gives a reason; it is never Strong or Worth exploring. +3. The reply leaves the recorded decision to keep the rules in-process standing, or names the conflict explicitly if it questions it. + +FAIL if the reply proposes a new interface, a split, or extracted modules for the pricing engine as a recommended change, gives it a Strong or Worth exploring badge, or overrides the decision record without naming the conflict. diff --git a/plugins/architecture/evals/deep-module-not-over-flagged/graders/says-already-deep.md b/plugins/architecture/evals/deep-module-not-over-flagged/graders/says-already-deep.md new file mode 100644 index 0000000000..a6fa1c5914 --- /dev/null +++ b/plugins/architecture/evals/deep-module-not-over-flagged/graders/says-already-deep.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: 'already\s+(a\s+|quite\s+|very\s+)?deep|(just|only|merely|simply)\s+(move|moves|moving|relocate|relocates|shift|shifts)\b' +flags: i +--- diff --git a/plugins/architecture/evals/deep-module-not-over-flagged/prompt.md b/plugins/architecture/evals/deep-module-not-over-flagged/prompt.md new file mode 100644 index 0000000000..bb46d94638 --- /dev/null +++ b/plugins/architecture/evals/deep-module-not-over-flagged/prompt.md @@ -0,0 +1,11 @@ +--- +description: "An already-deep pricing module is not proposed for deepening; at most a Speculative card with its reason" +tags: [architecture, improve, row19, converted] +runs: 3 +max_turns: 30 +timeout_seconds: 900 +allowed_tools: [Read, Glob, Grep, Skill, Agent] +expected_outcome: "Judges the pricing engine already deep (small interface, large tested implementation), says deepening would only move complexity, gives it no Strong or Worth exploring badge, and leaves the recorded in-process decision standing" +--- + +/architecture:improve deepening — review the pricing engine: a single Price(order) entry point backed by ~600 lines of rules, tax tables, and rounding policy, with a focused test suite that drives it only through Price(order). An architecture decision record already records keeping the rules in-process. diff --git a/plugins/architecture/evals/deep-module-not-over-flagged/samples/no-promoted-badge.json b/plugins/architecture/evals/deep-module-not-over-flagged/samples/no-promoted-badge.json new file mode 100644 index 0000000000..1e1fffa477 --- /dev/null +++ b/plugins/architecture/evals/deep-module-not-over-flagged/samples/no-promoted-badge.json @@ -0,0 +1,22 @@ +{ + "pass": [ + { + "answer": "No deepening candidates. The pricing engine is already deep and the ADR keeps its rules in-process.", + "why": "no badge at all" + }, + { + "answer": "One Speculative note: the rounding table could live beside the tax tables. Badge: Speculative, because the interface is already small.", + "why": "Speculative only" + } + ], + "fail": [ + { + "answer": "Candidate 1, pricing engine. Badge: Strong. Split the rules into three modules.", + "why": "Strong badge on a deep module" + }, + { + "answer": "**Worth exploring**: extract the tax tables behind a TaxPolicy seam.", + "why": "Worth exploring card" + } + ] +} diff --git a/plugins/architecture/evals/deep-module-not-over-flagged/samples/not-flagged-judge.json b/plugins/architecture/evals/deep-module-not-over-flagged/samples/not-flagged-judge.json new file mode 100644 index 0000000000..3dfd8c64b1 --- /dev/null +++ b/plugins/architecture/evals/deep-module-not-over-flagged/samples/not-flagged-judge.json @@ -0,0 +1,14 @@ +{ + "pass": [ + { + "answer": "Price(order) is a deep module: one small interface hides about 600 lines of rules, tax tables and rounding, and the tests drive only that interface. The deletion test says deleting it would spread the rules across callers, so complexity would only move. No deepening candidate here, and the ADR keeping the rules in-process stands.", + "why": "not flagged, decision respected" + } + ], + "fail": [ + { + "answer": "Strong candidate: break the pricing engine into RuleEngine, TaxTables and Rounding modules with their own interfaces, and move the rules out of process into a rules service.", + "why": "invents shallowness and overrides the ADR silently" + } + ] +} diff --git a/plugins/architecture/evals/deep-module-not-over-flagged/samples/says-already-deep.json b/plugins/architecture/evals/deep-module-not-over-flagged/samples/says-already-deep.json new file mode 100644 index 0000000000..0289a9bdf1 --- /dev/null +++ b/plugins/architecture/evals/deep-module-not-over-flagged/samples/says-already-deep.json @@ -0,0 +1,18 @@ +{ + "pass": [ + { + "answer": "Price(order) is already deep: one entry point over roughly 600 lines of rules. The deletion test says removing it would just move the rules into every caller, so it is not a candidate.", + "why": "names it deep and the deletion-test result" + } + ], + "fail": [ + { + "answer": "", + "why": "empty reply" + }, + { + "answer": "Candidate: split Price(order) into TaxPolicy, RoundingPolicy and RuleSet modules so each can be tested on its own.", + "why": "proposes deepening without judging depth" + } + ] +} diff --git a/plugins/architecture/evals/no-findings-deep-tax-package/case.yaml b/plugins/architecture/evals/no-findings-deep-tax-package/case.yaml new file mode 100644 index 0000000000..9ec3646ef2 --- /dev/null +++ b/plugins/architecture/evals/no-findings-deep-tax-package/case.yaml @@ -0,0 +1,4 @@ +schema_version: "1.1" +name: no-findings-deep-tax-package +context: + scaffold_script: scaffold.sh diff --git a/plugins/architecture/evals/no-findings-deep-tax-package/graders/no-findings-judge.md b/plugins/architecture/evals/no-findings-deep-tax-package/graders/no-findings-judge.md new file mode 100644 index 0000000000..27725bc699 --- /dev/null +++ b/plugins/architecture/evals/no-findings-deep-tax-package/graders/no-findings-judge.md @@ -0,0 +1,13 @@ +--- +type: llm +--- + +The repository holds one module, `tax`, whose only public entry point is `compute_tax(order)`; the rate tables, exemptions and rounding sit in private files behind it, the tests call only `compute_tax`, and an ADR keeps the rules in-process. `invoice.py` is a thin command-line caller. + +PASS if every one of these holds: + +1. The reply states that the scan found no deepening candidate (or no shallow module worth deepening) in this repository. +2. The reply presents no module as a recommended candidate: no card or numbered candidate with a Strong or Worth exploring badge, and no proposed new interface for `tax`. +3. Any module the reply mentions is described accurately (for example, `tax` as already deep, `invoice.py` as a thin caller), not as shallow. + +FAIL if the reply lists one or more candidates to deepen, proposes splitting or wrapping `tax`, calls `tax` shallow, or ends by asking which candidate to explore when it named none. diff --git a/plugins/architecture/evals/no-findings-deep-tax-package/graders/no-promoted-badge.md b/plugins/architecture/evals/no-findings-deep-tax-package/graders/no-promoted-badge.md new file mode 100644 index 0000000000..cd02090f94 --- /dev/null +++ b/plugins/architecture/evals/no-findings-deep-tax-package/graders/no-promoted-badge.md @@ -0,0 +1,6 @@ +--- +type: regex +pattern: '(badge|recommendation|rating)\W{0,4}(Strong|Worth exploring)\b|\[(Strong|Worth exploring)\]|\*\*(Strong|Worth exploring)\*\*' +flags: i +match: not_contains +--- diff --git a/plugins/architecture/evals/no-findings-deep-tax-package/graders/skill-fired.md b/plugins/architecture/evals/no-findings-deep-tax-package/graders/skill-fired.md new file mode 100644 index 0000000000..133b70a6a5 --- /dev/null +++ b/plugins/architecture/evals/no-findings-deep-tax-package/graders/skill-fired.md @@ -0,0 +1,7 @@ +--- +type: tool_used +tool: Skill +input_match: 'improve' +min: 1 +arm: with-only +--- diff --git a/plugins/architecture/evals/no-findings-deep-tax-package/graders/states-no-candidates.md b/plugins/architecture/evals/no-findings-deep-tax-package/graders/states-no-candidates.md new file mode 100644 index 0000000000..04e230b729 --- /dev/null +++ b/plugins/architecture/evals/no-findings-deep-tax-package/graders/states-no-candidates.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: '\b(no|zero|0|not any)\b[^.\n]{0,40}\b(deepening\s+)?(candidates?|opportunit(y|ies)|shallow modules?)\b|\bnothing\b[^.\n]{0,30}\b(to\s+deepen|worth\s+deepening|to\s+recommend)|\bnone\b[^.\n]{0,40}\b(qualif|warrant|candidate|shallow)' +flags: i +--- diff --git a/plugins/architecture/evals/no-findings-deep-tax-package/prompt.md b/plugins/architecture/evals/no-findings-deep-tax-package/prompt.md new file mode 100644 index 0000000000..db2ce18611 --- /dev/null +++ b/plugins/architecture/evals/no-findings-deep-tax-package/prompt.md @@ -0,0 +1,11 @@ +--- +description: "A repo whose only module is already deep gets an explicit no-candidates verdict, not invented cards" +tags: [architecture, improve, row19, scaffold] +runs: 3 +max_turns: 40 +timeout_seconds: 900 +allowed_tools: [Read, Glob, Grep, Skill, Agent] +expected_outcome: "Scans the repo, finds tax already deep (compute_tax is the one entry point over rates, exemptions and rounding, tested only through it), and says plainly that there is no deepening candidate, with no Strong or Worth exploring card" +--- + +/architecture:improve deepening — scan this repository for deepening opportunities. diff --git a/plugins/architecture/evals/no-findings-deep-tax-package/samples/no-findings-judge.json b/plugins/architecture/evals/no-findings-deep-tax-package/samples/no-findings-judge.json new file mode 100644 index 0000000000..5158870b60 --- /dev/null +++ b/plugins/architecture/evals/no-findings-deep-tax-package/samples/no-findings-judge.json @@ -0,0 +1,14 @@ +{ + "pass": [ + { + "answer": "Scanned tax/, invoice.py and tests/. No deepening candidates: `tax` exposes one function, compute_tax, over the rate tables, exemptions and rounding, and the tests drive only that function. invoice.py is a thin caller. Deleting tax would spread its rules into every caller, so it is earning its keep. Nothing to report as a card.", + "why": "explicit no-findings verdict, modules described accurately" + } + ], + "fail": [ + { + "answer": "Three candidates. 1 (Strong): _rates.py and _rounding.py are shallow helpers; fold them into a TaxPolicy module. 2 (Worth exploring): invoice.py should own formatting through a Presenter seam. 3 (Speculative): compute_tax takes a dict; give it an Order type. Which of these would you like to explore?", + "why": "invents candidates on an already-deep module" + } + ] +} diff --git a/plugins/architecture/evals/no-findings-deep-tax-package/samples/no-promoted-badge.json b/plugins/architecture/evals/no-findings-deep-tax-package/samples/no-promoted-badge.json new file mode 100644 index 0000000000..f8187e8fe0 --- /dev/null +++ b/plugins/architecture/evals/no-findings-deep-tax-package/samples/no-promoted-badge.json @@ -0,0 +1,18 @@ +{ + "pass": [ + { + "answer": "No deepening candidates in this repository; `tax` is already deep.", + "why": "no badge" + } + ], + "fail": [ + { + "answer": "Candidate 1: tax rate lookup. Badge: Worth exploring. Pull rate_for behind a RateProvider seam.", + "why": "Worth exploring card" + }, + { + "answer": "[Strong] invoice.py and tax: merge the CLI into the package.", + "why": "Strong card" + } + ] +} diff --git a/plugins/architecture/evals/no-findings-deep-tax-package/samples/skill-fired.json b/plugins/architecture/evals/no-findings-deep-tax-package/samples/skill-fired.json new file mode 100644 index 0000000000..22e1e83ab7 --- /dev/null +++ b/plugins/architecture/evals/no-findings-deep-tax-package/samples/skill-fired.json @@ -0,0 +1,33 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "architecture:improve", + "args": "deepening" + } + } + ], + "why": "the improve skill was invoked" + } + ], + "fail": [ + { + "answer": [], + "why": "no calls" + }, + { + "answer": [ + { + "tool": "Grep", + "input": { + "pattern": "def " + } + } + ], + "why": "scanned by hand, skill never invoked" + } + ] +} diff --git a/plugins/architecture/evals/no-findings-deep-tax-package/samples/states-no-candidates.json b/plugins/architecture/evals/no-findings-deep-tax-package/samples/states-no-candidates.json new file mode 100644 index 0000000000..19c0bfcb02 --- /dev/null +++ b/plugins/architecture/evals/no-findings-deep-tax-package/samples/states-no-candidates.json @@ -0,0 +1,22 @@ +{ + "pass": [ + { + "answer": "Scan complete: no deepening candidates. `tax` is already deep: compute_tax is its one entry point and the tests drive only it.", + "why": "explicit no-candidates line" + }, + { + "answer": "I found nothing worth deepening here. The tax package hides its rate tables and rounding behind one function.", + "why": "nothing worth deepening" + } + ], + "fail": [ + { + "answer": "", + "why": "empty reply" + }, + { + "answer": "Candidate 1 (Worth exploring): the rates and rounding helpers are separate files; merge them into one TaxPolicy module. Which candidate would you like to explore?", + "why": "invents a candidate" + } + ] +} diff --git a/plugins/architecture/evals/no-findings-deep-tax-package/scaffold.sh b/plugins/architecture/evals/no-findings-deep-tax-package/scaffold.sh new file mode 100755 index 0000000000..f60033a235 --- /dev/null +++ b/plugins/architecture/evals/no-findings-deep-tax-package/scaffold.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# A small invoicing tool whose one module, tax, is already deep: compute_tax(order) is the only +# entry point, the rate tables, exemptions and rounding sit behind it, and the tests drive only it. +set -euo pipefail + +git init -q -b main +git config user.name eval +git config user.email eval@example.invalid +mkdir -p tax tests docs/adr +: > tests/__init__.py + +cat > tax/__init__.py <<'PY' +"""Sales tax for an order. compute_tax is the whole public interface.""" +from tax._engine import TaxResult, compute_tax + +__all__ = ["TaxResult", "compute_tax"] +PY + +cat > tax/_engine.py <<'PY' +from dataclasses import dataclass +from decimal import Decimal + +from tax._rates import exempt, rate_for +from tax._rounding import round_line + + +@dataclass(frozen=True) +class TaxResult: + lines: tuple + total_tax: Decimal + total: Decimal + + +def compute_tax(order): + """Tax every line of an order for its ship-to region and return per-line and total tax.""" + region = order["ship_to"]["region"] + lines = [] + subtotal = Decimal("0") + total_tax = Decimal("0") + for line in order["lines"]: + amount = Decimal(str(line["unit_price"])) * line["quantity"] + amount -= Decimal(str(line.get("discount", 0))) + if amount < 0: + amount = Decimal("0") + if exempt(line["category"], region, order.get("customer_type", "retail")): + tax = Decimal("0") + else: + tax = round_line(amount * rate_for(line["category"], region), region) + lines.append((line["sku"], amount, tax)) + subtotal += amount + total_tax += tax + return TaxResult(tuple(lines), total_tax, subtotal + total_tax) +PY + +cat > tax/_rates.py <<'PY' +from decimal import Decimal + +_BASE = { + "WA": Decimal("0.065"), + "OR": Decimal("0"), + "CA": Decimal("0.0725"), + "NY": Decimal("0.04"), + "TX": Decimal("0.0625"), +} + +_CATEGORY_OVERRIDES = { + ("CA", "groceries"): Decimal("0"), + ("NY", "clothing"): Decimal("0"), + ("TX", "groceries"): Decimal("0"), + ("WA", "digital"): Decimal("0.065"), + ("NY", "digital"): Decimal("0.04"), +} + +_EXEMPT_CUSTOMERS = {"nonprofit", "government"} + + +def rate_for(category, region): + if region not in _BASE: + raise ValueError("no tax table for region %s" % region) + return _CATEGORY_OVERRIDES.get((region, category), _BASE[region]) + + +def exempt(category, region, customer_type): + if customer_type in _EXEMPT_CUSTOMERS: + return True + return category == "resale" and region != "OR" +PY + +cat > tax/_rounding.py <<'PY' +from decimal import ROUND_HALF_EVEN, ROUND_HALF_UP, Decimal + +_CENT = Decimal("0.01") +_BANKERS = {"NY"} + + +def round_line(value, region): + mode = ROUND_HALF_EVEN if region in _BANKERS else ROUND_HALF_UP + return value.quantize(_CENT, rounding=mode) +PY + +cat > invoice.py <<'PY' +import json +import sys + +from tax import compute_tax + + +def main(path): + with open(path) as handle: + order = json.load(handle) + result = compute_tax(order) + for sku, amount, tax in result.lines: + print("%-10s %10s %8s" % (sku, amount, tax)) + print("tax %s total %s" % (result.total_tax, result.total)) + + +if __name__ == "__main__": + main(sys.argv[1]) +PY + +cat > tests/test_tax.py <<'PY' +import unittest +from decimal import Decimal + +from tax import compute_tax + + +def order(region, *lines, customer_type="retail"): + return {"ship_to": {"region": region}, "customer_type": customer_type, "lines": list(lines)} + + +def line(sku, price, qty, category="general", discount=0): + return {"sku": sku, "unit_price": price, "quantity": qty, "category": category, "discount": discount} + + +class ComputeTax(unittest.TestCase): + def test_washington_general_goods(self): + self.assertEqual(compute_tax(order("WA", line("A", 10, 2))).total_tax, Decimal("1.30")) + + def test_oregon_has_no_sales_tax(self): + self.assertEqual(compute_tax(order("OR", line("A", 99.99, 1))).total_tax, Decimal("0")) + + def test_california_groceries_are_zero_rated(self): + self.assertEqual(compute_tax(order("CA", line("G", 5, 4, "groceries"))).total_tax, Decimal("0")) + + def test_nonprofit_is_exempt(self): + result = compute_tax(order("TX", line("A", 100, 1), customer_type="nonprofit")) + self.assertEqual(result.total, Decimal("100")) + + def test_new_york_rounds_half_even(self): + self.assertEqual(compute_tax(order("NY", line("A", "0.625", 1))).total_tax, Decimal("0.02")) + + def test_discount_never_goes_negative(self): + result = compute_tax(order("WA", line("A", 5, 1, discount=10))) + self.assertEqual(result.total, Decimal("0")) + + def test_unknown_region_is_rejected(self): + with self.assertRaises(ValueError): + compute_tax(order("ZZ", line("A", 1, 1))) + + +if __name__ == "__main__": + unittest.main() +PY + +cat > docs/adr/0001-keep-tax-rules-in-process.md <<'MD' +# 1. Keep tax rules in process + +Status: accepted + +The rate tables change a few times a year and ship with a release. They stay in the `tax` package +behind `compute_tax`; no rules service. +MD + +cat > README.md <<'MD' +# invoice + +`python3 invoice.py order.json` prints per-line tax and the order total. Tests: `python3 -m unittest`. +MD + +git add . +git commit -q -m "feat: invoice tax calculation" diff --git a/plugins/architecture/evals/no-findings-deep-toc-library/case.yaml b/plugins/architecture/evals/no-findings-deep-toc-library/case.yaml new file mode 100644 index 0000000000..f0303b1ced --- /dev/null +++ b/plugins/architecture/evals/no-findings-deep-toc-library/case.yaml @@ -0,0 +1,4 @@ +schema_version: "1.1" +name: no-findings-deep-toc-library +context: + scaffold_script: scaffold.sh diff --git a/plugins/architecture/evals/no-findings-deep-toc-library/graders/no-findings-judge.md b/plugins/architecture/evals/no-findings-deep-toc-library/graders/no-findings-judge.md new file mode 100644 index 0000000000..59de2b81e5 --- /dev/null +++ b/plugins/architecture/evals/no-findings-deep-toc-library/graders/no-findings-judge.md @@ -0,0 +1,13 @@ +--- +type: llm +--- + +The repository is a small JavaScript library. Its only export is `buildToc(markdown, options)` in `src/index.js`; `src/parse.js` (heading parsing, code fences, setext headings), `src/slug.js` (slug de-duplication) and `src/tree.js` (nesting and rendering) are imported only by `src/index.js`, and the tests call only `buildToc`. + +PASS if every one of these holds: + +1. The reply states that the scan found no deepening candidate (or no shallow module worth deepening) in this repository. +2. The reply presents no module as a recommended candidate: no card or numbered candidate with a Strong or Worth exploring badge, and no proposed new interface for `buildToc` or its helpers. +3. Any module the reply mentions is described accurately (for example, `buildToc` as one small interface over the helpers), not as a shallow pass-through. + +FAIL if the reply lists one or more candidates to deepen, proposes merging, exporting or wrapping the helpers as a recommended change, calls a helper a shallow module, or ends by asking which candidate to explore when it named none. diff --git a/plugins/architecture/evals/no-findings-deep-toc-library/graders/no-promoted-badge.md b/plugins/architecture/evals/no-findings-deep-toc-library/graders/no-promoted-badge.md new file mode 100644 index 0000000000..cd02090f94 --- /dev/null +++ b/plugins/architecture/evals/no-findings-deep-toc-library/graders/no-promoted-badge.md @@ -0,0 +1,6 @@ +--- +type: regex +pattern: '(badge|recommendation|rating)\W{0,4}(Strong|Worth exploring)\b|\[(Strong|Worth exploring)\]|\*\*(Strong|Worth exploring)\*\*' +flags: i +match: not_contains +--- diff --git a/plugins/architecture/evals/no-findings-deep-toc-library/graders/skill-fired.md b/plugins/architecture/evals/no-findings-deep-toc-library/graders/skill-fired.md new file mode 100644 index 0000000000..133b70a6a5 --- /dev/null +++ b/plugins/architecture/evals/no-findings-deep-toc-library/graders/skill-fired.md @@ -0,0 +1,7 @@ +--- +type: tool_used +tool: Skill +input_match: 'improve' +min: 1 +arm: with-only +--- diff --git a/plugins/architecture/evals/no-findings-deep-toc-library/graders/states-no-candidates.md b/plugins/architecture/evals/no-findings-deep-toc-library/graders/states-no-candidates.md new file mode 100644 index 0000000000..04e230b729 --- /dev/null +++ b/plugins/architecture/evals/no-findings-deep-toc-library/graders/states-no-candidates.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: '\b(no|zero|0|not any)\b[^.\n]{0,40}\b(deepening\s+)?(candidates?|opportunit(y|ies)|shallow modules?)\b|\bnothing\b[^.\n]{0,30}\b(to\s+deepen|worth\s+deepening|to\s+recommend)|\bnone\b[^.\n]{0,40}\b(qualif|warrant|candidate|shallow)' +flags: i +--- diff --git a/plugins/architecture/evals/no-findings-deep-toc-library/prompt.md b/plugins/architecture/evals/no-findings-deep-toc-library/prompt.md new file mode 100644 index 0000000000..e2f03e1d8d --- /dev/null +++ b/plugins/architecture/evals/no-findings-deep-toc-library/prompt.md @@ -0,0 +1,11 @@ +--- +description: "A library with one deep export gets an explicit no-candidates verdict; its private helpers are not carded as shallow" +tags: [architecture, improve, row19, scaffold] +runs: 3 +max_turns: 40 +timeout_seconds: 900 +allowed_tools: [Read, Glob, Grep, Skill, Agent] +expected_outcome: "Scans the library, finds buildToc already deep (one export over parsing, slug de-duplication and nesting, tested only through it), and says plainly that there is no deepening candidate, with no Strong or Worth exploring card" +--- + +/architecture:improve find deepening opportunities in this codebase. diff --git a/plugins/architecture/evals/no-findings-deep-toc-library/samples/no-findings-judge.json b/plugins/architecture/evals/no-findings-deep-toc-library/samples/no-findings-judge.json new file mode 100644 index 0000000000..ec741fbdc0 --- /dev/null +++ b/plugins/architecture/evals/no-findings-deep-toc-library/samples/no-findings-judge.json @@ -0,0 +1,14 @@ +{ + "pass": [ + { + "answer": "Scanned src/ and test/. No deepening candidates: buildToc is one function with three options, and behind it parse.js, slug.js and tree.js handle fences, setext headings, duplicate slugs and nesting. Only index.js imports them and the tests go through buildToc alone. Deleting any helper would push its logic into index.js, not remove it.", + "why": "explicit no-findings verdict, modules described accurately" + } + ], + "fail": [ + { + "answer": "Two candidates. 1 (Worth exploring): slug.js is a shallow module, one function; merge it into tree.js. 2 (Speculative): export parseHeadings so callers can reuse it. Which of these would you like to explore?", + "why": "cards private helpers as shallow and invents candidates" + } + ] +} diff --git a/plugins/architecture/evals/no-findings-deep-toc-library/samples/no-promoted-badge.json b/plugins/architecture/evals/no-findings-deep-toc-library/samples/no-promoted-badge.json new file mode 100644 index 0000000000..f2458f26d6 --- /dev/null +++ b/plugins/architecture/evals/no-findings-deep-toc-library/samples/no-promoted-badge.json @@ -0,0 +1,18 @@ +{ + "pass": [ + { + "answer": "No deepening candidates; buildToc already hides parsing, slugging and nesting.", + "why": "no badge" + } + ], + "fail": [ + { + "answer": "Candidate 1: slug.js. Badge: Worth exploring. Merge it into tree.js so nesting owns slugs.", + "why": "Worth exploring card" + }, + { + "answer": "**Strong**: parse.js and tree.js are a shallow pair; combine them.", + "why": "Strong card" + } + ] +} diff --git a/plugins/architecture/evals/no-findings-deep-toc-library/samples/skill-fired.json b/plugins/architecture/evals/no-findings-deep-toc-library/samples/skill-fired.json new file mode 100644 index 0000000000..9dbb457ccd --- /dev/null +++ b/plugins/architecture/evals/no-findings-deep-toc-library/samples/skill-fired.json @@ -0,0 +1,33 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "architecture:improve", + "args": "find deepening opportunities in this codebase." + } + } + ], + "why": "the improve skill was invoked" + } + ], + "fail": [ + { + "answer": [], + "why": "no calls" + }, + { + "answer": [ + { + "tool": "Read", + "input": { + "file_path": "/tmp/ws/src/index.js" + } + } + ], + "why": "read files by hand, skill never invoked" + } + ] +} diff --git a/plugins/architecture/evals/no-findings-deep-toc-library/samples/states-no-candidates.json b/plugins/architecture/evals/no-findings-deep-toc-library/samples/states-no-candidates.json new file mode 100644 index 0000000000..2ee20e6010 --- /dev/null +++ b/plugins/architecture/evals/no-findings-deep-toc-library/samples/states-no-candidates.json @@ -0,0 +1,22 @@ +{ + "pass": [ + { + "answer": "No deepening opportunities found. buildToc is the one export, and parse, slug and tree are private helpers behind it.", + "why": "explicit no-opportunities line" + }, + { + "answer": "None of the modules qualify as a candidate: each helper does real work and only index.js imports it.", + "why": "none qualify" + } + ], + "fail": [ + { + "answer": "", + "why": "empty reply" + }, + { + "answer": "Candidate 1 (Speculative): slug.js is small; fold it into tree.js. Which of these would you like to explore?", + "why": "invents a candidate" + } + ] +} diff --git a/plugins/architecture/evals/no-findings-deep-toc-library/scaffold.sh b/plugins/architecture/evals/no-findings-deep-toc-library/scaffold.sh new file mode 100755 index 0000000000..b9a3deb595 --- /dev/null +++ b/plugins/architecture/evals/no-findings-deep-toc-library/scaffold.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# A small JavaScript library whose one export, buildToc(markdown, options), hides heading parsing, +# slug de-duplication and nesting. The internals are imported only by index.js, and the tests call +# only buildToc. +set -euo pipefail + +git init -q -b main +git config user.name eval +git config user.email eval@example.invalid +mkdir -p src test + +cat > package.json <<'JSON' +{ + "name": "md-toc", + "version": "1.4.0", + "type": "module", + "main": "src/index.js", + "exports": { ".": "./src/index.js" }, + "scripts": { "test": "node --test test/" } +} +JSON + +cat > src/index.js <<'JS' +import { parseHeadings } from './parse.js'; +import { createSlugger } from './slug.js'; +import { nest, render } from './tree.js'; + +/** + * Build a Markdown table of contents for a document. + * @param {string} markdown + * @param {{ minDepth?: number, maxDepth?: number, bullet?: string }} [options] + * @returns {string} + */ +export function buildToc(markdown, { minDepth = 1, maxDepth = 3, bullet = '-' } = {}) { + const slugger = createSlugger(); + const headings = parseHeadings(markdown) + .filter((h) => h.depth >= minDepth && h.depth <= maxDepth) + .map((h) => ({ ...h, slug: slugger(h.text) })); + return render(nest(headings), bullet); +} +JS + +cat > src/parse.js <<'JS' +const ATX = /^(#{1,6})[ \t]+(.+?)[ \t]*#*[ \t]*$/; +const FENCE = /^(```|~~~)/; + +export function parseHeadings(markdown) { + const headings = []; + let fence = null; + const lines = markdown.split(/\r?\n/); + for (let i = 0; i < lines.length; i += 1) { + const line = lines[i]; + const open = line.match(FENCE); + if (open) { + if (fence === null) fence = open[1]; + else if (open[1] === fence) fence = null; + continue; + } + if (fence !== null) continue; + const atx = line.match(ATX); + if (atx) { + headings.push({ depth: atx[1].length, text: stripInline(atx[2]) }); + continue; + } + const next = lines[i + 1] ?? ''; + if (line.trim() && /^=+\s*$/.test(next)) headings.push({ depth: 1, text: stripInline(line.trim()) }); + else if (line.trim() && /^-+\s*$/.test(next)) headings.push({ depth: 2, text: stripInline(line.trim()) }); + } + return headings; +} + +function stripInline(text) { + return text + .replace(/!?\[([^\]]*)\]\([^)]*\)/g, '$1') + .replace(/`([^`]*)`/g, '$1') + .replace(/[*_]{1,3}([^*_]+)[*_]{1,3}/g, '$1'); +} +JS + +cat > src/slug.js <<'JS' +export function createSlugger() { + const seen = new Map(); + return (text) => { + const base = text + .toLowerCase() + .normalize('NFKD') + .replace(/[̀-ͯ]/g, '') + .replace(/[^\p{L}\p{N}\s-]/gu, '') + .trim() + .replace(/\s+/g, '-'); + const count = seen.get(base) ?? 0; + seen.set(base, count + 1); + return count === 0 ? base : `${base}-${count}`; + }; +} +JS + +cat > src/tree.js <<'JS' +export function nest(headings) { + const root = { depth: 0, children: [] }; + const stack = [root]; + for (const heading of headings) { + const node = { ...heading, children: [] }; + while (stack.length > 1 && stack[stack.length - 1].depth >= heading.depth) stack.pop(); + stack[stack.length - 1].children.push(node); + stack.push(node); + } + return root.children; +} + +export function render(nodes, bullet, indent = '') { + return nodes + .map((n) => [`${indent}${bullet} [${n.text}](#${n.slug})`, render(n.children, bullet, `${indent} `)] + .filter(Boolean) + .join('\n')) + .join('\n'); +} +JS + +cat > test/toc.test.js <<'JS' +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { buildToc } from '../src/index.js'; + +test('nests by heading depth', () => { + assert.equal(buildToc('# A\n## B\n## C\n# D'), '- [A](#a)\n - [B](#b)\n - [C](#c)\n- [D](#d)'); +}); + +test('repeated headings get numbered slugs', () => { + assert.equal(buildToc('## Usage\n## Usage'), '- [Usage](#usage)\n- [Usage](#usage-1)'); +}); + +test('headings inside code fences are ignored', () => { + assert.equal(buildToc('# Real\n```\n# not a heading\n```'), '- [Real](#real)'); +}); + +test('setext headings count', () => { + assert.equal(buildToc('Title\n=====\nPart\n----'), '- [Title](#title)\n - [Part](#part)'); +}); + +test('maxDepth drops deeper headings', () => { + assert.equal(buildToc('# A\n## B\n### C', { maxDepth: 2 }), '- [A](#a)\n - [B](#b)'); +}); + +test('inline code and links are stripped from the text', () => { + assert.equal(buildToc('## Use `run()` with [docs](x.md)'), '- [Use run() with docs](#use-run-with-docs)'); +}); +JS + +cat > README.md <<'MD' +# md-toc + +`buildToc(markdown, { minDepth, maxDepth, bullet })` returns a Markdown table of contents. +Run the tests with `npm test`. +MD + +git add . +git commit -q -m "feat: markdown table of contents" From 41028112228fbf5099a6423fc89ce23a8d6c4b45 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 4 Oct 2026 13:06:04 -0400 Subject: [PATCH 2/2] test(architecture): drop slash-command skill-fired grader, widen deep regex A paid row 19 run was INVALID because the slash-command prompts load the skill without a Skill tool call, so the tool_used: Skill grader failed every with-plugin run. The slash command guarantees the load, so the grader is removed from both no-findings cases. The says-already-deep regex missed valid with-plugin wording on 2 of 3 runs; it now matches "already the deep module", "already what the deepening lens aims for" and "would push ... out", with those replies added as pass samples. Co-authored-by: Claude Opus 5.5 --- plugins/architecture/CHANGELOG.md | 7 +++- .../graders/says-already-deep.md | 2 +- .../samples/says-already-deep.json | 12 +++++++ .../graders/skill-fired.md | 7 ---- .../samples/skill-fired.json | 33 ------------------- .../graders/skill-fired.md | 7 ---- .../samples/skill-fired.json | 33 ------------------- 7 files changed, 19 insertions(+), 82 deletions(-) delete mode 100644 plugins/architecture/evals/no-findings-deep-tax-package/graders/skill-fired.md delete mode 100644 plugins/architecture/evals/no-findings-deep-tax-package/samples/skill-fired.json delete mode 100644 plugins/architecture/evals/no-findings-deep-toc-library/graders/skill-fired.md delete mode 100644 plugins/architecture/evals/no-findings-deep-toc-library/samples/skill-fired.json diff --git a/plugins/architecture/CHANGELOG.md b/plugins/architecture/CHANGELOG.md index 8243023071..f5d9e9f424 100644 --- a/plugins/architecture/CHANGELOG.md +++ b/plugins/architecture/CHANGELOG.md @@ -11,7 +11,12 @@ All notable changes to the `architecture` plugin are documented here. Format fol `evals/`, tagged `row19`: the `deep-module-not-over-flagged` case converted from `evals.json`, two scaffolded repositories whose modules are already deep (the reply must say there is no candidate and show no Strong or Worth exploring card), and a control repository with a shallow pass-through - chain that the reply must name. No skill text changed. + chain that the reply must name. No skill text changed. The cases carry no `tool_used: Skill` + grader: every prompt starts with `/architecture:improve`, which loads the skill without a Skill + tool call, so that grader failed every with-plugin run; the slash command guarantees the load, + and the cases measure the verdict, not triggering. The `says-already-deep` regex also matches + the with-plugin wording a paid run produced ("already the deep module", "would push complexity + outward"). ## [0.22.1] - 2026-10-04 diff --git a/plugins/architecture/evals/deep-module-not-over-flagged/graders/says-already-deep.md b/plugins/architecture/evals/deep-module-not-over-flagged/graders/says-already-deep.md index a6fa1c5914..12e51c8e9e 100644 --- a/plugins/architecture/evals/deep-module-not-over-flagged/graders/says-already-deep.md +++ b/plugins/architecture/evals/deep-module-not-over-flagged/graders/says-already-deep.md @@ -1,5 +1,5 @@ --- type: regex -pattern: 'already\s+(a\s+|quite\s+|very\s+)?deep|(just|only|merely|simply)\s+(move|moves|moving|relocate|relocates|shift|shifts)\b' +pattern: 'already\s+(a\s+|the\s+|quite\s+|very\s+)?deep|already\s+what\s+the\s+deepening|(just|only|merely|simply)\s+(move|moves|moving|relocate|relocates|shift|shifts)\b|would\s+push\s+(\S+\s+){0,3}out(ward)?\b' flags: i --- diff --git a/plugins/architecture/evals/deep-module-not-over-flagged/samples/says-already-deep.json b/plugins/architecture/evals/deep-module-not-over-flagged/samples/says-already-deep.json index 0289a9bdf1..63764a8b4f 100644 --- a/plugins/architecture/evals/deep-module-not-over-flagged/samples/says-already-deep.json +++ b/plugins/architecture/evals/deep-module-not-over-flagged/samples/says-already-deep.json @@ -3,6 +3,14 @@ { "answer": "Price(order) is already deep: one entry point over roughly 600 lines of rules. The deletion test says removing it would just move the rules into every caller, so it is not a candidate.", "why": "names it deep and the deletion-test result" + }, + { + "answer": "**Going only on your description, the pricing engine has no deepening candidate. It's already the deep module this check looks for.**\n\n- **Deleting it would push complexity outward:** without `Price`, every caller would have to know rule order, tax lookup and rounding.", + "why": "with-plugin wording from a real run: 'already the deep module' and 'push complexity outward'" + }, + { + "answer": "**What your description suggests (unverified):** as described, this engine is already what the deepening lens aims for, not a candidate for it.\n- **Deletion test:** deleting the module would push pricing logic out into every caller.", + "why": "with-plugin wording from a real run: 'already what the deepening lens aims for' and 'push pricing logic out'" } ], "fail": [ @@ -13,6 +21,10 @@ { "answer": "Candidate: split Price(order) into TaxPolicy, RoundingPolicy and RuleSet modules so each can be tested on its own.", "why": "proposes deepening without judging depth" + }, + { + "answer": "Candidate (Worth exploring): the pricing engine is not deep enough yet. Pull TaxTable out behind its own interface so callers can push tax lookups through it.", + "why": "proposes deepening; mentions 'deep' and 'push' without judging it already deep" } ] } diff --git a/plugins/architecture/evals/no-findings-deep-tax-package/graders/skill-fired.md b/plugins/architecture/evals/no-findings-deep-tax-package/graders/skill-fired.md deleted file mode 100644 index 133b70a6a5..0000000000 --- a/plugins/architecture/evals/no-findings-deep-tax-package/graders/skill-fired.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: tool_used -tool: Skill -input_match: 'improve' -min: 1 -arm: with-only ---- diff --git a/plugins/architecture/evals/no-findings-deep-tax-package/samples/skill-fired.json b/plugins/architecture/evals/no-findings-deep-tax-package/samples/skill-fired.json deleted file mode 100644 index 22e1e83ab7..0000000000 --- a/plugins/architecture/evals/no-findings-deep-tax-package/samples/skill-fired.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "pass": [ - { - "answer": [ - { - "tool": "Skill", - "input": { - "skill": "architecture:improve", - "args": "deepening" - } - } - ], - "why": "the improve skill was invoked" - } - ], - "fail": [ - { - "answer": [], - "why": "no calls" - }, - { - "answer": [ - { - "tool": "Grep", - "input": { - "pattern": "def " - } - } - ], - "why": "scanned by hand, skill never invoked" - } - ] -} diff --git a/plugins/architecture/evals/no-findings-deep-toc-library/graders/skill-fired.md b/plugins/architecture/evals/no-findings-deep-toc-library/graders/skill-fired.md deleted file mode 100644 index 133b70a6a5..0000000000 --- a/plugins/architecture/evals/no-findings-deep-toc-library/graders/skill-fired.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: tool_used -tool: Skill -input_match: 'improve' -min: 1 -arm: with-only ---- diff --git a/plugins/architecture/evals/no-findings-deep-toc-library/samples/skill-fired.json b/plugins/architecture/evals/no-findings-deep-toc-library/samples/skill-fired.json deleted file mode 100644 index 9dbb457ccd..0000000000 --- a/plugins/architecture/evals/no-findings-deep-toc-library/samples/skill-fired.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "pass": [ - { - "answer": [ - { - "tool": "Skill", - "input": { - "skill": "architecture:improve", - "args": "find deepening opportunities in this codebase." - } - } - ], - "why": "the improve skill was invoked" - } - ], - "fail": [ - { - "answer": [], - "why": "no calls" - }, - { - "answer": [ - { - "tool": "Read", - "input": { - "file_path": "/tmp/ws/src/index.js" - } - } - ], - "why": "read files by hand, skill never invoked" - } - ] -}