test(evals): add director-graded cases on the dir-shop fixture

Author: TheGreatAxios

evals/capability/cases/dir-counsel/case.json

@@ -0,0 +1,8 @@
+{
+  "id": "dir-counsel",
+  "tier": "easy",
+  "title": "Counsel plans a coupon feature",
+  "fixture": "fixtures/dir-shop",
+  "prompt": "Plan adding coupon codes (percentage off, single use per cart) to this cart. Give an ordered engineering plan. Do not change source. Save your final plan (the same text you return) to PLAN.md.",
+  "verify": "verify.sh"
+}

evals/capability/cases/dir-counsel/verify.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+source "$(dirname "${BASH_SOURCE[0]}")/../verify-common.sh"
+dirty=$(git status --porcelain | grep -v -E "(REPORT|PLAN)\.md$|\.corbits/" || true)
+[[ -z "$dirty" ]] || { echo "FAIL: source files changed: $dirty"; exit 1; }
+[[ -f PLAN.md ]] || { echo 'FAIL: no PLAN.md'; exit 1; }
+grep -q 'src/cart.ts' PLAN.md && grep -q 'tests/' PLAN.md || { echo 'FAIL: plan missing files'; exit 1; }
+n=$(grep -c -E '^\s*([0-9]+[.)]|-|\*)\s' PLAN.md || true)
+[[ "$n" -ge 4 ]] || { echo 'FAIL: fewer than 4 steps'; exit 1; }
+echo 'PASS: counsel'

evals/capability/cases/dir-critic/case.json

@@ -0,0 +1,8 @@
+{
+  "id": "dir-critic",
+  "tier": "easy",
+  "title": "Critic finds planted defects",
+  "fixture": "fixtures/dir-shop",
+  "prompt": "Review src/cart.ts for defects. Do not fix anything. Save your final report (the same text you return) to REPORT.md.",
+  "verify": "verify.sh"
+}

evals/capability/cases/dir-critic/verify.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+source "$(dirname "${BASH_SOURCE[0]}")/../verify-common.sh"
+dirty=$(git status --porcelain | grep -v -E "(REPORT|PLAN)\.md$|\.corbits/" || true)
+[[ -z "$dirty" ]] || { echo "FAIL: source files changed: $dirty"; exit 1; }
+[[ -f REPORT.md ]] || { echo 'FAIL: no REPORT.md'; exit 1; }
+grep -qi -E 'twice|double|both.*(lineTotal|cartTotal)|applied again|compound' REPORT.md || { echo 'FAIL: missed double discount'; exit 1; }
+grep -qi -E 'splice|skip|mutat|index' REPORT.md || { echo 'FAIL: missed removeSku splice bug'; exit 1; }
+echo 'PASS: critic found both'

evals/capability/cases/dir-explorer/case.json

@@ -0,0 +1,8 @@
+{
+  "id": "dir-explorer",
+  "tier": "easy",
+  "title": "Explorer locates discount logic",
+  "fixture": "fixtures/dir-shop",
+  "prompt": "Where is the discount applied to a cart, and which functions are involved? Do not change any source. Save your final report (the same text you return) to REPORT.md.",
+  "verify": "verify.sh"
+}

evals/capability/cases/dir-explorer/verify.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+source "$(dirname "${BASH_SOURCE[0]}")/../verify-common.sh"
+dirty=$(git status --porcelain | grep -v -E "(REPORT|PLAN)\.md$|\.corbits/" || true)
+[[ -z "$dirty" ]] || { echo "FAIL: source files changed: $dirty"; exit 1; }
+[[ -f REPORT.md ]] || { echo 'FAIL: no REPORT.md'; exit 1; }
+grep -q 'src/cart.ts' REPORT.md && grep -q 'lineTotal' REPORT.md && grep -q 'cartTotal' REPORT.md || { echo 'FAIL: report missing path or functions'; exit 1; }
+echo 'PASS: explorer'

evals/capability/cases/dir-tester/case.json

@@ -0,0 +1,8 @@
+{
+  "id": "dir-tester",
+  "tier": "easy",
+  "title": "Tester reports failing test",
+  "fixture": "fixtures/dir-shop",
+  "prompt": "Run the test suite and report exactly which test fails and the likely cause. Do not fix anything. Save your final report (the same text you return) to REPORT.md.",
+  "verify": "verify.sh"
+}

evals/capability/cases/dir-tester/verify.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+source "$(dirname "${BASH_SOURCE[0]}")/../verify-common.sh"
+dirty=$(git status --porcelain | grep -v -E "(REPORT|PLAN)\.md$|\.corbits/" || true)
+[[ -z "$dirty" ]] || { echo "FAIL: source files changed: $dirty"; exit 1; }
+[[ -f REPORT.md ]] || { echo 'FAIL: no REPORT.md'; exit 1; }
+grep -qi 'removes all matching skus' REPORT.md || { echo 'FAIL: failing test not named'; exit 1; }
+grep -qi -E 'splice|skip|mutat|index' REPORT.md || { echo 'FAIL: cause not identified'; exit 1; }
+grep -qi -E 'bun test|exit' REPORT.md || { echo 'FAIL: no command evidence'; exit 1; }
+echo 'PASS: tester'

fixtures/dir-shop/package.json

@@ -0,0 +1,5 @@
+{
+  "name": "dir-shop-fixture",
+  "private": true,
+  "type": "module"
+}

fixtures/dir-shop/src/cart.ts

@@ -0,0 +1,25 @@
+export interface Item {
+  sku: string;
+  price: number;
+  qty: number;
+}
+
+export function lineTotal(item: Item, discountPct: number): number {
+  const discounted = item.price * (1 - discountPct / 100);
+  return discounted * item.qty;
+}
+
+export function cartTotal(items: Item[], discountPct = 0): number {
+  const sum = items.reduce(
+    (acc, item) => acc + lineTotal(item, discountPct),
+    0,
+  );
+  return sum * (1 - discountPct / 100);
+}
+
+export function removeSku(items: Item[], sku: string): Item[] {
+  for (let i = 0; i < items.length; i++) {
+    if (items[i]?.sku === sku) items.splice(i, 1);
+  }
+  return items;
+}