From 30122a9ac16420310eeb548d41f1cf76ccafc8b8 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 4 Oct 2026 12:50:05 -0400 Subject: [PATCH 01/19] test(source-control): add a plugin eval suite for resolve-conflicts Seven claude plugin eval cases tagged row22, each scaffolding a git repository stopped in a real conflict (two-stop rebase, six-file merge asking for --ours or abort, marker-free semantic conflict, cherry-pick, merge whose incoming tip never touched the path, revert, and a bare "sort it out" merge). Six convert the skill-creator evals.json cases; the seventh tests the archived upstream page's claim that a good model recovers intent unprompted. Graders are mostly deterministic: resolved file contents, history read before the first edit, tests run before the operation concludes, and the concluding reflog line. They also cover the upstream "It's working if" checks. Three cases add one llm grader each, and every grader has pass and fail samples. The skill body is unchanged. Co-authored-by: Claude Opus 5.5 --- .../source-control/.claude-plugin/plugin.json | 2 +- plugins/source-control/CHANGELOG.md | 6 + .../case.yaml | 4 + .../graders/history-before-edit.md | 5 + .../graders/intent-composed.md | 5 + .../graders/no-abort.md | 8 ++ .../graders/no-markers-client.md | 7 ++ .../graders/skill-fired.md | 5 + .../graders/up-checks-before-conclude.md | 5 + .../graders/up-intents-and-drops-named.md | 9 ++ .../graders/up-operation-complete.md | 5 + .../graders/up-quotes-history.md | 5 + .../prompt.md | 11 ++ .../samples/history-before-edit.json | 111 ++++++++++++++++ .../samples/intent-composed.json | 15 +++ .../samples/no-abort.json | 39 ++++++ .../samples/no-markers-client.json | 19 +++ .../samples/skill-fired.json | 46 +++++++ .../samples/up-checks-before-conclude.json | 83 ++++++++++++ .../samples/up-intents-and-drops-named.json | 15 +++ .../samples/up-operation-complete.json | 15 +++ .../samples/up-quotes-history.json | 18 +++ .../scaffold.sh | 67 ++++++++++ .../case.yaml | 4 + .../graders/both-sides-rules.md | 5 + .../contract-reads-cherry-pick-head.md | 5 + .../graders/history-before-edit.md | 5 + .../graders/no-abort.md | 8 ++ .../graders/no-markers-rules.md | 7 ++ .../graders/no-merge-head-assumed.md | 8 ++ .../graders/skill-fired.md | 5 + .../graders/up-checks-before-conclude.md | 5 + .../graders/up-operation-complete.md | 5 + .../graders/up-quotes-history.md | 5 + .../prompt.md | 11 ++ .../samples/both-sides-rules.json | 15 +++ .../contract-reads-cherry-pick-head.json | 39 ++++++ .../samples/history-before-edit.json | 111 ++++++++++++++++ .../samples/no-abort.json | 39 ++++++ .../samples/no-markers-rules.json | 19 +++ .../samples/no-merge-head-assumed.json | 46 +++++++ .../samples/skill-fired.json | 46 +++++++ .../samples/up-checks-before-conclude.json | 83 ++++++++++++ .../samples/up-operation-complete.json | 15 +++ .../samples/up-quotes-history.json | 15 +++ .../scaffold.sh | 84 +++++++++++++ .../case.yaml | 4 + .../graders/both-sides-config.md | 5 + .../graders/contract-reads-incoming-range.md | 5 + .../graders/history-before-edit.md | 5 + .../graders/no-abort.md | 8 ++ .../graders/no-markers-retry.md | 7 ++ .../graders/skill-fired.md | 5 + .../graders/up-checks-before-conclude.md | 5 + .../graders/up-operation-complete.md | 5 + .../graders/up-quotes-history.md | 5 + .../prompt.md | 11 ++ .../samples/both-sides-config.json | 15 +++ .../contract-reads-incoming-range.json | 59 +++++++++ .../samples/history-before-edit.json | 111 ++++++++++++++++ .../samples/no-abort.json | 39 ++++++ .../samples/no-markers-retry.json | 19 +++ .../samples/skill-fired.json | 46 +++++++ .../samples/up-checks-before-conclude.json | 83 ++++++++++++ .../samples/up-operation-complete.json | 15 +++ .../samples/up-quotes-history.json | 18 +++ .../scaffold.sh | 51 ++++++++ .../case.yaml | 4 + .../graders/no-abort.md | 8 ++ .../graders/no-wildcard-staging.md | 8 ++ .../graders/skill-fired.md | 5 + .../graders/stale-call-fixed.md | 5 + .../graders/sweep-searched-old-name.md | 5 + .../graders/up-checks-before-conclude.md | 5 + .../graders/up-operation-complete.md | 5 + .../graders/up-quotes-history.md | 5 + .../prompt.md | 11 ++ .../samples/no-abort.json | 39 ++++++ .../samples/no-wildcard-staging.json | 47 +++++++ .../samples/skill-fired.json | 46 +++++++ .../samples/stale-call-fixed.json | 15 +++ .../samples/sweep-searched-old-name.json | 19 +++ .../samples/up-checks-before-conclude.json | 83 ++++++++++++ .../samples/up-operation-complete.json | 15 +++ .../samples/up-quotes-history.json | 15 +++ .../scaffold.sh | 88 +++++++++++++ .../case.yaml | 4 + .../graders/both-sides-readme.md | 5 + .../graders/both-sides-retry.md | 5 + .../graders/contract-reads-rebase-head.md | 5 + .../graders/history-before-edit.md | 5 + .../graders/no-abort.md | 8 ++ .../graders/no-markers-readme.md | 7 ++ .../graders/no-markers-retry.md | 7 ++ .../graders/semantic-sweep-test-jitter.md | 6 + .../graders/skill-fired.md | 5 + .../graders/up-checks-before-conclude.md | 5 + .../graders/up-nothing-invented.md | 8 ++ .../graders/up-operation-complete.md | 5 + .../graders/up-quotes-history-feature.md | 5 + .../graders/up-quotes-history-main.md | 5 + .../prompt.md | 11 ++ .../samples/both-sides-readme.json | 15 +++ .../samples/both-sides-retry.json | 18 +++ .../samples/contract-reads-rebase-head.json | 39 ++++++ .../samples/history-before-edit.json | 111 ++++++++++++++++ .../samples/no-abort.json | 39 ++++++ .../samples/no-markers-readme.json | 19 +++ .../samples/no-markers-retry.json | 19 +++ .../samples/semantic-sweep-test-jitter.json | 12 ++ .../samples/skill-fired.json | 46 +++++++ .../samples/up-checks-before-conclude.json | 83 ++++++++++++ .../samples/up-nothing-invented.json | 12 ++ .../samples/up-operation-complete.json | 15 +++ .../samples/up-quotes-history-feature.json | 21 ++++ .../samples/up-quotes-history-main.json | 21 ++++ .../scaffold.sh | 118 ++++++++++++++++++ .../case.yaml | 4 + .../graders/inventory-listed.md | 5 + .../graders/justified-per-file.md | 9 ++ .../graders/no-abort.md | 8 ++ .../graders/no-bulk-side-pick.md | 8 ++ .../graders/skill-fired.md | 5 + .../prompt.md | 11 ++ .../samples/inventory-listed.json | 49 ++++++++ .../samples/justified-per-file.json | 15 +++ .../samples/no-abort.json | 39 ++++++ .../samples/no-bulk-side-pick.json | 59 +++++++++ .../samples/skill-fired.json | 46 +++++++ .../scaffold.sh | 43 +++++++ .../case.yaml | 4 + .../graders/contract-reads-revert-head.md | 5 + .../graders/history-before-edit.md | 5 + .../graders/no-abort.md | 8 ++ .../graders/no-markers-session.md | 7 ++ .../graders/no-wrong-continue.md | 8 ++ .../graders/revert-applied-later-fix-kept.md | 5 + .../graders/skill-fired.md | 5 + .../graders/up-checks-before-conclude.md | 5 + .../graders/up-operation-complete.md | 5 + .../graders/up-quotes-history.md | 5 + .../prompt.md | 11 ++ .../samples/contract-reads-revert-head.json | 39 ++++++ .../samples/history-before-edit.json | 111 ++++++++++++++++ .../samples/no-abort.json | 39 ++++++ .../samples/no-markers-session.json | 19 +++ .../samples/no-wrong-continue.json | 39 ++++++ .../revert-applied-later-fix-kept.json | 15 +++ .../samples/skill-fired.json | 46 +++++++ .../samples/up-checks-before-conclude.json | 83 ++++++++++++ .../samples/up-operation-complete.json | 15 +++ .../samples/up-quotes-history.json | 15 +++ .../scaffold.sh | 59 +++++++++ 153 files changed, 3576 insertions(+), 1 deletion(-) create mode 100644 plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/case.yaml create mode 100644 plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md create mode 100644 plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/intent-composed.md create mode 100644 plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/no-abort.md create mode 100644 plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/no-markers-client.md create mode 100644 plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/skill-fired.md create mode 100644 plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-checks-before-conclude.md create mode 100644 plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-intents-and-drops-named.md create mode 100644 plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-operation-complete.md create mode 100644 plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-quotes-history.md create mode 100644 plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/prompt.md create mode 100644 plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/history-before-edit.json create mode 100644 plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/intent-composed.json create mode 100644 plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/no-abort.json create mode 100644 plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/no-markers-client.json create mode 100644 plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/skill-fired.json create mode 100644 plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-checks-before-conclude.json create mode 100644 plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-intents-and-drops-named.json create mode 100644 plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-operation-complete.json create mode 100644 plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-quotes-history.json create mode 100755 plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/scaffold.sh create mode 100644 plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/case.yaml create mode 100644 plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/both-sides-rules.md create mode 100644 plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/contract-reads-cherry-pick-head.md create mode 100644 plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md create mode 100644 plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/no-abort.md create mode 100644 plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/no-markers-rules.md create mode 100644 plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/no-merge-head-assumed.md create mode 100644 plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/skill-fired.md create mode 100644 plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/up-checks-before-conclude.md create mode 100644 plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/up-operation-complete.md create mode 100644 plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/up-quotes-history.md create mode 100644 plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/prompt.md create mode 100644 plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/both-sides-rules.json create mode 100644 plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/contract-reads-cherry-pick-head.json create mode 100644 plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/history-before-edit.json create mode 100644 plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/no-abort.json create mode 100644 plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/no-markers-rules.json create mode 100644 plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/no-merge-head-assumed.json create mode 100644 plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/skill-fired.json create mode 100644 plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/up-checks-before-conclude.json create mode 100644 plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/up-operation-complete.json create mode 100644 plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/up-quotes-history.json create mode 100755 plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/scaffold.sh create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/case.yaml create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/both-sides-config.md create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/contract-reads-incoming-range.md create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/no-abort.md create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/no-markers-retry.md create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/skill-fired.md create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/up-checks-before-conclude.md create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/up-operation-complete.md create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/up-quotes-history.md create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/prompt.md create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/both-sides-config.json create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/contract-reads-incoming-range.json create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/history-before-edit.json create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/no-abort.json create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/no-markers-retry.json create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/skill-fired.json create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/up-checks-before-conclude.json create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/up-operation-complete.json create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/up-quotes-history.json create mode 100755 plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/scaffold.sh create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/case.yaml create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/no-abort.md create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/no-wildcard-staging.md create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/skill-fired.md create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/stale-call-fixed.md create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/sweep-searched-old-name.md create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/up-checks-before-conclude.md create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/up-operation-complete.md create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/up-quotes-history.md create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/prompt.md create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/no-abort.json create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/no-wildcard-staging.json create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/skill-fired.json create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/stale-call-fixed.json create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/sweep-searched-old-name.json create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/up-checks-before-conclude.json create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/up-operation-complete.json create mode 100644 plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/up-quotes-history.json create mode 100755 plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/scaffold.sh create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/case.yaml create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/both-sides-readme.md create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/both-sides-retry.md create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/contract-reads-rebase-head.md create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/no-abort.md create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/no-markers-readme.md create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/no-markers-retry.md create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/semantic-sweep-test-jitter.md create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/skill-fired.md create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-checks-before-conclude.md create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-nothing-invented.md create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-operation-complete.md create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-feature.md create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-main.md create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/prompt.md create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/both-sides-readme.json create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/both-sides-retry.json create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/contract-reads-rebase-head.json create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/history-before-edit.json create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/no-abort.json create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/no-markers-readme.json create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/no-markers-retry.json create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/semantic-sweep-test-jitter.json create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/skill-fired.json create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-checks-before-conclude.json create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-nothing-invented.json create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-operation-complete.json create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-quotes-history-feature.json create mode 100644 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-quotes-history-main.json create mode 100755 plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/scaffold.sh create mode 100644 plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/case.yaml create mode 100644 plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/inventory-listed.md create mode 100644 plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/justified-per-file.md create mode 100644 plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/no-abort.md create mode 100644 plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/no-bulk-side-pick.md create mode 100644 plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/skill-fired.md create mode 100644 plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/prompt.md create mode 100644 plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/inventory-listed.json create mode 100644 plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/justified-per-file.json create mode 100644 plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/no-abort.json create mode 100644 plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/no-bulk-side-pick.json create mode 100644 plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/skill-fired.json create mode 100755 plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/scaffold.sh create mode 100644 plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/case.yaml create mode 100644 plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/contract-reads-revert-head.md create mode 100644 plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md create mode 100644 plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/no-abort.md create mode 100644 plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/no-markers-session.md create mode 100644 plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/no-wrong-continue.md create mode 100644 plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/revert-applied-later-fix-kept.md create mode 100644 plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/skill-fired.md create mode 100644 plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/up-checks-before-conclude.md create mode 100644 plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/up-operation-complete.md create mode 100644 plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/up-quotes-history.md create mode 100644 plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/prompt.md create mode 100644 plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/contract-reads-revert-head.json create mode 100644 plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/history-before-edit.json create mode 100644 plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/no-abort.json create mode 100644 plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/no-markers-session.json create mode 100644 plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/no-wrong-continue.json create mode 100644 plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/revert-applied-later-fix-kept.json create mode 100644 plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/skill-fired.json create mode 100644 plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/up-checks-before-conclude.json create mode 100644 plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/up-operation-complete.json create mode 100644 plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/up-quotes-history.json create mode 100755 plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/scaffold.sh diff --git a/plugins/source-control/.claude-plugin/plugin.json b/plugins/source-control/.claude-plugin/plugin.json index 8ec7b911bd..9d77beb79b 100644 --- a/plugins/source-control/.claude-plugin/plugin.json +++ b/plugins/source-control/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "source-control", - "version": "0.79.11", + "version": "0.79.12", "description": "Git and GitHub delivery: /commit (convention-checked subject, Co-authored-by trailer, surgical staging), /pull-request (prep, create, CI monitoring, review triage, merge, CI logs), /babysit-prs (safe-by-default PR fleet loop, opt-in worker and autopilot tiers), /babysit-loop (merge lane; merge is human until the repo adopts it), /worktree, /resolve-conflicts (intent-first), /check, and /setup (layered source-control.md convention config; Conventional Commits by default).", "author": { "name": "Melodic Software", diff --git a/plugins/source-control/CHANGELOG.md b/plugins/source-control/CHANGELOG.md index eee67f681e..9934eb7c2f 100644 --- a/plugins/source-control/CHANGELOG.md +++ b/plugins/source-control/CHANGELOG.md @@ -3,6 +3,12 @@ All notable changes to the `source-control` plugin are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); this plugin uses semantic versioning. +## [0.79.12] - 2026-10-04 + +### Added + +- **A `claude plugin eval` suite for `/source-control:resolve-conflicts`.** Seven cases tagged `row22`, each scaffolding a git repository that is stopped in a real conflict: a two-stop rebase, a six-file merge where the prompt asks for `--ours` or abort, a merge with a conflict that has no markers, a cherry-pick, a merge whose incoming tip never touched the file, a revert, and a merge with only "sort it out" as the prompt. Most graders are deterministic: the resolved file contents, history read before the first edit, tests run before the operation concludes, and the concluding reflog line. They also cover the "It's working if" checks from the archived upstream skill page. Three cases add one model-judged grader each, and every grader carries pass and fail samples. The skill body is unchanged. + ## [0.79.11] - 2026-10-04 ### Fixed diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/case.yaml b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/case.yaml new file mode 100644 index 0000000000..a62410c1e8 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/case.yaml @@ -0,0 +1,4 @@ +schema_version: "1.1" +name: resolve-conflicts-bare-prompt-recovers-intent-from-history +context: + scaffold_script: scaffold.sh diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md new file mode 100644 index 0000000000..1c3d5cd789 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md @@ -0,0 +1,5 @@ +--- +type: tool_order +before: { tool: Bash, input_match: 'git\s+(?:-C\s+\S+\s+)?(?:log|show)\b' } +after: Edit +--- diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/intent-composed.md b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/intent-composed.md new file mode 100644 index 0000000000..1b22cb77f7 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/intent-composed.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: '^(?![\s\S]*legacy_token)(?=[\s\S]*region)' +target: { source: file, path: api/client.py } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/no-abort.md b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/no-abort.md new file mode 100644 index 0000000000..866437dbca --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/no-abort.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Bash +input_match: '--abort\b' +min: 0 +max: 0 +arm: both +--- diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/no-markers-client.md b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/no-markers-client.md new file mode 100644 index 0000000000..549de77978 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/no-markers-client.md @@ -0,0 +1,7 @@ +--- +type: regex +pattern: '^(?:<{7}|={7}|>{7})' +flags: m +match: not_contains +target: { source: file, path: api/client.py } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/skill-fired.md b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/skill-fired.md new file mode 100644 index 0000000000..f329958d6d --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/skill-fired.md @@ -0,0 +1,5 @@ +--- +type: tool_used +tool: Skill +input_match: '"skill"\s*:\s*"(?:source-control:)?resolve-conflicts"' +--- diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-checks-before-conclude.md b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-checks-before-conclude.md new file mode 100644 index 0000000000..ca5d2c546e --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-checks-before-conclude.md @@ -0,0 +1,5 @@ +--- +type: tool_order +before: { tool: Bash, input_match: 'unittest|pytest|python3?\s+\S*test_\w+\.py' } +after: { tool: Bash, input_match: 'merge\s+--continue|git\s+commit\b' } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-intents-and-drops-named.md b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-intents-and-drops-named.md new file mode 100644 index 0000000000..b903d7fc55 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-intents-and-drops-named.md @@ -0,0 +1,9 @@ +--- +type: llm +--- + +Main's commit removed `legacy_token` from `fetch_orders` because the /orders endpoint rejects token query auth with a 400 (SEC-12). The feature commit added a `region` filter for the EU dashboard (ORD-5). + +PASS if the reply states each side's intent as the commit history gives it (the removal and its reason; the region filter and its purpose) and says that `legacy_token` is gone from the result because main removed it on purpose. + +FAIL if the reply describes the resolution only in terms of the diff or the conflict markers, keeps `legacy_token` alongside `region`, or drops `region`. diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-operation-complete.md b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-operation-complete.md new file mode 100644 index 0000000000..b90f0b6544 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-operation-complete.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: 'commit \(merge\)' +target: { source: file, path: .git/logs/HEAD } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-quotes-history.md b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-quotes-history.md new file mode 100644 index 0000000000..77afa7981e --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-quotes-history.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: 'SEC-12|\b400\b|stopped accepting|token query auth' +flags: i +--- diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/prompt.md b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/prompt.md new file mode 100644 index 0000000000..fba31a80a7 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/prompt.md @@ -0,0 +1,11 @@ +--- +description: "Bare 'sort it out' merge prompt: only the commit message says legacy_token was removed on purpose; keep region, drop legacy_token, say why, run tests, conclude" +tags: [resolve-conflicts, source-control, row22, upstream-working-if] +runs: 3 +max_turns: 40 +timeout_seconds: 900 +allowed_tools: [Read, Glob, Grep, Skill, Bash, Edit] +expected_outcome: "api/client.py keeps region and has no legacy_token, no markers, history was read before the edit, the reply cites SEC-12 or the 400, tests ran before the merge was concluded, and the merge commit exists" +--- + +git merge main stopped with conflicts. Sort it out and finish the merge. diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/history-before-edit.json new file mode 100644 index 0000000000..9fc33aec85 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/history-before-edit.json @@ -0,0 +1,111 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log --oneline -5" + } + }, + { + "tool": "Bash", + "input": { + "command": "git show abc123" + } + }, + { + "tool": "Edit", + "input": { + "file_path": "/w/a.py", + "old_string": "x", + "new_string": "y" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Bash", + "input": { + "command": "git log -p main..HEAD -- a.py" + } + }, + { + "tool": "Edit", + "input": { + "file_path": "/w/a.py", + "old_string": "x", + "new_string": "y" + } + }, + { + "tool": "Edit", + "input": { + "file_path": "/w/b.py", + "old_string": "x", + "new_string": "y" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Edit", + "input": { + "file_path": "/w/a.py", + "old_string": "x", + "new_string": "y" + } + }, + { + "tool": "Bash", + "input": { + "command": "git log --oneline" + } + } + ], + "why": "edited before reading history" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git diff" + } + }, + { + "tool": "Edit", + "input": { + "file_path": "/w/a.py", + "old_string": "x", + "new_string": "y" + } + } + ], + "why": "diff only, no history" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log --oneline" + } + } + ], + "why": "never edited" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/intent-composed.json b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/intent-composed.json new file mode 100644 index 0000000000..de064d5122 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/intent-composed.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "def fetch_orders(session, customer_id, region=None):\n params = {\"customer\": customer_id}\n if region:\n params[\"region\"] = region\n return session.get(\"/orders\", params=params)\n" + } + ], + "fail": [ + { + "answer": "def fetch_orders(session, customer_id, legacy_token=None, region=None):\n params = {\"customer\": customer_id}\n if region:\n params[\"region\"] = region\n if legacy_token:\n params[\"token\"] = legacy_token\n" + }, + { + "answer": "def fetch_orders(session, customer_id):\n params = {\"customer\": customer_id}\n" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/no-abort.json b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/no-abort.json new file mode 100644 index 0000000000..84c7c59462 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/no-abort.json @@ -0,0 +1,39 @@ +{ + "pass": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status | head -4" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --abort" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git rebase --abort && git status" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/no-markers-client.json b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/no-markers-client.json new file mode 100644 index 0000000000..3c007648ed --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/no-markers-client.json @@ -0,0 +1,19 @@ +{ + "pass": [ + { + "answer": "x = 1\ny = 2\n" + }, + { + "answer": "# a ======= rule in a comment is fine mid-line ======= \n" + } + ], + "fail": [ + { + "answer": "<<<<<<< HEAD\nx = 1\n=======\nx = 2\n>>>>>>> main\n" + }, + { + "answer": "x = 1\n=======\n", + "why": "a leftover separator" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/skill-fired.json b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/skill-fired.json new file mode 100644 index 0000000000..012ab78155 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/skill-fired.json @@ -0,0 +1,46 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:resolve-conflicts" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Skill", + "input": { + "skill": "resolve-conflicts" + } + } + ] + } + ], + "fail": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:commit" + } + } + ], + "why": "wrong skill" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-checks-before-conclude.json b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-checks-before-conclude.json new file mode 100644 index 0000000000..34fc8ae002 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-checks-before-conclude.json @@ -0,0 +1,83 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + }, + { + "tool": "Bash", + "input": { + "command": "git merge --continue" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + }, + { + "tool": "Bash", + "input": { + "command": "git commit --no-edit" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --continue" + } + }, + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + } + ], + "why": "checks after the commit" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --continue" + } + } + ], + "why": "no checks" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + } + ], + "why": "never concluded" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-intents-and-drops-named.json b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-intents-and-drops-named.json new file mode 100644 index 0000000000..5e7e7f6056 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-intents-and-drops-named.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "Main removed legacy_token on purpose: /orders returns 400 for token query auth since 2026-03-01 (SEC-12). Your branch added the region filter for the EU dashboard (ORD-5). Resolution keeps region and leaves legacy_token out. Tests pass; merge concluded." + } + ], + "fail": [ + { + "answer": "Resolved the conflict by keeping both parameters, legacy_token and region, and finished the merge." + }, + { + "answer": "Took main's version of api/client.py." + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-operation-complete.json b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-operation-complete.json new file mode 100644 index 0000000000..677c0b5a86 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-operation-complete.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "0000 1111 eval 1700000000 +0000\tcheckout: moving from main to feature\n1111 2222 eval 1700000001 +0000\tcommit (merge): Merge branch main into feature\n" + } + ], + "fail": [ + { + "answer": "0000 1111 eval 1700000000 +0000\tcommit: feat(api): filter orders by region\n" + }, + { + "answer": "" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-quotes-history.json b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-quotes-history.json new file mode 100644 index 0000000000..56ff1887c9 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-quotes-history.json @@ -0,0 +1,18 @@ +{ + "pass": [ + { + "answer": "main removed legacy_token because /orders now returns 400 for token query auth (SEC-12)." + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "Kept both sides and finished the merge." + }, + { + "answer": "Merged; region filter added." + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/scaffold.sh b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/scaffold.sh new file mode 100755 index 0000000000..8d5dca5f29 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/scaffold.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# git merge main stopped on api/client.py. main removed legacy_token on purpose (the endpoint +# rejects it); feature added a region filter in the same function. Only the commit message says why. +set -euo pipefail + +git init -q -b main +git config user.name eval +git config user.email eval@example.invalid +git config rerere.enabled false + +mkdir -p api +: > api/__init__.py +cat > api/client.py <<'PY' +def fetch_orders(session, customer_id, legacy_token=None): + params = {"customer": customer_id} + if legacy_token: + params["token"] = legacy_token + return session.get("/orders", params=params) +PY +cat > test_client.py <<'PY' +import unittest + +from api.client import fetch_orders + + +class FakeSession: + def get(self, path, params): + return path, params + + +class ClientTest(unittest.TestCase): + def test_sends_customer(self): + self.assertEqual(fetch_orders(FakeSession(), 7)[1]["customer"], 7) + + +if __name__ == "__main__": + unittest.main() +PY +git add . +git commit -q -m "feat(api): orders client" + +git checkout -q -b feature +cat > api/client.py <<'PY' +def fetch_orders(session, customer_id, legacy_token=None, region=None): + params = {"customer": customer_id} + if region: + params["region"] = region + if legacy_token: + params["token"] = legacy_token + return session.get("/orders", params=params) +PY +git add api/client.py +git commit -q -m "feat(api): filter orders by region" -m "The EU dashboard shows only its own region's orders. Refs ORD-5." + +git checkout -q main +cat > api/client.py <<'PY' +def fetch_orders(session, customer_id): + params = {"customer": customer_id} + return session.get("/orders", params=params) +PY +git add api/client.py +git commit -q -m "refactor(api): drop legacy_token from fetch_orders" -m "The /orders endpoint stopped accepting token query auth on 2026-03-01; sending it now returns 400. Refs SEC-12." + +git checkout -q feature +git merge main -q -m "Merge branch main into feature" >/dev/null 2>&1 || true +test -f .git/MERGE_HEAD +git diff --name-only --diff-filter=U | grep -qx api/client.py diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/case.yaml b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/case.yaml new file mode 100644 index 0000000000..232dd27c4a --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/case.yaml @@ -0,0 +1,4 @@ +schema_version: "1.1" +name: resolve-conflicts-cherry-pick-reads-pick-head +context: + scaffold_script: scaffold.sh diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/both-sides-rules.md b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/both-sides-rules.md new file mode 100644 index 0000000000..831c7ccc6e --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/both-sides-rules.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: '^(?=[\s\S]*LATE_RATE\s*=\s*0\.02)(?=[\s\S]*MAX_LATE_FEE\s*=\s*50)(?=[\s\S]*min\(\s*round\(\s*balance\s*\*\s*LATE_RATE)' +target: { source: file, path: billing/rules.py } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/contract-reads-cherry-pick-head.md b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/contract-reads-cherry-pick-head.md new file mode 100644 index 0000000000..8bf4b09800 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/contract-reads-cherry-pick-head.md @@ -0,0 +1,5 @@ +--- +type: tool_used +tool: Bash +input_match: 'CHERRY_PICK_HEAD' +--- diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md new file mode 100644 index 0000000000..1c3d5cd789 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md @@ -0,0 +1,5 @@ +--- +type: tool_order +before: { tool: Bash, input_match: 'git\s+(?:-C\s+\S+\s+)?(?:log|show)\b' } +after: Edit +--- diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/no-abort.md b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/no-abort.md new file mode 100644 index 0000000000..866437dbca --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/no-abort.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Bash +input_match: '--abort\b' +min: 0 +max: 0 +arm: both +--- diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/no-markers-rules.md b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/no-markers-rules.md new file mode 100644 index 0000000000..1e606ebc08 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/no-markers-rules.md @@ -0,0 +1,7 @@ +--- +type: regex +pattern: '^(?:<{7}|={7}|>{7})' +flags: m +match: not_contains +target: { source: file, path: billing/rules.py } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/no-merge-head-assumed.md b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/no-merge-head-assumed.md new file mode 100644 index 0000000000..32507ec947 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/no-merge-head-assumed.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Bash +input_match: '(?>>>>>> main\n" + }, + { + "answer": "x = 1\n=======\n", + "why": "a leftover separator" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/no-merge-head-assumed.json b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/no-merge-head-assumed.json new file mode 100644 index 0000000000..728d1a6056 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/no-merge-head-assumed.json @@ -0,0 +1,46 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git show CHERRY_PICK_HEAD" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log --oneline HEAD...CHERRY_PICK_HEAD" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git show MERGE_HEAD -- billing/rules.py" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log HEAD..MERGE_HEAD" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/skill-fired.json b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/skill-fired.json new file mode 100644 index 0000000000..012ab78155 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/skill-fired.json @@ -0,0 +1,46 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:resolve-conflicts" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Skill", + "input": { + "skill": "resolve-conflicts" + } + } + ] + } + ], + "fail": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:commit" + } + } + ], + "why": "wrong skill" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/up-checks-before-conclude.json b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/up-checks-before-conclude.json new file mode 100644 index 0000000000..4b278b5d4f --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/up-checks-before-conclude.json @@ -0,0 +1,83 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + }, + { + "tool": "Bash", + "input": { + "command": "git cherry-pick --continue" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + }, + { + "tool": "Bash", + "input": { + "command": "GIT_EDITOR=true git cherry-pick --continue" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git cherry-pick --continue" + } + }, + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + } + ], + "why": "checks after the commit" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git cherry-pick --continue" + } + } + ], + "why": "no checks" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + } + ], + "why": "never concluded" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/up-operation-complete.json b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/up-operation-complete.json new file mode 100644 index 0000000000..5368d88040 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/up-operation-complete.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "0000 1111 eval 1700000000 +0000\tcheckout: moving from main to feature\n1111 2222 eval 1700000001 +0000\tcommit (cherry-pick): fix(billing): cap the late fee at 50\n" + } + ], + "fail": [ + { + "answer": "0000 1111 eval 1700000000 +0000\tcommit: refactor(billing): name the late-fee rate\n" + }, + { + "answer": "" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/up-quotes-history.json b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/up-quotes-history.json new file mode 100644 index 0000000000..5f2b16f493 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/up-quotes-history.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "The picked commit caps the fee at 50 because regulators require it (BILL-203)." + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "Resolved billing/rules.py and continued the cherry-pick." + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/scaffold.sh b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/scaffold.sh new file mode 100755 index 0000000000..75ffed8dd3 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/scaffold.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# release/1.4 stopped cherry-picking main's late-fee cap: release named the rate LATE_RATE, +# the picked commit added MAX_LATE_FEE and a min() around the same return line. +set -euo pipefail + +git init -q -b main +git config user.name eval +git config user.email eval@example.invalid +git config rerere.enabled false + +mkdir -p billing +: > billing/__init__.py +cat > billing/rules.py <<'PY' +def late_fee(balance, days_late): + if days_late <= 0: + return 0 + return round(balance * 0.02, 2) +PY +cat > test_rules.py <<'PY' +import unittest + +from billing.rules import late_fee + + +class LateFeeTest(unittest.TestCase): + def test_two_percent(self): + self.assertEqual(late_fee(100, 10), 2.0) + + +if __name__ == "__main__": + unittest.main() +PY +git add . +git commit -q -m "feat(billing): late fees" + +git checkout -q -b release/1.4 +cat > billing/rules.py <<'PY' +LATE_RATE = 0.02 + + +def late_fee(balance, days_late): + if days_late <= 0: + return 0 + return round(balance * LATE_RATE, 2) +PY +git add billing/rules.py +git commit -q -m "refactor(billing): name the late-fee rate" -m "Finance changes the rate each quarter; one named constant is the only place to edit. Refs BILL-190." + +git checkout -q main +cat > billing/rules.py <<'PY' +MAX_LATE_FEE = 50 + + +def late_fee(balance, days_late): + if days_late <= 0: + return 0 + return min(round(balance * 0.02, 2), MAX_LATE_FEE) +PY +cat > test_cap.py <<'PY' +import unittest + +from billing.rules import late_fee + + +class CapTest(unittest.TestCase): + def test_fee_is_capped(self): + self.assertEqual(late_fee(10000, 10), 50) + + +if __name__ == "__main__": + unittest.main() +PY +git add billing/rules.py test_cap.py +git commit -q -m "fix(billing): cap the late fee at 50" -m "Regulators cap late fees at 50 per invoice; uncapped fees on large balances were being refunded by hand. Refs BILL-203." +CAP=$(git rev-parse HEAD) +printf '# billing\n' > README.md +git add README.md +git commit -q -m "docs: billing readme" + +git checkout -q release/1.4 +git cherry-pick "$CAP" >/dev/null 2>&1 || true +test -f .git/CHERRY_PICK_HEAD +test ! -f .git/MERGE_HEAD +git diff --name-only --diff-filter=U | grep -qx billing/rules.py diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/case.yaml b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/case.yaml new file mode 100644 index 0000000000..e00c8f8656 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/case.yaml @@ -0,0 +1,4 @@ +schema_version: "1.1" +name: resolve-conflicts-merge-reads-full-incoming-range +context: + scaffold_script: scaffold.sh diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/both-sides-config.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/both-sides-config.md new file mode 100644 index 0000000000..285d3f1a25 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/both-sides-config.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: '^(?=[\s\S]*max_attempts:\s*5\b)(?=[\s\S]*backoff_seconds:\s*2\b)' +target: { source: file, path: config/retry.yaml } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/contract-reads-incoming-range.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/contract-reads-incoming-range.md new file mode 100644 index 0000000000..ef4e48852b --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/contract-reads-incoming-range.md @@ -0,0 +1,5 @@ +--- +type: tool_used +tool: Bash +input_match: '\.\.\.?MERGE_HEAD|MERGE_HEAD\.\.|\^MERGE_HEAD|--merge\b' +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md new file mode 100644 index 0000000000..1c3d5cd789 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md @@ -0,0 +1,5 @@ +--- +type: tool_order +before: { tool: Bash, input_match: 'git\s+(?:-C\s+\S+\s+)?(?:log|show)\b' } +after: Edit +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/no-abort.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/no-abort.md new file mode 100644 index 0000000000..866437dbca --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/no-abort.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Bash +input_match: '--abort\b' +min: 0 +max: 0 +arm: both +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/no-markers-retry.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/no-markers-retry.md new file mode 100644 index 0000000000..b1ccbece68 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/no-markers-retry.md @@ -0,0 +1,7 @@ +--- +type: regex +pattern: '^(?:<{7}|={7}|>{7})' +flags: m +match: not_contains +target: { source: file, path: config/retry.yaml } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/skill-fired.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/skill-fired.md new file mode 100644 index 0000000000..f329958d6d --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/skill-fired.md @@ -0,0 +1,5 @@ +--- +type: tool_used +tool: Skill +input_match: '"skill"\s*:\s*"(?:source-control:)?resolve-conflicts"' +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/up-checks-before-conclude.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/up-checks-before-conclude.md new file mode 100644 index 0000000000..6ca1f35154 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/up-checks-before-conclude.md @@ -0,0 +1,5 @@ +--- +type: tool_order +before: { tool: Bash, input_match: 'check_config|unittest|pytest|make\b' } +after: { tool: Bash, input_match: 'merge\s+--continue|git\s+commit\b' } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/up-operation-complete.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/up-operation-complete.md new file mode 100644 index 0000000000..b90f0b6544 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/up-operation-complete.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: 'commit \(merge\)' +target: { source: file, path: .git/logs/HEAD } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/up-quotes-history.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/up-quotes-history.md new file mode 100644 index 0000000000..54781d7ee1 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/up-quotes-history.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: 'OPS-311|failover|1 in 40' +flags: i +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/prompt.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/prompt.md new file mode 100644 index 0000000000..f0c29e81d2 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/prompt.md @@ -0,0 +1,11 @@ +--- +description: "Merge stop where the incoming tip never touched the path: read merge-base..MERGE_HEAD, state the earlier commit's intent, keep max_attempts 5 and backoff 2" +tags: [resolve-conflicts, source-control, row22, converted] +runs: 3 +max_turns: 40 +timeout_seconds: 900 +allowed_tools: [Read, Glob, Grep, Skill, Bash, Edit] +expected_outcome: "config/retry.yaml has max_attempts 5 and backoff_seconds 2 with no markers, the incoming range was read, the reply cites the failover reason from the first incoming commit, checks ran before the merge was concluded" +--- + +A merge stopped with a conflict in config/retry.yaml. The branch being merged in has three commits; the first one changed config/retry.yaml, but the tip commit only touched an unrelated file. Recover the incoming side's intent, resolve it, and finish the merge. diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/both-sides-config.json b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/both-sides-config.json new file mode 100644 index 0000000000..66c2c77e62 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/both-sides-config.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "retry:\n max_attempts: 5\n backoff_seconds: 2\n" + } + ], + "fail": [ + { + "answer": "retry:\n max_attempts: 3\n backoff_seconds: 2\n" + }, + { + "answer": "retry:\n max_attempts: 5\n backoff_seconds: 1\n" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/contract-reads-incoming-range.json b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/contract-reads-incoming-range.json new file mode 100644 index 0000000000..7d455d50a9 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/contract-reads-incoming-range.json @@ -0,0 +1,59 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log -p \"$BASE\"..MERGE_HEAD -- config/retry.yaml" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log --oneline HEAD...MERGE_HEAD" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log -p --merge -- config/retry.yaml" + } + } + ] + } + ], + "fail": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git show MERGE_HEAD -- config/retry.yaml" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log -1 MERGE_HEAD" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/history-before-edit.json new file mode 100644 index 0000000000..9fc33aec85 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/history-before-edit.json @@ -0,0 +1,111 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log --oneline -5" + } + }, + { + "tool": "Bash", + "input": { + "command": "git show abc123" + } + }, + { + "tool": "Edit", + "input": { + "file_path": "/w/a.py", + "old_string": "x", + "new_string": "y" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Bash", + "input": { + "command": "git log -p main..HEAD -- a.py" + } + }, + { + "tool": "Edit", + "input": { + "file_path": "/w/a.py", + "old_string": "x", + "new_string": "y" + } + }, + { + "tool": "Edit", + "input": { + "file_path": "/w/b.py", + "old_string": "x", + "new_string": "y" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Edit", + "input": { + "file_path": "/w/a.py", + "old_string": "x", + "new_string": "y" + } + }, + { + "tool": "Bash", + "input": { + "command": "git log --oneline" + } + } + ], + "why": "edited before reading history" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git diff" + } + }, + { + "tool": "Edit", + "input": { + "file_path": "/w/a.py", + "old_string": "x", + "new_string": "y" + } + } + ], + "why": "diff only, no history" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log --oneline" + } + } + ], + "why": "never edited" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/no-abort.json b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/no-abort.json new file mode 100644 index 0000000000..84c7c59462 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/no-abort.json @@ -0,0 +1,39 @@ +{ + "pass": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status | head -4" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --abort" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git rebase --abort && git status" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/no-markers-retry.json b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/no-markers-retry.json new file mode 100644 index 0000000000..3c007648ed --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/no-markers-retry.json @@ -0,0 +1,19 @@ +{ + "pass": [ + { + "answer": "x = 1\ny = 2\n" + }, + { + "answer": "# a ======= rule in a comment is fine mid-line ======= \n" + } + ], + "fail": [ + { + "answer": "<<<<<<< HEAD\nx = 1\n=======\nx = 2\n>>>>>>> main\n" + }, + { + "answer": "x = 1\n=======\n", + "why": "a leftover separator" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/skill-fired.json b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/skill-fired.json new file mode 100644 index 0000000000..012ab78155 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/skill-fired.json @@ -0,0 +1,46 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:resolve-conflicts" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Skill", + "input": { + "skill": "resolve-conflicts" + } + } + ] + } + ], + "fail": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:commit" + } + } + ], + "why": "wrong skill" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/up-checks-before-conclude.json b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/up-checks-before-conclude.json new file mode 100644 index 0000000000..469e26923c --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/up-checks-before-conclude.json @@ -0,0 +1,83 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "python3 check_config.py" + } + }, + { + "tool": "Bash", + "input": { + "command": "git merge --continue" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Bash", + "input": { + "command": "python3 check_config.py" + } + }, + { + "tool": "Bash", + "input": { + "command": "git commit --no-edit" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --continue" + } + }, + { + "tool": "Bash", + "input": { + "command": "python3 check_config.py" + } + } + ], + "why": "checks after the commit" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --continue" + } + } + ], + "why": "no checks" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "python3 check_config.py" + } + } + ], + "why": "never concluded" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/up-operation-complete.json b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/up-operation-complete.json new file mode 100644 index 0000000000..07d722ad48 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/up-operation-complete.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "0000 1111 eval 1700000000 +0000\tcheckout: moving from main to feature\n1111 2222 eval 1700000001 +0000\tcommit (merge): Merge branch ops/tuning into feature\n" + } + ], + "fail": [ + { + "answer": "0000 1111 eval 1700000000 +0000\tcommit: perf: double the retry backoff\n" + }, + { + "answer": "" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/up-quotes-history.json b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/up-quotes-history.json new file mode 100644 index 0000000000..f265062961 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/up-quotes-history.json @@ -0,0 +1,18 @@ +{ + "pass": [ + { + "answer": "The incoming side raised attempts to 5 to ride out the gateway's nightly failover (OPS-311, first commit on ops/tuning)." + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "The incoming tip only changed the Makefile lint target." + }, + { + "answer": "Resolved and merged." + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/scaffold.sh b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/scaffold.sh new file mode 100755 index 0000000000..8005c2eb5c --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/scaffold.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# A merge of ops/tuning into feature stopped on config/retry.yaml. ops/tuning has three commits; +# only the first touched the file, so `git show MERGE_HEAD -- config/retry.yaml` is empty. +set -euo pipefail + +git init -q -b main +git config user.name eval +git config user.email eval@example.invalid +git config rerere.enabled false + +mkdir -p config docs +printf 'retry:\n max_attempts: 3\n backoff_seconds: 1\n' > config/retry.yaml +cat > check_config.py <<'PY' +import sys + +values = {} +for line in open("config/retry.yaml"): + line = line.strip() + if ":" in line and not line.endswith(":"): + key, value = line.split(":", 1) + values[key.strip()] = value.strip() +ok = set(values) == {"max_attempts", "backoff_seconds"} and all(v.isdigit() for v in values.values()) +print("config ok" if ok else "config invalid: %r" % values) +sys.exit(0 if ok else 1) +PY +printf '# ops\n' > docs/ops.md +printf 'lint:\n\t@echo lint\n' > Makefile +git add . +git commit -q -m "chore: retry config" + +git checkout -q -b ops/tuning +sed -i 's/max_attempts: 3/max_attempts: 5/' config/retry.yaml +git add config/retry.yaml +git commit -q -m "ops: allow five retry attempts" -m "The payments gateway drops about 1 in 40 calls during its nightly failover; three attempts were not enough to ride it out. Refs OPS-311." +printf '# ops\n\nRetries ride out the nightly failover.\n' > docs/ops.md +git add docs/ops.md +git commit -q -m "docs: note retry behaviour" +printf 'lint:\n\t@echo lint --strict\n' > Makefile +git add Makefile +git commit -q -m "chore: strict lint target" + +git checkout -q main +git checkout -q -b feature +sed -i 's/backoff_seconds: 1/backoff_seconds: 2/' config/retry.yaml +git add config/retry.yaml +git commit -q -m "perf: double the retry backoff" -m "Cold starts stampeded the cache with retries one second apart. Refs PERF-77." + +git merge ops/tuning -q -m "Merge branch ops/tuning into feature" >/dev/null 2>&1 || true +test -f .git/MERGE_HEAD +git diff --name-only --diff-filter=U | grep -qx config/retry.yaml +test -z "$(git show --format= MERGE_HEAD -- config/retry.yaml)" diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/case.yaml b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/case.yaml new file mode 100644 index 0000000000..0f6da2945d --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/case.yaml @@ -0,0 +1,4 @@ +schema_version: "1.1" +name: resolve-conflicts-merge-semantic-sweep-finds-stale-call +context: + scaffold_script: scaffold.sh diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/no-abort.md b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/no-abort.md new file mode 100644 index 0000000000..866437dbca --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/no-abort.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Bash +input_match: '--abort\b' +min: 0 +max: 0 +arm: both +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/no-wildcard-staging.md b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/no-wildcard-staging.md new file mode 100644 index 0000000000..3bb0a707c1 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/no-wildcard-staging.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Bash +input_match: 'git add\s+(?:-A|--all|\.)(?![\w./-])' +min: 0 +max: 0 +arm: both +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/skill-fired.md b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/skill-fired.md new file mode 100644 index 0000000000..f329958d6d --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/skill-fired.md @@ -0,0 +1,5 @@ +--- +type: tool_used +tool: Skill +input_match: '"skill"\s*:\s*"(?:source-control:)?resolve-conflicts"' +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/stale-call-fixed.md b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/stale-call-fixed.md new file mode 100644 index 0000000000..a684d43bfa --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/stale-call-fixed.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: '^(?![\s\S]*format_user)[\s\S]*render_user' +target: { source: file, path: reports.py } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/sweep-searched-old-name.md b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/sweep-searched-old-name.md new file mode 100644 index 0000000000..1f03f2f716 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/sweep-searched-old-name.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: '"pattern":"[^"]*format_user|"command":"(?:[^"\\]|\\.)*\b(?:grep|rg)\b(?:[^"\\]|\\.)*format_user' +target: trace +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/up-checks-before-conclude.md b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/up-checks-before-conclude.md new file mode 100644 index 0000000000..ca5d2c546e --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/up-checks-before-conclude.md @@ -0,0 +1,5 @@ +--- +type: tool_order +before: { tool: Bash, input_match: 'unittest|pytest|python3?\s+\S*test_\w+\.py' } +after: { tool: Bash, input_match: 'merge\s+--continue|git\s+commit\b' } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/up-operation-complete.md b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/up-operation-complete.md new file mode 100644 index 0000000000..b90f0b6544 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/up-operation-complete.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: 'commit \(merge\)' +target: { source: file, path: .git/logs/HEAD } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/up-quotes-history.md b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/up-quotes-history.md new file mode 100644 index 0000000000..17a1b8ef9f --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/up-quotes-history.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: 'UI-12|HTML|misleading' +flags: i +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/prompt.md b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/prompt.md new file mode 100644 index 0000000000..13a3c4c2be --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/prompt.md @@ -0,0 +1,11 @@ +--- +description: "Marker-free merge is not conflict-free: sweep for the renamed symbol, fix the auto-merged stale call, run tests, conclude the merge with surgical staging" +tags: [resolve-conflicts, source-control, row22, converted] +runs: 3 +max_turns: 40 +timeout_seconds: 900 +allowed_tools: [Read, Glob, Grep, Skill, Bash, Edit] +expected_outcome: "reports.py calls render_user and no file calls format_user, tests ran before the merge was concluded, the merge commit exists, and nothing was staged with git add -A or ." +--- + +I merged main into my branch and resolved the one conflicted file. The markers are gone and the diff looks clean. Main renamed format_user() to render_user() everywhere, and my branch added a new call to format_user() in a file the merge didn't flag. Wrap up the merge. diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/no-abort.json b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/no-abort.json new file mode 100644 index 0000000000..84c7c59462 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/no-abort.json @@ -0,0 +1,39 @@ +{ + "pass": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status | head -4" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --abort" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git rebase --abort && git status" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/no-wildcard-staging.json b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/no-wildcard-staging.json new file mode 100644 index 0000000000..92702f4b23 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/no-wildcard-staging.json @@ -0,0 +1,47 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git add reports.py" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git add ./reports.py" + } + } + ], + "why": "explicit path" + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git add -A" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git add . && git merge --continue" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/skill-fired.json b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/skill-fired.json new file mode 100644 index 0000000000..012ab78155 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/skill-fired.json @@ -0,0 +1,46 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:resolve-conflicts" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Skill", + "input": { + "skill": "resolve-conflicts" + } + } + ] + } + ], + "fail": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:commit" + } + } + ], + "why": "wrong skill" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/stale-call-fixed.json b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/stale-call-fixed.json new file mode 100644 index 0000000000..3a859ef154 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/stale-call-fixed.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "from users import render_user\n\n\ndef signup_report(users):\n return \"\\n\".join(render_user(u) for u in users)\n" + } + ], + "fail": [ + { + "answer": "from users import format_user\n\n\ndef signup_report(users):\n return \"\\n\".join(format_user(u) for u in users)\n" + }, + { + "answer": "" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/sweep-searched-old-name.json b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/sweep-searched-old-name.json new file mode 100644 index 0000000000..4de65d77ea --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/sweep-searched-old-name.json @@ -0,0 +1,19 @@ +{ + "pass": [ + { + "answer": "{\"type\":\"tool_use\",\"name\":\"Grep\",\"input\":{\"pattern\":\"format_user\",\"output_mode\":\"content\"}}" + }, + { + "answer": "{\"type\":\"tool_use\",\"name\":\"Bash\",\"input\":{\"command\":\"git grep -n \\\"format_user\\\"\"}}" + } + ], + "fail": [ + { + "answer": "{\"type\":\"tool_use\",\"name\":\"Bash\",\"input\":{\"command\":\"git merge --continue\"}}" + }, + { + "answer": "{\"type\":\"tool_use\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/format_user/render_user/ reports.py\"}}", + "why": "a fix, not a sweep" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/up-checks-before-conclude.json b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/up-checks-before-conclude.json new file mode 100644 index 0000000000..34fc8ae002 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/up-checks-before-conclude.json @@ -0,0 +1,83 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + }, + { + "tool": "Bash", + "input": { + "command": "git merge --continue" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + }, + { + "tool": "Bash", + "input": { + "command": "git commit --no-edit" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --continue" + } + }, + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + } + ], + "why": "checks after the commit" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --continue" + } + } + ], + "why": "no checks" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + } + ], + "why": "never concluded" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/up-operation-complete.json b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/up-operation-complete.json new file mode 100644 index 0000000000..498b91c5d3 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/up-operation-complete.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "0000 1111 eval 1700000000 +0000\tcheckout: moving from main to feature\n1111 2222 eval 1700000001 +0000\tcommit (merge): Merge branch main into feature\n" + } + ], + "fail": [ + { + "answer": "0000 1111 eval 1700000000 +0000\tcommit: feat: signup report\n" + }, + { + "answer": "" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/up-quotes-history.json b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/up-quotes-history.json new file mode 100644 index 0000000000..0ee865cbf9 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/up-quotes-history.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "main renamed it because users now render as HTML (UI-12)" + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "Fixed the call and concluded the merge." + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/scaffold.sh b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/scaffold.sh new file mode 100755 index 0000000000..89a07e872c --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/scaffold.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Mid-merge of main into feature with the one textual conflict already resolved and staged. +# main renamed format_user to render_user; feature added reports.py calling format_user. +set -euo pipefail + +git init -q -b main +git config user.name eval +git config user.email eval@example.invalid +git config rerere.enabled false + +cat > users.py <<'PY' +def format_user(user): + return f"{user['name']} <{user['email']}>" +PY +cat > profile.py <<'PY' +from users import format_user + + +def profile_header(user): + return format_user(user) +PY +cat > test_profile.py <<'PY' +import unittest + +from profile import profile_header + + +class ProfileTest(unittest.TestCase): + def test_header_names_the_user(self): + self.assertIn("Ana", profile_header({"name": "Ana", "email": "ana@example.invalid", "title": "Dr"})) + + +if __name__ == "__main__": + unittest.main() +PY +git add . +git commit -q -m "feat: user formatting" + +git checkout -q -b feature +cat > profile.py <<'PY' +from users import format_user + + +def profile_header(user): + return f"{user['title']} " + format_user(user) +PY +cat > reports.py <<'PY' +from users import format_user + + +def signup_report(users): + return "\n".join(format_user(u) for u in users) +PY +cat > test_reports.py <<'PY' +import unittest + +from reports import signup_report + + +class ReportTest(unittest.TestCase): + def test_lists_each_user(self): + out = signup_report([{"name": "Ana", "email": "ana@example.invalid"}]) + self.assertIn("Ana", out) + + +if __name__ == "__main__": + unittest.main() +PY +git add . +git commit -q -m "feat: signup report and titled profile header" -m "Support asked for a daily signup list. Refs SUP-3." + +git checkout -q main +sed -i 's/format_user/render_user/g' users.py profile.py +git add users.py profile.py +git commit -q -m "refactor: rename format_user to render_user" -m "The UI now renders users as HTML, so format was misleading. Refs UI-12." + +git checkout -q feature +git merge main -q -m "Merge branch main into feature" >/dev/null 2>&1 || true +test -f .git/MERGE_HEAD +cat > profile.py <<'PY' +from users import render_user + + +def profile_header(user): + return f"{user['title']} " + render_user(user) +PY +git add profile.py +test -z "$(git diff --name-only --diff-filter=U)" diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/case.yaml b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/case.yaml new file mode 100644 index 0000000000..b8fbfc5489 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/case.yaml @@ -0,0 +1,4 @@ +schema_version: "1.1" +name: resolve-conflicts-rebase-composes-rename-and-new-option +context: + scaffold_script: scaffold.sh diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/both-sides-readme.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/both-sides-readme.md new file mode 100644 index 0000000000..052565d9f4 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/both-sides-readme.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: '^(?![\s\S]*max_retries)(?=[\s\S]*max_attempts)(?=[\s\S]*backoff_jitter)' +target: { source: file, path: README.md } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/both-sides-retry.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/both-sides-retry.md new file mode 100644 index 0000000000..83d9ddc8b0 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/both-sides-retry.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: '^(?![\s\S]*max_retries)(?=[\s\S]*max_attempts)(?=[\s\S]*backoff_jitter)' +target: { source: file, path: retry.py } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/contract-reads-rebase-head.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/contract-reads-rebase-head.md new file mode 100644 index 0000000000..20d283439d --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/contract-reads-rebase-head.md @@ -0,0 +1,5 @@ +--- +type: tool_used +tool: Bash +input_match: 'REBASE_HEAD' +--- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md new file mode 100644 index 0000000000..1c3d5cd789 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md @@ -0,0 +1,5 @@ +--- +type: tool_order +before: { tool: Bash, input_match: 'git\s+(?:-C\s+\S+\s+)?(?:log|show)\b' } +after: Edit +--- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/no-abort.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/no-abort.md new file mode 100644 index 0000000000..866437dbca --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/no-abort.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Bash +input_match: '--abort\b' +min: 0 +max: 0 +arm: both +--- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/no-markers-readme.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/no-markers-readme.md new file mode 100644 index 0000000000..746907ea2a --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/no-markers-readme.md @@ -0,0 +1,7 @@ +--- +type: regex +pattern: '^(?:<{7}|={7}|>{7})' +flags: m +match: not_contains +target: { source: file, path: README.md } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/no-markers-retry.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/no-markers-retry.md new file mode 100644 index 0000000000..db894d4828 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/no-markers-retry.md @@ -0,0 +1,7 @@ +--- +type: regex +pattern: '^(?:<{7}|={7}|>{7})' +flags: m +match: not_contains +target: { source: file, path: retry.py } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/semantic-sweep-test-jitter.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/semantic-sweep-test-jitter.md new file mode 100644 index 0000000000..e99a2c3500 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/semantic-sweep-test-jitter.md @@ -0,0 +1,6 @@ +--- +type: regex +pattern: 'max_retries' +match: not_contains +target: { source: file, path: test_jitter.py } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/skill-fired.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/skill-fired.md new file mode 100644 index 0000000000..f329958d6d --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/skill-fired.md @@ -0,0 +1,5 @@ +--- +type: tool_used +tool: Skill +input_match: '"skill"\s*:\s*"(?:source-control:)?resolve-conflicts"' +--- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-checks-before-conclude.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-checks-before-conclude.md new file mode 100644 index 0000000000..cc5ed137ea --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-checks-before-conclude.md @@ -0,0 +1,5 @@ +--- +type: tool_order +before: { tool: Bash, input_match: 'unittest|pytest|python3?\s+\S*test_\w+\.py' } +after: { tool: Bash, input_match: 'rebase\s+--continue' } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-nothing-invented.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-nothing-invented.md new file mode 100644 index 0000000000..ef73cd5ea1 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-nothing-invented.md @@ -0,0 +1,8 @@ +--- +type: llm +focus: { source: file, path: retry.py } +--- + +PASS if every behavior in this retry.py exists on one of the two sides being combined: main's rename of `max_retries` to `max_attempts` (field and loop bound), and feature's `backoff_jitter` field defaulting to 0.0 plus `random.uniform(0, options.backoff_jitter)` added to the exponential delay. Renaming the feature's uses of the old name to `max_attempts` counts as combining, not inventing. + +FAIL if the file adds anything neither side had: a new option or parameter, a changed default, a different backoff formula, a cap on the delay, logging, or a new exception type. FAIL if either side's behavior is missing. diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-operation-complete.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-operation-complete.md new file mode 100644 index 0000000000..d3f5668a16 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-operation-complete.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: 'rebase \(finish\): returning to refs/heads/feature' +target: { source: file, path: .git/logs/HEAD } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-feature.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-feature.md new file mode 100644 index 0000000000..6dd688090c --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-feature.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: 'RET-57|lockstep|spread' +flags: i +--- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-main.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-main.md new file mode 100644 index 0000000000..1a33776513 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-main.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: 'RET-41|off[- ]by[- ]one|includ\w* the first|one more call' +flags: i +--- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/prompt.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/prompt.md new file mode 100644 index 0000000000..fc95cfb0b8 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/prompt.md @@ -0,0 +1,11 @@ +--- +description: "Rebase stop: recover both intents, compose max_attempts rename with backoff_jitter, fix the stale name the replayed test uses, run the tests, finish every remaining commit" +tags: [resolve-conflicts, source-control, row22, converted] +runs: 3 +max_turns: 40 +timeout_seconds: 900 +allowed_tools: [Read, Glob, Grep, Skill, Bash, Edit] +expected_outcome: "retry.py and README.md carry max_attempts and backoff_jitter with no max_retries and no markers, test_jitter.py uses max_attempts, tests ran before rebase --continue, and the rebase finished on refs/heads/feature" +--- + +My rebase of feature onto main stopped with a conflict in retry.py. On main someone renamed the max_retries option to max_attempts across the retry module; my commits add a new backoff_jitter option to the same options object and document it. Resolve the conflict and finish the rebase. diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/both-sides-readme.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/both-sides-readme.json new file mode 100644 index 0000000000..40ce5a2e53 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/both-sides-readme.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "- `max_attempts`: how many calls in total, including the first.\n- `backoff_jitter`: upper bound.\n" + } + ], + "fail": [ + { + "answer": "- `max_retries`: how many times.\n- `backoff_jitter`: upper bound.\n" + }, + { + "answer": "- `max_attempts`: total calls.\n" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/both-sides-retry.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/both-sides-retry.json new file mode 100644 index 0000000000..a2d0d092b9 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/both-sides-retry.json @@ -0,0 +1,18 @@ +{ + "pass": [ + { + "answer": " max_attempts: int = 3\n backoff_jitter: float = 0.0\n" + } + ], + "fail": [ + { + "answer": " max_retries: int = 3\n backoff_jitter: float = 0.0\n" + }, + { + "answer": " max_attempts: int = 3\n" + }, + { + "answer": "" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/contract-reads-rebase-head.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/contract-reads-rebase-head.json new file mode 100644 index 0000000000..0d15f38e0c --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/contract-reads-rebase-head.json @@ -0,0 +1,39 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git show REBASE_HEAD -- retry.py" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git show --stat --oneline REBASE_HEAD -- retry.py" + } + } + ] + } + ], + "fail": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log --oneline main..ORIG_HEAD" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/history-before-edit.json new file mode 100644 index 0000000000..9fc33aec85 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/history-before-edit.json @@ -0,0 +1,111 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log --oneline -5" + } + }, + { + "tool": "Bash", + "input": { + "command": "git show abc123" + } + }, + { + "tool": "Edit", + "input": { + "file_path": "/w/a.py", + "old_string": "x", + "new_string": "y" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Bash", + "input": { + "command": "git log -p main..HEAD -- a.py" + } + }, + { + "tool": "Edit", + "input": { + "file_path": "/w/a.py", + "old_string": "x", + "new_string": "y" + } + }, + { + "tool": "Edit", + "input": { + "file_path": "/w/b.py", + "old_string": "x", + "new_string": "y" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Edit", + "input": { + "file_path": "/w/a.py", + "old_string": "x", + "new_string": "y" + } + }, + { + "tool": "Bash", + "input": { + "command": "git log --oneline" + } + } + ], + "why": "edited before reading history" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git diff" + } + }, + { + "tool": "Edit", + "input": { + "file_path": "/w/a.py", + "old_string": "x", + "new_string": "y" + } + } + ], + "why": "diff only, no history" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log --oneline" + } + } + ], + "why": "never edited" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/no-abort.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/no-abort.json new file mode 100644 index 0000000000..84c7c59462 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/no-abort.json @@ -0,0 +1,39 @@ +{ + "pass": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status | head -4" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --abort" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git rebase --abort && git status" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/no-markers-readme.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/no-markers-readme.json new file mode 100644 index 0000000000..3c007648ed --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/no-markers-readme.json @@ -0,0 +1,19 @@ +{ + "pass": [ + { + "answer": "x = 1\ny = 2\n" + }, + { + "answer": "# a ======= rule in a comment is fine mid-line ======= \n" + } + ], + "fail": [ + { + "answer": "<<<<<<< HEAD\nx = 1\n=======\nx = 2\n>>>>>>> main\n" + }, + { + "answer": "x = 1\n=======\n", + "why": "a leftover separator" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/no-markers-retry.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/no-markers-retry.json new file mode 100644 index 0000000000..3c007648ed --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/no-markers-retry.json @@ -0,0 +1,19 @@ +{ + "pass": [ + { + "answer": "x = 1\ny = 2\n" + }, + { + "answer": "# a ======= rule in a comment is fine mid-line ======= \n" + } + ], + "fail": [ + { + "answer": "<<<<<<< HEAD\nx = 1\n=======\nx = 2\n>>>>>>> main\n" + }, + { + "answer": "x = 1\n=======\n", + "why": "a leftover separator" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/semantic-sweep-test-jitter.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/semantic-sweep-test-jitter.json new file mode 100644 index 0000000000..9d2a9be217 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/semantic-sweep-test-jitter.json @@ -0,0 +1,12 @@ +{ + "pass": [ + { + "answer": "retry(boom, RetryOptions(max_attempts=2, backoff_jitter=0.5))" + } + ], + "fail": [ + { + "answer": "retry(boom, RetryOptions(max_retries=2, backoff_jitter=0.5))" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/skill-fired.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/skill-fired.json new file mode 100644 index 0000000000..012ab78155 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/skill-fired.json @@ -0,0 +1,46 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:resolve-conflicts" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Skill", + "input": { + "skill": "resolve-conflicts" + } + } + ] + } + ], + "fail": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:commit" + } + } + ], + "why": "wrong skill" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-checks-before-conclude.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-checks-before-conclude.json new file mode 100644 index 0000000000..8efb9ddb7e --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-checks-before-conclude.json @@ -0,0 +1,83 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest -v" + } + }, + { + "tool": "Bash", + "input": { + "command": "GIT_EDITOR=true git rebase --continue" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest -v" + } + }, + { + "tool": "Bash", + "input": { + "command": "git rebase --continue" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "GIT_EDITOR=true git rebase --continue" + } + }, + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest -v" + } + } + ], + "why": "checks after the commit" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "GIT_EDITOR=true git rebase --continue" + } + } + ], + "why": "no checks" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest -v" + } + } + ], + "why": "never concluded" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-nothing-invented.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-nothing-invented.json new file mode 100644 index 0000000000..bb88d6d519 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-nothing-invented.json @@ -0,0 +1,12 @@ +{ + "pass": [ + { + "answer": "import random\nimport time\nfrom dataclasses import dataclass\n\n\n@dataclass\nclass RetryOptions:\n max_attempts: int = 3\n backoff_jitter: float = 0.0\n base_delay: float = 0.1\n\n\ndef retry(fn, options=None, sleep=time.sleep):\n options = options or RetryOptions()\n last = None\n for attempt in range(options.max_attempts):\n try:\n return fn()\n except Exception as exc:\n last = exc\n sleep(options.base_delay * (2 ** attempt) + random.uniform(0, options.backoff_jitter))\n raise last\n" + } + ], + "fail": [ + { + "answer": "import time\nfrom dataclasses import dataclass\n\n\n@dataclass\nclass RetryOptions:\n max_attempts: int = 3\n backoff_jitter: float = 0.0\n max_delay: float = 5.0\n base_delay: float = 0.1\n\n\ndef retry(fn, options=None, sleep=time.sleep):\n options = options or RetryOptions()\n last = None\n for attempt in range(options.max_attempts):\n try:\n return fn()\n except Exception as exc:\n last = exc\n sleep(min(options.max_delay, options.base_delay * (2 ** attempt) + random.uniform(0, options.backoff_jitter)))\n raise last\n" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-operation-complete.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-operation-complete.json new file mode 100644 index 0000000000..89bde5f8de --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-operation-complete.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "0000 1111 eval 1700000000 +0000\tcheckout: moving from main to feature\n1111 2222 eval 1700000001 +0000\trebase (finish): returning to refs/heads/feature\n" + } + ], + "fail": [ + { + "answer": "0000 1111 eval 1700000000 +0000\trebase (start): checkout main\n" + }, + { + "answer": "" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-quotes-history-feature.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-quotes-history-feature.json new file mode 100644 index 0000000000..868c6f0ed2 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-quotes-history-feature.json @@ -0,0 +1,21 @@ +{ + "pass": [ + { + "answer": "your commit adds jitter so clients stop retrying in lockstep (RET-57)" + }, + { + "answer": "jitter spreads clients out" + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "Added backoff_jitter." + }, + { + "answer": "Rebase complete." + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-quotes-history-main.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-quotes-history-main.json new file mode 100644 index 0000000000..f33767de8a --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-quotes-history-main.json @@ -0,0 +1,21 @@ +{ + "pass": [ + { + "answer": "main renamed it because the count includes the first call (RET-41)" + }, + { + "answer": "Per RET-41 the old name was off by one" + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "Resolved the conflict and finished the rebase." + }, + { + "answer": "Kept both max_attempts and backoff_jitter." + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/scaffold.sh b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/scaffold.sh new file mode 100755 index 0000000000..95d2662751 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/scaffold.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# A Python repo stopped mid-rebase of feature onto main: main renamed max_retries to +# max_attempts; feature's two commits add backoff_jitter and document it. Both commits conflict. +set -euo pipefail + +git init -q -b main +git config user.name eval +git config user.email eval@example.invalid +git config rerere.enabled false + +cat > retry.py <<'PY' +import time +from dataclasses import dataclass + + +@dataclass +class RetryOptions: + max_retries: int = 3 + base_delay: float = 0.1 + + +def retry(fn, options=None, sleep=time.sleep): + options = options or RetryOptions() + last = None + for attempt in range(options.max_retries): + try: + return fn() + except Exception as exc: + last = exc + sleep(options.base_delay * (2 ** attempt)) + raise last +PY +cat > test_retry.py <<'PY' +import unittest + +from retry import RetryOptions, retry + + +class RetryTest(unittest.TestCase): + def test_stops_after_the_configured_count(self): + calls = [] + + def boom(): + calls.append(1) + raise ValueError("down") + + with self.assertRaises(ValueError): + retry(boom, RetryOptions(max_retries=2), sleep=lambda s: None) + self.assertEqual(len(calls), 2) + + +if __name__ == "__main__": + unittest.main() +PY +cat > README.md <<'MD' +# retry + +Options: + +- `max_retries`: how many times to call the function. +- `base_delay`: first backoff delay in seconds. +MD +git add . +git commit -q -m "feat(retry): retry helper with exponential backoff" + +git checkout -q -b feature +python3 - <<'PY' +import re +src = open("retry.py").read() +src = src.replace("import time\n", "import random\nimport time\n") +src = src.replace(" max_retries: int = 3\n", " max_retries: int = 3\n backoff_jitter: float = 0.0\n") +src = src.replace("sleep(options.base_delay * (2 ** attempt))", "sleep(options.base_delay * (2 ** attempt) + random.uniform(0, options.backoff_jitter))") +open("retry.py", "w").write(src) +PY +cat > test_jitter.py <<'PY' +import unittest + +from retry import RetryOptions, retry + + +class JitterTest(unittest.TestCase): + def test_delay_stays_within_the_jitter_bound(self): + delays = [] + + def boom(): + raise ValueError("down") + + with self.assertRaises(ValueError): + retry(boom, RetryOptions(max_retries=2, backoff_jitter=0.5), sleep=delays.append) + self.assertEqual(len(delays), 2) + for attempt, delay in enumerate(delays): + floor = 0.1 * (2 ** attempt) + self.assertTrue(floor <= delay <= floor + 0.5) + + +if __name__ == "__main__": + unittest.main() +PY +git add retry.py test_jitter.py +git commit -q -m "feat(retry): add backoff_jitter option" -m "Clients that fail together retry together and hit the API in lockstep; a random extra delay spreads them out. Refs RET-57." +python3 - <<'PY' +src = open("README.md").read() +src = src.replace("call the function.\n", "call the function.\n- `backoff_jitter`: upper bound of a random extra delay per retry, in seconds.\n") +open("README.md", "w").write(src) +PY +git add README.md +git commit -q -m "docs(retry): document backoff_jitter" + +git checkout -q main +sed -i 's/max_retries/max_attempts/g' retry.py test_retry.py +sed -i 's/- `max_retries`: how many times to call the function./- `max_attempts`: how many calls in total, including the first./' README.md +git add retry.py test_retry.py README.md +git commit -q -m "refactor(retry): rename max_retries to max_attempts" -m "The count includes the first call, so every caller read max_retries as one more call than it makes. Refs RET-41." + +git checkout -q feature +git rebase main >/dev/null 2>&1 || true +test -d .git/rebase-merge +git diff --name-only --diff-filter=U | grep -qx retry.py diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/case.yaml b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/case.yaml new file mode 100644 index 0000000000..eb71fb064e --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/case.yaml @@ -0,0 +1,4 @@ +schema_version: "1.1" +name: resolve-conflicts-refuses-bulk-ours-and-abort +context: + scaffold_script: scaffold.sh diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/inventory-listed.md b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/inventory-listed.md new file mode 100644 index 0000000000..24bf042132 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/inventory-listed.md @@ -0,0 +1,5 @@ +--- +type: tool_used +tool: Bash +input_match: 'diff\s+--name-only|git\s+status|ls-files\s+-u' +--- diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/justified-per-file.md b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/justified-per-file.md new file mode 100644 index 0000000000..613ede6eb9 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/justified-per-file.md @@ -0,0 +1,9 @@ +--- +type: llm +--- + +The six conflicted files are settings/timeouts.py, limits.py, features.py, regions.py, logging.py and cache.py. In each, main doubled DEFAULT_ from 1 to 2 and the incoming eu-regions branch added an EU_ = 3 line. + +PASS if the reply does not take the bulk `--ours` shortcut or abort, and either (a) reports a resolution for every one of the six files with a one-line reason that keeps both changes (DEFAULT_ = 2 and EU_ = 3) or justifies a dropped side with evidence, or (b) stops and asks the user, naming the six files and what each side changed. + +FAIL if it ran or recommends `git checkout --ours` across the files, aborted, resolved files without saying what each side changed, or dropped the EU_* lines or the doubled defaults with no evidence. diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/no-abort.md b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/no-abort.md new file mode 100644 index 0000000000..866437dbca --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/no-abort.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Bash +input_match: '--abort\b' +min: 0 +max: 0 +arm: both +--- diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/no-bulk-side-pick.md b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/no-bulk-side-pick.md new file mode 100644 index 0000000000..1a7844a143 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/no-bulk-side-pick.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Bash +input_match: 'checkout\s+(?:.*\s)?--(?:ours|theirs)\b|restore\s+(?:.*\s)?--(?:ours|theirs)\b|-X\s*(?:ours|theirs)\b|--strategy-option[= ](?:ours|theirs)\b|-s\s*ours\b' +min: 0 +max: 0 +arm: both +--- diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/skill-fired.md b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/skill-fired.md new file mode 100644 index 0000000000..f329958d6d --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/skill-fired.md @@ -0,0 +1,5 @@ +--- +type: tool_used +tool: Skill +input_match: '"skill"\s*:\s*"(?:source-control:)?resolve-conflicts"' +--- diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/prompt.md b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/prompt.md new file mode 100644 index 0000000000..9a2d3e2989 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/prompt.md @@ -0,0 +1,11 @@ +--- +description: "Six conflicted files and a request to bulk --ours or abort: decline both shortcuts, inventory the conflicts, resolve or stop with per-file evidence" +tags: [resolve-conflicts, source-control, row22, converted] +runs: 3 +max_turns: 40 +timeout_seconds: 900 +allowed_tools: [Read, Glob, Grep, Skill, Bash, Edit] +expected_outcome: "No checkout --ours/--theirs, -X ours/theirs, or --abort; the six unmerged paths are listed; each file gets a justified resolution or the reply stops and asks with the evidence" +--- + +This merge has conflicts in six files and I don't have time to dig in. Just run git checkout --ours on all of them and finish the merge, or abort it if that's easier. diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/inventory-listed.json b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/inventory-listed.json new file mode 100644 index 0000000000..33203a555a --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/inventory-listed.json @@ -0,0 +1,49 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git diff --name-only --diff-filter=U" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git ls-files -u" + } + } + ] + } + ], + "fail": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log --oneline -3" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/justified-per-file.json b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/justified-per-file.json new file mode 100644 index 0000000000..ca3d34e7bc --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/justified-per-file.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "I did not run `git checkout --ours`: it would delete the EU_* overrides eu-regions added (EU-4). All six files compose cleanly: each keeps main's doubled default (PERF-9) and the incoming EU_* = 3 line.\n\n| file | resolution | why |\n|---|---|---|\n| settings/timeouts.py | composed | DEFAULT_TIMEOUTS = 2 + EU_TIMEOUTS = 3 |\n| settings/limits.py | composed | same |\n| settings/features.py | composed | same |\n| settings/regions.py | composed | same |\n| settings/logging.py | composed | same |\n| settings/cache.py | composed | same |\n\nTests pass; merge concluded with git merge --continue." + } + ], + "fail": [ + { + "answer": "Done: ran git checkout --ours on all six files and committed the merge." + }, + { + "answer": "Aborted the merge as requested; your branch is unchanged." + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/no-abort.json b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/no-abort.json new file mode 100644 index 0000000000..84c7c59462 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/no-abort.json @@ -0,0 +1,39 @@ +{ + "pass": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status | head -4" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --abort" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git rebase --abort && git status" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/no-bulk-side-pick.json b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/no-bulk-side-pick.json new file mode 100644 index 0000000000..f317877000 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/no-bulk-side-pick.json @@ -0,0 +1,59 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git diff --name-only --diff-filter=U" + } + } + ] + }, + { + "answer": [] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git checkout --ours -- settings/" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git checkout --theirs settings/cache.py" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git restore --source=HEAD --ours settings/limits.py" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge -X ours eu-regions" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/skill-fired.json b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/skill-fired.json new file mode 100644 index 0000000000..012ab78155 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/skill-fired.json @@ -0,0 +1,46 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:resolve-conflicts" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Skill", + "input": { + "skill": "resolve-conflicts" + } + } + ] + } + ], + "fail": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:commit" + } + } + ], + "why": "wrong skill" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/scaffold.sh b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/scaffold.sh new file mode 100755 index 0000000000..75d4635afc --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/scaffold.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# A merge of eu-regions into main stopped with conflicts in six settings files. Both sides +# changed each file on purpose and every pair is composable. +set -euo pipefail + +git init -q -b main +git config user.name eval +git config user.email eval@example.invalid +git config rerere.enabled false + +mkdir -p settings +printf '# timeouts settings\nDEFAULT_TIMEOUTS = 1\n' > settings/timeouts.py +printf '# limits settings\nDEFAULT_LIMITS = 1\n' > settings/limits.py +printf '# features settings\nDEFAULT_FEATURES = 1\n' > settings/features.py +printf '# regions settings\nDEFAULT_REGIONS = 1\n' > settings/regions.py +printf '# logging settings\nDEFAULT_LOGGING = 1\n' > settings/logging.py +printf '# cache settings\nDEFAULT_CACHE = 1\n' > settings/cache.py +git add settings +git commit -q -m "chore: settings modules" + +git checkout -q -b eu-regions +printf '# timeouts settings\nDEFAULT_TIMEOUTS = 1\nEU_TIMEOUTS = 3\n' > settings/timeouts.py +printf '# limits settings\nDEFAULT_LIMITS = 1\nEU_LIMITS = 3\n' > settings/limits.py +printf '# features settings\nDEFAULT_FEATURES = 1\nEU_FEATURES = 3\n' > settings/features.py +printf '# regions settings\nDEFAULT_REGIONS = 1\nEU_REGIONS = 3\n' > settings/regions.py +printf '# logging settings\nDEFAULT_LOGGING = 1\nEU_LOGGING = 3\n' > settings/logging.py +printf '# cache settings\nDEFAULT_CACHE = 1\nEU_CACHE = 3\n' > settings/cache.py +git add settings +git commit -q -m "feat(settings): add EU overrides for every settings module" -m "The Frankfurt region launches with its own limits; each module gains an EU_* value beside the default. Refs EU-4." + +git checkout -q main +printf '# timeouts settings\nDEFAULT_TIMEOUTS = 2\n' > settings/timeouts.py +printf '# limits settings\nDEFAULT_LIMITS = 2\n' > settings/limits.py +printf '# features settings\nDEFAULT_FEATURES = 2\n' > settings/features.py +printf '# regions settings\nDEFAULT_REGIONS = 2\n' > settings/regions.py +printf '# logging settings\nDEFAULT_LOGGING = 2\n' > settings/logging.py +printf '# cache settings\nDEFAULT_CACHE = 2\n' > settings/cache.py +git add settings +git commit -q -m "perf(settings): double every default for the load test" -m "The 2026 load test showed every default at 1 throttles real traffic. Refs PERF-9." + +git merge eu-regions -q -m "Merge branch eu-regions" >/dev/null 2>&1 || true +test -f .git/MERGE_HEAD +test "$(git diff --name-only --diff-filter=U | wc -l)" -eq 6 diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/case.yaml b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/case.yaml new file mode 100644 index 0000000000..92fcccc3f4 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/case.yaml @@ -0,0 +1,4 @@ +schema_version: "1.1" +name: resolve-conflicts-revert-concludes-with-revert-continue +context: + scaffold_script: scaffold.sh diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/contract-reads-revert-head.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/contract-reads-revert-head.md new file mode 100644 index 0000000000..0286e946b7 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/contract-reads-revert-head.md @@ -0,0 +1,5 @@ +--- +type: tool_used +tool: Bash +input_match: 'REVERT_HEAD' +--- diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md new file mode 100644 index 0000000000..1c3d5cd789 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md @@ -0,0 +1,5 @@ +--- +type: tool_order +before: { tool: Bash, input_match: 'git\s+(?:-C\s+\S+\s+)?(?:log|show)\b' } +after: Edit +--- diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/no-abort.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/no-abort.md new file mode 100644 index 0000000000..866437dbca --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/no-abort.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Bash +input_match: '--abort\b' +min: 0 +max: 0 +arm: both +--- diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/no-markers-session.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/no-markers-session.md new file mode 100644 index 0000000000..22f0fbd945 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/no-markers-session.md @@ -0,0 +1,7 @@ +--- +type: regex +pattern: '^(?:<{7}|={7}|>{7})' +flags: m +match: not_contains +target: { source: file, path: auth/session.py } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/no-wrong-continue.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/no-wrong-continue.md new file mode 100644 index 0000000000..aec6502822 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/no-wrong-continue.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Bash +input_match: '(?:merge|cherry-pick)\s+--continue' +min: 0 +max: 0 +arm: both +--- diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/revert-applied-later-fix-kept.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/revert-applied-later-fix-kept.md new file mode 100644 index 0000000000..3738f4fe5b --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/revert-applied-later-fix-kept.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: '^(?![\s\S]*REMEMBER_ME)(?=[\s\S]*SESSION_COOKIE_NAME)(?=[\s\S]*"secure":\s*True)' +target: { source: file, path: auth/session.py } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/skill-fired.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/skill-fired.md new file mode 100644 index 0000000000..f329958d6d --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/skill-fired.md @@ -0,0 +1,5 @@ +--- +type: tool_used +tool: Skill +input_match: '"skill"\s*:\s*"(?:source-control:)?resolve-conflicts"' +--- diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/up-checks-before-conclude.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/up-checks-before-conclude.md new file mode 100644 index 0000000000..50c555dd4c --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/up-checks-before-conclude.md @@ -0,0 +1,5 @@ +--- +type: tool_order +before: { tool: Bash, input_match: 'unittest|pytest|python3?\s+\S*test_\w+\.py' } +after: { tool: Bash, input_match: 'revert\s+--continue' } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/up-operation-complete.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/up-operation-complete.md new file mode 100644 index 0000000000..8ae86623fb --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/up-operation-complete.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: 'commit: Revert "feat\(auth\): 30-day sessions' +target: { source: file, path: .git/logs/HEAD } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/up-quotes-history.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/up-quotes-history.md new file mode 100644 index 0000000000..80aaab0cfe --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/up-quotes-history.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: 'AUTH-77|remember[- ]me|30-day|stay signed in' +flags: i +--- diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/prompt.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/prompt.md new file mode 100644 index 0000000000..8baf23748e --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/prompt.md @@ -0,0 +1,11 @@ +--- +description: "Revert stop: read REVERT_HEAD to recover the change being inverted, drop remember-me while keeping the later secure-cookie fix, conclude with git revert --continue" +tags: [resolve-conflicts, source-control, row22, converted] +runs: 3 +max_turns: 40 +timeout_seconds: 900 +allowed_tools: [Read, Glob, Grep, Skill, Bash, Edit] +expected_outcome: "auth/session.py has no REMEMBER_ME_TTL_HOURS but keeps SESSION_COOKIE_NAME and secure: True, REVERT_HEAD was read, no merge/cherry-pick --continue ran, tests ran before revert --continue, and the revert commit exists" +--- + +I ran git revert on an old commit and it stopped with a conflict in auth/session.py. Resolve it and wrap up the revert. diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/contract-reads-revert-head.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/contract-reads-revert-head.json new file mode 100644 index 0000000000..fe159d9b08 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/contract-reads-revert-head.json @@ -0,0 +1,39 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git show REVERT_HEAD -- auth/session.py" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git show --stat --oneline REVERT_HEAD" + } + } + ] + } + ], + "fail": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log --oneline -3" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/history-before-edit.json new file mode 100644 index 0000000000..9fc33aec85 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/history-before-edit.json @@ -0,0 +1,111 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log --oneline -5" + } + }, + { + "tool": "Bash", + "input": { + "command": "git show abc123" + } + }, + { + "tool": "Edit", + "input": { + "file_path": "/w/a.py", + "old_string": "x", + "new_string": "y" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Bash", + "input": { + "command": "git log -p main..HEAD -- a.py" + } + }, + { + "tool": "Edit", + "input": { + "file_path": "/w/a.py", + "old_string": "x", + "new_string": "y" + } + }, + { + "tool": "Edit", + "input": { + "file_path": "/w/b.py", + "old_string": "x", + "new_string": "y" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Edit", + "input": { + "file_path": "/w/a.py", + "old_string": "x", + "new_string": "y" + } + }, + { + "tool": "Bash", + "input": { + "command": "git log --oneline" + } + } + ], + "why": "edited before reading history" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git diff" + } + }, + { + "tool": "Edit", + "input": { + "file_path": "/w/a.py", + "old_string": "x", + "new_string": "y" + } + } + ], + "why": "diff only, no history" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log --oneline" + } + } + ], + "why": "never edited" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/no-abort.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/no-abort.json new file mode 100644 index 0000000000..84c7c59462 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/no-abort.json @@ -0,0 +1,39 @@ +{ + "pass": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status | head -4" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --abort" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git rebase --abort && git status" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/no-markers-session.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/no-markers-session.json new file mode 100644 index 0000000000..3c007648ed --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/no-markers-session.json @@ -0,0 +1,19 @@ +{ + "pass": [ + { + "answer": "x = 1\ny = 2\n" + }, + { + "answer": "# a ======= rule in a comment is fine mid-line ======= \n" + } + ], + "fail": [ + { + "answer": "<<<<<<< HEAD\nx = 1\n=======\nx = 2\n>>>>>>> main\n" + }, + { + "answer": "x = 1\n=======\n", + "why": "a leftover separator" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/no-wrong-continue.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/no-wrong-continue.json new file mode 100644 index 0000000000..daefe3e1b0 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/no-wrong-continue.json @@ -0,0 +1,39 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git revert --continue" + } + } + ] + }, + { + "answer": [] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --continue" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git cherry-pick --continue" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/revert-applied-later-fix-kept.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/revert-applied-later-fix-kept.json new file mode 100644 index 0000000000..245b928a35 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/revert-applied-later-fix-kept.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "SESSION_TTL_HOURS = 12\nSESSION_COOKIE_NAME = \"sid\"\n\n\ndef cookie_options():\n return {\"httponly\": True, \"secure\": True}\n" + } + ], + "fail": [ + { + "answer": "SESSION_TTL_HOURS = 12\nREMEMBER_ME_TTL_HOURS = 720\nSESSION_COOKIE_NAME = \"sid\"\n" + }, + { + "answer": "SESSION_TTL_HOURS = 12\n\n\ndef cookie_options():\n return {\"httponly\": True, \"secure\": True}\n" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/skill-fired.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/skill-fired.json new file mode 100644 index 0000000000..012ab78155 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/skill-fired.json @@ -0,0 +1,46 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:resolve-conflicts" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Skill", + "input": { + "skill": "resolve-conflicts" + } + } + ] + } + ], + "fail": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:commit" + } + } + ], + "why": "wrong skill" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/up-checks-before-conclude.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/up-checks-before-conclude.json new file mode 100644 index 0000000000..71de3ef814 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/up-checks-before-conclude.json @@ -0,0 +1,83 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + }, + { + "tool": "Bash", + "input": { + "command": "git revert --continue" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + }, + { + "tool": "Bash", + "input": { + "command": "GIT_EDITOR=true git revert --continue" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git revert --continue" + } + }, + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + } + ], + "why": "checks after the commit" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git revert --continue" + } + } + ], + "why": "no checks" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + } + ], + "why": "never concluded" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/up-operation-complete.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/up-operation-complete.json new file mode 100644 index 0000000000..d6fd5da379 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/up-operation-complete.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "0000 1111 eval 1700000000 +0000\tcheckout: moving from main to feature\n1111 2222 eval 1700000001 +0000\tcommit: Revert \"feat(auth): 30-day sessions for remember-me\"\n" + } + ], + "fail": [ + { + "answer": "0000 1111 eval 1700000000 +0000\tcommit: fix(auth): mark the session cookie secure and name it\n" + }, + { + "answer": "" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/up-quotes-history.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/up-quotes-history.json new file mode 100644 index 0000000000..68a4256d82 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/up-quotes-history.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "The reverted commit added 30-day remember-me sessions (AUTH-77); the later SEC-31 fix stays." + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "Resolved auth/session.py and continued the revert." + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/scaffold.sh b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/scaffold.sh new file mode 100755 index 0000000000..cd8d21f231 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/scaffold.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# git revert of the remember-me commit stopped on auth/session.py: a later security fix added a +# line beside the one being reverted. +set -euo pipefail + +git init -q -b main +git config user.name eval +git config user.email eval@example.invalid +git config rerere.enabled false + +mkdir -p auth +: > auth/__init__.py +cat > auth/session.py <<'PY' +SESSION_TTL_HOURS = 12 + + +def cookie_options(): + return {"httponly": True} +PY +cat > test_session.py <<'PY' +import unittest + +from auth.session import SESSION_TTL_HOURS, cookie_options + + +class SessionTest(unittest.TestCase): + def test_cookie_is_httponly(self): + self.assertTrue(cookie_options()["httponly"]) + + def test_default_ttl(self): + self.assertEqual(SESSION_TTL_HOURS, 12) + + +if __name__ == "__main__": + unittest.main() +PY +git add . +git commit -q -m "feat(auth): session cookie" + +sed -i 's/^SESSION_TTL_HOURS = 12$/SESSION_TTL_HOURS = 12\nREMEMBER_ME_TTL_HOURS = 720/' auth/session.py +git add auth/session.py +git commit -q -m "feat(auth): 30-day sessions for remember-me" -m "Users asked to stay signed in on their own devices. Refs AUTH-77." +REMEMBER=$(git rev-parse HEAD) + +sed -i 's/^REMEMBER_ME_TTL_HOURS = 720$/REMEMBER_ME_TTL_HOURS = 720\nSESSION_COOKIE_NAME = "sid"/' auth/session.py +sed -i 's/return {"httponly": True}/return {"httponly": True, "secure": True}/' auth/session.py +cat >> test_session.py <<'PY' + + +class SecureCookieTest(unittest.TestCase): + def test_cookie_is_secure(self): + self.assertTrue(cookie_options()["secure"]) +PY +git add auth/session.py test_session.py +git commit -q -m "fix(auth): mark the session cookie secure and name it" -m "The cookie went over plain HTTP on the staging redirect. Refs SEC-31." + +git revert --no-edit "$REMEMBER" >/dev/null 2>&1 || true +test -f .git/REVERT_HEAD +git diff --name-only --diff-filter=U | grep -qx auth/session.py From fd78214370f83ee801c15b3653c5456a8192ffa8 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 4 Oct 2026 13:59:47 -0400 Subject: [PATCH 02/19] test(source-control): order history reads before staging the resolution The history-before-edit graders required an Edit call, but every completed run in the interrupted pass, with and without the plugin, rewrote the conflicted file with Write and then ran git add, so the grader failed on "after tool Edit never called". tool_order takes one tool per side, so the after side is now the Bash git add every resolution passes through. Samples updated to match. Co-Authored-By: Claude Opus 5.5 --- .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 49 +++++++++++-------- .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 49 +++++++++++-------- .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 49 +++++++++++-------- .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 49 +++++++++++-------- .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 49 +++++++++++-------- 10 files changed, 145 insertions(+), 110 deletions(-) diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md index 1c3d5cd789..179cdca816 100644 --- a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: tool_order before: { tool: Bash, input_match: 'git\s+(?:-C\s+\S+\s+)?(?:log|show)\b' } -after: Edit +after: { tool: Bash, input_match: 'git\s+(?:-C\s+\S+\s+)?add\b' } --- diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/history-before-edit.json index 9fc33aec85..a2e0bfaeec 100644 --- a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/history-before-edit.json +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/history-before-edit.json @@ -5,21 +5,20 @@ { "tool": "Bash", "input": { - "command": "git log --oneline -5" + "command": "BASE=$(git merge-base HEAD MERGE_HEAD); git log -p \"$BASE\"..MERGE_HEAD -- a.py" } }, { - "tool": "Bash", + "tool": "Write", "input": { - "command": "git show abc123" + "file_path": "/w/a.py", + "content": "y\n" } }, { - "tool": "Edit", + "tool": "Bash", "input": { - "file_path": "/w/a.py", - "old_string": "x", - "new_string": "y" + "command": "git add a.py && GIT_EDITOR=true git merge --continue" } } ] @@ -35,7 +34,7 @@ { "tool": "Bash", "input": { - "command": "git log -p main..HEAD -- a.py" + "command": "git show abc123" } }, { @@ -47,11 +46,9 @@ } }, { - "tool": "Edit", + "tool": "Bash", "input": { - "file_path": "/w/b.py", - "old_string": "x", - "new_string": "y" + "command": "git -C /w add a.py" } } ] @@ -61,11 +58,9 @@ { "answer": [ { - "tool": "Edit", + "tool": "Bash", "input": { - "file_path": "/w/a.py", - "old_string": "x", - "new_string": "y" + "command": "git add a.py" } }, { @@ -75,7 +70,7 @@ } } ], - "why": "edited before reading history" + "why": "staged the resolution before reading history" }, { "answer": [ @@ -86,11 +81,16 @@ } }, { - "tool": "Edit", + "tool": "Write", "input": { "file_path": "/w/a.py", - "old_string": "x", - "new_string": "y" + "content": "y\n" + } + }, + { + "tool": "Bash", + "input": { + "command": "git add a.py" } } ], @@ -103,9 +103,16 @@ "input": { "command": "git log --oneline" } + }, + { + "tool": "Write", + "input": { + "file_path": "/w/a.py", + "content": "y\n" + } } ], - "why": "never edited" + "why": "never staged a resolution" } ] } diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md index 1c3d5cd789..179cdca816 100644 --- a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: tool_order before: { tool: Bash, input_match: 'git\s+(?:-C\s+\S+\s+)?(?:log|show)\b' } -after: Edit +after: { tool: Bash, input_match: 'git\s+(?:-C\s+\S+\s+)?add\b' } --- diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/history-before-edit.json index 9fc33aec85..a2e0bfaeec 100644 --- a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/history-before-edit.json +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/history-before-edit.json @@ -5,21 +5,20 @@ { "tool": "Bash", "input": { - "command": "git log --oneline -5" + "command": "BASE=$(git merge-base HEAD MERGE_HEAD); git log -p \"$BASE\"..MERGE_HEAD -- a.py" } }, { - "tool": "Bash", + "tool": "Write", "input": { - "command": "git show abc123" + "file_path": "/w/a.py", + "content": "y\n" } }, { - "tool": "Edit", + "tool": "Bash", "input": { - "file_path": "/w/a.py", - "old_string": "x", - "new_string": "y" + "command": "git add a.py && GIT_EDITOR=true git merge --continue" } } ] @@ -35,7 +34,7 @@ { "tool": "Bash", "input": { - "command": "git log -p main..HEAD -- a.py" + "command": "git show abc123" } }, { @@ -47,11 +46,9 @@ } }, { - "tool": "Edit", + "tool": "Bash", "input": { - "file_path": "/w/b.py", - "old_string": "x", - "new_string": "y" + "command": "git -C /w add a.py" } } ] @@ -61,11 +58,9 @@ { "answer": [ { - "tool": "Edit", + "tool": "Bash", "input": { - "file_path": "/w/a.py", - "old_string": "x", - "new_string": "y" + "command": "git add a.py" } }, { @@ -75,7 +70,7 @@ } } ], - "why": "edited before reading history" + "why": "staged the resolution before reading history" }, { "answer": [ @@ -86,11 +81,16 @@ } }, { - "tool": "Edit", + "tool": "Write", "input": { "file_path": "/w/a.py", - "old_string": "x", - "new_string": "y" + "content": "y\n" + } + }, + { + "tool": "Bash", + "input": { + "command": "git add a.py" } } ], @@ -103,9 +103,16 @@ "input": { "command": "git log --oneline" } + }, + { + "tool": "Write", + "input": { + "file_path": "/w/a.py", + "content": "y\n" + } } ], - "why": "never edited" + "why": "never staged a resolution" } ] } diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md index 1c3d5cd789..179cdca816 100644 --- a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: tool_order before: { tool: Bash, input_match: 'git\s+(?:-C\s+\S+\s+)?(?:log|show)\b' } -after: Edit +after: { tool: Bash, input_match: 'git\s+(?:-C\s+\S+\s+)?add\b' } --- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/history-before-edit.json index 9fc33aec85..a2e0bfaeec 100644 --- a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/history-before-edit.json +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/history-before-edit.json @@ -5,21 +5,20 @@ { "tool": "Bash", "input": { - "command": "git log --oneline -5" + "command": "BASE=$(git merge-base HEAD MERGE_HEAD); git log -p \"$BASE\"..MERGE_HEAD -- a.py" } }, { - "tool": "Bash", + "tool": "Write", "input": { - "command": "git show abc123" + "file_path": "/w/a.py", + "content": "y\n" } }, { - "tool": "Edit", + "tool": "Bash", "input": { - "file_path": "/w/a.py", - "old_string": "x", - "new_string": "y" + "command": "git add a.py && GIT_EDITOR=true git merge --continue" } } ] @@ -35,7 +34,7 @@ { "tool": "Bash", "input": { - "command": "git log -p main..HEAD -- a.py" + "command": "git show abc123" } }, { @@ -47,11 +46,9 @@ } }, { - "tool": "Edit", + "tool": "Bash", "input": { - "file_path": "/w/b.py", - "old_string": "x", - "new_string": "y" + "command": "git -C /w add a.py" } } ] @@ -61,11 +58,9 @@ { "answer": [ { - "tool": "Edit", + "tool": "Bash", "input": { - "file_path": "/w/a.py", - "old_string": "x", - "new_string": "y" + "command": "git add a.py" } }, { @@ -75,7 +70,7 @@ } } ], - "why": "edited before reading history" + "why": "staged the resolution before reading history" }, { "answer": [ @@ -86,11 +81,16 @@ } }, { - "tool": "Edit", + "tool": "Write", "input": { "file_path": "/w/a.py", - "old_string": "x", - "new_string": "y" + "content": "y\n" + } + }, + { + "tool": "Bash", + "input": { + "command": "git add a.py" } } ], @@ -103,9 +103,16 @@ "input": { "command": "git log --oneline" } + }, + { + "tool": "Write", + "input": { + "file_path": "/w/a.py", + "content": "y\n" + } } ], - "why": "never edited" + "why": "never staged a resolution" } ] } diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md index 1c3d5cd789..179cdca816 100644 --- a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: tool_order before: { tool: Bash, input_match: 'git\s+(?:-C\s+\S+\s+)?(?:log|show)\b' } -after: Edit +after: { tool: Bash, input_match: 'git\s+(?:-C\s+\S+\s+)?add\b' } --- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/history-before-edit.json index 9fc33aec85..a2e0bfaeec 100644 --- a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/history-before-edit.json +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/history-before-edit.json @@ -5,21 +5,20 @@ { "tool": "Bash", "input": { - "command": "git log --oneline -5" + "command": "BASE=$(git merge-base HEAD MERGE_HEAD); git log -p \"$BASE\"..MERGE_HEAD -- a.py" } }, { - "tool": "Bash", + "tool": "Write", "input": { - "command": "git show abc123" + "file_path": "/w/a.py", + "content": "y\n" } }, { - "tool": "Edit", + "tool": "Bash", "input": { - "file_path": "/w/a.py", - "old_string": "x", - "new_string": "y" + "command": "git add a.py && GIT_EDITOR=true git merge --continue" } } ] @@ -35,7 +34,7 @@ { "tool": "Bash", "input": { - "command": "git log -p main..HEAD -- a.py" + "command": "git show abc123" } }, { @@ -47,11 +46,9 @@ } }, { - "tool": "Edit", + "tool": "Bash", "input": { - "file_path": "/w/b.py", - "old_string": "x", - "new_string": "y" + "command": "git -C /w add a.py" } } ] @@ -61,11 +58,9 @@ { "answer": [ { - "tool": "Edit", + "tool": "Bash", "input": { - "file_path": "/w/a.py", - "old_string": "x", - "new_string": "y" + "command": "git add a.py" } }, { @@ -75,7 +70,7 @@ } } ], - "why": "edited before reading history" + "why": "staged the resolution before reading history" }, { "answer": [ @@ -86,11 +81,16 @@ } }, { - "tool": "Edit", + "tool": "Write", "input": { "file_path": "/w/a.py", - "old_string": "x", - "new_string": "y" + "content": "y\n" + } + }, + { + "tool": "Bash", + "input": { + "command": "git add a.py" } } ], @@ -103,9 +103,16 @@ "input": { "command": "git log --oneline" } + }, + { + "tool": "Write", + "input": { + "file_path": "/w/a.py", + "content": "y\n" + } } ], - "why": "never edited" + "why": "never staged a resolution" } ] } diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md index 1c3d5cd789..179cdca816 100644 --- a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: tool_order before: { tool: Bash, input_match: 'git\s+(?:-C\s+\S+\s+)?(?:log|show)\b' } -after: Edit +after: { tool: Bash, input_match: 'git\s+(?:-C\s+\S+\s+)?add\b' } --- diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/history-before-edit.json index 9fc33aec85..a2e0bfaeec 100644 --- a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/history-before-edit.json +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/history-before-edit.json @@ -5,21 +5,20 @@ { "tool": "Bash", "input": { - "command": "git log --oneline -5" + "command": "BASE=$(git merge-base HEAD MERGE_HEAD); git log -p \"$BASE\"..MERGE_HEAD -- a.py" } }, { - "tool": "Bash", + "tool": "Write", "input": { - "command": "git show abc123" + "file_path": "/w/a.py", + "content": "y\n" } }, { - "tool": "Edit", + "tool": "Bash", "input": { - "file_path": "/w/a.py", - "old_string": "x", - "new_string": "y" + "command": "git add a.py && GIT_EDITOR=true git merge --continue" } } ] @@ -35,7 +34,7 @@ { "tool": "Bash", "input": { - "command": "git log -p main..HEAD -- a.py" + "command": "git show abc123" } }, { @@ -47,11 +46,9 @@ } }, { - "tool": "Edit", + "tool": "Bash", "input": { - "file_path": "/w/b.py", - "old_string": "x", - "new_string": "y" + "command": "git -C /w add a.py" } } ] @@ -61,11 +58,9 @@ { "answer": [ { - "tool": "Edit", + "tool": "Bash", "input": { - "file_path": "/w/a.py", - "old_string": "x", - "new_string": "y" + "command": "git add a.py" } }, { @@ -75,7 +70,7 @@ } } ], - "why": "edited before reading history" + "why": "staged the resolution before reading history" }, { "answer": [ @@ -86,11 +81,16 @@ } }, { - "tool": "Edit", + "tool": "Write", "input": { "file_path": "/w/a.py", - "old_string": "x", - "new_string": "y" + "content": "y\n" + } + }, + { + "tool": "Bash", + "input": { + "command": "git add a.py" } } ], @@ -103,9 +103,16 @@ "input": { "command": "git log --oneline" } + }, + { + "tool": "Write", + "input": { + "file_path": "/w/a.py", + "content": "y\n" + } } ], - "why": "never edited" + "why": "never staged a resolution" } ] } From 51db27f8af0e053bba15cb76133400af9c73f4a8 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 9 Oct 2026 20:04:59 +0000 Subject: [PATCH 03/19] test(source-control): give the nothing-invented judge the full expected retry.py The rubric listed only the two sides' changes, so behavior from the shared base (the dataclass, base_delay, the backoff, raise last) was on neither side as written, and the judge failed a correct file in every run. Show the judge the composed file and grade equivalence. The must-fail sample now fails only for its invented cap, and a second one drops the jitter. Co-authored-by: ksextonmelodic --- .../graders/up-nothing-invented.md | 31 +++++++++++++++++-- .../samples/up-nothing-invented.json | 7 ++++- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-nothing-invented.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-nothing-invented.md index ef73cd5ea1..7d7efc46e8 100644 --- a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-nothing-invented.md +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-nothing-invented.md @@ -3,6 +3,33 @@ type: llm focus: { source: file, path: retry.py } --- -PASS if every behavior in this retry.py exists on one of the two sides being combined: main's rename of `max_retries` to `max_attempts` (field and loop bound), and feature's `backoff_jitter` field defaulting to 0.0 plus `random.uniform(0, options.backoff_jitter)` added to the exponential delay. Renaming the feature's uses of the old name to `max_attempts` counts as combining, not inventing. +The judged file is retry.py after a rebase that combines two branches. Both started from the same base file. Main renamed `max_retries` to `max_attempts` (the field and the loop bound). Feature added `import random`, a `backoff_jitter: float = 0.0` field, and `+ random.uniform(0, options.backoff_jitter)` on the exponential delay. Everything else comes unchanged from the shared base, so it counts as both sides' behavior. The correct combination is this file: -FAIL if the file adds anything neither side had: a new option or parameter, a changed default, a different backoff formula, a cap on the delay, logging, or a new exception type. FAIL if either side's behavior is missing. +```python +import random +import time +from dataclasses import dataclass + + +@dataclass +class RetryOptions: + max_attempts: int = 3 + backoff_jitter: float = 0.0 + base_delay: float = 0.1 + + +def retry(fn, options=None, sleep=time.sleep): + options = options or RetryOptions() + last = None + for attempt in range(options.max_attempts): + try: + return fn() + except Exception as exc: + last = exc + sleep(options.base_delay * (2 ** attempt) + random.uniform(0, options.backoff_jitter)) + raise last +``` + +PASS if the judged file behaves the same as that reference. Differences in formatting, field order, import order, blank lines, or comments do not matter. + +FAIL if the judged file adds behavior the reference lacks: a new option or parameter, a changed default, a different backoff formula, a cap on the delay, logging, or a new exception type. Also FAIL if it lacks behavior the reference has: `max_retries` is still present, `max_attempts` is missing, or the `backoff_jitter` field or its `random.uniform` term is missing. diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-nothing-invented.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-nothing-invented.json index bb88d6d519..fc74bdcc96 100644 --- a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-nothing-invented.json +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-nothing-invented.json @@ -6,7 +6,12 @@ ], "fail": [ { - "answer": "import time\nfrom dataclasses import dataclass\n\n\n@dataclass\nclass RetryOptions:\n max_attempts: int = 3\n backoff_jitter: float = 0.0\n max_delay: float = 5.0\n base_delay: float = 0.1\n\n\ndef retry(fn, options=None, sleep=time.sleep):\n options = options or RetryOptions()\n last = None\n for attempt in range(options.max_attempts):\n try:\n return fn()\n except Exception as exc:\n last = exc\n sleep(min(options.max_delay, options.base_delay * (2 ** attempt) + random.uniform(0, options.backoff_jitter)))\n raise last\n" + "answer": "import random\nimport time\nfrom dataclasses import dataclass\n\n\n@dataclass\nclass RetryOptions:\n max_attempts: int = 3\n backoff_jitter: float = 0.0\n max_delay: float = 5.0\n base_delay: float = 0.1\n\n\ndef retry(fn, options=None, sleep=time.sleep):\n options = options or RetryOptions()\n last = None\n for attempt in range(options.max_attempts):\n try:\n return fn()\n except Exception as exc:\n last = exc\n sleep(min(options.max_delay, options.base_delay * (2 ** attempt) + random.uniform(0, options.backoff_jitter)))\n raise last\n", + "why": "invents a max_delay cap neither side had" + }, + { + "answer": "import time\nfrom dataclasses import dataclass\n\n\n@dataclass\nclass RetryOptions:\n max_attempts: int = 3\n base_delay: float = 0.1\n\n\ndef retry(fn, options=None, sleep=time.sleep):\n options = options or RetryOptions()\n last = None\n for attempt in range(options.max_attempts):\n try:\n return fn()\n except Exception as exc:\n last = exc\n sleep(options.base_delay * (2 ** attempt))\n raise last\n", + "why": "drops feature's backoff_jitter" } ] } From 8495e242828e03e3f17cd05d05995c394dc3bbcf Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 9 Oct 2026 20:06:00 +0000 Subject: [PATCH 04/19] test(source-control): grade up-quotes-history on assistant text across the trace Upstream's check is that the agent quotes history at you while resolving, but the graders read only the final message, which is usually a short summary; they failed in every run of both arms. Match the history terms in an assistant text block anywhere in the trace, and never in tool output or a tool call, so running git log alone does not pass. Samples are trace lines, with must-fail cases for each of those. Co-authored-by: ksextonmelodic --- .../graders/up-quotes-history.md | 5 +++-- .../samples/up-quotes-history.json | 18 ++++++++++++++--- .../graders/up-quotes-history.md | 5 +++-- .../samples/up-quotes-history.json | 16 +++++++++++++-- .../graders/up-quotes-history.md | 5 +++-- .../samples/up-quotes-history.json | 18 ++++++++++++++--- .../graders/up-quotes-history.md | 5 +++-- .../samples/up-quotes-history.json | 16 +++++++++++++-- .../graders/up-quotes-history-feature.md | 5 +++-- .../graders/up-quotes-history-main.md | 5 +++-- .../samples/up-quotes-history-feature.json | 20 +++++++++++++++---- .../samples/up-quotes-history-main.json | 20 +++++++++++++++---- .../graders/up-quotes-history.md | 5 +++-- .../samples/up-quotes-history.json | 16 +++++++++++++-- 14 files changed, 125 insertions(+), 34 deletions(-) diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-quotes-history.md b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-quotes-history.md index 77afa7981e..fff8e6079a 100644 --- a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-quotes-history.md +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-quotes-history.md @@ -1,5 +1,6 @@ --- type: regex -pattern: 'SEC-12|\b400\b|stopped accepting|token query auth' -flags: i +pattern: '^\{"type":"assistant".*"type":"text","text":"(?:[^"\\]|\\.)*?(?:SEC-12|\b400\b|stopped accepting|token query auth)' +flags: mi +target: trace --- diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-quotes-history.json b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-quotes-history.json index 56ff1887c9..c7c8c24f46 100644 --- a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-quotes-history.json +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-quotes-history.json @@ -1,7 +1,7 @@ { "pass": [ { - "answer": "main removed legacy_token because /orders now returns 400 for token query auth (SEC-12)." + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"main removed legacy_token because /orders now returns 400 for token query auth (SEC-12).\"},{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git add -- x\"}}]}}" } ], "fail": [ @@ -9,10 +9,22 @@ "answer": "" }, { - "answer": "Kept both sides and finished the merge." + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Kept both sides and finished the merge.\"}]}}" }, { - "answer": "Merged; region filter added." + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Merged; region filter added.\"}]}}" + }, + { + "answer": "{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n refactor(api): drop legacy_token from fetch_orders\\n\\n The /orders endpoint stopped accepting token query auth on 2026-03-01; sending it now returns 400. Refs SEC-12.\\n\"}]}}", + "why": "the history is only in tool output; the agent never quoted it" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"},{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --grep=SEC-12\"}}]}}", + "why": "the ref is in a tool call after a text block, not quoted to the user" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n refactor(api): drop legacy_token from fetch_orders\\n\\n The /orders endpoint stopped accepting token query auth on 2026-03-01; sending it now returns 400. Refs SEC-12.\\n\"}]}}", + "why": "an assistant line, then the history only in the next line's tool output" } ] } diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/up-quotes-history.md b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/up-quotes-history.md index 59e532a939..d0012455dd 100644 --- a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/up-quotes-history.md +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/up-quotes-history.md @@ -1,5 +1,6 @@ --- type: regex -pattern: 'BILL-203|regulator|refunded' -flags: i +pattern: '^\{"type":"assistant".*"type":"text","text":"(?:[^"\\]|\\.)*?(?:BILL-203|regulator|refunded)' +flags: mi +target: trace --- diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/up-quotes-history.json b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/up-quotes-history.json index 5f2b16f493..17691d4e6e 100644 --- a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/up-quotes-history.json +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/up-quotes-history.json @@ -1,7 +1,7 @@ { "pass": [ { - "answer": "The picked commit caps the fee at 50 because regulators require it (BILL-203)." + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"The picked commit caps the fee at 50 because regulators require it (BILL-203).\"},{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git add -- x\"}}]}}" } ], "fail": [ @@ -9,7 +9,19 @@ "answer": "" }, { - "answer": "Resolved billing/rules.py and continued the cherry-pick." + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Resolved billing/rules.py and continued the cherry-pick.\"}]}}" + }, + { + "answer": "{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n fix(billing): cap the late fee at 50\\n\\n Regulators cap late fees at 50 per invoice; uncapped fees on large balances were being refunded by hand. Refs BILL-203.\\n\"}]}}", + "why": "the history is only in tool output; the agent never quoted it" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"},{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --grep=BILL-203\"}}]}}", + "why": "the ref is in a tool call after a text block, not quoted to the user" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n fix(billing): cap the late fee at 50\\n\\n Regulators cap late fees at 50 per invoice; uncapped fees on large balances were being refunded by hand. Refs BILL-203.\\n\"}]}}", + "why": "an assistant line, then the history only in the next line's tool output" } ] } diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/up-quotes-history.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/up-quotes-history.md index 54781d7ee1..1016170b57 100644 --- a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/up-quotes-history.md +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/up-quotes-history.md @@ -1,5 +1,6 @@ --- type: regex -pattern: 'OPS-311|failover|1 in 40' -flags: i +pattern: '^\{"type":"assistant".*"type":"text","text":"(?:[^"\\]|\\.)*?(?:OPS-311|failover|1 in 40)' +flags: mi +target: trace --- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/up-quotes-history.json b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/up-quotes-history.json index f265062961..96309dad12 100644 --- a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/up-quotes-history.json +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/up-quotes-history.json @@ -1,7 +1,7 @@ { "pass": [ { - "answer": "The incoming side raised attempts to 5 to ride out the gateway's nightly failover (OPS-311, first commit on ops/tuning)." + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"The incoming side raised attempts to 5 to ride out the gateway's nightly failover (OPS-311, first commit on ops/tuning).\"},{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git add -- x\"}}]}}" } ], "fail": [ @@ -9,10 +9,22 @@ "answer": "" }, { - "answer": "The incoming tip only changed the Makefile lint target." + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"The incoming tip only changed the Makefile lint target.\"}]}}" }, { - "answer": "Resolved and merged." + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Resolved and merged.\"}]}}" + }, + { + "answer": "{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n ops: allow five retry attempts\\n\\n The payments gateway drops about 1 in 40 calls during its nightly failover; three attempts were not enough to ride it out. Refs OPS-311.\\n\"}]}}", + "why": "the history is only in tool output; the agent never quoted it" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"},{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --grep=OPS-311\"}}]}}", + "why": "the ref is in a tool call after a text block, not quoted to the user" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n ops: allow five retry attempts\\n\\n The payments gateway drops about 1 in 40 calls during its nightly failover; three attempts were not enough to ride it out. Refs OPS-311.\\n\"}]}}", + "why": "an assistant line, then the history only in the next line's tool output" } ] } diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/up-quotes-history.md b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/up-quotes-history.md index 17a1b8ef9f..fd359c6827 100644 --- a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/up-quotes-history.md +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/up-quotes-history.md @@ -1,5 +1,6 @@ --- type: regex -pattern: 'UI-12|HTML|misleading' -flags: i +pattern: '^\{"type":"assistant".*"type":"text","text":"(?:[^"\\]|\\.)*?(?:UI-12|HTML|misleading)' +flags: mi +target: trace --- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/up-quotes-history.json b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/up-quotes-history.json index 0ee865cbf9..7a67b12071 100644 --- a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/up-quotes-history.json +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/up-quotes-history.json @@ -1,7 +1,7 @@ { "pass": [ { - "answer": "main renamed it because users now render as HTML (UI-12)" + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"main renamed it because users now render as HTML (UI-12)\"},{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git add -- x\"}}]}}" } ], "fail": [ @@ -9,7 +9,19 @@ "answer": "" }, { - "answer": "Fixed the call and concluded the merge." + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Fixed the call and concluded the merge.\"}]}}" + }, + { + "answer": "{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n refactor: rename format_user to render_user\\n\\n The UI now renders users as HTML, so format was misleading. Refs UI-12.\\n\"}]}}", + "why": "the history is only in tool output; the agent never quoted it" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"},{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --grep=UI-12\"}}]}}", + "why": "the ref is in a tool call after a text block, not quoted to the user" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n refactor: rename format_user to render_user\\n\\n The UI now renders users as HTML, so format was misleading. Refs UI-12.\\n\"}]}}", + "why": "an assistant line, then the history only in the next line's tool output" } ] } diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-feature.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-feature.md index 6dd688090c..6fb92515b1 100644 --- a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-feature.md +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-feature.md @@ -1,5 +1,6 @@ --- type: regex -pattern: 'RET-57|lockstep|spread' -flags: i +pattern: '^\{"type":"assistant".*"type":"text","text":"(?:[^"\\]|\\.)*?(?:RET-57|lockstep|spread)' +flags: mi +target: trace --- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-main.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-main.md index 1a33776513..a3bb0927ed 100644 --- a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-main.md +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-main.md @@ -1,5 +1,6 @@ --- type: regex -pattern: 'RET-41|off[- ]by[- ]one|includ\w* the first|one more call' -flags: i +pattern: '^\{"type":"assistant".*"type":"text","text":"(?:[^"\\]|\\.)*?(?:RET-41|off[- ]by[- ]one|includ\w* the first|one more call)' +flags: mi +target: trace --- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-quotes-history-feature.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-quotes-history-feature.json index 868c6f0ed2..cafbb0b0bf 100644 --- a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-quotes-history-feature.json +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-quotes-history-feature.json @@ -1,10 +1,10 @@ { "pass": [ { - "answer": "your commit adds jitter so clients stop retrying in lockstep (RET-57)" + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"your commit adds jitter so clients stop retrying in lockstep (RET-57)\"},{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git add -- x\"}}]}}" }, { - "answer": "jitter spreads clients out" + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"jitter spreads clients out\"}]}}" } ], "fail": [ @@ -12,10 +12,22 @@ "answer": "" }, { - "answer": "Added backoff_jitter." + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Added backoff_jitter.\"}]}}" }, { - "answer": "Rebase complete." + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Rebase complete.\"}]}}" + }, + { + "answer": "{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n feat(retry): add backoff_jitter option\\n\\n Clients that fail together retry together and hit the API in lockstep; a random extra delay spreads them out. Refs RET-57.\\n\"}]}}", + "why": "the history is only in tool output; the agent never quoted it" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"},{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --grep=RET-57\"}}]}}", + "why": "the ref is in a tool call after a text block, not quoted to the user" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n feat(retry): add backoff_jitter option\\n\\n Clients that fail together retry together and hit the API in lockstep; a random extra delay spreads them out. Refs RET-57.\\n\"}]}}", + "why": "an assistant line, then the history only in the next line's tool output" } ] } diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-quotes-history-main.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-quotes-history-main.json index f33767de8a..0b8313c07c 100644 --- a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-quotes-history-main.json +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-quotes-history-main.json @@ -1,10 +1,10 @@ { "pass": [ { - "answer": "main renamed it because the count includes the first call (RET-41)" + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"main renamed it because the count includes the first call (RET-41)\"},{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git add -- x\"}}]}}" }, { - "answer": "Per RET-41 the old name was off by one" + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Per RET-41 the old name was off by one\"}]}}" } ], "fail": [ @@ -12,10 +12,22 @@ "answer": "" }, { - "answer": "Resolved the conflict and finished the rebase." + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Resolved the conflict and finished the rebase.\"}]}}" }, { - "answer": "Kept both max_attempts and backoff_jitter." + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Kept both max_attempts and backoff_jitter.\"}]}}" + }, + { + "answer": "{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n refactor(retry): rename max_retries to max_attempts\\n\\n The count includes the first call, so every caller read max_retries as one more call than it makes. Refs RET-41.\\n\"}]}}", + "why": "the history is only in tool output; the agent never quoted it" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"},{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --grep=RET-41\"}}]}}", + "why": "the ref is in a tool call after a text block, not quoted to the user" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n refactor(retry): rename max_retries to max_attempts\\n\\n The count includes the first call, so every caller read max_retries as one more call than it makes. Refs RET-41.\\n\"}]}}", + "why": "an assistant line, then the history only in the next line's tool output" } ] } diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/up-quotes-history.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/up-quotes-history.md index 80aaab0cfe..5d0e097d29 100644 --- a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/up-quotes-history.md +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/up-quotes-history.md @@ -1,5 +1,6 @@ --- type: regex -pattern: 'AUTH-77|remember[- ]me|30-day|stay signed in' -flags: i +pattern: '^\{"type":"assistant".*"type":"text","text":"(?:[^"\\]|\\.)*?(?:AUTH-77|remember[- ]me|30-day|stay signed in)' +flags: mi +target: trace --- diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/up-quotes-history.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/up-quotes-history.json index 68a4256d82..382f00f7bc 100644 --- a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/up-quotes-history.json +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/up-quotes-history.json @@ -1,7 +1,7 @@ { "pass": [ { - "answer": "The reverted commit added 30-day remember-me sessions (AUTH-77); the later SEC-31 fix stays." + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"The reverted commit added 30-day remember-me sessions (AUTH-77); the later SEC-31 fix stays.\"},{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git add -- x\"}}]}}" } ], "fail": [ @@ -9,7 +9,19 @@ "answer": "" }, { - "answer": "Resolved auth/session.py and continued the revert." + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Resolved auth/session.py and continued the revert.\"}]}}" + }, + { + "answer": "{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n feat(auth): 30-day sessions for remember-me\\n\\n Users asked to stay signed in on their own devices. Refs AUTH-77.\\n\"}]}}", + "why": "the history is only in tool output; the agent never quoted it" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"},{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --grep=AUTH-77\"}}]}}", + "why": "the ref is in a tool call after a text block, not quoted to the user" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n feat(auth): 30-day sessions for remember-me\\n\\n Users asked to stay signed in on their own devices. Refs AUTH-77.\\n\"}]}}", + "why": "an assistant line, then the history only in the next line's tool output" } ] } From f3bef9adeee7c2a8f528e9bc06f4fc39839528aa Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 9 Oct 2026 20:06:28 +0000 Subject: [PATCH 05/19] test(source-control): word two prompts as an unresolved conflict so the skill fires The validity gate failed because the skill fired in 1 of 3 with-arm runs of merge-semantic-sweep-finds-stale-call and 0 of 3 of refuses-bulk-ours-and-abort. Every case that fired describes a stopped operation still being resolved; these two read as already resolved ("wrap up the merge") or as an order to run one command. Reword both as a merge conflict still being resolved and keep what each case tests: the stale call no marker shows, and the bulk --ours or abort request. Co-authored-by: ksextonmelodic --- .../prompt.md | 2 +- .../resolve-conflicts-refuses-bulk-ours-and-abort/prompt.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/prompt.md b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/prompt.md index 13a3c4c2be..e6efa3082e 100644 --- a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/prompt.md +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/prompt.md @@ -8,4 +8,4 @@ allowed_tools: [Read, Glob, Grep, Skill, Bash, Edit] expected_outcome: "reports.py calls render_user and no file calls format_user, tests ran before the merge was concluded, the merge commit exists, and nothing was staged with git add -A or ." --- -I merged main into my branch and resolved the one conflicted file. The markers are gone and the diff looks clean. Main renamed format_user() to render_user() everywhere, and my branch added a new call to format_user() in a file the merge didn't flag. Wrap up the merge. +I'm partway through resolving a merge conflict from merging main into my branch. I fixed the one conflicted file, the markers are gone, and the diff looks clean. Main renamed format_user() to render_user() everywhere, and my branch added a new call to format_user() in a file the merge didn't flag. Finish resolving the merge and conclude it. diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/prompt.md b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/prompt.md index 9a2d3e2989..9afef48690 100644 --- a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/prompt.md +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/prompt.md @@ -8,4 +8,4 @@ allowed_tools: [Read, Glob, Grep, Skill, Bash, Edit] expected_outcome: "No checkout --ours/--theirs, -X ours/theirs, or --abort; the six unmerged paths are listed; each file gets a justified resolution or the reply stops and asks with the evidence" --- -This merge has conflicts in six files and I don't have time to dig in. Just run git checkout --ours on all of them and finish the merge, or abort it if that's easier. +This merge stopped with conflicts in six files and I don't have time to resolve them properly. Just run git checkout --ours on all of them and finish the merge, or abort it if that's easier. From 9a2154327985d40139ef7cc8bd2d78f87c7e4315 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 9 Oct 2026 20:06:52 +0000 Subject: [PATCH 06/19] test(source-control): label every branch of the justified-per-file rubric The judge's samples covered one PASS branch and two FAIL clauses. Add a must-pass stop-and-ask reply naming the six files and each side's change, and must-fail replies that resolve without saying what either side changed or drop every EU_* line with no evidence, so calibration exercises the whole rubric. Co-authored-by: ksextonmelodic --- .../samples/justified-per-file.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/justified-per-file.json b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/justified-per-file.json index ca3d34e7bc..27eccdebf1 100644 --- a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/justified-per-file.json +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/justified-per-file.json @@ -2,6 +2,10 @@ "pass": [ { "answer": "I did not run `git checkout --ours`: it would delete the EU_* overrides eu-regions added (EU-4). All six files compose cleanly: each keeps main's doubled default (PERF-9) and the incoming EU_* = 3 line.\n\n| file | resolution | why |\n|---|---|---|\n| settings/timeouts.py | composed | DEFAULT_TIMEOUTS = 2 + EU_TIMEOUTS = 3 |\n| settings/limits.py | composed | same |\n| settings/features.py | composed | same |\n| settings/regions.py | composed | same |\n| settings/logging.py | composed | same |\n| settings/cache.py | composed | same |\n\nTests pass; merge concluded with git merge --continue." + }, + { + "answer": "I haven't run `git checkout --ours` or aborted: --ours would silently drop the EU_* overrides the eu-regions branch added for the Frankfurt launch (EU-4). Here is what each side changed:\n\n| file | main (PERF-9) | eu-regions (EU-4) |\n|---|---|---|\n| settings/timeouts.py | DEFAULT_TIMEOUTS 1 -> 2 | adds EU_TIMEOUTS = 3 |\n| settings/limits.py | DEFAULT_LIMITS 1 -> 2 | adds EU_LIMITS = 3 |\n| settings/features.py | DEFAULT_FEATURES 1 -> 2 | adds EU_FEATURES = 3 |\n| settings/regions.py | DEFAULT_REGIONS 1 -> 2 | adds EU_REGIONS = 3 |\n| settings/logging.py | DEFAULT_LOGGING 1 -> 2 | adds EU_LOGGING = 3 |\n| settings/cache.py | DEFAULT_CACHE 1 -> 2 | adds EU_CACHE = 3 |\n\nEvery file can keep both changes. Should I compose all six that way and conclude the merge, or do you want to abandon it?", + "why": "branch (b): stops and asks, naming the six files and each side's change" } ], "fail": [ @@ -10,6 +14,14 @@ }, { "answer": "Aborted the merge as requested; your branch is unchanged." + }, + { + "answer": "Resolved all six conflicted files and committed the merge. Tests pass.", + "why": "resolved without saying what either side changed" + }, + { + "answer": "I didn't use --ours or abort. I resolved each file by keeping main's doubled default:\n\n- settings/timeouts.py: DEFAULT_TIMEOUTS = 2\n- settings/limits.py: DEFAULT_LIMITS = 2\n- settings/features.py: DEFAULT_FEATURES = 2\n- settings/regions.py: DEFAULT_REGIONS = 2\n- settings/logging.py: DEFAULT_LOGGING = 2\n- settings/cache.py: DEFAULT_CACHE = 2\n\nMerge concluded.", + "why": "drops every EU_* line with no evidence" } ] } From 36f42512ab429caadf1ed268f3ef8d48bd903c58 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 9 Oct 2026 20:07:45 +0000 Subject: [PATCH 07/19] test(source-control): clear the two typos findings in the resolve-conflicts suite Spell out includes/including in the RET-41 pattern instead of the includ\w* stem, which matches the same replies, and use behavior in a scaffold commit message no grader reads. Co-authored-by: ksextonmelodic --- .../scaffold.sh | 2 +- .../graders/up-quotes-history-main.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/scaffold.sh b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/scaffold.sh index 8005c2eb5c..5ee0e51001 100755 --- a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/scaffold.sh +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/scaffold.sh @@ -34,7 +34,7 @@ git add config/retry.yaml git commit -q -m "ops: allow five retry attempts" -m "The payments gateway drops about 1 in 40 calls during its nightly failover; three attempts were not enough to ride it out. Refs OPS-311." printf '# ops\n\nRetries ride out the nightly failover.\n' > docs/ops.md git add docs/ops.md -git commit -q -m "docs: note retry behaviour" +git commit -q -m "docs: note retry behavior" printf 'lint:\n\t@echo lint --strict\n' > Makefile git add Makefile git commit -q -m "chore: strict lint target" diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-main.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-main.md index a3bb0927ed..dfd049570d 100644 --- a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-main.md +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-main.md @@ -1,6 +1,6 @@ --- type: regex -pattern: '^\{"type":"assistant".*"type":"text","text":"(?:[^"\\]|\\.)*?(?:RET-41|off[- ]by[- ]one|includ\w* the first|one more call)' +pattern: '^\{"type":"assistant".*"type":"text","text":"(?:[^"\\]|\\.)*?(?:RET-41|off[- ]by[- ]one|(?:includes?|including) the first|one more call)' flags: mi target: trace --- From e30d897b53dbf43ad040e7254bd6bb5ae24580b1 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 9 Oct 2026 20:07:45 +0000 Subject: [PATCH 08/19] docs(source-control): say the fragment's history grader orders reads before staging fd7821437 moved the history-before-edit graders' after-step from the first Edit to git add, so the entry's "before the first edit" no longer held. Co-authored-by: ksextonmelodic --- .../test-resolve-conflicts-plugin-eval-suite-ee7a239e.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/source-control/test-resolve-conflicts-plugin-eval-suite-ee7a239e.md b/.changes/source-control/test-resolve-conflicts-plugin-eval-suite-ee7a239e.md index 5de4962ea6..e510e007eb 100644 --- a/.changes/source-control/test-resolve-conflicts-plugin-eval-suite-ee7a239e.md +++ b/.changes/source-control/test-resolve-conflicts-plugin-eval-suite-ee7a239e.md @@ -4,4 +4,4 @@ bump: patch ### Added -- **A `claude plugin eval` suite for `/source-control:resolve-conflicts`.** Seven cases tagged `row22`, each scaffolding a git repository that is stopped in a real conflict: a two-stop rebase, a six-file merge where the prompt asks for `--ours` or abort, a merge with a conflict that has no markers, a cherry-pick, a merge whose incoming tip never touched the file, a revert, and a merge with only "sort it out" as the prompt. Most graders are deterministic: the resolved file contents, history read before the first edit, tests run before the operation concludes, and the concluding reflog line. They also cover the "It's working if" checks from the archived upstream skill page. Three cases add one model-judged grader each, and every grader carries pass and fail samples. The skill body is unchanged. +- **A `claude plugin eval` suite for `/source-control:resolve-conflicts`.** Seven cases tagged `row22`, each scaffolding a git repository that is stopped in a real conflict: a two-stop rebase, a six-file merge where the prompt asks for `--ours` or abort, a merge with a conflict that has no markers, a cherry-pick, a merge whose incoming tip never touched the file, a revert, and a merge with only "sort it out" as the prompt. Most graders are deterministic: the resolved file contents, history read before the resolution is staged, tests run before the operation concludes, and the concluding reflog line. They also cover the "It's working if" checks from the archived upstream skill page. Three cases add one model-judged grader each, and every grader carries pass and fail samples. The skill body is unchanged. From bdf81d7564aa1e636beb34cfbe9e28e3ac781953 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 9 Oct 2026 20:10:30 +0000 Subject: [PATCH 09/19] test(source-control): mark the rebase scaffold's literal backticks for ShellCheck lint-shell reports every severity, so the info-level SC2016 on the README sed would fail it once the pull request leaves draft. Co-authored-by: ksextonmelodic --- .../scaffold.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/scaffold.sh b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/scaffold.sh index 95d2662751..8035cb3d2f 100755 --- a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/scaffold.sh +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/scaffold.sh @@ -108,6 +108,7 @@ git commit -q -m "docs(retry): document backoff_jitter" git checkout -q main sed -i 's/max_retries/max_attempts/g' retry.py test_retry.py +# shellcheck disable=SC2016 # markdown code spans; the backticks are literal sed -i 's/- `max_retries`: how many times to call the function./- `max_attempts`: how many calls in total, including the first./' README.md git add retry.py test_retry.py README.md git commit -q -m "refactor(retry): rename max_retries to max_attempts" -m "The count includes the first call, so every caller read max_retries as one more call than it makes. Refs RET-41." From d02533fa27547d6708bf1c8f46964afa53cddde0 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 9 Oct 2026 20:18:39 +0000 Subject: [PATCH 10/19] test(source-control): catch wildcard staging after -- or git -C The grader required the wildcard right after `add`, so `git add -- .`, `git -C add -A` and `git add --verbose --all` staged the whole tree and still passed. Allow options, the pathspec separator and -C, and add must-fail samples for each plus :/. Co-authored-by: ksextonmelodic --- .../graders/no-wildcard-staging.md | 2 +- .../samples/no-wildcard-staging.json | 72 +++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/no-wildcard-staging.md b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/no-wildcard-staging.md index 3bb0a707c1..4d2eceaa93 100644 --- a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/no-wildcard-staging.md +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/no-wildcard-staging.md @@ -1,7 +1,7 @@ --- type: tool_used tool: Bash -input_match: 'git add\s+(?:-A|--all|\.)(?![\w./-])' +input_match: '\bgit\s+(?:-C\s+\S+\s+)?add(?:\s+-{1,2}[\w-]+)*?\s+(?:-A\b|--all\b|(?:--\s+)?(?:\.|:/)(?![\w./-]))' min: 0 max: 0 arm: both diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/no-wildcard-staging.json b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/no-wildcard-staging.json index 92702f4b23..edf825fb98 100644 --- a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/no-wildcard-staging.json +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/no-wildcard-staging.json @@ -20,6 +20,37 @@ } ], "why": "explicit path" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git add -- reports.py" + } + } + ], + "why": "explicit path after the separator" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git -C /w add reports.py" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git add -p reports.py" + } + } + ] } ], "fail": [ @@ -42,6 +73,47 @@ } } ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git add -- ." + } + } + ], + "why": "the whole tree after the pathspec separator" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git -C /w add -A" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git add --verbose --all" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git add :/" + } + } + ] } ] } From 9efa00423e0dc5b51480e294aeb7af9059d8c2cb Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 9 Oct 2026 20:19:05 +0000 Subject: [PATCH 11/19] test(source-control): require a history read of the pseudoref, not a mention The contract graders matched CHERRY_PICK_HEAD, REBASE_HEAD or REVERT_HEAD anywhere in a Bash call, so `git rev-parse --verify CHERRY_PICK_HEAD` passed without the picked commit being read. Match only git show, log or diff naming the pseudoref in the same simple command. The cherry-pick sample that used the pseudoref only for merge-base moves to must-fail. Co-authored-by: ksextonmelodic --- .../contract-reads-cherry-pick-head.md | 2 +- .../contract-reads-cherry-pick-head.json | 51 ++++++++++++++++++- .../graders/contract-reads-rebase-head.md | 2 +- .../samples/contract-reads-rebase-head.json | 48 +++++++++++++++++ .../graders/contract-reads-revert-head.md | 2 +- .../samples/contract-reads-revert-head.json | 48 +++++++++++++++++ 6 files changed, 149 insertions(+), 4 deletions(-) diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/contract-reads-cherry-pick-head.md b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/contract-reads-cherry-pick-head.md index 8bf4b09800..85afb2f66a 100644 --- a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/contract-reads-cherry-pick-head.md +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/contract-reads-cherry-pick-head.md @@ -1,5 +1,5 @@ --- type: tool_used tool: Bash -input_match: 'CHERRY_PICK_HEAD' +input_match: '\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:show|log|diff)\b[^;&|\n]*\bCHERRY_PICK_HEAD\b' --- diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/contract-reads-cherry-pick-head.json b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/contract-reads-cherry-pick-head.json index ba6979a036..7130c2281b 100644 --- a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/contract-reads-cherry-pick-head.json +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/contract-reads-cherry-pick-head.json @@ -15,7 +15,17 @@ { "tool": "Bash", "input": { - "command": "BASE=$(git merge-base HEAD CHERRY_PICK_HEAD) && git log --oneline \"$BASE\"..HEAD -- billing/rules.py" + "command": "git -C /w log -p -1 CHERRY_PICK_HEAD -- billing/rules.py" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git diff CHERRY_PICK_HEAD~1 CHERRY_PICK_HEAD -- billing/rules.py" } } ] @@ -34,6 +44,45 @@ } } ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "BASE=$(git merge-base HEAD CHERRY_PICK_HEAD) && git log --oneline \"$BASE\"..HEAD -- billing/rules.py" + } + } + ], + "why": "names the pseudoref only to find the base, then reads HEAD's side" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log --all --oneline" + } + }, + { + "tool": "Bash", + "input": { + "command": "git rev-parse --verify CHERRY_PICK_HEAD" + } + } + ], + "why": "confirms the pseudoref exists without reading its commit" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "test -f .git/CHERRY_PICK_HEAD && echo yes" + } + } + ], + "why": "checks the stop, reads no history" } ] } diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/contract-reads-rebase-head.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/contract-reads-rebase-head.md index 20d283439d..3977d8f151 100644 --- a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/contract-reads-rebase-head.md +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/contract-reads-rebase-head.md @@ -1,5 +1,5 @@ --- type: tool_used tool: Bash -input_match: 'REBASE_HEAD' +input_match: '\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:show|log|diff)\b[^;&|\n]*\bREBASE_HEAD\b' --- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/contract-reads-rebase-head.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/contract-reads-rebase-head.json index 0d15f38e0c..5b8832fa55 100644 --- a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/contract-reads-rebase-head.json +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/contract-reads-rebase-head.json @@ -19,6 +19,26 @@ } } ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git -C /w log -p -1 REBASE_HEAD -- retry.py" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git diff REBASE_HEAD~1 REBASE_HEAD -- retry.py" + } + } + ] } ], "fail": [ @@ -34,6 +54,34 @@ } } ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log --all --oneline" + } + }, + { + "tool": "Bash", + "input": { + "command": "git rev-parse --verify REBASE_HEAD" + } + } + ], + "why": "confirms the pseudoref exists without reading its commit" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "test -f .git/REBASE_HEAD && echo yes" + } + } + ], + "why": "checks the stop, reads no history" } ] } diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/contract-reads-revert-head.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/contract-reads-revert-head.md index 0286e946b7..bec9c21ade 100644 --- a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/contract-reads-revert-head.md +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/contract-reads-revert-head.md @@ -1,5 +1,5 @@ --- type: tool_used tool: Bash -input_match: 'REVERT_HEAD' +input_match: '\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:show|log|diff)\b[^;&|\n]*\bREVERT_HEAD\b' --- diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/contract-reads-revert-head.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/contract-reads-revert-head.json index fe159d9b08..c54db4fb3b 100644 --- a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/contract-reads-revert-head.json +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/contract-reads-revert-head.json @@ -19,6 +19,26 @@ } } ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git -C /w log -p -1 REVERT_HEAD -- auth/session.py" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git diff REVERT_HEAD~1 REVERT_HEAD -- auth/session.py" + } + } + ] } ], "fail": [ @@ -34,6 +54,34 @@ } } ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log --all --oneline" + } + }, + { + "tool": "Bash", + "input": { + "command": "git rev-parse --verify REVERT_HEAD" + } + } + ], + "why": "confirms the pseudoref exists without reading its commit" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "test -f .git/REVERT_HEAD && echo yes" + } + } + ], + "why": "checks the stop, reads no history" } ] } From 465f7ccecdb96ef185decca762d37d96bd9771ea Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 9 Oct 2026 20:20:28 +0000 Subject: [PATCH 12/19] test(source-control): order history reads before the first edit, not before staging fd7821437 moved the after-step to git add because runs rewrote the file with Write, not Edit, and tool_order takes one tool per side. That let a run edit first, read history, then stage, and pass. Grade the trace instead: from its start, a git log or git show call must come before any Edit, Write, MultiEdit or NotebookEdit call and before a Bash rewrite or side pick (sed -i, perl -i, cat >, tee, checkout or restore --ours or --theirs). Samples are trace lines. Co-authored-by: ksextonmelodic --- .../graders/history-before-edit.md | 6 +- .../samples/history-before-edit.json | 127 ++++-------------- .../graders/history-before-edit.md | 6 +- .../samples/history-before-edit.json | 127 ++++-------------- .../graders/history-before-edit.md | 6 +- .../samples/history-before-edit.json | 127 ++++-------------- .../graders/history-before-edit.md | 6 +- .../samples/history-before-edit.json | 127 ++++-------------- .../graders/history-before-edit.md | 6 +- .../samples/history-before-edit.json | 127 ++++-------------- 10 files changed, 145 insertions(+), 520 deletions(-) diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md index 179cdca816..464a2636ab 100644 --- a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- -type: tool_order -before: { tool: Bash, input_match: 'git\s+(?:-C\s+\S+\s+)?(?:log|show)\b' } -after: { tool: Bash, input_match: 'git\s+(?:-C\s+\S+\s+)?add\b' } +type: regex +pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|\bcat\s*>|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' +target: trace --- diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/history-before-edit.json index a2e0bfaeec..ad68a30691 100644 --- a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/history-before-edit.json +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/history-before-edit.json @@ -1,118 +1,43 @@ { "pass": [ { - "answer": [ - { - "tool": "Bash", - "input": { - "command": "BASE=$(git merge-base HEAD MERGE_HEAD); git log -p \"$BASE\"..MERGE_HEAD -- a.py" - } - }, - { - "tool": "Write", - "input": { - "file_path": "/w/a.py", - "content": "y\n" - } - }, - { - "tool": "Bash", - "input": { - "command": "git add a.py && GIT_EDITOR=true git merge --continue" - } - } - ] + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"BASE=$(git merge-base HEAD MERGE_HEAD); git log -p \\\"$BASE\\\"..MERGE_HEAD -- a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/a.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}" }, { - "answer": [ - { - "tool": "Bash", - "input": { - "command": "git status" - } - }, - { - "tool": "Bash", - "input": { - "command": "git show abc123" - } - }, - { - "tool": "Edit", - "input": { - "file_path": "/w/a.py", - "old_string": "x", - "new_string": "y" - } - }, - { - "tool": "Bash", - "input": { - "command": "git -C /w add a.py" - } - } - ] + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/a.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_3\",\"name\":\"Bash\",\"input\":{\"command\":\"git -C /w add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_3\",\"content\":\"ok\"}]}}" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git --no-pager log --oneline -5 -- a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "history read; nothing edited yet" } ], "fail": [ { - "answer": [ - { - "tool": "Bash", - "input": { - "command": "git add a.py" - } - }, - { - "tool": "Bash", - "input": { - "command": "git log --oneline" - } - } - ], - "why": "staged the resolution before reading history" + "answer": "" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/a.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", + "why": "rewrote the file before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/a.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "edited before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log -p -- a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "rewrote the file with Bash before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git checkout --theirs a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "took a side before reading history" }, { - "answer": [ - { - "tool": "Bash", - "input": { - "command": "git diff" - } - }, - { - "tool": "Write", - "input": { - "file_path": "/w/a.py", - "content": "y\n" - } - }, - { - "tool": "Bash", - "input": { - "command": "git add a.py" - } - } - ], + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git diff\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/a.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", "why": "diff only, no history" }, { - "answer": [ - { - "tool": "Bash", - "input": { - "command": "git log --oneline" - } - }, - { - "tool": "Write", - "input": { - "file_path": "/w/a.py", - "content": "y\n" - } - } - ], - "why": "never staged a resolution" + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_9\",\"content\":\"try git log next\"}]}}", + "why": "git log only in tool output" } ] } diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md index 179cdca816..464a2636ab 100644 --- a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- -type: tool_order -before: { tool: Bash, input_match: 'git\s+(?:-C\s+\S+\s+)?(?:log|show)\b' } -after: { tool: Bash, input_match: 'git\s+(?:-C\s+\S+\s+)?add\b' } +type: regex +pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|\bcat\s*>|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' +target: trace --- diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/history-before-edit.json index a2e0bfaeec..ad68a30691 100644 --- a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/history-before-edit.json +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/history-before-edit.json @@ -1,118 +1,43 @@ { "pass": [ { - "answer": [ - { - "tool": "Bash", - "input": { - "command": "BASE=$(git merge-base HEAD MERGE_HEAD); git log -p \"$BASE\"..MERGE_HEAD -- a.py" - } - }, - { - "tool": "Write", - "input": { - "file_path": "/w/a.py", - "content": "y\n" - } - }, - { - "tool": "Bash", - "input": { - "command": "git add a.py && GIT_EDITOR=true git merge --continue" - } - } - ] + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"BASE=$(git merge-base HEAD MERGE_HEAD); git log -p \\\"$BASE\\\"..MERGE_HEAD -- a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/a.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}" }, { - "answer": [ - { - "tool": "Bash", - "input": { - "command": "git status" - } - }, - { - "tool": "Bash", - "input": { - "command": "git show abc123" - } - }, - { - "tool": "Edit", - "input": { - "file_path": "/w/a.py", - "old_string": "x", - "new_string": "y" - } - }, - { - "tool": "Bash", - "input": { - "command": "git -C /w add a.py" - } - } - ] + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/a.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_3\",\"name\":\"Bash\",\"input\":{\"command\":\"git -C /w add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_3\",\"content\":\"ok\"}]}}" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git --no-pager log --oneline -5 -- a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "history read; nothing edited yet" } ], "fail": [ { - "answer": [ - { - "tool": "Bash", - "input": { - "command": "git add a.py" - } - }, - { - "tool": "Bash", - "input": { - "command": "git log --oneline" - } - } - ], - "why": "staged the resolution before reading history" + "answer": "" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/a.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", + "why": "rewrote the file before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/a.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "edited before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log -p -- a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "rewrote the file with Bash before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git checkout --theirs a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "took a side before reading history" }, { - "answer": [ - { - "tool": "Bash", - "input": { - "command": "git diff" - } - }, - { - "tool": "Write", - "input": { - "file_path": "/w/a.py", - "content": "y\n" - } - }, - { - "tool": "Bash", - "input": { - "command": "git add a.py" - } - } - ], + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git diff\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/a.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", "why": "diff only, no history" }, { - "answer": [ - { - "tool": "Bash", - "input": { - "command": "git log --oneline" - } - }, - { - "tool": "Write", - "input": { - "file_path": "/w/a.py", - "content": "y\n" - } - } - ], - "why": "never staged a resolution" + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_9\",\"content\":\"try git log next\"}]}}", + "why": "git log only in tool output" } ] } diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md index 179cdca816..464a2636ab 100644 --- a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- -type: tool_order -before: { tool: Bash, input_match: 'git\s+(?:-C\s+\S+\s+)?(?:log|show)\b' } -after: { tool: Bash, input_match: 'git\s+(?:-C\s+\S+\s+)?add\b' } +type: regex +pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|\bcat\s*>|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' +target: trace --- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/history-before-edit.json index a2e0bfaeec..ad68a30691 100644 --- a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/history-before-edit.json +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/history-before-edit.json @@ -1,118 +1,43 @@ { "pass": [ { - "answer": [ - { - "tool": "Bash", - "input": { - "command": "BASE=$(git merge-base HEAD MERGE_HEAD); git log -p \"$BASE\"..MERGE_HEAD -- a.py" - } - }, - { - "tool": "Write", - "input": { - "file_path": "/w/a.py", - "content": "y\n" - } - }, - { - "tool": "Bash", - "input": { - "command": "git add a.py && GIT_EDITOR=true git merge --continue" - } - } - ] + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"BASE=$(git merge-base HEAD MERGE_HEAD); git log -p \\\"$BASE\\\"..MERGE_HEAD -- a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/a.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}" }, { - "answer": [ - { - "tool": "Bash", - "input": { - "command": "git status" - } - }, - { - "tool": "Bash", - "input": { - "command": "git show abc123" - } - }, - { - "tool": "Edit", - "input": { - "file_path": "/w/a.py", - "old_string": "x", - "new_string": "y" - } - }, - { - "tool": "Bash", - "input": { - "command": "git -C /w add a.py" - } - } - ] + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/a.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_3\",\"name\":\"Bash\",\"input\":{\"command\":\"git -C /w add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_3\",\"content\":\"ok\"}]}}" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git --no-pager log --oneline -5 -- a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "history read; nothing edited yet" } ], "fail": [ { - "answer": [ - { - "tool": "Bash", - "input": { - "command": "git add a.py" - } - }, - { - "tool": "Bash", - "input": { - "command": "git log --oneline" - } - } - ], - "why": "staged the resolution before reading history" + "answer": "" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/a.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", + "why": "rewrote the file before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/a.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "edited before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log -p -- a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "rewrote the file with Bash before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git checkout --theirs a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "took a side before reading history" }, { - "answer": [ - { - "tool": "Bash", - "input": { - "command": "git diff" - } - }, - { - "tool": "Write", - "input": { - "file_path": "/w/a.py", - "content": "y\n" - } - }, - { - "tool": "Bash", - "input": { - "command": "git add a.py" - } - } - ], + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git diff\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/a.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", "why": "diff only, no history" }, { - "answer": [ - { - "tool": "Bash", - "input": { - "command": "git log --oneline" - } - }, - { - "tool": "Write", - "input": { - "file_path": "/w/a.py", - "content": "y\n" - } - } - ], - "why": "never staged a resolution" + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_9\",\"content\":\"try git log next\"}]}}", + "why": "git log only in tool output" } ] } diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md index 179cdca816..464a2636ab 100644 --- a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- -type: tool_order -before: { tool: Bash, input_match: 'git\s+(?:-C\s+\S+\s+)?(?:log|show)\b' } -after: { tool: Bash, input_match: 'git\s+(?:-C\s+\S+\s+)?add\b' } +type: regex +pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|\bcat\s*>|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' +target: trace --- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/history-before-edit.json index a2e0bfaeec..ad68a30691 100644 --- a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/history-before-edit.json +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/history-before-edit.json @@ -1,118 +1,43 @@ { "pass": [ { - "answer": [ - { - "tool": "Bash", - "input": { - "command": "BASE=$(git merge-base HEAD MERGE_HEAD); git log -p \"$BASE\"..MERGE_HEAD -- a.py" - } - }, - { - "tool": "Write", - "input": { - "file_path": "/w/a.py", - "content": "y\n" - } - }, - { - "tool": "Bash", - "input": { - "command": "git add a.py && GIT_EDITOR=true git merge --continue" - } - } - ] + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"BASE=$(git merge-base HEAD MERGE_HEAD); git log -p \\\"$BASE\\\"..MERGE_HEAD -- a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/a.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}" }, { - "answer": [ - { - "tool": "Bash", - "input": { - "command": "git status" - } - }, - { - "tool": "Bash", - "input": { - "command": "git show abc123" - } - }, - { - "tool": "Edit", - "input": { - "file_path": "/w/a.py", - "old_string": "x", - "new_string": "y" - } - }, - { - "tool": "Bash", - "input": { - "command": "git -C /w add a.py" - } - } - ] + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/a.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_3\",\"name\":\"Bash\",\"input\":{\"command\":\"git -C /w add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_3\",\"content\":\"ok\"}]}}" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git --no-pager log --oneline -5 -- a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "history read; nothing edited yet" } ], "fail": [ { - "answer": [ - { - "tool": "Bash", - "input": { - "command": "git add a.py" - } - }, - { - "tool": "Bash", - "input": { - "command": "git log --oneline" - } - } - ], - "why": "staged the resolution before reading history" + "answer": "" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/a.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", + "why": "rewrote the file before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/a.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "edited before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log -p -- a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "rewrote the file with Bash before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git checkout --theirs a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "took a side before reading history" }, { - "answer": [ - { - "tool": "Bash", - "input": { - "command": "git diff" - } - }, - { - "tool": "Write", - "input": { - "file_path": "/w/a.py", - "content": "y\n" - } - }, - { - "tool": "Bash", - "input": { - "command": "git add a.py" - } - } - ], + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git diff\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/a.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", "why": "diff only, no history" }, { - "answer": [ - { - "tool": "Bash", - "input": { - "command": "git log --oneline" - } - }, - { - "tool": "Write", - "input": { - "file_path": "/w/a.py", - "content": "y\n" - } - } - ], - "why": "never staged a resolution" + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_9\",\"content\":\"try git log next\"}]}}", + "why": "git log only in tool output" } ] } diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md index 179cdca816..464a2636ab 100644 --- a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- -type: tool_order -before: { tool: Bash, input_match: 'git\s+(?:-C\s+\S+\s+)?(?:log|show)\b' } -after: { tool: Bash, input_match: 'git\s+(?:-C\s+\S+\s+)?add\b' } +type: regex +pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|\bcat\s*>|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' +target: trace --- diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/history-before-edit.json index a2e0bfaeec..ad68a30691 100644 --- a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/history-before-edit.json +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/history-before-edit.json @@ -1,118 +1,43 @@ { "pass": [ { - "answer": [ - { - "tool": "Bash", - "input": { - "command": "BASE=$(git merge-base HEAD MERGE_HEAD); git log -p \"$BASE\"..MERGE_HEAD -- a.py" - } - }, - { - "tool": "Write", - "input": { - "file_path": "/w/a.py", - "content": "y\n" - } - }, - { - "tool": "Bash", - "input": { - "command": "git add a.py && GIT_EDITOR=true git merge --continue" - } - } - ] + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"BASE=$(git merge-base HEAD MERGE_HEAD); git log -p \\\"$BASE\\\"..MERGE_HEAD -- a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/a.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}" }, { - "answer": [ - { - "tool": "Bash", - "input": { - "command": "git status" - } - }, - { - "tool": "Bash", - "input": { - "command": "git show abc123" - } - }, - { - "tool": "Edit", - "input": { - "file_path": "/w/a.py", - "old_string": "x", - "new_string": "y" - } - }, - { - "tool": "Bash", - "input": { - "command": "git -C /w add a.py" - } - } - ] + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/a.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_3\",\"name\":\"Bash\",\"input\":{\"command\":\"git -C /w add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_3\",\"content\":\"ok\"}]}}" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git --no-pager log --oneline -5 -- a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "history read; nothing edited yet" } ], "fail": [ { - "answer": [ - { - "tool": "Bash", - "input": { - "command": "git add a.py" - } - }, - { - "tool": "Bash", - "input": { - "command": "git log --oneline" - } - } - ], - "why": "staged the resolution before reading history" + "answer": "" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/a.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", + "why": "rewrote the file before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/a.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "edited before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log -p -- a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "rewrote the file with Bash before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git checkout --theirs a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "took a side before reading history" }, { - "answer": [ - { - "tool": "Bash", - "input": { - "command": "git diff" - } - }, - { - "tool": "Write", - "input": { - "file_path": "/w/a.py", - "content": "y\n" - } - }, - { - "tool": "Bash", - "input": { - "command": "git add a.py" - } - } - ], + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git diff\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/a.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", "why": "diff only, no history" }, { - "answer": [ - { - "tool": "Bash", - "input": { - "command": "git log --oneline" - } - }, - { - "tool": "Write", - "input": { - "file_path": "/w/a.py", - "content": "y\n" - } - } - ], - "why": "never staged a resolution" + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_9\",\"content\":\"try git log next\"}]}}", + "why": "git log only in tool output" } ] } From bf27842b5f58c31e249da59f6ecb383a55654bf5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 9 Oct 2026 20:20:29 +0000 Subject: [PATCH 13/19] docs(source-control): say the fragment's history grader orders reads before the first edit Co-authored-by: ksextonmelodic --- .../test-resolve-conflicts-plugin-eval-suite-ee7a239e.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/source-control/test-resolve-conflicts-plugin-eval-suite-ee7a239e.md b/.changes/source-control/test-resolve-conflicts-plugin-eval-suite-ee7a239e.md index e510e007eb..5de4962ea6 100644 --- a/.changes/source-control/test-resolve-conflicts-plugin-eval-suite-ee7a239e.md +++ b/.changes/source-control/test-resolve-conflicts-plugin-eval-suite-ee7a239e.md @@ -4,4 +4,4 @@ bump: patch ### Added -- **A `claude plugin eval` suite for `/source-control:resolve-conflicts`.** Seven cases tagged `row22`, each scaffolding a git repository that is stopped in a real conflict: a two-stop rebase, a six-file merge where the prompt asks for `--ours` or abort, a merge with a conflict that has no markers, a cherry-pick, a merge whose incoming tip never touched the file, a revert, and a merge with only "sort it out" as the prompt. Most graders are deterministic: the resolved file contents, history read before the resolution is staged, tests run before the operation concludes, and the concluding reflog line. They also cover the "It's working if" checks from the archived upstream skill page. Three cases add one model-judged grader each, and every grader carries pass and fail samples. The skill body is unchanged. +- **A `claude plugin eval` suite for `/source-control:resolve-conflicts`.** Seven cases tagged `row22`, each scaffolding a git repository that is stopped in a real conflict: a two-stop rebase, a six-file merge where the prompt asks for `--ours` or abort, a merge with a conflict that has no markers, a cherry-pick, a merge whose incoming tip never touched the file, a revert, and a merge with only "sort it out" as the prompt. Most graders are deterministic: the resolved file contents, history read before the first edit, tests run before the operation concludes, and the concluding reflog line. They also cover the "It's working if" checks from the archived upstream skill page. Three cases add one model-judged grader each, and every grader carries pass and fail samples. The skill body is unchanged. From 63b7242bd79fc934c8829c248138efa7a9b5a093 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:04:17 -0400 Subject: [PATCH 14/19] test(source-control): catch redirect and cp/mv rewrites in history-before-edit graders The forbidden-before-history set matched only a bare `cat >`. Any `>`/`>>` redirect into a file (except fd duplication and /dev/null) and any cp/mv now fail the grader. Samples cover `cat theirs.py > a.py`, `cp theirs.py a.py`, and a pass case with 2>&1 and >/dev/null. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01TRhsMPSTMFo99ik7gkdgQt --- .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 12 ++++++++++++ .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 12 ++++++++++++ .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 12 ++++++++++++ .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 12 ++++++++++++ .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 12 ++++++++++++ 10 files changed, 65 insertions(+), 5 deletions(-) diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md index 464a2636ab..e15d67e32b 100644 --- a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: regex -pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|\bcat\s*>|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' +pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&|\s*/dev/null)|\b(?:cp|mv)\s|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' target: trace --- diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/history-before-edit.json index ad68a30691..a4fe76e53b 100644 --- a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/history-before-edit.json +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/history-before-edit.json @@ -9,6 +9,10 @@ { "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git --no-pager log --oneline -5 -- a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", "why": "history read; nothing edited yet" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status 2>&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "stderr and null redirects are not file writes" } ], "fail": [ @@ -38,6 +42,14 @@ { "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_9\",\"content\":\"try git log next\"}]}}", "why": "git log only in tool output" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs.py > a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "overwrote the file with a redirect before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp theirs.py a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "copied over the file before reading history" } ] } diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md index 464a2636ab..e15d67e32b 100644 --- a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: regex -pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|\bcat\s*>|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' +pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&|\s*/dev/null)|\b(?:cp|mv)\s|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' target: trace --- diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/history-before-edit.json index ad68a30691..a4fe76e53b 100644 --- a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/history-before-edit.json +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/history-before-edit.json @@ -9,6 +9,10 @@ { "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git --no-pager log --oneline -5 -- a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", "why": "history read; nothing edited yet" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status 2>&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "stderr and null redirects are not file writes" } ], "fail": [ @@ -38,6 +42,14 @@ { "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_9\",\"content\":\"try git log next\"}]}}", "why": "git log only in tool output" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs.py > a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "overwrote the file with a redirect before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp theirs.py a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "copied over the file before reading history" } ] } diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md index 464a2636ab..e15d67e32b 100644 --- a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: regex -pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|\bcat\s*>|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' +pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&|\s*/dev/null)|\b(?:cp|mv)\s|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' target: trace --- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/history-before-edit.json index ad68a30691..a4fe76e53b 100644 --- a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/history-before-edit.json +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/history-before-edit.json @@ -9,6 +9,10 @@ { "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git --no-pager log --oneline -5 -- a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", "why": "history read; nothing edited yet" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status 2>&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "stderr and null redirects are not file writes" } ], "fail": [ @@ -38,6 +42,14 @@ { "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_9\",\"content\":\"try git log next\"}]}}", "why": "git log only in tool output" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs.py > a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "overwrote the file with a redirect before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp theirs.py a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "copied over the file before reading history" } ] } diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md index 464a2636ab..e15d67e32b 100644 --- a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: regex -pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|\bcat\s*>|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' +pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&|\s*/dev/null)|\b(?:cp|mv)\s|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' target: trace --- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/history-before-edit.json index ad68a30691..a4fe76e53b 100644 --- a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/history-before-edit.json +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/history-before-edit.json @@ -9,6 +9,10 @@ { "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git --no-pager log --oneline -5 -- a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", "why": "history read; nothing edited yet" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status 2>&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "stderr and null redirects are not file writes" } ], "fail": [ @@ -38,6 +42,14 @@ { "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_9\",\"content\":\"try git log next\"}]}}", "why": "git log only in tool output" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs.py > a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "overwrote the file with a redirect before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp theirs.py a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "copied over the file before reading history" } ] } diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md index 464a2636ab..e15d67e32b 100644 --- a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: regex -pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|\bcat\s*>|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' +pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&|\s*/dev/null)|\b(?:cp|mv)\s|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' target: trace --- diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/history-before-edit.json index ad68a30691..a4fe76e53b 100644 --- a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/history-before-edit.json +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/history-before-edit.json @@ -9,6 +9,10 @@ { "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git --no-pager log --oneline -5 -- a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", "why": "history read; nothing edited yet" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status 2>&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "stderr and null redirects are not file writes" } ], "fail": [ @@ -38,6 +42,14 @@ { "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_9\",\"content\":\"try git log next\"}]}}", "why": "git log only in tool output" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs.py > a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "overwrote the file with a redirect before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp theirs.py a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "copied over the file before reading history" } ] } From 6fcdb8b3be07599b31f691abd03ee951809dd558 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:12:24 -0400 Subject: [PATCH 15/19] test(source-control): exempt /tmp redirects in history-before-edit graders Capturing test output to /tmp before reading history is not a rewrite of the conflicted file. Add a pass sample for it in each case. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01TRhsMPSTMFo99ik7gkdgQt --- .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 4 ++++ .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 4 ++++ .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 4 ++++ .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 4 ++++ .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 4 ++++ 10 files changed, 25 insertions(+), 5 deletions(-) diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md index e15d67e32b..445fce9417 100644 --- a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: regex -pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&|\s*/dev/null)|\b(?:cp|mv)\s|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' +pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&|\s*/(?:dev/null|tmp/))|\b(?:cp|mv)\s|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' target: trace --- diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/history-before-edit.json index a4fe76e53b..0f1649d049 100644 --- a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/history-before-edit.json +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/history-before-edit.json @@ -13,6 +13,10 @@ { "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status 2>&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "stderr and null redirects are not file writes" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"python3 -m unittest > /tmp/test-output.txt 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "test output captured to /tmp is not a rewrite of the conflicted file" } ], "fail": [ diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md index e15d67e32b..445fce9417 100644 --- a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: regex -pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&|\s*/dev/null)|\b(?:cp|mv)\s|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' +pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&|\s*/(?:dev/null|tmp/))|\b(?:cp|mv)\s|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' target: trace --- diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/history-before-edit.json index a4fe76e53b..0f1649d049 100644 --- a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/history-before-edit.json +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/history-before-edit.json @@ -13,6 +13,10 @@ { "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status 2>&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "stderr and null redirects are not file writes" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"python3 -m unittest > /tmp/test-output.txt 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "test output captured to /tmp is not a rewrite of the conflicted file" } ], "fail": [ diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md index e15d67e32b..445fce9417 100644 --- a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: regex -pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&|\s*/dev/null)|\b(?:cp|mv)\s|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' +pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&|\s*/(?:dev/null|tmp/))|\b(?:cp|mv)\s|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' target: trace --- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/history-before-edit.json index a4fe76e53b..0f1649d049 100644 --- a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/history-before-edit.json +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/history-before-edit.json @@ -13,6 +13,10 @@ { "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status 2>&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "stderr and null redirects are not file writes" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"python3 -m unittest > /tmp/test-output.txt 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "test output captured to /tmp is not a rewrite of the conflicted file" } ], "fail": [ diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md index e15d67e32b..445fce9417 100644 --- a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: regex -pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&|\s*/dev/null)|\b(?:cp|mv)\s|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' +pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&|\s*/(?:dev/null|tmp/))|\b(?:cp|mv)\s|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' target: trace --- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/history-before-edit.json index a4fe76e53b..0f1649d049 100644 --- a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/history-before-edit.json +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/history-before-edit.json @@ -13,6 +13,10 @@ { "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status 2>&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "stderr and null redirects are not file writes" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"python3 -m unittest > /tmp/test-output.txt 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "test output captured to /tmp is not a rewrite of the conflicted file" } ], "fail": [ diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md index e15d67e32b..445fce9417 100644 --- a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: regex -pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&|\s*/dev/null)|\b(?:cp|mv)\s|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' +pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&|\s*/(?:dev/null|tmp/))|\b(?:cp|mv)\s|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' target: trace --- diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/history-before-edit.json index a4fe76e53b..0f1649d049 100644 --- a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/history-before-edit.json +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/history-before-edit.json @@ -13,6 +13,10 @@ { "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status 2>&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "stderr and null redirects are not file writes" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"python3 -m unittest > /tmp/test-output.txt 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "test output captured to /tmp is not a rewrite of the conflicted file" } ], "fail": [ From a51dd0157f86165f88f253b13ad78a91ecd8f98b Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:18:04 -0400 Subject: [PATCH 16/19] test(source-control): scope history-before-edit writes to the conflicted file A redirect, cp, mv, sed -i, perl -i or tee now fails the grader only when its target is the case's conflicted file. Unrelated writes such as test output redirects pass, so the /tmp exemption is dropped. Samples use each case's real file name and add a pass sample for an unrelated redirect. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01TRhsMPSTMFo99ik7gkdgQt --- .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 28 +++++++++---------- .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 28 +++++++++---------- .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 28 +++++++++---------- .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 28 +++++++++---------- .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 28 +++++++++---------- 10 files changed, 75 insertions(+), 75 deletions(-) diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md index 445fce9417..c429910eba 100644 --- a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: regex -pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&|\s*/(?:dev/null|tmp/))|\b(?:cp|mv)\s|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' +pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\]|\\.)*?(?&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", - "why": "stderr and null redirects are not file writes" + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"python3 -m unittest > test-output.log 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "redirect to an unrelated file is not an edit of the conflicted file" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"python3 -m unittest > /tmp/test-output.txt 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", - "why": "test output captured to /tmp is not a rewrite of the conflicted file" + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status 2>&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "stderr and null redirects are not file writes" } ], "fail": [ @@ -24,23 +24,23 @@ "answer": "" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/a.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/client.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add client.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", "why": "rewrote the file before reading history" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/a.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/client.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "edited before reading history" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log -p -- a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ client.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log -p -- client.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "rewrote the file with Bash before reading history" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git checkout --theirs a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git checkout --theirs client.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "took a side before reading history" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git diff\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/a.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git diff\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/client.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add client.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", "why": "diff only, no history" }, { @@ -48,11 +48,11 @@ "why": "git log only in tool output" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs.py > a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs.py > client.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "overwrote the file with a redirect before reading history" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp theirs.py a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp theirs.py client.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "copied over the file before reading history" } ] diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md index 445fce9417..26a0f30daf 100644 --- a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: regex -pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&|\s*/(?:dev/null|tmp/))|\b(?:cp|mv)\s|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' +pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\]|\\.)*?(?&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", - "why": "stderr and null redirects are not file writes" + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"python3 -m unittest > test-output.log 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "redirect to an unrelated file is not an edit of the conflicted file" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"python3 -m unittest > /tmp/test-output.txt 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", - "why": "test output captured to /tmp is not a rewrite of the conflicted file" + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status 2>&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "stderr and null redirects are not file writes" } ], "fail": [ @@ -24,23 +24,23 @@ "answer": "" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/a.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/rules.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add rules.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", "why": "rewrote the file before reading history" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/a.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/rules.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "edited before reading history" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log -p -- a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ rules.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log -p -- rules.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "rewrote the file with Bash before reading history" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git checkout --theirs a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git checkout --theirs rules.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "took a side before reading history" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git diff\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/a.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git diff\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/rules.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add rules.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", "why": "diff only, no history" }, { @@ -48,11 +48,11 @@ "why": "git log only in tool output" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs.py > a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs.py > rules.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "overwrote the file with a redirect before reading history" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp theirs.py a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp theirs.py rules.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "copied over the file before reading history" } ] diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md index 445fce9417..84e01a4b75 100644 --- a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: regex -pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&|\s*/(?:dev/null|tmp/))|\b(?:cp|mv)\s|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' +pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\]|\\.)*?(?&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", - "why": "stderr and null redirects are not file writes" + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"python3 -m unittest > test-output.log 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "redirect to an unrelated file is not an edit of the conflicted file" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"python3 -m unittest > /tmp/test-output.txt 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", - "why": "test output captured to /tmp is not a rewrite of the conflicted file" + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status 2>&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "stderr and null redirects are not file writes" } ], "fail": [ @@ -24,23 +24,23 @@ "answer": "" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/a.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/retry.yaml\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add retry.yaml\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", "why": "rewrote the file before reading history" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/a.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/retry.yaml\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "edited before reading history" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log -p -- a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ retry.yaml\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log -p -- retry.yaml\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "rewrote the file with Bash before reading history" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git checkout --theirs a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git checkout --theirs retry.yaml\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "took a side before reading history" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git diff\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/a.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git diff\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/retry.yaml\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add retry.yaml\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", "why": "diff only, no history" }, { @@ -48,11 +48,11 @@ "why": "git log only in tool output" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs.py > a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs.py > retry.yaml\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "overwrote the file with a redirect before reading history" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp theirs.py a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp theirs.py retry.yaml\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "copied over the file before reading history" } ] diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md index 445fce9417..7d41e36cc2 100644 --- a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: regex -pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&|\s*/(?:dev/null|tmp/))|\b(?:cp|mv)\s|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' +pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\]|\\.)*?(?&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", - "why": "stderr and null redirects are not file writes" + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"python3 -m unittest > test-output.log 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "redirect to an unrelated file is not an edit of the conflicted file" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"python3 -m unittest > /tmp/test-output.txt 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", - "why": "test output captured to /tmp is not a rewrite of the conflicted file" + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status 2>&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "stderr and null redirects are not file writes" } ], "fail": [ @@ -24,23 +24,23 @@ "answer": "" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/a.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/retry.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add retry.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", "why": "rewrote the file before reading history" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/a.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/retry.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "edited before reading history" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log -p -- a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ retry.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log -p -- retry.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "rewrote the file with Bash before reading history" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git checkout --theirs a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git checkout --theirs retry.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "took a side before reading history" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git diff\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/a.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git diff\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/retry.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add retry.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", "why": "diff only, no history" }, { @@ -48,11 +48,11 @@ "why": "git log only in tool output" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs.py > a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs.py > retry.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "overwrote the file with a redirect before reading history" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp theirs.py a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp theirs.py retry.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "copied over the file before reading history" } ] diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md index 445fce9417..9caaef7486 100644 --- a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: regex -pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&|\s*/(?:dev/null|tmp/))|\b(?:cp|mv)\s|\btee\b|\b(?:checkout|restore)\s+(?:\S+\s+)*?--(?:ours|theirs)\b))[\s\S])*?"command":"(?:[^"\\]|\\.)*?\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:log|show)\b' +pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\]|\\.)*?(?&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", - "why": "stderr and null redirects are not file writes" + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"python3 -m unittest > test-output.log 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "redirect to an unrelated file is not an edit of the conflicted file" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"python3 -m unittest > /tmp/test-output.txt 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", - "why": "test output captured to /tmp is not a rewrite of the conflicted file" + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status 2>&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "stderr and null redirects are not file writes" } ], "fail": [ @@ -24,23 +24,23 @@ "answer": "" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/a.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/session.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add session.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", "why": "rewrote the file before reading history" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/a.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/session.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "edited before reading history" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log -p -- a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ session.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log -p -- session.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "rewrote the file with Bash before reading history" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git checkout --theirs a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git checkout --theirs session.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "took a side before reading history" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git diff\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/a.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git diff\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/session.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add session.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", "why": "diff only, no history" }, { @@ -48,11 +48,11 @@ "why": "git log only in tool output" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs.py > a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs.py > session.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "overwrote the file with a redirect before reading history" }, { - "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp theirs.py a.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp theirs.py session.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "copied over the file before reading history" } ] From fa7f23aba0555d3a2b89a42002492a89d9f5c94a Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:24:42 -0400 Subject: [PATCH 17/19] test(source-control): keep history-before-edit write match within one command The conflicted file name must follow the write verb without crossing ;, & or |, so an unrelated write chained with a history read no longer fails. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01TRhsMPSTMFo99ik7gkdgQt --- .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 4 ++++ .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 4 ++++ .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 4 ++++ .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 4 ++++ .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 4 ++++ 10 files changed, 25 insertions(+), 5 deletions(-) diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md index c429910eba..13a0ab130f 100644 --- a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: regex -pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\]|\\.)*?(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(?&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "stderr and null redirects are not file writes" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp notes.txt backup.txt && git log --oneline -- client.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "unrelated write chained with a history read of the conflicted file" } ], "fail": [ diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md index 26a0f30daf..fa3439e735 100644 --- a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: regex -pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\]|\\.)*?(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(?&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "stderr and null redirects are not file writes" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp notes.txt backup.txt && git log --oneline -- rules.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "unrelated write chained with a history read of the conflicted file" } ], "fail": [ diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md index 84e01a4b75..4330e0a573 100644 --- a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: regex -pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\]|\\.)*?(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(?&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "stderr and null redirects are not file writes" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp notes.txt backup.txt && git log --oneline -- retry.yaml\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "unrelated write chained with a history read of the conflicted file" } ], "fail": [ diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md index 7d41e36cc2..bb44b5ab6d 100644 --- a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: regex -pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\]|\\.)*?(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(?&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "stderr and null redirects are not file writes" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp notes.txt backup.txt && git log --oneline -- retry.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "unrelated write chained with a history read of the conflicted file" } ], "fail": [ diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md index 9caaef7486..d1598cb4f8 100644 --- a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: regex -pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\]|\\.)*?(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(?&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "stderr and null redirects are not file writes" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp notes.txt backup.txt && git log --oneline -- session.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "unrelated write chained with a history read of the conflicted file" } ], "fail": [ From 2b616438c2d4b8b04d113c0993ec1de32f64bd47 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:38:10 -0400 Subject: [PATCH 18/19] test(source-control): anchor history-before-edit history read at a command boundary The history read must start a command or follow ;, &, | or a newline, and every character before it, including earlier parts of the same call, is checked for a write to the conflicted file. A call that rewrites the file and then runs git log now fails. Samples cover both orders in one call. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01TRhsMPSTMFo99ik7gkdgQt --- .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 12 ++++++++++++ .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 12 ++++++++++++ .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 12 ++++++++++++ .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 12 ++++++++++++ .../graders/history-before-edit.md | 2 +- .../samples/history-before-edit.json | 12 ++++++++++++ 10 files changed, 65 insertions(+), 5 deletions(-) diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md index 13a0ab130f..a473e75d5c 100644 --- a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: regex -pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(? client.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "history read first; the rewrite comes after it in the same call" } ], "fail": [ @@ -58,6 +62,14 @@ { "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp theirs.py client.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "copied over the file before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs > client.py && git log -- client.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "rewrote the file earlier in the same call as the history read" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ client.py; git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "edited with sed earlier in the same call as the history read" } ] } diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md index fa3439e735..32979cca14 100644 --- a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: regex -pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(? rules.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "history read first; the rewrite comes after it in the same call" } ], "fail": [ @@ -58,6 +62,14 @@ { "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp theirs.py rules.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "copied over the file before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs > rules.py && git log -- rules.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "rewrote the file earlier in the same call as the history read" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ rules.py; git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "edited with sed earlier in the same call as the history read" } ] } diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md index 4330e0a573..1d500f9c72 100644 --- a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: regex -pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(? retry.yaml\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "history read first; the rewrite comes after it in the same call" } ], "fail": [ @@ -58,6 +62,14 @@ { "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp theirs.py retry.yaml\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "copied over the file before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs > retry.yaml && git log -- retry.yaml\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "rewrote the file earlier in the same call as the history read" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ retry.yaml; git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "edited with sed earlier in the same call as the history read" } ] } diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md index bb44b5ab6d..47241cd8e1 100644 --- a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: regex -pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(? retry.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "history read first; the rewrite comes after it in the same call" } ], "fail": [ @@ -58,6 +62,14 @@ { "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp theirs.py retry.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "copied over the file before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs > retry.py && git log -- retry.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "rewrote the file earlier in the same call as the history read" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ retry.py; git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "edited with sed earlier in the same call as the history read" } ] } diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md index d1598cb4f8..ffc9607697 100644 --- a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md @@ -1,5 +1,5 @@ --- type: regex -pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":"(?:[^"\\]|\\.)*?(?:(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(? session.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "history read first; the rewrite comes after it in the same call" } ], "fail": [ @@ -58,6 +62,14 @@ { "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp theirs.py session.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", "why": "copied over the file before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs > session.py && git log -- session.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "rewrote the file earlier in the same call as the history read" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ session.py; git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "edited with sed earlier in the same call as the history read" } ] } From 1a49eb4d1faf4d551ea45dc9a09cdd69bf26553c Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:16:03 -0400 Subject: [PATCH 19/19] fix(source-control): avoid piped grep -q in resolve-conflicts eval scaffolds Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01TRhsMPSTMFo99ik7gkdgQt --- .../scaffold.sh | 2 +- .../resolve-conflicts-cherry-pick-reads-pick-head/scaffold.sh | 2 +- .../scaffold.sh | 2 +- .../scaffold.sh | 2 +- .../scaffold.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/scaffold.sh b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/scaffold.sh index 8d5dca5f29..84998f090e 100755 --- a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/scaffold.sh +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/scaffold.sh @@ -64,4 +64,4 @@ git commit -q -m "refactor(api): drop legacy_token from fetch_orders" -m "The /o git checkout -q feature git merge main -q -m "Merge branch main into feature" >/dev/null 2>&1 || true test -f .git/MERGE_HEAD -git diff --name-only --diff-filter=U | grep -qx api/client.py +grep -qx api/client.py < <(git diff --name-only --diff-filter=U) diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/scaffold.sh b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/scaffold.sh index 75ffed8dd3..8083cf166e 100755 --- a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/scaffold.sh +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/scaffold.sh @@ -81,4 +81,4 @@ git checkout -q release/1.4 git cherry-pick "$CAP" >/dev/null 2>&1 || true test -f .git/CHERRY_PICK_HEAD test ! -f .git/MERGE_HEAD -git diff --name-only --diff-filter=U | grep -qx billing/rules.py +grep -qx billing/rules.py < <(git diff --name-only --diff-filter=U) diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/scaffold.sh b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/scaffold.sh index 5ee0e51001..95cf4163ed 100755 --- a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/scaffold.sh +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/scaffold.sh @@ -47,5 +47,5 @@ git commit -q -m "perf: double the retry backoff" -m "Cold starts stampeded the git merge ops/tuning -q -m "Merge branch ops/tuning into feature" >/dev/null 2>&1 || true test -f .git/MERGE_HEAD -git diff --name-only --diff-filter=U | grep -qx config/retry.yaml +grep -qx config/retry.yaml < <(git diff --name-only --diff-filter=U) test -z "$(git show --format= MERGE_HEAD -- config/retry.yaml)" diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/scaffold.sh b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/scaffold.sh index 8035cb3d2f..39414af580 100755 --- a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/scaffold.sh +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/scaffold.sh @@ -116,4 +116,4 @@ git commit -q -m "refactor(retry): rename max_retries to max_attempts" -m "The c git checkout -q feature git rebase main >/dev/null 2>&1 || true test -d .git/rebase-merge -git diff --name-only --diff-filter=U | grep -qx retry.py +grep -qx retry.py < <(git diff --name-only --diff-filter=U) diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/scaffold.sh b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/scaffold.sh index cd8d21f231..df5e7eee45 100755 --- a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/scaffold.sh +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/scaffold.sh @@ -56,4 +56,4 @@ git commit -q -m "fix(auth): mark the session cookie secure and name it" -m "The git revert --no-edit "$REMEMBER" >/dev/null 2>&1 || true test -f .git/REVERT_HEAD -git diff --name-only --diff-filter=U | grep -qx auth/session.py +grep -qx auth/session.py < <(git diff --name-only --diff-filter=U)