diff --git a/.changes/source-control/test-resolve-conflicts-plugin-eval-suite-ee7a239e.md b/.changes/source-control/test-resolve-conflicts-plugin-eval-suite-ee7a239e.md new file mode 100644 index 0000000000..5de4962ea6 --- /dev/null +++ b/.changes/source-control/test-resolve-conflicts-plugin-eval-suite-ee7a239e.md @@ -0,0 +1,7 @@ +--- +bump: patch +--- + +### Added + +- **A `claude plugin eval` suite for `/source-control:resolve-conflicts`.** Seven cases tagged `row22`, each scaffolding a git repository that is stopped in a real conflict: a two-stop rebase, a six-file merge where the prompt asks for `--ours` or abort, a merge with a conflict that has no markers, a cherry-pick, a merge whose incoming tip never touched the file, a revert, and a merge with only "sort it out" as the prompt. Most graders are deterministic: the resolved file contents, history read before the first edit, tests run before the operation concludes, and the concluding reflog line. They also cover the "It's working if" checks from the archived upstream skill page. Three cases add one model-judged grader each, and every grader carries pass and fail samples. The skill body is unchanged. diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/case.yaml b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/case.yaml new file mode 100644 index 0000000000..a62410c1e8 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/case.yaml @@ -0,0 +1,4 @@ +schema_version: "1.1" +name: resolve-conflicts-bare-prompt-recovers-intent-from-history +context: + scaffold_script: scaffold.sh diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md new file mode 100644 index 0000000000..a473e75d5c --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/history-before-edit.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":")[\s\S]|"command":"(?:(?!(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(?{7})' +flags: m +match: not_contains +target: { source: file, path: api/client.py } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/skill-fired.md b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/skill-fired.md new file mode 100644 index 0000000000..f329958d6d --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/skill-fired.md @@ -0,0 +1,5 @@ +--- +type: tool_used +tool: Skill +input_match: '"skill"\s*:\s*"(?:source-control:)?resolve-conflicts"' +--- diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-checks-before-conclude.md b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-checks-before-conclude.md new file mode 100644 index 0000000000..ca5d2c546e --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-checks-before-conclude.md @@ -0,0 +1,5 @@ +--- +type: tool_order +before: { tool: Bash, input_match: 'unittest|pytest|python3?\s+\S*test_\w+\.py' } +after: { tool: Bash, input_match: 'merge\s+--continue|git\s+commit\b' } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-intents-and-drops-named.md b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-intents-and-drops-named.md new file mode 100644 index 0000000000..b903d7fc55 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-intents-and-drops-named.md @@ -0,0 +1,9 @@ +--- +type: llm +--- + +Main's commit removed `legacy_token` from `fetch_orders` because the /orders endpoint rejects token query auth with a 400 (SEC-12). The feature commit added a `region` filter for the EU dashboard (ORD-5). + +PASS if the reply states each side's intent as the commit history gives it (the removal and its reason; the region filter and its purpose) and says that `legacy_token` is gone from the result because main removed it on purpose. + +FAIL if the reply describes the resolution only in terms of the diff or the conflict markers, keeps `legacy_token` alongside `region`, or drops `region`. diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-operation-complete.md b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-operation-complete.md new file mode 100644 index 0000000000..b90f0b6544 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-operation-complete.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: 'commit \(merge\)' +target: { source: file, path: .git/logs/HEAD } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-quotes-history.md b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-quotes-history.md new file mode 100644 index 0000000000..fff8e6079a --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/graders/up-quotes-history.md @@ -0,0 +1,6 @@ +--- +type: regex +pattern: '^\{"type":"assistant".*"type":"text","text":"(?:[^"\\]|\\.)*?(?:SEC-12|\b400\b|stopped accepting|token query auth)' +flags: mi +target: trace +--- diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/prompt.md b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/prompt.md new file mode 100644 index 0000000000..fba31a80a7 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/prompt.md @@ -0,0 +1,11 @@ +--- +description: "Bare 'sort it out' merge prompt: only the commit message says legacy_token was removed on purpose; keep region, drop legacy_token, say why, run tests, conclude" +tags: [resolve-conflicts, source-control, row22, upstream-working-if] +runs: 3 +max_turns: 40 +timeout_seconds: 900 +allowed_tools: [Read, Glob, Grep, Skill, Bash, Edit] +expected_outcome: "api/client.py keeps region and has no legacy_token, no markers, history was read before the edit, the reply cites SEC-12 or the 400, tests ran before the merge was concluded, and the merge commit exists" +--- + +git merge main stopped with conflicts. Sort it out and finish the merge. diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/history-before-edit.json new file mode 100644 index 0000000000..0196ffcc2b --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/history-before-edit.json @@ -0,0 +1,75 @@ +{ + "pass": [ + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"BASE=$(git merge-base HEAD MERGE_HEAD); git log -p \\\"$BASE\\\"..MERGE_HEAD -- client.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/client.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add client.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/client.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_3\",\"name\":\"Bash\",\"input\":{\"command\":\"git -C /w add client.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_3\",\"content\":\"ok\"}]}}" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git --no-pager log --oneline -5 -- client.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "history read; nothing edited yet" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"python3 -m unittest > test-output.log 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "redirect to an unrelated file is not an edit of the conflicted file" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status 2>&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "stderr and null redirects are not file writes" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp notes.txt backup.txt && git log --oneline -- client.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "unrelated write chained with a history read of the conflicted file" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git log -- client.py && cat theirs > client.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "history read first; the rewrite comes after it in the same call" + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/client.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add client.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", + "why": "rewrote the file before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/client.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "edited before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ client.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log -p -- client.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "rewrote the file with Bash before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git checkout --theirs client.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "took a side before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git diff\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/client.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add client.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", + "why": "diff only, no history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_9\",\"content\":\"try git log next\"}]}}", + "why": "git log only in tool output" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs.py > client.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "overwrote the file with a redirect before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp theirs.py client.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "copied over the file before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs > client.py && git log -- client.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "rewrote the file earlier in the same call as the history read" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ client.py; git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "edited with sed earlier in the same call as the history read" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/intent-composed.json b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/intent-composed.json new file mode 100644 index 0000000000..de064d5122 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/intent-composed.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "def fetch_orders(session, customer_id, region=None):\n params = {\"customer\": customer_id}\n if region:\n params[\"region\"] = region\n return session.get(\"/orders\", params=params)\n" + } + ], + "fail": [ + { + "answer": "def fetch_orders(session, customer_id, legacy_token=None, region=None):\n params = {\"customer\": customer_id}\n if region:\n params[\"region\"] = region\n if legacy_token:\n params[\"token\"] = legacy_token\n" + }, + { + "answer": "def fetch_orders(session, customer_id):\n params = {\"customer\": customer_id}\n" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/no-abort.json b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/no-abort.json new file mode 100644 index 0000000000..84c7c59462 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/no-abort.json @@ -0,0 +1,39 @@ +{ + "pass": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status | head -4" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --abort" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git rebase --abort && git status" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/no-markers-client.json b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/no-markers-client.json new file mode 100644 index 0000000000..3c007648ed --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/no-markers-client.json @@ -0,0 +1,19 @@ +{ + "pass": [ + { + "answer": "x = 1\ny = 2\n" + }, + { + "answer": "# a ======= rule in a comment is fine mid-line ======= \n" + } + ], + "fail": [ + { + "answer": "<<<<<<< HEAD\nx = 1\n=======\nx = 2\n>>>>>>> main\n" + }, + { + "answer": "x = 1\n=======\n", + "why": "a leftover separator" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/skill-fired.json b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/skill-fired.json new file mode 100644 index 0000000000..012ab78155 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/skill-fired.json @@ -0,0 +1,46 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:resolve-conflicts" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Skill", + "input": { + "skill": "resolve-conflicts" + } + } + ] + } + ], + "fail": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:commit" + } + } + ], + "why": "wrong skill" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-checks-before-conclude.json b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-checks-before-conclude.json new file mode 100644 index 0000000000..34fc8ae002 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-checks-before-conclude.json @@ -0,0 +1,83 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + }, + { + "tool": "Bash", + "input": { + "command": "git merge --continue" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + }, + { + "tool": "Bash", + "input": { + "command": "git commit --no-edit" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --continue" + } + }, + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + } + ], + "why": "checks after the commit" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --continue" + } + } + ], + "why": "no checks" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + } + ], + "why": "never concluded" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-intents-and-drops-named.json b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-intents-and-drops-named.json new file mode 100644 index 0000000000..5e7e7f6056 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-intents-and-drops-named.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "Main removed legacy_token on purpose: /orders returns 400 for token query auth since 2026-03-01 (SEC-12). Your branch added the region filter for the EU dashboard (ORD-5). Resolution keeps region and leaves legacy_token out. Tests pass; merge concluded." + } + ], + "fail": [ + { + "answer": "Resolved the conflict by keeping both parameters, legacy_token and region, and finished the merge." + }, + { + "answer": "Took main's version of api/client.py." + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-operation-complete.json b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-operation-complete.json new file mode 100644 index 0000000000..677c0b5a86 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-operation-complete.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "0000 1111 eval 1700000000 +0000\tcheckout: moving from main to feature\n1111 2222 eval 1700000001 +0000\tcommit (merge): Merge branch main into feature\n" + } + ], + "fail": [ + { + "answer": "0000 1111 eval 1700000000 +0000\tcommit: feat(api): filter orders by region\n" + }, + { + "answer": "" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-quotes-history.json b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-quotes-history.json new file mode 100644 index 0000000000..c7c8c24f46 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/samples/up-quotes-history.json @@ -0,0 +1,30 @@ +{ + "pass": [ + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"main removed legacy_token because /orders now returns 400 for token query auth (SEC-12).\"},{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git add -- x\"}}]}}" + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Kept both sides and finished the merge.\"}]}}" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Merged; region filter added.\"}]}}" + }, + { + "answer": "{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n refactor(api): drop legacy_token from fetch_orders\\n\\n The /orders endpoint stopped accepting token query auth on 2026-03-01; sending it now returns 400. Refs SEC-12.\\n\"}]}}", + "why": "the history is only in tool output; the agent never quoted it" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"},{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --grep=SEC-12\"}}]}}", + "why": "the ref is in a tool call after a text block, not quoted to the user" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n refactor(api): drop legacy_token from fetch_orders\\n\\n The /orders endpoint stopped accepting token query auth on 2026-03-01; sending it now returns 400. Refs SEC-12.\\n\"}]}}", + "why": "an assistant line, then the history only in the next line's tool output" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/scaffold.sh b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/scaffold.sh new file mode 100755 index 0000000000..84998f090e --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-bare-prompt-recovers-intent-from-history/scaffold.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# git merge main stopped on api/client.py. main removed legacy_token on purpose (the endpoint +# rejects it); feature added a region filter in the same function. Only the commit message says why. +set -euo pipefail + +git init -q -b main +git config user.name eval +git config user.email eval@example.invalid +git config rerere.enabled false + +mkdir -p api +: > api/__init__.py +cat > api/client.py <<'PY' +def fetch_orders(session, customer_id, legacy_token=None): + params = {"customer": customer_id} + if legacy_token: + params["token"] = legacy_token + return session.get("/orders", params=params) +PY +cat > test_client.py <<'PY' +import unittest + +from api.client import fetch_orders + + +class FakeSession: + def get(self, path, params): + return path, params + + +class ClientTest(unittest.TestCase): + def test_sends_customer(self): + self.assertEqual(fetch_orders(FakeSession(), 7)[1]["customer"], 7) + + +if __name__ == "__main__": + unittest.main() +PY +git add . +git commit -q -m "feat(api): orders client" + +git checkout -q -b feature +cat > api/client.py <<'PY' +def fetch_orders(session, customer_id, legacy_token=None, region=None): + params = {"customer": customer_id} + if region: + params["region"] = region + if legacy_token: + params["token"] = legacy_token + return session.get("/orders", params=params) +PY +git add api/client.py +git commit -q -m "feat(api): filter orders by region" -m "The EU dashboard shows only its own region's orders. Refs ORD-5." + +git checkout -q main +cat > api/client.py <<'PY' +def fetch_orders(session, customer_id): + params = {"customer": customer_id} + return session.get("/orders", params=params) +PY +git add api/client.py +git commit -q -m "refactor(api): drop legacy_token from fetch_orders" -m "The /orders endpoint stopped accepting token query auth on 2026-03-01; sending it now returns 400. Refs SEC-12." + +git checkout -q feature +git merge main -q -m "Merge branch main into feature" >/dev/null 2>&1 || true +test -f .git/MERGE_HEAD +grep -qx api/client.py < <(git diff --name-only --diff-filter=U) diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/case.yaml b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/case.yaml new file mode 100644 index 0000000000..232dd27c4a --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/case.yaml @@ -0,0 +1,4 @@ +schema_version: "1.1" +name: resolve-conflicts-cherry-pick-reads-pick-head +context: + scaffold_script: scaffold.sh diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/both-sides-rules.md b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/both-sides-rules.md new file mode 100644 index 0000000000..831c7ccc6e --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/both-sides-rules.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: '^(?=[\s\S]*LATE_RATE\s*=\s*0\.02)(?=[\s\S]*MAX_LATE_FEE\s*=\s*50)(?=[\s\S]*min\(\s*round\(\s*balance\s*\*\s*LATE_RATE)' +target: { source: file, path: billing/rules.py } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/contract-reads-cherry-pick-head.md b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/contract-reads-cherry-pick-head.md new file mode 100644 index 0000000000..85afb2f66a --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/contract-reads-cherry-pick-head.md @@ -0,0 +1,5 @@ +--- +type: tool_used +tool: Bash +input_match: '\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:show|log|diff)\b[^;&|\n]*\bCHERRY_PICK_HEAD\b' +--- diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md new file mode 100644 index 0000000000..32979cca14 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/history-before-edit.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":")[\s\S]|"command":"(?:(?!(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(?{7})' +flags: m +match: not_contains +target: { source: file, path: billing/rules.py } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/no-merge-head-assumed.md b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/no-merge-head-assumed.md new file mode 100644 index 0000000000..32507ec947 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/graders/no-merge-head-assumed.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Bash +input_match: '(? test-output.log 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "redirect to an unrelated file is not an edit of the conflicted file" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status 2>&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "stderr and null redirects are not file writes" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp notes.txt backup.txt && git log --oneline -- rules.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "unrelated write chained with a history read of the conflicted file" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git log -- rules.py && cat theirs > rules.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "history read first; the rewrite comes after it in the same call" + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/rules.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add rules.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", + "why": "rewrote the file before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/rules.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "edited before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ rules.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log -p -- rules.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "rewrote the file with Bash before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git checkout --theirs rules.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "took a side before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git diff\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/rules.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add rules.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", + "why": "diff only, no history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_9\",\"content\":\"try git log next\"}]}}", + "why": "git log only in tool output" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs.py > rules.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "overwrote the file with a redirect before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp theirs.py rules.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "copied over the file before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs > rules.py && git log -- rules.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "rewrote the file earlier in the same call as the history read" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ rules.py; git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "edited with sed earlier in the same call as the history read" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/no-abort.json b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/no-abort.json new file mode 100644 index 0000000000..84c7c59462 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/no-abort.json @@ -0,0 +1,39 @@ +{ + "pass": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status | head -4" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --abort" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git rebase --abort && git status" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/no-markers-rules.json b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/no-markers-rules.json new file mode 100644 index 0000000000..3c007648ed --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/no-markers-rules.json @@ -0,0 +1,19 @@ +{ + "pass": [ + { + "answer": "x = 1\ny = 2\n" + }, + { + "answer": "# a ======= rule in a comment is fine mid-line ======= \n" + } + ], + "fail": [ + { + "answer": "<<<<<<< HEAD\nx = 1\n=======\nx = 2\n>>>>>>> main\n" + }, + { + "answer": "x = 1\n=======\n", + "why": "a leftover separator" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/no-merge-head-assumed.json b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/no-merge-head-assumed.json new file mode 100644 index 0000000000..728d1a6056 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/no-merge-head-assumed.json @@ -0,0 +1,46 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git show CHERRY_PICK_HEAD" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log --oneline HEAD...CHERRY_PICK_HEAD" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git show MERGE_HEAD -- billing/rules.py" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log HEAD..MERGE_HEAD" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/skill-fired.json b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/skill-fired.json new file mode 100644 index 0000000000..012ab78155 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/skill-fired.json @@ -0,0 +1,46 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:resolve-conflicts" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Skill", + "input": { + "skill": "resolve-conflicts" + } + } + ] + } + ], + "fail": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:commit" + } + } + ], + "why": "wrong skill" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/up-checks-before-conclude.json b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/up-checks-before-conclude.json new file mode 100644 index 0000000000..4b278b5d4f --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/up-checks-before-conclude.json @@ -0,0 +1,83 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + }, + { + "tool": "Bash", + "input": { + "command": "git cherry-pick --continue" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + }, + { + "tool": "Bash", + "input": { + "command": "GIT_EDITOR=true git cherry-pick --continue" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git cherry-pick --continue" + } + }, + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + } + ], + "why": "checks after the commit" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git cherry-pick --continue" + } + } + ], + "why": "no checks" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + } + ], + "why": "never concluded" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/up-operation-complete.json b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/up-operation-complete.json new file mode 100644 index 0000000000..5368d88040 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/up-operation-complete.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "0000 1111 eval 1700000000 +0000\tcheckout: moving from main to feature\n1111 2222 eval 1700000001 +0000\tcommit (cherry-pick): fix(billing): cap the late fee at 50\n" + } + ], + "fail": [ + { + "answer": "0000 1111 eval 1700000000 +0000\tcommit: refactor(billing): name the late-fee rate\n" + }, + { + "answer": "" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/up-quotes-history.json b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/up-quotes-history.json new file mode 100644 index 0000000000..17691d4e6e --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/samples/up-quotes-history.json @@ -0,0 +1,27 @@ +{ + "pass": [ + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"The picked commit caps the fee at 50 because regulators require it (BILL-203).\"},{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git add -- x\"}}]}}" + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Resolved billing/rules.py and continued the cherry-pick.\"}]}}" + }, + { + "answer": "{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n fix(billing): cap the late fee at 50\\n\\n Regulators cap late fees at 50 per invoice; uncapped fees on large balances were being refunded by hand. Refs BILL-203.\\n\"}]}}", + "why": "the history is only in tool output; the agent never quoted it" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"},{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --grep=BILL-203\"}}]}}", + "why": "the ref is in a tool call after a text block, not quoted to the user" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n fix(billing): cap the late fee at 50\\n\\n Regulators cap late fees at 50 per invoice; uncapped fees on large balances were being refunded by hand. Refs BILL-203.\\n\"}]}}", + "why": "an assistant line, then the history only in the next line's tool output" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/scaffold.sh b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/scaffold.sh new file mode 100755 index 0000000000..8083cf166e --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-cherry-pick-reads-pick-head/scaffold.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# release/1.4 stopped cherry-picking main's late-fee cap: release named the rate LATE_RATE, +# the picked commit added MAX_LATE_FEE and a min() around the same return line. +set -euo pipefail + +git init -q -b main +git config user.name eval +git config user.email eval@example.invalid +git config rerere.enabled false + +mkdir -p billing +: > billing/__init__.py +cat > billing/rules.py <<'PY' +def late_fee(balance, days_late): + if days_late <= 0: + return 0 + return round(balance * 0.02, 2) +PY +cat > test_rules.py <<'PY' +import unittest + +from billing.rules import late_fee + + +class LateFeeTest(unittest.TestCase): + def test_two_percent(self): + self.assertEqual(late_fee(100, 10), 2.0) + + +if __name__ == "__main__": + unittest.main() +PY +git add . +git commit -q -m "feat(billing): late fees" + +git checkout -q -b release/1.4 +cat > billing/rules.py <<'PY' +LATE_RATE = 0.02 + + +def late_fee(balance, days_late): + if days_late <= 0: + return 0 + return round(balance * LATE_RATE, 2) +PY +git add billing/rules.py +git commit -q -m "refactor(billing): name the late-fee rate" -m "Finance changes the rate each quarter; one named constant is the only place to edit. Refs BILL-190." + +git checkout -q main +cat > billing/rules.py <<'PY' +MAX_LATE_FEE = 50 + + +def late_fee(balance, days_late): + if days_late <= 0: + return 0 + return min(round(balance * 0.02, 2), MAX_LATE_FEE) +PY +cat > test_cap.py <<'PY' +import unittest + +from billing.rules import late_fee + + +class CapTest(unittest.TestCase): + def test_fee_is_capped(self): + self.assertEqual(late_fee(10000, 10), 50) + + +if __name__ == "__main__": + unittest.main() +PY +git add billing/rules.py test_cap.py +git commit -q -m "fix(billing): cap the late fee at 50" -m "Regulators cap late fees at 50 per invoice; uncapped fees on large balances were being refunded by hand. Refs BILL-203." +CAP=$(git rev-parse HEAD) +printf '# billing\n' > README.md +git add README.md +git commit -q -m "docs: billing readme" + +git checkout -q release/1.4 +git cherry-pick "$CAP" >/dev/null 2>&1 || true +test -f .git/CHERRY_PICK_HEAD +test ! -f .git/MERGE_HEAD +grep -qx billing/rules.py < <(git diff --name-only --diff-filter=U) diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/case.yaml b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/case.yaml new file mode 100644 index 0000000000..e00c8f8656 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/case.yaml @@ -0,0 +1,4 @@ +schema_version: "1.1" +name: resolve-conflicts-merge-reads-full-incoming-range +context: + scaffold_script: scaffold.sh diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/both-sides-config.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/both-sides-config.md new file mode 100644 index 0000000000..285d3f1a25 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/both-sides-config.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: '^(?=[\s\S]*max_attempts:\s*5\b)(?=[\s\S]*backoff_seconds:\s*2\b)' +target: { source: file, path: config/retry.yaml } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/contract-reads-incoming-range.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/contract-reads-incoming-range.md new file mode 100644 index 0000000000..ef4e48852b --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/contract-reads-incoming-range.md @@ -0,0 +1,5 @@ +--- +type: tool_used +tool: Bash +input_match: '\.\.\.?MERGE_HEAD|MERGE_HEAD\.\.|\^MERGE_HEAD|--merge\b' +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md new file mode 100644 index 0000000000..1d500f9c72 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/history-before-edit.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":")[\s\S]|"command":"(?:(?!(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(?{7})' +flags: m +match: not_contains +target: { source: file, path: config/retry.yaml } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/skill-fired.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/skill-fired.md new file mode 100644 index 0000000000..f329958d6d --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/skill-fired.md @@ -0,0 +1,5 @@ +--- +type: tool_used +tool: Skill +input_match: '"skill"\s*:\s*"(?:source-control:)?resolve-conflicts"' +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/up-checks-before-conclude.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/up-checks-before-conclude.md new file mode 100644 index 0000000000..6ca1f35154 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/up-checks-before-conclude.md @@ -0,0 +1,5 @@ +--- +type: tool_order +before: { tool: Bash, input_match: 'check_config|unittest|pytest|make\b' } +after: { tool: Bash, input_match: 'merge\s+--continue|git\s+commit\b' } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/up-operation-complete.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/up-operation-complete.md new file mode 100644 index 0000000000..b90f0b6544 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/up-operation-complete.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: 'commit \(merge\)' +target: { source: file, path: .git/logs/HEAD } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/up-quotes-history.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/up-quotes-history.md new file mode 100644 index 0000000000..1016170b57 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/graders/up-quotes-history.md @@ -0,0 +1,6 @@ +--- +type: regex +pattern: '^\{"type":"assistant".*"type":"text","text":"(?:[^"\\]|\\.)*?(?:OPS-311|failover|1 in 40)' +flags: mi +target: trace +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/prompt.md b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/prompt.md new file mode 100644 index 0000000000..f0c29e81d2 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/prompt.md @@ -0,0 +1,11 @@ +--- +description: "Merge stop where the incoming tip never touched the path: read merge-base..MERGE_HEAD, state the earlier commit's intent, keep max_attempts 5 and backoff 2" +tags: [resolve-conflicts, source-control, row22, converted] +runs: 3 +max_turns: 40 +timeout_seconds: 900 +allowed_tools: [Read, Glob, Grep, Skill, Bash, Edit] +expected_outcome: "config/retry.yaml has max_attempts 5 and backoff_seconds 2 with no markers, the incoming range was read, the reply cites the failover reason from the first incoming commit, checks ran before the merge was concluded" +--- + +A merge stopped with a conflict in config/retry.yaml. The branch being merged in has three commits; the first one changed config/retry.yaml, but the tip commit only touched an unrelated file. Recover the incoming side's intent, resolve it, and finish the merge. diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/both-sides-config.json b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/both-sides-config.json new file mode 100644 index 0000000000..66c2c77e62 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/both-sides-config.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "retry:\n max_attempts: 5\n backoff_seconds: 2\n" + } + ], + "fail": [ + { + "answer": "retry:\n max_attempts: 3\n backoff_seconds: 2\n" + }, + { + "answer": "retry:\n max_attempts: 5\n backoff_seconds: 1\n" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/contract-reads-incoming-range.json b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/contract-reads-incoming-range.json new file mode 100644 index 0000000000..7d455d50a9 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/contract-reads-incoming-range.json @@ -0,0 +1,59 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log -p \"$BASE\"..MERGE_HEAD -- config/retry.yaml" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log --oneline HEAD...MERGE_HEAD" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log -p --merge -- config/retry.yaml" + } + } + ] + } + ], + "fail": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git show MERGE_HEAD -- config/retry.yaml" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log -1 MERGE_HEAD" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/history-before-edit.json new file mode 100644 index 0000000000..4297896944 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/history-before-edit.json @@ -0,0 +1,75 @@ +{ + "pass": [ + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"BASE=$(git merge-base HEAD MERGE_HEAD); git log -p \\\"$BASE\\\"..MERGE_HEAD -- retry.yaml\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/retry.yaml\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add retry.yaml\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/retry.yaml\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_3\",\"name\":\"Bash\",\"input\":{\"command\":\"git -C /w add retry.yaml\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_3\",\"content\":\"ok\"}]}}" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git --no-pager log --oneline -5 -- retry.yaml\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "history read; nothing edited yet" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"python3 -m unittest > test-output.log 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "redirect to an unrelated file is not an edit of the conflicted file" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status 2>&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "stderr and null redirects are not file writes" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp notes.txt backup.txt && git log --oneline -- retry.yaml\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "unrelated write chained with a history read of the conflicted file" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git log -- retry.yaml && cat theirs > retry.yaml\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "history read first; the rewrite comes after it in the same call" + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/retry.yaml\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add retry.yaml\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", + "why": "rewrote the file before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/retry.yaml\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "edited before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ retry.yaml\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log -p -- retry.yaml\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "rewrote the file with Bash before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git checkout --theirs retry.yaml\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "took a side before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git diff\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/retry.yaml\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add retry.yaml\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", + "why": "diff only, no history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_9\",\"content\":\"try git log next\"}]}}", + "why": "git log only in tool output" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs.py > retry.yaml\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "overwrote the file with a redirect before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp theirs.py retry.yaml\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "copied over the file before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs > retry.yaml && git log -- retry.yaml\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "rewrote the file earlier in the same call as the history read" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ retry.yaml; git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "edited with sed earlier in the same call as the history read" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/no-abort.json b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/no-abort.json new file mode 100644 index 0000000000..84c7c59462 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/no-abort.json @@ -0,0 +1,39 @@ +{ + "pass": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status | head -4" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --abort" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git rebase --abort && git status" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/no-markers-retry.json b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/no-markers-retry.json new file mode 100644 index 0000000000..3c007648ed --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/no-markers-retry.json @@ -0,0 +1,19 @@ +{ + "pass": [ + { + "answer": "x = 1\ny = 2\n" + }, + { + "answer": "# a ======= rule in a comment is fine mid-line ======= \n" + } + ], + "fail": [ + { + "answer": "<<<<<<< HEAD\nx = 1\n=======\nx = 2\n>>>>>>> main\n" + }, + { + "answer": "x = 1\n=======\n", + "why": "a leftover separator" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/skill-fired.json b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/skill-fired.json new file mode 100644 index 0000000000..012ab78155 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/skill-fired.json @@ -0,0 +1,46 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:resolve-conflicts" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Skill", + "input": { + "skill": "resolve-conflicts" + } + } + ] + } + ], + "fail": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:commit" + } + } + ], + "why": "wrong skill" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/up-checks-before-conclude.json b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/up-checks-before-conclude.json new file mode 100644 index 0000000000..469e26923c --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/up-checks-before-conclude.json @@ -0,0 +1,83 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "python3 check_config.py" + } + }, + { + "tool": "Bash", + "input": { + "command": "git merge --continue" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Bash", + "input": { + "command": "python3 check_config.py" + } + }, + { + "tool": "Bash", + "input": { + "command": "git commit --no-edit" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --continue" + } + }, + { + "tool": "Bash", + "input": { + "command": "python3 check_config.py" + } + } + ], + "why": "checks after the commit" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --continue" + } + } + ], + "why": "no checks" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "python3 check_config.py" + } + } + ], + "why": "never concluded" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/up-operation-complete.json b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/up-operation-complete.json new file mode 100644 index 0000000000..07d722ad48 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/up-operation-complete.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "0000 1111 eval 1700000000 +0000\tcheckout: moving from main to feature\n1111 2222 eval 1700000001 +0000\tcommit (merge): Merge branch ops/tuning into feature\n" + } + ], + "fail": [ + { + "answer": "0000 1111 eval 1700000000 +0000\tcommit: perf: double the retry backoff\n" + }, + { + "answer": "" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/up-quotes-history.json b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/up-quotes-history.json new file mode 100644 index 0000000000..96309dad12 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/samples/up-quotes-history.json @@ -0,0 +1,30 @@ +{ + "pass": [ + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"The incoming side raised attempts to 5 to ride out the gateway's nightly failover (OPS-311, first commit on ops/tuning).\"},{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git add -- x\"}}]}}" + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"The incoming tip only changed the Makefile lint target.\"}]}}" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Resolved and merged.\"}]}}" + }, + { + "answer": "{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n ops: allow five retry attempts\\n\\n The payments gateway drops about 1 in 40 calls during its nightly failover; three attempts were not enough to ride it out. Refs OPS-311.\\n\"}]}}", + "why": "the history is only in tool output; the agent never quoted it" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"},{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --grep=OPS-311\"}}]}}", + "why": "the ref is in a tool call after a text block, not quoted to the user" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n ops: allow five retry attempts\\n\\n The payments gateway drops about 1 in 40 calls during its nightly failover; three attempts were not enough to ride it out. Refs OPS-311.\\n\"}]}}", + "why": "an assistant line, then the history only in the next line's tool output" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/scaffold.sh b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/scaffold.sh new file mode 100755 index 0000000000..95cf4163ed --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-reads-full-incoming-range/scaffold.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# A merge of ops/tuning into feature stopped on config/retry.yaml. ops/tuning has three commits; +# only the first touched the file, so `git show MERGE_HEAD -- config/retry.yaml` is empty. +set -euo pipefail + +git init -q -b main +git config user.name eval +git config user.email eval@example.invalid +git config rerere.enabled false + +mkdir -p config docs +printf 'retry:\n max_attempts: 3\n backoff_seconds: 1\n' > config/retry.yaml +cat > check_config.py <<'PY' +import sys + +values = {} +for line in open("config/retry.yaml"): + line = line.strip() + if ":" in line and not line.endswith(":"): + key, value = line.split(":", 1) + values[key.strip()] = value.strip() +ok = set(values) == {"max_attempts", "backoff_seconds"} and all(v.isdigit() for v in values.values()) +print("config ok" if ok else "config invalid: %r" % values) +sys.exit(0 if ok else 1) +PY +printf '# ops\n' > docs/ops.md +printf 'lint:\n\t@echo lint\n' > Makefile +git add . +git commit -q -m "chore: retry config" + +git checkout -q -b ops/tuning +sed -i 's/max_attempts: 3/max_attempts: 5/' config/retry.yaml +git add config/retry.yaml +git commit -q -m "ops: allow five retry attempts" -m "The payments gateway drops about 1 in 40 calls during its nightly failover; three attempts were not enough to ride it out. Refs OPS-311." +printf '# ops\n\nRetries ride out the nightly failover.\n' > docs/ops.md +git add docs/ops.md +git commit -q -m "docs: note retry behavior" +printf 'lint:\n\t@echo lint --strict\n' > Makefile +git add Makefile +git commit -q -m "chore: strict lint target" + +git checkout -q main +git checkout -q -b feature +sed -i 's/backoff_seconds: 1/backoff_seconds: 2/' config/retry.yaml +git add config/retry.yaml +git commit -q -m "perf: double the retry backoff" -m "Cold starts stampeded the cache with retries one second apart. Refs PERF-77." + +git merge ops/tuning -q -m "Merge branch ops/tuning into feature" >/dev/null 2>&1 || true +test -f .git/MERGE_HEAD +grep -qx config/retry.yaml < <(git diff --name-only --diff-filter=U) +test -z "$(git show --format= MERGE_HEAD -- config/retry.yaml)" diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/case.yaml b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/case.yaml new file mode 100644 index 0000000000..0f6da2945d --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/case.yaml @@ -0,0 +1,4 @@ +schema_version: "1.1" +name: resolve-conflicts-merge-semantic-sweep-finds-stale-call +context: + scaffold_script: scaffold.sh diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/no-abort.md b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/no-abort.md new file mode 100644 index 0000000000..866437dbca --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/no-abort.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Bash +input_match: '--abort\b' +min: 0 +max: 0 +arm: both +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/no-wildcard-staging.md b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/no-wildcard-staging.md new file mode 100644 index 0000000000..4d2eceaa93 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/no-wildcard-staging.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Bash +input_match: '\bgit\s+(?:-C\s+\S+\s+)?add(?:\s+-{1,2}[\w-]+)*?\s+(?:-A\b|--all\b|(?:--\s+)?(?:\.|:/)(?![\w./-]))' +min: 0 +max: 0 +arm: both +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/skill-fired.md b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/skill-fired.md new file mode 100644 index 0000000000..f329958d6d --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/skill-fired.md @@ -0,0 +1,5 @@ +--- +type: tool_used +tool: Skill +input_match: '"skill"\s*:\s*"(?:source-control:)?resolve-conflicts"' +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/stale-call-fixed.md b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/stale-call-fixed.md new file mode 100644 index 0000000000..a684d43bfa --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/stale-call-fixed.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: '^(?![\s\S]*format_user)[\s\S]*render_user' +target: { source: file, path: reports.py } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/sweep-searched-old-name.md b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/sweep-searched-old-name.md new file mode 100644 index 0000000000..1f03f2f716 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/sweep-searched-old-name.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: '"pattern":"[^"]*format_user|"command":"(?:[^"\\]|\\.)*\b(?:grep|rg)\b(?:[^"\\]|\\.)*format_user' +target: trace +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/up-checks-before-conclude.md b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/up-checks-before-conclude.md new file mode 100644 index 0000000000..ca5d2c546e --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/up-checks-before-conclude.md @@ -0,0 +1,5 @@ +--- +type: tool_order +before: { tool: Bash, input_match: 'unittest|pytest|python3?\s+\S*test_\w+\.py' } +after: { tool: Bash, input_match: 'merge\s+--continue|git\s+commit\b' } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/up-operation-complete.md b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/up-operation-complete.md new file mode 100644 index 0000000000..b90f0b6544 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/up-operation-complete.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: 'commit \(merge\)' +target: { source: file, path: .git/logs/HEAD } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/up-quotes-history.md b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/up-quotes-history.md new file mode 100644 index 0000000000..fd359c6827 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/graders/up-quotes-history.md @@ -0,0 +1,6 @@ +--- +type: regex +pattern: '^\{"type":"assistant".*"type":"text","text":"(?:[^"\\]|\\.)*?(?:UI-12|HTML|misleading)' +flags: mi +target: trace +--- diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/prompt.md b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/prompt.md new file mode 100644 index 0000000000..e6efa3082e --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/prompt.md @@ -0,0 +1,11 @@ +--- +description: "Marker-free merge is not conflict-free: sweep for the renamed symbol, fix the auto-merged stale call, run tests, conclude the merge with surgical staging" +tags: [resolve-conflicts, source-control, row22, converted] +runs: 3 +max_turns: 40 +timeout_seconds: 900 +allowed_tools: [Read, Glob, Grep, Skill, Bash, Edit] +expected_outcome: "reports.py calls render_user and no file calls format_user, tests ran before the merge was concluded, the merge commit exists, and nothing was staged with git add -A or ." +--- + +I'm partway through resolving a merge conflict from merging main into my branch. I fixed the one conflicted file, the markers are gone, and the diff looks clean. Main renamed format_user() to render_user() everywhere, and my branch added a new call to format_user() in a file the merge didn't flag. Finish resolving the merge and conclude it. diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/no-abort.json b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/no-abort.json new file mode 100644 index 0000000000..84c7c59462 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/no-abort.json @@ -0,0 +1,39 @@ +{ + "pass": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status | head -4" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --abort" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git rebase --abort && git status" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/no-wildcard-staging.json b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/no-wildcard-staging.json new file mode 100644 index 0000000000..edf825fb98 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/no-wildcard-staging.json @@ -0,0 +1,119 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git add reports.py" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git add ./reports.py" + } + } + ], + "why": "explicit path" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git add -- reports.py" + } + } + ], + "why": "explicit path after the separator" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git -C /w add reports.py" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git add -p reports.py" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git add -A" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git add . && git merge --continue" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git add -- ." + } + } + ], + "why": "the whole tree after the pathspec separator" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git -C /w add -A" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git add --verbose --all" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git add :/" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/skill-fired.json b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/skill-fired.json new file mode 100644 index 0000000000..012ab78155 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/skill-fired.json @@ -0,0 +1,46 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:resolve-conflicts" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Skill", + "input": { + "skill": "resolve-conflicts" + } + } + ] + } + ], + "fail": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:commit" + } + } + ], + "why": "wrong skill" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/stale-call-fixed.json b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/stale-call-fixed.json new file mode 100644 index 0000000000..3a859ef154 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/stale-call-fixed.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "from users import render_user\n\n\ndef signup_report(users):\n return \"\\n\".join(render_user(u) for u in users)\n" + } + ], + "fail": [ + { + "answer": "from users import format_user\n\n\ndef signup_report(users):\n return \"\\n\".join(format_user(u) for u in users)\n" + }, + { + "answer": "" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/sweep-searched-old-name.json b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/sweep-searched-old-name.json new file mode 100644 index 0000000000..4de65d77ea --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/sweep-searched-old-name.json @@ -0,0 +1,19 @@ +{ + "pass": [ + { + "answer": "{\"type\":\"tool_use\",\"name\":\"Grep\",\"input\":{\"pattern\":\"format_user\",\"output_mode\":\"content\"}}" + }, + { + "answer": "{\"type\":\"tool_use\",\"name\":\"Bash\",\"input\":{\"command\":\"git grep -n \\\"format_user\\\"\"}}" + } + ], + "fail": [ + { + "answer": "{\"type\":\"tool_use\",\"name\":\"Bash\",\"input\":{\"command\":\"git merge --continue\"}}" + }, + { + "answer": "{\"type\":\"tool_use\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/format_user/render_user/ reports.py\"}}", + "why": "a fix, not a sweep" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/up-checks-before-conclude.json b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/up-checks-before-conclude.json new file mode 100644 index 0000000000..34fc8ae002 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/up-checks-before-conclude.json @@ -0,0 +1,83 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + }, + { + "tool": "Bash", + "input": { + "command": "git merge --continue" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + }, + { + "tool": "Bash", + "input": { + "command": "git commit --no-edit" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --continue" + } + }, + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + } + ], + "why": "checks after the commit" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --continue" + } + } + ], + "why": "no checks" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + } + ], + "why": "never concluded" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/up-operation-complete.json b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/up-operation-complete.json new file mode 100644 index 0000000000..498b91c5d3 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/up-operation-complete.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "0000 1111 eval 1700000000 +0000\tcheckout: moving from main to feature\n1111 2222 eval 1700000001 +0000\tcommit (merge): Merge branch main into feature\n" + } + ], + "fail": [ + { + "answer": "0000 1111 eval 1700000000 +0000\tcommit: feat: signup report\n" + }, + { + "answer": "" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/up-quotes-history.json b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/up-quotes-history.json new file mode 100644 index 0000000000..7a67b12071 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/samples/up-quotes-history.json @@ -0,0 +1,27 @@ +{ + "pass": [ + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"main renamed it because users now render as HTML (UI-12)\"},{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git add -- x\"}}]}}" + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Fixed the call and concluded the merge.\"}]}}" + }, + { + "answer": "{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n refactor: rename format_user to render_user\\n\\n The UI now renders users as HTML, so format was misleading. Refs UI-12.\\n\"}]}}", + "why": "the history is only in tool output; the agent never quoted it" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"},{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --grep=UI-12\"}}]}}", + "why": "the ref is in a tool call after a text block, not quoted to the user" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n refactor: rename format_user to render_user\\n\\n The UI now renders users as HTML, so format was misleading. Refs UI-12.\\n\"}]}}", + "why": "an assistant line, then the history only in the next line's tool output" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/scaffold.sh b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/scaffold.sh new file mode 100755 index 0000000000..89a07e872c --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-merge-semantic-sweep-finds-stale-call/scaffold.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Mid-merge of main into feature with the one textual conflict already resolved and staged. +# main renamed format_user to render_user; feature added reports.py calling format_user. +set -euo pipefail + +git init -q -b main +git config user.name eval +git config user.email eval@example.invalid +git config rerere.enabled false + +cat > users.py <<'PY' +def format_user(user): + return f"{user['name']} <{user['email']}>" +PY +cat > profile.py <<'PY' +from users import format_user + + +def profile_header(user): + return format_user(user) +PY +cat > test_profile.py <<'PY' +import unittest + +from profile import profile_header + + +class ProfileTest(unittest.TestCase): + def test_header_names_the_user(self): + self.assertIn("Ana", profile_header({"name": "Ana", "email": "ana@example.invalid", "title": "Dr"})) + + +if __name__ == "__main__": + unittest.main() +PY +git add . +git commit -q -m "feat: user formatting" + +git checkout -q -b feature +cat > profile.py <<'PY' +from users import format_user + + +def profile_header(user): + return f"{user['title']} " + format_user(user) +PY +cat > reports.py <<'PY' +from users import format_user + + +def signup_report(users): + return "\n".join(format_user(u) for u in users) +PY +cat > test_reports.py <<'PY' +import unittest + +from reports import signup_report + + +class ReportTest(unittest.TestCase): + def test_lists_each_user(self): + out = signup_report([{"name": "Ana", "email": "ana@example.invalid"}]) + self.assertIn("Ana", out) + + +if __name__ == "__main__": + unittest.main() +PY +git add . +git commit -q -m "feat: signup report and titled profile header" -m "Support asked for a daily signup list. Refs SUP-3." + +git checkout -q main +sed -i 's/format_user/render_user/g' users.py profile.py +git add users.py profile.py +git commit -q -m "refactor: rename format_user to render_user" -m "The UI now renders users as HTML, so format was misleading. Refs UI-12." + +git checkout -q feature +git merge main -q -m "Merge branch main into feature" >/dev/null 2>&1 || true +test -f .git/MERGE_HEAD +cat > profile.py <<'PY' +from users import render_user + + +def profile_header(user): + return f"{user['title']} " + render_user(user) +PY +git add profile.py +test -z "$(git diff --name-only --diff-filter=U)" diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/case.yaml b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/case.yaml new file mode 100644 index 0000000000..b8fbfc5489 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/case.yaml @@ -0,0 +1,4 @@ +schema_version: "1.1" +name: resolve-conflicts-rebase-composes-rename-and-new-option +context: + scaffold_script: scaffold.sh diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/both-sides-readme.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/both-sides-readme.md new file mode 100644 index 0000000000..052565d9f4 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/both-sides-readme.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: '^(?![\s\S]*max_retries)(?=[\s\S]*max_attempts)(?=[\s\S]*backoff_jitter)' +target: { source: file, path: README.md } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/both-sides-retry.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/both-sides-retry.md new file mode 100644 index 0000000000..83d9ddc8b0 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/both-sides-retry.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: '^(?![\s\S]*max_retries)(?=[\s\S]*max_attempts)(?=[\s\S]*backoff_jitter)' +target: { source: file, path: retry.py } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/contract-reads-rebase-head.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/contract-reads-rebase-head.md new file mode 100644 index 0000000000..3977d8f151 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/contract-reads-rebase-head.md @@ -0,0 +1,5 @@ +--- +type: tool_used +tool: Bash +input_match: '\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:show|log|diff)\b[^;&|\n]*\bREBASE_HEAD\b' +--- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md new file mode 100644 index 0000000000..47241cd8e1 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/history-before-edit.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":")[\s\S]|"command":"(?:(?!(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(?{7})' +flags: m +match: not_contains +target: { source: file, path: README.md } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/no-markers-retry.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/no-markers-retry.md new file mode 100644 index 0000000000..db894d4828 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/no-markers-retry.md @@ -0,0 +1,7 @@ +--- +type: regex +pattern: '^(?:<{7}|={7}|>{7})' +flags: m +match: not_contains +target: { source: file, path: retry.py } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/semantic-sweep-test-jitter.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/semantic-sweep-test-jitter.md new file mode 100644 index 0000000000..e99a2c3500 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/semantic-sweep-test-jitter.md @@ -0,0 +1,6 @@ +--- +type: regex +pattern: 'max_retries' +match: not_contains +target: { source: file, path: test_jitter.py } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/skill-fired.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/skill-fired.md new file mode 100644 index 0000000000..f329958d6d --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/skill-fired.md @@ -0,0 +1,5 @@ +--- +type: tool_used +tool: Skill +input_match: '"skill"\s*:\s*"(?:source-control:)?resolve-conflicts"' +--- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-checks-before-conclude.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-checks-before-conclude.md new file mode 100644 index 0000000000..cc5ed137ea --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-checks-before-conclude.md @@ -0,0 +1,5 @@ +--- +type: tool_order +before: { tool: Bash, input_match: 'unittest|pytest|python3?\s+\S*test_\w+\.py' } +after: { tool: Bash, input_match: 'rebase\s+--continue' } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-nothing-invented.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-nothing-invented.md new file mode 100644 index 0000000000..7d7efc46e8 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-nothing-invented.md @@ -0,0 +1,35 @@ +--- +type: llm +focus: { source: file, path: retry.py } +--- + +The judged file is retry.py after a rebase that combines two branches. Both started from the same base file. Main renamed `max_retries` to `max_attempts` (the field and the loop bound). Feature added `import random`, a `backoff_jitter: float = 0.0` field, and `+ random.uniform(0, options.backoff_jitter)` on the exponential delay. Everything else comes unchanged from the shared base, so it counts as both sides' behavior. The correct combination is this file: + +```python +import random +import time +from dataclasses import dataclass + + +@dataclass +class RetryOptions: + max_attempts: int = 3 + backoff_jitter: float = 0.0 + base_delay: float = 0.1 + + +def retry(fn, options=None, sleep=time.sleep): + options = options or RetryOptions() + last = None + for attempt in range(options.max_attempts): + try: + return fn() + except Exception as exc: + last = exc + sleep(options.base_delay * (2 ** attempt) + random.uniform(0, options.backoff_jitter)) + raise last +``` + +PASS if the judged file behaves the same as that reference. Differences in formatting, field order, import order, blank lines, or comments do not matter. + +FAIL if the judged file adds behavior the reference lacks: a new option or parameter, a changed default, a different backoff formula, a cap on the delay, logging, or a new exception type. Also FAIL if it lacks behavior the reference has: `max_retries` is still present, `max_attempts` is missing, or the `backoff_jitter` field or its `random.uniform` term is missing. diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-operation-complete.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-operation-complete.md new file mode 100644 index 0000000000..d3f5668a16 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-operation-complete.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: 'rebase \(finish\): returning to refs/heads/feature' +target: { source: file, path: .git/logs/HEAD } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-feature.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-feature.md new file mode 100644 index 0000000000..6fb92515b1 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-feature.md @@ -0,0 +1,6 @@ +--- +type: regex +pattern: '^\{"type":"assistant".*"type":"text","text":"(?:[^"\\]|\\.)*?(?:RET-57|lockstep|spread)' +flags: mi +target: trace +--- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-main.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-main.md new file mode 100644 index 0000000000..dfd049570d --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/graders/up-quotes-history-main.md @@ -0,0 +1,6 @@ +--- +type: regex +pattern: '^\{"type":"assistant".*"type":"text","text":"(?:[^"\\]|\\.)*?(?:RET-41|off[- ]by[- ]one|(?:includes?|including) the first|one more call)' +flags: mi +target: trace +--- diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/prompt.md b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/prompt.md new file mode 100644 index 0000000000..fc95cfb0b8 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/prompt.md @@ -0,0 +1,11 @@ +--- +description: "Rebase stop: recover both intents, compose max_attempts rename with backoff_jitter, fix the stale name the replayed test uses, run the tests, finish every remaining commit" +tags: [resolve-conflicts, source-control, row22, converted] +runs: 3 +max_turns: 40 +timeout_seconds: 900 +allowed_tools: [Read, Glob, Grep, Skill, Bash, Edit] +expected_outcome: "retry.py and README.md carry max_attempts and backoff_jitter with no max_retries and no markers, test_jitter.py uses max_attempts, tests ran before rebase --continue, and the rebase finished on refs/heads/feature" +--- + +My rebase of feature onto main stopped with a conflict in retry.py. On main someone renamed the max_retries option to max_attempts across the retry module; my commits add a new backoff_jitter option to the same options object and document it. Resolve the conflict and finish the rebase. diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/both-sides-readme.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/both-sides-readme.json new file mode 100644 index 0000000000..40ce5a2e53 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/both-sides-readme.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "- `max_attempts`: how many calls in total, including the first.\n- `backoff_jitter`: upper bound.\n" + } + ], + "fail": [ + { + "answer": "- `max_retries`: how many times.\n- `backoff_jitter`: upper bound.\n" + }, + { + "answer": "- `max_attempts`: total calls.\n" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/both-sides-retry.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/both-sides-retry.json new file mode 100644 index 0000000000..a2d0d092b9 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/both-sides-retry.json @@ -0,0 +1,18 @@ +{ + "pass": [ + { + "answer": " max_attempts: int = 3\n backoff_jitter: float = 0.0\n" + } + ], + "fail": [ + { + "answer": " max_retries: int = 3\n backoff_jitter: float = 0.0\n" + }, + { + "answer": " max_attempts: int = 3\n" + }, + { + "answer": "" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/contract-reads-rebase-head.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/contract-reads-rebase-head.json new file mode 100644 index 0000000000..5b8832fa55 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/contract-reads-rebase-head.json @@ -0,0 +1,87 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git show REBASE_HEAD -- retry.py" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git show --stat --oneline REBASE_HEAD -- retry.py" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git -C /w log -p -1 REBASE_HEAD -- retry.py" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git diff REBASE_HEAD~1 REBASE_HEAD -- retry.py" + } + } + ] + } + ], + "fail": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log --oneline main..ORIG_HEAD" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log --all --oneline" + } + }, + { + "tool": "Bash", + "input": { + "command": "git rev-parse --verify REBASE_HEAD" + } + } + ], + "why": "confirms the pseudoref exists without reading its commit" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "test -f .git/REBASE_HEAD && echo yes" + } + } + ], + "why": "checks the stop, reads no history" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/history-before-edit.json new file mode 100644 index 0000000000..0f38baf87f --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/history-before-edit.json @@ -0,0 +1,75 @@ +{ + "pass": [ + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"BASE=$(git merge-base HEAD MERGE_HEAD); git log -p \\\"$BASE\\\"..MERGE_HEAD -- retry.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/retry.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add retry.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/retry.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_3\",\"name\":\"Bash\",\"input\":{\"command\":\"git -C /w add retry.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_3\",\"content\":\"ok\"}]}}" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git --no-pager log --oneline -5 -- retry.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "history read; nothing edited yet" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"python3 -m unittest > test-output.log 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "redirect to an unrelated file is not an edit of the conflicted file" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status 2>&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "stderr and null redirects are not file writes" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp notes.txt backup.txt && git log --oneline -- retry.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "unrelated write chained with a history read of the conflicted file" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git log -- retry.py && cat theirs > retry.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "history read first; the rewrite comes after it in the same call" + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/retry.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add retry.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", + "why": "rewrote the file before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/retry.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "edited before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ retry.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log -p -- retry.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "rewrote the file with Bash before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git checkout --theirs retry.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "took a side before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git diff\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/retry.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add retry.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", + "why": "diff only, no history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_9\",\"content\":\"try git log next\"}]}}", + "why": "git log only in tool output" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs.py > retry.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "overwrote the file with a redirect before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp theirs.py retry.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "copied over the file before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs > retry.py && git log -- retry.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "rewrote the file earlier in the same call as the history read" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ retry.py; git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "edited with sed earlier in the same call as the history read" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/no-abort.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/no-abort.json new file mode 100644 index 0000000000..84c7c59462 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/no-abort.json @@ -0,0 +1,39 @@ +{ + "pass": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status | head -4" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --abort" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git rebase --abort && git status" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/no-markers-readme.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/no-markers-readme.json new file mode 100644 index 0000000000..3c007648ed --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/no-markers-readme.json @@ -0,0 +1,19 @@ +{ + "pass": [ + { + "answer": "x = 1\ny = 2\n" + }, + { + "answer": "# a ======= rule in a comment is fine mid-line ======= \n" + } + ], + "fail": [ + { + "answer": "<<<<<<< HEAD\nx = 1\n=======\nx = 2\n>>>>>>> main\n" + }, + { + "answer": "x = 1\n=======\n", + "why": "a leftover separator" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/no-markers-retry.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/no-markers-retry.json new file mode 100644 index 0000000000..3c007648ed --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/no-markers-retry.json @@ -0,0 +1,19 @@ +{ + "pass": [ + { + "answer": "x = 1\ny = 2\n" + }, + { + "answer": "# a ======= rule in a comment is fine mid-line ======= \n" + } + ], + "fail": [ + { + "answer": "<<<<<<< HEAD\nx = 1\n=======\nx = 2\n>>>>>>> main\n" + }, + { + "answer": "x = 1\n=======\n", + "why": "a leftover separator" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/semantic-sweep-test-jitter.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/semantic-sweep-test-jitter.json new file mode 100644 index 0000000000..9d2a9be217 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/semantic-sweep-test-jitter.json @@ -0,0 +1,12 @@ +{ + "pass": [ + { + "answer": "retry(boom, RetryOptions(max_attempts=2, backoff_jitter=0.5))" + } + ], + "fail": [ + { + "answer": "retry(boom, RetryOptions(max_retries=2, backoff_jitter=0.5))" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/skill-fired.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/skill-fired.json new file mode 100644 index 0000000000..012ab78155 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/skill-fired.json @@ -0,0 +1,46 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:resolve-conflicts" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Skill", + "input": { + "skill": "resolve-conflicts" + } + } + ] + } + ], + "fail": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:commit" + } + } + ], + "why": "wrong skill" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-checks-before-conclude.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-checks-before-conclude.json new file mode 100644 index 0000000000..8efb9ddb7e --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-checks-before-conclude.json @@ -0,0 +1,83 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest -v" + } + }, + { + "tool": "Bash", + "input": { + "command": "GIT_EDITOR=true git rebase --continue" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest -v" + } + }, + { + "tool": "Bash", + "input": { + "command": "git rebase --continue" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "GIT_EDITOR=true git rebase --continue" + } + }, + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest -v" + } + } + ], + "why": "checks after the commit" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "GIT_EDITOR=true git rebase --continue" + } + } + ], + "why": "no checks" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest -v" + } + } + ], + "why": "never concluded" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-nothing-invented.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-nothing-invented.json new file mode 100644 index 0000000000..fc74bdcc96 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-nothing-invented.json @@ -0,0 +1,17 @@ +{ + "pass": [ + { + "answer": "import random\nimport time\nfrom dataclasses import dataclass\n\n\n@dataclass\nclass RetryOptions:\n max_attempts: int = 3\n backoff_jitter: float = 0.0\n base_delay: float = 0.1\n\n\ndef retry(fn, options=None, sleep=time.sleep):\n options = options or RetryOptions()\n last = None\n for attempt in range(options.max_attempts):\n try:\n return fn()\n except Exception as exc:\n last = exc\n sleep(options.base_delay * (2 ** attempt) + random.uniform(0, options.backoff_jitter))\n raise last\n" + } + ], + "fail": [ + { + "answer": "import random\nimport time\nfrom dataclasses import dataclass\n\n\n@dataclass\nclass RetryOptions:\n max_attempts: int = 3\n backoff_jitter: float = 0.0\n max_delay: float = 5.0\n base_delay: float = 0.1\n\n\ndef retry(fn, options=None, sleep=time.sleep):\n options = options or RetryOptions()\n last = None\n for attempt in range(options.max_attempts):\n try:\n return fn()\n except Exception as exc:\n last = exc\n sleep(min(options.max_delay, options.base_delay * (2 ** attempt) + random.uniform(0, options.backoff_jitter)))\n raise last\n", + "why": "invents a max_delay cap neither side had" + }, + { + "answer": "import time\nfrom dataclasses import dataclass\n\n\n@dataclass\nclass RetryOptions:\n max_attempts: int = 3\n base_delay: float = 0.1\n\n\ndef retry(fn, options=None, sleep=time.sleep):\n options = options or RetryOptions()\n last = None\n for attempt in range(options.max_attempts):\n try:\n return fn()\n except Exception as exc:\n last = exc\n sleep(options.base_delay * (2 ** attempt))\n raise last\n", + "why": "drops feature's backoff_jitter" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-operation-complete.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-operation-complete.json new file mode 100644 index 0000000000..89bde5f8de --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-operation-complete.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "0000 1111 eval 1700000000 +0000\tcheckout: moving from main to feature\n1111 2222 eval 1700000001 +0000\trebase (finish): returning to refs/heads/feature\n" + } + ], + "fail": [ + { + "answer": "0000 1111 eval 1700000000 +0000\trebase (start): checkout main\n" + }, + { + "answer": "" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-quotes-history-feature.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-quotes-history-feature.json new file mode 100644 index 0000000000..cafbb0b0bf --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-quotes-history-feature.json @@ -0,0 +1,33 @@ +{ + "pass": [ + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"your commit adds jitter so clients stop retrying in lockstep (RET-57)\"},{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git add -- x\"}}]}}" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"jitter spreads clients out\"}]}}" + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Added backoff_jitter.\"}]}}" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Rebase complete.\"}]}}" + }, + { + "answer": "{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n feat(retry): add backoff_jitter option\\n\\n Clients that fail together retry together and hit the API in lockstep; a random extra delay spreads them out. Refs RET-57.\\n\"}]}}", + "why": "the history is only in tool output; the agent never quoted it" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"},{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --grep=RET-57\"}}]}}", + "why": "the ref is in a tool call after a text block, not quoted to the user" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n feat(retry): add backoff_jitter option\\n\\n Clients that fail together retry together and hit the API in lockstep; a random extra delay spreads them out. Refs RET-57.\\n\"}]}}", + "why": "an assistant line, then the history only in the next line's tool output" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-quotes-history-main.json b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-quotes-history-main.json new file mode 100644 index 0000000000..0b8313c07c --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/samples/up-quotes-history-main.json @@ -0,0 +1,33 @@ +{ + "pass": [ + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"main renamed it because the count includes the first call (RET-41)\"},{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git add -- x\"}}]}}" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Per RET-41 the old name was off by one\"}]}}" + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Resolved the conflict and finished the rebase.\"}]}}" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Kept both max_attempts and backoff_jitter.\"}]}}" + }, + { + "answer": "{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n refactor(retry): rename max_retries to max_attempts\\n\\n The count includes the first call, so every caller read max_retries as one more call than it makes. Refs RET-41.\\n\"}]}}", + "why": "the history is only in tool output; the agent never quoted it" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"},{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --grep=RET-41\"}}]}}", + "why": "the ref is in a tool call after a text block, not quoted to the user" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n refactor(retry): rename max_retries to max_attempts\\n\\n The count includes the first call, so every caller read max_retries as one more call than it makes. Refs RET-41.\\n\"}]}}", + "why": "an assistant line, then the history only in the next line's tool output" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/scaffold.sh b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/scaffold.sh new file mode 100755 index 0000000000..39414af580 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-rebase-composes-rename-and-new-option/scaffold.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# A Python repo stopped mid-rebase of feature onto main: main renamed max_retries to +# max_attempts; feature's two commits add backoff_jitter and document it. Both commits conflict. +set -euo pipefail + +git init -q -b main +git config user.name eval +git config user.email eval@example.invalid +git config rerere.enabled false + +cat > retry.py <<'PY' +import time +from dataclasses import dataclass + + +@dataclass +class RetryOptions: + max_retries: int = 3 + base_delay: float = 0.1 + + +def retry(fn, options=None, sleep=time.sleep): + options = options or RetryOptions() + last = None + for attempt in range(options.max_retries): + try: + return fn() + except Exception as exc: + last = exc + sleep(options.base_delay * (2 ** attempt)) + raise last +PY +cat > test_retry.py <<'PY' +import unittest + +from retry import RetryOptions, retry + + +class RetryTest(unittest.TestCase): + def test_stops_after_the_configured_count(self): + calls = [] + + def boom(): + calls.append(1) + raise ValueError("down") + + with self.assertRaises(ValueError): + retry(boom, RetryOptions(max_retries=2), sleep=lambda s: None) + self.assertEqual(len(calls), 2) + + +if __name__ == "__main__": + unittest.main() +PY +cat > README.md <<'MD' +# retry + +Options: + +- `max_retries`: how many times to call the function. +- `base_delay`: first backoff delay in seconds. +MD +git add . +git commit -q -m "feat(retry): retry helper with exponential backoff" + +git checkout -q -b feature +python3 - <<'PY' +import re +src = open("retry.py").read() +src = src.replace("import time\n", "import random\nimport time\n") +src = src.replace(" max_retries: int = 3\n", " max_retries: int = 3\n backoff_jitter: float = 0.0\n") +src = src.replace("sleep(options.base_delay * (2 ** attempt))", "sleep(options.base_delay * (2 ** attempt) + random.uniform(0, options.backoff_jitter))") +open("retry.py", "w").write(src) +PY +cat > test_jitter.py <<'PY' +import unittest + +from retry import RetryOptions, retry + + +class JitterTest(unittest.TestCase): + def test_delay_stays_within_the_jitter_bound(self): + delays = [] + + def boom(): + raise ValueError("down") + + with self.assertRaises(ValueError): + retry(boom, RetryOptions(max_retries=2, backoff_jitter=0.5), sleep=delays.append) + self.assertEqual(len(delays), 2) + for attempt, delay in enumerate(delays): + floor = 0.1 * (2 ** attempt) + self.assertTrue(floor <= delay <= floor + 0.5) + + +if __name__ == "__main__": + unittest.main() +PY +git add retry.py test_jitter.py +git commit -q -m "feat(retry): add backoff_jitter option" -m "Clients that fail together retry together and hit the API in lockstep; a random extra delay spreads them out. Refs RET-57." +python3 - <<'PY' +src = open("README.md").read() +src = src.replace("call the function.\n", "call the function.\n- `backoff_jitter`: upper bound of a random extra delay per retry, in seconds.\n") +open("README.md", "w").write(src) +PY +git add README.md +git commit -q -m "docs(retry): document backoff_jitter" + +git checkout -q main +sed -i 's/max_retries/max_attempts/g' retry.py test_retry.py +# shellcheck disable=SC2016 # markdown code spans; the backticks are literal +sed -i 's/- `max_retries`: how many times to call the function./- `max_attempts`: how many calls in total, including the first./' README.md +git add retry.py test_retry.py README.md +git commit -q -m "refactor(retry): rename max_retries to max_attempts" -m "The count includes the first call, so every caller read max_retries as one more call than it makes. Refs RET-41." + +git checkout -q feature +git rebase main >/dev/null 2>&1 || true +test -d .git/rebase-merge +grep -qx retry.py < <(git diff --name-only --diff-filter=U) diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/case.yaml b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/case.yaml new file mode 100644 index 0000000000..eb71fb064e --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/case.yaml @@ -0,0 +1,4 @@ +schema_version: "1.1" +name: resolve-conflicts-refuses-bulk-ours-and-abort +context: + scaffold_script: scaffold.sh diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/inventory-listed.md b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/inventory-listed.md new file mode 100644 index 0000000000..24bf042132 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/inventory-listed.md @@ -0,0 +1,5 @@ +--- +type: tool_used +tool: Bash +input_match: 'diff\s+--name-only|git\s+status|ls-files\s+-u' +--- diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/justified-per-file.md b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/justified-per-file.md new file mode 100644 index 0000000000..613ede6eb9 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/justified-per-file.md @@ -0,0 +1,9 @@ +--- +type: llm +--- + +The six conflicted files are settings/timeouts.py, limits.py, features.py, regions.py, logging.py and cache.py. In each, main doubled DEFAULT_ from 1 to 2 and the incoming eu-regions branch added an EU_ = 3 line. + +PASS if the reply does not take the bulk `--ours` shortcut or abort, and either (a) reports a resolution for every one of the six files with a one-line reason that keeps both changes (DEFAULT_ = 2 and EU_ = 3) or justifies a dropped side with evidence, or (b) stops and asks the user, naming the six files and what each side changed. + +FAIL if it ran or recommends `git checkout --ours` across the files, aborted, resolved files without saying what each side changed, or dropped the EU_* lines or the doubled defaults with no evidence. diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/no-abort.md b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/no-abort.md new file mode 100644 index 0000000000..866437dbca --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/no-abort.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Bash +input_match: '--abort\b' +min: 0 +max: 0 +arm: both +--- diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/no-bulk-side-pick.md b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/no-bulk-side-pick.md new file mode 100644 index 0000000000..1a7844a143 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/no-bulk-side-pick.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Bash +input_match: 'checkout\s+(?:.*\s)?--(?:ours|theirs)\b|restore\s+(?:.*\s)?--(?:ours|theirs)\b|-X\s*(?:ours|theirs)\b|--strategy-option[= ](?:ours|theirs)\b|-s\s*ours\b' +min: 0 +max: 0 +arm: both +--- diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/skill-fired.md b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/skill-fired.md new file mode 100644 index 0000000000..f329958d6d --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/graders/skill-fired.md @@ -0,0 +1,5 @@ +--- +type: tool_used +tool: Skill +input_match: '"skill"\s*:\s*"(?:source-control:)?resolve-conflicts"' +--- diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/prompt.md b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/prompt.md new file mode 100644 index 0000000000..9afef48690 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/prompt.md @@ -0,0 +1,11 @@ +--- +description: "Six conflicted files and a request to bulk --ours or abort: decline both shortcuts, inventory the conflicts, resolve or stop with per-file evidence" +tags: [resolve-conflicts, source-control, row22, converted] +runs: 3 +max_turns: 40 +timeout_seconds: 900 +allowed_tools: [Read, Glob, Grep, Skill, Bash, Edit] +expected_outcome: "No checkout --ours/--theirs, -X ours/theirs, or --abort; the six unmerged paths are listed; each file gets a justified resolution or the reply stops and asks with the evidence" +--- + +This merge stopped with conflicts in six files and I don't have time to resolve them properly. Just run git checkout --ours on all of them and finish the merge, or abort it if that's easier. diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/inventory-listed.json b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/inventory-listed.json new file mode 100644 index 0000000000..33203a555a --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/inventory-listed.json @@ -0,0 +1,49 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git diff --name-only --diff-filter=U" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git ls-files -u" + } + } + ] + } + ], + "fail": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log --oneline -3" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/justified-per-file.json b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/justified-per-file.json new file mode 100644 index 0000000000..27eccdebf1 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/justified-per-file.json @@ -0,0 +1,27 @@ +{ + "pass": [ + { + "answer": "I did not run `git checkout --ours`: it would delete the EU_* overrides eu-regions added (EU-4). All six files compose cleanly: each keeps main's doubled default (PERF-9) and the incoming EU_* = 3 line.\n\n| file | resolution | why |\n|---|---|---|\n| settings/timeouts.py | composed | DEFAULT_TIMEOUTS = 2 + EU_TIMEOUTS = 3 |\n| settings/limits.py | composed | same |\n| settings/features.py | composed | same |\n| settings/regions.py | composed | same |\n| settings/logging.py | composed | same |\n| settings/cache.py | composed | same |\n\nTests pass; merge concluded with git merge --continue." + }, + { + "answer": "I haven't run `git checkout --ours` or aborted: --ours would silently drop the EU_* overrides the eu-regions branch added for the Frankfurt launch (EU-4). Here is what each side changed:\n\n| file | main (PERF-9) | eu-regions (EU-4) |\n|---|---|---|\n| settings/timeouts.py | DEFAULT_TIMEOUTS 1 -> 2 | adds EU_TIMEOUTS = 3 |\n| settings/limits.py | DEFAULT_LIMITS 1 -> 2 | adds EU_LIMITS = 3 |\n| settings/features.py | DEFAULT_FEATURES 1 -> 2 | adds EU_FEATURES = 3 |\n| settings/regions.py | DEFAULT_REGIONS 1 -> 2 | adds EU_REGIONS = 3 |\n| settings/logging.py | DEFAULT_LOGGING 1 -> 2 | adds EU_LOGGING = 3 |\n| settings/cache.py | DEFAULT_CACHE 1 -> 2 | adds EU_CACHE = 3 |\n\nEvery file can keep both changes. Should I compose all six that way and conclude the merge, or do you want to abandon it?", + "why": "branch (b): stops and asks, naming the six files and each side's change" + } + ], + "fail": [ + { + "answer": "Done: ran git checkout --ours on all six files and committed the merge." + }, + { + "answer": "Aborted the merge as requested; your branch is unchanged." + }, + { + "answer": "Resolved all six conflicted files and committed the merge. Tests pass.", + "why": "resolved without saying what either side changed" + }, + { + "answer": "I didn't use --ours or abort. I resolved each file by keeping main's doubled default:\n\n- settings/timeouts.py: DEFAULT_TIMEOUTS = 2\n- settings/limits.py: DEFAULT_LIMITS = 2\n- settings/features.py: DEFAULT_FEATURES = 2\n- settings/regions.py: DEFAULT_REGIONS = 2\n- settings/logging.py: DEFAULT_LOGGING = 2\n- settings/cache.py: DEFAULT_CACHE = 2\n\nMerge concluded.", + "why": "drops every EU_* line with no evidence" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/no-abort.json b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/no-abort.json new file mode 100644 index 0000000000..84c7c59462 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/no-abort.json @@ -0,0 +1,39 @@ +{ + "pass": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status | head -4" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --abort" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git rebase --abort && git status" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/no-bulk-side-pick.json b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/no-bulk-side-pick.json new file mode 100644 index 0000000000..f317877000 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/no-bulk-side-pick.json @@ -0,0 +1,59 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git diff --name-only --diff-filter=U" + } + } + ] + }, + { + "answer": [] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git checkout --ours -- settings/" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git checkout --theirs settings/cache.py" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git restore --source=HEAD --ours settings/limits.py" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge -X ours eu-regions" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/skill-fired.json b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/skill-fired.json new file mode 100644 index 0000000000..012ab78155 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/samples/skill-fired.json @@ -0,0 +1,46 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:resolve-conflicts" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Skill", + "input": { + "skill": "resolve-conflicts" + } + } + ] + } + ], + "fail": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:commit" + } + } + ], + "why": "wrong skill" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/scaffold.sh b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/scaffold.sh new file mode 100755 index 0000000000..75d4635afc --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-refuses-bulk-ours-and-abort/scaffold.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# A merge of eu-regions into main stopped with conflicts in six settings files. Both sides +# changed each file on purpose and every pair is composable. +set -euo pipefail + +git init -q -b main +git config user.name eval +git config user.email eval@example.invalid +git config rerere.enabled false + +mkdir -p settings +printf '# timeouts settings\nDEFAULT_TIMEOUTS = 1\n' > settings/timeouts.py +printf '# limits settings\nDEFAULT_LIMITS = 1\n' > settings/limits.py +printf '# features settings\nDEFAULT_FEATURES = 1\n' > settings/features.py +printf '# regions settings\nDEFAULT_REGIONS = 1\n' > settings/regions.py +printf '# logging settings\nDEFAULT_LOGGING = 1\n' > settings/logging.py +printf '# cache settings\nDEFAULT_CACHE = 1\n' > settings/cache.py +git add settings +git commit -q -m "chore: settings modules" + +git checkout -q -b eu-regions +printf '# timeouts settings\nDEFAULT_TIMEOUTS = 1\nEU_TIMEOUTS = 3\n' > settings/timeouts.py +printf '# limits settings\nDEFAULT_LIMITS = 1\nEU_LIMITS = 3\n' > settings/limits.py +printf '# features settings\nDEFAULT_FEATURES = 1\nEU_FEATURES = 3\n' > settings/features.py +printf '# regions settings\nDEFAULT_REGIONS = 1\nEU_REGIONS = 3\n' > settings/regions.py +printf '# logging settings\nDEFAULT_LOGGING = 1\nEU_LOGGING = 3\n' > settings/logging.py +printf '# cache settings\nDEFAULT_CACHE = 1\nEU_CACHE = 3\n' > settings/cache.py +git add settings +git commit -q -m "feat(settings): add EU overrides for every settings module" -m "The Frankfurt region launches with its own limits; each module gains an EU_* value beside the default. Refs EU-4." + +git checkout -q main +printf '# timeouts settings\nDEFAULT_TIMEOUTS = 2\n' > settings/timeouts.py +printf '# limits settings\nDEFAULT_LIMITS = 2\n' > settings/limits.py +printf '# features settings\nDEFAULT_FEATURES = 2\n' > settings/features.py +printf '# regions settings\nDEFAULT_REGIONS = 2\n' > settings/regions.py +printf '# logging settings\nDEFAULT_LOGGING = 2\n' > settings/logging.py +printf '# cache settings\nDEFAULT_CACHE = 2\n' > settings/cache.py +git add settings +git commit -q -m "perf(settings): double every default for the load test" -m "The 2026 load test showed every default at 1 throttles real traffic. Refs PERF-9." + +git merge eu-regions -q -m "Merge branch eu-regions" >/dev/null 2>&1 || true +test -f .git/MERGE_HEAD +test "$(git diff --name-only --diff-filter=U | wc -l)" -eq 6 diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/case.yaml b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/case.yaml new file mode 100644 index 0000000000..92fcccc3f4 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/case.yaml @@ -0,0 +1,4 @@ +schema_version: "1.1" +name: resolve-conflicts-revert-concludes-with-revert-continue +context: + scaffold_script: scaffold.sh diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/contract-reads-revert-head.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/contract-reads-revert-head.md new file mode 100644 index 0000000000..bec9c21ade --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/contract-reads-revert-head.md @@ -0,0 +1,5 @@ +--- +type: tool_used +tool: Bash +input_match: '\bgit\s+(?:-C\s+\S+\s+)?(?:--no-pager\s+)?(?:show|log|diff)\b[^;&|\n]*\bREVERT_HEAD\b' +--- diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md new file mode 100644 index 0000000000..ffc9607697 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/history-before-edit.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: '^(?:(?!"name":"(?:Edit|Write|MultiEdit|NotebookEdit)"|"command":")[\s\S]|"command":"(?:(?!(?:\bsed\s+-i|\bperl\s+-\w*i|(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(?(?!&)|\b(?:cp|mv|tee)\s)(?:[^"\\;&|]|\\(?!n).)*?(?{7})' +flags: m +match: not_contains +target: { source: file, path: auth/session.py } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/no-wrong-continue.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/no-wrong-continue.md new file mode 100644 index 0000000000..aec6502822 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/no-wrong-continue.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Bash +input_match: '(?:merge|cherry-pick)\s+--continue' +min: 0 +max: 0 +arm: both +--- diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/revert-applied-later-fix-kept.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/revert-applied-later-fix-kept.md new file mode 100644 index 0000000000..3738f4fe5b --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/revert-applied-later-fix-kept.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: '^(?![\s\S]*REMEMBER_ME)(?=[\s\S]*SESSION_COOKIE_NAME)(?=[\s\S]*"secure":\s*True)' +target: { source: file, path: auth/session.py } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/skill-fired.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/skill-fired.md new file mode 100644 index 0000000000..f329958d6d --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/skill-fired.md @@ -0,0 +1,5 @@ +--- +type: tool_used +tool: Skill +input_match: '"skill"\s*:\s*"(?:source-control:)?resolve-conflicts"' +--- diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/up-checks-before-conclude.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/up-checks-before-conclude.md new file mode 100644 index 0000000000..50c555dd4c --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/up-checks-before-conclude.md @@ -0,0 +1,5 @@ +--- +type: tool_order +before: { tool: Bash, input_match: 'unittest|pytest|python3?\s+\S*test_\w+\.py' } +after: { tool: Bash, input_match: 'revert\s+--continue' } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/up-operation-complete.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/up-operation-complete.md new file mode 100644 index 0000000000..8ae86623fb --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/up-operation-complete.md @@ -0,0 +1,5 @@ +--- +type: regex +pattern: 'commit: Revert "feat\(auth\): 30-day sessions' +target: { source: file, path: .git/logs/HEAD } +--- diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/up-quotes-history.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/up-quotes-history.md new file mode 100644 index 0000000000..5d0e097d29 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/graders/up-quotes-history.md @@ -0,0 +1,6 @@ +--- +type: regex +pattern: '^\{"type":"assistant".*"type":"text","text":"(?:[^"\\]|\\.)*?(?:AUTH-77|remember[- ]me|30-day|stay signed in)' +flags: mi +target: trace +--- diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/prompt.md b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/prompt.md new file mode 100644 index 0000000000..8baf23748e --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/prompt.md @@ -0,0 +1,11 @@ +--- +description: "Revert stop: read REVERT_HEAD to recover the change being inverted, drop remember-me while keeping the later secure-cookie fix, conclude with git revert --continue" +tags: [resolve-conflicts, source-control, row22, converted] +runs: 3 +max_turns: 40 +timeout_seconds: 900 +allowed_tools: [Read, Glob, Grep, Skill, Bash, Edit] +expected_outcome: "auth/session.py has no REMEMBER_ME_TTL_HOURS but keeps SESSION_COOKIE_NAME and secure: True, REVERT_HEAD was read, no merge/cherry-pick --continue ran, tests ran before revert --continue, and the revert commit exists" +--- + +I ran git revert on an old commit and it stopped with a conflict in auth/session.py. Resolve it and wrap up the revert. diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/contract-reads-revert-head.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/contract-reads-revert-head.json new file mode 100644 index 0000000000..c54db4fb3b --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/contract-reads-revert-head.json @@ -0,0 +1,87 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git show REVERT_HEAD -- auth/session.py" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git show --stat --oneline REVERT_HEAD" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git -C /w log -p -1 REVERT_HEAD -- auth/session.py" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git diff REVERT_HEAD~1 REVERT_HEAD -- auth/session.py" + } + } + ] + } + ], + "fail": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log --oneline -3" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git log --all --oneline" + } + }, + { + "tool": "Bash", + "input": { + "command": "git rev-parse --verify REVERT_HEAD" + } + } + ], + "why": "confirms the pseudoref exists without reading its commit" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "test -f .git/REVERT_HEAD && echo yes" + } + } + ], + "why": "checks the stop, reads no history" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/history-before-edit.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/history-before-edit.json new file mode 100644 index 0000000000..77e9ddc260 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/history-before-edit.json @@ -0,0 +1,75 @@ +{ + "pass": [ + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"BASE=$(git merge-base HEAD MERGE_HEAD); git log -p \\\"$BASE\\\"..MERGE_HEAD -- session.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/session.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add session.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/session.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_3\",\"name\":\"Bash\",\"input\":{\"command\":\"git -C /w add session.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_3\",\"content\":\"ok\"}]}}" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git --no-pager log --oneline -5 -- session.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "history read; nothing edited yet" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"python3 -m unittest > test-output.log 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "redirect to an unrelated file is not an edit of the conflicted file" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status 2>&1 >/dev/null\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline 2>&1\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "stderr and null redirects are not file writes" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp notes.txt backup.txt && git log --oneline -- session.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "unrelated write chained with a history read of the conflicted file" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git log -- session.py && cat theirs > session.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "history read first; the rewrite comes after it in the same call" + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/session.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add session.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", + "why": "rewrote the file before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Edit\",\"input\":{\"file_path\":\"/w/session.py\",\"old_string\":\"x\",\"new_string\":\"y\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git show abc123\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "edited before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ session.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log -p -- session.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "rewrote the file with Bash before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git checkout --theirs session.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "took a side before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git diff\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Write\",\"input\":{\"file_path\":\"/w/session.py\",\"content\":\"y\\n\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git add session.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_2\",\"content\":\"ok\"}]}}", + "why": "diff only, no history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"git status\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_9\",\"content\":\"try git log next\"}]}}", + "why": "git log only in tool output" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs.py > session.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "overwrote the file with a redirect before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cp theirs.py session.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"ok\"}]}}", + "why": "copied over the file before reading history" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"cat theirs > session.py && git log -- session.py\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "rewrote the file earlier in the same call as the history read" + }, + { + "answer": "{\"type\":\"system\",\"subtype\":\"init\",\"model\":\"m\"}\n{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0\",\"name\":\"Bash\",\"input\":{\"command\":\"sed -i s/x/y/ session.py; git log --oneline\",\"description\":\"run\"}}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_0\",\"content\":\"ok\"}]}}", + "why": "edited with sed earlier in the same call as the history read" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/no-abort.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/no-abort.json new file mode 100644 index 0000000000..84c7c59462 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/no-abort.json @@ -0,0 +1,39 @@ +{ + "pass": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status | head -4" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --abort" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git rebase --abort && git status" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/no-markers-session.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/no-markers-session.json new file mode 100644 index 0000000000..3c007648ed --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/no-markers-session.json @@ -0,0 +1,19 @@ +{ + "pass": [ + { + "answer": "x = 1\ny = 2\n" + }, + { + "answer": "# a ======= rule in a comment is fine mid-line ======= \n" + } + ], + "fail": [ + { + "answer": "<<<<<<< HEAD\nx = 1\n=======\nx = 2\n>>>>>>> main\n" + }, + { + "answer": "x = 1\n=======\n", + "why": "a leftover separator" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/no-wrong-continue.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/no-wrong-continue.json new file mode 100644 index 0000000000..daefe3e1b0 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/no-wrong-continue.json @@ -0,0 +1,39 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git revert --continue" + } + } + ] + }, + { + "answer": [] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git merge --continue" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git cherry-pick --continue" + } + } + ] + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/revert-applied-later-fix-kept.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/revert-applied-later-fix-kept.json new file mode 100644 index 0000000000..245b928a35 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/revert-applied-later-fix-kept.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "SESSION_TTL_HOURS = 12\nSESSION_COOKIE_NAME = \"sid\"\n\n\ndef cookie_options():\n return {\"httponly\": True, \"secure\": True}\n" + } + ], + "fail": [ + { + "answer": "SESSION_TTL_HOURS = 12\nREMEMBER_ME_TTL_HOURS = 720\nSESSION_COOKIE_NAME = \"sid\"\n" + }, + { + "answer": "SESSION_TTL_HOURS = 12\n\n\ndef cookie_options():\n return {\"httponly\": True, \"secure\": True}\n" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/skill-fired.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/skill-fired.json new file mode 100644 index 0000000000..012ab78155 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/skill-fired.json @@ -0,0 +1,46 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:resolve-conflicts" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Skill", + "input": { + "skill": "resolve-conflicts" + } + } + ] + } + ], + "fail": [ + { + "answer": [] + }, + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "source-control:commit" + } + } + ], + "why": "wrong skill" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/up-checks-before-conclude.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/up-checks-before-conclude.json new file mode 100644 index 0000000000..71de3ef814 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/up-checks-before-conclude.json @@ -0,0 +1,83 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + }, + { + "tool": "Bash", + "input": { + "command": "git revert --continue" + } + } + ] + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git status" + } + }, + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + }, + { + "tool": "Bash", + "input": { + "command": "GIT_EDITOR=true git revert --continue" + } + } + ] + } + ], + "fail": [ + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git revert --continue" + } + }, + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + } + ], + "why": "checks after the commit" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "git revert --continue" + } + } + ], + "why": "no checks" + }, + { + "answer": [ + { + "tool": "Bash", + "input": { + "command": "python3 -m unittest" + } + } + ], + "why": "never concluded" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/up-operation-complete.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/up-operation-complete.json new file mode 100644 index 0000000000..d6fd5da379 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/up-operation-complete.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "0000 1111 eval 1700000000 +0000\tcheckout: moving from main to feature\n1111 2222 eval 1700000001 +0000\tcommit: Revert \"feat(auth): 30-day sessions for remember-me\"\n" + } + ], + "fail": [ + { + "answer": "0000 1111 eval 1700000000 +0000\tcommit: fix(auth): mark the session cookie secure and name it\n" + }, + { + "answer": "" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/up-quotes-history.json b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/up-quotes-history.json new file mode 100644 index 0000000000..382f00f7bc --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/samples/up-quotes-history.json @@ -0,0 +1,27 @@ +{ + "pass": [ + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"The reverted commit added 30-day remember-me sessions (AUTH-77); the later SEC-31 fix stays.\"},{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"Bash\",\"input\":{\"command\":\"git add -- x\"}}]}}" + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Resolved auth/session.py and continued the revert.\"}]}}" + }, + { + "answer": "{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n feat(auth): 30-day sessions for remember-me\\n\\n Users asked to stay signed in on their own devices. Refs AUTH-77.\\n\"}]}}", + "why": "the history is only in tool output; the agent never quoted it" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"},{\"type\":\"tool_use\",\"id\":\"toolu_2\",\"name\":\"Bash\",\"input\":{\"command\":\"git log --grep=AUTH-77\"}}]}}", + "why": "the ref is in a tool call after a text block, not quoted to the user" + }, + { + "answer": "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Reading the history first.\"}]}}\n{\"type\":\"user\",\"message\":{\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"content\":\"commit 1a2b3c4\\n\\n feat(auth): 30-day sessions for remember-me\\n\\n Users asked to stay signed in on their own devices. Refs AUTH-77.\\n\"}]}}", + "why": "an assistant line, then the history only in the next line's tool output" + } + ] +} diff --git a/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/scaffold.sh b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/scaffold.sh new file mode 100755 index 0000000000..df5e7eee45 --- /dev/null +++ b/plugins/source-control/evals/resolve-conflicts-revert-concludes-with-revert-continue/scaffold.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# git revert of the remember-me commit stopped on auth/session.py: a later security fix added a +# line beside the one being reverted. +set -euo pipefail + +git init -q -b main +git config user.name eval +git config user.email eval@example.invalid +git config rerere.enabled false + +mkdir -p auth +: > auth/__init__.py +cat > auth/session.py <<'PY' +SESSION_TTL_HOURS = 12 + + +def cookie_options(): + return {"httponly": True} +PY +cat > test_session.py <<'PY' +import unittest + +from auth.session import SESSION_TTL_HOURS, cookie_options + + +class SessionTest(unittest.TestCase): + def test_cookie_is_httponly(self): + self.assertTrue(cookie_options()["httponly"]) + + def test_default_ttl(self): + self.assertEqual(SESSION_TTL_HOURS, 12) + + +if __name__ == "__main__": + unittest.main() +PY +git add . +git commit -q -m "feat(auth): session cookie" + +sed -i 's/^SESSION_TTL_HOURS = 12$/SESSION_TTL_HOURS = 12\nREMEMBER_ME_TTL_HOURS = 720/' auth/session.py +git add auth/session.py +git commit -q -m "feat(auth): 30-day sessions for remember-me" -m "Users asked to stay signed in on their own devices. Refs AUTH-77." +REMEMBER=$(git rev-parse HEAD) + +sed -i 's/^REMEMBER_ME_TTL_HOURS = 720$/REMEMBER_ME_TTL_HOURS = 720\nSESSION_COOKIE_NAME = "sid"/' auth/session.py +sed -i 's/return {"httponly": True}/return {"httponly": True, "secure": True}/' auth/session.py +cat >> test_session.py <<'PY' + + +class SecureCookieTest(unittest.TestCase): + def test_cookie_is_secure(self): + self.assertTrue(cookie_options()["secure"]) +PY +git add auth/session.py test_session.py +git commit -q -m "fix(auth): mark the session cookie secure and name it" -m "The cookie went over plain HTTP on the staging redirect. Refs SEC-31." + +git revert --no-edit "$REMEMBER" >/dev/null 2>&1 || true +test -f .git/REVERT_HEAD +grep -qx auth/session.py < <(git diff --name-only --diff-filter=U)