diff --git a/plugins/planning/.claude-plugin/plugin.json b/plugins/planning/.claude-plugin/plugin.json index 40c0bb164c..83899da0c8 100644 --- a/plugins/planning/.claude-plugin/plugin.json +++ b/plugins/planning/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "planning", - "version": "0.69.3", + "version": "0.69.4", "userConfig": { "surface": { "type": "string", diff --git a/plugins/planning/CHANGELOG.md b/plugins/planning/CHANGELOG.md index 39fae9d8bb..677e646b8b 100644 --- a/plugins/planning/CHANGELOG.md +++ b/plugins/planning/CHANGELOG.md @@ -3,6 +3,12 @@ All notable changes to the `planning` plugin are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); this plugin uses semantic versioning. +## [0.69.4] - 2026-10-04 + +### Fixed + +- **The SDK out-of-scope regex in three `/planning:wayfind` eval cases now needs the SDK and the exclusion in the same clause.** It used to accept any SDK mention within 240 characters of any exclusion phrase, so "The partner SDK work remains open; separately, the batch job is out of scope" passed though the SDK was never excluded. The text between them may no longer cross a sentence end, a semicolon, a line break, a contrastive join such as ", but", or an "and" that opens a new clause, with or without a comma and in either order ("the batch job is out of scope and the SDKs stay in scope"); an SDK listed under an "Out of scope" label or after "Out of scope:" still passes. New fail samples cover the cross-clause pairing, and `/evals:validate` grades every sample correctly. + ## [0.69.3] - 2026-10-04 ### Added diff --git a/plugins/planning/evals/work-brief-respects-out-of-scope/case.yaml b/plugins/planning/evals/work-brief-respects-out-of-scope/case.yaml index 4aa00a81a8..ef47dae9b8 100644 --- a/plugins/planning/evals/work-brief-respects-out-of-scope/case.yaml +++ b/plugins/planning/evals/work-brief-respects-out-of-scope/case.yaml @@ -13,7 +13,7 @@ execution: graders: - name: sdk-fenced-out type: regex - pattern: '(?:SDK|client (?:librar|code))[\s\S]{0,240}(?:out of scope|out-of-scope|excluded|exclusion|not (?:in scope|ship|build|provide)|never ship|won''t ship|will not ship)|(?:out of scope|out-of-scope|excluded|exclusion|not (?:in scope|ship|build|provide)|never ship|won''t ship|will not ship)[\s\S]{0,240}(?:SDK|client (?:librar|code))' + pattern: '(?:SDK|client (?:librar|code))(?:(?!,\s*(?:but|while|whereas|although|though|however|separately|yet)\b|(?:,\s*|\s+)and\s+(?:the|a|an|our|we|they|it|this|that|its|their)\b)[^.!?;\n]){0,200}(?:out of scope|out-of-scope|excluded|exclusion|not (?:in scope|ship|build|provide)|never ship|won''t ship|will not ship)|(?:out of scope|out-of-scope|excluded|exclusion|not (?:in scope|ship|build|provide)|never ship|won''t ship|will not ship)(?:\s*:(?:(?!,\s*(?:but|while|whereas|although|though|however|separately|yet)\b|,\s*and\s+(?:the|a|an|our|we|they|it|this|that|its|their)\b)[^.!?;\n]){0,200}(?:SDK|client (?:librar|code))(?![^.!?;,\n]*\bin scope\b)|(?:(?!,\s*(?:but|while|whereas|although|though|however|separately|yet)\b|(?:,\s*|\s+)and\s+(?:the|a|an|our|we|they|it|this|that|its|their)\b)[^.!?;\n]){0,200}(?:SDK|client (?:librar|code)))|(?:^|\n)[ \t]*(?:#+[ \t]*|\*\*)?(?:out of scope|out-of-scope|exclusions?)[*_:]*[ \t]*\n\s*(?:(?:[-*+]|\d+\.)[^\n]*\n\s*){0,5}(?:[-*+]|\d+\.)[^\n]*?(?:SDK|client (?:librar|code))' flags: i arm: both - name: brief-respects-out-of-scope diff --git a/plugins/planning/evals/work-brief-respects-out-of-scope/samples/sdk-fenced-out.json b/plugins/planning/evals/work-brief-respects-out-of-scope/samples/sdk-fenced-out.json index c9cdf626fc..ee33370a87 100644 --- a/plugins/planning/evals/work-brief-respects-out-of-scope/samples/sdk-fenced-out.json +++ b/plugins/planning/evals/work-brief-respects-out-of-scope/samples/sdk-fenced-out.json @@ -1,10 +1,14 @@ { "pass": [ {"answer": "Scope: partner client SDKs or libraries are out of scope; we publish a feed contract only.", "why": "exclusion stated"}, - {"answer": "Do not recommend building SDKs: the map excludes them, since we never ship client code.", "why": "exclusion stated before the term"} + {"answer": "Do not recommend building SDKs: the map excludes them, since we never ship client code.", "why": "exclusion stated before the term"}, + {"answer": "Out of scope: batch replay and the partner SDKs.", "why": "a colon label governs every item in its list"} ], "fail": [ {"answer": "Evaluate which client SDKs (Python, Java, .NET) we should build to make adoption easiest.", "why": "tasks SDK work, no exclusion"}, - {"answer": "Find out what tooling and integration help each partner needs. Return findings per partner.", "why": "no exclusion at all"} + {"answer": "Find out what tooling and integration help each partner needs. Return findings per partner.", "why": "no exclusion at all"}, + {"answer": "Survey which client SDKs partners would adopt. Separately, the nightly batch job is out of scope and will not ship.", "why": "exclusion is about the batch job, in another sentence"}, + {"answer": "The nightly batch job is out of scope and the partner SDKs stay in scope.", "why": "exclusion first, coordinate clause without a comma keeps the SDK in scope"}, + {"answer": "Out of scope: the batch job and the partner SDKs stay in scope.", "why": "colon label, but the trailing clause keeps the SDK in scope"} ] } diff --git a/plugins/planning/evals/work-design-handoff-carries-map/case.yaml b/plugins/planning/evals/work-design-handoff-carries-map/case.yaml index 336dfb6552..dc225b328f 100644 --- a/plugins/planning/evals/work-design-handoff-carries-map/case.yaml +++ b/plugins/planning/evals/work-design-handoff-carries-map/case.yaml @@ -23,7 +23,7 @@ graders: arm: both - name: names-sdk-exclusion type: regex - pattern: '(?:SDK|client (?:librar|code))[\s\S]{0,240}(?:out of scope|out-of-scope|excluded|exclusion|not (?:in scope|ship|build|provide)|never ship|won''t ship|will not ship)|(?:out of scope|out-of-scope|excluded|exclusion|not (?:in scope|ship|build|provide)|never ship|won''t ship|will not ship)[\s\S]{0,240}(?:SDK|client (?:librar|code))' + pattern: '(?:SDK|client (?:librar|code))(?:(?!,\s*(?:but|while|whereas|although|though|however|separately|yet)\b|(?:,\s*|\s+)and\s+(?:the|a|an|our|we|they|it|this|that|its|their)\b)[^.!?;\n]){0,200}(?:out of scope|out-of-scope|excluded|exclusion|not (?:in scope|ship|build|provide)|never ship|won''t ship|will not ship)|(?:out of scope|out-of-scope|excluded|exclusion|not (?:in scope|ship|build|provide)|never ship|won''t ship|will not ship)(?:\s*:(?:(?!,\s*(?:but|while|whereas|although|though|however|separately|yet)\b|,\s*and\s+(?:the|a|an|our|we|they|it|this|that|its|their)\b)[^.!?;\n]){0,200}(?:SDK|client (?:librar|code))(?![^.!?;,\n]*\bin scope\b)|(?:(?!,\s*(?:but|while|whereas|although|though|however|separately|yet)\b|(?:,\s*|\s+)and\s+(?:the|a|an|our|we|they|it|this|that|its|their)\b)[^.!?;\n]){0,200}(?:SDK|client (?:librar|code)))|(?:^|\n)[ \t]*(?:#+[ \t]*|\*\*)?(?:out of scope|out-of-scope|exclusions?)[*_:]*[ \t]*\n\s*(?:(?:[-*+]|\d+\.)[^\n]*\n\s*){0,5}(?:[-*+]|\d+\.)[^\n]*?(?:SDK|client (?:librar|code))' flags: i arm: both - name: handoff-carries-map-context diff --git a/plugins/planning/evals/work-design-handoff-carries-map/samples/names-sdk-exclusion.json b/plugins/planning/evals/work-design-handoff-carries-map/samples/names-sdk-exclusion.json new file mode 100644 index 0000000000..5d492dffc6 --- /dev/null +++ b/plugins/planning/evals/work-design-handoff-carries-map/samples/names-sdk-exclusion.json @@ -0,0 +1,13 @@ +{ + "pass": [ + {"answer": "Settled: Debezium on Postgres logical replication. Out of scope: partner client SDKs or libraries; we only publish the feed contract.", "why": "exclusion governs the SDK in one clause"}, + {"answer": "Out of scope: batch replay and the partner SDKs.", "why": "a colon label governs every item in its list"} + ], + "fail": [ + {"answer": "Decision: webhook push or pulled cursor (#307). Partner SDKs remain open; separately, the nightly CSV retirement plan is out of scope.", "why": "exclusion is about something else, in another clause"}, + {"answer": "Decision: webhook push or pulled cursor (#307). Explore which client SDKs partners would want.", "why": "no exclusion"}, + {"answer": "Decision: webhook push or pulled cursor (#307). Partner SDKs stay open, and the nightly CSV retirement plan is out of scope.", "why": "coordinate clause excludes something else"}, + {"answer": "The nightly batch job is out of scope and the partner SDKs stay in scope.", "why": "exclusion first, coordinate clause without a comma keeps the SDK in scope"}, + {"answer": "Out of scope: the batch job and the partner SDKs stay in scope.", "why": "colon label, but the trailing clause keeps the SDK in scope"} + ] +} diff --git a/plugins/planning/evals/work-research-brief-carries-map/case.yaml b/plugins/planning/evals/work-research-brief-carries-map/case.yaml index d744a6879c..9eb205b555 100644 --- a/plugins/planning/evals/work-research-brief-carries-map/case.yaml +++ b/plugins/planning/evals/work-research-brief-carries-map/case.yaml @@ -23,7 +23,7 @@ graders: arm: both - name: names-sdk-exclusion type: regex - pattern: '(?:SDK|client (?:librar|code))[\s\S]{0,240}(?:out of scope|out-of-scope|excluded|exclusion|not (?:in scope|ship|build|provide)|never ship|won''t ship|will not ship)|(?:out of scope|out-of-scope|excluded|exclusion|not (?:in scope|ship|build|provide)|never ship|won''t ship|will not ship)[\s\S]{0,240}(?:SDK|client (?:librar|code))' + pattern: '(?:SDK|client (?:librar|code))(?:(?!,\s*(?:but|while|whereas|although|though|however|separately|yet)\b|(?:,\s*|\s+)and\s+(?:the|a|an|our|we|they|it|this|that|its|their)\b)[^.!?;\n]){0,200}(?:out of scope|out-of-scope|excluded|exclusion|not (?:in scope|ship|build|provide)|never ship|won''t ship|will not ship)|(?:out of scope|out-of-scope|excluded|exclusion|not (?:in scope|ship|build|provide)|never ship|won''t ship|will not ship)(?:\s*:(?:(?!,\s*(?:but|while|whereas|although|though|however|separately|yet)\b|,\s*and\s+(?:the|a|an|our|we|they|it|this|that|its|their)\b)[^.!?;\n]){0,200}(?:SDK|client (?:librar|code))(?![^.!?;,\n]*\bin scope\b)|(?:(?!,\s*(?:but|while|whereas|although|though|however|separately|yet)\b|(?:,\s*|\s+)and\s+(?:the|a|an|our|we|they|it|this|that|its|their)\b)[^.!?;\n]){0,200}(?:SDK|client (?:librar|code)))|(?:^|\n)[ \t]*(?:#+[ \t]*|\*\*)?(?:out of scope|out-of-scope|exclusions?)[*_:]*[ \t]*\n\s*(?:(?:[-*+]|\d+\.)[^\n]*\n\s*){0,5}(?:[-*+]|\d+\.)[^\n]*?(?:SDK|client (?:librar|code))' flags: i arm: both - name: skill-fired diff --git a/plugins/planning/evals/work-research-brief-carries-map/samples/names-sdk-exclusion.json b/plugins/planning/evals/work-research-brief-carries-map/samples/names-sdk-exclusion.json new file mode 100644 index 0000000000..179d3085bd --- /dev/null +++ b/plugins/planning/evals/work-research-brief-carries-map/samples/names-sdk-exclusion.json @@ -0,0 +1,20 @@ +{ + "pass": [ + {"answer": "Out of scope: partner client SDKs or libraries; we publish a feed contract only.", "why": "exclusion governs the SDK in one clause"}, + {"answer": "## Out-of-scope\n\n- Partner client SDKs or libraries: we publish a feed contract only and never ship client code", "why": "the map's own out-of-scope line"}, + {"answer": "Out of scope:\n- Replay windows\n- Partner client SDKs or libraries", "why": "SDK listed under an out-of-scope label"}, + {"answer": "Partner SDKs, client libraries, and plugins are out of scope.", "why": "serial-comma list ending in and stays one clause"}, + {"answer": "Out of scope: batch replay and the partner SDKs.", "why": "a colon label governs every item in its list"} + ], + "fail": [ + {"answer": "For Q3, given Debezium, recommend which partner SDKs we should build and ship.", "why": "tasks SDK work, no exclusion"}, + {"answer": "The partner SDK work remains open; separately, shipping the nightly batch job is explicitly out of scope and will not ship this quarter.", "why": "exclusion is about the batch job, in another clause"}, + {"answer": "Partner SDKs stay in scope. The nightly batch job is out of scope.", "why": "exclusion is in a different sentence"}, + {"answer": "We will build the partner SDKs, but the batch export is out of scope.", "why": "contrastive clause excludes something else"}, + {"answer": "Partner SDKs stay in scope, and the nightly batch job is out of scope.", "why": "coordinate clause excludes something else"}, + {"answer": "Partner SDKs stay in scope and the nightly batch job is out of scope.", "why": "coordinate clause without a comma excludes something else"}, + {"answer": "The nightly batch job is out of scope, and the partner SDKs stay in scope.", "why": "exclusion first, coordinate clause keeps the SDK in scope"}, + {"answer": "The nightly batch job is out of scope and the partner SDKs stay in scope.", "why": "exclusion first, coordinate clause without a comma keeps the SDK in scope"}, + {"answer": "Out of scope: the batch job and the partner SDKs stay in scope.", "why": "colon label, but the trailing clause keeps the SDK in scope"} + ] +}