diff --git a/README.md b/README.md index 8895a6db..f8302c81 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,12 @@ Agent Skills are modular, text-based playbooks that teach an agent how to perfor | `metrics/launchdarkly-metric-create` | Create metrics and instrument tracking events | | `metrics/launchdarkly-metric-instrument` | Add tracking calls to code for existing metrics | +### AWS DevOps Agent + +| Skill | Description | +|-------|-------------| +| `aws-devops-agent/experiment-orchestration` | Take a product goal through a LaunchDarkly experiment and a rollout (guarded, if the plan has it) in AWS DevOps Agent. `ldcli aws-devops-agent setup` installs it with the `experiment-orchestrator` custom agent. See [`skills/aws-devops-agent`](skills/aws-devops-agent/README.md). | + ## Install as a Claude Code Plugin This repo is a [Claude Code plugin](https://code.claude.com/docs/en/create-plugins). Installing it gives you all the skills above plus the LaunchDarkly MCP server. diff --git a/evals/README.md b/evals/README.md index 603fc282..a9a3fb2d 100644 --- a/evals/README.md +++ b/evals/README.md @@ -34,6 +34,7 @@ npm run eval:agentcontrol-tools # agentcontrol/tools npm run eval:configs-variations # agentcontrol/configs-variations npm run eval:flag-create # feature-flags/launchdarkly-flag-create npm run eval:onboarding # onboarding +npm run eval:experiment-orchestration # aws-devops-agent/experiment-orchestration # Quick smoke check — first test case only (~15-20s, ~$0.05) npm run eval:configs-create:single @@ -42,6 +43,7 @@ npm run eval:agentcontrol-tools:single npm run eval:configs-variations:single npm run eval:flag-create:single npm run eval:onboarding:single +npm run eval:experiment-orchestration:single # Aggregate and CI operations npm run eval:all # Run every suite and rebuild ../eval-scores.json diff --git a/evals/experiment-orchestration/mocks.json b/evals/experiment-orchestration/mocks.json new file mode 100644 index 00000000..7ff6722d --- /dev/null +++ b/evals/experiment-orchestration/mocks.json @@ -0,0 +1,113 @@ +{ + "start-guarded-rollout": { + "flagKey": "{{flagKey}}", + "environmentKey": "{{environmentKey}}", + "status": "monitoring", + "currentStage": 1, + "stageCount": 3 + }, + "update-rollout": { + "key": "{{flagKey}}", + "environment": "{{environmentKey}}", + "fallthrough": { + "variation": 0 + } + }, + "stop-experiment-iteration": { + "key": "{{experimentKey}}", + "environmentKey": "{{environmentKey}}", + "currentIteration": { + "status": "stopped" + } + }, + "get-experiment-results": { + "experimentKey": "{{experimentKey}}", + "metrics": [ + { + "metricKey": "checkout-completed", + "isPrimary": true, + "treatments": [ + { + "treatmentId": "treat-control", + "name": "Control", + "isBest": false, + "analyzedUnitCount": 4200 + }, + { + "treatmentId": "treat-treatment", + "name": "Treatment", + "isBest": true, + "relativeDifference": 0.081, + "isSignificant": true, + "analyzedUnitCount": 4180 + } + ] + } + ] + }, + "get-experiment-metric-results": { + "experimentKey": "{{experimentKey}}", + "metricKey": "{{metricKey}}", + "treatments": [ + { + "treatmentId": "treat-control", + "name": "Control", + "analyzedUnitCount": 4200, + "trafficCount": 4200, + "conversionCount": 630, + "mean": 0.15 + }, + { + "treatmentId": "treat-treatment", + "name": "Treatment", + "analyzedUnitCount": 4180, + "trafficCount": 4180, + "conversionCount": 678, + "mean": 0.162, + "statistics": { + "relativeDifferences": [ + { + "estimate": 0.081, + "lower": 0.02, + "upper": 0.14, + "pValue": 0.008 + } + ], + "isSignificant": true + } + } + ] + }, + "create-approval-request": { + "_id": "apr-0001", + "flagKey": "{{flagKey}}", + "environmentKey": "{{environmentKey}}", + "status": "pending", + "reviewStatus": "pending" + }, + "apply-approval-request": { + "_id": "apr-0001", + "status": "completed", + "reviewStatus": "approved" + }, + "list-approval-requests": { + "items": [ + { + "_id": "apr-0001", + "flagKey": "{{flagKey}}", + "status": "pending", + "reviewStatus": "pending" + } + ], + "totalCount": 1 + }, + "list-metric-events": { + "events": [ + { + "eventKey": "checkout-completed", + "lastSeen": "2026-10-28T12:00:00Z" + } + ], + "totalCount": 1 + } +} diff --git a/evals/experiment-orchestration/promptfooconfig.yaml b/evals/experiment-orchestration/promptfooconfig.yaml new file mode 100644 index 00000000..ea5aedf2 --- /dev/null +++ b/evals/experiment-orchestration/promptfooconfig.yaml @@ -0,0 +1,634 @@ +# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json +# +# Run with shared defaults: +# promptfoo eval -c shared/defaults.yaml -c experiment-orchestration/promptfooconfig.yaml +# +# experiment-orchestration (skills/aws-devops-agent/) takes a product goal +# through a 10% experiment and, if it wins, a guarded rollout, using only the +# LaunchDarkly MCP tools. It runs in AWS DevOps Agent, which these evals can't +# reproduce: they score the skill's decisions under the Claude Agent SDK with +# mocked tools. Lean mode (allow_builtins:false) also stands in for an agent +# space with no GitHub connected, so the code change is the user's. +# +# Each test after the first starts mid-flow: the user request states what the +# earlier steps settled, and `mock_overrides` sets the tool responses for the +# branch under test (an approval requirement, a plan without guarded +# rollouts, a losing result). +description: "Evaluation of the experiment-orchestration skill (AWS DevOps Agent)" + +prompts: + - file://../../skills/aws-devops-agent/experiment-orchestration/SKILL.md + +providers: + - id: file://../providers/claude-skill-agent-sdk.js + label: mcp + config: + skill_slug: experiment-orchestration + allow_builtins: false + expose_ask_question: true + # The agent's system prompt loads the skill up front in AWS DevOps Agent. + inline_skill: true + # Older duplicates the hosted LaunchDarkly MCP server doesn't have; the + # agent would otherwise pick create-feature-flag, which has no `defaults`. + exclude_mcp_tools: [create-feature-flag, update-feature-flag, get-feature-flag, list-feature-flags] + # This suite's tools and mocks, matched to the hosted LaunchDarkly MCP + # server, so other suites keep the shared definitions unchanged. + tool_definitions: experiment-orchestration/tools.json + tool_mocks: experiment-orchestration/mocks.json + +tests: + # ================================================================== + # GOAL CLARIFICATION — a goal with no metric or target. The skill must + # ask before it creates or changes anything (Step 1, Safety Rules). The + # user dismisses every question, so no answer ever arrives. + # `asked_with_tool` is informational (weight 0): it records whether the + # agent used a structured question tool, which review asked about. + # ================================================================== + - description: "Vague goal: asks for the metric and target before creating anything" + vars: + user_request: > + Project `storefront`, environment `production`. I want more people to + finish checkout. Can you run an experiment for that? + max_turns: 8 + mock_ask_question_answers: [[], [], [], [], []] + assert: + - type: javascript + value: | + const tools = output.tools_called || []; + const writes = ['create-flag','toggle-flag','create-experiment','start-experiment-iteration','create-metric','update-rollout','start-guarded-rollout']; + const called = writes.filter(w => tools.includes(w)); + const asked = tools.some(t => String(t).includes('ask-question')) || /\?/.test(String(output.response || '')); + const pass = called.length === 0 && asked; + return { pass, score: pass ? 1 : 0, reason: !asked ? 'asked nothing' : pass ? 'asked and made no changes' : 'changed before asking: ' + called.join(', ') }; + metric: no_writes_before_goal + weight: 4 + - type: javascript + value: | + const tools = output.tools_called || []; + const pass = tools.some(t => String(t).includes('ask-question')); + return { pass, score: pass ? 1 : 0, reason: pass ? 'used ask-question' : 'asked in prose (or not at all)' }; + metric: asked_with_tool + weight: 0 + - type: llm-rubric + value: | + The user asked to "get more people to finish checkout" with no metric or + target. Evaluate the agent's questions (in prose or via a question tool): + 1. Does it ask for, or propose and ask to confirm, the success metric? + 2. Does it ask for the target improvement? + 3. Does it avoid claiming it already created a flag or experiment? + Score 1.0 if all three. Deduct 0.34 per criterion missed. + metric: asks_required_questions + weight: 3 + + # ================================================================== + # IMPLEMENTATION WITHOUT GITHUB — goal and hypothesis are settled. The + # skill creates the boolean flag with both defaults on `false`, then + # hands the code change to the user instead of turning anything on. + # ================================================================== + - description: "No GitHub: creates the boolean flag serving false and hands the code change to the user" + vars: + user_request: > + Project `storefront`, environment `production`. Goal confirmed: + checkout conversion (metric `checkout-completed`) up 5%, randomized by + user, roll back on a 2% error-rate regression. GitHub is not connected + to this agent space and there is no coding agent. Hypothesis approved: + changing the checkout button copy from "Continue" to "Place order" + raises conversion. No flag exists yet; use the key + `exp-checkout-conversion-button-copy`. Continue from Step 3. + max_turns: 10 + mock_overrides: + list-metrics: &metrics_with_error + metrics: + - { key: "checkout-completed", name: "Checkout completed", measureType: "occurrence", successCriteria: "HigherThanBaseline", eventKey: "checkout-completed", randomizationUnits: ["user"] } + - { key: "storefront-telemetry-default-error-rate", name: "Error rate", measureType: "occurrence", successCriteria: "LowerThanBaseline", eventKey: "$ld:telemetry:error", randomizationUnits: ["user"] } + totalCount: 2 + get-flag: + error: "flag not found" + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const c = traj.find(t => t.tool === 'create-flag'); + if (!c) return { pass: false, score: 0, reason: 'never created the flag' }; + const a = c.arguments || {}; + const d = a.defaults || {}; + const boolKind = !a.kind || a.kind === 'boolean'; + const pass = boolKind && d.onVariation === 1 && d.offVariation === 1; + return { pass, score: pass ? 1 : 0, reason: `kind=${a.kind} defaults=${JSON.stringify(d)} (want boolean, 1/1)` }; + metric: flag_defaults_serve_false + weight: 4 + - type: javascript + value: | + const tools = output.tools_called || []; + const early = ['toggle-flag','create-experiment','start-experiment-iteration']; + const called = early.filter(w => tools.includes(w)); + const created = tools.includes('create-flag'); + const pass = created && called.length === 0; + return { pass, score: pass ? 1 : 0, reason: !created ? 'never created the flag' : pass ? 'waited for the deploy' : 'ran ahead of the code change: ' + called.join(', ') }; + metric: waits_for_deploy + weight: 4 + - type: llm-rubric + value: | + GitHub isn't connected, so the user makes the code change. Evaluate the response: + 1. Does it give the user the flag key `exp-checkout-conversion-button-copy` + and say the change must be gated behind it? + 2. Does it ask the user to confirm when the change is reviewed, merged, and + deployed, rather than claiming it will do those itself? + Score 1.0 if both, 0.5 if one, 0 if neither. + metric: hands_off_code_change + weight: 3 + + # ================================================================== + # MULTIVARIATE FLAG — the hypothesis has three versions of the change. + # The skill creates a multivariate flag with the control first and both + # defaults on it, so the flag serves the control even when it is on. + # ================================================================== + - description: "Several versions: creates a multivariate flag with the control first and defaults on it" + vars: + user_request: > + Project `storefront`, environment `production`. Goal confirmed: + checkout conversion (metric `checkout-completed`) up 5%, randomized by + user, roll back on a 2% error-rate regression. GitHub is not connected + to this agent space and there is no coding agent. Hypothesis approved: + the checkout button label affects conversion; test the current label + "Continue" against "Place order" and "Buy now". No flag exists yet; use + the key `exp-checkout-conversion-button-label`. Continue from Step 3. + max_turns: 10 + mock_overrides: + list-metrics: *metrics_with_error + get-flag: + error: "flag not found" + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const c = traj.find(t => t.tool === 'create-flag'); + if (!c) return { pass: false, score: 0, reason: 'never created the flag' }; + const a = c.arguments || {}; + const vals = (a.variations || []).map(v => String(v.value)); + const d = a.defaults || {}; + const checks = { + multivariate: a.kind === 'multivariate', + threeVariations: vals.length === 3, + controlFirst: /continue/i.test(vals[0] || ''), + defaultsOnControl: d.onVariation === 0 && d.offVariation === 0, + }; + const failed = Object.keys(checks).filter(k => !checks[k]); + const score = 1 - failed.length / Object.keys(checks).length; + return { pass: failed.length === 0, score, reason: `kind=${a.kind} values=${JSON.stringify(vals)} defaults=${JSON.stringify(d)}` + (failed.length ? ' failed: ' + failed.join(', ') : '') }; + metric: multivariate_flag_shape + weight: 5 + - type: javascript + value: | + const tools = output.tools_called || []; + const early = ['toggle-flag','create-experiment','start-experiment-iteration'].filter(w => tools.includes(w)); + const created = tools.includes('create-flag'); + const pass = created && early.length === 0; + return { pass, score: pass ? 1 : 0, reason: !created ? 'never created the flag' : pass ? 'waited for the deploy' : 'ran ahead of the code change: ' + early.join(', ') }; + metric: waits_for_deploy + weight: 3 + + # ================================================================== + # 10% EXPERIMENT — code is deployed. The skill turns the flag on, then + # creates the experiment on the fallthrough with 5 + 5 allocation, the + # control for everyone outside it, and sequential frequentist analysis. + # Guards the "50/50 allocated to 10%" wording that sent 50 + 50 (100%). + # ================================================================== + - description: "Deployed: creates a fallthrough experiment at 5 + 5 with the control outside it" + vars: + user_request: > + Project `storefront`, environment `production`. Goal confirmed: + checkout conversion (metric `checkout-completed`) up 5%, randomized by + user, roll back on a 2% error-rate regression. Guardrail: the built-in + error-rate metric `storefront-telemetry-default-error-rate`. Hypothesis + approved: changing the checkout button copy to "Place order" raises + conversion. Flag `exp-checkout-conversion-button-copy` exists and the + gated change is reviewed, merged, and deployed. Continue from Step 6: + set up and start the experiment. + max_turns: 12 + mock_overrides: + list-metrics: *metrics_with_error + assert: + - type: javascript + value: | + const tools = output.tools_called || []; + const on = tools.indexOf('toggle-flag'); + const ce = tools.indexOf('create-experiment'); + const pass = on >= 0 && ce > on; + return { pass, score: pass ? 1 : 0, reason: `toggle-flag@${on} create-experiment@${ce}` }; + metric: flag_on_before_experiment + weight: 2 + - type: javascript + value: | + const traj = output.trajectory || []; + const c = traj.find(t => t.tool === 'create-experiment'); + if (!c) return { pass: false, score: 0, reason: 'never created the experiment' }; + const it = (c.arguments || {}).iteration || {}; + const ts = it.treatments || []; + const total = ts.reduce((s, t) => s + Number(t.allocationPercent || 0), 0); + const each = ts.map(t => Number(t.allocationPercent)); + const pass = ts.length === 2 && Math.abs(total - 10) < 0.01 && each.every(x => Math.abs(x - 5) < 0.01); + return { pass, score: pass ? 1 : 0, reason: `allocations=${JSON.stringify(each)} total=${total} (want 5 + 5 = 10)` }; + metric: ten_percent_allocation + weight: 5 + - type: javascript + value: | + const traj = output.trajectory || []; + const c = traj.find(t => t.tool === 'create-experiment'); + if (!c) return { pass: false, score: 0, reason: 'never created the experiment' }; + const it = (c.arguments || {}).iteration || {}; + // var-false is the false variation's _id in the shared get-flag mock. + const flag = (it.flags || {})['exp-checkout-conversion-button-copy'] || {}; + const control = (it.treatments || []).find(t => t.baseline) || {}; + const controlVar = ((control.parameters || [])[0] || {}).variationId; + const checks = { + fallthrough: flag.ruleId === 'fallthrough', + outsideIsControl: flag.notInExperimentVariationId === 'var-false', + controlIsFalse: controlVar === 'var-false', + }; + const failed = Object.keys(checks).filter(k => !checks[k]); + const pass = failed.length === 0; + return { pass, score: 1 - failed.length / 3, reason: pass ? 'fallthrough, control outside and as baseline' : 'failed: ' + failed.join(', ') + ' | flags=' + JSON.stringify(it.flags) }; + metric: fallthrough_control_outside + weight: 4 + - type: javascript + value: | + const traj = output.trajectory || []; + const c = traj.find(t => t.tool === 'create-experiment'); + if (!c) return { pass: false, score: 0, reason: 'never created the experiment' }; + const a = c.arguments || {}; + const it = a.iteration || {}; + const keys = (it.metrics || []).map(m => m.key); + const checks = { + frequentist: a.methodology === 'frequentist', + sequential: (a.analysisConfig || {}).sequentialTestingEnabled === true, + primary: it.primarySingleMetricKey === 'checkout-completed', + errorGuardrail: keys.includes('storefront-telemetry-default-error-rate'), + noReshuffle: it.canReshuffleTraffic === false, + }; + const failed = Object.keys(checks).filter(k => !checks[k]); + const score = 1 - failed.length / Object.keys(checks).length; + return { pass: failed.length === 0, score, reason: failed.length ? 'failed: ' + failed.join(', ') : 'analysis and metrics set' }; + metric: analysis_and_metrics + weight: 3 + + # ================================================================== + # MULTIVARIATE EXPERIMENT — three variations share the 10%: 3.33 each, + # the control outside the experiment, and Benjamini-Hochberg with the + # scope LaunchDarkly requires alongside it. + # ================================================================== + - description: "Multivariate deployed: splits 10% evenly across three variations with the comparison correction" + vars: + user_request: > + Project `storefront`, environment `production`. Goal confirmed: + checkout conversion (metric `checkout-completed`) up 5%, randomized by + user, roll back on a 2% error-rate regression. Guardrail: the built-in + error-rate metric `storefront-telemetry-default-error-rate`. Hypothesis + approved: test the checkout button label "Continue" (current) against + "Place order" and "Buy now". Multivariate flag + `exp-checkout-conversion-button-label` exists and the gated change is + reviewed, merged, and deployed. Continue from Step 6: set up and start + the experiment. + max_turns: 12 + mock_overrides: + list-metrics: *metrics_with_error + get-flag: + key: "exp-checkout-conversion-button-label" + kind: "multivariate" + variations: + - { _id: "var-continue", value: "Continue", name: "Continue" } + - { _id: "var-place-order", value: "Place order", name: "Place order" } + - { _id: "var-buy-now", value: "Buy now", name: "Buy now" } + defaults: { onVariation: 0, offVariation: 0 } + environment: { on: false, version: 2, rules: [], targets: [], fallthrough: { variation: 0 }, offVariation: 0 } + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const c = traj.find(t => t.tool === 'create-experiment'); + if (!c) return { pass: false, score: 0, reason: 'never created the experiment' }; + const it = (c.arguments || {}).iteration || {}; + const ts = it.treatments || []; + const each = ts.map(t => Number(t.allocationPercent)); + const total = each.reduce((a, b) => a + b, 0); + const pass = ts.length === 3 && each.every(x => Math.abs(x - 3.33) < 0.011) && total <= 10 + 1e-9; + return { pass, score: pass ? 1 : 0, reason: `allocations=${JSON.stringify(each)} total=${total} (want 3 x 3.33, <= 10)` }; + metric: even_split_within_ten + weight: 5 + - type: javascript + value: | + const traj = output.trajectory || []; + const c = traj.find(t => t.tool === 'create-experiment'); + if (!c) return { pass: false, score: 0, reason: 'never created the experiment' }; + const a = c.arguments || {}; + const it = a.iteration || {}; + const flag = (it.flags || {})['exp-checkout-conversion-button-label'] || {}; + const control = (it.treatments || []).find(t => t.baseline) || {}; + const controlVar = ((control.parameters || [])[0] || {}).variationId; + const ac = a.analysisConfig || {}; + const checks = { + fallthrough: flag.ruleId === 'fallthrough', + outsideIsControl: flag.notInExperimentVariationId === 'var-continue', + controlBaseline: controlVar === 'var-continue', + correction: ac.multipleComparisonCorrectionMethod === 'benjamini-hochberg', + scope: typeof ac.multipleComparisonCorrectionScope === 'string' && ac.multipleComparisonCorrectionScope.length > 0, + }; + const failed = Object.keys(checks).filter(k => !checks[k]); + const score = 1 - failed.length / Object.keys(checks).length; + return { pass: failed.length === 0, score, reason: failed.length ? 'failed: ' + failed.join(', ') + ' | analysisConfig=' + JSON.stringify(ac) : 'control outside and as baseline, correction with scope' }; + metric: multivariate_experiment_config + weight: 4 + + # ================================================================== + # LAUNCHDARKLY APPROVAL — turning the flag on returns requiresApproval. + # The skill says the change wasn't made, submits an approval request + # with the returned instructions, and doesn't retry the direct call. + # ================================================================== + - description: "Approval required: requests approval instead of retrying the flag change" + vars: + user_request: > + Project `storefront`, environment `production`. Goal confirmed: + checkout conversion (metric `checkout-completed`) up 5%, randomized by + user, roll back on a 2% error-rate regression. Guardrail: the built-in + error-rate metric `storefront-telemetry-default-error-rate`. Hypothesis + approved: changing the checkout button copy to "Place order" raises + conversion. Flag `exp-checkout-conversion-button-copy` exists and the + gated change is reviewed, merged, and deployed. If LaunchDarkly needs + an approval, the reviewer is member `m-4821`. Continue from Step 6. + max_turns: 10 + mock_overrides: + list-metrics: *metrics_with_error + toggle-flag: + requiresApproval: true + message: "This environment requires approval for flag changes." + instructions: [{ kind: "turnFlagOn" }] + list-approval-requests: + items: [{ _id: "apr-0001", flagKey: "exp-checkout-conversion-button-copy", status: "pending", reviewStatus: "pending" }] + totalCount: 1 + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const toggles = traj.filter(t => t.tool === 'toggle-flag').length; + const reqIdx = traj.findIndex(t => t.tool === 'create-approval-request'); + const togIdx = traj.findIndex(t => t.tool === 'toggle-flag'); + const pass = reqIdx > togIdx && togIdx >= 0 && toggles === 1; + return { pass, score: pass ? 1 : 0, reason: `toggle-flag x${toggles}, create-approval-request@${reqIdx}` }; + metric: requests_approval_once + weight: 5 + - type: javascript + value: | + const traj = output.trajectory || []; + const r = traj.find(t => t.tool === 'create-approval-request'); + if (!r) return { pass: false, score: 0, reason: 'no approval request' }; + const a = r.arguments || {}; + const ins = JSON.stringify(a.instructions || []); + const pass = a.flagKey === 'exp-checkout-conversion-button-copy' && (a.notifyMemberIds || []).includes('m-4821') && /turnFlagOn/.test(ins); + return { pass, score: pass ? 1 : 0, reason: `flagKey=${a.flagKey} reviewers=${JSON.stringify(a.notifyMemberIds)} instructions=${ins}` }; + metric: approval_request_contents + weight: 3 + - type: javascript + value: | + const tools = output.tools_called || []; + const requested = tools.includes('create-approval-request'); + const early = ['apply-approval-request','create-experiment','start-experiment-iteration'].filter(w => tools.includes(w)); + const pass = requested && early.length === 0; + return { pass, score: pass ? 1 : 0, reason: !requested ? 'never requested approval' : pass ? 'waited for the pending approval' : 'acted before approval: ' + early.join(', ') }; + metric: waits_for_approval + weight: 3 + + # ================================================================== + # WINNING EXPERIMENT -> GUARDED ROLLOUT — results are significant and + # positive (default mocks). The skill stops with the control as winner, + # then starts a 3-stage guarded rollout below 50% in thousandths. + # ================================================================== + - description: "Winner: stops with control, then a 3-stage guarded rollout below 50%" + vars: + user_request: &winner_ready > + Project `storefront`, environment `production`. Experiment + `exp-checkout-button-copy` on flag `exp-checkout-conversion-button-copy` + (primary metric `checkout-completed`, guardrail + `storefront-telemetry-default-error-rate`, randomized by user, roll + back on a 2% regression) has run for two weeks. Continue from Step 7: + check the results and act on them. + max_turns: 14 + mock_overrides: + get-experiment: + key: "exp-checkout-button-copy" + environmentKey: "production" + currentIteration: + status: "running" + startedAt: "2026-10-14T00:00:00Z" + treatments: + - { _id: "treat-control", name: "Control", baseline: true, allocationPercent: 5, parameters: [{ flagKey: "exp-checkout-conversion-button-copy", variationId: "var-false" }] } + - { _id: "treat-treatment", name: "Treatment", baseline: false, allocationPercent: 5, parameters: [{ flagKey: "exp-checkout-conversion-button-copy", variationId: "var-true" }] } + get-flag: + key: "exp-checkout-conversion-button-copy" + kind: "boolean" + variations: [{ _id: "var-true", value: true, name: "true" }, { _id: "var-false", value: false, name: "false" }] + defaults: { onVariation: 1, offVariation: 1 } + environment: { on: true, version: 5, rules: [], targets: [], fallthrough: { variation: 1 }, offVariation: 1 } + get-experiment-metric-results: + by_arg: metricKey + cases: + checkout-completed: + metricKey: "checkout-completed" + treatments: + - { treatmentId: "treat-control", name: "Control", analyzedUnitCount: 4200, trafficCount: 4200, conversionCount: 630, mean: 0.15 } + - { treatmentId: "treat-treatment", name: "Treatment", analyzedUnitCount: 4180, trafficCount: 4180, conversionCount: 678, mean: 0.162, statistics: { relativeDifferences: [{ estimate: 0.081, lower: 0.02, upper: 0.14, pValue: 0.008 }], isSignificant: true } } + default: + metricKey: "{{metricKey}}" + treatments: + - { treatmentId: "treat-control", name: "Control", analyzedUnitCount: 4200, trafficCount: 4200, conversionCount: 42, mean: 0.010 } + - { treatmentId: "treat-treatment", name: "Treatment", analyzedUnitCount: 4180, trafficCount: 4180, conversionCount: 41, mean: 0.0098, statistics: { relativeDifferences: [{ estimate: -0.02, lower: -0.4, upper: 0.4, pValue: 0.91 }], isSignificant: false } } + get-experiment-results: + experimentKey: "exp-checkout-button-copy" + metrics: + - { metricKey: "checkout-completed", isPrimary: true, treatments: [{ treatmentId: "treat-control", name: "Control", isBest: false, analyzedUnitCount: 4200 }, { treatmentId: "treat-treatment", name: "Treatment", isBest: true, relativeDifference: 0.081, isSignificant: true, analyzedUnitCount: 4180 }] } + - { metricKey: "storefront-telemetry-default-error-rate", isPrimary: false, treatments: [{ treatmentId: "treat-control", name: "Control", analyzedUnitCount: 4200 }, { treatmentId: "treat-treatment", name: "Treatment", relativeDifference: -0.02, isSignificant: false, analyzedUnitCount: 4180 }] } + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const s = traj.find(t => t.tool === 'stop-experiment-iteration'); + const g = traj.findIndex(t => t.tool === 'start-guarded-rollout'); + const sIdx = traj.findIndex(t => t.tool === 'stop-experiment-iteration'); + const winner = s && (s.arguments || {}).winningTreatmentId; + const pass = winner === 'treat-control' && g > sIdx; + return { pass, score: pass ? 1 : 0, reason: `winner=${winner} stop@${sIdx} rollout@${g} (want control, stop before rollout)` }; + metric: stop_with_control_then_rollout + weight: 4 + - type: javascript + value: | + const traj = output.trajectory || []; + const g = traj.find(t => t.tool === 'start-guarded-rollout'); + if (!g) return { pass: false, score: 0, reason: 'no guarded rollout' }; + const a = g.arguments || {}; + const w = (a.stages || []).map(s => Number(s.rolloutWeight)); + const checks = { + threeStages: w.length === 3, + belowHalf: w.every(x => x > 0 && x < 50000), + thousandths: w.every(x => x >= 1000), + variations: a.testVariationId === 'var-true' && a.controlVariationId === 'var-false', + completes: a.updateDefaultRuleOnCompletion === true, + }; + const failed = Object.keys(checks).filter(k => !checks[k]); + const score = 1 - failed.length / Object.keys(checks).length; + return { pass: failed.length === 0, score, reason: `weights=${JSON.stringify(w)} ` + (failed.length ? 'failed: ' + failed.join(', ') : 'ok') }; + metric: rollout_stages + weight: 4 + - type: javascript + value: | + const traj = output.trajectory || []; + const g = traj.find(t => t.tool === 'start-guarded-rollout'); + if (!g) return { pass: false, score: 0, reason: 'no guarded rollout' }; + const ms = (g.arguments || {}).metrics || []; + const keys = ms.map(m => m.metricKey); + const allRollback = ms.length > 0 && ms.every(m => m.onRegression && m.onRegression.rollback === true && typeof m.regressionThreshold === 'number'); + const pass = keys.includes('storefront-telemetry-default-error-rate') && allRollback; + return { pass, score: pass ? 1 : 0, reason: `metrics=${JSON.stringify(ms)}` }; + metric: rollout_guardrails + weight: 3 + + # ================================================================== + # NO GUARDED ROLLOUTS ON THE PLAN — start-guarded-rollout fails with a + # plan error. The skill falls back to a regular rollout of the treatment + # on the default rule instead of stopping or retrying. + # ================================================================== + - description: "No guarded rollouts on the plan: falls back to a regular rollout of the treatment" + vars: + user_request: *winner_ready + max_turns: 14 + mock_overrides: + get-experiment: + key: "exp-checkout-button-copy" + environmentKey: "production" + currentIteration: + status: "running" + startedAt: "2026-10-14T00:00:00Z" + treatments: + - { _id: "treat-control", name: "Control", baseline: true, allocationPercent: 5, parameters: [{ flagKey: "exp-checkout-conversion-button-copy", variationId: "var-false" }] } + - { _id: "treat-treatment", name: "Treatment", baseline: false, allocationPercent: 5, parameters: [{ flagKey: "exp-checkout-conversion-button-copy", variationId: "var-true" }] } + get-flag: + key: "exp-checkout-conversion-button-copy" + kind: "boolean" + variations: [{ _id: "var-true", value: true, name: "true" }, { _id: "var-false", value: false, name: "false" }] + defaults: { onVariation: 1, offVariation: 1 } + environment: { on: true, version: 5, rules: [], targets: [], fallthrough: { variation: 1 }, offVariation: 1 } + get-experiment-metric-results: + by_arg: metricKey + cases: + checkout-completed: + metricKey: "checkout-completed" + treatments: + - { treatmentId: "treat-control", name: "Control", analyzedUnitCount: 4200, trafficCount: 4200, conversionCount: 630, mean: 0.15 } + - { treatmentId: "treat-treatment", name: "Treatment", analyzedUnitCount: 4180, trafficCount: 4180, conversionCount: 678, mean: 0.162, statistics: { relativeDifferences: [{ estimate: 0.081, lower: 0.02, upper: 0.14, pValue: 0.008 }], isSignificant: true } } + default: + metricKey: "{{metricKey}}" + treatments: + - { treatmentId: "treat-control", name: "Control", analyzedUnitCount: 4200, trafficCount: 4200, conversionCount: 42, mean: 0.010 } + - { treatmentId: "treat-treatment", name: "Treatment", analyzedUnitCount: 4180, trafficCount: 4180, conversionCount: 41, mean: 0.0098, statistics: { relativeDifferences: [{ estimate: -0.02, lower: -0.4, upper: 0.4, pValue: 0.91 }], isSignificant: false } } + get-experiment-results: + experimentKey: "exp-checkout-button-copy" + metrics: + - { metricKey: "checkout-completed", isPrimary: true, treatments: [{ treatmentId: "treat-control", name: "Control", isBest: false, analyzedUnitCount: 4200 }, { treatmentId: "treat-treatment", name: "Treatment", isBest: true, relativeDifference: 0.081, isSignificant: true, analyzedUnitCount: 4180 }] } + - { metricKey: "storefront-telemetry-default-error-rate", isPrimary: false, treatments: [{ treatmentId: "treat-control", name: "Control", analyzedUnitCount: 4200 }, { treatmentId: "treat-treatment", name: "Treatment", relativeDifference: -0.02, isSignificant: false, analyzedUnitCount: 4180 }] } + start-guarded-rollout: + error: "Guarded rollouts are not available on your LaunchDarkly plan." + code: "feature_not_available" + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const gr = traj.filter(t => t.tool === 'start-guarded-rollout').length; + const g = traj.findIndex(t => t.tool === 'start-guarded-rollout'); + const u = traj.find((t, i) => t.tool === 'update-rollout' && i > g); + const a = (u && u.arguments) || {}; + const pass = g >= 0 && gr === 1 && a.rolloutType === 'variation' && a.variationIndex === 0; + return { pass, score: pass ? 1 : 0, reason: `start-guarded-rollout x${gr}; update-rollout=${JSON.stringify(a)} (want variation, index 0)` }; + metric: regular_rollout_fallback + weight: 5 + - type: llm-rubric + value: | + Guarded rollouts weren't available, so the agent used a regular rollout. + Evaluate the response: + 1. Does it say the regular rollout serves the treatment to everyone with no + automatic rollback? + 2. Does it say it will watch the guardrail metrics and revert to false on a + regression? + Score 1.0 if both, 0.5 if one, 0 if neither. + metric: explains_no_auto_rollback + weight: 2 + + # ================================================================== + # LOSING EXPERIMENT — the primary metric drops significantly. The skill + # stops with the control, turns the flag off, starts no rollout, and + # writes the Step 8 report. + # ================================================================== + - description: "Loser: stops with control, turns the flag off, no rollout, and reports" + vars: + user_request: *winner_ready + max_turns: 12 + mock_overrides: + get-experiment: + key: "exp-checkout-button-copy" + environmentKey: "production" + currentIteration: + status: "running" + startedAt: "2026-10-14T00:00:00Z" + treatments: + - { _id: "treat-control", name: "Control", baseline: true, allocationPercent: 5, parameters: [{ flagKey: "exp-checkout-conversion-button-copy", variationId: "var-false" }] } + - { _id: "treat-treatment", name: "Treatment", baseline: false, allocationPercent: 5, parameters: [{ flagKey: "exp-checkout-conversion-button-copy", variationId: "var-true" }] } + get-flag: + key: "exp-checkout-conversion-button-copy" + kind: "boolean" + variations: [{ _id: "var-true", value: true, name: "true" }, { _id: "var-false", value: false, name: "false" }] + defaults: { onVariation: 1, offVariation: 1 } + environment: { on: true, version: 5, rules: [], targets: [], fallthrough: { variation: 1 }, offVariation: 1 } + get-experiment-metric-results: + by_arg: metricKey + cases: + checkout-completed: + metricKey: "checkout-completed" + treatments: + - { treatmentId: "treat-control", name: "Control", analyzedUnitCount: 4200, trafficCount: 4200, conversionCount: 630, mean: 0.15 } + - { treatmentId: "treat-treatment", name: "Treatment", analyzedUnitCount: 4180, trafficCount: 4180, conversionCount: 543, mean: 0.13, statistics: { relativeDifferences: [{ estimate: -0.133, lower: -0.19, upper: -0.07, pValue: 0.001 }], isSignificant: true } } + default: + metricKey: "{{metricKey}}" + treatments: + - { treatmentId: "treat-control", name: "Control", analyzedUnitCount: 4200, trafficCount: 4200, conversionCount: 42, mean: 0.010 } + - { treatmentId: "treat-treatment", name: "Treatment", analyzedUnitCount: 4180, trafficCount: 4180, conversionCount: 41, mean: 0.0098, statistics: { relativeDifferences: [{ estimate: -0.02, lower: -0.4, upper: 0.4, pValue: 0.91 }], isSignificant: false } } + get-experiment-results: + experimentKey: "exp-checkout-button-copy" + metrics: + - { metricKey: "checkout-completed", isPrimary: true, treatments: [{ treatmentId: "treat-control", name: "Control", isBest: true, analyzedUnitCount: 4200 }, { treatmentId: "treat-treatment", name: "Treatment", isBest: false, relativeDifference: -0.133, isSignificant: true, analyzedUnitCount: 4180 }] } + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const s = traj.find(t => t.tool === 'stop-experiment-iteration'); + const off = traj.find(t => t.tool === 'toggle-flag' && (t.arguments || {}).on === false); + const winner = s && (s.arguments || {}).winningTreatmentId; + const pass = winner === 'treat-control' && Boolean(off); + return { pass, score: pass ? 1 : 0, reason: `winner=${winner} flagOff=${Boolean(off)}` }; + metric: stops_and_turns_off + weight: 4 + - type: javascript + value: | + const traj = output.trajectory || []; + const stopped = traj.some(t => t.tool === 'stop-experiment-iteration'); + const bad = traj.filter(t => t.tool === 'start-guarded-rollout' || (t.tool === 'update-rollout' && (t.arguments || {}).variationIndex === 0) || t.tool === 'delete-flag'); + const pass = stopped && bad.length === 0; + return { pass, score: pass ? 1 : 0, reason: !stopped ? 'never stopped the experiment' : pass ? 'no rollout, flag kept' : 'called: ' + bad.map(t => t.tool).join(', ') }; + metric: no_rollout_after_loss + weight: 4 + - type: llm-rubric + value: | + The experiment lost: checkout conversion fell about 13%, significantly. + Evaluate the response: + 1. Does it include a structured report with an "Experiment Report" heading + and sections for the hypothesis, the experiment phase, and next steps? + 2. Does it mark the experiment as a loser and say no rollout happened? + Score 1.0 if both, 0.5 if one, 0 if neither. + metric: loss_report + weight: 3 diff --git a/evals/experiment-orchestration/tools.json b/evals/experiment-orchestration/tools.json new file mode 100644 index 00000000..8e68828b --- /dev/null +++ b/evals/experiment-orchestration/tools.json @@ -0,0 +1,823 @@ +[ + { + "name": "create-flag", + "description": "Create a new feature flag in a project. Defaults to a boolean temporary flag. After creation the flag is OFF in all environments: use toggle-flag to enable it. An OFF flag serves `defaults.offVariation` and an ON flag that matches no rule serves `defaults.onVariation`; boolean flags default to OFF serving false.", + "input_schema": { + "type": "object", + "properties": { + "projectKey": { + "type": "string", + "description": "The project key" + }, + "name": { + "type": "string", + "description": "Human-readable flag name" + }, + "key": { + "type": "string", + "description": "Unique flag key (immutable)" + }, + "kind": { + "type": "string", + "enum": [ + "boolean", + "multivariate" + ], + "description": "Flag kind (default: boolean)" + }, + "description": { + "type": "string", + "description": "Description of the flag's purpose" + }, + "temporary": { + "type": "boolean", + "description": "Whether the flag is temporary (default: true)" + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Tags for organization" + }, + "variations": { + "type": "array", + "items": { + "type": "object", + "properties": { + "value": { + "description": "The variation value" + }, + "name": { + "type": "string", + "description": "Variation name" + } + } + }, + "description": "Custom variations (for multivariate flags)" + }, + "clientSideAvailability": { + "type": "object", + "properties": { + "usingMobileKey": { + "type": "boolean" + }, + "usingEnvironmentId": { + "type": "boolean" + } + }, + "description": "Client-side SDK availability" + }, + "defaults": { + "type": "object", + "properties": { + "onVariation": { + "type": "number" + }, + "offVariation": { + "type": "number" + } + }, + "required": [ + "onVariation", + "offVariation" + ] + } + }, + "required": [ + "projectKey", + "name", + "key" + ] + } + }, + { + "name": "toggle-flag", + "description": "Turn a feature flag's targeting on or off in a specific environment. Returns the previous and new state. If the environment requires approval, the response includes requiresApproval: true and the attempted instructions; call `create-approval-request` with the returned instructions to submit it for review.", + "input_schema": { + "type": "object", + "properties": { + "projectKey": { + "type": "string", + "description": "The project key" + }, + "flagKey": { + "type": "string", + "description": "The flag key" + }, + "env": { + "type": "string", + "description": "Environment key" + }, + "on": { + "type": "boolean", + "description": "Whether to turn targeting on or off" + }, + "comment": { + "type": "string", + "description": "Optional comment" + } + }, + "required": [ + "projectKey", + "flagKey", + "env", + "on" + ] + } + }, + { + "name": "list-metrics", + "description": "List metrics in a project. Use to find existing metrics to reuse as an experiment's primary or guardrail metric before creating a new one.", + "input_schema": { + "type": "object", + "properties": { + "projectKey": { + "type": "string", + "description": "The project key" + }, + "query": { + "type": "string", + "description": "Search by metric name or key" + }, + "limit": { + "type": "number", + "description": "Max number of results (default 20)" + }, + "metricUsedIn": { + "type": "string" + }, + "eventKind": { + "type": "string" + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "projectKey" + ] + } + }, + { + "name": "create-metric", + "description": "Create a metric that measures an experiment outcome. measureType 'occurrence' is a conversion (did each user do it at least once), 'count' totals events, and 'value' averages or sums the number sent with track(). eventKey is the tracked event the metric counts.", + "input_schema": { + "type": "object", + "properties": { + "projectKey": { + "type": "string", + "description": "The project key" + }, + "key": { + "type": "string", + "description": "Unique metric key (immutable)" + }, + "name": { + "type": "string", + "description": "Human-readable metric name" + }, + "kind": { + "type": "string", + "enum": [ + "custom", + "pageview", + "click" + ] + }, + "measureType": { + "type": "string", + "enum": [ + "count", + "occurrence", + "value" + ], + "description": "occurrence = conversion, count = event total, value = numeric value" + }, + "valueAggregation": { + "type": "string", + "enum": [ + "sum", + "average" + ], + "description": "Only for measureType 'value': average (default) or sum" + }, + "unit": { + "type": "string", + "description": "Unit for value metrics, e.g. USD or ms" + }, + "eventKey": { + "type": "string", + "description": "The event key the metric counts (matches the track() call)" + }, + "urls": { + "type": "array", + "description": "Required for pageview and click metrics: pages to match", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "substring", + "exact", + "canonical", + "regex" + ] + }, + "substring": { + "type": "string" + }, + "url": { + "type": "string" + }, + "pattern": { + "type": "string" + } + }, + "required": [ + "kind" + ] + } + }, + "selector": { + "type": "string", + "description": "Required for click metrics: CSS selector that triggers the event" + }, + "successCriteria": { + "type": "string", + "enum": [ + "HigherThanBaseline", + "LowerThanBaseline" + ], + "description": "Which direction is success" + }, + "description": { + "type": "string", + "description": "What the metric measures" + }, + "randomizationUnits": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "projectKey", + "key", + "name", + "measureType", + "successCriteria" + ] + } + }, + { + "name": "create-experiment", + "description": "Create a new experiment on a flag. The iteration takes a hypothesis, metrics ({key}), primarySingleMetricKey, randomizationUnit, canReshuffleTraffic, treatments ({name, baseline, allocationPercent, parameters: [{flagKey, variationId}]}), and flags keyed by flag key ({ruleId, flagConfigVersion, notInExperimentVariationId}). allocationPercent is each treatment's share of all traffic on the rule; the rest is served notInExperimentVariationId. The experiment is created as a draft and collects no data until an iteration is started.", + "input_schema": { + "type": "object", + "properties": { + "projectKey": { + "type": "string" + }, + "environmentKey": { + "type": "string" + }, + "key": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "methodology": { + "type": "string", + "enum": [ + "bayesian", + "frequentist", + "export_only" + ] + }, + "analysisConfig": { + "type": "object", + "properties": { + "sequentialTestingEnabled": { + "type": "boolean" + }, + "significanceThreshold": { + "type": "string" + }, + "testDirection": { + "type": "string" + }, + "multipleComparisonCorrectionMethod": { + "type": "string", + "enum": [ + "bonferroni", + "benjamini-hochberg" + ] + }, + "multipleComparisonCorrectionScope": { + "type": "string", + "enum": [ + "variations", + "variations-and-metrics", + "metrics" + ] + } + } + }, + "iteration": { + "type": "object", + "properties": { + "hypothesis": { + "type": "string" + }, + "canReshuffleTraffic": { + "type": "boolean" + }, + "randomizationUnit": { + "type": "string" + }, + "primarySingleMetricKey": { + "type": "string" + }, + "metrics": { + "type": "array", + "items": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "isGroup": { + "type": "boolean" + } + } + } + }, + "treatments": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "baseline": { + "type": "boolean" + }, + "allocationPercent": { + "type": "number" + }, + "parameters": { + "type": "array", + "items": { + "type": "object", + "properties": { + "flagKey": { + "type": "string" + }, + "variationId": { + "type": "string" + } + } + } + } + } + } + }, + "flags": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "ruleId": { + "type": "string" + }, + "flagConfigVersion": { + "type": "number" + }, + "notInExperimentVariationId": { + "type": "string" + } + } + } + } + } + } + }, + "required": [ + "projectKey", + "environmentKey", + "key", + "name", + "iteration" + ] + } + }, + { + "name": "start-guarded-rollout", + "description": "Start a guarded rollout on a flag's default rule (fallthrough). A guarded rollout progressively increases traffic to the test variation through a series of stages while monitoring metrics for regressions. Each stage specifies a rolloutWeight (percentage in thousandths, e.g. 10000 = 10%) and a monitoring window in milliseconds. Metrics define what to monitor and whether to notify or rollback on regression. Requires the flag to be on and not already have an active guarded rollout. If the environment requires approval, the response includes requiresApproval: true and the attempted instructions; call `create-approval-request` with the returned instructions to submit it for review.", + "input_schema": { + "type": "object", + "properties": { + "projectKey": { + "type": "string" + }, + "flagKey": { + "type": "string" + }, + "environmentKey": { + "type": "string" + }, + "testVariationId": { + "type": "string" + }, + "controlVariationId": { + "type": "string" + }, + "randomizationUnit": { + "type": "string" + }, + "stages": { + "type": "array", + "items": { + "type": "object", + "properties": { + "rolloutWeight": { + "type": "number" + }, + "monitoringWindowMilliseconds": { + "type": "number" + } + }, + "required": [ + "rolloutWeight", + "monitoringWindowMilliseconds" + ] + } + }, + "metrics": { + "type": "array", + "items": { + "type": "object", + "properties": { + "metricKey": { + "type": "string" + }, + "onRegression": { + "type": "object", + "properties": { + "notify": { + "type": "boolean" + }, + "rollback": { + "type": "boolean" + } + }, + "required": [ + "notify", + "rollback" + ] + }, + "regressionThreshold": { + "type": "number" + } + }, + "required": [ + "metricKey", + "onRegression", + "regressionThreshold" + ] + } + }, + "metricSources": { + "type": "array", + "items": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "isGroup": { + "type": "boolean" + } + }, + "required": [ + "key", + "isGroup" + ] + } + }, + "updateDefaultRuleOnCompletion": { + "type": "boolean" + }, + "comment": { + "type": "string" + } + }, + "required": [ + "projectKey", + "flagKey", + "environmentKey", + "testVariationId", + "controlVariationId", + "randomizationUnit", + "stages", + "metrics" + ] + } + }, + { + "name": "update-rollout", + "description": "Change the default rule (fallthrough) for a flag. Set a percentage rollout across variations or serve a single variation to all unmatched users. Weights must sum to 100. Use human-friendly percentages (e.g., 80 for 80%). If the environment requires approval, the response includes requiresApproval: true and the attempted instructions; call `create-approval-request` with the returned instructions to submit it for review.", + "input_schema": { + "type": "object", + "properties": { + "projectKey": { + "type": "string" + }, + "flagKey": { + "type": "string" + }, + "env": { + "type": "string" + }, + "rolloutType": { + "type": "string", + "enum": [ + "percentage", + "variation" + ] + }, + "variationIndex": { + "type": "number" + }, + "weights": { + "type": "array", + "items": { + "type": "object", + "properties": { + "variationIndex": { + "type": "number" + }, + "percent": { + "type": "number" + } + }, + "required": [ + "variationIndex", + "percent" + ] + } + }, + "contextKind": { + "type": "string" + }, + "comment": { + "type": "string" + } + }, + "required": [ + "projectKey", + "flagKey", + "env", + "rolloutType" + ] + } + }, + { + "name": "stop-experiment-iteration", + "description": "Stop the currently running iteration of an experiment. Data collection ends and the iteration moves to the `stopped` status. A winning treatment is required to stop: pass `winningTreatmentId` (the `_id` of one of the iteration's treatments, available from get-experiment) and an optional `winningReason` explaining the call. If the experiment was inconclusive, declare the baseline/control treatment as the winner.", + "input_schema": { + "type": "object", + "properties": { + "projectKey": { + "type": "string" + }, + "environmentKey": { + "type": "string" + }, + "experimentKey": { + "type": "string" + }, + "winningTreatmentId": { + "type": "string" + }, + "winningReason": { + "type": "string" + } + }, + "required": [ + "projectKey", + "environmentKey", + "experimentKey", + "winningTreatmentId" + ] + } + }, + { + "name": "get-experiment-results", + "description": "Get an overall results summary for an experiment, across every metric on the current iteration. Returns a compact per-metric summary: which treatment is leading (isBest) — for frequentist, the lowest p-value versus control (note: not significance-gated, so a treatment can be flagged leading even when nothing is significant) — each treatment's lift versus control, whether it's statistically significant, and sample sizes. For the full per-treatment statistics on one metric, use get-experiment-metric-results.", + "input_schema": { + "type": "object", + "properties": { + "projectKey": { + "type": "string" + }, + "environmentKey": { + "type": "string" + }, + "experimentKey": { + "type": "string" + }, + "iterationId": { + "type": "string" + }, + "disableMultipleComparisonCorrection": { + "type": "boolean" + }, + "disableSequentialTesting": { + "type": "boolean" + } + }, + "required": [ + "projectKey", + "environmentKey", + "experimentKey" + ] + } + }, + { + "name": "get-experiment-metric-results", + "description": "Get detailed statistical results for a single metric on an experiment iteration. Returns, per treatment: sample sizes (analyzedUnitCount, trafficCount, conversionCount), the observed mean and standard deviation, and a `statistics` block with the lift versus control (relativeDifferences: estimate plus lower/upper interval bounds and a per-comparison pValue), the overall pValue, and an isSignificant flag.", + "input_schema": { + "type": "object", + "properties": { + "projectKey": { + "type": "string" + }, + "environmentKey": { + "type": "string" + }, + "experimentKey": { + "type": "string" + }, + "metricKey": { + "type": "string" + }, + "iterationId": { + "type": "string" + }, + "disableMultipleComparisonCorrection": { + "type": "boolean" + }, + "disableSequentialTesting": { + "type": "boolean" + } + }, + "required": [ + "projectKey", + "environmentKey", + "experimentKey", + "metricKey" + ] + } + }, + { + "name": "create-approval-request", + "description": "Create an approval request for a flag or segment change in an environment that requires approvals. Pass flagKey for a flag or segmentKey for a segment, never both. Requires at least one reviewer: notifyMemberIds and/or notifyTeamKeys. If the user named a person, resolve them with find-members and pass the returned id. If they did not name anyone, ask who should review before calling this tool. Provide the same semantic patch instructions you would use for a direct change. The request will be reviewed by approvers before taking effect. Does NOT approve the request: that must be done by a human reviewer.", + "input_schema": { + "type": "object", + "properties": { + "projectKey": { + "type": "string" + }, + "env": { + "type": "string" + }, + "flagKey": { + "type": "string" + }, + "segmentKey": { + "type": "string" + }, + "instructions": { + "type": "array", + "items": { + "type": "object" + } + }, + "description": { + "type": "string" + }, + "comment": { + "type": "string" + }, + "notifyMemberIds": { + "type": "array", + "items": { + "type": "string" + } + }, + "notifyTeamKeys": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "projectKey", + "env", + "instructions", + "description" + ] + } + }, + { + "name": "apply-approval-request", + "description": "Apply an already-approved flag or segment approval request. This executes the changes that were approved by reviewers. Pass flagKey for a flag or segmentKey for a segment, never both. Only works on requests with reviewStatus 'approved'. Does NOT approve requests: that must be done by a human reviewer. Use list-approval-requests to get the request id.", + "input_schema": { + "type": "object", + "properties": { + "projectKey": { + "type": "string" + }, + "env": { + "type": "string" + }, + "id": { + "type": "string" + }, + "flagKey": { + "type": "string" + }, + "segmentKey": { + "type": "string" + }, + "comment": { + "type": "string" + } + }, + "required": [ + "projectKey", + "env", + "id" + ] + } + }, + { + "name": "list-approval-requests", + "description": "List approval requests for a flag or a segment in an environment. Pass flagKey for a flag or segmentKey for a segment, never both. Shows status, review state, and who requested the change. Use to check on approval progress after creating a request.", + "input_schema": { + "type": "object", + "properties": { + "projectKey": { + "type": "string" + }, + "env": { + "type": "string" + }, + "flagKey": { + "type": "string" + }, + "segmentKey": { + "type": "string" + } + }, + "required": [ + "projectKey", + "env" + ] + } + }, + { + "name": "list-metric-events", + "description": "List recent event keys received by a LaunchDarkly project — used to check which events are actively flowing before creating a metric. Returns up to 50 event keys with last-seen timestamps. If the event key you need isn't here, it may not be instrumented yet. Supports fuzzy search via `query` to narrow results. Filter by `environmentKey` to check a specific environment.", + "input_schema": { + "type": "object", + "properties": { + "projectKey": { + "type": "string" + }, + "query": { + "type": "string" + }, + "environmentKey": { + "type": "string" + } + }, + "required": [ + "projectKey" + ] + } + } +] diff --git a/evals/package.json b/evals/package.json index 5b16f174..51f90ca4 100644 --- a/evals/package.json +++ b/evals/package.json @@ -27,6 +27,8 @@ "eval:onboarding:single": "promptfoo eval -c shared/defaults.yaml -c onboarding/promptfooconfig.yaml --env-file .env --no-cache --filter-first-n 1", "eval:experiment-guide": "promptfoo eval -c shared/defaults.yaml -c experiment-guide/promptfooconfig.yaml --env-file .env --no-cache -o experiment-guide/results.json", "eval:experiment-guide:single": "promptfoo eval -c shared/defaults.yaml -c experiment-guide/promptfooconfig.yaml --env-file .env --no-cache --filter-first-n 1", + "eval:experiment-orchestration": "promptfoo eval -c shared/defaults.yaml -c experiment-orchestration/promptfooconfig.yaml --env-file .env --no-cache -o experiment-orchestration/results.json", + "eval:experiment-orchestration:single": "promptfoo eval -c shared/defaults.yaml -c experiment-orchestration/promptfooconfig.yaml --env-file .env --no-cache --filter-first-n 1", "fixture:from-pr": "node scripts/build-pr-fixture.js", "eval:all": "node scripts/aggregate.js --run", "eval:aggregate": "node scripts/aggregate.js", diff --git a/evals/providers/_mock.js b/evals/providers/_mock.js index 831b70b1..f5ab11ee 100644 --- a/evals/providers/_mock.js +++ b/evals/providers/_mock.js @@ -378,4 +378,43 @@ function renderMockResponse(template, input, toolName, state) { return walk(template, replacements); } -module.exports = { renderMockResponse, createMockState }; +/** + * Pick a test's override for `toolName`, if the test set one through the + * `mock_overrides` var. An object value answers every call; an array answers + * successive calls in order and repeats its last entry. An object with + * `by_arg` answers by the value of that tool argument: `cases[value]`, else + * `default`. Returns undefined when the test has no override for the call, so + * the caller falls back to the shared template. + */ +function resolveMockOverride(overrides, toolName, callIndex, args) { + if (!overrides || typeof overrides !== "object") return undefined; + if (!Object.prototype.hasOwnProperty.call(overrides, toolName)) return undefined; + const value = overrides[toolName]; + if (Array.isArray(value)) { + if (value.length === 0) return undefined; + return value[Math.min(callIndex, value.length - 1)]; + } + if (value && typeof value === "object" && typeof value.by_arg === "string") { + const key = args ? args[value.by_arg] : undefined; + const cases = value.cases || {}; + if (key !== undefined && Object.prototype.hasOwnProperty.call(cases, key)) return cases[key]; + return value.default; + } + return value; +} + +/** + * Render an override with the same `{{placeholder}}` substitution as the + * shared templates, but without the stateful overlay: an override is exactly + * what the test wants the tool to return. + */ +function renderOverride(template, input) { + return walk(template, buildReplacements(input)); +} + +module.exports = { + renderMockResponse, + createMockState, + resolveMockOverride, + renderOverride, +}; diff --git a/evals/providers/claude-skill-agent-sdk.js b/evals/providers/claude-skill-agent-sdk.js index 3adabd65..d787f28f 100644 --- a/evals/providers/claude-skill-agent-sdk.js +++ b/evals/providers/claude-skill-agent-sdk.js @@ -27,6 +27,25 @@ * `/` to explicitly invoke the skill via * slash command. Use for routing/advisory skills whose * description-based activation is unreliable in eval. + * inline_skill - Default false. When true, put the skill's + * SKILL.md body into the agent's instructions + * instead of relying on the SDK to load it. Listing + * the skill in `skills` doesn't load its text when + * builtins are off, and force_skill_invocation + * interrupts MCP calls when the agent's first + * action is one (seen 2026-10-28). Use for skills a + * host loads up front, like an agent's system prompt. + * tool_definitions - Optional path, relative to evals/, of a JSON + * array of tool definitions for this suite only. + * Each replaces the shared definition of the same + * name or adds a tool, so a suite can match its + * skill's MCP server without changing other suites. + * tool_mocks - Optional path, relative to evals/, of a JSON + * object of mock templates for this suite only, + * merged over mocks/tool-responses.json. + * exclude_mcp_tools - Optional list of mock tool names to hide, e.g. + * older duplicates the skill's real MCP server + * doesn't have. * expose_ask_question - Default false. When true, expose a second in-process * MCP server with an `ask-question` tool that records * calls into the trajectory and returns a canned answer. @@ -45,10 +64,18 @@ * (true/false), e.g. to run one test with no * LaunchDarkly tools * mock_ask_question_answers - optional array of `selected` arrays returned by - * successive `ask-question` calls + * successive `ask-question` calls; an empty + * array is a dismissed question (no selection). + * Calls past the end get the first option. * mock_files - optional object mapping relative file paths to * string content, written into the isolated cwd * before the agent runs + * mock_overrides - optional object mapping a tool name to the + * response this test wants instead of the shared + * template: an object for every call, or an + * array for successive calls (the last entry + * repeats). Use for branches like + * `requiresApproval` or a plan error. * * Environment variables: * AGENT_MODEL - SUT model (default claude-sonnet-4-6). @@ -62,7 +89,12 @@ const path = require("node:path"); const { z } = require("zod"); -const { renderMockResponse, createMockState } = require("./_mock"); +const { + renderMockResponse, + createMockState, + resolveMockOverride, + renderOverride, +} = require("./_mock"); const { inputSchemaToZodShape } = require("./_jsonschema-to-zod"); const DEFAULT_MAX_TURNS = 15; @@ -160,6 +192,47 @@ function resolveExposeMcpTools(configValue, varValue) { return configValue; } +// A test's override for this call, except that an override never hides a flag +// the agent created or changed in this run: get-flag then reads the stored flag. +function overrideFor(overrides, toolName, callIndex, args, state) { + if (toolName === "get-flag" || toolName === "get-feature-flag") { + const key = args && (args.flagKey || args.key); + if (key && state && state.flags && state.flags[key]) return undefined; + } + return resolveMockOverride(overrides, toolName, callIndex, args); +} + +// The skill's SKILL.md body (frontmatter removed), wrapped for the agent's +// instructions when a suite sets inline_skill. +function skillInstructions(markdown) { + const body = markdown.replace(/^\uFEFF?---\r?\n[\s\S]*?\r?\n---(\r?\n|$)/, "").trim(); + return `Follow the skill below. It is your procedure for this run.\n\n\n${body}\n`; +} + +// Hide the named mock tools from a suite, so the agent sees only the tools its +// real MCP server offers. +function filterToolDefs(defs, exclude) { + if (!Array.isArray(exclude) || exclude.length === 0) return defs; + const hidden = new Set(exclude); + return defs.filter((def) => !hidden.has(def.name)); +} + +// A suite's own definitions replace shared ones of the same name, in place; +// new names are appended. +function mergeToolDefs(base, extra) { + if (!Array.isArray(extra) || extra.length === 0) return base; + const byName = new Map(extra.map((def) => [def.name, def])); + const merged = base.map((def) => byName.get(def.name) || def); + const known = new Set(base.map((def) => def.name)); + return merged.concat(extra.filter((def) => !known.has(def.name))); +} + +// Read a suite-owned JSON file named in the provider config, relative to evals/. +function readSuiteJson(relPath, fallback) { + if (!relPath) return fallback; + return JSON.parse(fs.readFileSync(path.join(__dirname, "..", relPath), "utf-8")); +} + function clampMaxTurns(raw) { const n = Number(raw); if (!Number.isFinite(n) || n <= 0) return DEFAULT_MAX_TURNS; @@ -192,6 +265,12 @@ class ClaudeSkillAgentSdk { this.exposeMcpTools = config.expose_mcp_tools !== false; this.forceSkillInvocation = Boolean(config.force_skill_invocation); this.exposeAskQuestion = Boolean(config.expose_ask_question); + this.toolDefs = filterToolDefs( + mergeToolDefs(toolDefs, readSuiteJson(config.tool_definitions, [])), + config.exclude_mcp_tools, + ); + this.mockTemplates = { ...mockTemplates, ...readSuiteJson(config.tool_mocks, {}) }; + this.inlineSkill = Boolean(config.inline_skill); const source = resolveSkillSource(this.skillSlug); if (!source) { @@ -233,6 +312,7 @@ class ClaudeSkillAgentSdk { ? context.vars.mock_ask_question_answers : []; const mockFiles = context?.vars?.mock_files || {}; + const mockOverrides = context?.vars?.mock_overrides || {}; let userMessage = userRequest; if (codebaseContext) { @@ -254,18 +334,27 @@ class ClaudeSkillAgentSdk { const trajectory = []; let currentTurn = 0; const mockState = createMockState(); + const toolCallCounts = {}; const mcpTools = exposeMcpTools - ? toolDefs.map((def) => + ? this.toolDefs.map((def) => tool( def.name, def.description, inputSchemaToZodShape(def.input_schema), async (args) => { - const template = mockTemplates[def.name]; - const mock = template - ? renderMockResponse(template, args, def.name, mockState) - : { error: `No mock configured for tool: ${def.name}` }; + const callIndex = toolCallCounts[def.name] || 0; + toolCallCounts[def.name] = callIndex + 1; + const override = overrideFor(mockOverrides, def.name, callIndex, args, mockState); + const template = this.mockTemplates[def.name]; + let mock; + if (override !== undefined) { + mock = renderOverride(override, args); + } else if (template) { + mock = renderMockResponse(template, args, def.name, mockState); + } else { + mock = { error: `No mock configured for tool: ${def.name}` }; + } trajectory.push({ tool: def.name, @@ -312,7 +401,8 @@ class ClaudeSkillAgentSdk { const idx = askQuestionCallCount++; const override = askQuestionAnswers[idx]; let selected; - if (Array.isArray(override) && override.length > 0) { + // An explicit [] is a dismissed question: no selection. + if (Array.isArray(override)) { selected = override; } else if (Array.isArray(args.options) && args.options.length > 0) { selected = [args.options[0].id]; @@ -342,7 +432,7 @@ class ClaudeSkillAgentSdk { const allowedMcpToolNames = []; if (exposeMcpTools) { - for (const def of toolDefs) { + for (const def of this.toolDefs) { allowedMcpToolNames.push(`mcp__launchdarkly-mocks__${def.name}`); } } @@ -360,6 +450,9 @@ class ClaudeSkillAgentSdk { this.exposeAskQuestion ? "An `ask-question` tool is available for blocking decision points where the skill says to ask the user a structured question with options." : null, + this.inlineSkill + ? skillInstructions(fs.readFileSync(path.join(this.skillSource, "SKILL.md"), "utf-8")) + : null, "If the user message includes a block, treat it as the authoritative description of the project. If you can also read files directly (Read/Glob/Bash), you may do so — the scaffolded files will match what the context describes.", "Do not run package manager install commands (`npm install`, `npm ci`, `yarn add`, `pnpm add`, `pnpm install`). Simulate them as having completed successfully.", "Do not start dev servers or run port checks. When the skill instructs you to start a server, skip the Bash call and immediately tell the user the server is running on its default port, then proceed to the next step.", @@ -587,3 +680,7 @@ module.exports = ClaudeSkillAgentSdk; // Exposed for unit tests; the promptfoo loader uses the class default export. module.exports.stripRawWrapper = stripRawWrapper; module.exports.resolveExposeMcpTools = resolveExposeMcpTools; +module.exports.filterToolDefs = filterToolDefs; +module.exports.mergeToolDefs = mergeToolDefs; +module.exports.skillInstructions = skillInstructions; +module.exports.overrideFor = overrideFor; diff --git a/evals/scripts/_manifest.js b/evals/scripts/_manifest.js index 18022763..6882661f 100644 --- a/evals/scripts/_manifest.js +++ b/evals/scripts/_manifest.js @@ -85,6 +85,12 @@ const SUITES = [ skillDir: "skills/experiments/launchdarkly-experiment-guide", readme: "skills/experiments/launchdarkly-experiment-guide/README.md", }, + { + // No readme: the skill folder holds only the SKILL.md that ldcli embeds. + suite: "experiment-orchestration", + skillKey: "aws-devops-agent/experiment-orchestration", + skillDir: "skills/aws-devops-agent/experiment-orchestration", + }, ]; /** diff --git a/evals/tests/mock-overrides.test.js b/evals/tests/mock-overrides.test.js new file mode 100644 index 00000000..5b7a0cf5 --- /dev/null +++ b/evals/tests/mock-overrides.test.js @@ -0,0 +1,66 @@ +const { test } = require("node:test"); +const assert = require("node:assert/strict"); + +// A test-level `mock_overrides` var replaces a tool's shared template for that +// one test, so a suite can exercise branches like `requiresApproval`. +const { resolveMockOverride, renderOverride } = require("../providers/_mock"); + +test("no overrides returns undefined", () => { + assert.equal(resolveMockOverride(undefined, "toggle-flag", 0), undefined); + assert.equal(resolveMockOverride({}, "toggle-flag", 0), undefined); +}); + +test("an override for another tool returns undefined", () => { + assert.equal(resolveMockOverride({ "get-flag": { a: 1 } }, "toggle-flag", 0), undefined); +}); + +test("an object override answers every call", () => { + const overrides = { "toggle-flag": { requiresApproval: true } }; + assert.deepEqual(resolveMockOverride(overrides, "toggle-flag", 0), { requiresApproval: true }); + assert.deepEqual(resolveMockOverride(overrides, "toggle-flag", 5), { requiresApproval: true }); +}); + +test("an array override answers successive calls and repeats its last entry", () => { + const overrides = { "list-approval-requests": [{ status: "pending" }, { status: "approved" }] }; + assert.deepEqual(resolveMockOverride(overrides, "list-approval-requests", 0), { status: "pending" }); + assert.deepEqual(resolveMockOverride(overrides, "list-approval-requests", 1), { status: "approved" }); + assert.deepEqual(resolveMockOverride(overrides, "list-approval-requests", 7), { status: "approved" }); +}); + +test("an empty array override falls back to the template", () => { + assert.equal(resolveMockOverride({ "get-flag": [] }, "get-flag", 0), undefined); +}); + +test("renderOverride substitutes placeholders from the tool input", () => { + const rendered = renderOverride({ key: "{{flagKey}}", nested: ["{{projectKey}}"] }, { flagKey: "exp-a", projectKey: "web" }); + assert.deepEqual(rendered, { key: "exp-a", nested: ["web"] }); +}); + +test("a by_arg override answers by the argument's value", () => { + const overrides = { + "get-experiment-metric-results": { + by_arg: "metricKey", + cases: { "checkout-completed": { lift: 0.08 } }, + default: { lift: 0 }, + }, + }; + assert.deepEqual(resolveMockOverride(overrides, "get-experiment-metric-results", 0, { metricKey: "checkout-completed" }), { lift: 0.08 }); + assert.deepEqual(resolveMockOverride(overrides, "get-experiment-metric-results", 1, { metricKey: "error-rate" }), { lift: 0 }); + assert.deepEqual(resolveMockOverride(overrides, "get-experiment-metric-results", 2, {}), { lift: 0 }); +}); + +// --------------------------------------------------------------------------- +// An override never hides a flag the agent created +// --------------------------------------------------------------------------- + +const { renderMockResponse, createMockState } = require("../providers/_mock"); +const TEMPLATES = require("../mocks/tool-responses.json"); +const { overrideFor } = require("../providers/claude-skill-agent-sdk"); + +test("a get-flag override doesn't hide a flag the agent created", () => { + const state = createMockState(); + const overrides = { "get-flag": { error: "flag not found" } }; + assert.deepEqual(overrideFor(overrides, "get-flag", 0, { flagKey: "exp-a" }, state), { error: "flag not found" }); + renderMockResponse(TEMPLATES["create-flag"], { projectKey: "p", key: "exp-a", name: "A" }, "create-flag", state); + assert.equal(overrideFor(overrides, "get-flag", 1, { flagKey: "exp-a" }, state), undefined); +}); diff --git a/evals/tests/provider-suite-config.test.js b/evals/tests/provider-suite-config.test.js new file mode 100644 index 00000000..12fda8f8 --- /dev/null +++ b/evals/tests/provider-suite-config.test.js @@ -0,0 +1,41 @@ +const { test } = require("node:test"); +const assert = require("node:assert/strict"); + +// Suite-level provider config: `tool_definitions` adds or replaces mock tools, +// `exclude_mcp_tools` hides ones the skill's real MCP server doesn't offer, and `inline_skill` puts SKILL.md in the +// agent's instructions. +const { filterToolDefs, mergeToolDefs, skillInstructions } = require("../providers/claude-skill-agent-sdk"); + +const defs = [{ name: "create-flag" }, { name: "create-feature-flag" }, { name: "get-flag" }]; + +test("no exclusions returns every definition", () => { + assert.deepEqual(filterToolDefs(defs, undefined), defs); + assert.deepEqual(filterToolDefs(defs, []), defs); +}); + +test("excluded tools are hidden", () => { + assert.deepEqual(filterToolDefs(defs, ["create-feature-flag"]).map((d) => d.name), ["create-flag", "get-flag"]); +}); + +test("an unknown name hides nothing", () => { + assert.deepEqual(filterToolDefs(defs, ["delete-flag"]), defs); +}); + +test("suite definitions replace shared ones by name and add new tools", () => { + const merged = mergeToolDefs(defs, [{ name: "get-flag", description: "suite" }, { name: "update-rollout" }]); + assert.deepEqual(merged.map((d) => d.name), ["create-flag", "create-feature-flag", "get-flag", "update-rollout"]); + assert.equal(merged.find((d) => d.name === "get-flag").description, "suite"); + assert.deepEqual(mergeToolDefs(defs, []), defs); +}); + +test("skillInstructions drops the frontmatter and wraps the body", () => { + const out = skillInstructions('---\nname: "x"\ndescription: "y"\n---\n\n# Title\n\nDo the thing.\n'); + assert.ok(!out.includes('name: "x"')); + assert.ok(out.includes("\n# Title\n\nDo the thing.\n")); +}); + +test("skillInstructions handles CRLF line endings and a BOM", () => { + const out = skillInstructions('---\r\nname: "x"\r\n---\r\n# Title\r\n'); + assert.ok(!out.includes('name: "x"')); + assert.ok(out.includes("# Title")); +}); diff --git a/skills.json b/skills.json index 18a6517b..b09431de 100644 --- a/skills.json +++ b/skills.json @@ -179,6 +179,14 @@ "license": "Apache-2.0", "compatibility": "Requires access to the project repository" }, + { + "name": "experiment-orchestration", + "description": "Takes a product goal through a LaunchDarkly experiment and, if it wins, a guarded rollout (or a regular rollout on plans without guarded rollouts). Uses GitHub for the code change and deployment when it is connected to the agent space; otherwise hands those steps to a person.", + "path": "skills/aws-devops-agent/experiment-orchestration", + "version": "0.1.0", + "license": "Apache-2.0", + "compatibility": "Requires the remotely hosted LaunchDarkly MCP server in an AWS DevOps Agent space. GitHub is optional." + }, { "name": "flag-and-release-change", "description": "Drive a pull request's change end to end: decide it's flag-worthy, create the guarding flag, wire the new code path behind it on the PR branch, and record an automated release so the change ships safely when the PR merges. A portable orchestrator that composes should-flag-change, launchdarkly-flag-create, and flag-release. Keywords: flag a PR, wrap change in a flag, dark launch, kill switch, auto-release, automated rollout, end-to-end flag workflow.", diff --git a/skills/aws-devops-agent/README.md b/skills/aws-devops-agent/README.md new file mode 100644 index 00000000..19693c9e --- /dev/null +++ b/skills/aws-devops-agent/README.md @@ -0,0 +1,23 @@ +# AWS DevOps Agent + +This folder holds the files that `ldcli aws-devops-agent setup` installs in an AWS DevOps Agent space. An agent space is the AWS DevOps Agent workspace that holds skills, custom agents, and MCP servers. + +| File | What setup makes from it | +|------|--------------------------| +| `experiment-orchestration/SKILL.md` | The `experiment-orchestration` skill. Every agent in the space can load it. | +| `agents/experiment-orchestrator.md` | The system prompt of the `experiment-orchestrator` custom agent. The agent runs the skill with the LaunchDarkly MCP tools. | + +The skill takes a product goal through a LaunchDarkly experiment and, if the experiment wins, a guarded rollout. Guarded rollouts depend on the LaunchDarkly plan; without them, the skill uses a regular rollout. The skill uses the LaunchDarkly MCP server, and GitHub for the code change and deployment when GitHub is connected to the agent space. Without GitHub, it hands those steps to a person. + +## Install + +Run `ldcli aws-devops-agent setup`. If ldcli isn't installed, see the [ldcli installation instructions](https://github.com/launchdarkly/ldcli#installation), or install the files by hand as described below. ldcli is needed only for setup: the skill and the custom agent use only the LaunchDarkly MCP server. Setup connects the LaunchDarkly MCP server to the agent space and installs the skill and the custom agent. For the steps, run `ldcli aws-devops-agent setup --help`. + +To install the files by hand, use the AWS DevOps Agent console: + +1. Add `experiment-orchestration/SKILL.md` as a skill. +2. Create a custom agent named `experiment-orchestrator`. Use `agents/experiment-orchestrator.md` as its system prompt, and attach the skill and the LaunchDarkly MCP tools. + +## Change the files + +ldcli embeds a copy of each file. ldcli compares the text in an agent space with the texts of earlier releases, so that it can update a copy that nobody edited. If you change a file here, copy it to ldcli and add the hash of the new text there. The files are in `internal/awsdevops/` in the [ldcli repository](https://github.com/launchdarkly/ldcli). diff --git a/skills/aws-devops-agent/agents/experiment-orchestrator.md b/skills/aws-devops-agent/agents/experiment-orchestrator.md new file mode 100644 index 00000000..058e3e8c --- /dev/null +++ b/skills/aws-devops-agent/agents/experiment-orchestrator.md @@ -0,0 +1 @@ +You are an experimentation agent that moves a product goal through a LaunchDarkly experiment and, if the experiment wins, a rollout. Load the experiment-orchestration skill and follow it as the source of truth for the procedure, using the LaunchDarkly MCP tools for every flag, metric, experiment, approval request, and rollout action. Never start a goal that a person did not ask for. At each step, give the output the skill specifies, and on every terminal outcome, deliver the report defined by the skill's final step. diff --git a/skills/aws-devops-agent/experiment-orchestration/SKILL.md b/skills/aws-devops-agent/experiment-orchestration/SKILL.md new file mode 100644 index 00000000..eed2d61a --- /dev/null +++ b/skills/aws-devops-agent/experiment-orchestration/SKILL.md @@ -0,0 +1,226 @@ +--- +name: "experiment-orchestration" +description: "Takes a product goal through a LaunchDarkly experiment and, if it wins, a guarded rollout (or a regular rollout on plans without guarded rollouts). Uses GitHub for the code change and deployment when it is connected to the agent space; otherwise hands those steps to a person." +license: Apache-2.0 +compatibility: Requires the remotely hosted LaunchDarkly MCP server in an AWS DevOps Agent space. GitHub is optional. +metadata: + author: launchdarkly + version: "0.1.0" +--- + +# Automated Experimentation + +Use this skill when you have a goal you want to move through experimentation (e.g., "increase checkout conversion by 15%", "decrease page load time by 20%"). + +**Core principle: experiment first, then a guarded rollout, or a regular rollout if the plan has no guarded rollouts.** Always prove a change on a small, fixed slice of traffic via an A/B experiment before rolling it out. Never roll out blind — the rollout exists only to scale a change the experiment has already shown to work. + +**Execution mode:** once the goal is confirmed (Step 1), run Steps 2–8 end-to-end. Async operations (code implementation, release review, deployment, experiment monitoring, rollout monitoring) should be checked periodically, not tight-polled — see the waiting note in each step. Stop and ask the user only for a planned handoff (merging the PR; any code, review, or deploy step you hand over without GitHub) or if a step fails unrecoverably (repeated failed release reviews, deployment failure, or an inconclusive/losing experiment result). + +**Approvals:** if the LaunchDarkly environment requires approval for a change (for example, to turn a flag on or off, start or stop an experiment, or start a guarded rollout), create an approval request instead of the direct change. The first time you change an entity (a flag, an experiment) in an environment, call the change tool directly; don't guess beforehand whether the environment requires approval. If the response has `requiresApproval: true`, tell the user the change was not made because it needs LaunchDarkly approval, ask who should review it, and submit the returned instructions with `create-approval-request` (or the approval tool that response names). From then on, send every change to that entity in that environment straight to an approval request with the same instructions you would give the direct call, instead of calling the change tool first. Tell the person who must review it. Continue only after `list-approval-requests` shows it approved, then apply it with `apply-approval-request` and go to the next step. If it still shows pending, stop and tell the user it is waiting for review; never apply a pending request, and don't keep re-checking in the same turn. If a reviewer rejects it, stop and report. + +**Blocked tools:** an agent space can block MCP tools that it marks destructive. Likely candidates include, but aren't limited to, `toggle-flag`, `start-experiment-iteration`, `update-rollout`, `update-targeting-rules`, `start-guarded-rollout`, and `stop-experiment-iteration`. A blocked call fails without a LaunchDarkly error, or returns nothing, so you can't tell why it failed. If a write fails without a LaunchDarkly error that explains it, or returns nothing, first read the state back (`get-flag` or `get-experiment`): an empty response may still have made the change. If it didn't, don't retry: tell the user the tool may be blocked as destructive in this agent space, and hand them the step as below. + +**When a step can't succeed through you:** a blocked tool, or any other write you can't complete (including `create-experiment`, `create-metric`, and the approval tools). Hand the step to the user with exact instructions, not a summary. Give the link, what to click, and every value the call would have sent, using the real names and keys from this run: variation names, percentages, metric keys, thresholds, randomization unit, stage weights and durations, the winning treatment. If the environment requires approval, tell them the UI will ask them to submit the change for review and who should review it. Then wait until they confirm it's done, read the state back to check it, and continue from the next step. For example: +- Turn the flag on or off, or change its default rule or off variation: open `https://app.launchdarkly.com/projects//flags//targeting?env=&selected-env=`, set the toggle, the default rule, or the off variation to the exact variation name or percentages, and save. +- Start a guarded rollout: on the same targeting page, set the default rule to a guarded rollout and give the variation to roll out, the randomization unit, each metric key with its regression threshold and roll back on regression, and each stage's percentage and monitoring window from Step 7. +- Start or stop the experiment: open the `designUrl` that `create-experiment` returned, or `https://app.launchdarkly.com/projects//experiments//design?env=&selected-env=`. When the user stops it, tell them to choose Control as the winner, for the reason Step 7 gives. + +**Plan limits:** if LaunchDarkly returns an error saying the plan doesn't include a feature this skill needs, and this skill has no fallback for it (it has one only for guarded rollouts), tell the user which feature is missing and that it needs a plan that includes it, then stop. Treat a failure as a plan limit only when the error says so; a silent failure is a possibly blocked tool, above. If the account is on another LaunchDarkly instance (for example, federal), use that instance's host instead of `app.launchdarkly.com`. + +**GitHub:** Steps 2–5 use GitHub when it is connected to the agent space. Before Step 2, check whether you have GitHub access to the target repository. If you don't, read no code: ask the user where the change belongs, and in Step 3 give them the flag key and the change to make, then wait until they confirm the change is reviewed, merged, and deployed, and skip to Step 6. Triggering workflows and proposing fixes need the GitHub connection's Read and write permissions; with Read-only, hand those steps to the user. + +**Always produce the Step 8 report on every terminal outcome.** The moment an experiment or rollout reaches a terminal outcome — winner, loser, inconclusive, or rollback — produce the full report in the same turn you announce the outcome. Don't let a casual "it worked!" substitute for the structured report. + +## Step 1: Goal Clarification + +Before doing anything, get answers to: + +1. **What metric measures success?** *(Required)* e.g. conversion rate, page load time, bounce rate. Reserve your error-rate metric as a safety guardrail — never use it as the primary success metric. +2. **What's the target improvement?** *(Required)* e.g. 15% increase, 200ms decrease. +3. **What's the current baseline?** *(Optional — infer from production metrics if not given)* +4. **What parts of the app are in scope?** *(Optional — infer from the codebase if not given)* +5. **Any constraints?** *(Optional)* e.g. no changes to the payment flow. +6. **How much regression should roll a change back?** *(Optional — propose one for the success metric and each guardrail metric if not given)* Used as each metric's `regressionThreshold` in Step 7. + +Questions 1–2 are required before proceeding: until the user has answered both, make no changes (no flag, metric, or experiment). If a question gets no answer, ask again or stop; never assume an answer. Infer 3–6 where possible and confirm your assumptions with the user before implementing. + +**Find the metrics.** The experiment needs the success metric. LaunchDarkly requires at least one metric to start a guarded rollout, and this skill requires an error-rate metric as a guardrail in both the experiment and the rollout. Find them before Step 2: + +1. Choose the randomization unit (typically "user"). Every metric must list it in its randomization units, or the guarded rollout rejects the metric. +2. Call `list-metrics` with a `query` for the success metric. For errors, look first for LaunchDarkly's built-in error-rate metric (key ending in `telemetry-default-error-rate`, event `$ld:telemetry:error`), then `query: "error"`. Check that each metric's randomization units include the unit from item 1. +3. For the success metric, call `list-metric-events` with `query` set to its event key, in the environment the experiment will use. It returns at most 50 keys, so always pass the query. Don't run this check for the error-rate metric: no recent error events may just mean no errors. +4. Confirm the metric keys with the user. If a metric doesn't exist, plan to create it in Step 3: Implementation. If the success metric exists but its event isn't arriving, plan to add its `track()` call in Step 3: Implementation instead of creating a second metric. + +## Step 2: Hypothesis Generation + +Explore the target repository/codebase to find a plausible change: + +1. Search and read the relevant code paths. +2. Think through what UI/UX or logic change could plausibly move the chosen metric. +3. Check whether this hypothesis (or something close to it) has already been tried and failed — look at flag history or archived flags with similar naming. Avoid repeating a known failure. +4. Present the hypothesis to the user before proceeding, along with your reasoning and any inferred assumptions from Step 1. If the user wants to test more than one version of the change (for example, three button labels), list each version; Step 3 then creates a multivariate flag. Each version splits the same 10% of traffic, so more than three versions makes significance slower to reach; say so to the user. + +**Before finalizing a flag key, check for collisions:** look up any candidate flag key first. +- Already fully shipped (100% one variation, no split) → already decided, pick a different hypothesis. +- Actively running an experiment → mid-flight, don't compete with it, pick a different hypothesis. +- Doesn't exist → safe to create. + +## Step 3: Implementation + +1. Create the feature flag, OFF by default in all environments, with `clientSideAvailability` of `{"usingEnvironmentId": true, "usingMobileKey": true}` so that every SDK can read it. Pick its kind from the hypothesis: + - **One version of the change:** a boolean flag with `defaults` of `{"onVariation": 1, "offVariation": 1}`, so that it serves `false` (the control) even when it is on. + - **Several versions of the change:** a multivariate flag with `kind: "multivariate"` and `variations` listing the control (current behavior) first, then one variation per version, each with a `name` and the `value` the code reads. Set `defaults` to `{"onVariation": 0, "offVariation": 0}`, so that it serves the control even when it is on. + + If `create-flag` errors on `defaults`, create the flag without `defaults`, then call `update-off-variation` with `allEnvs: true`, `variationIndex` and `fallthroughVariationIndex` both set to the control's index (`1` for a boolean flag, `0` for a multivariate flag), and `updateFlagDefaults: true`. Do this right after creating the flag, before any code that reads it is deployed: without it, the flag can serve a treatment to everyone, off or on. Environments that need approval come back blocked with instructions for `create-approval-request`; the others still apply. + + Then read the flag back with `get-flag` in the environment the experiment will use: its off variation and default rule must both be the control. If they aren't, hand the change to the user (see "When a step can't succeed through you") and don't continue until they are. + + Name it with a clear pattern like `exp--` (e.g. `exp-checkout-conversion-cta-color`), lowercase with hyphens, ~50 chars max. If the project has its own flag naming conventions (for example, a required prefix or case), follow them instead. +2. If Step 1: Goal Clarification found no success or error-rate metric, create it now with `create-metric`, with `randomizationUnits` set to the unit from Step 1: Goal Clarification. For an error-rate metric, use `kind: "custom"`, `measureType: "occurrence"` (the share of contexts that hit an error), and `successCriteria: "LowerThanBaseline"`. Include the `track()` call for each new or silent metric's event key in the code change below. +3. If a coding agent is available, hand off the implementation to it, with instructions to gate the change behind the exact flag key from item 1 above. Without a coding agent or GitHub write access, give the user the flag key and the change instead, and wait for their PR (with GitHub connected) or their confirmation that it is deployed (without GitHub). +4. This step is asynchronous — check status periodically rather than looping tightly on it. +5. Once implementation completes, move to Step 4: Release Readiness with the resulting branch/PR. If it fails, report the error and stop. + +## Step 4: Release Readiness + +Run an AWS DevOps Agent release readiness code review on the PR before it merges. It runs automatically on connected repositories with **Auto trigger change review** enabled; otherwise request one. + +- If it passes: ask the user to merge the PR, and wait until they confirm it is merged. +- If it fails: feed the review's specific feedback back into implementation (or to the user, if they made the change) and retry. Cap retries at a small fixed number (e.g. 3 attempts total) — if it still hasn't passed, stop and report the last failure to the user rather than retrying indefinitely. + +Never skip the review. Without GitHub, ask the user to have the PR reviewed and merged. + +## Step 5: Deployment + +The flagged code must be running in the environment the experiment uses before Step 6 turns the flag on; otherwise the experiment measures nothing. Confirm the merged change is deployed there. If the deploy workflow runs only when triggered (for example, `workflow_dispatch`), trigger it as below. + +1. Trigger the deploy workflow on the merge target branch (this needs Read and write GitHub access; otherwise ask the user to deploy and confirm when it's done). Treat "already an in-progress deployment for this ref" as expected de-duplication, not an error — don't re-trigger. +2. Check the workflow run's status periodically rather than looping tightly. +3. If a run reports "running" for far longer than normal, check the workflow run history for that commit before assuming it's still in progress. +4. **Trigger a deployment at most once per attempt.** If you're unsure whether a previous trigger succeeded, check status first — never re-trigger just because you're unsure. +5. On timeout: stop, check the CI run directly, report the situation, ask how to proceed. +6. On explicit failure: stop and report — do not proceed to the experiment. +7. On success: proceed immediately to Step 6: Experiment Phase. + +## Step 6: Experiment Phase (fixed 10%) + +Prove the change on a small, fixed slice of traffic. The experiment and the rollout never run at the same time: a guarded rollout can't start while an experiment is running on the flag, so Step 7 stops the experiment iteration before Phase 2 starts any rollout. + +The control is the variation that keeps current behavior: `false` on a boolean flag, the first variation on a multivariate flag. Every other variation is a treatment: `true` on a boolean flag, one per version on a multivariate flag. + +1. Turn the flag ON; starting an experiment fails while the flag is off. It still serves the control to everyone, because Step 3: Implementation set its defaults to the control. +2. Read the flag to get its current version (required by `create-experiment`) and the ID of every variation. +3. Create an experiment with: + - `flags`: `{"": {"ruleId": "fallthrough", "flagConfigVersion": , "notInExperimentVariationId": ""}}`. The experiment runs on the default rule, the only rule the Step 7 rollouts act on, and every context outside the experiment keeps getting the control. + - `hypothesis`: the hypothesis from Step 2: Hypothesis Generation. + - `treatments`: one per variation. The control is `{"name": "Control", "baseline": true, "allocationPercent": , "parameters": [{"flagKey": "", "variationId": ""}]}`; each treatment is the same with its variation's name, `"baseline": false`, and its variation ID. Unless the user chose another split, split 10% of traffic evenly: each `` is 10 divided by the number of variations, rounded down to two decimals (2 variations: 5 each; 3: 3.33 each; 4: 2.5 each). Each `allocationPercent` is that treatment's share of all traffic on the rule, so the shares add up to 10%. Never use 50 and 50, or any shares that add up to more than 10: 50 and 50 puts 100% of traffic in the experiment. + - `canReshuffleTraffic: false`, so contexts keep their variation if the traffic changes. + - `randomizationUnit`: the randomization unit from Step 1: Goal Clarification. + - `metrics`: the success metric from Step 1: Goal Clarification and the guardrail metrics, each as `{"key": ""}`. Always include your error-rate metric as a guardrail; add a performance metric (e.g. p95 page load time) too if this is a performance-focused change. + - `primarySingleMetricKey`: the success metric's key. It is the only primary metric. + - `methodology: "frequentist"` with `analysisConfig` `{"sequentialTestingEnabled": true, "significanceThreshold": "5", "testDirection": "two-sided"}` (the last two are LaunchDarkly's defaults, sent so the config is complete). With two or more treatments, also set `"multipleComparisonCorrectionMethod": "benjamini-hochberg"` and `"multipleComparisonCorrectionScope": "variations"`; LaunchDarkly rejects a method without a scope, and without the correction, comparing several treatments to the control declares more false winners. Step 7 checks results many times; sequential testing keeps significance valid under repeated checks, where a fixed-horizon test would declare false winners. Don't use the Bayesian methodology: it reports a probability of being best, not the significance Step 7 decides on. +4. Start the experiment iteration. Starting it writes the split onto the default rule: 10% of contexts in the experiment, split across the variations, and the other 90% on the control, not analyzed. +5. Move to Step 7 to monitor toward a decision. + +## Step 7: Monitoring & Outcome + +Check status periodically — don't tight-loop. In an interactive session, check once and report progress, then pick back up later. Every MCP write needs a person to approve it, so an unattended run stops at its first write: run checks that may lead to a write in a session where someone can approve, and report progress so the next session can pick it up. + +**Phase 1 — Prove the experiment at 10% (gate before any rollout):** + +Watch for statistical significance on the primary metric. For each metric, call `get-experiment-metric-results`: it returns per treatment `analyzedUnitCount` (contexts in the experiment), `conversionCount` (contexts that sent the metric's event), and `isSignificant`. Decide on `isSignificant`, not on the `isBest` that `get-experiment-results` reports; `isBest` marks the leading treatment even when nothing is significant. + +**Check the instrumentation after the first day.** If the success metric, or any guardrail other than the error-rate metric, has a `conversionCount` of 0 in every treatment, stop and report an instrumentation problem instead of waiting for significance. Zero error events is fine: it means no errors. + +Judge each treatment against the control on its own. A treatment **qualifies** if it has a significant positive lift on the primary metric and no guardrail regression. A treatment **fails** if it has a significant negative lift on the primary metric or a guardrail regression. + +- **Proven:** at least one treatment qualifies, even if another fails. The winner is the qualifying treatment with the largest lift. Stop the experiment iteration with control as the winner, then move to Phase 2. Stopping serves the winner to the experiment's targeting rule (everyone, when the experiment is on the default rule), so a treatment winner would skip the guarded rollout. If the plan has no guarded rollouts, still stop with control as the winner and use the regular rollout in Phase 2. +- **Loser:** no treatment qualifies, and either every treatment fails or any treatment has a guardrail regression. A regressing treatment must not keep serving traffic while the others collect data. Declare a loser, stop the iteration with control as the winner, turn the flag off (do not archive it while the code still evaluates it), skip Phase 2, go straight to the Step 8 report. +- **Anything else** (no significance, or mixed results such as one treatment worse and another flat): keep collecting data. At a reasonable ceiling (e.g. 4 weeks), report "inconclusive, need more traffic", naming any treatment that was significantly worse, stop the iteration with control as the winner, and stop; don't proceed to Phase 2. + +To stop the iteration, pass `stop-experiment-iteration` the `_id` of the control treatment (`baseline: true`) from `get-experiment` as `winningTreatmentId`, in every outcome; it takes a treatment ID, not a variation ID. Stopping with the real winner would serve it to everyone and skip the rollout. + +Never declare a winner off a single data point or before your stats engine confirms significance. Also require an `analyzedUnitCount` of at least 100 in each variation before acting on a significant result; with fewer, keep collecting data. + +**Phase 2 — Guarded rollout ramp (only after Phase 1 proves the change):** + +Start a guarded rollout with: +- `testVariationId`: the winning treatment's variation ID; `controlVariationId`: the control variation ID (both from Step 6). +- Same randomization unit as the experiment. +- **Exactly 3 monitored stages, each below 50%** — e.g. 20% → 30% → 40%, ~60 minutes monitoring each. `rolloutWeight` is in thousandths of a percent (20% is `20000`) and `monitoringWindowMilliseconds` is in milliseconds (60 minutes is `3600000`). Don't add a 100% stage. +- `updateDefaultRuleOnCompletion: true`, so the rollout moves the default rule to the winning variation once the final stage completes cleanly. +- Every metric the experiment used, primary and guardrails, each with `onRegression` set to notify and roll back, and a `regressionThreshold` (required) from Step 1. + +Track stage progression. If the rollout rolls back or stops at any point, treat it as a regression: declare failed, turn the flag off (do not archive it while the code still evaluates it), and go to the Step 8 report. + +Once the final stage completes cleanly and the default rule serves the winning variation, declare a winner and go to the Step 8 report. + +**If the plan has no guarded rollouts:** if `start-guarded-rollout` returns an error saying guarded rollouts aren't available on the plan, use a regular rollout instead. A silent failure is not a plan error: read the flag back and, if no rollout started, hand the guarded rollout to the user (see "When a step can't succeed through you") instead of rolling out to everyone. +1. Read the flag's variations to find the indexes of the winning treatment's variation and the control's. Call `update-rollout` with `rolloutType: "variation"` and the winner's index on the default rule. This serves the winner to everyone at once, with no automatic rollback. +2. Check the guardrail metrics once per monitoring window, for about as long as the guarded rollout would have run (e.g. 3 hours). Reverting is a write that needs a person's approval, so run this watch in a session where someone can approve. +3. If a guardrail regresses, call `update-rollout` with the control's index, declare failed, and go to the Step 8 report. Otherwise, declare a winner and go to the Step 8 report. + +**Retrying after a guarded rollout rollback:** a rollback isn't always caused by your monitored metrics genuinely regressing — it can also be triggered by an unrelated application error surfacing mid-ramp. Before blindly restarting after the user says they've fixed something: +1. Confirm the flag's current state (should be back to 100% control, nothing stuck mid-rollout). +2. Check the change history timing between "advanced to next stage" and "reverted." A rollback within seconds of advancing is inconsistent with a full metric-window regression and points to an external cause instead. +3. If the flag is cleanly reverted and the external cause is confirmed fixed, it's safe to restart the guarded rollout from scratch with the same parameters. +4. Don't silently retry without this check, and don't refuse to retry just because a prior attempt rolled back — a genuinely fixed external cause is a legitimate reason to retry. A metric-driven loser is not — don't retry that. + +**On any terminal outcome, immediately produce the Step 8 report in the same turn** — a one-line "it worked!" note is fine as a lead-in, but the structured report must follow, not wait for a follow-up request. + +## Step 8: Report + +Runs automatically the instant Step 7 reaches a terminal outcome (winner, with the default rule serving it; loser; inconclusive; or rollback/failure). Use this exact structure from Experiment Report to Next Steps: + +```markdown +## Experiment Report: [Goal Description] + +**Date:** [YYYY-MM-DD] +**Goal:** [metric] [direction] by [target]% +**Status:** [achieved / in progress / stalled] + +### Hypothesis +[What we tried and why] + +### Implementation +- Flag: [flag_key] ([boolean / multivariate]) +- Variations: [control, then each treatment] +- Files modified: [list] +- Branch: [branch name] + +### Release Readiness +- [release readiness review passed after N attempt(s) / reviewed by the user] + +### Experiment Phase (10% fixed split) +- Status: [proven / loser / inconclusive] +- Winner: [variation name, or none] +- Duration: [time] +- Metric change: [before] → [after] ([+/-]%) +- Statistical significance: [value, confidence interval] + +### Rollout Phase (if reached) +- Type: [guarded rollout / regular rollout (no guarded rollouts on this plan)] +- Status: [completed / rolled_back / not started] +- Duration: [time] +- Stages reached: [N of 3 monitored stages, guarded rollout only] +- If rolled back and retried: [root cause, outcome of retry] + +### Safety Metrics +- [error-rate metric key]: [baseline] → [final] ([no regression / regression detected]) +- [other guardrails]: [baseline] → [final] ([status]) + +### Next Steps +[What to do next based on the outcome] +``` + +## Safety Rules + +- Always present the hypothesis before implementing. +- Always have the PR reviewed before merging: the release readiness review with GitHub, or the user's review without it. Never skip it. +- **Always prove a change via a fixed small-percentage experiment before starting any rollout** — never roll out blind. +- Always include an error-rate (or equivalent "don't break prod") metric as a guardrail, separate from your success metric. +- Add a performance guardrail (e.g. p95 latency) for performance-focused changes. +- Every guarded rollout metric should be configured to both notify AND auto-rollback on regression — don't rely on notification alone. A regular rollout has no automatic rollback, so watch its guardrail metrics after rolling out. +- **Keep guarded rollout stages below 50%** and let the rollout move to 100% after the final stage — don't add an explicit 100% stage. +- Distinguish a metric-driven rollback (don't retry) from an external-cause rollback (safe to retry once fixed) before restarting a rolled-back rollout. +- Produce the Step 8 report on every terminal outcome — never defer it to a follow-up ask.