From 78a492bdabe3127d434b5f68d1d254f6bc9e51e8 Mon Sep 17 00:00:00 2001 From: Carmen Romero Date: Tue, 27 Oct 2026 12:08:44 -0700 Subject: [PATCH 1/3] feat(experiment-setup): add an eval suite for approval before create and start Score the skill's gates for a complete design, suggested versus asked fields, a missing event, and not starting or changing a flag until the user says so. Co-authored-by: Cursor --- evals/README.md | 2 + evals/experiment-setup/promptfooconfig.yaml | 354 ++++++++++++++++++++ evals/mocks/tool-responses.json | 33 ++ evals/package.json | 2 + evals/providers/_mock.js | 172 ++++++++++ evals/scripts/_manifest.js | 6 + evals/tests/mock.test.js | 162 +++++++++ evals/tools/definitions.json | 152 +++++++++ 8 files changed, 883 insertions(+) create mode 100644 evals/experiment-setup/promptfooconfig.yaml diff --git a/evals/README.md b/evals/README.md index 603fc282..1191f337 100644 --- a/evals/README.md +++ b/evals/README.md @@ -34,6 +34,7 @@ npm run eval:agentcontrol-tools # agentcontrol/tools npm run eval:configs-variations # agentcontrol/configs-variations npm run eval:flag-create # feature-flags/launchdarkly-flag-create npm run eval:onboarding # onboarding +npm run eval:experiment-setup # experiments/launchdarkly-experiment-setup # Quick smoke check — first test case only (~15-20s, ~$0.05) npm run eval:configs-create:single @@ -42,6 +43,7 @@ npm run eval:agentcontrol-tools:single npm run eval:configs-variations:single npm run eval:flag-create:single npm run eval:onboarding:single +npm run eval:experiment-setup:single # Aggregate and CI operations npm run eval:all # Run every suite and rebuild ../eval-scores.json diff --git a/evals/experiment-setup/promptfooconfig.yaml b/evals/experiment-setup/promptfooconfig.yaml new file mode 100644 index 00000000..ff7dad4f --- /dev/null +++ b/evals/experiment-setup/promptfooconfig.yaml @@ -0,0 +1,354 @@ +# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json +# +# Run with shared defaults: +# promptfoo eval -c shared/defaults.yaml -c experiment-setup/promptfooconfig.yaml +# +# Scores the approval gates in launchdarkly-experiment-setup: a complete design +# before create, suggestions versus questions for the control and randomization +# unit, the missing-event warning, and no flag change or iteration start until +# the user says so. Each case has one weight-3 tool-call check. Wording is a +# separate weight-2 rubric. +description: "Evaluation of the experiment-setup skill: design, approve, then create or start" + +prompts: + - file://../../skills/experiments/launchdarkly-experiment-setup/SKILL.md + +providers: + - id: file://../providers/claude-skill-agent-sdk.js + label: claude-skill-agent-sdk + config: + skill_slug: launchdarkly-experiment-setup + allow_builtins: false + expose_mcp_tools: true + expose_ask_question: true + force_skill_invocation: true + +tests: + # ------------------------------------------------------------------ + # 1. Complete design. The user already named every required piece, + # including the control and randomization unit. "yes" creates the + # experiment and does not start it. methodology stays omitted. + # checkout-flow uses the shared get-flag template: version 1, and the + # fallthrough rule id is the string "fallthrough". + # ------------------------------------------------------------------ + - description: "Complete design: yes creates without methodology, metric, or start" + vars: + user_request: > + Set up an experiment in project "my-project", environment "production". + Flag: checkout-flow. Variations: true (id var-true) and false (id var-false). + Control: the false variation. Hypothesis: the new checkout will lift + completion rate by 3%. Primary metric: the existing checkout-completed + metric. Randomization unit: user. Show one design. When I approve it, + create the experiment. Do not start it. + max_turns: 16 + mock_ask_question_answers: + - ["yes"] + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const tools = output.tools_called || []; + const createIdx = traj.findIndex(t => t.tool === 'create-experiment'); + if (createIdx === -1) return { pass: false, score: 0, reason: 'no create-experiment' }; + const args = traj[createIdx].arguments || {}; + const iteration = args.iteration || {}; + const methodologySent = args.methodology != null || iteration.methodology != null; + const flags = iteration.flags || {}; + const flagCfg = flags['checkout-flow'] || flags[Object.keys(flags)[0]]; + const flagOk = !!flagCfg && flagCfg.ruleId === 'fallthrough' && flagCfg.flagConfigVersion === 1; + const approved = traj.slice(0, createIdx).some(t => { + if (!String(t.tool).includes('ask-question')) return false; + let selected = []; + try { selected = JSON.parse(t.mock_response_preview || '{}').selected || []; } catch (e) {} + return selected.map(String).includes('yes'); + }); + const pass = approved && flagOk && !methodologySent && !tools.includes('create-metric') && !tools.includes('start-experiment-iteration'); + return { pass, score: pass ? 1 : 0, reason: 'approved=' + approved + ' flag=' + JSON.stringify(flagCfg) + ' methodology=' + methodologySent + ' tools=' + tools.join(' -> ') }; + metric: creates_without_methodology_or_start + weight: 3 + - type: llm-rubric + value: | + Evaluate `assistant_turns` and any `ask-question` prompts in `trajectory`. + Score 1.0 if the agent shows one complete design before the user approves, + and that design names the flag, both variations, the false variation as + control, the hypothesis, checkout-completed, and randomization unit user. + Score 0 if it creates with no design, or shows a second conflicting design + before the approval. + metric: one_complete_design + weight: 2 + + # ------------------------------------------------------------------ + # 2. Flag is on and active, and user is the only context kind. The + # skill suggests the control and the randomization unit instead of + # gathering them. + # ------------------------------------------------------------------ + - description: "Suggests control and randomization unit when the flag is active and only user contexts exist" + vars: + user_request: > + Set up an experiment in project "only-user", environment "production". + Flag: checkout-active. Compare its variations. Hypothesis: the new + checkout will lift completion rate by 3%. Primary metric: the existing + checkout-completed metric. I have not chosen a control variation or a + randomization unit. + max_turns: 12 + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const tools = output.tools_called || []; + const searches = traj.filter(t => t.tool === 'search-contexts'); + const userCall = searches.some(t => { + const a = t.arguments || {}; + return a.kind === 'user' || /kind\s+equals\s+"?user"?/i.test(String(a.filter || '')); + }); + const otherCall = searches.some(t => /kind\s+notEquals\s+"?user"?/i.test(String((t.arguments || {}).filter || ''))); + const gathered = traj.some(t => { + if (!String(t.tool).includes('ask-question')) return false; + const text = JSON.stringify(t.arguments || {}).toLowerCase(); + if (text.includes('suggested')) return false; + return /control|baseline|randomization|context kind/.test(text); + }); + const pass = tools.includes('get-flag') && tools.includes('get-flag-status-across-envs') && userCall && otherCall && !gathered && !tools.includes('create-experiment') && !tools.includes('create-metric'); + return { pass, score: pass ? 1 : 0, reason: 'userCall=' + userCall + ' otherCall=' + otherCall + ' gathered=' + gathered + ' tools=' + tools.join(' -> ') }; + metric: suggests_from_flag_and_contexts + weight: 3 + - type: llm-rubric + value: | + The user did not name a control or a randomization unit. The flag is on + and active, and user is the only context kind. Score 1.0 if the design + labels both the served variation ("Current checkout") and randomization + unit "user" as suggested, each with a short reason, and there is no + gather message asking the user to supply either value. Deduct 0.5 if + either value is missing the suggested label. Score 0 if it asks the user + to choose them before showing a design. + metric: labels_suggestions + weight: 2 + + # ------------------------------------------------------------------ + # 3. Flag is off and inactive, and another context kind is stored. + # Ask for the control and the randomization unit. Do not suggest them. + # ------------------------------------------------------------------ + - description: "Asks for control and randomization unit when the flag is inactive and contexts are mixed" + vars: + user_request: > + Set up an experiment in project "mixed-contexts", environment "production". + Flag: checkout-inactive. Compare its variations. Hypothesis: the new + checkout will lift completion rate by 3%. Primary metric: the existing + checkout-completed metric. I have not chosen a control variation or a + randomization unit. + max_turns: 12 + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const tools = output.tools_called || []; + const asks = traj.filter(t => String(t.tool).includes('ask-question')); + const pass = asks.length === 1 && !tools.includes('create-experiment') && !tools.includes('create-metric'); + return { pass, score: pass ? 1 : 0, reason: 'asks=' + asks.length + ' tools=' + tools.join(' -> ') }; + metric: asks_once_and_does_not_create + weight: 3 + - type: llm-rubric + value: | + The flag is off and inactive, and more than one context kind is stored. + Score 1.0 if the agent asks for both the control variation and the + randomization unit, leaves both blank, and does not label either as + suggested. Score 0 if it suggests either value or creates the experiment. + metric: asks_instead_of_suggesting + weight: 2 + + # ------------------------------------------------------------------ + # 4. Flag, hypothesis, and primary metric are all missing. One gather + # message, and nothing is created. + # ------------------------------------------------------------------ + - description: "Missing pieces: one gather message and no create" + vars: + user_request: > + I want to run an experiment in project "my-project", environment + "production". I don't have a flag, a hypothesis, or a primary metric yet. + max_turns: 8 + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const tools = output.tools_called || []; + const asks = traj.filter(t => String(t.tool).includes('ask-question')); + const pass = asks.length === 1 && !tools.includes('create-metric') && !tools.includes('create-experiment'); + return { pass, score: pass ? 1 : 0, reason: 'asks=' + asks.length + ' tools=' + tools.join(' -> ') }; + metric: one_gather_and_no_create + weight: 3 + - type: llm-rubric + value: | + Score 1.0 if a single message asks for the missing flag (and its + variations), the hypothesis, and the primary metric. Deduct 0.34 for + each of those three that the message leaves out, or if it splits them + across separate messages. Score 0 if it shows a design or creates anything. + metric: gather_names_missing_pieces + weight: 2 + + # ------------------------------------------------------------------ + # 5. The custom event has not arrived. The design warns, and "yes" + # still creates the metric and the experiment. list-metric-events + # takes `query`, not eventKey; any arguments count. + # ------------------------------------------------------------------ + - description: "Missing event: list-metric-events runs, then yes still creates" + vars: + user_request: > + Set up an experiment in project "my-project", environment "production". + Flag: checkout-flow. Variations: true (id var-true) and false (id var-false). + Control: the false variation. Hypothesis: the new checkout will lift + completion rate by 3%. Primary metric: a new custom metric "CTA clicked" + (key cta-clicked) measuring the share of users who sent the cta-clicked + event. Randomization unit: user. Show the design. When I approve it, + create the metric and the experiment. Do not start it. + max_turns: 16 + mock_ask_question_answers: + - ["yes"] + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const eventsIdx = traj.findIndex(t => t.tool === 'list-metric-events'); + const metricIdx = traj.findIndex(t => t.tool === 'create-metric'); + const expIdx = traj.findIndex(t => t.tool === 'create-experiment'); + const pass = eventsIdx !== -1 && metricIdx !== -1 && expIdx !== -1 && eventsIdx < metricIdx && eventsIdx < expIdx; + return { pass, score: pass ? 1 : 0, reason: 'events@' + eventsIdx + ' metric@' + metricIdx + ' experiment@' + expIdx }; + metric: lists_events_before_create + weight: 3 + - type: llm-rubric + value: | + The cta-clicked event is not in the list-metric-events result. Similar + keys cta-click and cta-clicked-v2 are. Score 1.0 if the design warns + that the event has not arrived in the environment and lists those + similar keys. Score 0 if it blocks creation or omits the warning. + metric: warns_and_lists_similar_keys + weight: 2 + + # ------------------------------------------------------------------ + # 6. The user says to start. The flag is off and the event is still + # missing. One question covers both. No flag write and no start. + # The scripted answer is "no" so the first option cannot approve a toggle. + # ------------------------------------------------------------------ + - description: "Start blocked: flag off and event missing, no flag change and no start" + vars: + user_request: > + Start the existing experiment "checkout-draft" in project "my-project", + environment "production". It uses flag checkout-flow and a custom metric + whose event key is cta-clicked. Start it now. + max_turns: 12 + mock_ask_question_answers: + - ["no"] + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const tools = output.tools_called || []; + const asks = traj.filter(t => String(t.tool).includes('ask-question')); + const forbidden = ['toggle-flag','update-flag-settings','update-feature-flag','create-flag','create-feature-flag','start-experiment-iteration']; + const called = forbidden.filter(name => tools.includes(name)); + const pass = asks.length === 1 && called.length === 0; + return { pass, score: pass ? 1 : 0, reason: 'asks=' + asks.length + (called.length ? ' called ' + called.join(', ') : ' no flag change or start') }; + metric: one_question_and_no_flag_or_start + weight: 3 + - type: llm-rubric + value: | + The flag is off and the cta-clicked event is not arriving. Score 1.0 if + one message covers both: the flag must be on before starting, and whether + to start anyway while the event is missing. Score 0 if it asks those as + separate messages, turns the flag on, or starts the iteration. + metric: one_message_covers_flag_and_event + weight: 2 + + # ------------------------------------------------------------------ + # 7. The named flag does not exist. Offer three choices. create-flag + # runs only after an explicit yes to the proposed details. The scripted + # answer is "no". + # ------------------------------------------------------------------ + - description: "Missing flag: three choices, and no is not a yes to create-flag" + vars: + user_request: > + Set up an experiment in project "my-project", environment "production". + Flag: missing-checkout. Variations: true and false. Control: false. + Hypothesis: the new checkout will lift completion rate by 3%. Primary + metric: the existing checkout-completed metric. Randomization unit: user. + max_turns: 12 + mock_ask_question_answers: + - ["no"] + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const lookedUp = traj.some(t => (t.tool === 'get-flag' || t.tool === 'get-feature-flag') && ((t.arguments || {}).flagKey === 'missing-checkout' || (t.arguments || {}).key === 'missing-checkout')); + const listed = traj.some(t => (t.tool === 'list-flags' || t.tool === 'list-feature-flags') && String((t.arguments || {}).query || '').toLowerCase() === 'missing-checkout'); + const asked = traj.some(t => String(t.tool).includes('ask-question')); + const creates = traj.filter(t => t.tool === 'create-flag' || t.tool === 'create-feature-flag'); + const onlyAfterYes = creates.every(call => { + const idx = traj.indexOf(call); + return traj.slice(0, idx).some(t => { + if (!String(t.tool).includes('ask-question')) return false; + let selected = []; + try { selected = JSON.parse(t.mock_response_preview || '{}').selected || []; } catch (e) {} + return selected.map(String).some(id => id === 'yes'); + }); + }); + const pass = lookedUp && listed && asked && onlyAfterYes && creates.length === 0; + return { pass, score: pass ? 1 : 0, reason: 'get-flag=' + lookedUp + ' list-flags=' + listed + ' asked=' + asked + ' creates=' + creates.length }; + metric: create_flag_only_after_yes + weight: 3 + - type: llm-rubric + value: | + The flag missing-checkout does not exist. Score 1.0 if one message offers + three choices: correct the key (mentioning similar flags such as + checkout-flow or new-checkout), create the flag themselves and then + continue, or have the agent create it. For that last choice the message + proposes the key, name, type, and variations and says the flag will be + created off. Score 0 if it calls create-flag or offers fewer than three + choices. + metric: offers_three_flag_choices + weight: 2 + + # ------------------------------------------------------------------ + # 8. A running experiment's allocation change starts a new iteration, + # so save-and-start-experiment-iteration waits for an explicit start. + # The user does not say to start. + # ------------------------------------------------------------------ + - description: "Running experiment: no save-and-start without a start" + vars: + user_request: > + The experiment "checkout-running" in project "my-project", environment + "production" is running. Change the treatment allocation to 30% control + and 70% treatment. Do not start a new iteration until I explicitly say + to start. I am not saying to start. + max_turns: 12 + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const sawRunning = traj.some(t => t.tool === 'get-experiment' && ((t.arguments || {}).experimentKey === 'checkout-running' || (t.arguments || {}).key === 'checkout-running')); + const saves = traj.filter(t => t.tool === 'save-and-start-experiment-iteration'); + const onlyAfterStart = saves.every(call => { + const idx = traj.indexOf(call); + return traj.slice(0, idx).some(t => { + if (!String(t.tool).includes('ask-question')) return false; + let selected = []; + try { selected = JSON.parse(t.mock_response_preview || '{}').selected || []; } catch (e) {} + const options = (t.arguments && t.arguments.options) || []; + return selected.some(id => { + const opt = options.find(o => o.id === id); + const label = opt ? String(opt.label) : String(id); + return /start/i.test(label) && !/\b(not|later|wait|don'?t|no)\b/i.test(label); + }); + }); + }); + const pass = sawRunning && onlyAfterStart && saves.length === 0; + return { pass, score: pass ? 1 : 0, reason: 'get-experiment=' + sawRunning + ' save-and-start=' + saves.length }; + metric: save_and_start_only_after_start + weight: 3 + - type: llm-rubric + value: | + The experiment is running and the user wants a new allocation but has + not said to start. Score 1.0 if the agent shows the allocation change + and waits instead of starting a new iteration. Score 0 if it calls + save-and-start-experiment-iteration or tells the user the new iteration + is already running. + metric: shows_change_and_waits + weight: 2 diff --git a/evals/mocks/tool-responses.json b/evals/mocks/tool-responses.json index b0669541..d5c3e8dd 100644 --- a/evals/mocks/tool-responses.json +++ b/evals/mocks/tool-responses.json @@ -590,5 +590,38 @@ { "_id": "treat-treatment", "name": "Treatment", "baseline": false, "allocationPercent": 50 } ] } + }, + "get-flag-status-across-envs": { + "flagKey": "{{flagKey}}", + "environments": [ + { "key": "test", "status": "active" }, + { "key": "production", "status": "active" } + ], + "summary": { + "active": ["test", "production"], + "inactive": [], + "new": [], + "launched": [] + } + }, + "search-contexts": { + "items": [ + { "kind": "user", "key": "user-1", "lastSeen": "2026-10-20T00:00:00.000Z" } + ], + "totalCount": 1 + }, + "list-metric-events": { + "events": [ + { "key": "checkout-completed", "lastSeen": "2026-10-20T00:00:00.000Z" }, + { "key": "page-load-ms", "lastSeen": "2026-10-20T00:00:00.000Z" }, + { "key": "cta-click", "lastSeen": "2026-10-18T00:00:00.000Z" }, + { "key": "cta-clicked-v2", "lastSeen": "2026-10-18T00:00:00.000Z" } + ], + "totalCount": 4 + }, + "save-and-start-experiment-iteration": { + "key": "{{experimentKey}}", + "environmentKey": "{{environmentKey}}", + "currentIteration": { "status": "running" } } } diff --git a/evals/package.json b/evals/package.json index 5b16f174..b33d3d4a 100644 --- a/evals/package.json +++ b/evals/package.json @@ -27,6 +27,8 @@ "eval:onboarding:single": "promptfoo eval -c shared/defaults.yaml -c onboarding/promptfooconfig.yaml --env-file .env --no-cache --filter-first-n 1", "eval:experiment-guide": "promptfoo eval -c shared/defaults.yaml -c experiment-guide/promptfooconfig.yaml --env-file .env --no-cache -o experiment-guide/results.json", "eval:experiment-guide:single": "promptfoo eval -c shared/defaults.yaml -c experiment-guide/promptfooconfig.yaml --env-file .env --no-cache --filter-first-n 1", + "eval:experiment-setup": "promptfoo eval -c shared/defaults.yaml -c experiment-setup/promptfooconfig.yaml --env-file .env --no-cache -o experiment-setup/results.json", + "eval:experiment-setup:single": "promptfoo eval -c shared/defaults.yaml -c experiment-setup/promptfooconfig.yaml --env-file .env --no-cache --filter-first-n 1", "fixture:from-pr": "node scripts/build-pr-fixture.js", "eval:all": "node scripts/aggregate.js --run", "eval:aggregate": "node scripts/aggregate.js", diff --git a/evals/providers/_mock.js b/evals/providers/_mock.js index 831b70b1..cb50c0e1 100644 --- a/evals/providers/_mock.js +++ b/evals/providers/_mock.js @@ -129,6 +129,147 @@ function createMockState() { }; } +// Lifecycle status for get-flag-status-across-envs. The default matches the +// environments in the get-environments mock and reports both as active, so +// flag-drift sees no cross-environment difference and flag-command gets no +// cleanup signal. `federal` is intentionally absent: the flag-drift mock +// flips that environment's fallthrough, and listing it here would make +// single-environment drift cases look divergent. +function flagStatusResponse(flagKey, productionStatus) { + const environments = [ + { key: "test", status: "active" }, + { key: "production", status: productionStatus }, + ]; + const summary = { active: [], inactive: [], new: [], launched: [] }; + for (const env of environments) { + if (summary[env.status]) summary[env.status].push(env.key); + } + return { flagKey, environments, summary }; +} + +const DEFAULT_METRIC_EVENTS = [ + { key: "checkout-completed", lastSeen: "2026-10-20T00:00:00.000Z" }, + { key: "page-load-ms", lastSeen: "2026-10-20T00:00:00.000Z" }, + { key: "cta-click", lastSeen: "2026-10-18T00:00:00.000Z" }, + { key: "cta-clicked-v2", lastSeen: "2026-10-18T00:00:00.000Z" }, +]; + +// list-metric-events has no eventKey. A query of cta-clicked returns the +// similar keys only; every other call, including no query, returns the +// default list (which also contains those similar keys). +function metricEventsResponse(input) { + const query = typeof input.query === "string" ? input.query.trim() : ""; + const events = + query.toLowerCase() === "cta-clicked" + ? DEFAULT_METRIC_EVENTS.filter( + (event) => event.key === "cta-click" || event.key === "cta-clicked-v2", + ) + : DEFAULT_METRIC_EVENTS; + return { events, totalCount: events.length }; +} + +function isUserKindQuery(input) { + if (input.kind === "user") return true; + return typeof input.filter === "string" && /kind\s+equals\s+"?user"?/i.test(input.filter); +} + +function isNotEqualsUser(input) { + return ( + typeof input.filter === "string" && + /kind\s+notEquals\s+"?user"?/i.test(input.filter) + ); +} + +function searchContextsResponse(input) { + const mixed = input.projectKey === "mixed-contexts"; + if (isNotEqualsUser(input)) { + if (mixed) { + return { + items: [ + { + kind: "organization", + key: "org-1", + lastSeen: "2026-10-20T00:00:00.000Z", + }, + ], + totalCount: 1, + }; + } + return { items: [], totalCount: 0 }; + } + if (isUserKindQuery(input) || (input.kind == null && input.filter == null)) { + return { + items: [{ kind: "user", key: "user-1", lastSeen: "2026-10-20T00:00:00.000Z" }], + totalCount: 1, + }; + } + return { items: [], totalCount: 0 }; +} + +function checkoutActiveFlag() { + return { + key: "checkout-active", + name: "Checkout active", + kind: "multivariate", + description: "Checkout experiment flag", + temporary: true, + tags: [], + variations: [ + { _id: "var-current", value: "current", name: "Current checkout" }, + { _id: "var-new", value: "new", name: "New checkout" }, + ], + defaults: { onVariation: 0, offVariation: 1 }, + environment: { + on: true, + version: 7, + rules: [], + targets: [], + fallthrough: { variation: 0 }, + offVariation: 1, + }, + }; +} + +function similarCheckoutFlags() { + const flag = (key, name, on) => ({ + key, + name, + kind: "boolean", + description: name, + temporary: true, + tags: ["checkout"], + on, + variations: [ + { value: true, name: "Enabled" }, + { value: false, name: "Disabled" }, + ], + }); + return { + flags: [ + flag("checkout-flow", "Checkout flow", true), + flag("new-checkout", "New checkout", false), + ], + totalCount: 2, + pageInfo: { limit: 20, offset: 0 }, + }; +} + +function runningExperiment(input) { + return { + key: "checkout-running", + name: "Checkout running", + environmentKey: input.environmentKey || input.env || "production", + currentIteration: { + status: "running", + hypothesis: "The new checkout will lift completion.", + treatments: [ + { _id: "treat-control", name: "Control", baseline: true, allocationPercent: 50 }, + { _id: "treat-treatment", name: "Treatment", baseline: false, allocationPercent: 50 }, + ], + }, + }; +} + /** * Render a mock response for `toolName` given `input` (raw tool arguments). * `state` is the per-invocation map from `createMockState()`. @@ -201,6 +342,17 @@ function renderMockResponse(template, input, toolName, state) { const key = input.flagKey || input.key; if (key && state.flags[key]) return state.flags[key]; + // Sentinel keys for the experiment-setup suite. Checked before the + // federal fallthrough hook so they do not inherit that override. + if (key === "missing-checkout") { + return { + error: "not_found", + status: 404, + message: "Flag 'missing-checkout' was not found in this project.", + }; + } + if (key === "checkout-active") return checkoutActiveFlag(); + // Cross-environment divergence hook: a `federal`-style environment serves // the opposite default (variation 0 / `true`) from every other environment // (which resolve to variation 1 / `false` via the static template). This @@ -374,6 +526,26 @@ function renderMockResponse(template, input, toolName, state) { return flag; } + if (toolName === "get-flag-status-across-envs") { + const flagKey = input.flagKey || "unknown-flag"; + const productionStatus = flagKey === "checkout-inactive" ? "inactive" : "active"; + return flagStatusResponse(flagKey, productionStatus); + } + + if (toolName === "search-contexts") return searchContextsResponse(input || {}); + + if (toolName === "list-metric-events") return metricEventsResponse(input || {}); + + if (toolName === "list-flags" || toolName === "list-feature-flags") { + const query = typeof input.query === "string" ? input.query.trim().toLowerCase() : ""; + if (query === "missing-checkout") return similarCheckoutFlags(); + } + + if (toolName === "get-experiment") { + const experimentKey = input.experimentKey || input.key; + if (experimentKey === "checkout-running") return runningExperiment(input); + } + // Default: stateless template render return walk(template, replacements); } diff --git a/evals/scripts/_manifest.js b/evals/scripts/_manifest.js index 18022763..4faa8c8a 100644 --- a/evals/scripts/_manifest.js +++ b/evals/scripts/_manifest.js @@ -85,6 +85,12 @@ const SUITES = [ skillDir: "skills/experiments/launchdarkly-experiment-guide", readme: "skills/experiments/launchdarkly-experiment-guide/README.md", }, + { + suite: "experiment-setup", + skillKey: "experiments/launchdarkly-experiment-setup", + skillDir: "skills/experiments/launchdarkly-experiment-setup", + readme: "skills/experiments/launchdarkly-experiment-setup/README.md", + }, ]; /** diff --git a/evals/tests/mock.test.js b/evals/tests/mock.test.js index 4a96127c..1d99beef 100644 --- a/evals/tests/mock.test.js +++ b/evals/tests/mock.test.js @@ -422,3 +422,165 @@ test("update-ai-config uses template substitution", () => { assert.equal(result.key, "bot"); assert.equal(result.name, "Bot"); }); + +// --------------------------------------------------------------------------- +// experiment-setup sentinels +// --------------------------------------------------------------------------- + +test("get-flag-status-across-envs defaults to active in test and production", () => { + const state = createMockState(); + const result = renderMockResponse(TEMPLATES["get-flag-status-across-envs"], { + projectKey: "payments", + flagKey: "billing-v2", + }, "get-flag-status-across-envs", state); + assert.deepEqual( + result.environments.map((env) => [env.key, env.status]), + [["test", "active"], ["production", "active"]], + ); + assert.deepEqual(result.summary.active, ["test", "production"]); + assert.deepEqual(result.summary.inactive, []); + assert.equal( + result.environments.some((env) => /federal/i.test(env.key)), + false, + ); +}); + +test("get-flag-status-across-envs reports checkout-inactive as inactive in production", () => { + const state = createMockState(); + const result = renderMockResponse(TEMPLATES["get-flag-status-across-envs"], { + projectKey: "mixed-contexts", + flagKey: "checkout-inactive", + }, "get-flag-status-across-envs", state); + const production = result.environments.find((env) => env.key === "production"); + assert.equal(production.status, "inactive"); + assert.deepEqual(result.summary.inactive, ["production"]); +}); + +test("get-flag returns checkout-active on, serving one variation", () => { + const state = createMockState(); + const result = renderMockResponse(TEMPLATES["get-flag"], { + projectKey: "only-user", + flagKey: "checkout-active", + environmentKey: "production", + }, "get-flag", state); + assert.equal(result.environment.on, true); + assert.equal(result.environment.version, 7); + assert.equal(result.environment.fallthrough.variation, 0); + assert.equal(result.environment.fallthrough.rollout, undefined); + assert.equal(result.variations[0].name, "Current checkout"); +}); + +test("get-flag returns not-found for missing-checkout", () => { + const state = createMockState(); + const result = renderMockResponse(TEMPLATES["get-flag"], { + projectKey: "my-project", + flagKey: "missing-checkout", + }, "get-flag", state); + assert.equal(result.status, 404); +}); + +test("get-flag returns a stored flag before the missing-checkout not-found branch", () => { + const state = createMockState(); + renderMockResponse(TEMPLATES["create-flag"], { + key: "missing-checkout", + name: "Missing", + }, "create-flag", state); + const result = renderMockResponse(TEMPLATES["get-flag"], { + flagKey: "missing-checkout", + }, "get-flag", state); + assert.equal(result.key, "missing-checkout"); + assert.equal(result.error, undefined); +}); + +test("list-flags query missing-checkout returns similar keys only", () => { + const state = createMockState(); + const result = renderMockResponse(TEMPLATES["list-flags"], { + projectKey: "my-project", + query: "missing-checkout", + }, "list-flags", state); + const keys = result.flags.map((flag) => flag.key); + assert.deepEqual(keys, ["checkout-flow", "new-checkout"]); +}); + +test("list-flags without that query keeps the shared flag list", () => { + const state = createMockState(); + const result = renderMockResponse(TEMPLATES["list-flags"], { + projectKey: "web-app", + query: "search-v2", + }, "list-flags", state); + assert.deepEqual( + result.flags.map((flag) => flag.key), + ["enable-notifications", "search-v2"], + ); +}); + +test("search-contexts suggests user only when no other kind is stored", () => { + const state = createMockState(); + const users = renderMockResponse(TEMPLATES["search-contexts"], { + projectKey: "only-user", + environmentKey: "production", + kind: "user", + }, "search-contexts", state); + const others = renderMockResponse(TEMPLATES["search-contexts"], { + projectKey: "only-user", + environmentKey: "production", + filter: 'kind notEquals "user"', + }, "search-contexts", state); + assert.equal(users.totalCount, 1); + assert.equal(users.items[0].kind, "user"); + assert.equal(others.totalCount, 0); +}); + +test("search-contexts on mixed-contexts returns another kind", () => { + const state = createMockState(); + const others = renderMockResponse(TEMPLATES["search-contexts"], { + projectKey: "mixed-contexts", + environmentKey: "production", + filter: 'kind notEquals "user"', + }, "search-contexts", state); + assert.ok(others.totalCount > 0); + assert.equal(others.items[0].kind, "organization"); +}); + +test("list-metric-events default includes similar keys and omits cta-clicked", () => { + const state = createMockState(); + const result = renderMockResponse(TEMPLATES["list-metric-events"], { + projectKey: "my-project", + }, "list-metric-events", state); + const keys = result.events.map((event) => event.key); + assert.deepEqual(keys, [ + "checkout-completed", + "page-load-ms", + "cta-click", + "cta-clicked-v2", + ]); +}); + +test("list-metric-events query cta-clicked returns similar keys only", () => { + const state = createMockState(); + const result = renderMockResponse(TEMPLATES["list-metric-events"], { + projectKey: "my-project", + environmentKey: "production", + query: "cta-clicked", + }, "list-metric-events", state); + assert.deepEqual( + result.events.map((event) => event.key), + ["cta-click", "cta-clicked-v2"], + ); +}); + +test("get-experiment returns a running iteration for checkout-running", () => { + const state = createMockState(); + const running = renderMockResponse(TEMPLATES["get-experiment"], { + projectKey: "my-project", + environmentKey: "production", + experimentKey: "checkout-running", + }, "get-experiment", state); + const draft = renderMockResponse(TEMPLATES["get-experiment"], { + projectKey: "my-project", + environmentKey: "production", + experimentKey: "checkout-draft", + }, "get-experiment", state); + assert.equal(running.currentIteration.status, "running"); + assert.equal(draft.currentIteration.status, "not_started"); +}); diff --git a/evals/tools/definitions.json b/evals/tools/definitions.json index db990c83..5c7c53e6 100644 --- a/evals/tools/definitions.json +++ b/evals/tools/definitions.json @@ -685,5 +685,157 @@ }, "required": ["projectKey", "environmentKey", "experimentKey"] } + }, + { + "name": "get-flag-status-across-envs", + "description": "Check a flag's lifecycle status across all environments. Returns per-environment state (active, inactive, new, launched) and a summary of which environments the flag is active, inactive, or launched in. Use before cleanup to verify the flag is inactive everywhere.", + "input_schema": { + "type": "object", + "properties": { + "flagKey": { "type": "string" }, + "projectKey": { "type": "string" } + }, + "required": ["projectKey", "flagKey"] + } + }, + { + "name": "list-metric-events", + "description": "List recent event keys received by a LaunchDarkly project — used to check which events are actively flowing before creating a metric. Returns up to 50 event keys with last-seen timestamps. If the event key you need isn't here, it may not be instrumented yet. Use this BEFORE create-metric to: 1. Confirm the event key exists and is receiving data 2. Discover event keys when the user isn't sure what to use 3. Check the correct spelling of an event key Supports fuzzy search via `query` to narrow results. Filter by `environmentKey` to check a specific environment. If the desired event key is absent, surface a warning and ask whether the user wants to instrument the event first or proceed and wire it up later.", + "input_schema": { + "type": "object", + "properties": { + "query": { "type": "string" }, + "projectKey": { "type": "string" }, + "environmentKey": { "type": "string" } + }, + "required": ["projectKey"] + } + }, + { + "name": "search-contexts", + "description": "Search for stored LaunchDarkly contexts in an environment by kind, key, or attribute value. Returns context records including their stored attributes (e.g., merchantCountryCode, disbursementRail) and when the context was last seen. Convenience param: - `kind`: filter to a specific context kind (e.g. `Merchant`, `user`, `organization`). Emitted as the filter clause `kind equals \"\"`. Raw filter syntax (passed in `filter`) uses `field operator value` clauses. Operators: `equals`, `notEquals`, `anyOf`, `startsWith`, `exists`, `before`, `after`; `contains` and `notContains` are not supported. Combine clauses with `,` (AND) or `|` (OR); group with `()`. Supported fields: `kind`, `key`, `kindKey`, `name`, `applicationId`, `q` (fuzzy), and `.` for context attributes. Examples: - `kind equals \"Merchant\"` — contexts of a specific kind - `Merchant.merchantCountryCode equals \"US\"` — Merchant contexts whose attribute matches - `key startsWith \"merchant-\"` — contexts whose key has a prefix - `applicationId equals \"GoSDK/1.2\"` — contexts reported by a specific SDK/application Example combining both params: set `kind=Merchant` and `filter=Merchant.disbursementRail equals \"PIX\"` to find Merchant contexts on the PIX rail. Note: the contexts API cannot filter by which flag a context evaluated; there is no `flagEvaluated` field. Filter by kind, key, applicationId, or attribute values instead. Ordering is fixed by the API; there is no sort parameter. Pagination: responses include `totalCount` and `continuationToken`. Pass `continuationToken` from the previous response to get the next page.", + "input_schema": { + "type": "object", + "properties": { + "kind": { "type": "string" }, + "limit": { "type": "number" }, + "filter": { "type": "string" }, + "projectKey": { "type": "string" }, + "environmentKey": { "type": "string" }, + "continuationToken": { "type": "string" } + }, + "required": ["projectKey", "environmentKey"] + } + }, + { + "name": "save-and-start-experiment-iteration", + "description": "Stop the current running iteration, create a new draft iteration with the provided field updates applied, and start it — the API-recommended way to mutate treatments, metrics, methodology, or other fields that are locked while an iteration is running. Provide `changeJustification` to record why a new iteration is needed. Field inputs are the same as `update-experiment`; updates are checked against the new iteration's not_started mutability rules. Fields not allowed in not_started are returned under `skipped` rather than being sent.", + "input_schema": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "tags": { "type": "array", "items": { "type": "string" } }, + "flags": { + "type": "object", + "propertyNames": { "type": "string" }, + "additionalProperties": { + "type": "object", + "required": ["ruleId", "flagConfigVersion"], + "properties": { + "ruleId": { "type": "string" }, + "flagConfigVersion": { "type": "number" }, + "notInExperimentVariationId": { "type": "string" } + }, + "additionalProperties": false + } + }, + "metrics": { + "type": "array", + "items": { + "type": "object", + "required": ["key"], + "properties": { + "key": { "type": "string" }, + "isGroup": { "type": "boolean" } + }, + "additionalProperties": false + } + }, + "holdoutId": { "type": "string" }, + "attributes": { "type": "array", "items": { "type": "string" } }, + "dataSource": { "type": "string", "enum": ["launchdarkly", "snowflake", "databricks"] }, + "hypothesis": { "type": "string" }, + "projectKey": { "type": "string" }, + "treatments": { + "type": "array", + "items": { + "type": "object", + "required": ["name", "baseline", "allocationPercent", "parameters"], + "properties": { + "name": { "type": "string" }, + "baseline": { "type": "boolean" }, + "parameters": { + "type": "array", + "items": { + "type": "object", + "required": ["flagKey", "variationId"], + "properties": { + "flagKey": { "type": "string" }, + "variationId": { "type": "string" } + }, + "additionalProperties": false + } + }, + "allocationPercent": { "type": "number" } + }, + "additionalProperties": false + } + }, + "covariateId": { "type": "string" }, + "description": { "type": "string" }, + "methodology": { "type": "string", "enum": ["bayesian", "frequentist", "export_only"] }, + "flagSettings": { + "type": "object", + "propertyNames": { "type": "string" }, + "additionalProperties": { + "type": "object", + "required": ["ruleId", "flagConfigVersion"], + "properties": { + "ruleId": { "type": "string" }, + "flagConfigVersion": { "type": "number" }, + "notInExperimentVariationId": { "type": "string" } + }, + "additionalProperties": false + } + }, + "maintainerId": { "type": "string" }, + "experimentKey": { "type": "string" }, + "analysisConfig": { + "type": "object", + "properties": { + "testDirection": { "type": "string" }, + "bayesianThreshold": { "type": "string" }, + "significanceThreshold": { "type": "string" }, + "sequentialTestingEnabled": { "type": "boolean" }, + "multipleComparisonCorrectionScope": { + "type": "string", + "enum": ["variations", "variations-and-metrics", "metrics"] + }, + "multipleComparisonCorrectionMethod": { + "type": "string", + "enum": ["bonferroni", "benjamini-hochberg"] + } + }, + "additionalProperties": false + }, + "environmentKey": { "type": "string" }, + "primaryFunnelKey": { "type": "string" }, + "randomizationUnit": { "type": "string" }, + "canReshuffleTraffic": { "type": "boolean" }, + "changeJustification": { "type": "string" }, + "primarySingleMetricKey": { "type": "string" } + }, + "required": ["projectKey", "environmentKey", "experimentKey"] + } } ] From 158e1475dd5195fb073ad7ed5981a7ec412d5e7d Mon Sep 17 00:00:00 2001 From: Carmen Romero Date: Wed, 28 Oct 2026 13:49:27 -0700 Subject: [PATCH 2/3] fix(experiment-setup): score the design wait without a scripted approval Invoke the skill in the same turn as the request, and grade lookups and forbidden writes instead of ask-question answers that never arrive. Co-authored-by: Cursor --- evals/README.md | 1 + evals/experiment-setup/promptfooconfig.yaml | 347 +++++++++----------- evals/providers/_mock.js | 39 ++- evals/providers/claude-skill-agent-sdk.js | 46 ++- evals/tests/mock.test.js | 31 ++ evals/tests/provider-inline-skill.test.js | 59 ++++ 6 files changed, 320 insertions(+), 203 deletions(-) create mode 100644 evals/tests/provider-inline-skill.test.js diff --git a/evals/README.md b/evals/README.md index 1191f337..41d64c0f 100644 --- a/evals/README.md +++ b/evals/README.md @@ -356,6 +356,7 @@ Every `type: javascript` assertion must return `{ pass: boolean, score: number, | `allow_builtins` | `false` | When `true`, expose Claude Code's built-in tools (Read/Grep/Glob/Bash/Edit/Write). Use for skills that scan the codebase. | | `expose_mcp_tools` | `true` | When `false`, do not expose LaunchDarkly mock MCP tools. Use for routing/advisory skills. A test can also set `expose_mcp_tools` as a var to override this for itself, e.g. one "no LaunchDarkly access" case in a suite that otherwise has the mocks. | | `force_skill_invocation` | `false` | When `true`, set `initialPrompt` to `/` so the skill is invoked via slash command. Use for skills whose description-based activation is unreliable. | +| `inline_skill_invocation` | `false` | When `true`, send `/` and the user message as one turn, and do not set `initialPrompt`. | | `expose_ask_question` | `false` | When `true`, expose an `ask-question` MCP tool for testing blocking decision points. | ## Aggregated Quality Artifact (`eval-scores.json`) diff --git a/evals/experiment-setup/promptfooconfig.yaml b/evals/experiment-setup/promptfooconfig.yaml index ff7dad4f..a01bcee2 100644 --- a/evals/experiment-setup/promptfooconfig.yaml +++ b/evals/experiment-setup/promptfooconfig.yaml @@ -3,12 +3,12 @@ # Run with shared defaults: # promptfoo eval -c shared/defaults.yaml -c experiment-setup/promptfooconfig.yaml # -# Scores the approval gates in launchdarkly-experiment-setup: a complete design -# before create, suggestions versus questions for the control and randomization -# unit, the missing-event warning, and no flag change or iteration start until -# the user says so. Each case has one weight-3 tool-call check. Wording is a -# separate weight-2 rubric. -description: "Evaluation of the experiment-setup skill: design, approve, then create or start" +# Scores the approval gates in launchdarkly-experiment-setup. The user message +# is the whole turn: the agent shows a design or a question and waits. No case +# scripts an ask-question answer. Each case has one weight-3 tool-call check +# (lookups that should happen, and writes that must not) and a weight-1 rubric +# graded from the assistant's messages. The tool check decides pass or fail. +description: "Evaluation of the experiment-setup skill: look up, design, and wait" prompts: - file://../../skills/experiments/launchdarkly-experiment-setup/SKILL.md @@ -20,67 +20,59 @@ providers: skill_slug: launchdarkly-experiment-setup allow_builtins: false expose_mcp_tools: true - expose_ask_question: true - force_skill_invocation: true + # No case scripts an answer. An unscripted ask-question call returns the + # first option, which can approve a create, so leave the tool off. + expose_ask_question: false + # Send / and user_request as one turn. initialPrompt would + # deliver the request as a second turn and abort in-flight tool calls. + # The skills list only supplies the name and description. + force_skill_invocation: false + inline_skill_invocation: true tests: # ------------------------------------------------------------------ - # 1. Complete design. The user already named every required piece, - # including the control and randomization unit. "yes" creates the - # experiment and does not start it. methodology stays omitted. - # checkout-flow uses the shared get-flag template: version 1, and the - # fallthrough rule id is the string "fallthrough". + # 1. Every required piece is already named. Look up the flag and the + # metric, show one design, and wait. checkout-flow's get-flag template + # is version 1 and its fallthrough rule id is "fallthrough". # ------------------------------------------------------------------ - - description: "Complete design: yes creates without methodology, metric, or start" + - description: "Complete design: one design, then wait, with nothing created or started" vars: user_request: > Set up an experiment in project "my-project", environment "production". Flag: checkout-flow. Variations: true (id var-true) and false (id var-false). Control: the false variation. Hypothesis: the new checkout will lift completion rate by 3%. Primary metric: the existing checkout-completed - metric. Randomization unit: user. Show one design. When I approve it, - create the experiment. Do not start it. + metric. Randomization unit: user. max_turns: 16 - mock_ask_question_answers: - - ["yes"] assert: - type: javascript value: | const traj = output.trajectory || []; const tools = output.tools_called || []; - const createIdx = traj.findIndex(t => t.tool === 'create-experiment'); - if (createIdx === -1) return { pass: false, score: 0, reason: 'no create-experiment' }; - const args = traj[createIdx].arguments || {}; - const iteration = args.iteration || {}; - const methodologySent = args.methodology != null || iteration.methodology != null; - const flags = iteration.flags || {}; - const flagCfg = flags['checkout-flow'] || flags[Object.keys(flags)[0]]; - const flagOk = !!flagCfg && flagCfg.ruleId === 'fallthrough' && flagCfg.flagConfigVersion === 1; - const approved = traj.slice(0, createIdx).some(t => { - if (!String(t.tool).includes('ask-question')) return false; - let selected = []; - try { selected = JSON.parse(t.mock_response_preview || '{}').selected || []; } catch (e) {} - return selected.map(String).includes('yes'); - }); - const pass = approved && flagOk && !methodologySent && !tools.includes('create-metric') && !tools.includes('start-experiment-iteration'); - return { pass, score: pass ? 1 : 0, reason: 'approved=' + approved + ' flag=' + JSON.stringify(flagCfg) + ' methodology=' + methodologySent + ' tools=' + tools.join(' -> ') }; - metric: creates_without_methodology_or_start + const writes = ['create-metric','create-experiment','create-flag','start-experiment-iteration','save-and-start-experiment-iteration','toggle-flag','update-flag-settings','update-feature-flag','create-feature-flag']; + const called = writes.filter(name => tools.includes(name)); + const gotFlag = traj.some(t => t.tool === 'get-flag' && (t.arguments || {}).flagKey === 'checkout-flow'); + const listedMetrics = tools.includes('list-metrics'); + const pass = gotFlag && listedMetrics && called.length === 0; + return { pass, score: pass ? 1 : 0, reason: 'get-flag=' + gotFlag + ' list-metrics=' + listedMetrics + (called.length ? ' wrote ' + called.join(', ') : ' no writes') }; + metric: looks_up_flag_and_metric_and_does_not_write weight: 3 - type: llm-rubric value: | - Evaluate `assistant_turns` and any `ask-question` prompts in `trajectory`. - Score 1.0 if the agent shows one complete design before the user approves, - and that design names the flag, both variations, the false variation as - control, the hypothesis, checkout-completed, and randomization unit user. - Score 0 if it creates with no design, or shows a second conflicting design - before the approval. - metric: one_complete_design - weight: 2 + Read the assistant's messages. Score 1.0 if they show one complete + design and then wait for approval. That design names checkout-flow, + both variations, the false variation as control, the hypothesis, + checkout-completed, and randomization unit user. It includes ruleId + fallthrough and a flagConfigVersion. Statistics are the LaunchDarkly + default (frequentist, sequential testing). Score 0 if a required piece + is missing, statistics are Bayesian or another methodology, or the + message says the experiment was created or started. + metric: one_complete_design_then_waits + weight: 1 # ------------------------------------------------------------------ - # 2. Flag is on and active, and user is the only context kind. The - # skill suggests the control and the randomization unit instead of - # gathering them. + # 2. Flag is on and active, and user is the only context kind. Suggest + # the control and the randomization unit. Do not create. # ------------------------------------------------------------------ - description: "Suggests control and randomization unit when the flag is active and only user contexts exist" vars: @@ -89,44 +81,46 @@ tests: Flag: checkout-active. Compare its variations. Hypothesis: the new checkout will lift completion rate by 3%. Primary metric: the existing checkout-completed metric. I have not chosen a control variation or a - randomization unit. + randomization unit. Show the design and wait. Do not create anything yet. max_turns: 12 assert: - type: javascript value: | const traj = output.trajectory || []; const tools = output.tools_called || []; + const writes = ['create-metric','create-experiment','create-flag','start-experiment-iteration','save-and-start-experiment-iteration','toggle-flag','update-flag-settings','update-feature-flag','create-feature-flag']; + const called = writes.filter(name => tools.includes(name)); + const gotFlag = traj.some(t => t.tool === 'get-flag' && (t.arguments || {}).flagKey === 'checkout-active'); const searches = traj.filter(t => t.tool === 'search-contexts'); - const userCall = searches.some(t => { + const revealsOthers = searches.some(t => { const a = t.arguments || {}; - return a.kind === 'user' || /kind\s+equals\s+"?user"?/i.test(String(a.filter || '')); - }); - const otherCall = searches.some(t => /kind\s+notEquals\s+"?user"?/i.test(String((t.arguments || {}).filter || ''))); - const gathered = traj.some(t => { - if (!String(t.tool).includes('ask-question')) return false; - const text = JSON.stringify(t.arguments || {}).toLowerCase(); - if (text.includes('suggested')) return false; - return /control|baseline|randomization|context kind/.test(text); + const filter = String(a.filter || ''); + if (/kind\s+notEquals\s+"?user"?/i.test(filter)) return true; + if (a.kind === 'user' || /kind\s+equals\s+"?user"?/i.test(filter)) return false; + const kindBlank = a.kind == null || String(a.kind).trim() === ''; + const filterBlank = a.filter == null || filter.trim() === ''; + return kindBlank && filterBlank; }); - const pass = tools.includes('get-flag') && tools.includes('get-flag-status-across-envs') && userCall && otherCall && !gathered && !tools.includes('create-experiment') && !tools.includes('create-metric'); - return { pass, score: pass ? 1 : 0, reason: 'userCall=' + userCall + ' otherCall=' + otherCall + ' gathered=' + gathered + ' tools=' + tools.join(' -> ') }; - metric: suggests_from_flag_and_contexts + const pass = gotFlag && revealsOthers && called.length === 0; + return { pass, score: pass ? 1 : 0, reason: 'get-flag=' + gotFlag + ' revealsOthers=' + revealsOthers + (called.length ? ' wrote ' + called.join(', ') : ' no writes') }; + metric: looks_up_active_flag_and_contexts weight: 3 - type: llm-rubric value: | - The user did not name a control or a randomization unit. The flag is on - and active, and user is the only context kind. Score 1.0 if the design - labels both the served variation ("Current checkout") and randomization - unit "user" as suggested, each with a short reason, and there is no - gather message asking the user to supply either value. Deduct 0.5 if - either value is missing the suggested label. Score 0 if it asks the user - to choose them before showing a design. + Read the assistant's messages. The user did not name a control or a + randomization unit. Score 1.0 if the design labels both the served + variation ("Current checkout") and randomization unit "user" as + suggested, each with a short reason. Offering to confirm or change + those suggestions before approval is fine. Score 0 only if it asks + the user to supply the control or the randomization unit instead of + suggesting them, or omits the suggested label on either. metric: labels_suggestions - weight: 2 + weight: 1 # ------------------------------------------------------------------ # 3. Flag is off and inactive, and another context kind is stored. - # Ask for the control and the randomization unit. Do not suggest them. + # Ask for the control and the randomization unit. Do not suggest them + # and do not create. # ------------------------------------------------------------------ - description: "Asks for control and randomization unit when the flag is inactive and contexts are mixed" vars: @@ -135,30 +129,45 @@ tests: Flag: checkout-inactive. Compare its variations. Hypothesis: the new checkout will lift completion rate by 3%. Primary metric: the existing checkout-completed metric. I have not chosen a control variation or a - randomization unit. + randomization unit. Ask me for anything you still need. Do not create + anything yet. max_turns: 12 assert: - type: javascript value: | const traj = output.trajectory || []; const tools = output.tools_called || []; - const asks = traj.filter(t => String(t.tool).includes('ask-question')); - const pass = asks.length === 1 && !tools.includes('create-experiment') && !tools.includes('create-metric'); - return { pass, score: pass ? 1 : 0, reason: 'asks=' + asks.length + ' tools=' + tools.join(' -> ') }; - metric: asks_once_and_does_not_create + const writes = ['create-metric','create-experiment','create-flag','start-experiment-iteration','save-and-start-experiment-iteration','toggle-flag','update-flag-settings','update-feature-flag','create-feature-flag']; + const called = writes.filter(name => tools.includes(name)); + const gotFlag = traj.some(t => t.tool === 'get-flag' && (t.arguments || {}).flagKey === 'checkout-inactive'); + const gotStatus = traj.some(t => t.tool === 'get-flag-status-across-envs' && (t.arguments || {}).flagKey === 'checkout-inactive'); + const searches = traj.filter(t => t.tool === 'search-contexts'); + const revealsOthers = searches.some(t => { + const a = t.arguments || {}; + const filter = String(a.filter || ''); + if (/kind\s+notEquals\s+"?user"?/i.test(filter)) return true; + if (a.kind === 'user' || /kind\s+equals\s+"?user"?/i.test(filter)) return false; + const kindBlank = a.kind == null || String(a.kind).trim() === ''; + const filterBlank = a.filter == null || filter.trim() === ''; + return kindBlank && filterBlank; + }); + const pass = gotFlag && gotStatus && revealsOthers && called.length === 0; + return { pass, score: pass ? 1 : 0, reason: 'get-flag=' + gotFlag + ' status=' + gotStatus + ' revealsOthers=' + revealsOthers + (called.length ? ' wrote ' + called.join(', ') : ' no writes') }; + metric: looks_up_inactive_flag_and_mixed_contexts weight: 3 - type: llm-rubric value: | - The flag is off and inactive, and more than one context kind is stored. - Score 1.0 if the agent asks for both the control variation and the - randomization unit, leaves both blank, and does not label either as - suggested. Score 0 if it suggests either value or creates the experiment. + Read the assistant's messages. The flag is off and inactive, and more + than one context kind is stored. Score 1.0 if the agent asks for both + the control variation and the randomization unit, leaves both blank, + and does not label either as suggested. Score 0 if it suggests either + value or shows a design that fills either in. metric: asks_instead_of_suggesting - weight: 2 + weight: 1 # ------------------------------------------------------------------ # 4. Flag, hypothesis, and primary metric are all missing. One gather - # message, and nothing is created. + # message, and nothing is created. No lookup is required. # ------------------------------------------------------------------ - description: "Missing pieces: one gather message and no create" vars: @@ -169,28 +178,27 @@ tests: assert: - type: javascript value: | - const traj = output.trajectory || []; const tools = output.tools_called || []; - const asks = traj.filter(t => String(t.tool).includes('ask-question')); - const pass = asks.length === 1 && !tools.includes('create-metric') && !tools.includes('create-experiment'); - return { pass, score: pass ? 1 : 0, reason: 'asks=' + asks.length + ' tools=' + tools.join(' -> ') }; - metric: one_gather_and_no_create + const writes = ['create-metric','create-experiment','create-flag','start-experiment-iteration','save-and-start-experiment-iteration','toggle-flag','update-flag-settings','update-feature-flag','create-feature-flag']; + const called = writes.filter(name => tools.includes(name)); + const pass = called.length === 0; + return { pass, score: pass ? 1 : 0, reason: called.length ? 'wrote ' + called.join(', ') : 'no writes' }; + metric: does_not_write weight: 3 - type: llm-rubric value: | - Score 1.0 if a single message asks for the missing flag (and its - variations), the hypothesis, and the primary metric. Deduct 0.34 for - each of those three that the message leaves out, or if it splits them - across separate messages. Score 0 if it shows a design or creates anything. + Read the assistant's messages. Score 1.0 if a single message asks for + the missing flag and its variations, the hypothesis, and the primary + metric. Score 0 if any of those three is left out, they are split + across separate messages, or a design is shown. metric: gather_names_missing_pieces - weight: 2 + weight: 1 # ------------------------------------------------------------------ - # 5. The custom event has not arrived. The design warns, and "yes" - # still creates the metric and the experiment. list-metric-events - # takes `query`, not eventKey; any arguments count. + # 5. The custom event has not arrived. list-metric-events runs, the + # design warns and lists similar keys, and nothing is created. # ------------------------------------------------------------------ - - description: "Missing event: list-metric-events runs, then yes still creates" + - description: "Missing event: list-metric-events runs, the design warns, and nothing is created" vars: user_request: > Set up an experiment in project "my-project", environment "production". @@ -198,35 +206,32 @@ tests: Control: the false variation. Hypothesis: the new checkout will lift completion rate by 3%. Primary metric: a new custom metric "CTA clicked" (key cta-clicked) measuring the share of users who sent the cta-clicked - event. Randomization unit: user. Show the design. When I approve it, - create the metric and the experiment. Do not start it. + event. Randomization unit: user. max_turns: 16 - mock_ask_question_answers: - - ["yes"] assert: - type: javascript value: | - const traj = output.trajectory || []; - const eventsIdx = traj.findIndex(t => t.tool === 'list-metric-events'); - const metricIdx = traj.findIndex(t => t.tool === 'create-metric'); - const expIdx = traj.findIndex(t => t.tool === 'create-experiment'); - const pass = eventsIdx !== -1 && metricIdx !== -1 && expIdx !== -1 && eventsIdx < metricIdx && eventsIdx < expIdx; - return { pass, score: pass ? 1 : 0, reason: 'events@' + eventsIdx + ' metric@' + metricIdx + ' experiment@' + expIdx }; - metric: lists_events_before_create + const tools = output.tools_called || []; + const writes = ['create-metric','create-experiment','create-flag','start-experiment-iteration','save-and-start-experiment-iteration','toggle-flag','update-flag-settings','update-feature-flag','create-feature-flag']; + const called = writes.filter(name => tools.includes(name)); + const listedEvents = tools.includes('list-metric-events'); + const pass = listedEvents && called.length === 0; + return { pass, score: pass ? 1 : 0, reason: 'list-metric-events=' + listedEvents + (called.length ? ' wrote ' + called.join(', ') : ' no writes') }; + metric: lists_events_and_does_not_create weight: 3 - type: llm-rubric value: | - The cta-clicked event is not in the list-metric-events result. Similar - keys cta-click and cta-clicked-v2 are. Score 1.0 if the design warns - that the event has not arrived in the environment and lists those - similar keys. Score 0 if it blocks creation or omits the warning. + Read the assistant's messages. The cta-clicked event is not arriving. + Similar keys cta-click and cta-clicked-v2 are. Score 1.0 if the design + warns that the event has not arrived in the environment and lists those + similar keys. Score 0 if the warning is missing or neither similar key + is named. metric: warns_and_lists_similar_keys - weight: 2 + weight: 1 # ------------------------------------------------------------------ # 6. The user says to start. The flag is off and the event is still - # missing. One question covers both. No flag write and no start. - # The scripted answer is "no" so the first option cannot approve a toggle. + # missing. One message covers both. No flag write and no start. # ------------------------------------------------------------------ - description: "Start blocked: flag off and event missing, no flag change and no start" vars: @@ -235,35 +240,34 @@ tests: environment "production". It uses flag checkout-flow and a custom metric whose event key is cta-clicked. Start it now. max_turns: 12 - mock_ask_question_answers: - - ["no"] assert: - type: javascript value: | const traj = output.trajectory || []; const tools = output.tools_called || []; - const asks = traj.filter(t => String(t.tool).includes('ask-question')); - const forbidden = ['toggle-flag','update-flag-settings','update-feature-flag','create-flag','create-feature-flag','start-experiment-iteration']; - const called = forbidden.filter(name => tools.includes(name)); - const pass = asks.length === 1 && called.length === 0; - return { pass, score: pass ? 1 : 0, reason: 'asks=' + asks.length + (called.length ? ' called ' + called.join(', ') : ' no flag change or start') }; - metric: one_question_and_no_flag_or_start + const writes = ['create-metric','create-experiment','create-flag','start-experiment-iteration','save-and-start-experiment-iteration','toggle-flag','update-flag-settings','update-feature-flag','create-feature-flag']; + const called = writes.filter(name => tools.includes(name)); + const gotFlag = traj.some(t => t.tool === 'get-flag' && (t.arguments || {}).flagKey === 'checkout-flow'); + const listedEvents = tools.includes('list-metric-events'); + const pass = gotFlag && listedEvents && called.length === 0; + return { pass, score: pass ? 1 : 0, reason: 'get-flag=' + gotFlag + ' list-metric-events=' + listedEvents + (called.length ? ' wrote ' + called.join(', ') : ' no writes') }; + metric: looks_up_flag_and_event_and_does_not_start weight: 3 - type: llm-rubric value: | - The flag is off and the cta-clicked event is not arriving. Score 1.0 if - one message covers both: the flag must be on before starting, and whether - to start anyway while the event is missing. Score 0 if it asks those as - separate messages, turns the flag on, or starts the iteration. + Read the assistant's messages. The flag is off and the cta-clicked + event is not arriving. Score 1.0 if one message covers both: the flag + must be on before starting, and whether to start while the event is + missing. Score 0 if those are separate messages, or the message says + the flag was turned on or the iteration was started. metric: one_message_covers_flag_and_event - weight: 2 + weight: 1 # ------------------------------------------------------------------ - # 7. The named flag does not exist. Offer three choices. create-flag - # runs only after an explicit yes to the proposed details. The scripted - # answer is "no". + # 7. The named flag does not exist. Look up the key and similar flags. + # Offer three choices. Do not create the flag. # ------------------------------------------------------------------ - - description: "Missing flag: three choices, and no is not a yes to create-flag" + - description: "Missing flag: three choices, and the flag is not created" vars: user_request: > Set up an experiment in project "my-project", environment "production". @@ -271,84 +275,59 @@ tests: Hypothesis: the new checkout will lift completion rate by 3%. Primary metric: the existing checkout-completed metric. Randomization unit: user. max_turns: 12 - mock_ask_question_answers: - - ["no"] assert: - type: javascript value: | const traj = output.trajectory || []; - const lookedUp = traj.some(t => (t.tool === 'get-flag' || t.tool === 'get-feature-flag') && ((t.arguments || {}).flagKey === 'missing-checkout' || (t.arguments || {}).key === 'missing-checkout')); - const listed = traj.some(t => (t.tool === 'list-flags' || t.tool === 'list-feature-flags') && String((t.arguments || {}).query || '').toLowerCase() === 'missing-checkout'); - const asked = traj.some(t => String(t.tool).includes('ask-question')); - const creates = traj.filter(t => t.tool === 'create-flag' || t.tool === 'create-feature-flag'); - const onlyAfterYes = creates.every(call => { - const idx = traj.indexOf(call); - return traj.slice(0, idx).some(t => { - if (!String(t.tool).includes('ask-question')) return false; - let selected = []; - try { selected = JSON.parse(t.mock_response_preview || '{}').selected || []; } catch (e) {} - return selected.map(String).some(id => id === 'yes'); - }); - }); - const pass = lookedUp && listed && asked && onlyAfterYes && creates.length === 0; - return { pass, score: pass ? 1 : 0, reason: 'get-flag=' + lookedUp + ' list-flags=' + listed + ' asked=' + asked + ' creates=' + creates.length }; - metric: create_flag_only_after_yes + const tools = output.tools_called || []; + const writes = ['create-metric','create-experiment','create-flag','start-experiment-iteration','save-and-start-experiment-iteration','toggle-flag','update-flag-settings','update-feature-flag','create-feature-flag']; + const called = writes.filter(name => tools.includes(name)); + const lookedUp = traj.some(t => t.tool === 'get-flag' && (t.arguments || {}).flagKey === 'missing-checkout'); + const listed = traj.some(t => t.tool === 'list-flags'); + const pass = lookedUp && listed && called.length === 0; + return { pass, score: pass ? 1 : 0, reason: 'get-flag=' + lookedUp + ' list-flags=' + listed + (called.length ? ' wrote ' + called.join(', ') : ' no writes') }; + metric: looks_up_missing_flag_and_does_not_create weight: 3 - type: llm-rubric value: | - The flag missing-checkout does not exist. Score 1.0 if one message offers - three choices: correct the key (mentioning similar flags such as - checkout-flow or new-checkout), create the flag themselves and then - continue, or have the agent create it. For that last choice the message - proposes the key, name, type, and variations and says the flag will be - created off. Score 0 if it calls create-flag or offers fewer than three - choices. + Read the assistant's messages. The flag missing-checkout does not exist. + Score 1.0 if one message offers three choices: correct the key + (mentioning a similar flag such as checkout-flow or new-checkout), + create the flag themselves and then continue, or have the agent create + it. For that last choice the message proposes the key, name, type, and + variations and says the flag will be created off. Score 0 if it offers + fewer than three choices. metric: offers_three_flag_choices - weight: 2 + weight: 1 # ------------------------------------------------------------------ - # 8. A running experiment's allocation change starts a new iteration, - # so save-and-start-experiment-iteration waits for an explicit start. - # The user does not say to start. + # 8. A running experiment's allocation change would start a new + # iteration. Show the change and wait. The user has not said to start. # ------------------------------------------------------------------ - - description: "Running experiment: no save-and-start without a start" + - description: "Running experiment: show the allocation change and do not start" vars: user_request: > The experiment "checkout-running" in project "my-project", environment "production" is running. Change the treatment allocation to 30% control - and 70% treatment. Do not start a new iteration until I explicitly say - to start. I am not saying to start. + and 70% treatment. max_turns: 12 assert: - type: javascript value: | const traj = output.trajectory || []; + const tools = output.tools_called || []; + const writes = ['create-metric','create-experiment','create-flag','start-experiment-iteration','save-and-start-experiment-iteration','toggle-flag','update-flag-settings','update-feature-flag','create-feature-flag']; + const called = writes.filter(name => tools.includes(name)); const sawRunning = traj.some(t => t.tool === 'get-experiment' && ((t.arguments || {}).experimentKey === 'checkout-running' || (t.arguments || {}).key === 'checkout-running')); - const saves = traj.filter(t => t.tool === 'save-and-start-experiment-iteration'); - const onlyAfterStart = saves.every(call => { - const idx = traj.indexOf(call); - return traj.slice(0, idx).some(t => { - if (!String(t.tool).includes('ask-question')) return false; - let selected = []; - try { selected = JSON.parse(t.mock_response_preview || '{}').selected || []; } catch (e) {} - const options = (t.arguments && t.arguments.options) || []; - return selected.some(id => { - const opt = options.find(o => o.id === id); - const label = opt ? String(opt.label) : String(id); - return /start/i.test(label) && !/\b(not|later|wait|don'?t|no)\b/i.test(label); - }); - }); - }); - const pass = sawRunning && onlyAfterStart && saves.length === 0; - return { pass, score: pass ? 1 : 0, reason: 'get-experiment=' + sawRunning + ' save-and-start=' + saves.length }; - metric: save_and_start_only_after_start + const pass = sawRunning && called.length === 0; + return { pass, score: pass ? 1 : 0, reason: 'get-experiment=' + sawRunning + (called.length ? ' wrote ' + called.join(', ') : ' no writes') }; + metric: reads_running_experiment_and_does_not_start weight: 3 - type: llm-rubric value: | - The experiment is running and the user wants a new allocation but has - not said to start. Score 1.0 if the agent shows the allocation change - and waits instead of starting a new iteration. Score 0 if it calls - save-and-start-experiment-iteration or tells the user the new iteration - is already running. + Read the assistant's messages. The experiment is running and the user + wants a 30/70 allocation but has not said to start. Score 1.0 if the + agent shows that allocation change and waits. Score 0 if it says a new + iteration is already running. metric: shows_change_and_waits - weight: 2 + weight: 1 diff --git a/evals/providers/_mock.js b/evals/providers/_mock.js index cb50c0e1..9f1401ef 100644 --- a/evals/providers/_mock.js +++ b/evals/providers/_mock.js @@ -180,28 +180,33 @@ function isNotEqualsUser(input) { ); } +function contextItem(kind, key) { + return { kind, key, lastSeen: "2026-10-20T00:00:00.000Z" }; +} + +function isUnfilteredContextSearch(input) { + const kindBlank = input.kind == null || String(input.kind).trim() === ""; + const filterBlank = input.filter == null || String(input.filter).trim() === ""; + return kindBlank && filterBlank; +} + function searchContextsResponse(input) { const mixed = input.projectKey === "mixed-contexts"; if (isNotEqualsUser(input)) { if (mixed) { - return { - items: [ - { - kind: "organization", - key: "org-1", - lastSeen: "2026-10-20T00:00:00.000Z", - }, - ], - totalCount: 1, - }; + return { items: [contextItem("organization", "org-1")], totalCount: 1 }; } return { items: [], totalCount: 0 }; } - if (isUserKindQuery(input) || (input.kind == null && input.filter == null)) { - return { - items: [{ kind: "user", key: "user-1", lastSeen: "2026-10-20T00:00:00.000Z" }], - totalCount: 1, - }; + if (isUserKindQuery(input)) { + return { items: [contextItem("user", "user-1")], totalCount: 1 }; + } + // No kind and no filter lists every stored kind: user in both projects, + // and organization as well when other kinds are stored. + if (isUnfilteredContextSearch(input)) { + const items = [contextItem("user", "user-1")]; + if (mixed) items.push(contextItem("organization", "org-1")); + return { items, totalCount: items.length }; } return { items: [], totalCount: 0 }; } @@ -538,7 +543,9 @@ function renderMockResponse(template, input, toolName, state) { if (toolName === "list-flags" || toolName === "list-feature-flags") { const query = typeof input.query === "string" ? input.query.trim().toLowerCase() : ""; - if (query === "missing-checkout") return similarCheckoutFlags(); + // A search for a checkout flag returns the similar keys and omits the + // missing one, whatever the exact query string is. + if (query.includes("checkout")) return similarCheckoutFlags(); } if (toolName === "get-experiment") { diff --git a/evals/providers/claude-skill-agent-sdk.js b/evals/providers/claude-skill-agent-sdk.js index 3adabd65..6a824b48 100644 --- a/evals/providers/claude-skill-agent-sdk.js +++ b/evals/providers/claude-skill-agent-sdk.js @@ -27,6 +27,12 @@ * `/` to explicitly invoke the skill via * slash command. Use for routing/advisory skills whose * description-based activation is unreliable in eval. + * That slash command is its own first turn; the + * request arrives as a second turn. + * inline_skill_invocation - Default false. When true, send `/` + * and the user message as one turn, and do not set + * initialPrompt. Use when the skill must be invoked + * without a second message aborting tool calls. * expose_ask_question - Default false. When true, expose a second in-process * MCP server with an `ask-question` tool that records * calls into the trajectory and returns a canned answer. @@ -179,6 +185,31 @@ function loadSdk() { return sdkPromise; } +/** + * How the skill is invoked for one call. + * Inline sends `/` and the request as the single query prompt and leaves + * initialPrompt unset, even if force_skill_invocation is also set. + * Otherwise the prompt is the request alone, and force_skill_invocation sets + * initialPrompt to `/` as its own first turn. + */ +function resolveSkillDelivery({ + skillSlug, + userMessage, + forceSkillInvocation, + inlineSkillInvocation, +}) { + if (inlineSkillInvocation) { + return { + prompt: `/${skillSlug}\n\n${userMessage}`, + initialPrompt: undefined, + }; + } + return { + prompt: userMessage, + initialPrompt: forceSkillInvocation ? `/${skillSlug}` : undefined, + }; +} + class ClaudeSkillAgentSdk { constructor(options = {}) { const config = options.config || {}; @@ -191,6 +222,7 @@ class ClaudeSkillAgentSdk { this.allowBuiltins = Boolean(config.allow_builtins); this.exposeMcpTools = config.expose_mcp_tools !== false; this.forceSkillInvocation = Boolean(config.force_skill_invocation); + this.inlineSkillInvocation = Boolean(config.inline_skill_invocation); this.exposeAskQuestion = Boolean(config.expose_ask_question); const source = resolveSkillSource(this.skillSlug); @@ -242,6 +274,13 @@ class ClaudeSkillAgentSdk { userMessage += `\n\n\n${gitDiff}\n`; } + const delivery = resolveSkillDelivery({ + skillSlug: this.skillSlug, + userMessage, + forceSkillInvocation: this.forceSkillInvocation, + inlineSkillInvocation: this.inlineSkillInvocation, + }); + const { cwd, isolatedConfig } = createInvocationCwd( this.skillSlug, this.skillSource, @@ -390,8 +429,8 @@ class ClaudeSkillAgentSdk { description: "Evaluation harness agent for LaunchDarkly skills.", prompt: harnessPrompt, skills: [this.skillSlug], - ...(this.forceSkillInvocation - ? { initialPrompt: `/${this.skillSlug}` } + ...(delivery.initialPrompt + ? { initialPrompt: delivery.initialPrompt } : {}), ...(agentTools ? { tools: agentTools } : {}), }, @@ -447,7 +486,7 @@ class ClaudeSkillAgentSdk { }; try { - const q = query({ prompt: userMessage, options: queryOptions }); + const q = query({ prompt: delivery.prompt, options: queryOptions }); for await (const msg of q) { if (debug) allMessages.push(msg); if (msg.type === "assistant") { @@ -587,3 +626,4 @@ module.exports = ClaudeSkillAgentSdk; // Exposed for unit tests; the promptfoo loader uses the class default export. module.exports.stripRawWrapper = stripRawWrapper; module.exports.resolveExposeMcpTools = resolveExposeMcpTools; +module.exports.resolveSkillDelivery = resolveSkillDelivery; diff --git a/evals/tests/mock.test.js b/evals/tests/mock.test.js index 1d99beef..aff4dc21 100644 --- a/evals/tests/mock.test.js +++ b/evals/tests/mock.test.js @@ -502,6 +502,17 @@ test("list-flags query missing-checkout returns similar keys only", () => { assert.deepEqual(keys, ["checkout-flow", "new-checkout"]); }); +test("list-flags query containing checkout returns similar keys and omits missing-checkout", () => { + const state = createMockState(); + const result = renderMockResponse(TEMPLATES["list-flags"], { + projectKey: "my-project", + query: "Checkout", + }, "list-flags", state); + const keys = result.flags.map((flag) => flag.key); + assert.deepEqual(keys, ["checkout-flow", "new-checkout"]); + assert.equal(keys.includes("missing-checkout"), false); +}); + test("list-flags without that query keeps the shared flag list", () => { const state = createMockState(); const result = renderMockResponse(TEMPLATES["list-flags"], { @@ -531,6 +542,26 @@ test("search-contexts suggests user only when no other kind is stored", () => { assert.equal(others.totalCount, 0); }); +test("search-contexts with no kind filter lists every stored kind", () => { + const state = createMockState(); + const onlyUser = renderMockResponse(TEMPLATES["search-contexts"], { + projectKey: "only-user", + environmentKey: "production", + }, "search-contexts", state); + assert.deepEqual(onlyUser.items.map((item) => item.kind), ["user"]); + assert.equal(onlyUser.totalCount, 1); + + const mixed = renderMockResponse(TEMPLATES["search-contexts"], { + projectKey: "mixed-contexts", + environmentKey: "production", + }, "search-contexts", state); + assert.deepEqual( + mixed.items.map((item) => item.kind).sort(), + ["organization", "user"], + ); + assert.equal(mixed.totalCount, 2); +}); + test("search-contexts on mixed-contexts returns another kind", () => { const state = createMockState(); const others = renderMockResponse(TEMPLATES["search-contexts"], { diff --git a/evals/tests/provider-inline-skill.test.js b/evals/tests/provider-inline-skill.test.js new file mode 100644 index 00000000..759d71c4 --- /dev/null +++ b/evals/tests/provider-inline-skill.test.js @@ -0,0 +1,59 @@ +const { test } = require("node:test"); +const assert = require("node:assert/strict"); + +// inline_skill_invocation puts the slash command and the request in one +// message. force_skill_invocation still uses initialPrompt as its own turn. +const { resolveSkillDelivery } = require("../providers/claude-skill-agent-sdk"); + +const slug = "launchdarkly-experiment-setup"; +const request = 'Set up an experiment on flag "checkout-flow".'; + +test("inline invocation prepends the slash command and leaves initialPrompt unset", () => { + const delivery = resolveSkillDelivery({ + skillSlug: slug, + userMessage: request, + forceSkillInvocation: false, + inlineSkillInvocation: true, + }); + + assert.equal(delivery.prompt, `/${slug}\n\n${request}`); + assert.equal(delivery.prompt.startsWith(`/${slug}\n\n`), true); + assert.equal(delivery.prompt.endsWith(request), true); + assert.equal(delivery.initialPrompt, undefined); +}); + +test("inline invocation wins when force_skill_invocation is also set", () => { + const delivery = resolveSkillDelivery({ + skillSlug: slug, + userMessage: request, + forceSkillInvocation: true, + inlineSkillInvocation: true, + }); + + assert.equal(delivery.prompt, `/${slug}\n\n${request}`); + assert.equal(delivery.initialPrompt, undefined); +}); + +test("with inline off, force_skill_invocation still sets initialPrompt and leaves the request unchanged", () => { + const delivery = resolveSkillDelivery({ + skillSlug: slug, + userMessage: request, + forceSkillInvocation: true, + inlineSkillInvocation: false, + }); + + assert.equal(delivery.prompt, request); + assert.equal(delivery.initialPrompt, `/${slug}`); +}); + +test("with both off, the request is sent alone and initialPrompt is unset", () => { + const delivery = resolveSkillDelivery({ + skillSlug: slug, + userMessage: request, + forceSkillInvocation: false, + inlineSkillInvocation: false, + }); + + assert.equal(delivery.prompt, request); + assert.equal(delivery.initialPrompt, undefined); +}); From b28a9110a269f7842557d03d7be481b5e8fc7e95 Mon Sep 17 00:00:00 2001 From: Carmen Romero Date: Fri, 30 Oct 2026 12:24:14 -0700 Subject: [PATCH 3/3] refactor(experiment-setup): keep suite-only tools out of the shared catalog The four tools this suite added, and their default templates, now load only for this suite. Sentinel responses stay in the shared mock renderer. Co-authored-by: Cursor --- evals/experiment-setup/mocks.json | 62 ++++ evals/experiment-setup/promptfooconfig.yaml | 4 + evals/experiment-setup/tools.json | 301 ++++++++++++++++++++ evals/mocks/tool-responses.json | 33 --- evals/tests/mock.test.js | 49 +++- evals/tools/definitions.json | 152 ---------- 6 files changed, 407 insertions(+), 194 deletions(-) create mode 100644 evals/experiment-setup/mocks.json create mode 100644 evals/experiment-setup/tools.json diff --git a/evals/experiment-setup/mocks.json b/evals/experiment-setup/mocks.json new file mode 100644 index 00000000..e002be5d --- /dev/null +++ b/evals/experiment-setup/mocks.json @@ -0,0 +1,62 @@ +{ + "get-flag-status-across-envs": { + "flagKey": "{{flagKey}}", + "environments": [ + { + "key": "test", + "status": "active" + }, + { + "key": "production", + "status": "active" + } + ], + "summary": { + "active": [ + "test", + "production" + ], + "inactive": [], + "new": [], + "launched": [] + } + }, + "list-metric-events": { + "events": [ + { + "key": "checkout-completed", + "lastSeen": "2026-10-20T00:00:00.000Z" + }, + { + "key": "page-load-ms", + "lastSeen": "2026-10-20T00:00:00.000Z" + }, + { + "key": "cta-click", + "lastSeen": "2026-10-18T00:00:00.000Z" + }, + { + "key": "cta-clicked-v2", + "lastSeen": "2026-10-18T00:00:00.000Z" + } + ], + "totalCount": 4 + }, + "search-contexts": { + "items": [ + { + "kind": "user", + "key": "user-1", + "lastSeen": "2026-10-20T00:00:00.000Z" + } + ], + "totalCount": 1 + }, + "save-and-start-experiment-iteration": { + "key": "{{experimentKey}}", + "environmentKey": "{{environmentKey}}", + "currentIteration": { + "status": "running" + } + } +} diff --git a/evals/experiment-setup/promptfooconfig.yaml b/evals/experiment-setup/promptfooconfig.yaml index 55328781..4bbc627c 100644 --- a/evals/experiment-setup/promptfooconfig.yaml +++ b/evals/experiment-setup/promptfooconfig.yaml @@ -28,6 +28,10 @@ providers: # skills list only supplies the name and description. force_skill_invocation: false inline_skill: true + # Schemas and default templates for the tools only this suite needs. + # Sentinel responses still come from the shared mock renderer. + tool_definitions: experiment-setup/tools.json + tool_mocks: experiment-setup/mocks.json tests: # ------------------------------------------------------------------ diff --git a/evals/experiment-setup/tools.json b/evals/experiment-setup/tools.json new file mode 100644 index 00000000..aa765987 --- /dev/null +++ b/evals/experiment-setup/tools.json @@ -0,0 +1,301 @@ +[ + { + "name": "get-flag-status-across-envs", + "description": "Check a flag's lifecycle status across all environments. Returns per-environment state (active, inactive, new, launched) and a summary of which environments the flag is active, inactive, or launched in. Use before cleanup to verify the flag is inactive everywhere.", + "input_schema": { + "type": "object", + "properties": { + "flagKey": { + "type": "string" + }, + "projectKey": { + "type": "string" + } + }, + "required": [ + "projectKey", + "flagKey" + ] + } + }, + { + "name": "list-metric-events", + "description": "List recent event keys received by a LaunchDarkly project \u2014 used to check which events are actively flowing before creating a metric. Returns up to 50 event keys with last-seen timestamps. If the event key you need isn't here, it may not be instrumented yet. Use this BEFORE create-metric to: 1. Confirm the event key exists and is receiving data 2. Discover event keys when the user isn't sure what to use 3. Check the correct spelling of an event key Supports fuzzy search via `query` to narrow results. Filter by `environmentKey` to check a specific environment. If the desired event key is absent, surface a warning and ask whether the user wants to instrument the event first or proceed and wire it up later.", + "input_schema": { + "type": "object", + "properties": { + "query": { + "type": "string" + }, + "projectKey": { + "type": "string" + }, + "environmentKey": { + "type": "string" + } + }, + "required": [ + "projectKey" + ] + } + }, + { + "name": "search-contexts", + "description": "Search for stored LaunchDarkly contexts in an environment by kind, key, or attribute value. Returns context records including their stored attributes (e.g., merchantCountryCode, disbursementRail) and when the context was last seen. Convenience param: - `kind`: filter to a specific context kind (e.g. `Merchant`, `user`, `organization`). Emitted as the filter clause `kind equals \"\"`. Raw filter syntax (passed in `filter`) uses `field operator value` clauses. Operators: `equals`, `notEquals`, `anyOf`, `startsWith`, `exists`, `before`, `after`; `contains` and `notContains` are not supported. Combine clauses with `,` (AND) or `|` (OR); group with `()`. Supported fields: `kind`, `key`, `kindKey`, `name`, `applicationId`, `q` (fuzzy), and `.` for context attributes. Examples: - `kind equals \"Merchant\"` \u2014 contexts of a specific kind - `Merchant.merchantCountryCode equals \"US\"` \u2014 Merchant contexts whose attribute matches - `key startsWith \"merchant-\"` \u2014 contexts whose key has a prefix - `applicationId equals \"GoSDK/1.2\"` \u2014 contexts reported by a specific SDK/application Example combining both params: set `kind=Merchant` and `filter=Merchant.disbursementRail equals \"PIX\"` to find Merchant contexts on the PIX rail. Note: the contexts API cannot filter by which flag a context evaluated; there is no `flagEvaluated` field. Filter by kind, key, applicationId, or attribute values instead. Ordering is fixed by the API; there is no sort parameter. Pagination: responses include `totalCount` and `continuationToken`. Pass `continuationToken` from the previous response to get the next page.", + "input_schema": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "limit": { + "type": "number" + }, + "filter": { + "type": "string" + }, + "projectKey": { + "type": "string" + }, + "environmentKey": { + "type": "string" + }, + "continuationToken": { + "type": "string" + } + }, + "required": [ + "projectKey", + "environmentKey" + ] + } + }, + { + "name": "save-and-start-experiment-iteration", + "description": "Stop the current running iteration, create a new draft iteration with the provided field updates applied, and start it \u2014 the API-recommended way to mutate treatments, metrics, methodology, or other fields that are locked while an iteration is running. Provide `changeJustification` to record why a new iteration is needed. Field inputs are the same as `update-experiment`; updates are checked against the new iteration's not_started mutability rules. Fields not allowed in not_started are returned under `skipped` rather than being sent.", + "input_schema": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + }, + "flags": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "required": [ + "ruleId", + "flagConfigVersion" + ], + "properties": { + "ruleId": { + "type": "string" + }, + "flagConfigVersion": { + "type": "number" + }, + "notInExperimentVariationId": { + "type": "string" + } + }, + "additionalProperties": false + } + }, + "metrics": { + "type": "array", + "items": { + "type": "object", + "required": [ + "key" + ], + "properties": { + "key": { + "type": "string" + }, + "isGroup": { + "type": "boolean" + } + }, + "additionalProperties": false + } + }, + "holdoutId": { + "type": "string" + }, + "attributes": { + "type": "array", + "items": { + "type": "string" + } + }, + "dataSource": { + "type": "string", + "enum": [ + "launchdarkly", + "snowflake", + "databricks" + ] + }, + "hypothesis": { + "type": "string" + }, + "projectKey": { + "type": "string" + }, + "treatments": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name", + "baseline", + "allocationPercent", + "parameters" + ], + "properties": { + "name": { + "type": "string" + }, + "baseline": { + "type": "boolean" + }, + "parameters": { + "type": "array", + "items": { + "type": "object", + "required": [ + "flagKey", + "variationId" + ], + "properties": { + "flagKey": { + "type": "string" + }, + "variationId": { + "type": "string" + } + }, + "additionalProperties": false + } + }, + "allocationPercent": { + "type": "number" + } + }, + "additionalProperties": false + } + }, + "covariateId": { + "type": "string" + }, + "description": { + "type": "string" + }, + "methodology": { + "type": "string", + "enum": [ + "bayesian", + "frequentist", + "export_only" + ] + }, + "flagSettings": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "required": [ + "ruleId", + "flagConfigVersion" + ], + "properties": { + "ruleId": { + "type": "string" + }, + "flagConfigVersion": { + "type": "number" + }, + "notInExperimentVariationId": { + "type": "string" + } + }, + "additionalProperties": false + } + }, + "maintainerId": { + "type": "string" + }, + "experimentKey": { + "type": "string" + }, + "analysisConfig": { + "type": "object", + "properties": { + "testDirection": { + "type": "string" + }, + "bayesianThreshold": { + "type": "string" + }, + "significanceThreshold": { + "type": "string" + }, + "sequentialTestingEnabled": { + "type": "boolean" + }, + "multipleComparisonCorrectionScope": { + "type": "string", + "enum": [ + "variations", + "variations-and-metrics", + "metrics" + ] + }, + "multipleComparisonCorrectionMethod": { + "type": "string", + "enum": [ + "bonferroni", + "benjamini-hochberg" + ] + } + }, + "additionalProperties": false + }, + "environmentKey": { + "type": "string" + }, + "primaryFunnelKey": { + "type": "string" + }, + "randomizationUnit": { + "type": "string" + }, + "canReshuffleTraffic": { + "type": "boolean" + }, + "changeJustification": { + "type": "string" + }, + "primarySingleMetricKey": { + "type": "string" + } + }, + "required": [ + "projectKey", + "environmentKey", + "experimentKey" + ] + } + } +] diff --git a/evals/mocks/tool-responses.json b/evals/mocks/tool-responses.json index d5c3e8dd..b0669541 100644 --- a/evals/mocks/tool-responses.json +++ b/evals/mocks/tool-responses.json @@ -590,38 +590,5 @@ { "_id": "treat-treatment", "name": "Treatment", "baseline": false, "allocationPercent": 50 } ] } - }, - "get-flag-status-across-envs": { - "flagKey": "{{flagKey}}", - "environments": [ - { "key": "test", "status": "active" }, - { "key": "production", "status": "active" } - ], - "summary": { - "active": ["test", "production"], - "inactive": [], - "new": [], - "launched": [] - } - }, - "search-contexts": { - "items": [ - { "kind": "user", "key": "user-1", "lastSeen": "2026-10-20T00:00:00.000Z" } - ], - "totalCount": 1 - }, - "list-metric-events": { - "events": [ - { "key": "checkout-completed", "lastSeen": "2026-10-20T00:00:00.000Z" }, - { "key": "page-load-ms", "lastSeen": "2026-10-20T00:00:00.000Z" }, - { "key": "cta-click", "lastSeen": "2026-10-18T00:00:00.000Z" }, - { "key": "cta-clicked-v2", "lastSeen": "2026-10-18T00:00:00.000Z" } - ], - "totalCount": 4 - }, - "save-and-start-experiment-iteration": { - "key": "{{experimentKey}}", - "environmentKey": "{{environmentKey}}", - "currentIteration": { "status": "running" } } } diff --git a/evals/tests/mock.test.js b/evals/tests/mock.test.js index aff4dc21..6eb907a3 100644 --- a/evals/tests/mock.test.js +++ b/evals/tests/mock.test.js @@ -4,6 +4,9 @@ const assert = require("node:assert/strict"); const { renderMockResponse, createMockState } = require("../providers/_mock"); const TEMPLATES = require("../mocks/tool-responses.json"); +// These four tools' default templates live with the experiment-setup suite. +// The sentinel branches in _mock.js still override them. +const SUITE_TEMPLATES = require("../experiment-setup/mocks.json"); // --------------------------------------------------------------------------- // createMockState @@ -427,9 +430,37 @@ test("update-ai-config uses template substitution", () => { // experiment-setup sentinels // --------------------------------------------------------------------------- +test("experiment-setup suite files own the four tools the shared catalog no longer has", () => { + const sharedDefs = require("../tools/definitions.json").map((def) => def.name); + const suiteDefs = require("../experiment-setup/tools.json").map((def) => def.name); + const owned = [ + "get-flag-status-across-envs", + "list-metric-events", + "search-contexts", + "save-and-start-experiment-iteration", + ]; + for (const name of owned) { + assert.equal(sharedDefs.includes(name), false); + assert.equal(TEMPLATES[name], undefined); + assert.equal(suiteDefs.includes(name), true); + assert.ok(SUITE_TEMPLATES[name]); + } +}); + +test("save-and-start returns the suite template when no sentinel matches", () => { + const state = createMockState(); + const result = renderMockResponse(SUITE_TEMPLATES["save-and-start-experiment-iteration"], { + projectKey: "my-project", + environmentKey: "production", + experimentKey: "checkout-running", + }, "save-and-start-experiment-iteration", state); + assert.equal(result.key, "checkout-running"); + assert.equal(result.currentIteration.status, "running"); +}); + test("get-flag-status-across-envs defaults to active in test and production", () => { const state = createMockState(); - const result = renderMockResponse(TEMPLATES["get-flag-status-across-envs"], { + const result = renderMockResponse(SUITE_TEMPLATES["get-flag-status-across-envs"], { projectKey: "payments", flagKey: "billing-v2", }, "get-flag-status-across-envs", state); @@ -447,7 +478,7 @@ test("get-flag-status-across-envs defaults to active in test and production", () test("get-flag-status-across-envs reports checkout-inactive as inactive in production", () => { const state = createMockState(); - const result = renderMockResponse(TEMPLATES["get-flag-status-across-envs"], { + const result = renderMockResponse(SUITE_TEMPLATES["get-flag-status-across-envs"], { projectKey: "mixed-contexts", flagKey: "checkout-inactive", }, "get-flag-status-across-envs", state); @@ -527,12 +558,12 @@ test("list-flags without that query keeps the shared flag list", () => { test("search-contexts suggests user only when no other kind is stored", () => { const state = createMockState(); - const users = renderMockResponse(TEMPLATES["search-contexts"], { + const users = renderMockResponse(SUITE_TEMPLATES["search-contexts"], { projectKey: "only-user", environmentKey: "production", kind: "user", }, "search-contexts", state); - const others = renderMockResponse(TEMPLATES["search-contexts"], { + const others = renderMockResponse(SUITE_TEMPLATES["search-contexts"], { projectKey: "only-user", environmentKey: "production", filter: 'kind notEquals "user"', @@ -544,14 +575,14 @@ test("search-contexts suggests user only when no other kind is stored", () => { test("search-contexts with no kind filter lists every stored kind", () => { const state = createMockState(); - const onlyUser = renderMockResponse(TEMPLATES["search-contexts"], { + const onlyUser = renderMockResponse(SUITE_TEMPLATES["search-contexts"], { projectKey: "only-user", environmentKey: "production", }, "search-contexts", state); assert.deepEqual(onlyUser.items.map((item) => item.kind), ["user"]); assert.equal(onlyUser.totalCount, 1); - const mixed = renderMockResponse(TEMPLATES["search-contexts"], { + const mixed = renderMockResponse(SUITE_TEMPLATES["search-contexts"], { projectKey: "mixed-contexts", environmentKey: "production", }, "search-contexts", state); @@ -564,7 +595,7 @@ test("search-contexts with no kind filter lists every stored kind", () => { test("search-contexts on mixed-contexts returns another kind", () => { const state = createMockState(); - const others = renderMockResponse(TEMPLATES["search-contexts"], { + const others = renderMockResponse(SUITE_TEMPLATES["search-contexts"], { projectKey: "mixed-contexts", environmentKey: "production", filter: 'kind notEquals "user"', @@ -575,7 +606,7 @@ test("search-contexts on mixed-contexts returns another kind", () => { test("list-metric-events default includes similar keys and omits cta-clicked", () => { const state = createMockState(); - const result = renderMockResponse(TEMPLATES["list-metric-events"], { + const result = renderMockResponse(SUITE_TEMPLATES["list-metric-events"], { projectKey: "my-project", }, "list-metric-events", state); const keys = result.events.map((event) => event.key); @@ -589,7 +620,7 @@ test("list-metric-events default includes similar keys and omits cta-clicked", ( test("list-metric-events query cta-clicked returns similar keys only", () => { const state = createMockState(); - const result = renderMockResponse(TEMPLATES["list-metric-events"], { + const result = renderMockResponse(SUITE_TEMPLATES["list-metric-events"], { projectKey: "my-project", environmentKey: "production", query: "cta-clicked", diff --git a/evals/tools/definitions.json b/evals/tools/definitions.json index 5c7c53e6..db990c83 100644 --- a/evals/tools/definitions.json +++ b/evals/tools/definitions.json @@ -685,157 +685,5 @@ }, "required": ["projectKey", "environmentKey", "experimentKey"] } - }, - { - "name": "get-flag-status-across-envs", - "description": "Check a flag's lifecycle status across all environments. Returns per-environment state (active, inactive, new, launched) and a summary of which environments the flag is active, inactive, or launched in. Use before cleanup to verify the flag is inactive everywhere.", - "input_schema": { - "type": "object", - "properties": { - "flagKey": { "type": "string" }, - "projectKey": { "type": "string" } - }, - "required": ["projectKey", "flagKey"] - } - }, - { - "name": "list-metric-events", - "description": "List recent event keys received by a LaunchDarkly project — used to check which events are actively flowing before creating a metric. Returns up to 50 event keys with last-seen timestamps. If the event key you need isn't here, it may not be instrumented yet. Use this BEFORE create-metric to: 1. Confirm the event key exists and is receiving data 2. Discover event keys when the user isn't sure what to use 3. Check the correct spelling of an event key Supports fuzzy search via `query` to narrow results. Filter by `environmentKey` to check a specific environment. If the desired event key is absent, surface a warning and ask whether the user wants to instrument the event first or proceed and wire it up later.", - "input_schema": { - "type": "object", - "properties": { - "query": { "type": "string" }, - "projectKey": { "type": "string" }, - "environmentKey": { "type": "string" } - }, - "required": ["projectKey"] - } - }, - { - "name": "search-contexts", - "description": "Search for stored LaunchDarkly contexts in an environment by kind, key, or attribute value. Returns context records including their stored attributes (e.g., merchantCountryCode, disbursementRail) and when the context was last seen. Convenience param: - `kind`: filter to a specific context kind (e.g. `Merchant`, `user`, `organization`). Emitted as the filter clause `kind equals \"\"`. Raw filter syntax (passed in `filter`) uses `field operator value` clauses. Operators: `equals`, `notEquals`, `anyOf`, `startsWith`, `exists`, `before`, `after`; `contains` and `notContains` are not supported. Combine clauses with `,` (AND) or `|` (OR); group with `()`. Supported fields: `kind`, `key`, `kindKey`, `name`, `applicationId`, `q` (fuzzy), and `.` for context attributes. Examples: - `kind equals \"Merchant\"` — contexts of a specific kind - `Merchant.merchantCountryCode equals \"US\"` — Merchant contexts whose attribute matches - `key startsWith \"merchant-\"` — contexts whose key has a prefix - `applicationId equals \"GoSDK/1.2\"` — contexts reported by a specific SDK/application Example combining both params: set `kind=Merchant` and `filter=Merchant.disbursementRail equals \"PIX\"` to find Merchant contexts on the PIX rail. Note: the contexts API cannot filter by which flag a context evaluated; there is no `flagEvaluated` field. Filter by kind, key, applicationId, or attribute values instead. Ordering is fixed by the API; there is no sort parameter. Pagination: responses include `totalCount` and `continuationToken`. Pass `continuationToken` from the previous response to get the next page.", - "input_schema": { - "type": "object", - "properties": { - "kind": { "type": "string" }, - "limit": { "type": "number" }, - "filter": { "type": "string" }, - "projectKey": { "type": "string" }, - "environmentKey": { "type": "string" }, - "continuationToken": { "type": "string" } - }, - "required": ["projectKey", "environmentKey"] - } - }, - { - "name": "save-and-start-experiment-iteration", - "description": "Stop the current running iteration, create a new draft iteration with the provided field updates applied, and start it — the API-recommended way to mutate treatments, metrics, methodology, or other fields that are locked while an iteration is running. Provide `changeJustification` to record why a new iteration is needed. Field inputs are the same as `update-experiment`; updates are checked against the new iteration's not_started mutability rules. Fields not allowed in not_started are returned under `skipped` rather than being sent.", - "input_schema": { - "type": "object", - "properties": { - "name": { "type": "string" }, - "tags": { "type": "array", "items": { "type": "string" } }, - "flags": { - "type": "object", - "propertyNames": { "type": "string" }, - "additionalProperties": { - "type": "object", - "required": ["ruleId", "flagConfigVersion"], - "properties": { - "ruleId": { "type": "string" }, - "flagConfigVersion": { "type": "number" }, - "notInExperimentVariationId": { "type": "string" } - }, - "additionalProperties": false - } - }, - "metrics": { - "type": "array", - "items": { - "type": "object", - "required": ["key"], - "properties": { - "key": { "type": "string" }, - "isGroup": { "type": "boolean" } - }, - "additionalProperties": false - } - }, - "holdoutId": { "type": "string" }, - "attributes": { "type": "array", "items": { "type": "string" } }, - "dataSource": { "type": "string", "enum": ["launchdarkly", "snowflake", "databricks"] }, - "hypothesis": { "type": "string" }, - "projectKey": { "type": "string" }, - "treatments": { - "type": "array", - "items": { - "type": "object", - "required": ["name", "baseline", "allocationPercent", "parameters"], - "properties": { - "name": { "type": "string" }, - "baseline": { "type": "boolean" }, - "parameters": { - "type": "array", - "items": { - "type": "object", - "required": ["flagKey", "variationId"], - "properties": { - "flagKey": { "type": "string" }, - "variationId": { "type": "string" } - }, - "additionalProperties": false - } - }, - "allocationPercent": { "type": "number" } - }, - "additionalProperties": false - } - }, - "covariateId": { "type": "string" }, - "description": { "type": "string" }, - "methodology": { "type": "string", "enum": ["bayesian", "frequentist", "export_only"] }, - "flagSettings": { - "type": "object", - "propertyNames": { "type": "string" }, - "additionalProperties": { - "type": "object", - "required": ["ruleId", "flagConfigVersion"], - "properties": { - "ruleId": { "type": "string" }, - "flagConfigVersion": { "type": "number" }, - "notInExperimentVariationId": { "type": "string" } - }, - "additionalProperties": false - } - }, - "maintainerId": { "type": "string" }, - "experimentKey": { "type": "string" }, - "analysisConfig": { - "type": "object", - "properties": { - "testDirection": { "type": "string" }, - "bayesianThreshold": { "type": "string" }, - "significanceThreshold": { "type": "string" }, - "sequentialTestingEnabled": { "type": "boolean" }, - "multipleComparisonCorrectionScope": { - "type": "string", - "enum": ["variations", "variations-and-metrics", "metrics"] - }, - "multipleComparisonCorrectionMethod": { - "type": "string", - "enum": ["bonferroni", "benjamini-hochberg"] - } - }, - "additionalProperties": false - }, - "environmentKey": { "type": "string" }, - "primaryFunnelKey": { "type": "string" }, - "randomizationUnit": { "type": "string" }, - "canReshuffleTraffic": { "type": "boolean" }, - "changeJustification": { "type": "string" }, - "primarySingleMetricKey": { "type": "string" } - }, - "required": ["projectKey", "environmentKey", "experimentKey"] - } } ]