diff --git a/evals/README.md b/evals/README.md index a9a3fb2..a275436 100644 --- a/evals/README.md +++ b/evals/README.md @@ -34,6 +34,7 @@ npm run eval:agentcontrol-tools # agentcontrol/tools npm run eval:configs-variations # agentcontrol/configs-variations npm run eval:flag-create # feature-flags/launchdarkly-flag-create npm run eval:onboarding # onboarding +npm run eval:experiment-setup # experiments/launchdarkly-experiment-setup npm run eval:experiment-orchestration # aws-devops-agent/experiment-orchestration # Quick smoke check — first test case only (~15-20s, ~$0.05) @@ -43,6 +44,7 @@ npm run eval:agentcontrol-tools:single npm run eval:configs-variations:single npm run eval:flag-create:single npm run eval:onboarding:single +npm run eval:experiment-setup:single npm run eval:experiment-orchestration:single # Aggregate and CI operations diff --git a/evals/experiment-setup/mocks.json b/evals/experiment-setup/mocks.json new file mode 100644 index 0000000..e002be5 --- /dev/null +++ b/evals/experiment-setup/mocks.json @@ -0,0 +1,62 @@ +{ + "get-flag-status-across-envs": { + "flagKey": "{{flagKey}}", + "environments": [ + { + "key": "test", + "status": "active" + }, + { + "key": "production", + "status": "active" + } + ], + "summary": { + "active": [ + "test", + "production" + ], + "inactive": [], + "new": [], + "launched": [] + } + }, + "list-metric-events": { + "events": [ + { + "key": "checkout-completed", + "lastSeen": "2026-10-20T00:00:00.000Z" + }, + { + "key": "page-load-ms", + "lastSeen": "2026-10-20T00:00:00.000Z" + }, + { + "key": "cta-click", + "lastSeen": "2026-10-18T00:00:00.000Z" + }, + { + "key": "cta-clicked-v2", + "lastSeen": "2026-10-18T00:00:00.000Z" + } + ], + "totalCount": 4 + }, + "search-contexts": { + "items": [ + { + "kind": "user", + "key": "user-1", + "lastSeen": "2026-10-20T00:00:00.000Z" + } + ], + "totalCount": 1 + }, + "save-and-start-experiment-iteration": { + "key": "{{experimentKey}}", + "environmentKey": "{{environmentKey}}", + "currentIteration": { + "status": "running" + } + } +} diff --git a/evals/experiment-setup/promptfooconfig.yaml b/evals/experiment-setup/promptfooconfig.yaml new file mode 100644 index 0000000..4bbc627 --- /dev/null +++ b/evals/experiment-setup/promptfooconfig.yaml @@ -0,0 +1,337 @@ +# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json +# +# Run with shared defaults: +# promptfoo eval -c shared/defaults.yaml -c experiment-setup/promptfooconfig.yaml +# +# Scores the approval gates in launchdarkly-experiment-setup. The user message +# is the whole turn: the agent shows a design or a question and waits. No case +# scripts an ask-question answer. Each case has one weight-3 tool-call check +# (lookups that should happen, and writes that must not) and a weight-1 rubric +# graded from the assistant's messages. The tool check decides pass or fail. +description: "Evaluation of the experiment-setup skill: look up, design, and wait" + +prompts: + - file://../../skills/experiments/launchdarkly-experiment-setup/SKILL.md + +providers: + - id: file://../providers/claude-skill-agent-sdk.js + label: claude-skill-agent-sdk + config: + skill_slug: launchdarkly-experiment-setup + allow_builtins: false + expose_mcp_tools: true + # No case scripts an answer. An unscripted ask-question call returns the + # first option, which can approve a create, so leave the tool off. + expose_ask_question: false + # Paste SKILL.md into the agent instructions. A slash-command first turn + # arrives before the request and aborts in-flight tool calls, and the + # skills list only supplies the name and description. + force_skill_invocation: false + inline_skill: true + # Schemas and default templates for the tools only this suite needs. + # Sentinel responses still come from the shared mock renderer. + tool_definitions: experiment-setup/tools.json + tool_mocks: experiment-setup/mocks.json + +tests: + # ------------------------------------------------------------------ + # 1. Every required piece is already named. Look up the flag and the + # metric, show one design, and wait. checkout-flow's get-flag template + # is version 1 and its fallthrough rule id is "fallthrough". + # ------------------------------------------------------------------ + - description: "Complete design: one design, then wait, with nothing created or started" + vars: + user_request: > + Set up an experiment in project "my-project", environment "production". + Flag: checkout-flow. Variations: true (id var-true) and false (id var-false). + Control: the false variation. Hypothesis: the new checkout will lift + completion rate by 3%. Primary metric: the existing checkout-completed + metric. Randomization unit: user. + max_turns: 16 + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const tools = output.tools_called || []; + const writes = ['create-metric','create-experiment','create-flag','start-experiment-iteration','save-and-start-experiment-iteration','toggle-flag','update-flag-settings','update-feature-flag','create-feature-flag']; + const called = writes.filter(name => tools.includes(name)); + const gotFlag = traj.some(t => t.tool === 'get-flag' && (t.arguments || {}).flagKey === 'checkout-flow'); + const listedMetrics = tools.includes('list-metrics'); + const pass = gotFlag && listedMetrics && called.length === 0; + return { pass, score: pass ? 1 : 0, reason: 'get-flag=' + gotFlag + ' list-metrics=' + listedMetrics + (called.length ? ' wrote ' + called.join(', ') : ' no writes') }; + metric: looks_up_flag_and_metric_and_does_not_write + weight: 3 + - type: llm-rubric + value: | + Read the assistant's messages. Score 1.0 if they show one complete + design and then wait for approval. That design names checkout-flow, + both variations, the false variation as control, the hypothesis, + checkout-completed, and randomization unit user. It includes ruleId + fallthrough and a flagConfigVersion. Statistics are the LaunchDarkly + default (frequentist, sequential testing). Score 0 if a required piece + is missing, statistics are Bayesian or another methodology, or the + message says the experiment was created or started. + metric: one_complete_design_then_waits + weight: 1 + + # ------------------------------------------------------------------ + # 2. Flag is on and active, and user is the only context kind. Suggest + # the control and the randomization unit. Do not create. + # ------------------------------------------------------------------ + - description: "Suggests control and randomization unit when the flag is active and only user contexts exist" + vars: + user_request: > + Set up an experiment in project "only-user", environment "production". + Flag: checkout-active. Compare its variations. Hypothesis: the new + checkout will lift completion rate by 3%. Primary metric: the existing + checkout-completed metric. I have not chosen a control variation or a + randomization unit. Show the design and wait. Do not create anything yet. + max_turns: 12 + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const tools = output.tools_called || []; + const writes = ['create-metric','create-experiment','create-flag','start-experiment-iteration','save-and-start-experiment-iteration','toggle-flag','update-flag-settings','update-feature-flag','create-feature-flag']; + const called = writes.filter(name => tools.includes(name)); + const gotFlag = traj.some(t => t.tool === 'get-flag' && (t.arguments || {}).flagKey === 'checkout-active'); + const searches = traj.filter(t => t.tool === 'search-contexts'); + const revealsOthers = searches.some(t => { + const a = t.arguments || {}; + const filter = String(a.filter || ''); + if (/kind\s+notEquals\s+"?user"?/i.test(filter)) return true; + if (a.kind === 'user' || /kind\s+equals\s+"?user"?/i.test(filter)) return false; + const kindBlank = a.kind == null || String(a.kind).trim() === ''; + const filterBlank = a.filter == null || filter.trim() === ''; + return kindBlank && filterBlank; + }); + const pass = gotFlag && revealsOthers && called.length === 0; + return { pass, score: pass ? 1 : 0, reason: 'get-flag=' + gotFlag + ' revealsOthers=' + revealsOthers + (called.length ? ' wrote ' + called.join(', ') : ' no writes') }; + metric: looks_up_active_flag_and_contexts + weight: 3 + - type: llm-rubric + value: | + Read the assistant's messages. The user did not name a control or a + randomization unit. Score 1.0 if the design labels both the served + variation ("Current checkout") and randomization unit "user" as + suggested, each with a short reason. Offering to confirm or change + those suggestions before approval is fine. Score 0 only if it asks + the user to supply the control or the randomization unit instead of + suggesting them, or omits the suggested label on either. + metric: labels_suggestions + weight: 1 + + # ------------------------------------------------------------------ + # 3. Flag is off and inactive, and another context kind is stored. + # Ask for the control and the randomization unit. Do not suggest them + # and do not create. + # ------------------------------------------------------------------ + - description: "Asks for control and randomization unit when the flag is inactive and contexts are mixed" + vars: + user_request: > + Set up an experiment in project "mixed-contexts", environment "production". + Flag: checkout-inactive. Compare its variations. Hypothesis: the new + checkout will lift completion rate by 3%. Primary metric: the existing + checkout-completed metric. I have not chosen a control variation or a + randomization unit. Ask me for anything you still need. Do not create + anything yet. + max_turns: 12 + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const tools = output.tools_called || []; + const writes = ['create-metric','create-experiment','create-flag','start-experiment-iteration','save-and-start-experiment-iteration','toggle-flag','update-flag-settings','update-feature-flag','create-feature-flag']; + const called = writes.filter(name => tools.includes(name)); + const gotFlag = traj.some(t => t.tool === 'get-flag' && (t.arguments || {}).flagKey === 'checkout-inactive'); + const gotStatus = traj.some(t => t.tool === 'get-flag-status-across-envs' && (t.arguments || {}).flagKey === 'checkout-inactive'); + const searches = traj.filter(t => t.tool === 'search-contexts'); + const revealsOthers = searches.some(t => { + const a = t.arguments || {}; + const filter = String(a.filter || ''); + if (/kind\s+notEquals\s+"?user"?/i.test(filter)) return true; + if (a.kind === 'user' || /kind\s+equals\s+"?user"?/i.test(filter)) return false; + const kindBlank = a.kind == null || String(a.kind).trim() === ''; + const filterBlank = a.filter == null || filter.trim() === ''; + return kindBlank && filterBlank; + }); + const pass = gotFlag && gotStatus && revealsOthers && called.length === 0; + return { pass, score: pass ? 1 : 0, reason: 'get-flag=' + gotFlag + ' status=' + gotStatus + ' revealsOthers=' + revealsOthers + (called.length ? ' wrote ' + called.join(', ') : ' no writes') }; + metric: looks_up_inactive_flag_and_mixed_contexts + weight: 3 + - type: llm-rubric + value: | + Read the assistant's messages. The flag is off and inactive, and more + than one context kind is stored. Score 1.0 if the agent asks for both + the control variation and the randomization unit, leaves both blank, + and does not label either as suggested. Score 0 if it suggests either + value or shows a design that fills either in. + metric: asks_instead_of_suggesting + weight: 1 + + # ------------------------------------------------------------------ + # 4. Flag, hypothesis, and primary metric are all missing. One gather + # message, and nothing is created. No lookup is required. + # ------------------------------------------------------------------ + - description: "Missing pieces: one gather message and no create" + vars: + user_request: > + I want to run an experiment in project "my-project", environment + "production". I don't have a flag, a hypothesis, or a primary metric yet. + max_turns: 8 + assert: + - type: javascript + value: | + const tools = output.tools_called || []; + const writes = ['create-metric','create-experiment','create-flag','start-experiment-iteration','save-and-start-experiment-iteration','toggle-flag','update-flag-settings','update-feature-flag','create-feature-flag']; + const called = writes.filter(name => tools.includes(name)); + const pass = called.length === 0; + return { pass, score: pass ? 1 : 0, reason: called.length ? 'wrote ' + called.join(', ') : 'no writes' }; + metric: does_not_write + weight: 3 + - type: llm-rubric + value: | + Read the assistant's messages. Score 1.0 if a single message asks for + the missing flag and its variations, the hypothesis, and the primary + metric. Score 0 if any of those three is left out, they are split + across separate messages, or a design is shown. + metric: gather_names_missing_pieces + weight: 1 + + # ------------------------------------------------------------------ + # 5. The custom event has not arrived. list-metric-events runs, the + # design warns and lists similar keys, and nothing is created. + # ------------------------------------------------------------------ + - description: "Missing event: list-metric-events runs, the design warns, and nothing is created" + vars: + user_request: > + Set up an experiment in project "my-project", environment "production". + Flag: checkout-flow. Variations: true (id var-true) and false (id var-false). + Control: the false variation. Hypothesis: the new checkout will lift + completion rate by 3%. Primary metric: a new custom metric "CTA clicked" + (key cta-clicked) measuring the share of users who sent the cta-clicked + event. Randomization unit: user. + max_turns: 16 + assert: + - type: javascript + value: | + const tools = output.tools_called || []; + const writes = ['create-metric','create-experiment','create-flag','start-experiment-iteration','save-and-start-experiment-iteration','toggle-flag','update-flag-settings','update-feature-flag','create-feature-flag']; + const called = writes.filter(name => tools.includes(name)); + const listedEvents = tools.includes('list-metric-events'); + const pass = listedEvents && called.length === 0; + return { pass, score: pass ? 1 : 0, reason: 'list-metric-events=' + listedEvents + (called.length ? ' wrote ' + called.join(', ') : ' no writes') }; + metric: lists_events_and_does_not_create + weight: 3 + - type: llm-rubric + value: | + Read the assistant's messages. The cta-clicked event is not arriving. + Similar keys cta-click and cta-clicked-v2 are. Score 1.0 if the design + warns that the event has not arrived in the environment and lists those + similar keys. Score 0 if the warning is missing or neither similar key + is named. + metric: warns_and_lists_similar_keys + weight: 1 + + # ------------------------------------------------------------------ + # 6. The user says to start. The flag is off and the event is still + # missing. One message covers both. No flag write and no start. + # ------------------------------------------------------------------ + - description: "Start blocked: flag off and event missing, no flag change and no start" + vars: + user_request: > + Start the existing experiment "checkout-draft" in project "my-project", + environment "production". It uses flag checkout-flow and a custom metric + whose event key is cta-clicked. Start it now. + max_turns: 12 + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const tools = output.tools_called || []; + const writes = ['create-metric','create-experiment','create-flag','start-experiment-iteration','save-and-start-experiment-iteration','toggle-flag','update-flag-settings','update-feature-flag','create-feature-flag']; + const called = writes.filter(name => tools.includes(name)); + const gotFlag = traj.some(t => t.tool === 'get-flag' && (t.arguments || {}).flagKey === 'checkout-flow'); + const listedEvents = tools.includes('list-metric-events'); + const pass = gotFlag && listedEvents && called.length === 0; + return { pass, score: pass ? 1 : 0, reason: 'get-flag=' + gotFlag + ' list-metric-events=' + listedEvents + (called.length ? ' wrote ' + called.join(', ') : ' no writes') }; + metric: looks_up_flag_and_event_and_does_not_start + weight: 3 + - type: llm-rubric + value: | + Read the assistant's messages. The flag is off and the cta-clicked + event is not arriving. Score 1.0 if one message covers both: the flag + must be on before starting, and whether to start while the event is + missing. Score 0 if those are separate messages, or the message says + the flag was turned on or the iteration was started. + metric: one_message_covers_flag_and_event + weight: 1 + + # ------------------------------------------------------------------ + # 7. The named flag does not exist. Look up the key and similar flags. + # Offer three choices. Do not create the flag. + # ------------------------------------------------------------------ + - description: "Missing flag: three choices, and the flag is not created" + vars: + user_request: > + Set up an experiment in project "my-project", environment "production". + Flag: missing-checkout. Variations: true and false. Control: false. + Hypothesis: the new checkout will lift completion rate by 3%. Primary + metric: the existing checkout-completed metric. Randomization unit: user. + max_turns: 12 + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const tools = output.tools_called || []; + const writes = ['create-metric','create-experiment','create-flag','start-experiment-iteration','save-and-start-experiment-iteration','toggle-flag','update-flag-settings','update-feature-flag','create-feature-flag']; + const called = writes.filter(name => tools.includes(name)); + const lookedUp = traj.some(t => t.tool === 'get-flag' && (t.arguments || {}).flagKey === 'missing-checkout'); + const listed = traj.some(t => t.tool === 'list-flags'); + const pass = lookedUp && listed && called.length === 0; + return { pass, score: pass ? 1 : 0, reason: 'get-flag=' + lookedUp + ' list-flags=' + listed + (called.length ? ' wrote ' + called.join(', ') : ' no writes') }; + metric: looks_up_missing_flag_and_does_not_create + weight: 3 + - type: llm-rubric + value: | + Read the assistant's messages. The flag missing-checkout does not exist. + Score 1.0 if one message offers three choices: correct the key + (mentioning a similar flag such as checkout-flow or new-checkout), + create the flag themselves and then continue, or have the agent create + it. For that last choice the message proposes the key, name, type, and + variations and says the flag will be created off. Score 0 if it offers + fewer than three choices. + metric: offers_three_flag_choices + weight: 1 + + # ------------------------------------------------------------------ + # 8. A running experiment's allocation change would start a new + # iteration. Show the change and wait. The user has not said to start. + # ------------------------------------------------------------------ + - description: "Running experiment: show the allocation change and do not start" + vars: + user_request: > + The experiment "checkout-running" in project "my-project", environment + "production" is running. Change the treatment allocation to 30% control + and 70% treatment. + max_turns: 12 + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const tools = output.tools_called || []; + const writes = ['create-metric','create-experiment','create-flag','start-experiment-iteration','save-and-start-experiment-iteration','toggle-flag','update-flag-settings','update-feature-flag','create-feature-flag']; + const called = writes.filter(name => tools.includes(name)); + const sawRunning = traj.some(t => t.tool === 'get-experiment' && ((t.arguments || {}).experimentKey === 'checkout-running' || (t.arguments || {}).key === 'checkout-running')); + const pass = sawRunning && called.length === 0; + return { pass, score: pass ? 1 : 0, reason: 'get-experiment=' + sawRunning + (called.length ? ' wrote ' + called.join(', ') : ' no writes') }; + metric: reads_running_experiment_and_does_not_start + weight: 3 + - type: llm-rubric + value: | + Read the assistant's messages. The experiment is running and the user + wants a 30/70 allocation but has not said to start. Score 1.0 if the + agent shows that allocation change and waits. Score 0 if it says a new + iteration is already running. + metric: shows_change_and_waits + weight: 1 diff --git a/evals/experiment-setup/tools.json b/evals/experiment-setup/tools.json new file mode 100644 index 0000000..aa76598 --- /dev/null +++ b/evals/experiment-setup/tools.json @@ -0,0 +1,301 @@ +[ + { + "name": "get-flag-status-across-envs", + "description": "Check a flag's lifecycle status across all environments. Returns per-environment state (active, inactive, new, launched) and a summary of which environments the flag is active, inactive, or launched in. Use before cleanup to verify the flag is inactive everywhere.", + "input_schema": { + "type": "object", + "properties": { + "flagKey": { + "type": "string" + }, + "projectKey": { + "type": "string" + } + }, + "required": [ + "projectKey", + "flagKey" + ] + } + }, + { + "name": "list-metric-events", + "description": "List recent event keys received by a LaunchDarkly project \u2014 used to check which events are actively flowing before creating a metric. Returns up to 50 event keys with last-seen timestamps. If the event key you need isn't here, it may not be instrumented yet. Use this BEFORE create-metric to: 1. Confirm the event key exists and is receiving data 2. Discover event keys when the user isn't sure what to use 3. Check the correct spelling of an event key Supports fuzzy search via `query` to narrow results. Filter by `environmentKey` to check a specific environment. If the desired event key is absent, surface a warning and ask whether the user wants to instrument the event first or proceed and wire it up later.", + "input_schema": { + "type": "object", + "properties": { + "query": { + "type": "string" + }, + "projectKey": { + "type": "string" + }, + "environmentKey": { + "type": "string" + } + }, + "required": [ + "projectKey" + ] + } + }, + { + "name": "search-contexts", + "description": "Search for stored LaunchDarkly contexts in an environment by kind, key, or attribute value. Returns context records including their stored attributes (e.g., merchantCountryCode, disbursementRail) and when the context was last seen. Convenience param: - `kind`: filter to a specific context kind (e.g. `Merchant`, `user`, `organization`). Emitted as the filter clause `kind equals \"\"`. Raw filter syntax (passed in `filter`) uses `field operator value` clauses. Operators: `equals`, `notEquals`, `anyOf`, `startsWith`, `exists`, `before`, `after`; `contains` and `notContains` are not supported. Combine clauses with `,` (AND) or `|` (OR); group with `()`. Supported fields: `kind`, `key`, `kindKey`, `name`, `applicationId`, `q` (fuzzy), and `.` for context attributes. Examples: - `kind equals \"Merchant\"` \u2014 contexts of a specific kind - `Merchant.merchantCountryCode equals \"US\"` \u2014 Merchant contexts whose attribute matches - `key startsWith \"merchant-\"` \u2014 contexts whose key has a prefix - `applicationId equals \"GoSDK/1.2\"` \u2014 contexts reported by a specific SDK/application Example combining both params: set `kind=Merchant` and `filter=Merchant.disbursementRail equals \"PIX\"` to find Merchant contexts on the PIX rail. Note: the contexts API cannot filter by which flag a context evaluated; there is no `flagEvaluated` field. Filter by kind, key, applicationId, or attribute values instead. Ordering is fixed by the API; there is no sort parameter. Pagination: responses include `totalCount` and `continuationToken`. Pass `continuationToken` from the previous response to get the next page.", + "input_schema": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "limit": { + "type": "number" + }, + "filter": { + "type": "string" + }, + "projectKey": { + "type": "string" + }, + "environmentKey": { + "type": "string" + }, + "continuationToken": { + "type": "string" + } + }, + "required": [ + "projectKey", + "environmentKey" + ] + } + }, + { + "name": "save-and-start-experiment-iteration", + "description": "Stop the current running iteration, create a new draft iteration with the provided field updates applied, and start it \u2014 the API-recommended way to mutate treatments, metrics, methodology, or other fields that are locked while an iteration is running. Provide `changeJustification` to record why a new iteration is needed. Field inputs are the same as `update-experiment`; updates are checked against the new iteration's not_started mutability rules. Fields not allowed in not_started are returned under `skipped` rather than being sent.", + "input_schema": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + }, + "flags": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "required": [ + "ruleId", + "flagConfigVersion" + ], + "properties": { + "ruleId": { + "type": "string" + }, + "flagConfigVersion": { + "type": "number" + }, + "notInExperimentVariationId": { + "type": "string" + } + }, + "additionalProperties": false + } + }, + "metrics": { + "type": "array", + "items": { + "type": "object", + "required": [ + "key" + ], + "properties": { + "key": { + "type": "string" + }, + "isGroup": { + "type": "boolean" + } + }, + "additionalProperties": false + } + }, + "holdoutId": { + "type": "string" + }, + "attributes": { + "type": "array", + "items": { + "type": "string" + } + }, + "dataSource": { + "type": "string", + "enum": [ + "launchdarkly", + "snowflake", + "databricks" + ] + }, + "hypothesis": { + "type": "string" + }, + "projectKey": { + "type": "string" + }, + "treatments": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name", + "baseline", + "allocationPercent", + "parameters" + ], + "properties": { + "name": { + "type": "string" + }, + "baseline": { + "type": "boolean" + }, + "parameters": { + "type": "array", + "items": { + "type": "object", + "required": [ + "flagKey", + "variationId" + ], + "properties": { + "flagKey": { + "type": "string" + }, + "variationId": { + "type": "string" + } + }, + "additionalProperties": false + } + }, + "allocationPercent": { + "type": "number" + } + }, + "additionalProperties": false + } + }, + "covariateId": { + "type": "string" + }, + "description": { + "type": "string" + }, + "methodology": { + "type": "string", + "enum": [ + "bayesian", + "frequentist", + "export_only" + ] + }, + "flagSettings": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "required": [ + "ruleId", + "flagConfigVersion" + ], + "properties": { + "ruleId": { + "type": "string" + }, + "flagConfigVersion": { + "type": "number" + }, + "notInExperimentVariationId": { + "type": "string" + } + }, + "additionalProperties": false + } + }, + "maintainerId": { + "type": "string" + }, + "experimentKey": { + "type": "string" + }, + "analysisConfig": { + "type": "object", + "properties": { + "testDirection": { + "type": "string" + }, + "bayesianThreshold": { + "type": "string" + }, + "significanceThreshold": { + "type": "string" + }, + "sequentialTestingEnabled": { + "type": "boolean" + }, + "multipleComparisonCorrectionScope": { + "type": "string", + "enum": [ + "variations", + "variations-and-metrics", + "metrics" + ] + }, + "multipleComparisonCorrectionMethod": { + "type": "string", + "enum": [ + "bonferroni", + "benjamini-hochberg" + ] + } + }, + "additionalProperties": false + }, + "environmentKey": { + "type": "string" + }, + "primaryFunnelKey": { + "type": "string" + }, + "randomizationUnit": { + "type": "string" + }, + "canReshuffleTraffic": { + "type": "boolean" + }, + "changeJustification": { + "type": "string" + }, + "primarySingleMetricKey": { + "type": "string" + } + }, + "required": [ + "projectKey", + "environmentKey", + "experimentKey" + ] + } + } +] diff --git a/evals/package.json b/evals/package.json index 51f90ca..953d865 100644 --- a/evals/package.json +++ b/evals/package.json @@ -27,6 +27,8 @@ "eval:onboarding:single": "promptfoo eval -c shared/defaults.yaml -c onboarding/promptfooconfig.yaml --env-file .env --no-cache --filter-first-n 1", "eval:experiment-guide": "promptfoo eval -c shared/defaults.yaml -c experiment-guide/promptfooconfig.yaml --env-file .env --no-cache -o experiment-guide/results.json", "eval:experiment-guide:single": "promptfoo eval -c shared/defaults.yaml -c experiment-guide/promptfooconfig.yaml --env-file .env --no-cache --filter-first-n 1", + "eval:experiment-setup": "promptfoo eval -c shared/defaults.yaml -c experiment-setup/promptfooconfig.yaml --env-file .env --no-cache -o experiment-setup/results.json", + "eval:experiment-setup:single": "promptfoo eval -c shared/defaults.yaml -c experiment-setup/promptfooconfig.yaml --env-file .env --no-cache --filter-first-n 1", "eval:experiment-orchestration": "promptfoo eval -c shared/defaults.yaml -c experiment-orchestration/promptfooconfig.yaml --env-file .env --no-cache -o experiment-orchestration/results.json", "eval:experiment-orchestration:single": "promptfoo eval -c shared/defaults.yaml -c experiment-orchestration/promptfooconfig.yaml --env-file .env --no-cache --filter-first-n 1", "fixture:from-pr": "node scripts/build-pr-fixture.js", diff --git a/evals/providers/_mock.js b/evals/providers/_mock.js index f5ab11e..b8ea9c1 100644 --- a/evals/providers/_mock.js +++ b/evals/providers/_mock.js @@ -129,6 +129,152 @@ function createMockState() { }; } +// Lifecycle status for get-flag-status-across-envs. The default matches the +// environments in the get-environments mock and reports both as active, so +// flag-drift sees no cross-environment difference and flag-command gets no +// cleanup signal. `federal` is intentionally absent: the flag-drift mock +// flips that environment's fallthrough, and listing it here would make +// single-environment drift cases look divergent. +function flagStatusResponse(flagKey, productionStatus) { + const environments = [ + { key: "test", status: "active" }, + { key: "production", status: productionStatus }, + ]; + const summary = { active: [], inactive: [], new: [], launched: [] }; + for (const env of environments) { + if (summary[env.status]) summary[env.status].push(env.key); + } + return { flagKey, environments, summary }; +} + +const DEFAULT_METRIC_EVENTS = [ + { key: "checkout-completed", lastSeen: "2026-10-20T00:00:00.000Z" }, + { key: "page-load-ms", lastSeen: "2026-10-20T00:00:00.000Z" }, + { key: "cta-click", lastSeen: "2026-10-18T00:00:00.000Z" }, + { key: "cta-clicked-v2", lastSeen: "2026-10-18T00:00:00.000Z" }, +]; + +// list-metric-events has no eventKey. A query of cta-clicked returns the +// similar keys only; every other call, including no query, returns the +// default list (which also contains those similar keys). +function metricEventsResponse(input) { + const query = typeof input.query === "string" ? input.query.trim() : ""; + const events = + query.toLowerCase() === "cta-clicked" + ? DEFAULT_METRIC_EVENTS.filter( + (event) => event.key === "cta-click" || event.key === "cta-clicked-v2", + ) + : DEFAULT_METRIC_EVENTS; + return { events, totalCount: events.length }; +} + +function isUserKindQuery(input) { + if (input.kind === "user") return true; + return typeof input.filter === "string" && /kind\s+equals\s+"?user"?/i.test(input.filter); +} + +function isNotEqualsUser(input) { + return ( + typeof input.filter === "string" && + /kind\s+notEquals\s+"?user"?/i.test(input.filter) + ); +} + +function contextItem(kind, key) { + return { kind, key, lastSeen: "2026-10-20T00:00:00.000Z" }; +} + +function isUnfilteredContextSearch(input) { + const kindBlank = input.kind == null || String(input.kind).trim() === ""; + const filterBlank = input.filter == null || String(input.filter).trim() === ""; + return kindBlank && filterBlank; +} + +function searchContextsResponse(input) { + const mixed = input.projectKey === "mixed-contexts"; + if (isNotEqualsUser(input)) { + if (mixed) { + return { items: [contextItem("organization", "org-1")], totalCount: 1 }; + } + return { items: [], totalCount: 0 }; + } + if (isUserKindQuery(input)) { + return { items: [contextItem("user", "user-1")], totalCount: 1 }; + } + // No kind and no filter lists every stored kind: user in both projects, + // and organization as well when other kinds are stored. + if (isUnfilteredContextSearch(input)) { + const items = [contextItem("user", "user-1")]; + if (mixed) items.push(contextItem("organization", "org-1")); + return { items, totalCount: items.length }; + } + return { items: [], totalCount: 0 }; +} + +function checkoutActiveFlag() { + return { + key: "checkout-active", + name: "Checkout active", + kind: "multivariate", + description: "Checkout experiment flag", + temporary: true, + tags: [], + variations: [ + { _id: "var-current", value: "current", name: "Current checkout" }, + { _id: "var-new", value: "new", name: "New checkout" }, + ], + defaults: { onVariation: 0, offVariation: 1 }, + environment: { + on: true, + version: 7, + rules: [], + targets: [], + fallthrough: { variation: 0 }, + offVariation: 1, + }, + }; +} + +function similarCheckoutFlags() { + const flag = (key, name, on) => ({ + key, + name, + kind: "boolean", + description: name, + temporary: true, + tags: ["checkout"], + on, + variations: [ + { value: true, name: "Enabled" }, + { value: false, name: "Disabled" }, + ], + }); + return { + flags: [ + flag("checkout-flow", "Checkout flow", true), + flag("new-checkout", "New checkout", false), + ], + totalCount: 2, + pageInfo: { limit: 20, offset: 0 }, + }; +} + +function runningExperiment(input) { + return { + key: "checkout-running", + name: "Checkout running", + environmentKey: input.environmentKey || input.env || "production", + currentIteration: { + status: "running", + hypothesis: "The new checkout will lift completion.", + treatments: [ + { _id: "treat-control", name: "Control", baseline: true, allocationPercent: 50 }, + { _id: "treat-treatment", name: "Treatment", baseline: false, allocationPercent: 50 }, + ], + }, + }; +} + /** * Render a mock response for `toolName` given `input` (raw tool arguments). * `state` is the per-invocation map from `createMockState()`. @@ -201,6 +347,17 @@ function renderMockResponse(template, input, toolName, state) { const key = input.flagKey || input.key; if (key && state.flags[key]) return state.flags[key]; + // Sentinel keys for the experiment-setup suite. Checked before the + // federal fallthrough hook so they do not inherit that override. + if (key === "missing-checkout") { + return { + error: "not_found", + status: 404, + message: "Flag 'missing-checkout' was not found in this project.", + }; + } + if (key === "checkout-active") return checkoutActiveFlag(); + // Cross-environment divergence hook: a `federal`-style environment serves // the opposite default (variation 0 / `true`) from every other environment // (which resolve to variation 1 / `false` via the static template). This @@ -374,6 +531,28 @@ function renderMockResponse(template, input, toolName, state) { return flag; } + if (toolName === "get-flag-status-across-envs") { + const flagKey = input.flagKey || "unknown-flag"; + const productionStatus = flagKey === "checkout-inactive" ? "inactive" : "active"; + return flagStatusResponse(flagKey, productionStatus); + } + + if (toolName === "search-contexts") return searchContextsResponse(input || {}); + + if (toolName === "list-metric-events") return metricEventsResponse(input || {}); + + if (toolName === "list-flags" || toolName === "list-feature-flags") { + const query = typeof input.query === "string" ? input.query.trim().toLowerCase() : ""; + // A search for a checkout flag returns the similar keys and omits the + // missing one, whatever the exact query string is. + if (query.includes("checkout")) return similarCheckoutFlags(); + } + + if (toolName === "get-experiment") { + const experimentKey = input.experimentKey || input.key; + if (experimentKey === "checkout-running") return runningExperiment(input); + } + // Default: stateless template render return walk(template, replacements); } diff --git a/evals/scripts/_manifest.js b/evals/scripts/_manifest.js index 6882661..47b3d84 100644 --- a/evals/scripts/_manifest.js +++ b/evals/scripts/_manifest.js @@ -85,6 +85,12 @@ const SUITES = [ skillDir: "skills/experiments/launchdarkly-experiment-guide", readme: "skills/experiments/launchdarkly-experiment-guide/README.md", }, + { + suite: "experiment-setup", + skillKey: "experiments/launchdarkly-experiment-setup", + skillDir: "skills/experiments/launchdarkly-experiment-setup", + readme: "skills/experiments/launchdarkly-experiment-setup/README.md", + }, { // No readme: the skill folder holds only the SKILL.md that ldcli embeds. suite: "experiment-orchestration", diff --git a/evals/tests/mock.test.js b/evals/tests/mock.test.js index 4a96127..6eb907a 100644 --- a/evals/tests/mock.test.js +++ b/evals/tests/mock.test.js @@ -4,6 +4,9 @@ const assert = require("node:assert/strict"); const { renderMockResponse, createMockState } = require("../providers/_mock"); const TEMPLATES = require("../mocks/tool-responses.json"); +// These four tools' default templates live with the experiment-setup suite. +// The sentinel branches in _mock.js still override them. +const SUITE_TEMPLATES = require("../experiment-setup/mocks.json"); // --------------------------------------------------------------------------- // createMockState @@ -422,3 +425,224 @@ test("update-ai-config uses template substitution", () => { assert.equal(result.key, "bot"); assert.equal(result.name, "Bot"); }); + +// --------------------------------------------------------------------------- +// experiment-setup sentinels +// --------------------------------------------------------------------------- + +test("experiment-setup suite files own the four tools the shared catalog no longer has", () => { + const sharedDefs = require("../tools/definitions.json").map((def) => def.name); + const suiteDefs = require("../experiment-setup/tools.json").map((def) => def.name); + const owned = [ + "get-flag-status-across-envs", + "list-metric-events", + "search-contexts", + "save-and-start-experiment-iteration", + ]; + for (const name of owned) { + assert.equal(sharedDefs.includes(name), false); + assert.equal(TEMPLATES[name], undefined); + assert.equal(suiteDefs.includes(name), true); + assert.ok(SUITE_TEMPLATES[name]); + } +}); + +test("save-and-start returns the suite template when no sentinel matches", () => { + const state = createMockState(); + const result = renderMockResponse(SUITE_TEMPLATES["save-and-start-experiment-iteration"], { + projectKey: "my-project", + environmentKey: "production", + experimentKey: "checkout-running", + }, "save-and-start-experiment-iteration", state); + assert.equal(result.key, "checkout-running"); + assert.equal(result.currentIteration.status, "running"); +}); + +test("get-flag-status-across-envs defaults to active in test and production", () => { + const state = createMockState(); + const result = renderMockResponse(SUITE_TEMPLATES["get-flag-status-across-envs"], { + projectKey: "payments", + flagKey: "billing-v2", + }, "get-flag-status-across-envs", state); + assert.deepEqual( + result.environments.map((env) => [env.key, env.status]), + [["test", "active"], ["production", "active"]], + ); + assert.deepEqual(result.summary.active, ["test", "production"]); + assert.deepEqual(result.summary.inactive, []); + assert.equal( + result.environments.some((env) => /federal/i.test(env.key)), + false, + ); +}); + +test("get-flag-status-across-envs reports checkout-inactive as inactive in production", () => { + const state = createMockState(); + const result = renderMockResponse(SUITE_TEMPLATES["get-flag-status-across-envs"], { + projectKey: "mixed-contexts", + flagKey: "checkout-inactive", + }, "get-flag-status-across-envs", state); + const production = result.environments.find((env) => env.key === "production"); + assert.equal(production.status, "inactive"); + assert.deepEqual(result.summary.inactive, ["production"]); +}); + +test("get-flag returns checkout-active on, serving one variation", () => { + const state = createMockState(); + const result = renderMockResponse(TEMPLATES["get-flag"], { + projectKey: "only-user", + flagKey: "checkout-active", + environmentKey: "production", + }, "get-flag", state); + assert.equal(result.environment.on, true); + assert.equal(result.environment.version, 7); + assert.equal(result.environment.fallthrough.variation, 0); + assert.equal(result.environment.fallthrough.rollout, undefined); + assert.equal(result.variations[0].name, "Current checkout"); +}); + +test("get-flag returns not-found for missing-checkout", () => { + const state = createMockState(); + const result = renderMockResponse(TEMPLATES["get-flag"], { + projectKey: "my-project", + flagKey: "missing-checkout", + }, "get-flag", state); + assert.equal(result.status, 404); +}); + +test("get-flag returns a stored flag before the missing-checkout not-found branch", () => { + const state = createMockState(); + renderMockResponse(TEMPLATES["create-flag"], { + key: "missing-checkout", + name: "Missing", + }, "create-flag", state); + const result = renderMockResponse(TEMPLATES["get-flag"], { + flagKey: "missing-checkout", + }, "get-flag", state); + assert.equal(result.key, "missing-checkout"); + assert.equal(result.error, undefined); +}); + +test("list-flags query missing-checkout returns similar keys only", () => { + const state = createMockState(); + const result = renderMockResponse(TEMPLATES["list-flags"], { + projectKey: "my-project", + query: "missing-checkout", + }, "list-flags", state); + const keys = result.flags.map((flag) => flag.key); + assert.deepEqual(keys, ["checkout-flow", "new-checkout"]); +}); + +test("list-flags query containing checkout returns similar keys and omits missing-checkout", () => { + const state = createMockState(); + const result = renderMockResponse(TEMPLATES["list-flags"], { + projectKey: "my-project", + query: "Checkout", + }, "list-flags", state); + const keys = result.flags.map((flag) => flag.key); + assert.deepEqual(keys, ["checkout-flow", "new-checkout"]); + assert.equal(keys.includes("missing-checkout"), false); +}); + +test("list-flags without that query keeps the shared flag list", () => { + const state = createMockState(); + const result = renderMockResponse(TEMPLATES["list-flags"], { + projectKey: "web-app", + query: "search-v2", + }, "list-flags", state); + assert.deepEqual( + result.flags.map((flag) => flag.key), + ["enable-notifications", "search-v2"], + ); +}); + +test("search-contexts suggests user only when no other kind is stored", () => { + const state = createMockState(); + const users = renderMockResponse(SUITE_TEMPLATES["search-contexts"], { + projectKey: "only-user", + environmentKey: "production", + kind: "user", + }, "search-contexts", state); + const others = renderMockResponse(SUITE_TEMPLATES["search-contexts"], { + projectKey: "only-user", + environmentKey: "production", + filter: 'kind notEquals "user"', + }, "search-contexts", state); + assert.equal(users.totalCount, 1); + assert.equal(users.items[0].kind, "user"); + assert.equal(others.totalCount, 0); +}); + +test("search-contexts with no kind filter lists every stored kind", () => { + const state = createMockState(); + const onlyUser = renderMockResponse(SUITE_TEMPLATES["search-contexts"], { + projectKey: "only-user", + environmentKey: "production", + }, "search-contexts", state); + assert.deepEqual(onlyUser.items.map((item) => item.kind), ["user"]); + assert.equal(onlyUser.totalCount, 1); + + const mixed = renderMockResponse(SUITE_TEMPLATES["search-contexts"], { + projectKey: "mixed-contexts", + environmentKey: "production", + }, "search-contexts", state); + assert.deepEqual( + mixed.items.map((item) => item.kind).sort(), + ["organization", "user"], + ); + assert.equal(mixed.totalCount, 2); +}); + +test("search-contexts on mixed-contexts returns another kind", () => { + const state = createMockState(); + const others = renderMockResponse(SUITE_TEMPLATES["search-contexts"], { + projectKey: "mixed-contexts", + environmentKey: "production", + filter: 'kind notEquals "user"', + }, "search-contexts", state); + assert.ok(others.totalCount > 0); + assert.equal(others.items[0].kind, "organization"); +}); + +test("list-metric-events default includes similar keys and omits cta-clicked", () => { + const state = createMockState(); + const result = renderMockResponse(SUITE_TEMPLATES["list-metric-events"], { + projectKey: "my-project", + }, "list-metric-events", state); + const keys = result.events.map((event) => event.key); + assert.deepEqual(keys, [ + "checkout-completed", + "page-load-ms", + "cta-click", + "cta-clicked-v2", + ]); +}); + +test("list-metric-events query cta-clicked returns similar keys only", () => { + const state = createMockState(); + const result = renderMockResponse(SUITE_TEMPLATES["list-metric-events"], { + projectKey: "my-project", + environmentKey: "production", + query: "cta-clicked", + }, "list-metric-events", state); + assert.deepEqual( + result.events.map((event) => event.key), + ["cta-click", "cta-clicked-v2"], + ); +}); + +test("get-experiment returns a running iteration for checkout-running", () => { + const state = createMockState(); + const running = renderMockResponse(TEMPLATES["get-experiment"], { + projectKey: "my-project", + environmentKey: "production", + experimentKey: "checkout-running", + }, "get-experiment", state); + const draft = renderMockResponse(TEMPLATES["get-experiment"], { + projectKey: "my-project", + environmentKey: "production", + experimentKey: "checkout-draft", + }, "get-experiment", state); + assert.equal(running.currentIteration.status, "running"); + assert.equal(draft.currentIteration.status, "not_started"); +});