From 10f748d14e126893e777b76b6174b797e33ff12d Mon Sep 17 00:00:00 2001
From: Farhan Sareshwala <fsareshwala@launchdarkly.com>
Date: Fri, 30 Oct 2026 14:46:22 -0700
Subject: [PATCH 1/3] fix(release-autopilot-onboarding): defer browser changes,
 gate GitHub and mapping, clear opt-out overrides

Follow-ups from review on #229:
- Bring your own key, auto-cleanup, and auto-archival change settings in
  the browser, so they now happen only after 'Looks good'; Preview lists
  them and never asks for them.
- GitHub runs only for auto-flagging, auto-releasing, or auto-cleanup, and
  repo mapping only for auto-flagging or auto-releasing; auto-archival
  alone picks projects directly.
- A picked capability that a repo override keeps off is turned back on,
  and verification checks the effective value.
- Evals: mappings must name the exact project, the policy case must take
  the 403-then-policies path, and plan mode must write the repo mapping.
  New cases for archival-only, preview with cleanup, and the repo override.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 CHANGELOG.md                                  |   1 +
 .../assertions.js                             |  49 +++++++-
 .../promptfooconfig.yaml                      | 109 +++++++++++++++++-
 skills.json                                   |   2 +-
 .../SKILL.md                                  |  64 +++++-----
 .../marketplace.json                          |   2 +-
 .../references/edge-cases.md                  |   1 +
 .../references/modes.md                       |   2 +-
 8 files changed, 190 insertions(+), 40 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b0d5e39..4c6ab5a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,7 @@ All notable changes to this repository will be documented in this file.
 
 ## Unreleased
 
+- `release-autopilot/launchdarkly-release-autopilot-onboarding` 0.3.0-experimental: browser steps that change settings (bring your own key, auto-cleanup, auto-archival) now happen only after "Looks good", and Preview lists them instead of asking for them; GitHub and repo mapping run only for the capabilities that need them (auto-archival alone needs neither); a picked capability turned off by a repo override is turned back on and verified. Evals require the exact project on mappings, the policy lookup path, and the repo write in plan mode, and add archival-only, preview-with-cleanup, and repo-override cases (23 total).
 - Add `release-autopilot/launchdarkly-release-autopilot-onboarding` 0.2.0-experimental: a step-by-step wizard with setup, preview, and resume modes that asks which capabilities to set up (auto-flagging, auto-releasing, auto-cleanup, auto-archival), hands off the GitHub App install or authorize link, maps chosen repos, guides managed or bring-your-own-key Vega inference, and reviews or creates environment release plans for auto-releasing, or uses the project's release policies where plans are not available. Release policies are required for auto-releasing today; the skill will be updated to use environment release plans by GA, when plans replace policies. One confirmed plan lists every write before any change. Ships in Claude Code plugin 1.0.3 and Cursor plugin 1.0.4.
 - `release-autopilot/launchdarkly-release-autopilot-settings` 1.4.0-experimental: route first-time setup to the onboarding skill.
 - evals: add the `release-autopilot-onboarding` suite (20 cases: confirm before writing, repo overrides, policy and plan modes, reader and writer roles, preview, resume, GitHub install and authorize paths, deprecated MCP, a failed write, auto-cleanup, auto-archival, a browser signed in to a different account, an account without the add-on, a diagnosis on an empty account). The suite keeps its tool definitions and mocks in its own folder (`tool_definitions`, `tool_mocks`). The provider gains `mock_ask_question_match` with label matching, stateful Release Autopilot mocks, and `CLAUDE_CODE_OAUTH_TOKEN` as an alternative to `ANTHROPIC_API_KEY`.
diff --git a/evals/release-autopilot-onboarding/assertions.js b/evals/release-autopilot-onboarding/assertions.js
index 7ffe522..1882429 100644
--- a/evals/release-autopilot-onboarding/assertions.js
+++ b/evals/release-autopilot-onboarding/assertions.js
@@ -45,7 +45,7 @@ module.exports.confirmsBeforeWriting = (output) => {
 const mapsRepoWithOverrides = (repo, project, caps) => (output) => {
   const maps = traj(output).filter((t) => t.tool === "update-release-autopilot-repo-settings");
   if (!maps.length) return result(false, "never wrote a repo mapping");
-  const wrong = maps.filter((m) => (m.arguments || {}).repo !== repo || (project && (m.arguments || {}).projectKey && m.arguments.projectKey !== project));
+  const wrong = maps.filter((m) => (m.arguments || {}).repo !== repo || (project && (m.arguments || {}).projectKey !== project));
   if (wrong.length) return result(false, `wrote ${wrong.map((m) => `${m.arguments.repo}@${m.arguments.projectKey}`).join(", ")}; want only ${repo}@${project}`);
   const bad = maps.filter((m) => caps.some((c) => !(m.arguments[c] && m.arguments[c].enabled === false)));
   return result(bad.length === 0, bad.length ? `missing ${caps.join("/")} enabled:false` : `${repo} written with ${caps.join("/")} off`);
@@ -88,7 +88,10 @@ module.exports.planBeforeRepoWrite = (output) => {
   const plan = steps.findIndex((t) => t.tool === "create-release-plan");
   const repo = steps.findIndex((t) => t.tool === "update-release-autopilot-repo-settings");
   if (plan === -1) return result(false, "never created a release plan");
-  return result(repo === -1 || plan < repo, `plan at ${plan}, repo write at ${repo}`);
+  if (repo === -1) return result(false, "never mapped acme/demo");
+  const args = steps[repo].arguments || {};
+  const pass = plan < repo && args.repo === "acme/demo" && args.projectKey === "default";
+  return result(pass, `plan at ${plan}, repo write at ${repo} (${args.repo}@${args.projectKey})`);
 };
 
 module.exports.plansHaveClauses = (output) => {
@@ -352,3 +355,45 @@ module.exports.pausesBeforeMapping = (output) => {
   const pass = repoQ.length === 0 && resume;
   return result(pass, `repo questions: ${repoQ.length}; resume phrase in final message: ${resume}`);
 };
+
+// Policy mode: exactly one release-plans lookup (the 403 is account-wide) and
+// at least one release-policies lookup.
+module.exports.policyLookupPath = (output) => {
+  const plans = traj(output).filter((t) => t.tool === "list-release-plans").length;
+  const policies = traj(output).filter((t) => t.tool === "list-release-policies").length;
+  return result(plans === 1 && policies >= 1, `release-plans lookups: ${plans}, release-policies lookups: ${policies}`);
+};
+
+const isBrowserStepQuestion = (t) => t.tool === "ask-question" &&
+  /\/projects\/[^/\s]+\/settings\/(lifecycle|vega)/.test(String((t.arguments && t.arguments.prompt) || ""));
+
+// Browser steps that change settings come only after "Looks good".
+module.exports.browserStepsAfterConfirm = (output) => {
+  const steps = traj(output);
+  const browser = steps.findIndex(isBrowserStepQuestion);
+  if (browser === -1) return result(false, "no browser step question");
+  const confirm = steps.findIndex(isConfirmQuestion);
+  const pass = confirm !== -1 && confirm < browser;
+  return result(pass, `confirm at ${confirm}, first browser step at ${browser}`);
+};
+
+// Preview never asks the user to change something in the browser, and lists it.
+module.exports.previewListsBrowserSteps = (output) => {
+  const asked = traj(output).filter(isBrowserStepQuestion).length;
+  const listed = /settings\/lifecycle/.test(finalText(output));
+  return result(asked === 0 && listed, `browser step questions: ${asked}; lifecycle link listed: ${listed}`);
+};
+
+// Archival-only setup needs no GitHub step and no repo mapping.
+module.exports.noGithubOrMapping = (output) => {
+  const steps = traj(output);
+  const repoWrites = steps.filter((t) => /^(update|delete)-release-autopilot-repo-settings$/.test(t.tool)).length;
+  const ghQ = steps.filter((t) => t.tool === "ask-question" && /github|repo/i.test(String((t.arguments && t.arguments.prompt) || ""))).length;
+  return result(repoWrites === 0 && ghQ === 0, `repo writes: ${repoWrites}; GitHub or repo questions: ${ghQ}`);
+};
+
+module.exports.storefrontAutoFlaggingOn = (output) => {
+  const maps = traj(output).filter((t) => t.tool === "update-release-autopilot-repo-settings" && (t.arguments || {}).repo === "acme/storefront");
+  const on = maps.some((m) => m.arguments.autoFlagging && m.arguments.autoFlagging.enabled === true);
+  return result(on, on ? "turned auto-flagging on for acme/storefront" : "did not clear the repo's auto-flagging override");
+};
diff --git a/evals/release-autopilot-onboarding/promptfooconfig.yaml b/evals/release-autopilot-onboarding/promptfooconfig.yaml
index e608f72..bf36f75 100644
--- a/evals/release-autopilot-onboarding/promptfooconfig.yaml
+++ b/evals/release-autopilot-onboarding/promptfooconfig.yaml
@@ -150,7 +150,7 @@ tests:
         metric: no_unpicked_archival
         weight: 3
       - type: javascript
-        value: file://../release-autopilot-onboarding/assertions.js:releasePlansCalledAtMostOnce
+        value: file://../release-autopilot-onboarding/assertions.js:policyLookupPath
         metric: plans_403_account_wide
         weight: 3
       - type: javascript
@@ -620,6 +620,10 @@ tests:
         - ["Done"]
         - ["Looks good"]
     assert:
+      - type: javascript
+        value: file://../release-autopilot-onboarding/assertions.js:browserStepsAfterConfirm
+        metric: browser_after_confirm
+        weight: 3
       - type: javascript
         value: file://../release-autopilot-onboarding/assertions.js:noReleaseSetup
         metric: no_release_setup
@@ -769,6 +773,10 @@ tests:
         - ["Done"]
         - ["Looks good"]
     assert:
+      - type: javascript
+        value: file://../release-autopilot-onboarding/assertions.js:browserStepsAfterConfirm
+        metric: browser_after_confirm
+        weight: 3
       - type: javascript
         value: file://../release-autopilot-onboarding/assertions.js:noReleaseSetup
         metric: no_release_setup
@@ -837,3 +845,102 @@ tests:
         value: file://../release-autopilot-onboarding/assertions.js:noWrites
         metric: no_writes
         weight: 3
+
+  # ------------------------------------------------------------------
+  # Auto-archival only: project-level, no GitHub, no repo mapping. The
+  # Lifecycle settings step comes after "Looks good".
+  # ------------------------------------------------------------------
+  - providers: ["mcp"]
+    description: "Archival only: no GitHub or repo mapping, browser step after confirm"
+    vars:
+      user_request: "Set up Release Autopilot auto-archival for project default."
+      max_turns: 16
+      mock_overrides:
+        get-release-autopilot-settings:
+          autoFlagging: { enabled: false, approvalRequired: true }
+          autoReleasing: { enabled: false }
+          version: 1
+        list-release-autopilot-repo-settings: { repos: [], totalCount: 0 }
+        list-release-autopilot-github-repos:
+          installed: false
+          userAuthorized: false
+          repositories: []
+          totalCount: 0
+          githubConnectUrl: "https://app.launchdarkly.com/github-connect?flow=install"
+      mock_ask_question_match: true
+      mock_ask_question_answers:
+        - ["Auto-archival"]
+        - ["default"]
+        - ["Looks good"]
+        - ["Done"]
+    assert:
+      - type: javascript
+        value: file://../release-autopilot-onboarding/assertions.js:noGithubOrMapping
+        metric: no_github_or_mapping
+        weight: 4
+      - type: javascript
+        value: file://../release-autopilot-onboarding/assertions.js:browserStepsAfterConfirm
+        metric: browser_after_confirm
+        weight: 3
+      - type: javascript
+        value: file://../release-autopilot-onboarding/assertions.js:archivalQuestionHasLifecycleLink
+        metric: lifecycle_link_in_question
+        weight: 2
+
+  # ------------------------------------------------------------------
+  # Preview with auto-cleanup picked: no writes, no browser changes asked
+  # for, and the cleanup step is listed with its link.
+  # ------------------------------------------------------------------
+  - providers: ["mcp-auto"]
+    description: "Preview with auto-cleanup: lists the browser step, asks for no browser change"
+    vars:
+      user_request: "Do a dry run: preview what you'd change to set up Release Autopilot auto-flagging and auto-cleanup for acme/demo in project default. Don't change anything."
+      max_turns: 20
+      mock_ask_question_match: true
+      mock_ask_question_answers:
+        - ["Add more"]
+        - ["Auto-flagging", "Auto-cleanup"]
+        - ["LaunchDarkly's"]
+    assert:
+      - type: javascript
+        value: file://../release-autopilot-onboarding/assertions.js:noWrites
+        metric: no_writes
+        weight: 4
+      - type: javascript
+        value: file://../release-autopilot-onboarding/assertions.js:previewListsBrowserSteps
+        metric: preview_lists_browser_steps
+        weight: 4
+
+  # ------------------------------------------------------------------
+  # acme/storefront is mapped with auto-flagging turned off by a repo
+  # override, and the account has it on. Picking auto-flagging must clear
+  # that override, not leave the repo opted out.
+  # ------------------------------------------------------------------
+  - providers: ["mcp"]
+    description: "Picked capability off by a repo override: turns it on for the repo"
+    vars:
+      user_request: "Set up Release Autopilot auto-flagging for acme/storefront in project default. I don't want auto-releasing."
+      max_turns: 22
+      mock_overrides:
+        list-release-autopilot-repo-settings:
+          repos:
+            - githubRepoId: 101
+              projectKey: default
+              autoFlagging: { enabled: false, approvalRequired: true, enabledOverride: true }
+              autoReleasing: { enabled: false, enabledOverride: true }
+          totalCount: 1
+      mock_ask_question_match: true
+      mock_ask_question_answers:
+        - ["Add more"]
+        - ["Auto-flagging"]
+        - ["LaunchDarkly's"]
+        - ["Looks good"]
+    assert:
+      - type: javascript
+        value: file://../release-autopilot-onboarding/assertions.js:storefrontAutoFlaggingOn
+        metric: clears_repo_override
+        weight: 4
+      - type: javascript
+        value: file://../release-autopilot-onboarding/assertions.js:confirmsBeforeWriting
+        metric: confirms_before_writing
+        weight: 3
diff --git a/skills.json b/skills.json
index 742d1f8..31679eb 100644
--- a/skills.json
+++ b/skills.json
@@ -431,7 +431,7 @@
       "name": "launchdarkly-release-autopilot-onboarding",
       "description": "Step-by-step onboarding wizard for LaunchDarkly Release Autopilot: pick capabilities (auto-flagging, auto-releasing, auto-cleanup, auto-archival), connect the GitHub App, map repositories to projects, choose which AI model account runs the agents (LaunchDarkly's by default, or your own Anthropic or OpenAI key), review release policies (release plans by GA), and turn on scheduled flag cleanup and archival. Use when the user wants to set up, onboard to, preview, or continue setting up Release Autopilot. For a single settings change or 'why didn't my PR get classified?', use launchdarkly-release-autopilot-settings instead.",
       "path": "skills/release-autopilot/launchdarkly-release-autopilot-onboarding",
-      "version": "0.2.0-experimental",
+      "version": "0.3.0-experimental",
       "license": "Apache-2.0",
       "compatibility": "Requires the remotely hosted LaunchDarkly MCP server",
       "tags": [
diff --git a/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/SKILL.md b/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/SKILL.md
index 52f578b..89d2ffb 100644
--- a/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/SKILL.md
+++ b/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/SKILL.md
@@ -5,7 +5,7 @@ license: Apache-2.0
 compatibility: Requires the remotely hosted LaunchDarkly MCP server
 metadata:
   author: launchdarkly
-  version: "0.2.0-experimental"
+  version: "0.3.0-experimental"
 ---
 
 # LaunchDarkly Release Autopilot Onboarding
@@ -44,18 +44,18 @@ Pick the mode from the user's request before Step 0:
 | Mode | Trigger | What changes |
 |------|---------|--------------|
 | **Setup** | "set up", "onboard", "turn on" | The full flow |
-| **Preview** | "preview", "what would you change", "dry run", or a reader (see Step 1) | Run Steps 1–7 and end with the plan as a checklist for an admin. Make no writes |
+| **Preview** | "preview", "what would you change", "dry run", or a reader (see Step 1) | Run Steps 1–7 and end with the plan as a checklist for an admin. Make no writes, and never ask the user to change anything in the browser; list those steps instead |
 | **Resume** | "continue", "pick up where we left off", or a return after a browser step | Skip the intro, re-read live state, and continue at the first unfinished step. Still stops at Step 7 and waits for "Looks good". Details: [references/modes.md](references/modes.md) |
 
 Every mode is safe to run again. A re-run reads live state and proposes only the writes that are still needed.
 
 ## Core Principles
 
-1. **Only set up what they picked.** Release setup matters only for auto-releasing. If the user picks auto-flagging only, skip Step 6.
+1. **Only set up what they picked.** GitHub is needed only for auto-flagging, auto-releasing, and auto-cleanup; repo mapping only for auto-flagging and auto-releasing; release setup only for auto-releasing. Skip every step the picks do not need.
 2. **Least privilege by default.** Recommend auto-flagging, which always waits for a person to approve its proposed change. Auto-releasing is opt-in and you must say it can ship code.
 3. **Gather, confirm, then write.** No write happens before [Step 7](#step-7-confirm-the-plan). The confirmation lists every write with its current and proposed state. This holds in every mode, including Resume and requests that already state every answer: a restated pick is an answer, never approval. Step 1's reads, including the role check, always run first.
 4. **Don't ask what the account already answers.** If live state already settles a question, state the answer and move on.
-5. **Never act in the user's browser.** GitHub install, GitHub authorization, release policy edits, and bring-your-own-key setup are browser steps. Give the exact link and wait for the user to say they finished.
+5. **Never act in the user's browser, and defer browser changes.** GitHub install and authorization come early, because repo mapping needs them. Bring your own key, auto-cleanup, and auto-archival change settings in the browser, so they happen only after "Looks good", in [Step 8](#step-8-apply). Give the exact link and wait for the user to say they finished.
 6. **Only touch what the user named.** Never map every repo on the install. Never create release plans in environments the user did not pick.
 
 ## Speaking to the user
@@ -84,7 +84,7 @@ Decide the mode from the user's first message before you write anything. Never a
 
 > **Release Autopilot Setup**
 >
-> I'll set up Release Autopilot in a few short steps: check your account, pick capabilities, connect GitHub, map repositories, choose whose AI model account runs the agents, then set up releases, flag cleanup, and archival for the capabilities you pick.
+> I'll set up Release Autopilot in a few short steps: check your account, pick capabilities, connect GitHub and map repositories if your picks need them, choose whose AI model account runs the agents, then set up releases, flag cleanup, and archival for the capabilities you pick.
 >
 > Nothing changes until you approve the full plan.
 
@@ -127,8 +127,12 @@ Auto-cleanup and auto-archival are opt-in like auto-releasing: never recommend t
 
 ### Step 3: Connect GitHub
 
+Only if the user picked auto-flagging, auto-releasing, or auto-cleanup. Auto-archival alone does not need GitHub: skip Steps 3 and 4 and choose projects directly (see [Step 4](#step-4-map-repositories)).
+
 Skip this step without mentioning GitHub if Step 1 returned `installed: true`, `userAuthorized: true`, and no `githubConnectUrl`.
 
+**Preview mode:** do not ask the user to install or authorize. List "Connect GitHub" in the Step 7 checklist with the link, and continue with what the reads show.
+
 Otherwise read the URL's `flow`:
 
 - `flow=install`: the LaunchDarkly GitHub App is not installed on their GitHub organization. None of these capabilities can see a pull request until it is.
@@ -155,26 +159,25 @@ Never go on to Step 4 while `list-release-autopilot-github-repos` still returns
 
 ### Step 4: Map repositories
 
+Only if the user picked auto-flagging or auto-releasing. If they picked only auto-cleanup or auto-archival, do not map repos: ask which projects to set up (header "Projects", `multiSelect: true`, from `list-projects`, or ask for the keys) and go to Step 5.
+
 1. **Choose repos.** If the current workspace has a `git remote` on the install list, offer it first. Use `AskUserQuestion` with header "Repos", `multiSelect: true`, and up to four repos from the install list, as `owner/name`. If the install has more than four repos, say the user can type any other `owner/name` from the list in Other. Use only repos that came back from `list-release-autopilot-github-repos`.
 
    If a repo the user wants is not on the list, its owner decides the fix: a listed owner means the App needs access to that repo, and a new owner means an org owner must install the App there (no `githubConnectUrl` appears for this case). See [references/edge-cases.md](references/edge-cases.md). Wait for "Done" and list again.
 2. **Choose a project for each repo.** Each repo maps to one LaunchDarkly project, which is where auto-flagging creates flags and metrics for that repo. If `list-projects` is available, offer the projects as options. Otherwise ask for the project key.
-3. **Check existing mappings** with the join from Step 1 (an unmapped repo returns 404, which is expected). Already mapped to the chosen project with matching capabilities: write nothing. Mapped to the chosen project with an unpicked capability on: record an override that turns it off. Mapped to a different project: never remap here; send the user to the settings skill to unmap first. Details: [references/edge-cases.md](references/edge-cases.md#existing-mappings).
+3. **Check existing mappings** with the join from Step 1 (an unmapped repo returns 404, which is expected). Already mapped to the chosen project with matching capabilities: write nothing. Mapped to the chosen project with an unpicked capability on: record an override that turns it off. Mapped to the chosen project with a picked capability effectively off because of a repo override: record a repo write that turns it on (`{ "enabled": true }`), plus the account write if the account has it off, and say so in Step 7. Mapped to a different project: never remap here; send the user to the settings skill to unmap first. Details: [references/edge-cases.md](references/edge-cases.md#existing-mappings).
 4. **Say what starts when.** If the account already has a capability on, mapping a repo starts that automation in the repo as soon as Step 8 maps it.
 
 ### Step 5: Choose the AI model account
 
+Skip this step if the user picked only auto-archival; it runs no agents.
+
 The agents behind these capabilities (Vega) use an AI model to read pull requests and write code changes. This step decides whose model account they use; the product calls it "inference". Most users keep the default. Before asking, tell the user in one sentence what the choice means, for example: "The agents behind auto-flagging use an AI model to read your pull requests. By default they run on LaunchDarkly's model account; you can use your own Anthropic or OpenAI key instead." Then use `AskUserQuestion` with header "AI model":
 
 - label: "LaunchDarkly's (default)", description: "The agents use an AI model that LaunchDarkly provides and pays for. No setup"
 - label: "Bring your own key", description: "The agents use your own Anthropic or OpenAI API key, for example to keep usage on your provider contract. Your provider bills you for usage"
 
-For "Bring your own key", this is a browser step (no MCP tool reads or writes these settings). For each project, follow [references/byok.md](references/byok.md), then ask header "AI model":
-
-- label: "Done", description: "I saved my key in the Vega settings for every project"
-- label: "Use LaunchDarkly's for now", description: "Skip this. I'll switch to my own key later"
-
-Record the answer as reported by the user, not as verified.
+For "Bring your own key", record the choice only. It is a browser step that changes settings, so it happens after "Looks good", in [Step 8](#step-8-apply).
 
 ### Step 6: Set up releases and cleanup
 
@@ -215,27 +218,11 @@ To create a plan, follow [references/release-plans.md](references/release-plans.
 
 #### Auto-cleanup
 
-Only if the user picked auto-cleanup.
-
-
-Auto-cleanup is the per-project **Automated flag cleanup** setting, on the Lifecycle settings page (or the Vega settings page, for accounts where Vega has its own project settings). The MCP settings tools do not manage it yet, so never send `autoCleanup`; this is a browser step. If a later version of `get-release-autopilot-settings` returns `autoCleanup` and the update tool accepts it, set it there instead. For each project from Step 4, the link is exactly `/projects/{projectKey}/settings/lifecycle` on their LaunchDarkly instance (or `/projects/{projectKey}/settings/vega` if that page has no "Automated flag cleanup" section). Follow [references/auto-cleanup.md](references/auto-cleanup.md), then ask header "Auto-cleanup", with the link in the question text:
-
-- label: "Done", description: "I turned on automated flag cleanup for every project"
-- label: "Later", description: "Skip this. I'll turn it on from Lifecycle settings later"
-
-Record the answer as reported by the user, and list "Done" under browser steps in Step 7.
+Only if the user picked auto-cleanup. Record it for each project. It is a browser step that changes settings, so it happens after "Looks good", in [Step 8](#step-8-apply). Never send `autoCleanup`.
 
 #### Auto-archival
 
-Only if the user picked auto-archival.
-
-
-Auto-archival is the per-project **Automatic archival** setting on the Lifecycle settings page. No MCP tool manages it, so this is a browser step. For each project from Step 4, the link is exactly `/projects/{projectKey}/settings/lifecycle` on their LaunchDarkly instance. Follow [references/auto-archival.md](references/auto-archival.md), then ask header "Auto-archival", with the link in the question text:
-
-- label: "Done", description: "I turned on automatic archival for every project"
-- label: "Later", description: "Skip this, or the section isn't there yet. I'll turn it on from Lifecycle settings later"
-
-Record the answer as reported by the user, and list "Done" under browser steps in Step 7.
+Only if the user picked auto-archival. Record it for each project. It is a browser step that changes settings, so it happens after "Looks good", in [Step 8](#step-8-apply).
 
 ### Step 7: Confirm the plan
 
@@ -247,16 +234,17 @@ List every write in the order you will make it. For each write, state the curren
 2. **Repo mappings and overrides**: one line per repo, naming the project and the **effective** auto-flagging and auto-releasing state after every write in this plan. For a capability the user did not pick, or that cannot work yet, the mapping sends a repo override that turns it off. Say so: "Auto-releasing: off for this repository (the account has it on; this project has no release policy yet)."
 3. **Account settings**: only fields that change. If auto-releasing turns on, say it can ship code.
    For a member who is not an admin or owner, list account writes under "Needs an admin" instead, and say an admin can make them with the settings skill. Do not name a settings page; there is none to point to.
-4. **Browser steps the user reported done**: GitHub, the AI model choice, auto-cleanup, and auto-archival, marked as reported, not verified.
+4. **Browser steps after these changes**: bring your own key, auto-cleanup, and auto-archival, one line per project with its link. GitHub, if the user connected it in Step 3, is marked as reported, not verified.
 5. **Follow-ups for the user**: release policies to add, metric events to instrument.
 
-If there are no writes, say so, list the follow-ups, and stop.
+If there are no writes and no browser steps, say so, list the follow-ups, and stop.
 
 **Preview mode:** the final message contains only the heading "Changes an admin would make", the checklist (one line per change), and any follow-ups. Do not repeat the account state, and do not ask for confirmation. For example:
 
 ```
 - [ ] Map acme/demo to project default, with auto-releasing off for this repository
 - [ ] Turn on auto-flagging for the account
+- [ ] In the browser: turn on automated flag cleanup at /projects/default/settings/lifecycle
 ```
 
 **Setup mode:** use `AskUserQuestion` with header "Confirm":
@@ -279,10 +267,18 @@ The order matters. When the plan creates release plans, they come first, so that
 
 If a write fails with 403, the member lacks that permission. Report which write failed, list the remaining writes as a checklist for an admin, and stop.
 
+**Browser steps.** After the writes, walk the user through each browser step from Step 7, one at a time, with the link in the question text:
+
+- **Bring your own key:** for each project, `/projects/{projectKey}/settings/vega`. Follow [references/byok.md](references/byok.md). Ask header "AI model": "Done" ("I saved my key in the Vega settings for every project") or "Use LaunchDarkly's for now" ("Skip this. I'll switch to my own key later").
+- **Auto-cleanup:** for each project, exactly `/projects/{projectKey}/settings/lifecycle` (or `/projects/{projectKey}/settings/vega` if that page has no "Automated flag cleanup" section). Follow [references/auto-cleanup.md](references/auto-cleanup.md). Ask header "Auto-cleanup": "Done" ("I turned on automated flag cleanup for every project") or "Later" ("Skip this. I'll turn it on from Lifecycle settings later").
+- **Auto-archival:** for each project, exactly `/projects/{projectKey}/settings/lifecycle`. Follow [references/auto-archival.md](references/auto-archival.md). Ask header "Auto-archival": "Done" ("I turned on automatic archival for every project") or "Later" ("Skip this, or the section isn't there yet. I'll turn it on from Lifecycle settings later").
+
+Record each answer as reported by the user. Never do these in Preview mode.
+
 ### Step 9: Verify
 
 1. `get-release-autopilot-settings`: the account gates match the plan.
-2. `get-release-autopilot-repo-settings` for each repo: mapped to the right project, effective auto-flagging and auto-releasing match the plan.
+2. `get-release-autopilot-repo-settings` for each repo: mapped to the right project, effective auto-flagging and auto-releasing match the plan. If a picked capability is still effectively off for a repo, say which and do not call the setup complete.
 3. With release plans: `list-release-plans` for each environment lists the new plans. With release policies: report the policy state from Step 6.
 
 If any change was left for an admin or failed, lead with "Partly set up: waiting on an admin" and the admin checklist, and never call the setup complete. Then report a short checklist with each item marked verified, reported by the user (browser steps the user did in this flow), or already set up (steps the flow skipped because they were already done), then the follow-ups. Tell the user how to see it work; if a change is waiting on an admin, make that conditional ("Once an admin turns on auto-flagging, open a pull request…"):
@@ -307,7 +303,7 @@ More cases: [references/edge-cases.md](references/edge-cases.md).
 - Don't write before Step 7's "Looks good", or at all in Preview mode.
 - Don't turn on auto-releasing unless the user picked it, and never send `approvalRequired`.
 - Don't map unpicked repos, iterate over the install list, or remap a repo to another project.
-- Don't act in the user's browser, change release policies, or report a browser step as verified.
+- Don't act in the user's browser, change release policies, or report a browser step as verified. Don't ask for a browser change before "Looks good" or in Preview mode.
 
 ## References
 
diff --git a/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/marketplace.json b/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/marketplace.json
index 32071da..4fd2a94 100644
--- a/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/marketplace.json
+++ b/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/marketplace.json
@@ -1,7 +1,7 @@
 {
   "name": "launchdarkly-release-autopilot-onboarding",
   "description": "Step-by-step Release Autopilot onboarding: pick capabilities, connect GitHub, map repos, choose inference, review release setup",
-  "version": "0.2.0-experimental",
+  "version": "0.3.0-experimental",
   "author": "LaunchDarkly",
   "repository": "https://github.com/launchdarkly/ai-tooling",
   "skills": ["./"],
diff --git a/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/references/edge-cases.md b/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/references/edge-cases.md
index 47cae1d..d26e6b1 100644
--- a/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/references/edge-cases.md
+++ b/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/references/edge-cases.md
@@ -25,4 +25,5 @@
 |-----------------|--------|
 | Mapped to the chosen project, effective capabilities match the user's picks | Tell the user it is already set up. Write nothing |
 | Mapped to the chosen project, a capability the user did not pick is effectively on | Record a repo override that turns it off, as its own line in Step 7 |
+| Mapped to the chosen project, a picked capability is effectively off because of a repo override | Record a repo write that turns it on, plus the account write if the account has it off. Verify the effective value in Step 9 |
 | Mapped to a different project | Do not remap. Remapping changes where its flags are created. The user unmaps it with [`launchdarkly-release-autopilot-settings`](../../launchdarkly-release-autopilot-settings/SKILL.md), then maps it in this flow |
diff --git a/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/references/modes.md b/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/references/modes.md
index e15b50e..b4b37fc 100644
--- a/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/references/modes.md
+++ b/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/references/modes.md
@@ -2,7 +2,7 @@
 
 ## Preview
 
-- Run Steps 1–7 with live reads, and make no writes.
+- Run Steps 1–7 with live reads, and make no writes. Never ask the user to install GitHub or change a setting in the browser; list those steps in the checklist.
 - In Step 1, leave out the "Change one setting" option, because that hand-off writes.
 - In Step 7, show the plan as a checklist under "Changes an admin would make", one line per change, and stop without asking for confirmation.
 - A member whose role is `reader` or `no_access` is always in Preview.

From 56b7323bd5952d948383783542d58014efd91165 Mon Sep 17 00:00:00 2001
From: Farhan Sareshwala <fsareshwala@launchdarkly.com>
Date: Fri, 30 Oct 2026 15:01:51 -0700
Subject: [PATCH 2/3] fix(release-autopilot-onboarding): close gaps from a
 recheck of the #229 review

Step 8 now sends enabled:true for a picked capability a repo override
kept off (unless it cannot work yet), and Step 7 describes that write.
Preview with GitHub disconnected lists mapping as a later step instead of
contradicting the no-mapping-without-GitHub rule. Resume re-lists pending
browser steps after the confirmation. Evals detect browser-step and
repo-picker questions by content, not only by link, require the
override case to read the repo back, and add a bring-your-own-key case
(24 total).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 CHANGELOG.md                                  |  2 +-
 .../assertions.js                             | 28 +++++++++++----
 .../promptfooconfig.yaml                      | 35 +++++++++++++++++++
 .../SKILL.md                                  |  8 ++---
 .../references/modes.md                       |  1 +
 5 files changed, 63 insertions(+), 11 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4c6ab5a..99dc6f8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,7 +4,7 @@ All notable changes to this repository will be documented in this file.
 
 ## Unreleased
 
-- `release-autopilot/launchdarkly-release-autopilot-onboarding` 0.3.0-experimental: browser steps that change settings (bring your own key, auto-cleanup, auto-archival) now happen only after "Looks good", and Preview lists them instead of asking for them; GitHub and repo mapping run only for the capabilities that need them (auto-archival alone needs neither); a picked capability turned off by a repo override is turned back on and verified. Evals require the exact project on mappings, the policy lookup path, and the repo write in plan mode, and add archival-only, preview-with-cleanup, and repo-override cases (23 total).
+- `release-autopilot/launchdarkly-release-autopilot-onboarding` 0.3.0-experimental: browser steps that change settings (bring your own key, auto-cleanup, auto-archival) now happen only after "Looks good", and Preview lists them instead of asking for them; GitHub and repo mapping run only for the capabilities that need them (auto-archival alone needs neither); a picked capability turned off by a repo override is turned back on and verified. Evals require the exact project on mappings, the policy lookup path, and the repo write in plan mode, and add archival-only, preview-with-cleanup, repo-override, and bring-your-own-key cases (24 total).
 - Add `release-autopilot/launchdarkly-release-autopilot-onboarding` 0.2.0-experimental: a step-by-step wizard with setup, preview, and resume modes that asks which capabilities to set up (auto-flagging, auto-releasing, auto-cleanup, auto-archival), hands off the GitHub App install or authorize link, maps chosen repos, guides managed or bring-your-own-key Vega inference, and reviews or creates environment release plans for auto-releasing, or uses the project's release policies where plans are not available. Release policies are required for auto-releasing today; the skill will be updated to use environment release plans by GA, when plans replace policies. One confirmed plan lists every write before any change. Ships in Claude Code plugin 1.0.3 and Cursor plugin 1.0.4.
 - `release-autopilot/launchdarkly-release-autopilot-settings` 1.4.0-experimental: route first-time setup to the onboarding skill.
 - evals: add the `release-autopilot-onboarding` suite (20 cases: confirm before writing, repo overrides, policy and plan modes, reader and writer roles, preview, resume, GitHub install and authorize paths, deprecated MCP, a failed write, auto-cleanup, auto-archival, a browser signed in to a different account, an account without the add-on, a diagnosis on an empty account). The suite keeps its tool definitions and mocks in its own folder (`tool_definitions`, `tool_mocks`). The provider gains `mock_ask_question_match` with label matching, stateful Release Autopilot mocks, and `CLAUDE_CODE_OAUTH_TOKEN` as an alternative to `ANTHROPIC_API_KEY`.
diff --git a/evals/release-autopilot-onboarding/assertions.js b/evals/release-autopilot-onboarding/assertions.js
index 1882429..eadce81 100644
--- a/evals/release-autopilot-onboarding/assertions.js
+++ b/evals/release-autopilot-onboarding/assertions.js
@@ -364,8 +364,13 @@ module.exports.policyLookupPath = (output) => {
   return result(plans === 1 && policies >= 1, `release-plans lookups: ${plans}, release-policies lookups: ${policies}`);
 };
 
+// A browser step question: a "Done" option and a settings topic (by link or by
+// words), so a pre-confirm question without the right link is still caught.
+const BROWSER_TOPIC = /settings\/(lifecycle|vega)|lifecycle settings|vega settings|own key|api key|flag cleanup|automatic archival|auto-archival|auto-cleanup/i;
 const isBrowserStepQuestion = (t) => t.tool === "ask-question" &&
-  /\/projects\/[^/\s]+\/settings\/(lifecycle|vega)/.test(String((t.arguments && t.arguments.prompt) || ""));
+  ((t.arguments && t.arguments.options) || []).some((o) => /^done\b/i.test(String(o.label || ""))) &&
+  BROWSER_TOPIC.test(String((t.arguments && t.arguments.prompt) || "")) &&
+  !/github/i.test(String((t.arguments && t.arguments.prompt) || ""));
 
 // Browser steps that change settings come only after "Looks good".
 module.exports.browserStepsAfterConfirm = (output) => {
@@ -388,12 +393,23 @@ module.exports.previewListsBrowserSteps = (output) => {
 module.exports.noGithubOrMapping = (output) => {
   const steps = traj(output);
   const repoWrites = steps.filter((t) => /^(update|delete)-release-autopilot-repo-settings$/.test(t.tool)).length;
-  const ghQ = steps.filter((t) => t.tool === "ask-question" && /github|repo/i.test(String((t.arguments && t.arguments.prompt) || ""))).length;
-  return result(repoWrites === 0 && ghQ === 0, `repo writes: ${repoWrites}; GitHub or repo questions: ${ghQ}`);
+  const ghQ = steps.filter((t) => t.tool === "ask-question" && (
+    /github/i.test(String((t.arguments && t.arguments.prompt) || "")) ||
+    ((t.arguments && t.arguments.options) || []).some((o) => /^[\w.-]+\/[\w.-]+$/.test(String(o.label || "")))
+  )).length;
+  return result(repoWrites === 0 && ghQ === 0, `repo writes: ${repoWrites}; GitHub or repo-picker questions: ${ghQ}`);
 };
 
 module.exports.storefrontAutoFlaggingOn = (output) => {
-  const maps = traj(output).filter((t) => t.tool === "update-release-autopilot-repo-settings" && (t.arguments || {}).repo === "acme/storefront");
-  const on = maps.some((m) => m.arguments.autoFlagging && m.arguments.autoFlagging.enabled === true);
-  return result(on, on ? "turned auto-flagging on for acme/storefront" : "did not clear the repo's auto-flagging override");
+  const steps = traj(output);
+  const w = steps.findIndex((t) => t.tool === "update-release-autopilot-repo-settings" && (t.arguments || {}).repo === "acme/storefront" &&
+    t.arguments.autoFlagging && t.arguments.autoFlagging.enabled === true);
+  if (w === -1) return result(false, "did not clear the repo's auto-flagging override");
+  const verified = steps.slice(w + 1).some((t) => t.tool === "get-release-autopilot-repo-settings" && (t.arguments || {}).repo === "acme/storefront");
+  return result(verified, verified ? "turned auto-flagging on for acme/storefront and read it back" : "turned it on but never verified the repo's effective value");
+};
+
+module.exports.byokQuestionHasVegaLink = (output) => {
+  const pass = askPrompts(output).some((p) => /\/projects\/[^/\s]+\/settings\/vega/.test(p));
+  return result(pass, pass ? "bring-your-own-key question contains the Vega settings link" : "no question contains /projects/{key}/settings/vega");
 };
diff --git a/evals/release-autopilot-onboarding/promptfooconfig.yaml b/evals/release-autopilot-onboarding/promptfooconfig.yaml
index bf36f75..96ebb9c 100644
--- a/evals/release-autopilot-onboarding/promptfooconfig.yaml
+++ b/evals/release-autopilot-onboarding/promptfooconfig.yaml
@@ -944,3 +944,38 @@ tests:
         value: file://../release-autopilot-onboarding/assertions.js:confirmsBeforeWriting
         metric: confirms_before_writing
         weight: 3
+
+  # ------------------------------------------------------------------
+  # Bring your own key is a browser change: it comes after "Looks good",
+  # with the project's Vega settings link in the question.
+  # ------------------------------------------------------------------
+  - providers: ["mcp"]
+    description: "Bring your own key: Vega settings step comes after confirm"
+    vars:
+      user_request: "Set up Release Autopilot auto-flagging for acme/demo in project default, using our own Anthropic key."
+      max_turns: 22
+      mock_overrides:
+        get-release-autopilot-settings:
+          autoFlagging: { enabled: false, approvalRequired: true }
+          autoReleasing: { enabled: false }
+          version: 1
+        list-release-autopilot-repo-settings: { repos: [], totalCount: 0 }
+      mock_ask_question_match: true
+      mock_ask_question_answers:
+        - ["Auto-flagging"]
+        - ["Bring your own key"]
+        - ["Looks good"]
+        - ["Done"]
+    assert:
+      - type: javascript
+        value: file://../release-autopilot-onboarding/assertions.js:browserStepsAfterConfirm
+        metric: browser_after_confirm
+        weight: 4
+      - type: javascript
+        value: file://../release-autopilot-onboarding/assertions.js:byokQuestionHasVegaLink
+        metric: vega_link_in_question
+        weight: 3
+      - type: javascript
+        value: file://../release-autopilot-onboarding/assertions.js:repoTurnsAutoReleasingOff
+        metric: repo_override_unpicked
+        weight: 2
diff --git a/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/SKILL.md b/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/SKILL.md
index 89d2ffb..5b8d59c 100644
--- a/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/SKILL.md
+++ b/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/SKILL.md
@@ -131,7 +131,7 @@ Only if the user picked auto-flagging, auto-releasing, or auto-cleanup. Auto-arc
 
 Skip this step without mentioning GitHub if Step 1 returned `installed: true`, `userAuthorized: true`, and no `githubConnectUrl`.
 
-**Preview mode:** do not ask the user to install or authorize. List "Connect GitHub" in the Step 7 checklist with the link, and continue with what the reads show.
+**Preview mode:** do not ask the user to install or authorize. List "Connect GitHub" in the Step 7 checklist with the link. Repo mapping cannot be planned until GitHub is connected, so skip Step 4 and any release setup that depends on it; list "After GitHub is connected: map the repos" in the checklist, and continue with the steps that do not need GitHub.
 
 Otherwise read the URL's `flow`:
 
@@ -165,7 +165,7 @@ Only if the user picked auto-flagging or auto-releasing. If they picked only aut
 
    If a repo the user wants is not on the list, its owner decides the fix: a listed owner means the App needs access to that repo, and a new owner means an org owner must install the App there (no `githubConnectUrl` appears for this case). See [references/edge-cases.md](references/edge-cases.md). Wait for "Done" and list again.
 2. **Choose a project for each repo.** Each repo maps to one LaunchDarkly project, which is where auto-flagging creates flags and metrics for that repo. If `list-projects` is available, offer the projects as options. Otherwise ask for the project key.
-3. **Check existing mappings** with the join from Step 1 (an unmapped repo returns 404, which is expected). Already mapped to the chosen project with matching capabilities: write nothing. Mapped to the chosen project with an unpicked capability on: record an override that turns it off. Mapped to the chosen project with a picked capability effectively off because of a repo override: record a repo write that turns it on (`{ "enabled": true }`), plus the account write if the account has it off, and say so in Step 7. Mapped to a different project: never remap here; send the user to the settings skill to unmap first. Details: [references/edge-cases.md](references/edge-cases.md#existing-mappings).
+3. **Check existing mappings** with the join from Step 1 (an unmapped repo returns 404, which is expected). Already mapped to the chosen project with matching capabilities: write nothing. Mapped to the chosen project with an unpicked capability on: record an override that turns it off. Mapped to the chosen project with a picked capability effectively off because of a repo override: record a repo write that turns it on (`{ "enabled": true }`), plus the account write if the account has it off, and say so in Step 7. The exception is a capability that cannot work yet, such as auto-releasing in a project with no complete release policy (Step 6): keep it off. Mapped to a different project: never remap here; send the user to the settings skill to unmap first. Details: [references/edge-cases.md](references/edge-cases.md#existing-mappings).
 4. **Say what starts when.** If the account already has a capability on, mapping a repo starts that automation in the repo as soon as Step 8 maps it.
 
 ### Step 5: Choose the AI model account
@@ -231,7 +231,7 @@ Only if the user picked auto-archival. Record it for each project. It is a brows
 List every write in the order you will make it. For each write, state the current and the proposed state and the blast radius, using the wording rules from the settings skill's [Confirm before write](../launchdarkly-release-autopilot-settings/SKILL.md#confirm-before-write):
 
 1. **Metrics and release plans** (release plans only): one line per metric to create and per environment, with the plan name, steps, advancement, and metrics.
-2. **Repo mappings and overrides**: one line per repo, naming the project and the **effective** auto-flagging and auto-releasing state after every write in this plan. For a capability the user did not pick, or that cannot work yet, the mapping sends a repo override that turns it off. Say so: "Auto-releasing: off for this repository (the account has it on; this project has no release policy yet)."
+2. **Repo mappings and overrides**: one line per repo, naming the project and the **effective** auto-flagging and auto-releasing state after every write in this plan. For a capability the user did not pick, or that cannot work yet, the mapping sends a repo override that turns it off. Say so: "Auto-releasing: off for this repository (the account has it on; this project has no release policy yet)." For a picked capability that a repo override kept off, say the write turns it back on: "Auto-flagging: off → on for this repository (it was turned off for this repository only)."
 3. **Account settings**: only fields that change. If auto-releasing turns on, say it can ship code.
    For a member who is not an admin or owner, list account writes under "Needs an admin" instead, and say an admin can make them with the settings skill. Do not name a settings page; there is none to point to.
 4. **Browser steps after these changes**: bring your own key, auto-cleanup, and auto-archival, one line per project with its link. GitHub, if the user connected it in Step 3, is marked as reported, not verified.
@@ -260,7 +260,7 @@ Make only the writes in the confirmed plan, in this order. Stop at the first fai
 
 1. Release plans only: metrics recorded in Step 6, created in LaunchDarkly with [`launchdarkly-metric-create`](../../metrics/launchdarkly-metric-create/SKILL.md). Skip its instrumentation steps.
 2. Release plans only: `create-release-plan` for each new plan.
-3. `update-release-autopilot-repo-settings` for each repo, with `repo: "owner/name"` and `projectKey`. For each capability the user did not pick, or that cannot work yet, send `{ "enabled": false }` for it, so the repo does not inherit it from the account.
+3. `update-release-autopilot-repo-settings` for each repo, with `repo: "owner/name"` and `projectKey`. For each capability the user did not pick, or that cannot work yet, send `{ "enabled": false }` for it, so the repo does not inherit it from the account. For a picked capability that a repo override keeps off, send `{ "enabled": true }` for it.
 4. `update-release-autopilot-settings` with only the confirmed fields. Never send `autoCleanup`.
 
 The order matters. When the plan creates release plans, they come first, so that no release can start without the plan the user chose. The account gates come last because turning a capability on starts automation in every mapped repo that inherits it.
diff --git a/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/references/modes.md b/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/references/modes.md
index b4b37fc..f991748 100644
--- a/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/references/modes.md
+++ b/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/references/modes.md
@@ -13,6 +13,7 @@
 - Skip Step 0 entirely. Re-read live state in Step 1, and say in one sentence what is already done.
 - Use the earlier picks if they are still in the conversation. If not, ask them again with live state as the default.
 - Writes that landed before a Step 8 failure show as already set up, so the new plan lists only what is left.
+- Browser steps that were still pending when the flow paused (bring your own key, auto-cleanup, auto-archival) come after the confirmation again: list them in the new plan and walk through them in Step 8.
 - A restated pick is an answer, never approval. Resume still stops at Step 7 and waits for "Looks good".
 
 Every mode is safe to run again: a re-run reads live state and proposes only the writes that are still needed.

From c6e67a443f4ae84011f59072e2b00ad7a280f108 Mon Sep 17 00:00:00 2001
From: Farhan Sareshwala <fsareshwala@launchdarkly.com>
Date: Fri, 30 Oct 2026 16:09:36 -0700
Subject: [PATCH 3/3] fix(release-autopilot-onboarding): address second review
 round on #237

- Browser steps count as approved only after a 'Looks good' answer, not
  after the confirmation question is merely shown.
- Release setup lookups must name the requested project and one of its
  environments; the check accepts either path, release policies (403
  then a policy lookup for the project) or environment release plans, so
  it keeps working at GA.
- Preview with GitHub disconnected and auto-releasing picked also lists
  the deferred release setup (environments and release policies or
  plans) after repo mapping. New eval case (25 total).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 CHANGELOG.md                                  |  2 +-
 .../assertions.js                             | 54 +++++++++++++++----
 .../promptfooconfig.yaml                      | 44 ++++++++++++++-
 .../SKILL.md                                  |  2 +-
 4 files changed, 88 insertions(+), 14 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 99dc6f8..3499a6a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,7 +4,7 @@ All notable changes to this repository will be documented in this file.
 
 ## Unreleased
 
-- `release-autopilot/launchdarkly-release-autopilot-onboarding` 0.3.0-experimental: browser steps that change settings (bring your own key, auto-cleanup, auto-archival) now happen only after "Looks good", and Preview lists them instead of asking for them; GitHub and repo mapping run only for the capabilities that need them (auto-archival alone needs neither); a picked capability turned off by a repo override is turned back on and verified. Evals require the exact project on mappings, the policy lookup path, and the repo write in plan mode, and add archival-only, preview-with-cleanup, repo-override, and bring-your-own-key cases (24 total).
+- `release-autopilot/launchdarkly-release-autopilot-onboarding` 0.3.0-experimental: browser steps that change settings (bring your own key, auto-cleanup, auto-archival) now happen only after "Looks good", and Preview lists them instead of asking for them; GitHub and repo mapping run only for the capabilities that need them (auto-archival alone needs neither); a picked capability turned off by a repo override is turned back on and verified. Evals require the exact project on mappings, the policy lookup path, and the repo write in plan mode, and add archival-only, preview-with-cleanup, repo-override, bring-your-own-key, and preview-without-GitHub cases (25 total); release setup lookups are checked for the right project and environments in both release policy and release plan modes.
 - Add `release-autopilot/launchdarkly-release-autopilot-onboarding` 0.2.0-experimental: a step-by-step wizard with setup, preview, and resume modes that asks which capabilities to set up (auto-flagging, auto-releasing, auto-cleanup, auto-archival), hands off the GitHub App install or authorize link, maps chosen repos, guides managed or bring-your-own-key Vega inference, and reviews or creates environment release plans for auto-releasing, or uses the project's release policies where plans are not available. Release policies are required for auto-releasing today; the skill will be updated to use environment release plans by GA, when plans replace policies. One confirmed plan lists every write before any change. Ships in Claude Code plugin 1.0.3 and Cursor plugin 1.0.4.
 - `release-autopilot/launchdarkly-release-autopilot-settings` 1.4.0-experimental: route first-time setup to the onboarding skill.
 - evals: add the `release-autopilot-onboarding` suite (20 cases: confirm before writing, repo overrides, policy and plan modes, reader and writer roles, preview, resume, GitHub install and authorize paths, deprecated MCP, a failed write, auto-cleanup, auto-archival, a browser signed in to a different account, an account without the add-on, a diagnosis on an empty account). The suite keeps its tool definitions and mocks in its own folder (`tool_definitions`, `tool_mocks`). The provider gains `mock_ask_question_match` with label matching, stateful Release Autopilot mocks, and `CLAUDE_CODE_OAUTH_TOKEN` as an alternative to `ANTHROPIC_API_KEY`.
diff --git a/evals/release-autopilot-onboarding/assertions.js b/evals/release-autopilot-onboarding/assertions.js
index eadce81..442be69 100644
--- a/evals/release-autopilot-onboarding/assertions.js
+++ b/evals/release-autopilot-onboarding/assertions.js
@@ -21,6 +21,20 @@ function isConfirmQuestion(step) {
   return labels.some((l) => l.includes("looks good"));
 }
 
+// Index of the last confirm question before `limit` whose answer was "Looks good",
+// or -1. Showing the confirm question is not approval; only that answer is.
+function approvedConfirmBefore(steps, limit) {
+  for (let i = limit - 1; i >= 0; i--) {
+    const step = steps[i];
+    if (!isConfirmQuestion(step)) continue;
+    const ok = ((step.arguments && step.arguments.options) || [])
+      .filter((o) => String(o.label || "").toLowerCase().includes("looks good"))
+      .map((o) => o.id);
+    if ((step.selected || []).some((id) => ok.includes(id))) return i;
+  }
+  return -1;
+}
+
 module.exports.noWrites = (output) => {
   const w = writes(output);
   return result(w.length === 0, w.length ? `wrote: ${w.map((t) => t.tool).join(", ")}` : "no writes");
@@ -356,13 +370,25 @@ module.exports.pausesBeforeMapping = (output) => {
   return result(pass, `repo questions: ${repoQ.length}; resume phrase in final message: ${resume}`);
 };
 
-// Policy mode: exactly one release-plans lookup (the 403 is account-wide) and
-// at least one release-policies lookup.
-module.exports.policyLookupPath = (output) => {
-  const plans = traj(output).filter((t) => t.tool === "list-release-plans").length;
-  const policies = traj(output).filter((t) => t.tool === "list-release-policies").length;
-  return result(plans === 1 && policies >= 1, `release-plans lookups: ${plans}, release-policies lookups: ${policies}`);
-};
+// Release setup lookups for one project, valid for both modes. Every
+// release-plans call names the project and one of its environments. If the
+// first call is the account-wide 403, there is exactly one plans call and a
+// release-policies lookup for the project (release policies, today). If it
+// succeeds, no policy lookup is required (environment release plans, at GA).
+const releaseSetupLookups = (projectKey, envs) => (output) => {
+  const steps = traj(output);
+  const plans = steps.filter((t) => t.tool === "list-release-plans");
+  if (!plans.length) return result(false, "never checked release plans");
+  const badPlan = plans.find((t) => (t.arguments || {}).projectKey !== projectKey || !envs.includes((t.arguments || {}).environmentKey));
+  if (badPlan) return result(false, `release-plans lookup with ${JSON.stringify(badPlan.arguments)}`);
+  const forbidden = /403|not allow/i.test(String(plans[0].mock_response_preview || ""));
+  if (!forbidden) return result(true, `release plans available; ${plans.length} lookup(s) for ${projectKey}`);
+  const policies = steps.filter((t) => t.tool === "list-release-policies");
+  const okPolicies = policies.filter((t) => (t.arguments || {}).projectKey === projectKey);
+  const pass = plans.length === 1 && okPolicies.length >= 1 && okPolicies.length === policies.length;
+  return result(pass, `release-plans lookups: ${plans.length}; release-policies lookups for ${projectKey}: ${okPolicies.length} of ${policies.length}`);
+};
+module.exports.releaseSetupLookupsDefault = releaseSetupLookups("default", ["test", "production"]);
 
 // A browser step question: a "Done" option and a settings topic (by link or by
 // words), so a pre-confirm question without the right link is still caught.
@@ -377,9 +403,8 @@ module.exports.browserStepsAfterConfirm = (output) => {
   const steps = traj(output);
   const browser = steps.findIndex(isBrowserStepQuestion);
   if (browser === -1) return result(false, "no browser step question");
-  const confirm = steps.findIndex(isConfirmQuestion);
-  const pass = confirm !== -1 && confirm < browser;
-  return result(pass, `confirm at ${confirm}, first browser step at ${browser}`);
+  const approved = approvedConfirmBefore(steps, browser);
+  return result(approved !== -1, approved !== -1 ? `approved at ${approved}, first browser step at ${browser}` : `browser step at ${browser} without a "Looks good" answer before it`);
 };
 
 // Preview never asks the user to change something in the browser, and lists it.
@@ -413,3 +438,12 @@ module.exports.byokQuestionHasVegaLink = (output) => {
   const pass = askPrompts(output).some((p) => /\/projects\/[^/\s]+\/settings\/vega/.test(p));
   return result(pass, pass ? "bring-your-own-key question contains the Vega settings link" : "no question contains /projects/{key}/settings/vega");
 };
+
+// Preview with GitHub disconnected and auto-releasing picked lists the
+// deferred release setup, not only repo mapping.
+module.exports.previewListsDeferredReleaseSetup = (output) => {
+  const text = finalText(output).toLowerCase();
+  const mapping = /map/.test(text) && /github/.test(text);
+  const release = /environment/.test(text) && /release (polic|plan)/.test(text);
+  return result(mapping && release, `lists mapping after GitHub: ${mapping}; lists environments and release policies/plans: ${release}`);
+};
diff --git a/evals/release-autopilot-onboarding/promptfooconfig.yaml b/evals/release-autopilot-onboarding/promptfooconfig.yaml
index 96ebb9c..641cd41 100644
--- a/evals/release-autopilot-onboarding/promptfooconfig.yaml
+++ b/evals/release-autopilot-onboarding/promptfooconfig.yaml
@@ -150,8 +150,8 @@ tests:
         metric: no_unpicked_archival
         weight: 3
       - type: javascript
-        value: file://../release-autopilot-onboarding/assertions.js:policyLookupPath
-        metric: plans_403_account_wide
+        value: file://../release-autopilot-onboarding/assertions.js:releaseSetupLookupsDefault
+        metric: release_setup_lookups
         weight: 3
       - type: javascript
         value: file://../release-autopilot-onboarding/assertions.js:skippedGithubNotReported
@@ -435,6 +435,10 @@ tests:
         - ["Leave rules"]
         - ["Looks good"]
     assert:
+      - type: javascript
+        value: file://../release-autopilot-onboarding/assertions.js:releaseSetupLookupsDefault
+        metric: release_setup_lookups
+        weight: 3
       - type: javascript
         value: file://../release-autopilot-onboarding/assertions.js:noCleanupQuestion
         metric: no_unpicked_cleanup
@@ -979,3 +983,39 @@ tests:
         value: file://../release-autopilot-onboarding/assertions.js:repoTurnsAutoReleasingOff
         metric: repo_override_unpicked
         weight: 2
+
+  # ------------------------------------------------------------------
+  # Preview, GitHub disconnected, auto-releasing picked: the checklist
+  # lists GitHub, the later repo mapping, and the later release setup
+  # (environments and release policies or plans). No writes.
+  # ------------------------------------------------------------------
+  - providers: ["mcp-auto"]
+    description: "Preview, GitHub disconnected, auto-releasing: lists deferred mapping and release setup"
+    vars:
+      user_request: "Do a dry run: preview what you'd change to set up Release Autopilot auto-flagging and auto-releasing for project default. Don't change anything."
+      max_turns: 18
+      mock_overrides:
+        get-release-autopilot-settings:
+          autoFlagging: { enabled: false, approvalRequired: true }
+          autoReleasing: { enabled: false }
+          version: 1
+        list-release-autopilot-repo-settings: { repos: [], totalCount: 0 }
+        list-release-autopilot-github-repos:
+          installed: false
+          userAuthorized: false
+          repositories: []
+          totalCount: 0
+          githubConnectUrl: "https://app.launchdarkly.com/github-connect?flow=install"
+      mock_ask_question_match: true
+      mock_ask_question_answers:
+        - ["Auto-flagging", "Auto-releasing"]
+        - ["LaunchDarkly's"]
+    assert:
+      - type: javascript
+        value: file://../release-autopilot-onboarding/assertions.js:noWrites
+        metric: no_writes
+        weight: 4
+      - type: javascript
+        value: file://../release-autopilot-onboarding/assertions.js:previewListsDeferredReleaseSetup
+        metric: deferred_release_setup
+        weight: 4
diff --git a/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/SKILL.md b/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/SKILL.md
index 5b8d59c..940df50 100644
--- a/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/SKILL.md
+++ b/skills/release-autopilot/launchdarkly-release-autopilot-onboarding/SKILL.md
@@ -131,7 +131,7 @@ Only if the user picked auto-flagging, auto-releasing, or auto-cleanup. Auto-arc
 
 Skip this step without mentioning GitHub if Step 1 returned `installed: true`, `userAuthorized: true`, and no `githubConnectUrl`.
 
-**Preview mode:** do not ask the user to install or authorize. List "Connect GitHub" in the Step 7 checklist with the link. Repo mapping cannot be planned until GitHub is connected, so skip Step 4 and any release setup that depends on it; list "After GitHub is connected: map the repos" in the checklist, and continue with the steps that do not need GitHub.
+**Preview mode:** do not ask the user to install or authorize. List "Connect GitHub" in the Step 7 checklist with the link. Repo mapping cannot be planned until GitHub is connected, so skip Step 4 and any release setup that depends on it; list "After GitHub is connected: map the repos" in the checklist. If the user picked auto-releasing, also list "After mapping: choose environments and check the release policies (or release plans) for project {projectKey}", because auto-releasing needs that before it can work. Continue with the steps that do not need GitHub.
 
 Otherwise read the URL's `flow`: