diff --git a/docs/setup/marketplace-install.md b/docs/setup/marketplace-install.md index f3ae26eff..5ca28d4db 100644 --- a/docs/setup/marketplace-install.md +++ b/docs/setup/marketplace-install.md @@ -126,7 +126,7 @@ Add further families the same way. Verify with `/plugins` inside Codex, or `codex plugin list` from the shell. Codex has no auto-update setting for marketplaces, so refreshing is a manual -step — `codex plugin marketplace upgrade` re-pulls the configured Git snapshots. +step — `codex plugin marketplace upgrade apache-magpie` re-pulls the Magpie Git snapshot. Run it when you want the newer skills; nothing moves on its own. Once the baseline is in, ask for the rest in plain language — *install the diff --git a/plugins/magpie-setup/skills/setup/install.md b/plugins/magpie-setup/skills/setup/install.md index 8c302741e..85556e621 100644 --- a/plugins/magpie-setup/skills/setup/install.md +++ b/plugins/magpie-setup/skills/setup/install.md @@ -399,7 +399,7 @@ Tell the user, in this order: 4. **How it updates** — Claude Code: `/plugin marketplace update apache-magpie` then `/plugin update @apache-magpie`; Codex: - `codex plugin update magpie`; Gemini: + `codex plugin marketplace upgrade apache-magpie`; Gemini: `gemini extensions update magpie`. The bundled `SessionStart` hook prompts on a version change where the client runs hooks. 5. **Configure it for yourself when a skill asks.** A marketplace diff --git a/plugins/magpie-setup/skills/setup/setup_preflight/cli.py b/plugins/magpie-setup/skills/setup/setup_preflight/cli.py index 261804ac0..a5e28ec92 100644 --- a/plugins/magpie-setup/skills/setup/setup_preflight/cli.py +++ b/plugins/magpie-setup/skills/setup/setup_preflight/cli.py @@ -33,8 +33,10 @@ import argparse import json +import os import subprocess import sys +from collections.abc import Mapping from pathlib import Path from .core import ( @@ -50,50 +52,121 @@ from .delegation import findings as delegation_findings from .lockfile import MalformedLock +#: How to notice which harness is running, and that harness's own +#: plugin-listing command (`install.md` Step M2/M5). The markers mirror +#: the canonical self-detection order in `tools/adversarial-review`: a +#: harness started from inside another inherits the outer one's variables, +#: so the innermost candidates are checked first and Claude Code's widely +#: inherited `CLAUDECODE` comes last. Only the running harness's listing +#: runs — a machine may have several installed, and whichever comes first +#: in a fixed probe order is not the one whose versions this session runs. +#: Copilot installs from a repo URL or catalogue and has no listing CLI, +#: so its entry has none: the floor stays unknown there rather than +#: guessing what a listing would print. +HARNESS_LISTINGS: tuple[tuple[str, tuple[tuple[str, str | None], ...], list[str] | None], ...] = ( + ("codex", (("CODEX_SANDBOX", None), ("CODEX_THREAD_ID", None)), ["codex", "plugin", "list", "--json"]), + ("copilot", (("COPILOT_CLI", None),), None), + ("gemini", (("GEMINI_CLI", "1"),), ["gemini", "extensions", "list", "-o", "json"]), + ("claude", (("CLAUDECODE", "1"),), ["claude", "plugin", "list", "--json"]), +) + + +def _running_harness(env: Mapping[str, str]) -> tuple[str | None, list[str] | None]: + """The harness these environment variables are running, and its listing.""" + for harness, markers, argv in HARNESS_LISTINGS: + for var, want in markers: + value = env.get(var) + if value and (want is None or value == want): + return harness, argv + return None, None + -def read_installed(explicit: str | None) -> dict[str, str] | None: - """The installed plugin map, or `None` for *unknown* — never `{}` for it. +def _parse_listing(text: str, harness: str | None = None) -> dict[str, str] | None: + """One harness listing, parsed into the shared `{name: version}` map. - `--plugin-list` lets the caller supply what it already read; otherwise - the harness CLI is asked directly. Every failure mode collapses to - `None`: the command missing, a non-zero exit, unparsable output, or a - sandbox that denies the plugin cache. The one case that is *not* - unknown is a listing that parsed and was genuinely empty, which only a - harness with no plugins installed produces. + Shared by every harness's listing command: `None` for unknown (the + output did not parse, or an empty parsed listing leaves the sandbox + read-denial ambiguity standing), never an empty map. """ - if explicit is not None: - text = sys.stdin.read() if explicit == "-" else Path(explicit).read_text(encoding="utf-8") - else: - try: - completed = subprocess.run( - ["claude", "plugin", "list", "--json"], - capture_output=True, - text=True, - timeout=20, - check=False, - ) - except (OSError, subprocess.SubprocessError): - return None - if completed.returncode != 0: - return None - text = completed.stdout try: - parsed = json.loads(text) + listing = json.loads(text) except json.JSONDecodeError: return None - if not isinstance(parsed, list): + # Codex 0.149.1 separates installed plugins from available catalogue + # entries. Only the former describe what this session can run. + if harness == "codex" and isinstance(listing, dict): + listing = listing.get("installed") + if not isinstance(listing, list): return None - installed: dict[str, str] = {} - for entry in parsed: - if not isinstance(entry, dict): + out: dict[str, str] = {} + for item in listing: + if not isinstance(item, dict): return None - name, version = entry.get("name"), entry.get("version") - if isinstance(name, str) and isinstance(version, str): - installed[name] = version - # An empty *parsed* list inside a sandbox is the read-denied case, and - # is indistinguishable from a genuinely plugin-free harness. Unknown is - # the safe reading of the ambiguity: it proposes nothing. - return installed or None + version = item.get("version") + if not isinstance(item.get("name"), str) or not isinstance(version, str): + return None + out[item["name"]] = version + # A listing that parses but is empty leaves the sandbox read-denial + # ambiguity standing, so we report unknown rather than an empty map. + return out or None + + +def read_installed( + explicit: str | None, env: Mapping[str, str] | None = None +) -> tuple[str | None, dict[str, str] | None]: + """The running harness and its installed plugin map, or `None` for + *unknown* — never `{}` for it. The pair is a coherent answer: both + are `None` unless the running harness's own listing was read and + parsed. + + `--plugin-list` lets the caller supply what it already read; the + harness is still detected from the environment, so its own parsing + and floor rules apply, and stays `None` only when no harness with a + listing command is running — the map then stands alone. Otherwise the + running harness is detected from its own environment markers and only + its listing command runs — no other harness's listing may answer for + it. Every failure mode collapses to `(None, None)`: the harness + undetectable, a listing CLI the running harness does not have + (Copilot), the command missing, a non-zero exit, a timeout or a + refusal from the sandbox, unparsable output, or a sandbox that denies + the plugin cache. An empty listing also stays unknown because it + cannot be distinguished from a denied cache read. + """ + if explicit is not None: + text = sys.stdin.read() if explicit == "-" else Path(explicit).read_text(encoding="utf-8") + # A supplied listing is still the running harness's own: without + # its id, a Gemini `magpie` extension reads as every floor plugin + # missing and a Codex envelope as unparsable. + harness, argv = _running_harness(os.environ if env is None else env) + if argv is None: + harness = None + parsed = _parse_listing(text, harness) + return (harness if parsed is not None else None), parsed + harness, argv = _running_harness(os.environ if env is None else env) + if argv is None: + # No harness is running here, or the one running has no listing + # CLI (Copilot installs from a repo URL): unknown, and no other + # harness's listing may be probed instead. + return None, None + try: + completed = subprocess.run( + argv, + check=False, + capture_output=True, + text=True, + timeout=20, + ) + except (OSError, subprocess.SubprocessError): + # The listing cannot be produced: the binary is missing from the + # harness it belongs to, it times out, or the sandbox refuses to + # execute it. Unknown — never another harness's answer. + return None, None + if completed.returncode != 0: + return None, None + parsed = _parse_listing(completed.stdout, harness) + if parsed is None: + return None, None + return harness, parsed def _delegation(root: Path, skill: str) -> list[Finding]: @@ -127,7 +200,7 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument( "--plugin-list", metavar="PATH", - help="`claude plugin list --json` output to use instead of running it ('-' for stdin)", + help="harness plugin listing JSON to use instead of running it ('-' for stdin)", ) parser.add_argument( "--verify-interval-days", @@ -165,13 +238,13 @@ def main(argv: list[str] | None = None) -> int: root: Path = args.project_root try: - installed = read_installed(args.plugin_list) + harness, installed = read_installed(args.plugin_list) if args.no_cache: from .core import project_findings - project, cached = project_findings(root, installed), False + project, cached = project_findings(root, installed, harness), False else: - project, cached = cached_project_findings(root, installed) + project, cached = cached_project_findings(root, installed, harness=harness) findings = [ *([] if args.no_welcome else welcome_findings(root, args.skill)), *project, diff --git a/plugins/magpie-setup/skills/setup/setup_preflight/core.py b/plugins/magpie-setup/skills/setup/setup_preflight/core.py index e125829de..7f744b8c6 100644 --- a/plugins/magpie-setup/skills/setup/setup_preflight/core.py +++ b/plugins/magpie-setup/skills/setup/setup_preflight/core.py @@ -152,13 +152,24 @@ def _snapshot_findings(lock: Lock, root: Path) -> list[Finding]: return [] -def _floor_findings(lock: Lock, installed: dict[str, str] | None) -> list[Finding]: +def _floor_findings(lock: Lock, installed: dict[str, str] | None, harness: str | None) -> list[Finding]: if lock.url and lock.url != TRUSTED_MARKETPLACE: return [Finding("project", "untrusted-marketplace", "step-3", {"url": lock.url})] if installed is None: # Unknown, never absent. Say nothing and let the run continue: this # is the ordinary state of a sandboxed session, not a fault. return [] + if harness == "gemini": + # Gemini ships all families in one extension. Its version carries + # every floor entry; an unrecognised extension name is unknown. + version = installed.get("magpie") + if version is None: + return [] + installed = dict.fromkeys(lock.plugins, version) + elif harness == "codex" and not any(name in installed for name in lock.plugins): + # Until a listing identifies an exact floor name, qualified names + # or an unfamiliar schema are not evidence that plugins are absent. + return [] missing = [name for name in lock.plugins if name not in installed] outdated = [] for name in lock.plugins: @@ -177,7 +188,7 @@ def _floor_findings(lock: Lock, installed: dict[str, str] | None) -> list[Findin "project", "below-floor", "step-3", - {"missing": missing, "outdated": outdated, "restart_required": True}, + {"missing": missing, "outdated": outdated, "restart_required": True, "harness": harness}, ) ] @@ -223,8 +234,16 @@ def legacy_layout_findings(root: Path) -> list[Finding]: ] -def project_findings(root: Path, installed: dict[str, str] | None) -> list[Finding]: - """Everything true of the checkout rather than of one skill.""" +def project_findings( + root: Path, installed: dict[str, str] | None, harness: str | None = None +) -> list[Finding]: + """Everything true of the checkout rather than of one skill. + + `harness` is the agent the session is running under, as the floor + check measured it — `None` when the listing came from `--plugin-list` + or could not be read. It travels with the findings so the rules can + name that harness's own install and update commands. + """ lock = load(root / LOCK_NAME) if lock is None: # A supported end state, not a fault: the marketplace install @@ -235,7 +254,7 @@ def project_findings(root: Path, installed: dict[str, str] | None) -> list[Findi if lock.method in SNAPSHOT_METHODS: return _snapshot_findings(lock, root) if lock.method == "marketplace": - return _floor_findings(lock, installed) + return _floor_findings(lock, installed, harness) if lock.method == LOCAL_METHOD: return [] return [Finding("project", "unknown-method", "step-2", {"method": lock.method})] @@ -483,7 +502,7 @@ def isolated_setup_findings( # --- memoising the project scope ---------------------------------------------------- -def _cache_key(root: Path, installed: dict[str, str] | None) -> str: +def _cache_key(root: Path, installed: dict[str, str] | None, harness: str | None) -> str: # Linked worktrees of an adopted repository can share their main # checkout's personal layer, and so this cache file: key it on the # checkout as well, so one worktree never serves another's verdict. @@ -496,6 +515,9 @@ def _cache_key(root: Path, installed: dict[str, str] | None) -> str: # The legacy-layout finding depends on the in-tree directory existing. parts.append("legacy:" + ("1" if legacy_local_dir(root) is not None else "-")) parts.append("installed:" + ("?" if installed is None else json.dumps(installed, sort_keys=True))) + # The below-floor finding names the harness the listing came from, so a + # verdict computed under one harness must not be served to another. + parts.append("harness:" + (harness or "-")) # The verdict is also a function of this code: an upgrade that refreshes # the checker must not keep serving the previous checker's answer. here = Path(__file__).parent @@ -512,6 +534,7 @@ def _cache_key(root: Path, installed: dict[str, str] | None) -> str: def cached_project_findings( root: Path, installed: dict[str, str] | None, + harness: str | None = None, *, now: float | None = None, ) -> tuple[list[Finding], bool]: @@ -519,19 +542,19 @@ def cached_project_findings( Returns `(findings, was_cached)`. The cache lives in the personal layer (`layers.personal_dir`, which in a linked worktree of an adopted - repository may be its main checkout's), is keyed on the checkout, the lock files' identity and - the plugin listing, and expires so that a plugin installed mid-session - is picked up by the next skill rather than at the end of the day. A - project whose personal layer does not exist is not cached at all — - creating it would create the very state whose absence the skill scope - reads as "never configured". + repository may be its main checkout's), is keyed on the checkout, the lock files' identity, + the plugin listing and the harness the listing came from, and expires + so that a plugin installed mid-session is picked up by the next skill + rather than at the end of the day. A project whose personal layer + does not exist is not cached at all — creating it would create the + very state whose absence the skill scope reads as "never configured". """ local_dir = personal_dir(root) if local_dir is None or not local_dir.is_dir(): - return project_findings(root, installed), False + return project_findings(root, installed, harness), False stamp_now = now if now is not None else time.time() - key = _cache_key(root, installed) + key = _cache_key(root, installed, harness) cache_path = local_dir / CACHE_NAME if cache_path.exists(): try: @@ -548,7 +571,7 @@ def cached_project_findings( if isinstance(entries, list): return [Finding(**entry) for entry in entries], True - findings = project_findings(root, installed) + findings = project_findings(root, installed, harness) payload = {"key": key, "at": stamp_now, "findings": [asdict(f) for f in findings]} tmp = cache_path.with_suffix(".tmp") tmp.write_text(json.dumps(payload), encoding="utf-8") diff --git a/plugins/magpie-setup/skills/setup/setup_preflight/guides/general.md b/plugins/magpie-setup/skills/setup/setup_preflight/guides/general.md index da7dee5e4..86fe58154 100644 --- a/plugins/magpie-setup/skills/setup/setup_preflight/guides/general.md +++ b/plugins/magpie-setup/skills/setup/setup_preflight/guides/general.md @@ -21,7 +21,7 @@ On a pinned-snapshot install the commands are shorter: `/magpie-setup config`, ` - ✋ **By hand:** `claude plugin marketplace update apache-magpie`, then `claude plugin update @apache-magpie` - 🤖 **Automatically (Claude Code):** `/plugin` → **Marketplaces** → **apache-magpie** → **Enable auto-update** — it is off by default for third-party marketplaces like this one -- 🧩 Gemini CLI takes `--auto-update` at install time; Codex has no auto-update, so run `codex plugin marketplace upgrade` +- 🧩 Gemini CLI takes `--auto-update` at install time; Codex has no auto-update, so run `codex plugin marketplace upgrade apache-magpie` After an update, the next skill you run checks your configuration against it and proposes any fixes; `/magpie-setup:setup reconcile` checks everything at once diff --git a/plugins/magpie-setup/skills/setup/setup_preflight/sections/step-3.md b/plugins/magpie-setup/skills/setup/setup_preflight/sections/step-3.md index 047f41b9c..9f4185d8b 100644 --- a/plugins/magpie-setup/skills/setup/setup_preflight/sections/step-3.md +++ b/plugins/magpie-setup/skills/setup/setup_preflight/sections/step-3.md @@ -16,16 +16,37 @@ the release segment. The reconciliation check in step 4 is gated on the fingerprint, never on this version delta. **Why an unreadable result is *unknown* rather than *absent*.** Inside a -sandboxed session the plugin cache is read-denied and `claude plugin list ---json` returns `[]` there — that reads exactly like "nothing installed" +sandboxed session the plugin cache is read-denied and the running harness's +own listing returns `[]` there — that reads exactly like "nothing installed" but is not. Acting on it would propose installing a project's entire floor on every sandboxed run. -**The actions.** +For Codex, read only the `installed` list in its JSON envelope; available +catalogue entries are not installed versions. An unfamiliar name form or +schema stays unknown. Require an exact floor plugin name before trusting +a Codex listing, and the exact `magpie` extension name on Gemini. -- a floor plugin absent → `claude plugin install @apache-magpie`; -- a floor plugin below `min_version` → `claude plugin update - @apache-magpie`. +**The actions.** The `below-floor` finding names the harness its listing +came from (`facts.harness`); the commands differ per client, and naming one +the operator cannot run in this session wastes their turn: + +| Harness | a floor plugin absent | a floor plugin below `min_version` | +|---|---|---| +| Claude Code | `claude plugin install @apache-magpie` | `claude plugin update @apache-magpie` | +| Codex | `codex plugin add @apache-magpie` | `codex plugin marketplace upgrade apache-magpie` | +| Gemini CLI | `gemini extensions install https://github.com/apache/magpie` | `gemini extensions update magpie` | + +Gemini installs the framework as one extension, so on it the floor is +carried by the `magpie` extension as a whole: update that one extension +rather than naming individual plugins. + +`facts.harness` is `null` when the caller supplied the listing and no +harness with a listing command is running. Use the row of the harness this +session runs in; if that is not known, print every row rather than guess. + +Copilot CLI installs from a repo URL or catalogue and has no plugin CLI to +read a listing or run an action with. There the floor stays unknown: run +nothing, and leave it to the user. Where there is no such CLI, run nothing and print the commands instead. diff --git a/plugins/magpie-setup/skills/setup/upgrade.md b/plugins/magpie-setup/skills/setup/upgrade.md index 923ec99a7..d8919bc4b 100644 --- a/plugins/magpie-setup/skills/setup/upgrade.md +++ b/plugins/magpie-setup/skills/setup/upgrade.md @@ -68,7 +68,7 @@ Both paths run the same flow. /plugin update @apache-magpie ``` - (Codex: `codex plugin update magpie`; Gemini: + (Codex: `codex plugin marketplace upgrade apache-magpie`; Gemini: `gemini extensions update magpie`.) After the plugin refresh, run `setup-isolated-setup-update` for any installed harness profile; plugin updates do not merge workspace configuration. The companion diff --git a/tools/dev/render-preflight-fixtures.py b/tools/dev/render-preflight-fixtures.py index 3501d52b0..9966176ba 100644 --- a/tools/dev/render-preflight-fixtures.py +++ b/tools/dev/render-preflight-fixtures.py @@ -26,12 +26,19 @@ * `lock` — the committed `.apache-magpie.lock` lines, or `null` for none. A lock also gets `.apache-magpie-overrides/project.md`. -* `plugins` — `[[name, version], ...]` for `claude plugin list --json`, or - the string `"unreadable"` for a listing that cannot be parsed (no `claude` - on `PATH`). +* `plugins` — `[[name, version], ...]` for the running harness's listing, + or the string `"unreadable"` for output that cannot be parsed. Codex + entries are wrapped in its `installed` / `available` JSON envelope. * `requires` — `requires_config:` entries passed as `--requires`. * `skill`, `hash` — the invoking skill's `name:` and `surface_hash:`. * `reconciled` — dict written to personal layer `reconciled.json`, or `null`. +* `harness` — the harness whose session the case runs in (`codex`, + `copilot`, `gemini`, or `claude`). The renderer puts that harness's + session markers in the environment and a stub listing command on `PATH` + that prints `plugins`, so the checker detects the harness itself and + reads the listing the way that harness's session would produce it. + Left out for no markers at all, which the checker reports as an unknown + harness. * `welcome` — `true` to include the skill family's first-run welcome guide. Left out otherwise (`--no-welcome`), so a floor case is about the floor and not about which family its skill happens to belong to. @@ -58,6 +65,16 @@ DEFAULT_SKILL = "issue-triage" DEFAULT_HASH = "sha256:2edc90a1b7f3440d" +#: The session markers each harness self-detects by, mirroring the +#: checker's own `HARNESS_LISTINGS` marker order (the listing command is +#: the checker's business, not the renderer's). +HARNESS_ENV: dict[str, dict[str, str]] = { + "codex": {"CODEX_SANDBOX": "1"}, + "copilot": {"COPILOT_CLI": "1"}, + "gemini": {"GEMINI_CLI": "1"}, + "claude": {"CLAUDECODE": "1"}, +} + def run_checker(state: dict[str, object], repo_root: Path, scratch: Path) -> str: """Build `state` as a repository under `scratch` and return the checker's stdout.""" @@ -84,15 +101,41 @@ def run_checker(state: dict[str, object], repo_root: Path, scratch: Path) -> str personal.mkdir(parents=True, exist_ok=True) (personal / "reconciled.json").write_text(json.dumps(reconciled), encoding="utf-8") + harness = state.get("harness") listing = scratch / "plugins.json" plugins = state.get("plugins") if plugins == "unreadable": - listing.write_text("zsh: command not found: claude\n", encoding="utf-8") + listing.write_text("listing unavailable\n", encoding="utf-8") elif isinstance(plugins, list): - listing.write_text(json.dumps([{"name": n, "version": v} for n, v in plugins]), encoding="utf-8") + entries = [{"name": n, "version": v} for n, v in plugins] + payload = {"installed": entries, "available": []} if harness == "codex" else entries + listing.write_text(json.dumps(payload), encoding="utf-8") else: raise ValueError('`plugins` must be a list of [name, version] or "unreadable"') + if harness is not None: + # The checker must detect this harness itself, from the same session + # markers a live session carries, and read the harness's own listing + # command — a stub on PATH that prints the case's listing. Passing + # `--plugin-list` would describe the listing as already read, which + # is exactly the "the harness is then not known" case. + if not isinstance(harness, str) or harness not in HARNESS_ENV: + raise ValueError(f"`harness` must be one of {', '.join(sorted(HARNESS_ENV))} or null") + markers: dict[str, str] = HARNESS_ENV[harness] + bin_dir = scratch / "bin" + bin_dir.mkdir() + listing_arg = str(listing).replace("\\", "/") + stub = bin_dir / harness + stub.write_text(f'#!/bin/sh\ncat "{listing_arg}"\n', encoding="utf-8") + os.chmod(stub, 0o700) # owner-only: the renderer runs it as itself + (bin_dir / f"{harness}.bat").write_text(f'@echo off\ntype "{listing}"\r\n', encoding="utf-8") + env = _env(scratch) + env["PATH"] = str(bin_dir) + os.pathsep + env["PATH"] + env.update(markers) + else: + env = _env(scratch) + env["PYTHONPATH"] = str(repo_root / CHECKER) + args = [ sys.executable, "-m", @@ -103,14 +146,15 @@ def run_checker(state: dict[str, object], repo_root: Path, scratch: Path) -> str str(state.get("hash", DEFAULT_HASH)), "--project-root", str(project), - "--plugin-list", - str(listing), "--no-cache", "--verify-interval-days", "0", "--isolated-setup-interval-days", "0", ] + if harness is None: + # No harness to detect: hand the checker the listing as already read. + args += ["--plugin-list", str(listing)] requires = state.get("requires", []) if not isinstance(requires, list): raise ValueError("`requires` must be a list") @@ -119,8 +163,19 @@ def run_checker(state: dict[str, object], repo_root: Path, scratch: Path) -> str if state.get("welcome") is not True: args.append("--no-welcome") - env = _env(scratch) - env["PYTHONPATH"] = str(repo_root / CHECKER) + if os.name == "nt" and harness is not None: + # CreateProcess does not search PATHEXT for a bare command name. + # Resolve only the fixture's .bat stub; keep harness detection, + # listing arguments, parsing and the checker itself unchanged. + launcher = ( + "from setup_preflight import cli; " + f"name = {harness!r}; stub = {str(bin_dir / (harness + '.bat'))!r}; " + "[(argv.__setitem__(0, stub)) for h, _, argv in cli.HARNESS_LISTINGS " + "if h == name and argv is not None]; " + "raise SystemExit(cli.main())" + ) + args[1:3] = ["-c", launcher] + result = subprocess.run(args, capture_output=True, text=True, env=env, cwd=project, check=False) if result.returncode != 0: raise RuntimeError(f"setup_preflight exited {result.returncode}: {result.stderr.strip()}") @@ -131,13 +186,19 @@ def _env(scratch: Path) -> dict[str, str]: """A minimal environment, so nothing on the host leaks into a verdict.""" home = scratch / "home" home.mkdir(exist_ok=True) - return { + env = { "PATH": os.environ.get("PATH", "/usr/bin:/bin"), "HOME": str(home), "GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": os.devnull, "LC_ALL": "C", } + if os.name == "nt": + # Windows needs these to launch batch stubs from the child Python. + for key in ("SystemRoot", "COMSPEC"): + if key in os.environ: + env[key] = os.environ[key] + return env def render_report(prose: str, verdict: str) -> str: @@ -180,7 +241,7 @@ def main(argv: list[str] | None = None) -> int: continue stale.append(report) if not args.check: - report.write_text(text, encoding="utf-8") + report.write_text(text, encoding="utf-8", newline="\n") if not stale: return 0 diff --git a/tools/setup-preflight/README.md b/tools/setup-preflight/README.md index dd45e627e..0a09f5c72 100644 --- a/tools/setup-preflight/README.md +++ b/tools/setup-preflight/README.md @@ -46,9 +46,10 @@ how to word it, and the prohibitions. Those live in each skill's - **Runtime** — Python 3.11+. Standard library only, deliberately: the module is copied into the user's personal configuration layer and run with bare `python3`, where nothing else is available. -- **CLIs** — none required. The harness CLI (`claude plugin list --json`) - is consulted when present and its absence is a supported state, not an - error. +- **CLIs** — none required. The running harness's own listing command + (detected from its environment markers, e.g. `claude plugin list + --json` under Claude Code) is consulted when present; its absence or + the absence of any harness marker is a supported state, not an error. - **Credentials / auth** — none. - **Network** — none. Every input is a file in the project. - **Optional** — `--plugin-list` accepts a listing the caller already @@ -191,11 +192,11 @@ is settled when the run finishes rather than before it starts. **Unknown is never absent.** A plugin listing that could not be read is `None`, not `{}`. Inside a sandboxed session the plugin cache is -read-denied and `claude plugin list --json` prints `[]`, which reads -exactly like "nothing installed"; acting on it would propose installing a -project's entire floor on every sandboxed run. An empty *parsed* listing -is treated as unknown for the same reason — it is indistinguishable from -the denied case. +read-denied and the running harness's own listing prints `[]`, which +reads exactly like "nothing installed"; acting on it would propose +installing a project's entire floor on every sandboxed run. An empty +*parsed* listing is treated as unknown for the same reason — it is +indistinguishable from the denied case. **A dev build is a version like any other.** Nothing strips or rounds a `.devN` segment. `0.2.0.dev202609110041` is below `0.2.0`, and `0.10.0` diff --git a/tools/setup-preflight/tests/fixtures/README.md b/tools/setup-preflight/tests/fixtures/README.md new file mode 100644 index 000000000..f7de882af --- /dev/null +++ b/tools/setup-preflight/tests/fixtures/README.md @@ -0,0 +1,15 @@ + + + + + + +`codex-0.149.1-list.json` was captured from `codex plugin list --json` +on a live Codex CLI 0.149.1 install on 2026-10-20. It retains one installed +entry; other entries and the `source` / `marketplaceSource` fields (local +filesystem paths) were removed. The installed/available envelope, field +names, and retained values are unchanged. This is a format fixture, not a +Magpie installation. Synthetic Magpie names in the CLI tests exercise floor +comparison separately. No live Gemini output was available: its listing +is trusted only when the exact `magpie` extension name is present. diff --git a/tools/setup-preflight/tests/fixtures/codex-0.149.1-list.json b/tools/setup-preflight/tests/fixtures/codex-0.149.1-list.json new file mode 100644 index 000000000..1686162da --- /dev/null +++ b/tools/setup-preflight/tests/fixtures/codex-0.149.1-list.json @@ -0,0 +1,15 @@ +{ + "installed": [ + { + "pluginId": "documents@openai-primary-runtime", + "name": "documents", + "marketplaceName": "openai-primary-runtime", + "version": "26.1015.12040", + "installed": true, + "enabled": true, + "installPolicy": "AVAILABLE", + "authPolicy": "ON_USE" + } + ], + "available": [] +} diff --git a/tools/setup-preflight/tests/test_cli.py b/tools/setup-preflight/tests/test_cli.py index 76c788cb1..90d7b2c72 100644 --- a/tools/setup-preflight/tests/test_cli.py +++ b/tools/setup-preflight/tests/test_cli.py @@ -21,6 +21,7 @@ from __future__ import annotations import json +import subprocess from pathlib import Path import pytest @@ -39,6 +40,14 @@ """ +@pytest.fixture(autouse=True) +def clean_harness_markers(monkeypatch: pytest.MonkeyPatch) -> None: + # A developer's surrounding Codex session must not override the + # harness a test explicitly selects. + for marker in ("CODEX_SANDBOX", "CODEX_THREAD_ID", "COPILOT_CLI", "GEMINI_CLI", "CLAUDECODE"): + monkeypatch.delenv(marker, raising=False) + + def run(capsys: pytest.CaptureFixture[str], *args: str) -> tuple[int, dict]: code = main(list(args)) out = capsys.readouterr().out @@ -108,7 +117,7 @@ def test_every_section_named_is_one_a_reader_can_find(project: Path, capsys) -> def test_unparsable_output_is_unknown(tmp_path: Path) -> None: bad = tmp_path / "bad.json" bad.write_text("not json at all") - assert read_installed(str(bad)) is None + assert read_installed(str(bad)) == (None, None) def test_an_empty_listing_is_unknown_not_an_empty_install(tmp_path: Path) -> None: @@ -117,24 +126,326 @@ def test_an_empty_listing_is_unknown_not_an_empty_install(tmp_path: Path) -> Non harness. Unknown is the safe reading: it proposes nothing.""" empty = tmp_path / "empty.json" empty.write_text("[]") - assert read_installed(str(empty)) is None + assert read_installed(str(empty)) == (None, None) def test_a_real_listing_parses_to_a_name_version_map(tmp_path: Path) -> None: listing = tmp_path / "list.json" listing.write_text('[{"name": "magpie-setup", "version": "0.2.0"}]') - assert read_installed(str(listing)) == {"magpie-setup": "0.2.0"} + assert read_installed(str(listing)) == (None, {"magpie-setup": "0.2.0"}) def test_a_listing_that_is_not_a_list_is_unknown(tmp_path: Path) -> None: listing = tmp_path / "list.json" listing.write_text('{"plugins": []}') - assert read_installed(str(listing)) is None + assert read_installed(str(listing)) == (None, None) + + +# --- read_installed: the running harness answers with its own listing ---------------- + + +def _listing_ok(argv: list[str], body: str) -> subprocess.CompletedProcess: + completed = subprocess.CompletedProcess[str](argv, 0) + completed.stdout = body + return completed + + +def test_the_claude_marker_runs_only_claudes_listing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The harness running this session is detected from its own marker, + and only its listing command runs.""" + seen: list[list[str]] = [] + + def run(argv: list[str], **_kwargs: object) -> subprocess.CompletedProcess: + seen.append(list(argv)) + return _listing_ok(argv, '[{"name": "magpie-setup", "version": "0.2.0"}]') + + monkeypatch.setattr("setup_preflight.cli.subprocess.run", run) + assert read_installed(None, {"CLAUDECODE": "1"}) == ("claude", {"magpie-setup": "0.2.0"}) + assert seen == [["claude", "plugin", "list", "--json"]] + + +def test_the_codex_marker_runs_codex_listing_with_its_json_flag( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """`codex plugin list` prints a terminal summary by default; only + `--json` makes it emit the array the shared parser reads.""" + seen: list[list[str]] = [] + + def run(argv: list[str], **_kwargs: object) -> subprocess.CompletedProcess: + seen.append(list(argv)) + return _listing_ok(argv, '[{"name": "magpie-setup", "version": "0.3.1"}]') + + monkeypatch.setattr("setup_preflight.cli.subprocess.run", run) + assert read_installed(None, {"CODEX_SANDBOX": "1"}) == ("codex", {"magpie-setup": "0.3.1"}) + assert seen == [["codex", "plugin", "list", "--json"]] + + +def test_the_gemini_marker_runs_gemini_listing_with_json_output( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """`gemini extensions list` prints plain text by default; only + `-o json` makes it emit the array the shared parser reads.""" + seen: list[list[str]] = [] + + def run(argv: list[str], **_kwargs: object) -> subprocess.CompletedProcess: + seen.append(list(argv)) + return _listing_ok(argv, '[{"name": "magpie", "version": "0.2.0"}]') + + monkeypatch.setattr("setup_preflight.cli.subprocess.run", run) + assert read_installed(None, {"GEMINI_CLI": "1"}) == ("gemini", {"magpie": "0.2.0"}) + assert seen == [["gemini", "extensions", "list", "-o", "json"]] + + +def test_the_innermost_marker_wins_when_two_harnesses_inherit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A harness started from inside another inherits the outer one's + variables; the innermost candidate answers, and `CLAUDECODE` — which + every child inherits — is checked last.""" + seen: list[list[str]] = [] + + def run(argv: list[str], **_kwargs: object) -> subprocess.CompletedProcess: + seen.append(list(argv)) + return _listing_ok(argv, '[{"name": "magpie", "version": "0.2.0"}]') + + monkeypatch.setattr("setup_preflight.cli.subprocess.run", run) + harness, _ = read_installed(None, {"CLAUDECODE": "1", "GEMINI_CLI": "1"}) + assert harness == "gemini" + assert seen == [["gemini", "extensions", "list", "-o", "json"]] + + +def test_no_marker_is_unknown_and_no_listing_runs( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A session under no detected harness stays unknown: nothing is + probed in a fixed order, so no harness can answer for another.""" + + def explode(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess: + raise AssertionError("no listing may run without a detected harness") + + monkeypatch.setattr("setup_preflight.cli.subprocess.run", explode) + assert read_installed(None, {}) == (None, None) + + +def test_the_copilot_marker_is_unknown_without_a_listing_cli( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Copilot installs from a repo URL and has no listing CLI: the floor + stays unknown rather than guessing what a listing would print.""" + + def explode(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess: + raise AssertionError("copilot has no listing command to run") + + monkeypatch.setattr("setup_preflight.cli.subprocess.run", explode) + assert read_installed(None, {"COPILOT_CLI": "1"}) == (None, None) + + +def test_a_timeout_of_the_running_harness_is_unknown( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A `claude` that exists and times out must not borrow another + harness's listing: unknown, and no other harness is ever probed.""" + + def explode(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess: + raise subprocess.TimeoutExpired(cmd=["claude"], timeout=20) + + monkeypatch.setattr("setup_preflight.cli.subprocess.run", explode) + assert read_installed(None, {"CLAUDECODE": "1"}) == (None, None) + + +def test_a_refused_binary_of_the_running_harness_is_unknown( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A sandbox that refuses to execute the harness's own binary reads as + unknown, not as a missing binary to probe around.""" + def explode(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess: + raise PermissionError("sandbox denies execution") -def test_a_missing_harness_cli_is_unknown(monkeypatch: pytest.MonkeyPatch) -> None: - def explode(*_args: object, **_kwargs: object) -> None: + monkeypatch.setattr("setup_preflight.cli.subprocess.run", explode) + assert read_installed(None, {"CLAUDECODE": "1"}) == (None, None) + + +def test_a_missing_binary_of_the_running_harness_is_unknown( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The marker says Claude Code but `claude` is not on PATH: unknown — + the checker never asks a different harness to answer for it.""" + + def explode(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess: raise FileNotFoundError("claude") monkeypatch.setattr("setup_preflight.cli.subprocess.run", explode) - assert read_installed(None) is None + assert read_installed(None, {"CLAUDECODE": "1"}) == (None, None) + + +def test_a_failed_listing_of_the_running_harness_is_unknown( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A detected harness whose listing exits non-zero stays unknown.""" + + def run(argv: list[str], **_kwargs: object) -> subprocess.CompletedProcess: + completed = subprocess.CompletedProcess[str](argv, 1) + completed.stdout = "" + return completed + + monkeypatch.setattr("setup_preflight.cli.subprocess.run", run) + assert read_installed(None, {"CLAUDECODE": "1"}) == (None, None) + + +def test_an_unparsable_listing_of_the_running_harness_is_unknown( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A detected harness whose listing is not the shared JSON shape reads + as unknown — the checker proposes nothing it cannot read.""" + + def run(argv: list[str], **_kwargs: object) -> subprocess.CompletedProcess: + return _listing_ok(argv, "magpie-setup 0.3.1 (marketplace)") + + monkeypatch.setattr("setup_preflight.cli.subprocess.run", run) + assert read_installed(None, {"CLAUDECODE": "1"}) == (None, None) + + +def test_the_finding_names_the_running_harness( + project: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The `below-floor` finding carries the harness its listing came + from, so the rules can name that harness's own commands.""" + write_lock(project, MARKETPLACE) + monkeypatch.setenv("CLAUDECODE", "1") + monkeypatch.setattr( + "setup_preflight.cli.subprocess.run", + lambda argv, **_kwargs: _listing_ok(argv, '[{"name": "magpie-setup", "version": "0.1.0"}]'), + ) + code, payload = run( + capsys, + "--skill", + "magpie-x", + "--project-root", + str(project), + "--verify-interval-days", + "0", + ) + assert code == 0 + finding = next(f for f in payload["findings"] if f["code"] == "below-floor") + assert finding["facts"]["harness"] == "claude" + + +def test_a_supplied_listing_reports_no_harness(project: Path, capsys) -> None: + """`--plugin-list` is what the caller already read; the map stands + alone and the finding names no harness.""" + write_lock(project, MARKETPLACE) + listing = project / "plugins.json" + listing.write_text('[{"name": "other", "version": "1.0.0"}]') + _, payload = run( + capsys, + "--skill", + "magpie-x", + "--project-root", + str(project), + "--plugin-list", + str(listing), + ) + finding = next(f for f in payload["findings"] if f["code"] == "below-floor") + assert finding["facts"]["harness"] is None + + +@pytest.mark.parametrize("version", ["0.2.0", "0.3.0", "0.10.0"]) +def test_gemini_extension_at_or_above_floor_does_not_block( + project: Path, capsys, monkeypatch, version +) -> None: + write_lock(project, MARKETPLACE + " - magpie-agent-guard\n") + monkeypatch.setenv("GEMINI_CLI", "1") + monkeypatch.setattr( + "setup_preflight.cli.subprocess.run", + lambda argv, **kwargs: _listing_ok(argv, json.dumps([{"name": "magpie", "version": version}])), + ) + code, payload = run(capsys, "--skill", "magpie-x", "--project-root", str(project), "--no-cache") + assert code == 0 + assert not any(f["code"] == "below-floor" for f in payload.get("findings", [])) + + +def test_gemini_extension_below_floor_is_outdated_for_every_family( + project: Path, capsys, monkeypatch +) -> None: + write_lock(project, MARKETPLACE + " - magpie-agent-guard\n") + monkeypatch.setenv("GEMINI_CLI", "1") + monkeypatch.setattr( + "setup_preflight.cli.subprocess.run", + lambda argv, **kwargs: _listing_ok(argv, '[{"name":"magpie","version":"0.1.0"}]'), + ) + _, payload = run(capsys, "--skill", "magpie-x", "--project-root", str(project), "--no-cache") + facts = next(f["facts"] for f in payload["findings"] if f["code"] == "below-floor") + assert facts["missing"] == [] + assert facts["outdated"] == [ + {"plugin": name, "installed": "0.1.0", "floor": "0.2.0"} + for name in ("magpie-setup", "magpie-agent-guard") + ] + + +def test_a_supplied_gemini_listing_keeps_the_single_extension_rule( + project: Path, capsys, monkeypatch +) -> None: + write_lock(project, MARKETPLACE + " - magpie-agent-guard\n") + monkeypatch.setenv("GEMINI_CLI", "1") + listing = project / "plugins.json" + listing.write_text('[{"name": "magpie", "version": "0.3.0"}]') + code, payload = run( + capsys, "--skill", "magpie-x", "--project-root", str(project), "--plugin-list", str(listing) + ) + assert code == 0 + assert not any(f["code"] == "below-floor" for f in payload.get("findings", [])) + + +def test_a_supplied_codex_envelope_is_unwrapped(monkeypatch, tmp_path: Path) -> None: + listing = tmp_path / "codex.json" + listing.write_text( + (Path(__file__).parent / "fixtures" / "codex-0.149.1-list.json").read_text(encoding="utf-8") + ) + assert read_installed(str(listing), {"CODEX_THREAD_ID": "test"}) == ( + "codex", + {"documents": "26.1015.12040"}, + ) + + +def test_codex_live_listing_reads_only_installed_entries(monkeypatch) -> None: + text = (Path(__file__).parent / "fixtures" / "codex-0.149.1-list.json").read_text(encoding="utf-8") + listing = json.loads(text) + listing["available"] = [{"name": "magpie-setup", "version": "0.1.0"}] + text = json.dumps(listing) + monkeypatch.setattr("setup_preflight.cli.subprocess.run", lambda argv, **kwargs: _listing_ok(argv, text)) + assert read_installed(None, {"CODEX_THREAD_ID": "test"}) == ("codex", {"documents": "26.1015.12040"}) + + +def test_codex_envelope_reports_a_below_floor_install(project: Path, capsys, monkeypatch) -> None: + write_lock(project, MARKETPLACE) + monkeypatch.setenv("CODEX_THREAD_ID", "test") + monkeypatch.setattr( + "setup_preflight.cli.subprocess.run", + lambda argv, **kwargs: _listing_ok( + argv, '{"installed":[{"name":"magpie-setup","version":"0.1.0"}],"available":[]}' + ), + ) + _, payload = run(capsys, "--skill", "magpie-x", "--project-root", str(project), "--no-cache") + facts = next(f["facts"] for f in payload["findings"] if f["code"] == "below-floor") + assert facts["harness"] == "codex" + assert facts["missing"] == [] + assert facts["outdated"] == [{"plugin": "magpie-setup", "installed": "0.1.0", "floor": "0.2.0"}] + + +@pytest.mark.parametrize( + "text", + [ + '[{"id":"magpie-setup","version":"0.1.0"}]', + '[{"name":"magpie-setup","version":"0.1.0"},{"plugin":"magpie-agent-guard","version":"0.1.0"}]', + '{"installed":[],"available":[{"name":"magpie-setup","version":"0.1.0"}]}', + '{"available":[{"name":"magpie-setup","version":"0.1.0"}]}', + ], +) +def test_unrecognised_or_empty_codex_listing_is_unknown(monkeypatch, text) -> None: + monkeypatch.setattr("setup_preflight.cli.subprocess.run", lambda argv, **kwargs: _listing_ok(argv, text)) + assert read_installed(None, {"CODEX_THREAD_ID": "test"}) == (None, None) diff --git a/tools/setup-preflight/tests/test_core.py b/tools/setup-preflight/tests/test_core.py index 9805269ae..f1ad72638 100644 --- a/tools/setup-preflight/tests/test_core.py +++ b/tools/setup-preflight/tests/test_core.py @@ -88,6 +88,33 @@ def test_a_genuinely_missing_plugin_is_a_finding(project: Path) -> None: found = project_findings(project, {"something-else": "9.9.9"}) assert codes(found) == ["below-floor"] assert found[0].facts["missing"] == ["magpie-setup"] + # The listing came from `--plugin-list` here, so no harness is named. + assert found[0].facts["harness"] is None + + +def test_the_floor_finding_names_the_harness_it_measured(project: Path) -> None: + """The finding carries the harness the listing came from, so the rules + can name that harness's own install and update commands.""" + write_lock(project, MARKETPLACE) + found = project_findings(project, {"magpie": "0.1.0"}, "gemini") + assert codes(found) == ["below-floor"] + assert found[0].facts["harness"] == "gemini" + assert found[0].facts["missing"] == [] + assert found[0].facts["outdated"] == [{"plugin": "magpie-setup", "installed": "0.1.0", "floor": "0.2.0"}] + + +@pytest.mark.parametrize( + "harness,installed", + [ + ("gemini", {"other-extension": "1.0.0"}), + ("gemini", {"magpie@apache-magpie": "1.0.0"}), + ("codex", {"magpie-setup@apache-magpie": "1.0.0"}), + ("codex", {"other-plugin": "1.0.0"}), + ], +) +def test_non_claude_names_must_match_before_reporting_absence(project: Path, harness, installed) -> None: + write_lock(project, MARKETPLACE) + assert project_findings(project, installed, harness) == [] def test_being_ahead_of_the_floor_is_never_a_finding(project: Path) -> None: @@ -400,6 +427,19 @@ def test_a_changed_plugin_listing_invalidates_the_cache(project: Path) -> None: assert cached is False and found == [] +def test_a_changed_harness_invalidates_the_cache(project: Path) -> None: + """The below-floor finding names the harness its listing came from, so + a verdict computed under one harness must not be served to another.""" + write_lock(project, MARKETPLACE) + (project / ".apache-magpie-local").mkdir() + first, cached = cached_project_findings(project, {"magpie-setup": "0.1.0"}, "claude") + assert cached is False and codes(first) == ["below-floor"] + assert first[0].facts["harness"] == "claude" + second, cached = cached_project_findings(project, {"magpie-setup": "0.1.0"}, "codex") + assert cached is False and codes(second) == ["below-floor"] + assert second[0].facts["harness"] == "codex" + + def test_a_refreshed_checker_invalidates_the_cache(project: Path) -> None: """`upgrade` replaces the checker's copy; a verdict the old code computed must not outlive it.""" diff --git a/tools/skill-evals/README.md b/tools/skill-evals/README.md index a10b91689..085241f4e 100644 --- a/tools/skill-evals/README.md +++ b/tools/skill-evals/README.md @@ -11,7 +11,7 @@ Behavioral eval harness for Apache Magpie skills. Each eval suite tests a skill Suites are currently implemented for: -- **setup** — 135 cases across 30 steps (step-verify-drift, step-overrides-surface, step-override-bypass, step-m3-baseline-pick, step-m4-install-gates, step-m5-no-repo-offer, step-m5-companion-offer, step-upgrade-companion-offer, step-adopt-settings-merge, verify-default-set, uninstall-default-set, lock-marketplace-parse, adopt-write-floor, setup-prefill-from-floor, preflight-floor, upgrade-adoption-split, verify-floor, adopt-review-process, adopt-security-list, step-reconcile, step-verify, step-config-stamp, step-config-adversarial, step-config-mail, step-adopt-stamp, step-upgrade-stamp, step-delegate-propose, config-vcs-backend, config-local-custom-parser) +- **setup** — 138 cases across 30 steps (step-verify-drift, step-overrides-surface, step-override-bypass, step-m3-baseline-pick, step-m4-install-gates, step-m5-no-repo-offer, step-m5-companion-offer, step-upgrade-companion-offer, step-adopt-settings-merge, verify-default-set, uninstall-default-set, lock-marketplace-parse, adopt-write-floor, setup-prefill-from-floor, preflight-floor, upgrade-adoption-split, verify-floor, adopt-review-process, adopt-security-list, step-reconcile, step-verify, step-config-stamp, step-config-adversarial, step-config-mail, step-adopt-stamp, step-upgrade-stamp, step-delegate-propose, config-vcs-backend, config-local-custom-parser) - **setup-isolated-setup-install** — 19 cases across 5 steps (runtime-routing, step-snapshot-drift, step-scope-confirm, step-hardware-key, step-roborev-exclusion) - **setup-privacy-llm** — 6 cases across 2 steps (step-1-resolve, step-4-gate) - **setup-shared-config-sync** — 12 cases across 2 steps (step-3-decide-action, step-5-draft-commit) diff --git a/tools/skill-evals/evals/setup/README.md b/tools/skill-evals/evals/setup/README.md index fd0369db6..99f836cd1 100644 --- a/tools/skill-evals/evals/setup/README.md +++ b/tools/skill-evals/evals/setup/README.md @@ -5,7 +5,7 @@ Behavioral evals for the `setup` skill. -## Suites (135 cases total) +## Suites (138 cases total) | Suite | Step | Cases | What it covers | |---|---|---|---| @@ -25,7 +25,7 @@ Behavioral evals for the `setup` skill. | adopt-review-process | adopt.md § Step 4c | 5 | a fresh adopt finding two deviations in the confirmed documents where one is accepted and one rejected, and a release policy left alone because that family is not in the floor; a re-adoption reviewing only the one family the floor diff added; a re-adoption that adds no family, where edited process documents are still not re-read; a sentence that would weaken a merge confirmation gate, dropped and named while a label deviation beside it is written; and a repo whose only candidate document the maintainer un-ticks, skipping the step | | adopt-security-list | adopt.md § Step 4b | 4 | an ASF project whose dedicated security list is in its docs and on the ASF page (recommend `security@`); an ASF project with no dedicated list (no recommendation, `private@` only as a flagged fallback); a non-ASF project, which is not asked; and conflicting evidence between `SECURITY.md` and the ASF page (no recommendation, flagged fallback) | | setup-prefill-from-floor | install.md § Step M0b | 4 | adopted (propose the floor), unadopted (framework defaults), a foreign marketplace called out, and a snapshot lock falling through | -| preflight-floor | preflight-block.md § Pre-flight | 15 | at floor (silent), below floor (update), plugin missing (install), foreign marketplace (ask first, run nothing), no `claude` CLI (print), unadopted (propose setup), `apache/magpie` in an alarming context (update anyway, unasked), unreadable `claude plugin list --json` (silent, treated as unknown not absent), first Magpie run (show the general welcome guide, record it, carry on), first utilities run (show the utilities welcome guide, record it, carry on), first mentoring run (show the mentoring welcome guide, record it, carry on), first pairing run (show the pairing welcome guide, record it, carry on), first issue run (show the issue welcome guide, record it, carry on), first pr-management run (show the pr-management welcome guide, record it, carry on), first contributor-growth run (show the contributor-growth welcome guide, record it, carry on) | +| preflight-floor | preflight-block.md § Pre-flight | 18 | at floor (silent), below floor (update), plugin missing (install), foreign marketplace (ask first, run nothing), no `claude` CLI (print), unadopted (propose setup), `apache/magpie` in an alarming context (update anyway, unasked), unreadable `claude plugin list --json` (silent, treated as unknown not absent), first Magpie run (show the general welcome guide, record it, carry on), first utilities run (show the utilities welcome guide, record it, carry on), first mentoring run (show the mentoring welcome guide, record it, carry on), first pairing run (show the pairing welcome guide, record it, carry on), first issue run (show the issue welcome guide, record it, carry on), first pr-management run (show the pr-management welcome guide, record it, carry on), first contributor-growth run (show the contributor-growth welcome guide, record it, carry on), below floor on Codex (marketplace upgrade), below floor on Gemini (update the extension once), at floor on Gemini (silent) | | upgrade-adoption-split | upgrade.md § Step 0b | 4 | not adopted (nothing staged), adopted (floor raised and staged), already ahead (floor unchanged), snapshot method (falls through) | | verify-floor | verify.md § Adoption floor | 4 | no lock (not a fault), ahead of floor with extra plugins (not a fault), a shortfall (a fault), a floor plugin the marketplace no longer ships (a fault, not installed around) | | step-reconcile | reconcile.md § The sweep | 3 | a clean sweep on a pinned-snapshot install (anchor present, config resolved — stamp written, nothing proposed), a renamed step heading stranding an override's anchor (one re-anchor proposal named), a marketplace install whose plugin cache is sandbox-denied (anchor resolution left `unchecked`, config resolution still completes) | diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-1-at-floor/state.json b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-1-at-floor/state.json index ed74e4567..3661111ea 100644 --- a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-1-at-floor/state.json +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-1-at-floor/state.json @@ -1,4 +1,5 @@ { + "harness": "claude", "lock": [ "method: marketplace", "url: apache/magpie", diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-10-utilities-first-run/state.json b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-10-utilities-first-run/state.json index 45c879d2a..b9c3eea2f 100644 --- a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-10-utilities-first-run/state.json +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-10-utilities-first-run/state.json @@ -1,4 +1,5 @@ { + "harness": "claude", "lock": null, "plugins": [ [ diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-16-below-floor-codex/case-meta.json b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-16-below-floor-codex/case-meta.json new file mode 100644 index 000000000..03128a3ee --- /dev/null +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-16-below-floor-codex/case-meta.json @@ -0,0 +1 @@ +{"tags":["local-smoke","smoke"]} diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-16-below-floor-codex/expected.json b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-16-below-floor-codex/expected.json new file mode 100644 index 000000000..3d7c35fa4 --- /dev/null +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-16-below-floor-codex/expected.json @@ -0,0 +1 @@ +{"action": "update", "commands": ["codex plugin marketplace upgrade apache-magpie"], "blocks": true, "mention_restart": true} diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-16-below-floor-codex/report.md b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-16-below-floor-codex/report.md new file mode 100644 index 000000000..d39a95369 --- /dev/null +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-16-below-floor-codex/report.md @@ -0,0 +1,57 @@ + + +`.apache-magpie.lock` contains `method: marketplace`, `url: apache/magpie`, +`min_version: 0.3.0`, and a `plugins` list of magpie-setup and +magpie-agent-guard. + +`.apache-magpie-overrides/` exists and contains `project.md`. + +The running agent is OpenAI Codex CLI (`CODEX_SANDBOX=1`). There is no +`claude` executable on PATH and no Gemini session markers in the +environment. `codex plugin list --json` reports magpie-setup 0.2.0 and +magpie-agent-guard 0.2.0, both from the `apache-magpie` marketplace. + +The pre-flight command printed: + +```json +{ + "findings": [ + { + "code": "below-floor", + "facts": { + "harness": "codex", + "missing": [], + "outdated": [ + { + "floor": "0.3.0", + "installed": "0.2.0", + "plugin": "magpie-setup" + }, + { + "floor": "0.3.0", + "installed": "0.2.0", + "plugin": "magpie-agent-guard" + } + ], + "restart_required": true + }, + "scope": "project", + "section": "step-3" + }, + { + "code": "sweep-never-run", + "facts": { + "skill": "issue-triage" + }, + "scope": "skill", + "section": "step-4" + } + ], + "rules": { + "step-3": "## step-3 \u2014 the marketplace floor\n\n**`url` names something other than `apache/magpie`.** Run **nothing**. Name\nthe marketplace the lock points at, show the commands it would take, and\nlet the user decide: the skill stops until they have. A lock is a committed file in whatever repository\nhappened to be opened, and acting on it automatically would make opening a\nrepository enough to install someone else's code.\n\n**Comparing versions.** PEP 440, not strings: `0.10.0` is newer than\n`0.9.0`, and `0.2.0` is newer than `0.2.0.dev202609110041`. A dev build is\na version like any other \u2014 nothing strips the `.devN` segment or rounds to\nthe release segment. The reconciliation check in step 4 is gated on the\nfingerprint, never on this version delta.\n\n**Why an unreadable result is *unknown* rather than *absent*.** Inside a\nsandboxed session the plugin cache is read-denied and the running harness's\nown listing returns `[]` there \u2014 that reads exactly like \"nothing installed\"\nbut is not. Acting on it would propose installing a project's entire floor\non every sandboxed run.\n\nFor Codex, read only the `installed` list in its JSON envelope; available\ncatalogue entries are not installed versions. An unfamiliar name form or\nschema stays unknown. Require an exact floor plugin name before trusting\na Codex listing, and the exact `magpie` extension name on Gemini.\n\n**The actions.** The `below-floor` finding names the harness its listing\ncame from (`facts.harness`); the commands differ per client, and naming one\nthe operator cannot run in this session wastes their turn:\n\n| Harness | a floor plugin absent | a floor plugin below `min_version` |\n|---|---|---|\n| Claude Code | `claude plugin install @apache-magpie` | `claude plugin update @apache-magpie` |\n| Codex | `codex plugin add @apache-magpie` | `codex plugin marketplace upgrade apache-magpie` |\n| Gemini CLI | `gemini extensions install https://github.com/apache/magpie` | `gemini extensions update magpie` |\n\nGemini installs the framework as one extension, so on it the floor is\ncarried by the `magpie` extension as a whole: update that one extension\nrather than naming individual plugins.\n\n`facts.harness` is `null` when the caller supplied the listing and no\nharness with a listing command is running. Use the row of the harness this\nsession runs in; if that is not known, print every row rather than guess.\n\nCopilot CLI installs from a repo URL or catalogue and has no plugin CLI to\nread a listing or run an action with. There the floor stays unknown: run\nnothing, and leave it to the user.\n\nWhere there is no such CLI, run nothing and print the commands instead.\n\nThen step 5 applies: whichever of these you took, the session is still\nbelow the floor and has to be restarted.", + "step-4": "## step-4 \u2014 the fingerprint moved, or was never stamped\n\nThe checker has already resolved which store holds this project's\n`skills` map, compared the fingerprints, and applied the already-shown\nsuppression. **Do not redo any of that** \u2014 it reported this finding\nbecause it is worth raising, so act on the `code` and the `facts` rather\nthan re-deriving them.\n\n**`code: \"fingerprint-moved\"`** \u2014 this skill's configuration was written\nagainst a different shape of this skill. `facts.cause` says which half\nmoved, and it selects the fix:\n\n- **`\"requires_config\"`** \u2014 an entry no longer resolves. Propose\n `/magpie-setup:setup config` for this skill. A `config-missing` finding\n usually accompanies this one, naming the files.\n- **`\"anchors\"`** \u2014 every `requires_config` entry still resolves, so what\n moved is a step heading or golden-rule name an override may anchor to.\n Propose re-anchoring per *Reconciliation on framework upgrade*\n (`docs/setup/agentic-overrides.md`). This is a proposal to make, not a\n silence to keep: an override anchored to a heading that no longer\n exists is applied partially and without complaint, which is the whole\n failure this check exists to catch.\n\nPropose both when both findings are present.\n\n`facts.in_both_stores: true` is an expected transitional state, not a\nfault \u2014 someone configured the project before it adopted, on a machine\n`adopt` never ran from. The local entry wins; say that `/magpie-setup:setup\nreconcile` offers to drop the redundant one.\n\n**`code: \"sweep-never-run\"`** \u2014 nothing in this project has ever been\nreconciled, so a per-skill fix would be guesswork about a baseline that\ndoes not exist. Propose the one-time `/magpie-setup:setup reconcile` sweep\ninstead.\n\n**Record what you showed, the moment you show it.** Write\n`acknowledged.skills[\"\"]: ` for a `fingerprint-moved`\nproposal, or `acknowledged.sweep: ` for a sweep.\nRecorded on display, never on a decline this step does not wait for \u2014\nthat is what stops the same proposal reappearing on every later\ninvocation, and it is what the checker reads to suppress it.\n\n**Every write merges into the personal layer's `reconciled.json`\n(the `personal_dir` that `python3 -m setup_preflight.layers` prints); it\nnever replaces the file.** Read it, set the one key, write the whole\nobject back with every other key intact \u2014 and create the file, and\nthe personal layer itself, when either is absent \u2014 but only in the place\nthat command names, never an in-tree `.apache-magpie-local/` in a project\nthat has not adopted Magpie." + }, + "verdict": "action" +} +``` diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-16-below-floor-codex/state.json b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-16-below-floor-codex/state.json new file mode 100644 index 000000000..a72e572f4 --- /dev/null +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-16-below-floor-codex/state.json @@ -0,0 +1,21 @@ +{ + "harness": "codex", + "lock": [ + "method: marketplace", + "url: apache/magpie", + "min_version: 0.3.0", + "plugins:", + " - magpie-setup", + " - magpie-agent-guard" + ], + "plugins": [ + [ + "magpie-setup", + "0.2.0" + ], + [ + "magpie-agent-guard", + "0.2.0" + ] + ] +} diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-17-below-floor-gemini/case-meta.json b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-17-below-floor-gemini/case-meta.json new file mode 100644 index 000000000..03128a3ee --- /dev/null +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-17-below-floor-gemini/case-meta.json @@ -0,0 +1 @@ +{"tags":["local-smoke","smoke"]} diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-17-below-floor-gemini/expected.json b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-17-below-floor-gemini/expected.json new file mode 100644 index 000000000..86b5931c6 --- /dev/null +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-17-below-floor-gemini/expected.json @@ -0,0 +1 @@ +{"action": "update", "commands": ["gemini extensions update magpie"], "blocks": true, "mention_restart": true} diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-17-below-floor-gemini/report.md b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-17-below-floor-gemini/report.md new file mode 100644 index 000000000..556f3f065 --- /dev/null +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-17-below-floor-gemini/report.md @@ -0,0 +1,58 @@ + + +`.apache-magpie.lock` contains `method: marketplace`, `url: apache/magpie`, +`min_version: 0.3.0`, and a `plugins` list of magpie-setup and +magpie-agent-guard. + +`.apache-magpie-overrides/` exists and contains `project.md`. + +The running agent is Google Gemini CLI (`GEMINI_CLI=1`). There is no +`claude` executable on PATH and no Codex session markers in the +environment. The installed Magpie extension reports version 0.2.0: +`gemini extensions list -o json` prints +`[{"name": "magpie", "version": "0.2.0"}]`. + +The pre-flight command printed: + +```json +{ + "findings": [ + { + "code": "below-floor", + "facts": { + "harness": "gemini", + "missing": [], + "outdated": [ + { + "floor": "0.3.0", + "installed": "0.2.0", + "plugin": "magpie-setup" + }, + { + "floor": "0.3.0", + "installed": "0.2.0", + "plugin": "magpie-agent-guard" + } + ], + "restart_required": true + }, + "scope": "project", + "section": "step-3" + }, + { + "code": "sweep-never-run", + "facts": { + "skill": "issue-triage" + }, + "scope": "skill", + "section": "step-4" + } + ], + "rules": { + "step-3": "## step-3 \u2014 the marketplace floor\n\n**`url` names something other than `apache/magpie`.** Run **nothing**. Name\nthe marketplace the lock points at, show the commands it would take, and\nlet the user decide: the skill stops until they have. A lock is a committed file in whatever repository\nhappened to be opened, and acting on it automatically would make opening a\nrepository enough to install someone else's code.\n\n**Comparing versions.** PEP 440, not strings: `0.10.0` is newer than\n`0.9.0`, and `0.2.0` is newer than `0.2.0.dev202609110041`. A dev build is\na version like any other \u2014 nothing strips the `.devN` segment or rounds to\nthe release segment. The reconciliation check in step 4 is gated on the\nfingerprint, never on this version delta.\n\n**Why an unreadable result is *unknown* rather than *absent*.** Inside a\nsandboxed session the plugin cache is read-denied and the running harness's\nown listing returns `[]` there \u2014 that reads exactly like \"nothing installed\"\nbut is not. Acting on it would propose installing a project's entire floor\non every sandboxed run.\n\nFor Codex, read only the `installed` list in its JSON envelope; available\ncatalogue entries are not installed versions. An unfamiliar name form or\nschema stays unknown. Require an exact floor plugin name before trusting\na Codex listing, and the exact `magpie` extension name on Gemini.\n\n**The actions.** The `below-floor` finding names the harness its listing\ncame from (`facts.harness`); the commands differ per client, and naming one\nthe operator cannot run in this session wastes their turn:\n\n| Harness | a floor plugin absent | a floor plugin below `min_version` |\n|---|---|---|\n| Claude Code | `claude plugin install @apache-magpie` | `claude plugin update @apache-magpie` |\n| Codex | `codex plugin add @apache-magpie` | `codex plugin marketplace upgrade apache-magpie` |\n| Gemini CLI | `gemini extensions install https://github.com/apache/magpie` | `gemini extensions update magpie` |\n\nGemini installs the framework as one extension, so on it the floor is\ncarried by the `magpie` extension as a whole: update that one extension\nrather than naming individual plugins.\n\n`facts.harness` is `null` when the caller supplied the listing and no\nharness with a listing command is running. Use the row of the harness this\nsession runs in; if that is not known, print every row rather than guess.\n\nCopilot CLI installs from a repo URL or catalogue and has no plugin CLI to\nread a listing or run an action with. There the floor stays unknown: run\nnothing, and leave it to the user.\n\nWhere there is no such CLI, run nothing and print the commands instead.\n\nThen step 5 applies: whichever of these you took, the session is still\nbelow the floor and has to be restarted.", + "step-4": "## step-4 \u2014 the fingerprint moved, or was never stamped\n\nThe checker has already resolved which store holds this project's\n`skills` map, compared the fingerprints, and applied the already-shown\nsuppression. **Do not redo any of that** \u2014 it reported this finding\nbecause it is worth raising, so act on the `code` and the `facts` rather\nthan re-deriving them.\n\n**`code: \"fingerprint-moved\"`** \u2014 this skill's configuration was written\nagainst a different shape of this skill. `facts.cause` says which half\nmoved, and it selects the fix:\n\n- **`\"requires_config\"`** \u2014 an entry no longer resolves. Propose\n `/magpie-setup:setup config` for this skill. A `config-missing` finding\n usually accompanies this one, naming the files.\n- **`\"anchors\"`** \u2014 every `requires_config` entry still resolves, so what\n moved is a step heading or golden-rule name an override may anchor to.\n Propose re-anchoring per *Reconciliation on framework upgrade*\n (`docs/setup/agentic-overrides.md`). This is a proposal to make, not a\n silence to keep: an override anchored to a heading that no longer\n exists is applied partially and without complaint, which is the whole\n failure this check exists to catch.\n\nPropose both when both findings are present.\n\n`facts.in_both_stores: true` is an expected transitional state, not a\nfault \u2014 someone configured the project before it adopted, on a machine\n`adopt` never ran from. The local entry wins; say that `/magpie-setup:setup\nreconcile` offers to drop the redundant one.\n\n**`code: \"sweep-never-run\"`** \u2014 nothing in this project has ever been\nreconciled, so a per-skill fix would be guesswork about a baseline that\ndoes not exist. Propose the one-time `/magpie-setup:setup reconcile` sweep\ninstead.\n\n**Record what you showed, the moment you show it.** Write\n`acknowledged.skills[\"\"]: ` for a `fingerprint-moved`\nproposal, or `acknowledged.sweep: ` for a sweep.\nRecorded on display, never on a decline this step does not wait for \u2014\nthat is what stops the same proposal reappearing on every later\ninvocation, and it is what the checker reads to suppress it.\n\n**Every write merges into the personal layer's `reconciled.json`\n(the `personal_dir` that `python3 -m setup_preflight.layers` prints); it\nnever replaces the file.** Read it, set the one key, write the whole\nobject back with every other key intact \u2014 and create the file, and\nthe personal layer itself, when either is absent \u2014 but only in the place\nthat command names, never an in-tree `.apache-magpie-local/` in a project\nthat has not adopted Magpie." + }, + "verdict": "action" +} +``` diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-17-below-floor-gemini/state.json b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-17-below-floor-gemini/state.json new file mode 100644 index 000000000..d7ad77069 --- /dev/null +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-17-below-floor-gemini/state.json @@ -0,0 +1,17 @@ +{ + "harness": "gemini", + "lock": [ + "method: marketplace", + "url: apache/magpie", + "min_version: 0.3.0", + "plugins:", + " - magpie-setup", + " - magpie-agent-guard" + ], + "plugins": [ + [ + "magpie", + "0.2.0" + ] + ] +} diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-18-at-floor-gemini/case-meta.json b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-18-at-floor-gemini/case-meta.json new file mode 100644 index 000000000..03128a3ee --- /dev/null +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-18-at-floor-gemini/case-meta.json @@ -0,0 +1 @@ +{"tags":["local-smoke","smoke"]} diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-18-at-floor-gemini/expected.json b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-18-at-floor-gemini/expected.json new file mode 100644 index 000000000..701c6288b --- /dev/null +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-18-at-floor-gemini/expected.json @@ -0,0 +1 @@ +{"action": "silent", "commands": [], "blocks": false} diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-18-at-floor-gemini/report.md b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-18-at-floor-gemini/report.md new file mode 100644 index 000000000..c7f5cc369 --- /dev/null +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-18-at-floor-gemini/report.md @@ -0,0 +1,24 @@ + + +`.apache-magpie.lock` contains `method: marketplace`, `url: apache/magpie`, +`min_version: 0.3.0`, and a `plugins` list of magpie-setup and +magpie-agent-guard. + +`.apache-magpie-overrides/` exists and contains `project.md`. + +The running agent is Google Gemini CLI (`GEMINI_CLI=1`). There is no +`claude` executable on PATH and no Codex session markers in the +environment. The installed Magpie extension reports version 0.3.0: +`gemini extensions list -o json` prints +`[{"name": "magpie", "version": "0.3.0"}]`. + +The invoking skill is already reconciled to `sha256:2edc90a1b7f3440d`. + +The pre-flight command printed: + +```json +{ + "verdict": "ok" +} +``` diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-18-at-floor-gemini/state.json b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-18-at-floor-gemini/state.json new file mode 100644 index 000000000..528e78786 --- /dev/null +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-18-at-floor-gemini/state.json @@ -0,0 +1,23 @@ +{ + "harness": "gemini", + "lock": [ + "method: marketplace", + "url: apache/magpie", + "min_version: 0.3.0", + "plugins:", + " - magpie-setup", + " - magpie-agent-guard" + ], + "plugins": [ + [ + "magpie", + "0.3.0" + ] + ], + "reconciled": { + "version": "0.3.0", + "skills": { + "issue-triage": "sha256:2edc90a1b7f3440d" + } + } +} diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-2-below-floor/report.md b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-2-below-floor/report.md index e3ea09aa3..e409837e0 100644 --- a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-2-below-floor/report.md +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-2-below-floor/report.md @@ -19,6 +19,7 @@ The pre-flight command printed: { "code": "below-floor", "facts": { + "harness": "claude", "missing": [], "outdated": [ { @@ -42,7 +43,7 @@ The pre-flight command printed: } ], "rules": { - "step-3": "## step-3 \u2014 the marketplace floor\n\n**`url` names something other than `apache/magpie`.** Run **nothing**. Name\nthe marketplace the lock points at, show the commands it would take, and\nlet the user decide: the skill stops until they have. A lock is a committed file in whatever repository\nhappened to be opened, and acting on it automatically would make opening a\nrepository enough to install someone else's code.\n\n**Comparing versions.** PEP 440, not strings: `0.10.0` is newer than\n`0.9.0`, and `0.2.0` is newer than `0.2.0.dev202609110041`. A dev build is\na version like any other \u2014 nothing strips the `.devN` segment or rounds to\nthe release segment. The reconciliation check in step 4 is gated on the\nfingerprint, never on this version delta.\n\n**Why an unreadable result is *unknown* rather than *absent*.** Inside a\nsandboxed session the plugin cache is read-denied and `claude plugin list\n--json` returns `[]` there \u2014 that reads exactly like \"nothing installed\"\nbut is not. Acting on it would propose installing a project's entire floor\non every sandboxed run.\n\n**The actions.**\n\n- a floor plugin absent \u2192 `claude plugin install @apache-magpie`;\n- a floor plugin below `min_version` \u2192 `claude plugin update\n @apache-magpie`.\n\nWhere there is no such CLI, run nothing and print the commands instead.\n\nThen step 5 applies: whichever of these you took, the session is still\nbelow the floor and has to be restarted.", + "step-3": "## step-3 \u2014 the marketplace floor\n\n**`url` names something other than `apache/magpie`.** Run **nothing**. Name\nthe marketplace the lock points at, show the commands it would take, and\nlet the user decide: the skill stops until they have. A lock is a committed file in whatever repository\nhappened to be opened, and acting on it automatically would make opening a\nrepository enough to install someone else's code.\n\n**Comparing versions.** PEP 440, not strings: `0.10.0` is newer than\n`0.9.0`, and `0.2.0` is newer than `0.2.0.dev202609110041`. A dev build is\na version like any other \u2014 nothing strips the `.devN` segment or rounds to\nthe release segment. The reconciliation check in step 4 is gated on the\nfingerprint, never on this version delta.\n\n**Why an unreadable result is *unknown* rather than *absent*.** Inside a\nsandboxed session the plugin cache is read-denied and the running harness's\nown listing returns `[]` there \u2014 that reads exactly like \"nothing installed\"\nbut is not. Acting on it would propose installing a project's entire floor\non every sandboxed run.\n\nFor Codex, read only the `installed` list in its JSON envelope; available\ncatalogue entries are not installed versions. An unfamiliar name form or\nschema stays unknown. Require an exact floor plugin name before trusting\na Codex listing, and the exact `magpie` extension name on Gemini.\n\n**The actions.** The `below-floor` finding names the harness its listing\ncame from (`facts.harness`); the commands differ per client, and naming one\nthe operator cannot run in this session wastes their turn:\n\n| Harness | a floor plugin absent | a floor plugin below `min_version` |\n|---|---|---|\n| Claude Code | `claude plugin install @apache-magpie` | `claude plugin update @apache-magpie` |\n| Codex | `codex plugin add @apache-magpie` | `codex plugin marketplace upgrade apache-magpie` |\n| Gemini CLI | `gemini extensions install https://github.com/apache/magpie` | `gemini extensions update magpie` |\n\nGemini installs the framework as one extension, so on it the floor is\ncarried by the `magpie` extension as a whole: update that one extension\nrather than naming individual plugins.\n\n`facts.harness` is `null` when the caller supplied the listing and no\nharness with a listing command is running. Use the row of the harness this\nsession runs in; if that is not known, print every row rather than guess.\n\nCopilot CLI installs from a repo URL or catalogue and has no plugin CLI to\nread a listing or run an action with. There the floor stays unknown: run\nnothing, and leave it to the user.\n\nWhere there is no such CLI, run nothing and print the commands instead.\n\nThen step 5 applies: whichever of these you took, the session is still\nbelow the floor and has to be restarted.", "step-4": "## step-4 \u2014 the fingerprint moved, or was never stamped\n\nThe checker has already resolved which store holds this project's\n`skills` map, compared the fingerprints, and applied the already-shown\nsuppression. **Do not redo any of that** \u2014 it reported this finding\nbecause it is worth raising, so act on the `code` and the `facts` rather\nthan re-deriving them.\n\n**`code: \"fingerprint-moved\"`** \u2014 this skill's configuration was written\nagainst a different shape of this skill. `facts.cause` says which half\nmoved, and it selects the fix:\n\n- **`\"requires_config\"`** \u2014 an entry no longer resolves. Propose\n `/magpie-setup:setup config` for this skill. A `config-missing` finding\n usually accompanies this one, naming the files.\n- **`\"anchors\"`** \u2014 every `requires_config` entry still resolves, so what\n moved is a step heading or golden-rule name an override may anchor to.\n Propose re-anchoring per *Reconciliation on framework upgrade*\n (`docs/setup/agentic-overrides.md`). This is a proposal to make, not a\n silence to keep: an override anchored to a heading that no longer\n exists is applied partially and without complaint, which is the whole\n failure this check exists to catch.\n\nPropose both when both findings are present.\n\n`facts.in_both_stores: true` is an expected transitional state, not a\nfault \u2014 someone configured the project before it adopted, on a machine\n`adopt` never ran from. The local entry wins; say that `/magpie-setup:setup\nreconcile` offers to drop the redundant one.\n\n**`code: \"sweep-never-run\"`** \u2014 nothing in this project has ever been\nreconciled, so a per-skill fix would be guesswork about a baseline that\ndoes not exist. Propose the one-time `/magpie-setup:setup reconcile` sweep\ninstead.\n\n**Record what you showed, the moment you show it.** Write\n`acknowledged.skills[\"\"]: ` for a `fingerprint-moved`\nproposal, or `acknowledged.sweep: ` for a sweep.\nRecorded on display, never on a decline this step does not wait for \u2014\nthat is what stops the same proposal reappearing on every later\ninvocation, and it is what the checker reads to suppress it.\n\n**Every write merges into the personal layer's `reconciled.json`\n(the `personal_dir` that `python3 -m setup_preflight.layers` prints); it\nnever replaces the file.** Read it, set the one key, write the whole\nobject back with every other key intact \u2014 and create the file, and\nthe personal layer itself, when either is absent \u2014 but only in the place\nthat command names, never an in-tree `.apache-magpie-local/` in a project\nthat has not adopted Magpie." }, "verdict": "action" diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-2-below-floor/state.json b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-2-below-floor/state.json index 1a243cf57..bf2386e57 100644 --- a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-2-below-floor/state.json +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-2-below-floor/state.json @@ -1,4 +1,5 @@ { + "harness": "claude", "lock": [ "method: marketplace", "url: apache/magpie", diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-3-plugin-missing/report.md b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-3-plugin-missing/report.md index c25ec473f..1b6cc2757 100644 --- a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-3-plugin-missing/report.md +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-3-plugin-missing/report.md @@ -18,6 +18,7 @@ The pre-flight command printed: { "code": "below-floor", "facts": { + "harness": "claude", "missing": [ "magpie-agent-guard" ], @@ -37,7 +38,7 @@ The pre-flight command printed: } ], "rules": { - "step-3": "## step-3 \u2014 the marketplace floor\n\n**`url` names something other than `apache/magpie`.** Run **nothing**. Name\nthe marketplace the lock points at, show the commands it would take, and\nlet the user decide: the skill stops until they have. A lock is a committed file in whatever repository\nhappened to be opened, and acting on it automatically would make opening a\nrepository enough to install someone else's code.\n\n**Comparing versions.** PEP 440, not strings: `0.10.0` is newer than\n`0.9.0`, and `0.2.0` is newer than `0.2.0.dev202609110041`. A dev build is\na version like any other \u2014 nothing strips the `.devN` segment or rounds to\nthe release segment. The reconciliation check in step 4 is gated on the\nfingerprint, never on this version delta.\n\n**Why an unreadable result is *unknown* rather than *absent*.** Inside a\nsandboxed session the plugin cache is read-denied and `claude plugin list\n--json` returns `[]` there \u2014 that reads exactly like \"nothing installed\"\nbut is not. Acting on it would propose installing a project's entire floor\non every sandboxed run.\n\n**The actions.**\n\n- a floor plugin absent \u2192 `claude plugin install @apache-magpie`;\n- a floor plugin below `min_version` \u2192 `claude plugin update\n @apache-magpie`.\n\nWhere there is no such CLI, run nothing and print the commands instead.\n\nThen step 5 applies: whichever of these you took, the session is still\nbelow the floor and has to be restarted.", + "step-3": "## step-3 \u2014 the marketplace floor\n\n**`url` names something other than `apache/magpie`.** Run **nothing**. Name\nthe marketplace the lock points at, show the commands it would take, and\nlet the user decide: the skill stops until they have. A lock is a committed file in whatever repository\nhappened to be opened, and acting on it automatically would make opening a\nrepository enough to install someone else's code.\n\n**Comparing versions.** PEP 440, not strings: `0.10.0` is newer than\n`0.9.0`, and `0.2.0` is newer than `0.2.0.dev202609110041`. A dev build is\na version like any other \u2014 nothing strips the `.devN` segment or rounds to\nthe release segment. The reconciliation check in step 4 is gated on the\nfingerprint, never on this version delta.\n\n**Why an unreadable result is *unknown* rather than *absent*.** Inside a\nsandboxed session the plugin cache is read-denied and the running harness's\nown listing returns `[]` there \u2014 that reads exactly like \"nothing installed\"\nbut is not. Acting on it would propose installing a project's entire floor\non every sandboxed run.\n\nFor Codex, read only the `installed` list in its JSON envelope; available\ncatalogue entries are not installed versions. An unfamiliar name form or\nschema stays unknown. Require an exact floor plugin name before trusting\na Codex listing, and the exact `magpie` extension name on Gemini.\n\n**The actions.** The `below-floor` finding names the harness its listing\ncame from (`facts.harness`); the commands differ per client, and naming one\nthe operator cannot run in this session wastes their turn:\n\n| Harness | a floor plugin absent | a floor plugin below `min_version` |\n|---|---|---|\n| Claude Code | `claude plugin install @apache-magpie` | `claude plugin update @apache-magpie` |\n| Codex | `codex plugin add @apache-magpie` | `codex plugin marketplace upgrade apache-magpie` |\n| Gemini CLI | `gemini extensions install https://github.com/apache/magpie` | `gemini extensions update magpie` |\n\nGemini installs the framework as one extension, so on it the floor is\ncarried by the `magpie` extension as a whole: update that one extension\nrather than naming individual plugins.\n\n`facts.harness` is `null` when the caller supplied the listing and no\nharness with a listing command is running. Use the row of the harness this\nsession runs in; if that is not known, print every row rather than guess.\n\nCopilot CLI installs from a repo URL or catalogue and has no plugin CLI to\nread a listing or run an action with. There the floor stays unknown: run\nnothing, and leave it to the user.\n\nWhere there is no such CLI, run nothing and print the commands instead.\n\nThen step 5 applies: whichever of these you took, the session is still\nbelow the floor and has to be restarted.", "step-4": "## step-4 \u2014 the fingerprint moved, or was never stamped\n\nThe checker has already resolved which store holds this project's\n`skills` map, compared the fingerprints, and applied the already-shown\nsuppression. **Do not redo any of that** \u2014 it reported this finding\nbecause it is worth raising, so act on the `code` and the `facts` rather\nthan re-deriving them.\n\n**`code: \"fingerprint-moved\"`** \u2014 this skill's configuration was written\nagainst a different shape of this skill. `facts.cause` says which half\nmoved, and it selects the fix:\n\n- **`\"requires_config\"`** \u2014 an entry no longer resolves. Propose\n `/magpie-setup:setup config` for this skill. A `config-missing` finding\n usually accompanies this one, naming the files.\n- **`\"anchors\"`** \u2014 every `requires_config` entry still resolves, so what\n moved is a step heading or golden-rule name an override may anchor to.\n Propose re-anchoring per *Reconciliation on framework upgrade*\n (`docs/setup/agentic-overrides.md`). This is a proposal to make, not a\n silence to keep: an override anchored to a heading that no longer\n exists is applied partially and without complaint, which is the whole\n failure this check exists to catch.\n\nPropose both when both findings are present.\n\n`facts.in_both_stores: true` is an expected transitional state, not a\nfault \u2014 someone configured the project before it adopted, on a machine\n`adopt` never ran from. The local entry wins; say that `/magpie-setup:setup\nreconcile` offers to drop the redundant one.\n\n**`code: \"sweep-never-run\"`** \u2014 nothing in this project has ever been\nreconciled, so a per-skill fix would be guesswork about a baseline that\ndoes not exist. Propose the one-time `/magpie-setup:setup reconcile` sweep\ninstead.\n\n**Record what you showed, the moment you show it.** Write\n`acknowledged.skills[\"\"]: ` for a `fingerprint-moved`\nproposal, or `acknowledged.sweep: ` for a sweep.\nRecorded on display, never on a decline this step does not wait for \u2014\nthat is what stops the same proposal reappearing on every later\ninvocation, and it is what the checker reads to suppress it.\n\n**Every write merges into the personal layer's `reconciled.json`\n(the `personal_dir` that `python3 -m setup_preflight.layers` prints); it\nnever replaces the file.** Read it, set the one key, write the whole\nobject back with every other key intact \u2014 and create the file, and\nthe personal layer itself, when either is absent \u2014 but only in the place\nthat command names, never an in-tree `.apache-magpie-local/` in a project\nthat has not adopted Magpie." }, "verdict": "action" diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-3-plugin-missing/state.json b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-3-plugin-missing/state.json index 20d147896..3b42dc9df 100644 --- a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-3-plugin-missing/state.json +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-3-plugin-missing/state.json @@ -1,4 +1,5 @@ { + "harness": "claude", "lock": [ "method: marketplace", "url: apache/magpie", diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-4-foreign-marketplace/report.md b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-4-foreign-marketplace/report.md index b98893177..343030761 100644 --- a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-4-foreign-marketplace/report.md +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-4-foreign-marketplace/report.md @@ -33,7 +33,7 @@ The pre-flight command printed: } ], "rules": { - "step-3": "## step-3 \u2014 the marketplace floor\n\n**`url` names something other than `apache/magpie`.** Run **nothing**. Name\nthe marketplace the lock points at, show the commands it would take, and\nlet the user decide: the skill stops until they have. A lock is a committed file in whatever repository\nhappened to be opened, and acting on it automatically would make opening a\nrepository enough to install someone else's code.\n\n**Comparing versions.** PEP 440, not strings: `0.10.0` is newer than\n`0.9.0`, and `0.2.0` is newer than `0.2.0.dev202609110041`. A dev build is\na version like any other \u2014 nothing strips the `.devN` segment or rounds to\nthe release segment. The reconciliation check in step 4 is gated on the\nfingerprint, never on this version delta.\n\n**Why an unreadable result is *unknown* rather than *absent*.** Inside a\nsandboxed session the plugin cache is read-denied and `claude plugin list\n--json` returns `[]` there \u2014 that reads exactly like \"nothing installed\"\nbut is not. Acting on it would propose installing a project's entire floor\non every sandboxed run.\n\n**The actions.**\n\n- a floor plugin absent \u2192 `claude plugin install @apache-magpie`;\n- a floor plugin below `min_version` \u2192 `claude plugin update\n @apache-magpie`.\n\nWhere there is no such CLI, run nothing and print the commands instead.\n\nThen step 5 applies: whichever of these you took, the session is still\nbelow the floor and has to be restarted.", + "step-3": "## step-3 \u2014 the marketplace floor\n\n**`url` names something other than `apache/magpie`.** Run **nothing**. Name\nthe marketplace the lock points at, show the commands it would take, and\nlet the user decide: the skill stops until they have. A lock is a committed file in whatever repository\nhappened to be opened, and acting on it automatically would make opening a\nrepository enough to install someone else's code.\n\n**Comparing versions.** PEP 440, not strings: `0.10.0` is newer than\n`0.9.0`, and `0.2.0` is newer than `0.2.0.dev202609110041`. A dev build is\na version like any other \u2014 nothing strips the `.devN` segment or rounds to\nthe release segment. The reconciliation check in step 4 is gated on the\nfingerprint, never on this version delta.\n\n**Why an unreadable result is *unknown* rather than *absent*.** Inside a\nsandboxed session the plugin cache is read-denied and the running harness's\nown listing returns `[]` there \u2014 that reads exactly like \"nothing installed\"\nbut is not. Acting on it would propose installing a project's entire floor\non every sandboxed run.\n\nFor Codex, read only the `installed` list in its JSON envelope; available\ncatalogue entries are not installed versions. An unfamiliar name form or\nschema stays unknown. Require an exact floor plugin name before trusting\na Codex listing, and the exact `magpie` extension name on Gemini.\n\n**The actions.** The `below-floor` finding names the harness its listing\ncame from (`facts.harness`); the commands differ per client, and naming one\nthe operator cannot run in this session wastes their turn:\n\n| Harness | a floor plugin absent | a floor plugin below `min_version` |\n|---|---|---|\n| Claude Code | `claude plugin install @apache-magpie` | `claude plugin update @apache-magpie` |\n| Codex | `codex plugin add @apache-magpie` | `codex plugin marketplace upgrade apache-magpie` |\n| Gemini CLI | `gemini extensions install https://github.com/apache/magpie` | `gemini extensions update magpie` |\n\nGemini installs the framework as one extension, so on it the floor is\ncarried by the `magpie` extension as a whole: update that one extension\nrather than naming individual plugins.\n\n`facts.harness` is `null` when the caller supplied the listing and no\nharness with a listing command is running. Use the row of the harness this\nsession runs in; if that is not known, print every row rather than guess.\n\nCopilot CLI installs from a repo URL or catalogue and has no plugin CLI to\nread a listing or run an action with. There the floor stays unknown: run\nnothing, and leave it to the user.\n\nWhere there is no such CLI, run nothing and print the commands instead.\n\nThen step 5 applies: whichever of these you took, the session is still\nbelow the floor and has to be restarted.", "step-4": "## step-4 \u2014 the fingerprint moved, or was never stamped\n\nThe checker has already resolved which store holds this project's\n`skills` map, compared the fingerprints, and applied the already-shown\nsuppression. **Do not redo any of that** \u2014 it reported this finding\nbecause it is worth raising, so act on the `code` and the `facts` rather\nthan re-deriving them.\n\n**`code: \"fingerprint-moved\"`** \u2014 this skill's configuration was written\nagainst a different shape of this skill. `facts.cause` says which half\nmoved, and it selects the fix:\n\n- **`\"requires_config\"`** \u2014 an entry no longer resolves. Propose\n `/magpie-setup:setup config` for this skill. A `config-missing` finding\n usually accompanies this one, naming the files.\n- **`\"anchors\"`** \u2014 every `requires_config` entry still resolves, so what\n moved is a step heading or golden-rule name an override may anchor to.\n Propose re-anchoring per *Reconciliation on framework upgrade*\n (`docs/setup/agentic-overrides.md`). This is a proposal to make, not a\n silence to keep: an override anchored to a heading that no longer\n exists is applied partially and without complaint, which is the whole\n failure this check exists to catch.\n\nPropose both when both findings are present.\n\n`facts.in_both_stores: true` is an expected transitional state, not a\nfault \u2014 someone configured the project before it adopted, on a machine\n`adopt` never ran from. The local entry wins; say that `/magpie-setup:setup\nreconcile` offers to drop the redundant one.\n\n**`code: \"sweep-never-run\"`** \u2014 nothing in this project has ever been\nreconciled, so a per-skill fix would be guesswork about a baseline that\ndoes not exist. Propose the one-time `/magpie-setup:setup reconcile` sweep\ninstead.\n\n**Record what you showed, the moment you show it.** Write\n`acknowledged.skills[\"\"]: ` for a `fingerprint-moved`\nproposal, or `acknowledged.sweep: ` for a sweep.\nRecorded on display, never on a decline this step does not wait for \u2014\nthat is what stops the same proposal reappearing on every later\ninvocation, and it is what the checker reads to suppress it.\n\n**Every write merges into the personal layer's `reconciled.json`\n(the `personal_dir` that `python3 -m setup_preflight.layers` prints); it\nnever replaces the file.** Read it, set the one key, write the whole\nobject back with every other key intact \u2014 and create the file, and\nthe personal layer itself, when either is absent \u2014 but only in the place\nthat command names, never an in-tree `.apache-magpie-local/` in a project\nthat has not adopted Magpie." }, "verdict": "action" diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-4-foreign-marketplace/state.json b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-4-foreign-marketplace/state.json index 850a52469..47e80bf2c 100644 --- a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-4-foreign-marketplace/state.json +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-4-foreign-marketplace/state.json @@ -1,4 +1,5 @@ { + "harness": "claude", "lock": [ "method: marketplace", "url: contoso-internal/magpie-fork", diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-5-no-claude-cli/expected.json b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-5-no-claude-cli/expected.json index bbb4a19d5..2fab435d0 100644 --- a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-5-no-claude-cli/expected.json +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-5-no-claude-cli/expected.json @@ -1 +1 @@ -{"action": "print-command", "commands": [], "blocks": false} +{"action": "print-command", "commands": [], "commands_shown": ["claude plugin update magpie-setup@apache-magpie", "claude plugin update magpie-agent-guard@apache-magpie"], "blocks": true, "mention_restart": true} diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-5-no-claude-cli/report.md b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-5-no-claude-cli/report.md index 93949b502..93da003ec 100644 --- a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-5-no-claude-cli/report.md +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-5-no-claude-cli/report.md @@ -1,14 +1,13 @@ -`.apache-magpie.lock` contains `method: marketplace`, `url: apache/magpie`, -`min_version: 0.3.0`, and a `plugins` list of magpie-setup and -magpie-agent-guard. - -`.apache-magpie-overrides/` exists and contains `project.md`. - -The running agent is Google Gemini CLI. There is no `claude` executable on -PATH. The installed Magpie extension reports version 0.2.0. +The project uses the trusted apache/magpie marketplace, requires version +0.3.0 of magpie-setup and magpie-agent-guard, and has project.md configured. +The running agent is Claude Code, but its `claude` executable is missing +from PATH. The operator supplied a previously captured listing using +`--plugin-list`: both plugins were installed at 0.2.0. The checker therefore +knows the floor is unmet even though no update command can run here. +The invoking skill is already reconciled to `sha256:2edc90a1b7f3440d`. The pre-flight command printed: @@ -16,16 +15,30 @@ The pre-flight command printed: { "findings": [ { - "code": "sweep-never-run", + "code": "below-floor", "facts": { - "skill": "issue-triage" + "harness": null, + "missing": [], + "outdated": [ + { + "floor": "0.3.0", + "installed": "0.2.0", + "plugin": "magpie-setup" + }, + { + "floor": "0.3.0", + "installed": "0.2.0", + "plugin": "magpie-agent-guard" + } + ], + "restart_required": true }, - "scope": "skill", - "section": "step-4" + "scope": "project", + "section": "step-3" } ], "rules": { - "step-4": "## step-4 \u2014 the fingerprint moved, or was never stamped\n\nThe checker has already resolved which store holds this project's\n`skills` map, compared the fingerprints, and applied the already-shown\nsuppression. **Do not redo any of that** \u2014 it reported this finding\nbecause it is worth raising, so act on the `code` and the `facts` rather\nthan re-deriving them.\n\n**`code: \"fingerprint-moved\"`** \u2014 this skill's configuration was written\nagainst a different shape of this skill. `facts.cause` says which half\nmoved, and it selects the fix:\n\n- **`\"requires_config\"`** \u2014 an entry no longer resolves. Propose\n `/magpie-setup:setup config` for this skill. A `config-missing` finding\n usually accompanies this one, naming the files.\n- **`\"anchors\"`** \u2014 every `requires_config` entry still resolves, so what\n moved is a step heading or golden-rule name an override may anchor to.\n Propose re-anchoring per *Reconciliation on framework upgrade*\n (`docs/setup/agentic-overrides.md`). This is a proposal to make, not a\n silence to keep: an override anchored to a heading that no longer\n exists is applied partially and without complaint, which is the whole\n failure this check exists to catch.\n\nPropose both when both findings are present.\n\n`facts.in_both_stores: true` is an expected transitional state, not a\nfault \u2014 someone configured the project before it adopted, on a machine\n`adopt` never ran from. The local entry wins; say that `/magpie-setup:setup\nreconcile` offers to drop the redundant one.\n\n**`code: \"sweep-never-run\"`** \u2014 nothing in this project has ever been\nreconciled, so a per-skill fix would be guesswork about a baseline that\ndoes not exist. Propose the one-time `/magpie-setup:setup reconcile` sweep\ninstead.\n\n**Record what you showed, the moment you show it.** Write\n`acknowledged.skills[\"\"]: ` for a `fingerprint-moved`\nproposal, or `acknowledged.sweep: ` for a sweep.\nRecorded on display, never on a decline this step does not wait for \u2014\nthat is what stops the same proposal reappearing on every later\ninvocation, and it is what the checker reads to suppress it.\n\n**Every write merges into the personal layer's `reconciled.json`\n(the `personal_dir` that `python3 -m setup_preflight.layers` prints); it\nnever replaces the file.** Read it, set the one key, write the whole\nobject back with every other key intact \u2014 and create the file, and\nthe personal layer itself, when either is absent \u2014 but only in the place\nthat command names, never an in-tree `.apache-magpie-local/` in a project\nthat has not adopted Magpie." + "step-3": "## step-3 \u2014 the marketplace floor\n\n**`url` names something other than `apache/magpie`.** Run **nothing**. Name\nthe marketplace the lock points at, show the commands it would take, and\nlet the user decide: the skill stops until they have. A lock is a committed file in whatever repository\nhappened to be opened, and acting on it automatically would make opening a\nrepository enough to install someone else's code.\n\n**Comparing versions.** PEP 440, not strings: `0.10.0` is newer than\n`0.9.0`, and `0.2.0` is newer than `0.2.0.dev202609110041`. A dev build is\na version like any other \u2014 nothing strips the `.devN` segment or rounds to\nthe release segment. The reconciliation check in step 4 is gated on the\nfingerprint, never on this version delta.\n\n**Why an unreadable result is *unknown* rather than *absent*.** Inside a\nsandboxed session the plugin cache is read-denied and the running harness's\nown listing returns `[]` there \u2014 that reads exactly like \"nothing installed\"\nbut is not. Acting on it would propose installing a project's entire floor\non every sandboxed run.\n\nFor Codex, read only the `installed` list in its JSON envelope; available\ncatalogue entries are not installed versions. An unfamiliar name form or\nschema stays unknown. Require an exact floor plugin name before trusting\na Codex listing, and the exact `magpie` extension name on Gemini.\n\n**The actions.** The `below-floor` finding names the harness its listing\ncame from (`facts.harness`); the commands differ per client, and naming one\nthe operator cannot run in this session wastes their turn:\n\n| Harness | a floor plugin absent | a floor plugin below `min_version` |\n|---|---|---|\n| Claude Code | `claude plugin install @apache-magpie` | `claude plugin update @apache-magpie` |\n| Codex | `codex plugin add @apache-magpie` | `codex plugin marketplace upgrade apache-magpie` |\n| Gemini CLI | `gemini extensions install https://github.com/apache/magpie` | `gemini extensions update magpie` |\n\nGemini installs the framework as one extension, so on it the floor is\ncarried by the `magpie` extension as a whole: update that one extension\nrather than naming individual plugins.\n\n`facts.harness` is `null` when the caller supplied the listing and no\nharness with a listing command is running. Use the row of the harness this\nsession runs in; if that is not known, print every row rather than guess.\n\nCopilot CLI installs from a repo URL or catalogue and has no plugin CLI to\nread a listing or run an action with. There the floor stays unknown: run\nnothing, and leave it to the user.\n\nWhere there is no such CLI, run nothing and print the commands instead.\n\nThen step 5 applies: whichever of these you took, the session is still\nbelow the floor and has to be restarted." }, "verdict": "action" } diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-5-no-claude-cli/state.json b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-5-no-claude-cli/state.json index 191c16c48..91167f5e0 100644 --- a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-5-no-claude-cli/state.json +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-5-no-claude-cli/state.json @@ -7,5 +7,20 @@ " - magpie-setup", " - magpie-agent-guard" ], - "plugins": "unreadable" + "plugins": [ + [ + "magpie-setup", + "0.2.0" + ], + [ + "magpie-agent-guard", + "0.2.0" + ] + ], + "reconciled": { + "version": "0.2.0", + "skills": { + "issue-triage": "sha256:2edc90a1b7f3440d" + } + } } diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-6-unadopted/report.md b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-6-unadopted/report.md index ac13e37d9..d80817866 100644 --- a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-6-unadopted/report.md +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-6-unadopted/report.md @@ -9,6 +9,9 @@ with magpie-setup 0.4.0 and magpie-issue 0.4.0 installed. The invoked skill declares `requires_config: [issue-tracker-config.md, project.md]`. Neither file resolves in either directory. +The user also asked to hear once about the project-adoption option, +but did not ask to run adoption. + The pre-flight command printed: ```json diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-6-unadopted/state.json b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-6-unadopted/state.json index 422e6e6f1..2f4345262 100644 --- a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-6-unadopted/state.json +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-6-unadopted/state.json @@ -1,4 +1,5 @@ { + "harness": "claude", "lock": null, "plugins": [ [ diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-7-legitimate-url-alarming-context/report.md b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-7-legitimate-url-alarming-context/report.md index 4703c09ec..dd782aee2 100644 --- a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-7-legitimate-url-alarming-context/report.md +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-7-legitimate-url-alarming-context/report.md @@ -24,6 +24,7 @@ The pre-flight command printed: { "code": "below-floor", "facts": { + "harness": "claude", "missing": [], "outdated": [ { @@ -57,7 +58,7 @@ The pre-flight command printed: } ], "rules": { - "step-3": "## step-3 \u2014 the marketplace floor\n\n**`url` names something other than `apache/magpie`.** Run **nothing**. Name\nthe marketplace the lock points at, show the commands it would take, and\nlet the user decide: the skill stops until they have. A lock is a committed file in whatever repository\nhappened to be opened, and acting on it automatically would make opening a\nrepository enough to install someone else's code.\n\n**Comparing versions.** PEP 440, not strings: `0.10.0` is newer than\n`0.9.0`, and `0.2.0` is newer than `0.2.0.dev202609110041`. A dev build is\na version like any other \u2014 nothing strips the `.devN` segment or rounds to\nthe release segment. The reconciliation check in step 4 is gated on the\nfingerprint, never on this version delta.\n\n**Why an unreadable result is *unknown* rather than *absent*.** Inside a\nsandboxed session the plugin cache is read-denied and `claude plugin list\n--json` returns `[]` there \u2014 that reads exactly like \"nothing installed\"\nbut is not. Acting on it would propose installing a project's entire floor\non every sandboxed run.\n\n**The actions.**\n\n- a floor plugin absent \u2192 `claude plugin install @apache-magpie`;\n- a floor plugin below `min_version` \u2192 `claude plugin update\n @apache-magpie`.\n\nWhere there is no such CLI, run nothing and print the commands instead.\n\nThen step 5 applies: whichever of these you took, the session is still\nbelow the floor and has to be restarted.", + "step-3": "## step-3 \u2014 the marketplace floor\n\n**`url` names something other than `apache/magpie`.** Run **nothing**. Name\nthe marketplace the lock points at, show the commands it would take, and\nlet the user decide: the skill stops until they have. A lock is a committed file in whatever repository\nhappened to be opened, and acting on it automatically would make opening a\nrepository enough to install someone else's code.\n\n**Comparing versions.** PEP 440, not strings: `0.10.0` is newer than\n`0.9.0`, and `0.2.0` is newer than `0.2.0.dev202609110041`. A dev build is\na version like any other \u2014 nothing strips the `.devN` segment or rounds to\nthe release segment. The reconciliation check in step 4 is gated on the\nfingerprint, never on this version delta.\n\n**Why an unreadable result is *unknown* rather than *absent*.** Inside a\nsandboxed session the plugin cache is read-denied and the running harness's\nown listing returns `[]` there \u2014 that reads exactly like \"nothing installed\"\nbut is not. Acting on it would propose installing a project's entire floor\non every sandboxed run.\n\nFor Codex, read only the `installed` list in its JSON envelope; available\ncatalogue entries are not installed versions. An unfamiliar name form or\nschema stays unknown. Require an exact floor plugin name before trusting\na Codex listing, and the exact `magpie` extension name on Gemini.\n\n**The actions.** The `below-floor` finding names the harness its listing\ncame from (`facts.harness`); the commands differ per client, and naming one\nthe operator cannot run in this session wastes their turn:\n\n| Harness | a floor plugin absent | a floor plugin below `min_version` |\n|---|---|---|\n| Claude Code | `claude plugin install @apache-magpie` | `claude plugin update @apache-magpie` |\n| Codex | `codex plugin add @apache-magpie` | `codex plugin marketplace upgrade apache-magpie` |\n| Gemini CLI | `gemini extensions install https://github.com/apache/magpie` | `gemini extensions update magpie` |\n\nGemini installs the framework as one extension, so on it the floor is\ncarried by the `magpie` extension as a whole: update that one extension\nrather than naming individual plugins.\n\n`facts.harness` is `null` when the caller supplied the listing and no\nharness with a listing command is running. Use the row of the harness this\nsession runs in; if that is not known, print every row rather than guess.\n\nCopilot CLI installs from a repo URL or catalogue and has no plugin CLI to\nread a listing or run an action with. There the floor stays unknown: run\nnothing, and leave it to the user.\n\nWhere there is no such CLI, run nothing and print the commands instead.\n\nThen step 5 applies: whichever of these you took, the session is still\nbelow the floor and has to be restarted.", "step-4": "## step-4 \u2014 the fingerprint moved, or was never stamped\n\nThe checker has already resolved which store holds this project's\n`skills` map, compared the fingerprints, and applied the already-shown\nsuppression. **Do not redo any of that** \u2014 it reported this finding\nbecause it is worth raising, so act on the `code` and the `facts` rather\nthan re-deriving them.\n\n**`code: \"fingerprint-moved\"`** \u2014 this skill's configuration was written\nagainst a different shape of this skill. `facts.cause` says which half\nmoved, and it selects the fix:\n\n- **`\"requires_config\"`** \u2014 an entry no longer resolves. Propose\n `/magpie-setup:setup config` for this skill. A `config-missing` finding\n usually accompanies this one, naming the files.\n- **`\"anchors\"`** \u2014 every `requires_config` entry still resolves, so what\n moved is a step heading or golden-rule name an override may anchor to.\n Propose re-anchoring per *Reconciliation on framework upgrade*\n (`docs/setup/agentic-overrides.md`). This is a proposal to make, not a\n silence to keep: an override anchored to a heading that no longer\n exists is applied partially and without complaint, which is the whole\n failure this check exists to catch.\n\nPropose both when both findings are present.\n\n`facts.in_both_stores: true` is an expected transitional state, not a\nfault \u2014 someone configured the project before it adopted, on a machine\n`adopt` never ran from. The local entry wins; say that `/magpie-setup:setup\nreconcile` offers to drop the redundant one.\n\n**`code: \"sweep-never-run\"`** \u2014 nothing in this project has ever been\nreconciled, so a per-skill fix would be guesswork about a baseline that\ndoes not exist. Propose the one-time `/magpie-setup:setup reconcile` sweep\ninstead.\n\n**Record what you showed, the moment you show it.** Write\n`acknowledged.skills[\"\"]: ` for a `fingerprint-moved`\nproposal, or `acknowledged.sweep: ` for a sweep.\nRecorded on display, never on a decline this step does not wait for \u2014\nthat is what stops the same proposal reappearing on every later\ninvocation, and it is what the checker reads to suppress it.\n\n**Every write merges into the personal layer's `reconciled.json`\n(the `personal_dir` that `python3 -m setup_preflight.layers` prints); it\nnever replaces the file.** Read it, set the one key, write the whole\nobject back with every other key intact \u2014 and create the file, and\nthe personal layer itself, when either is absent \u2014 but only in the place\nthat command names, never an in-tree `.apache-magpie-local/` in a project\nthat has not adopted Magpie." }, "verdict": "action" diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-7-legitimate-url-alarming-context/state.json b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-7-legitimate-url-alarming-context/state.json index fcee80fef..a86105b13 100644 --- a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-7-legitimate-url-alarming-context/state.json +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-7-legitimate-url-alarming-context/state.json @@ -1,4 +1,5 @@ { + "harness": "claude", "lock": [ "method: marketplace", "url: apache/magpie", diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-8-unknown-plugin-list/state.json b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-8-unknown-plugin-list/state.json index 76d2d1f0f..44ec98918 100644 --- a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-8-unknown-plugin-list/state.json +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-8-unknown-plugin-list/state.json @@ -1,4 +1,5 @@ { + "harness": "claude", "lock": [ "method: marketplace", "url: apache/magpie", diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-9-first-magpie-run/report.md b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-9-first-magpie-run/report.md index 5fb8803d6..5436337e8 100644 --- a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-9-first-magpie-run/report.md +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-9-first-magpie-run/report.md @@ -20,7 +20,7 @@ The pre-flight command printed: "code": "welcome", "facts": { "family": "general", - "guide": "# \ud83d\udc26 Hello from Magpie \u2014 your project's maintainer sidekick\n\nOne minute, then straight back to what you asked for.\n\n## \ud83d\ude80 Getting started\n\n- \ud83d\udce6 **Install** from your agent's marketplace (Claude Code shown):\n `/plugin marketplace add apache/magpie`,\n then `/plugin install magpie-setup@apache-magpie` and one `/plugin install magpie-@apache-magpie` per family you want\n- \ud83d\ude4b **Configure it for yourself** \u2014 personal settings, nothing committed \u2014 `/magpie-setup:setup config`\n- \ud83d\udc65 **Adopt it for your project** \u2014 the configuration is committed to your repo and shared with every contributor \u2014\n `/magpie-setup:setup adopt` (a maintainers' decision)\n\nOn a pinned-snapshot install the commands are shorter: `/magpie-setup config`, `/magpie-setup adopt`.\n\n## \ud83d\udd04 Staying current\n\n- \u270b **By hand:** `claude plugin marketplace update apache-magpie`, then `claude plugin update @apache-magpie`\n- \ud83e\udd16 **Automatically (Claude Code):** `/plugin` \u2192 **Marketplaces** \u2192 **apache-magpie** \u2192 **Enable auto-update** \u2014\n it is off by default for third-party marketplaces like this one\n- \ud83e\udde9 Gemini CLI takes `--auto-update` at install time; Codex has no auto-update, so run `codex plugin marketplace upgrade`\n\nAfter an update, the next skill you run checks your configuration against it and proposes any fixes;\n`/magpie-setup:setup reconcile` checks everything at once\n(on a pinned snapshot, `/magpie-setup upgrade` does the update).\n\n## \ud83d\udcac Just ask\n\nTalk to Magpie in plain language \u2014 no syntax to learn.\nNothing is written, posted, or sent without your OK.\nInterrupt at any time, say no, or change your mind halfway through.\n\n## \ud83c\udfa8 You are the creator\n\nDescribe what you want a skill to do for you:\n\n- *\"from now on, always ask me before closing a stale issue\"*\n- *\"when you triage PRs, leave the bot-authored ones alone\"*\n\nMagpie records it as an override for that skill (`/magpie-setup:setup override `).\nWorth sharing? Say *\"upstream this change to Magpie\"* \u2014\nit makes the change generic and opens a PR to `apache/magpie` for you to review (`/magpie-setup:override-upstream`).\n\n## \ud83d\udcda More guides\n\nEach family can bring its own short guide, shown the first time you use it:\n\n- `contributor-growth` \u2014 `/magpie-setup:welcome-guide contributor-growth`\n- `issue` \u2014 `/magpie-setup:welcome-guide issue`\n- `mentoring` \u2014 `/magpie-setup:welcome-guide mentoring`\n- `pairing` \u2014 `/magpie-setup:welcome-guide pairing`\n- `pr-management` \u2014 `/magpie-setup:welcome-guide pr-management`\n- `security` \u2014 `/magpie-setup:welcome-guide security`\n- `utilities` \u2014 `/magpie-setup:welcome-guide utilities`\n\n## \ud83d\udc4b About this guide\n\n- **Continue** \u2014 show it again next time (three times at most)\n- **Dismiss** \u2014 hide it until the next Magpie upgrade\n- **Don't show again** \u2014 hide it for good; say *\"show the Magpie welcome again\"* to bring it back\n\nShow this again any time: `/magpie-setup:welcome-guide []`\n(`/magpie-setup-welcome-guide []` on a pinned snapshot).\n\n\ud83c\udf08 Prefer it in colour? Run this yourself \u2014 the `!` prefix runs it as a shell command:\n\n```text\n! PYTHONPATH=\".apache-magpie-local:$(git rev-parse --git-common-dir)/../.apache-magpie-local:$(git rev-parse --git-common-dir)/apache-magpie\" python3 -m setup_preflight.welcome show --family general --color=always\n```\n", + "guide": "# \ud83d\udc26 Hello from Magpie \u2014 your project's maintainer sidekick\n\nOne minute, then straight back to what you asked for.\n\n## \ud83d\ude80 Getting started\n\n- \ud83d\udce6 **Install** from your agent's marketplace (Claude Code shown):\n `/plugin marketplace add apache/magpie`,\n then `/plugin install magpie-setup@apache-magpie` and one `/plugin install magpie-@apache-magpie` per family you want\n- \ud83d\ude4b **Configure it for yourself** \u2014 personal settings, nothing committed \u2014 `/magpie-setup:setup config`\n- \ud83d\udc65 **Adopt it for your project** \u2014 the configuration is committed to your repo and shared with every contributor \u2014\n `/magpie-setup:setup adopt` (a maintainers' decision)\n\nOn a pinned-snapshot install the commands are shorter: `/magpie-setup config`, `/magpie-setup adopt`.\n\n## \ud83d\udd04 Staying current\n\n- \u270b **By hand:** `claude plugin marketplace update apache-magpie`, then `claude plugin update @apache-magpie`\n- \ud83e\udd16 **Automatically (Claude Code):** `/plugin` \u2192 **Marketplaces** \u2192 **apache-magpie** \u2192 **Enable auto-update** \u2014\n it is off by default for third-party marketplaces like this one\n- \ud83e\udde9 Gemini CLI takes `--auto-update` at install time; Codex has no auto-update, so run `codex plugin marketplace upgrade apache-magpie`\n\nAfter an update, the next skill you run checks your configuration against it and proposes any fixes;\n`/magpie-setup:setup reconcile` checks everything at once\n(on a pinned snapshot, `/magpie-setup upgrade` does the update).\n\n## \ud83d\udcac Just ask\n\nTalk to Magpie in plain language \u2014 no syntax to learn.\nNothing is written, posted, or sent without your OK.\nInterrupt at any time, say no, or change your mind halfway through.\n\n## \ud83c\udfa8 You are the creator\n\nDescribe what you want a skill to do for you:\n\n- *\"from now on, always ask me before closing a stale issue\"*\n- *\"when you triage PRs, leave the bot-authored ones alone\"*\n\nMagpie records it as an override for that skill (`/magpie-setup:setup override `).\nWorth sharing? Say *\"upstream this change to Magpie\"* \u2014\nit makes the change generic and opens a PR to `apache/magpie` for you to review (`/magpie-setup:override-upstream`).\n\n## \ud83d\udcda More guides\n\nEach family can bring its own short guide, shown the first time you use it:\n\n- `contributor-growth` \u2014 `/magpie-setup:welcome-guide contributor-growth`\n- `issue` \u2014 `/magpie-setup:welcome-guide issue`\n- `mentoring` \u2014 `/magpie-setup:welcome-guide mentoring`\n- `pairing` \u2014 `/magpie-setup:welcome-guide pairing`\n- `pr-management` \u2014 `/magpie-setup:welcome-guide pr-management`\n- `security` \u2014 `/magpie-setup:welcome-guide security`\n- `utilities` \u2014 `/magpie-setup:welcome-guide utilities`\n\n## \ud83d\udc4b About this guide\n\n- **Continue** \u2014 show it again next time (three times at most)\n- **Dismiss** \u2014 hide it until the next Magpie upgrade\n- **Don't show again** \u2014 hide it for good; say *\"show the Magpie welcome again\"* to bring it back\n\nShow this again any time: `/magpie-setup:welcome-guide []`\n(`/magpie-setup-welcome-guide []` on a pinned snapshot).\n\n\ud83c\udf08 Prefer it in colour? Run this yourself \u2014 the `!` prefix runs it as a shell command:\n\n```text\n! PYTHONPATH=\".apache-magpie-local:$(git rev-parse --git-common-dir)/../.apache-magpie-local:$(git rev-parse --git-common-dir)/apache-magpie\" python3 -m setup_preflight.welcome show --family general --color=always\n```\n", "max_shown": 3, "reason": "first-run", "shown": 0 diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-9-first-magpie-run/state.json b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-9-first-magpie-run/state.json index 38f939084..9211e6d98 100644 --- a/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-9-first-magpie-run/state.json +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/case-9-first-magpie-run/state.json @@ -1,4 +1,5 @@ { + "harness": "claude", "lock": null, "plugins": [ [ diff --git a/tools/skill-evals/evals/setup/preflight-floor/fixtures/output-spec.md b/tools/skill-evals/evals/setup/preflight-floor/fixtures/output-spec.md index 383d3c8ed..4462f0ff8 100644 --- a/tools/skill-evals/evals/setup/preflight-floor/fixtures/output-spec.md +++ b/tools/skill-evals/evals/setup/preflight-floor/fixtures/output-spec.md @@ -32,13 +32,17 @@ guide (the general one or the skill family's), offer its three choices and recor carrying on. That one **does not block** either, and recording the answer is not an install command. -`commands` is the list of install/update commands the pre-flight actually -**runs**, in the order it runs them. Reading installed state — `claude plugin -list --json` or the running agent's equivalent — does not count; only commands -that change what is installed do. It is empty whenever the pre-flight changes +`commands` is the list of install/update commands, or the personal-configuration +command `/magpie-setup:setup config`, that the pre-flight actually **runs**, +in the order it runs them. Reading installed state — `claude plugin +list --json` or the running agent's equivalent — does not count. It is empty whenever the pre-flight changes nothing: when it passes silently, when it only prints commands for the user to run, and when it declines to act and asks first. +Welcome recording and acknowledgement writes are excluded from `commands`. +For a welcome-only verdict, `commands` must be `[]` even though +`setup_preflight.welcome record` records the user's choice. + `print-command` also covers proposing `/magpie-setup:setup reconcile` for a `sweep-never-run` finding: it shows a proposal and continues, without running the sweep. `ask-first` means the pre-flight stops for a decision about an