From edd986c114b6dd454e903ffe20568591af3f0873 Mon Sep 17 00:00:00 2001 From: Many Kasiriha Date: Thu, 12 Nov 2026 15:34:25 +0100 Subject: [PATCH 1/3] docs(openspec): fix-desktop-eval-turn-overhead; openrouter tier verified Adds the fix-desktop-eval-turn-overhead change: the sut-platynui README gives locators that don't work on a real desktop (checked on Windows 11 with PlatynUI 0.12.0.dev330), keywords_resolve fails suites that wrap library keywords in user keywords, and rf-core hooks add turns the baseline arm never pays. Ticks openrouter-ci-eval-tier tasks 5.2, 5.3, 6.1 and 6.2 with the runs that verified them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FHEmzL3ErAhGQykKWDx2wP --- .../.openspec.yaml | 2 + .../fix-desktop-eval-turn-overhead/design.md | 123 ++++++++++++++++++ .../proposal.md | 69 ++++++++++ .../specs/platynui-skill/spec.md | 9 ++ .../specs/rf-validation-hooks/spec.md | 14 ++ .../specs/skill-eval-harness/spec.md | 53 ++++++++ .../specs/stop-hook-safety/spec.md | 17 +++ .../fix-desktop-eval-turn-overhead/tasks.md | 62 +++++++++ .../changes/openrouter-ci-eval-tier/tasks.md | 16 ++- 9 files changed, 361 insertions(+), 4 deletions(-) create mode 100644 openspec/changes/fix-desktop-eval-turn-overhead/.openspec.yaml create mode 100644 openspec/changes/fix-desktop-eval-turn-overhead/design.md create mode 100644 openspec/changes/fix-desktop-eval-turn-overhead/proposal.md create mode 100644 openspec/changes/fix-desktop-eval-turn-overhead/specs/platynui-skill/spec.md create mode 100644 openspec/changes/fix-desktop-eval-turn-overhead/specs/rf-validation-hooks/spec.md create mode 100644 openspec/changes/fix-desktop-eval-turn-overhead/specs/skill-eval-harness/spec.md create mode 100644 openspec/changes/fix-desktop-eval-turn-overhead/specs/stop-hook-safety/spec.md create mode 100644 openspec/changes/fix-desktop-eval-turn-overhead/tasks.md diff --git a/openspec/changes/fix-desktop-eval-turn-overhead/.openspec.yaml b/openspec/changes/fix-desktop-eval-turn-overhead/.openspec.yaml new file mode 100644 index 0000000..55ab199 --- /dev/null +++ b/openspec/changes/fix-desktop-eval-turn-overhead/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-11-12 diff --git a/openspec/changes/fix-desktop-eval-turn-overhead/design.md b/openspec/changes/fix-desktop-eval-turn-overhead/design.md new file mode 100644 index 0000000..df3b1fa --- /dev/null +++ b/openspec/changes/fix-desktop-eval-turn-overhead/design.md @@ -0,0 +1,123 @@ +## Context + +See proposal.md (Why) for the measurements. The parts of the current code that shape the approach: + +- `scoring/keyword_resolution.py` collects every keyword call into one flat list (`_Parsed.calls`), with no note of whether a call sits in a test, a fixture or a user keyword body. `library_calls` is the number of those call sites that resolve to a spec library. The archived design of strengthen-skill-eval-harness gives `min_calls` one purpose: "so an empty or BuiltIn-only suite cannot pass". +- `validate_robot.mjs` runs three tiers per written file. Tier 2 (formatting drift, `robocop format --check --diff`) is informational. Its siblings, the per-file dry run (`RF_AGENTSKILLS_FILE_DRYRUN`) and project validation at Stop (`RF_AGENTSKILLS_PROJECT_VALIDATION`), are already opt-in through environment flags. +- `maybe_remind_robot_tests.mjs` streams the transcript and fires once per session if any `file_path` ends in `.robot` or `.resource`. It doesn't look at what happened after the write. +- `bundles/rf-agentskills/` holds generated copies of the rf-core scripts and hooks, checked in CI ("Check generated variants and plugin layout"). +- The partial task's gating check is `custom.partial_install:suggests_plugin_or_passes`, which takes one nested `check` and passes when the answer suggests the plugin or that check passes. +- A live run on Windows (2026-11-12, `C:\workspace\platynui-exp` on the maintainer's PC) showed what the static checks miss; see proposal.md (Why). PlatynUI's own attributes there were `@Id` (from UIA AutomationId), `@Name` (localised) and `@native:*` for the raw UIA properties. +- The eval's baseline arm stages no plugin and so runs no hooks; the skill arm runs rf-core's hooks with every plugin. Hook changes therefore move every plugin's skill arm. + +## Goals / Non-Goals + +**Goals:** +- Make the fixture's locators the ones that work on a real desktop, and make the static grader reject the ones that don't. +- Make the counted measure match what `min_calls` was meant to check, without making a BuiltIn-only suite pass. +- Remove hook output that leads the agent to work the task didn't ask for, while keeping it available to users who want it. +- Re-baseline once, after all parts land. + +**Non-Goals:** +- Changing the task prompts (they'd change the task hashes, and the prompt's "make sure every keyword exists" is a fair user request). +- Giving the baseline arm hooks, or turning hooks off in evals: the skill arm should measure what users install. +- The UserPromptSubmit routing to `rf-language`: it caused a second skill load, but the false failures that went with it come from the grader, which this change fixes. +- The partial tier staging only rf-core: that is the partial-install test by design. +- Fixing the robotcode `.json` libdoc crash; that goes upstream. +- Making the live check part of CI. + +## Decisions + +### D1. Count library calls by expanding user keywords from tests and fixtures + +The parser records calls per owner: each test, each suite/test fixture, and each user keyword body (a keyword's own `[Setup]`/`[Teardown]` belong to its body). The count starts from tests and suite/test fixtures. A library call adds 1. A user-keyword call adds that keyword's expanded count, computed once per keyword with memoisation, and a keyword on the current expansion stack adds 0. Name matching reuses the existing normalisation and embedded-argument patterns, so `Press Digit 1` resolves to `Press Digit ${digit}`. The summary keeps `calls` and `library_calls` (call sites) and adds `counted_library_calls`. + +Alternatives: +- Lower `min_calls` to 3 on the desktop tasks: fixes one task's symptom, leaves rf-mobile's `min_calls: 4` with the same trap, and lets a suite with three direct calls pass where the task flow needs four. +- Count distinct library keywords: a wrapped calculator suite has 3 (`Bring To Front`, `Pointer Click`, `Get Attribute`), so it still fails. +- Count call sites including user-keyword calls: an empty wrapper keyword would count, which breaks the BuiltIn-only guarantee. + +### D2. Formatting drift becomes opt-in (`RF_AGENTSKILLS_FORMAT_CHECK`) + +When the flag isn't truthy, tier 2 doesn't start the formatter. The flag is parsed like the other `RF_AGENTSKILLS_*` booleans. + +Alternatives: +- Reword the hint as optional: the agent acted on the hint in 19 of 36 sessions, and wording rarely stops a model from acting on a diff it has been shown. +- Show it at most once per session: it already fired about once per session; the cost is the follow-up, not repetition. +- Turn it off only in evals: the eval would no longer measure the installed behaviour. + +Deprecation findings (tier 1) and errors (tier 0) stay on; they change correctness, not style. + +### D3. The Stop reminder looks for a run after the last write + +The transcript scan tracks two positions: the last tool call that wrote a `.robot`/`.resource` file (as today), and the last Bash command matching a Robot Framework run (`robot`, `python -m robot`, `robotcode robot`, any options). If the run comes after the write, the hook exits without output. The once-per-session marker and the `stop_hook_active` short-circuit are unchanged. The first step's wording becomes "Run the affected tests where they can run: …". + +Alternative: detect "cannot run here" from the prompt. Too fragile, and the dry run the agent already did is the stronger signal. + +### D4. The skill's lookup line moves up + +A short "Check a keyword" line goes in `Library Import` (before `Core Keywords Quick Reference`), naming `robotcode doc keyword PlatynUI.BareMetal ""` and `robotcode doc keywords PlatynUI.BareMetal ""`. The closing line stays. The skill's version gets a minor bump. Whether DeepSeek follows it is measured in the re-baseline run (`robotcode doc` calls per session, from the transcripts), not assumed. + +### D5. The fixture README lists `@Id` locators, verified on Windows + +Verified with `platynui-cli query` on Windows 11 (German): `@Id='CalculatorResults'`, `@Id='num1Button'` and `@Id='num2Button'` match; `@AutomationId='CalculatorResults'`, `@Name='One'` and `Window[@Name='Calculator']` don't. The README's table becomes: + +| Element | XPath | +|---|---| +| Digit buttons | `//control:Button[@Id='num1Button']` … `[@Id='num9Button']` | +| Plus / Equals | `//control:Button[@Id='plusButton']`, `//control:Button[@Id='equalButton']` | +| Result display | `//control:Text[@Id='CalculatorResults']` (its `Name` holds the shown value, e.g. "Display is 3", localised) | +| Main window | `//control:Window[@Name='Calculator' or @Name='Rechner']`, noting that window names are localised | + +`plusButton` and `equalButton` are confirmed during implementation with the same query. A short note under the table says that `@Name` values follow the display language and that PlatynUI exposes the UIA AutomationId as `@Id`, matching the skill. The skill needs no correction. + +### D6. `platynui_locators` is a custom check driven by the task + +A new `custom.platynui:locators_ok` check parses the suite like `keywords_resolve` (suite, resources, variable values) and scans every string argument and variable value for XPath attribute tests. It fails on `@AutomationId` not preceded by `native:`, and on `@Name=''` where the task maps that value to an Id (`name_to_id: {One: num1Button, Two: num2Button, Plus: plusButton, Equals: equalButton}`), unless the same predicate also accepts an `@Id`, as in `[@Id='num1Button' or @Name='One']`. The mapping lives in the task YAML, so the check stays generic; the README is documentation, not grader input. + +Alternatives: +- A regex over the raw file: simpler, but it would also flag comments and documentation (one suite explained `@Id` in a comment and then used `@AutomationId`). +- Reading the mapping from the README: couples the grader to a Markdown table. +- Only the `@AutomationId` rule: catches 49 of the 58 failures in the localised live pass, but not the `@Name='One'` suites, which fail on any non-English desktop. + +### D7. The partial fallback takes a list of checks + +`suggests_plugin_or_passes` accepts `checks: [...]` besides `check`; with a list, all must pass. The partial task uses `[keywords_resolve, platynui_locators]`. The single-`check` form keeps working for the other plugins' partial tasks. + +### D8. The live check is a script run on the Windows machine + +`scripts/desktop-live-check.py` runs on the Windows desktop, inside the project venv with PlatynUI installed. It takes a downloaded `skill-eval-full-rf-desktop-` artifact directory. It: + +1. collects `workspace/tests/*.robot` from the PlatynUI narrow and partial sessions, plus the skill's example; +2. screens each suite with `robot.api.get_model` (allowed imports, banned keywords), skipping unsafe ones with a reason; +3. for each suite, closes and starts Calculator, runs `robot` with a 150 s limit, and reads `output.xml`; +4. prints a table per tier and arm and the top failure messages. + +The run must happen in the logged-in desktop session; the doc shows running it there directly, or over SSH through a scheduled task created with `schtasks /create … /it`, as was done for the measurements in proposal.md. It reports results only and never writes baselines. + +Alternatives: +- A CI job on a self-hosted Windows runner: needs an always-on, unlocked desktop session and a registered runner; worth it later if the manual check proves useful. +- Running the suites with `--dryrun`: proves nothing about locators. + +### D9. One branch, one re-baseline + +The change branches from `main` after PR #2 (OpenRouter tier) and PR #3 (its baselines) have merged. All parts land in one PR, then one OpenRouter dispatch (`runs=6`) regenerates every plugin's OpenRouter baselines, and one `provider: claude` dispatch, started by the maintainer, regenerates the Claude ones. Both go in the same PR so the gate never compares the new grader or hooks against old baselines. + +## Risks / Trade-offs + +- [D1 makes `min_calls` easier to meet: a wrapper called four times with one library call inside passes `min_calls: 4`] → `min_calls` is a floor against empty suites, not a coverage check; the tasks' `file_contains` checks still require the result check and the target window. +- [D2 removes formatting help users may have relied on] → Documented in the rf-core hook README and the release notes, with the flag to turn it back on. +- [D3 skips the reminder after a failing run] → Failing runs are reported by `report_pending_errors.mjs` and rf_error_hints, not by this reminder. +- [D4 may not change DeepSeek's behaviour] → Measured, not assumed; if it doesn't, the next step is a separate decision. +- [D6 only knows the attributes and names it is told about] → It targets the mistakes measured on a real desktop; the live check (D8) stays the way to find new ones. +- [D8 runs model-written suites on a maintainer's machine] → The screening step allows only PlatynUI and standard libraries and rejects `Evaluate`, process and file-system keywords; every suite in the measured runs passed that screen. +- [The live check depends on the display language] → It reports the language it ran under; the README and the locator check avoid `@Name` for controls, so suites that follow them pass under any language. +- [Re-baselining both tiers depends on the maintainer's Claude dispatch] → The PR stays open until both are in; the OpenRouter part can be verified first. + +## Migration Plan + +No data migration. Old baselines are replaced in the same PR. Rollback is a revert of the PR, which restores the old baselines with the old grader and hooks. + +## Open Questions + +- Whether `Pointer Click` should also count when called through `Run Keyword`-style BuiltIn keywords (`Run Keyword Pointer Click …`). No session in the analysed runs did this; the current code doesn't count it either. diff --git a/openspec/changes/fix-desktop-eval-turn-overhead/proposal.md b/openspec/changes/fix-desktop-eval-turn-overhead/proposal.md new file mode 100644 index 0000000..802773c --- /dev/null +++ b/openspec/changes/fix-desktop-eval-turn-overhead/proposal.md @@ -0,0 +1,69 @@ +## Why + +On the OpenRouter tier (DeepSeek V4.1 Flash, N=6, runs 42887127109, 42993062842 and 43167186288), rf-desktop's skill arm scores below the baseline arm on the static gate: narrow 11/18 against 14/18, partial 14/18 against 17/18. Running the 63 suites those sessions wrote against real Windows Calculator (German Windows 11, PlatynUI 0.12.0.dev330, 2026-11-12) turns that around: + +- **The `sut-platynui` fixture README gives locators that don't work.** PlatynUI exposes the UIA AutomationId as `@Id`, and plain `@AutomationId` matches nothing. `@Name` follows the display language (`Eins`, `Gleich`, `Rechner`). All 63 suites used the README's `@Name='Calculator'` window locator and failed as written. With the names translated, the only passing suites were the 5 that used `@Id` for the result display, all from the skill arm (narrow 5/16 against 0/14); every other failure was `@AutomationId='CalculatorResults'`. The skill's own example passed both times. The skill was right, and the README contradicted it. +- **The static gate can't see this.** `keywords_resolve` only checks that keywords exist, so it passed most of the suites that fail on a desktop. +- **It also fails correct suites.** 7 skill-arm sessions, and no baseline session, wrote a suite that wraps `Pointer Click` in a user keyword (the style `rf-platynui` and `rf-language` both teach) and failed with `library_calls=3`, below `min_calls: 4`. The check counts library call sites, not the calls a run makes. +- **rf-core hooks add turns the baseline arm never pays.** + - After a formatting-drift hint, the model ran `robocop format` and re-checked in 19 of 36 sessions. + - The Stop reminder to "run the affected tests" added 0–5 calls on a machine where nothing can run. +- **Both arms spend turns parsing the libdoc JSON.** Each session made 12–16 `python -c` calls on the 84 KB spec, often failing. No session used `robotcode doc keyword`, which the skill names only in its last line. + +Raising rf-desktop's turn factor to 2.0 didn't help (the sessions used the extra turns and still hit the limit). The fix is to correct the fixture, make the grader see locator mistakes and stop it failing correct suites, and remove the turn overhead. + +## What Changes + +- **The `sut-platynui` README uses working locators:** + - `@Id` for the buttons and the result display (`num1Button`, `num2Button`, `plusButton`, `equalButton`, `CalculatorResults`), as verified on Windows; + - a window locator that doesn't depend on the display language. + + This is the main fix. +- **A new static check, `platynui_locators`, for the PlatynUI tasks.** It fails a suite whose XPath uses a plain `@AutomationId`, or that finds a Calculator control only by an English `@Name`. It is gating on the narrow task and part of the partial task's fallback check. The task YAMLs change, so their hashes change. +- **`keywords_resolve` counts library calls reached through user keywords.** + - Each call of a user keyword adds the library calls in its body, recursively, with a cycle guard. + - `min_calls` keeps its purpose: an empty or BuiltIn-only suite still fails. + - The summary reports both the call sites and the counted calls. +- **The per-file validation hook's formatting-drift tier becomes opt-in** (`RF_AGENTSKILLS_FORMAT_CHECK=on`). Formatting is cosmetic, and the hint costs one to three turns per written file. **BREAKING (hook output):** sessions no longer see formatting diffs by default. +- **The Stop test reminder changes.** + - It is skipped when the session already ran the suite (`robot`, `robotcode robot`, including `--dryrun`) after its last `.robot`/`.resource` write. + - Its wording says to run the tests "where they can run". +- **`rf-platynui` points keyword checks to robotcode** (`robotcode doc keyword PlatynUI.BareMetal ""`) in its workflow, not only at the end of the file. +- **A documented, manual live check for rf-desktop.** It runs the eval sessions' PlatynUI suites, and the skill's example, on a Windows desktop and reports pass rates per arm. It is run before a release and whenever the fixture or the skill's locator guidance changes. It isn't part of CI. +- **All plugins are re-baselined on both tiers once, after these land together.** + - The grader and task changes move the rf-desktop and rf-mobile results. + - The hook changes move every skill arm. +- **Out of scope, tracked separately:** the robotcode crash on `robotcode doc keywords ` (`AttributeError: 'ResourceFile' object has no attribute 'specversion'`), to be reported upstream. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `skill-eval-harness`: + - adds a requirement for how `keywords_resolve` counts library calls; + - adds the `platynui_locators` check for PlatynUI tasks; + - adds a requirement for the manual live check of desktop suites. +- `rf-validation-hooks`: "Formatting drift check" becomes opt-in. +- `stop-hook-safety`: adds a requirement that the test reminder is skipped once the session has run the suite after its last write. +- `platynui-skill`: adds a requirement that the skill's workflow points keyword checks to `robotcode doc keyword`. + +## Impact + +- **Fixture and tasks:** + - `eval/fixtures/sut-platynui/README.md`; + - `plugins/rf-desktop/evals/tasks/narrow/narrow-platynui-calculator-01.yaml` and `.../partial/partial-platynui-calculator-01.yaml` (new check, new task hashes). +- **Code:** + - `src/rf_skill_eval/scoring/keyword_resolution.py`, a new custom check under `src/rf_skill_eval/scoring/custom/`, and their tests; + - `plugins/rf-core/scripts/validate_robot.mjs` and `maybe_remind_robot_tests.mjs` with their tests; + - the rf-core hook README; + - the generated copies under `bundles/`. +- **Content:** + - `plugins/rf-desktop/skills/rf-platynui/SKILL.md` (minor version bump); + - `plugins/rf-core/evals/tasks/README.md` (the `min_calls` wording); + - a new section in `docs/` for the desktop live check, with the runner script under `scripts/`. +- **Baselines:** every plugin's OpenRouter and Claude baselines are regenerated. The Claude ones need a manual `provider: claude` dispatch on the maintainer's subscription. +- **Releases:** rf-core and rf-desktop get new plugin versions. diff --git a/openspec/changes/fix-desktop-eval-turn-overhead/specs/platynui-skill/spec.md b/openspec/changes/fix-desktop-eval-turn-overhead/specs/platynui-skill/spec.md new file mode 100644 index 0000000..c3a0932 --- /dev/null +++ b/openspec/changes/fix-desktop-eval-turn-overhead/specs/platynui-skill/spec.md @@ -0,0 +1,9 @@ +## ADDED Requirements + +### Requirement: Keyword checks point to robotcode + +The skill SHALL tell the agent, in its main text before the keyword reference and not only in its closing section, to check a PlatynUI keyword's name and arguments with `robotcode doc keyword PlatynUI.BareMetal ""` (and to search with `robotcode doc keywords PlatynUI.BareMetal ""`), and not by parsing a libdoc file or reading library source. + +#### Scenario: Lookup guidance near the keyword reference +- **WHEN** SKILL.md is read up to the start of the Core Keywords Quick Reference +- **THEN** it has already named `robotcode doc keyword PlatynUI.BareMetal` as the way to verify a keyword and its arguments diff --git a/openspec/changes/fix-desktop-eval-turn-overhead/specs/rf-validation-hooks/spec.md b/openspec/changes/fix-desktop-eval-turn-overhead/specs/rf-validation-hooks/spec.md new file mode 100644 index 0000000..7762ce6 --- /dev/null +++ b/openspec/changes/fix-desktop-eval-turn-overhead/specs/rf-validation-hooks/spec.md @@ -0,0 +1,14 @@ +## MODIFIED Requirements + +### Requirement: Formatting drift check + +The plugin SHALL provide a non-blocking formatting check for written `.robot`/`.resource` files using Robocop's formatter in check mode (`robocop format --check`), surfacing the proposed diff as a suggestion without failing the operation. The check SHALL be off by default and SHALL run only when `RF_AGENTSKILLS_FORMAT_CHECK` is truthy, because a formatting suggestion leads the agent to reformat and re-check a file that is otherwise correct. + +#### Scenario: Formatting suggestion is surfaced +- **WHEN** `RF_AGENTSKILLS_FORMAT_CHECK` is truthy and a `.robot` file is written with inconsistent formatting (e.g. 2-space separators or trailing whitespace) +- **THEN** the hook surfaces the formatter's proposed changes as informational output +- **AND** the formatting difference alone does NOT cause a model-facing error (exit `2`) + +#### Scenario: Disabled by default +- **WHEN** `RF_AGENTSKILLS_FORMAT_CHECK` is not set and a `.robot` file with inconsistent formatting is written +- **THEN** the hook output contains no formatting suggestion, and no formatter process is started diff --git a/openspec/changes/fix-desktop-eval-turn-overhead/specs/skill-eval-harness/spec.md b/openspec/changes/fix-desktop-eval-turn-overhead/specs/skill-eval-harness/spec.md new file mode 100644 index 0000000..516acf4 --- /dev/null +++ b/openspec/changes/fix-desktop-eval-turn-overhead/specs/skill-eval-harness/spec.md @@ -0,0 +1,53 @@ +## ADDED Requirements + +### Requirement: Static keyword resolution counts library calls through user keywords + +The `keywords_resolve` check SHALL count, for `min_calls`, the calls into spec-covered libraries that the suite's tests and fixtures reach. A library call written directly in a test or fixture SHALL count once. A call of a user keyword (from the suite or its resources) SHALL count every library call reached through that keyword's body, following nested user keywords. A user keyword that is reached again while its own body is being counted SHALL add nothing more, so recursive keywords terminate. User keywords that no test or fixture calls SHALL NOT add to the count. Standard-library calls SHALL NOT count. The check's summary SHALL report the number of call sites, the number of library call sites and the counted library calls. + +#### Scenario: Library keyword wrapped in a user keyword +- **WHEN** a suite defines `Press Digit` whose body calls `Pointer Click` once, its test calls `Press Digit` three times and calls `Get Attribute` once, and the check has `min_calls: 4` +- **THEN** the counted library calls are 4 and the `min_calls` condition passes + +#### Scenario: BuiltIn-only suite still fails +- **WHEN** a suite imports the required library but its tests only call `Log` and `Should Be Equal` +- **THEN** the counted library calls are 0 and the check fails with a message naming the expected minimum + +#### Scenario: Unused user keyword adds nothing +- **WHEN** a suite defines a user keyword that calls four library keywords, but no test or fixture calls it +- **THEN** that keyword's calls do not count toward `min_calls` + +#### Scenario: Recursive user keyword terminates +- **WHEN** a user keyword calls itself and one library keyword +- **THEN** the check finishes and counts that library call once per call from a test + +### Requirement: PlatynUI tasks check locators statically + +PlatynUI eval tasks SHALL include a gating static locator check on the produced suite. The check SHALL fail when an XPath in the suite, its resources or its variables uses a plain `@AutomationId` attribute (PlatynUI exposes the UIA AutomationId as `@Id`; `@native:AutomationId` stays allowed), or when a control that the fixture README gives an `@Id` for is located only by an English `@Name` value. Its failure message SHALL name the offending locator and the attribute to use instead. In a partial-install task, the locator check SHALL be part of the fallback check that passes the task when no plugin is suggested. + +#### Scenario: Plain AutomationId rejected +- **WHEN** a suite locates the result display with `//control:Text[@AutomationId='CalculatorResults']` +- **THEN** the locator check fails and its message names that locator and `@Id` + +#### Scenario: Id locators accepted +- **WHEN** a suite locates the buttons with `@Id='num1Button'`, `@Id='plusButton'`, `@Id='equalButton'` and the display with `@Id='CalculatorResults'` +- **THEN** the locator check passes + +#### Scenario: Language-bound name rejected +- **WHEN** a suite clicks the digit buttons only through `//control:Button[@Name='One']` and `@Name='Two'` +- **THEN** the locator check fails and names the `@Id` the README gives for those buttons + +#### Scenario: Partial task falls back to both checks +- **WHEN** a partial-install session doesn't suggest rf-desktop and writes a suite whose keywords resolve but that uses `@AutomationId` +- **THEN** the task fails + +### Requirement: Manual live check of desktop suites + +The repository SHALL document and ship a manual live check for rf-desktop. On a Windows desktop session, it SHALL run the PlatynUI suites from a downloaded eval run's artifacts and the skill's example suite against the real application, one fresh application instance per suite, with a per-suite time limit. It SHALL report pass counts per tier and arm, and the most common failure messages. It SHALL NOT run in CI, and it SHALL refuse to run a suite that imports a library other than `PlatynUI.BareMetal` and the standard libraries `BuiltIn`, `Collections`, `String` and `Process` (for the skill example only), or that calls `Evaluate`, `Run Process` or file-system keywords. + +#### Scenario: Report per arm +- **WHEN** the check runs on the artifacts of a full eval run on a Windows desktop +- **THEN** it prints, for narrow and partial and each arm, how many suites passed against the real Calculator, and the skill example's result + +#### Scenario: Unsafe suite skipped +- **WHEN** a collected suite imports `OperatingSystem` or calls `Evaluate` +- **THEN** the check doesn't run it and lists it as skipped with the reason diff --git a/openspec/changes/fix-desktop-eval-turn-overhead/specs/stop-hook-safety/spec.md b/openspec/changes/fix-desktop-eval-turn-overhead/specs/stop-hook-safety/spec.md new file mode 100644 index 0000000..8e6af8d --- /dev/null +++ b/openspec/changes/fix-desktop-eval-turn-overhead/specs/stop-hook-safety/spec.md @@ -0,0 +1,17 @@ +## ADDED Requirements + +### Requirement: Test reminder is skipped once the suite has run + +The `maybe_remind_robot_tests.mjs` reminder SHALL NOT be emitted when the session transcript shows a Robot Framework run (`robot`, `python -m robot`, `robotcode robot`, with or without `--dryrun`) after the last tool call that wrote a `.robot` or `.resource` file. When it is emitted, its first step SHALL say to run the affected tests where they can run, so an agent on a machine that cannot run them does not keep trying. + +#### Scenario: Suite dry-run after the last write +- **WHEN** a session writes `tests/calculator.robot` and then runs `robot --dryrun tests/calculator.robot` +- **THEN** the Stop hook emits no reminder + +#### Scenario: Write after the last run +- **WHEN** a session runs `robot tests/` and later edits `tests/login.robot` without running it again +- **THEN** the Stop hook emits the reminder (subject to the once-per-session rule) + +#### Scenario: Reminder wording +- **WHEN** the reminder is emitted +- **THEN** its first step tells the agent to run the affected tests where they can run diff --git a/openspec/changes/fix-desktop-eval-turn-overhead/tasks.md b/openspec/changes/fix-desktop-eval-turn-overhead/tasks.md new file mode 100644 index 0000000..21f852f --- /dev/null +++ b/openspec/changes/fix-desktop-eval-turn-overhead/tasks.md @@ -0,0 +1,62 @@ +## 1. Branch + +- [x] 1.1 After PR #2 and PR #3 have merged, branch `change/fix-desktop-eval-turn-overhead` from `main`; verify `git log` shows both merges. Done: branched from 3756236, which merges #2 and contains #3 (73d1a74). + +## 2. Fixture README (D5) + +- [x] 2.1 Verify on a Windows desktop which attributes PlatynUI exposes for Calculator: done 2026-11-12 on German Windows 11 with PlatynUI 0.12.0.dev330. `@Id='CalculatorResults'`, `@Id='num1Button'` and `@Id='num2Button'` match; `@AutomationId`, `@Name='One'` and `Window[@Name='Calculator']` don't. +- [x] 2.2 Confirm `plusButton` and `equalButton` with `platynui-cli query` on the same machine: done 2026-11-12. `@Id='plusButton'` (Name "Plus"), `@Id='equalButton'` (Name "Gleich") and `@Id='num1Button'` (Name "Eins") each match one enabled Button, reported twice because an unanchored `//` query reaches it through both the desktop and the app tree. A cold Calculator start can take more than 4 s; queries made earlier return nothing, without an error. +- [ ] 2.3 Rewrite the element table in `eval/fixtures/sut-platynui/README.md` with the `@Id` locators and the localised-window note from D5; verify no `@AutomationId` and no button `@Name` remains (`grep`), and that the README and `rf-platynui` name the same attributes. + +## 3. Grader: library calls through user keywords (D1) + +- [ ] 3.1 Record keyword calls per owner (test, suite/test fixture, user keyword body) in `scoring/keyword_resolution.py`, and compute `counted_library_calls` by expanding user-keyword calls from tests and fixtures, memoised, with a cycle guard; `min_calls` compares against it. Verify with new tests in `tests/eval` for the four scenarios of "Static keyword resolution counts library calls through user keywords" in `specs/skill-eval-harness/spec.md`, plus an embedded-argument wrapper (`Press Digit ${digit}`). +- [ ] 3.2 Report call sites, library call sites and counted library calls in the check summary; verify a test asserts on the summary text. +- [ ] 3.3 Update the `keywords_resolve` row in `plugins/rf-core/evals/tasks/README.md` to say how `min_calls` counts; verify `scripts/check-docs-voice.py` passes. + +## 4. Grader: PlatynUI locator check (D6, D7) + +- [ ] 4.1 Add `custom.platynui:locators_ok` under `src/rf_skill_eval/scoring/custom/`, taking `target` and `name_to_id`; verify with tests for the four scenarios of "PlatynUI tasks check locators statically", plus `@native:AutomationId` accepted, an `[@Id=… or @Name=…]` predicate accepted, and an `@AutomationId` in a comment ignored. +- [ ] 4.2 Let `suggests_plugin_or_passes` take `checks: [...]` (all must pass) alongside `check`; verify a test for each form, and that the other plugins' partial tasks load and score unchanged. +- [ ] 4.3 Add the locator check to `narrow-platynui-calculator-01.yaml` (gating) and to the partial task's `checks` list; verify `rf-skill-eval doctor --tasks-dir plugins/rf-desktop/evals/tasks/narrow` passes and both tasks load. +- [ ] 4.4 Re-grade offline the 72 stored rf-desktop sessions of runs 42887127109, 42993062842 and 43167186288 with tasks 3 and 4 in place; verify the static verdicts match the Windows live results of 2026-11-12 (only the 5 suites that use `@Id` for the display pass `platynui_locators`), and the 7 `min_calls` false failures no longer fail on call count. + +## 5. Hooks: formatting drift opt-in (D2) + +- [ ] 5.1 In `validate_robot.mjs`, start the tier-2 formatter only when `RF_AGENTSKILLS_FORMAT_CHECK` is truthy; verify with `plugins/rf-core/tests/test_hook_scripts.py` cases for "enabled shows the diff" and "unset starts no formatter and shows no diff". +- [ ] 5.2 Document the flag in `plugins/rf-core/hooks/README.md` (tier table and env-flag table); verify the README lists it next to `RF_AGENTSKILLS_FILE_DRYRUN`. + +## 6. Hooks: Stop reminder after a run (D3) + +- [ ] 6.1 In `maybe_remind_robot_tests.mjs`, track the positions of the last `.robot`/`.resource` write and the last Robot Framework run command, and stay silent when the run comes later; change the first step's wording to "Run the affected tests where they can run". Verify with hook tests for the three scenarios in `specs/stop-hook-safety/spec.md`, and that the existing once-per-session and `stop_hook_active` tests still pass. + +## 7. Generated copies + +- [ ] 7.1 Regenerate `bundles/rf-agentskills/` and any other generated variants from the rf-core sources; verify `scripts/check-drift.sh` and the "Check generated variants and plugin layout" check pass locally. + +## 8. Skill (D4) + +- [ ] 8.1 Add the "Check a keyword" line with `robotcode doc keyword PlatynUI.BareMetal ""` and `robotcode doc keywords PlatynUI.BareMetal ""` to the `Library Import` section of `plugins/rf-desktop/skills/rf-platynui/SKILL.md` and bump its minor version; verify `scripts/validate-skills.py` and `scripts/check-skill-keywords.py` pass. + +## 9. Live check (D8) + +- [ ] 9.1 Write `scripts/desktop-live-check.py` (collect, screen, run with a fresh Calculator, waiting until `CalculatorResults` answers a query rather than a fixed sleep, a 150 s limit, report per tier and arm); verify unit tests for the collection and the screening (an `OperatingSystem` import and an `Evaluate` call are skipped with a reason) run on Linux without PlatynUI. +- [ ] 9.2 Document it in `docs/` (where the rf-desktop eval docs live): Windows setup with uv and the pinned PlatynUI and platynui-cli versions, running it in the desktop session directly or over SSH through an `/it` scheduled task, and how to read the report; verify `scripts/check-docs-voice.py` passes. +- [ ] 9.3 Run it on the maintainer's Windows PC (`desktop-7800x3d`) against the artifacts of run 42993062842; verify it reproduces the 2026-11-12 results (skill example passes; all eval suites fail as written under the German display language). + +## 10. Tests, versions, PR + +- [ ] 10.1 Bump rf-core and rf-desktop plugin versions with `scripts/bump-version.sh`; verify `scripts/validate-marketplace.py` passes. +- [ ] 10.2 Run `uv run pytest tests/eval plugins/rf-core/tests`, `ruff check src tests/eval` and `scripts/check-governance.py`; verify all pass. +- [ ] 10.3 Open the PR against `main` (ask before posting); verify the required checks pass. + +## 11. Re-baseline (D9) + +- [ ] 11.1 Dispatch `skill-evaluation.yml -f provider=openrouter -f runs=6` on the PR branch and commit the candidate baselines for every plugin; verify the follow-up PR eval reports pass, not rebaseline-needed. +- [ ] 11.2 From the run's transcripts, report rf-desktop's per-arm pass rates, turns per session, `robocop format` runs and `robotcode doc` calls against run 42993062842; verify the report is in the PR description. +- [ ] 11.3 Run the live check (9.3) on the new run's artifacts; verify its per-arm pass rates are in the PR description next to the static ones. +- [ ] 11.4 Ask the maintainer to dispatch `provider=claude` for all plugins, and commit those candidate baselines; verify the Claude baselines in the PR carry the new harness version. + +## 12. Upstream (separate from the PR) + +- [ ] 12.1 Draft a robotcode issue for `robotcode doc keywords ` failing with `AttributeError: 'ResourceFile' object has no attribute 'specversion'`, with a minimal reproduction and the AI-use disclosure; the maintainer reviews and posts it. diff --git a/openspec/changes/openrouter-ci-eval-tier/tasks.md b/openspec/changes/openrouter-ci-eval-tier/tasks.md index cf77fcf..6b7f1f6 100644 --- a/openspec/changes/openrouter-ci-eval-tier/tasks.md +++ b/openspec/changes/openrouter-ci-eval-tier/tasks.md @@ -32,7 +32,7 @@ ## 5. CI workflow - [x] 5.1 Make preflight in `skill-evaluation.yml` output `provider`, `model`, `turns_factor` and the baseline subdirectory: OpenRouter when `OPENROUTER_API_KEY` is set, otherwise Claude, with a `provider` dispatch input overriding both. Keep the fork-PR rule. Verify with `tests/eval/test_ci_preflight.py` cases and by checking that the workflow still parses (`uv run python -c "import yaml; yaml.safe_load(open('.github/workflows/skill-evaluation.yml'))"`). -- [ ] 5.2 Wire `pr-eval` and `full-eval` to the preflight outputs: +- [x] 5.2 Wire `pr-eval` and `full-eval` to the preflight outputs: - the model and the turn-limit factor; - the credentials for the provider only; - separate OpenRouter caps (`*_CAP_USD_OPENROUTER`); @@ -40,19 +40,27 @@ - the candidate-baseline artifact names, which include the model. Verify on a PR from a branch in this repository: the eval job runs on DeepSeek and gates against the OpenRouter baselines. -- [ ] 5.3 Add the steps for OpenRouter jobs: + + Done: PR #2's eval run 43203324158 (head 73d1a74) ran every plugin's narrow tier on `deepseek/deepseek-v4.1-flash` with the OpenRouter caps and gated against `evals/baselines/openrouter/deepseek-v4.1-flash/`; all five gates passed. +- [x] 5.3 Add the steps for OpenRouter jobs: - a one-session smoke step that fails fast; - "Spend before" and "Spend after" steps that read `/api/v1/key`; - a summary line with the key's spend during the job next to the priced total; - on the weekly job, a warning when they differ by more than 25%. Verify in the job summary of the PR run from 5.2. + + Done: in run 43203324158 every PR eval job ran the smoke step, "Spend before" and "Spend after" successfully. The weekly warning is unit-tested; its first live check is the next weekly run. - [ ] 5.4 Check the no-credentials path: dispatch with provider `openrouter` from a fork, or with the secret temporarily unavailable through an environment that lacks it, and see "not run: no credentials". Verify from the run summary. If testing this needs a secret removed, ask the maintainer first. ## 6. Baselines -- [ ] 6.1 Dispatch the evaluation with provider `openrouter`, all tiers, both arms, N=6 and factor 1.5, for every plugin with evals. Verify that the run completes within the OpenRouter caps and produces one candidate baseline per plugin. -- [ ] 6.2 Review the candidate baselines (flaky tasks, turn-limit stops, and whether 1.5 is enough), commit them under `plugins//evals/baselines/openrouter/deepseek-v4.1-flash/` in a reviewed PR, and record the reviewed spend. Verify that the next PR eval run reports the gate as `pass`, not `rebaseline-needed`. +- [x] 6.1 Dispatch the evaluation with provider `openrouter`, all tiers, both arms, N=6 and factor 1.5, for every plugin with evals. Verify that the run completes within the OpenRouter caps and produces one candidate baseline per plugin. + + Done: run 42993062842 (all plugins, N=6, factor 1.5) produced one candidate baseline per plugin. OpenRouter charged $8.34 against a priced $12.58. An rf-desktop retry at factor 2.0 (run 43167186288, $1.51 charged, $1.70 priced) took more turns without improving the pass rate, so rf-desktop stays at 1.5. +- [x] 6.2 Review the candidate baselines (flaky tasks, turn-limit stops, and whether 1.5 is enough), commit them under `plugins//evals/baselines/openrouter/deepseek-v4.1-flash/` in a reviewed PR, and record the reviewed spend. Verify that the next PR eval run reports the gate as `pass`, not `rebaseline-needed`. + + Done: reviewed and committed in PR #3 (rf-api, rf-core, rf-mobile, rf-web, then rf-desktop at 1.5), merged into PR #2; run 43203324158 reported every gate as `pass`. Reviewed spend: $8.34 + $1.51. rf-desktop's skill arm scores below its baseline arm; the causes are tracked in `fix-desktop-eval-turn-overhead`. ## 7. Docs and clean-up From 12205bb9e609c7e580ec3c5869248d7d470da6c2 Mon Sep 17 00:00:00 2001 From: Many Kasiriha Date: Thu, 12 Nov 2026 17:52:09 +0100 Subject: [PATCH 2/3] fix(rf-desktop): working fixture locators, locator check, less hook overhead - sut-platynui README: @Id locators (verified on Windows 11), language-independent window - keywords_resolve: min_calls counts library calls reached through user keywords - new custom.platynui:locators_ok check, gating on the narrow task and in the partial fallback (which now takes a list of checks) - rf-core: formatting drift opt-in (RF_AGENTSKILLS_FORMAT_CHECK); Stop reminder skipped once the suite ran after the last write - rf-platynui: robotcode doc keyword line in Library Import - scripts/desktop-live-check.py and docs/ci/desktop-live-check.md Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FHEmzL3ErAhGQykKWDx2wP --- bundles/rf-agentskills/hooks/README.md | 18 +- .../scripts/maybe_remind_robot_tests.mjs | 42 ++- .../rf-agentskills/scripts/validate_robot.mjs | 10 +- .../skills/rf-platynui/SKILL.md | 2 + docs/ci/desktop-live-check.md | 110 +++++++ docs/ci/usage.md | 2 + eval/fixtures/sut-platynui/README.md | 15 +- .../fix-desktop-eval-turn-overhead/tasks.md | 36 +-- plugins/rf-core/CHANGELOG.md | 2 + plugins/rf-core/evals/tasks/README.md | 2 +- plugins/rf-core/hooks/README.md | 18 +- .../scripts/maybe_remind_robot_tests.mjs | 42 ++- plugins/rf-core/scripts/validate_robot.mjs | 10 +- plugins/rf-core/tests/test_hook_scripts.py | 94 ++++++ plugins/rf-desktop/CHANGELOG.md | 1 + .../narrow/narrow-platynui-calculator-01.yaml | 11 +- .../partial-platynui-calculator-01.yaml | 10 +- .../rf-desktop/skills/rf-platynui/SKILL.md | 2 + scripts/desktop-live-check.py | 292 ++++++++++++++++++ .../scoring/custom/partial_install.py | 21 +- src/rf_skill_eval/scoring/custom/platynui.py | 112 +++++++ .../scoring/keyword_resolution.py | 104 +++++-- .../tests/calculator.robot | 12 +- tests/eval/test_keyword_resolution.py | 103 ++++++ tests/eval/test_partial_install.py | 13 + tests/eval/test_platynui_locators.py | 96 ++++++ tests/test_desktop_live_check.py | 97 ++++++ 27 files changed, 1182 insertions(+), 95 deletions(-) create mode 100644 docs/ci/desktop-live-check.md create mode 100644 scripts/desktop-live-check.py create mode 100644 src/rf_skill_eval/scoring/custom/platynui.py create mode 100644 tests/eval/test_platynui_locators.py create mode 100644 tests/test_desktop_live_check.py diff --git a/bundles/rf-agentskills/hooks/README.md b/bundles/rf-agentskills/hooks/README.md index 1b626e8..0879667 100644 --- a/bundles/rf-agentskills/hooks/README.md +++ b/bundles/rf-agentskills/hooks/README.md @@ -21,7 +21,7 @@ rather than written-and-broken. | Event | Script | Always fires? | Cost when no-op | Cost when active | Hook timeout | |---|---|---|---|---|---| | `SessionStart` | `scripts/check_rf_environment.mjs` | yes | — | ~0.5–1 s (import probes) | 15 s | -| `PostToolUse` (matcher: `Write\|Edit`) | `scripts/validate_robot.mjs` | only on Write/Edit | ~30ms (file extension check) | ~1–1.5 s on a 500-line file once robotcode's cache is warm (`robotcode analyze code`, Robocop check and format run in parallel); the first run in a project ~4 s; +0.4 s or more with the opt-in dry run | 60 s | +| `PostToolUse` (matcher: `Write\|Edit`) | `scripts/validate_robot.mjs` | only on Write/Edit | ~30ms (file extension check) | ~1–1.5 s on a 500-line file once robotcode's cache is warm (`robotcode analyze code` and Robocop check run in parallel, plus the formatter when enabled); the first run in a project ~4 s; +0.4 s or more with the opt-in dry run | 60 s | | `PostToolUse` (matcher: `Bash`) | `scripts/rf_error_hints.mjs` | only on Bash | ~30 ms (text scan) | ~30 ms | 10 s | | `UserPromptSubmit` | `scripts/maybe_inject_rf_context.mjs` | always invoked, conditional injection | ~30ms (regex over prompt) | same (≤ 450-character text) | 15 s | | `Stop` | `scripts/report_pending_errors.mjs` | always invoked, reports only files whose validation found errors or unresolved names that are still there | ~30 ms (reads the session record) | one `validate_robot.mjs` run per pending file | 60 s | @@ -106,9 +106,12 @@ Fires from `PostToolUse` after a Write/Edit of a `.robot`/`.resource` file. gets no `VAR` hint), and so does the project's Robocop config. - If Robocop is missing, times out (10 s), or prints nothing parsable while failing, the tier is silent — a tool failure is never a finding. -- **Tier 2 — formatting drift:** runs `robocop format --no-cache --check --diff`. - Purely informational — surfaces the proposed reformat as `additionalContext` - (exit 0). Formatting differences **never** cause exit 2. +- **Tier 2 — formatting drift (opt-in):** with `RF_AGENTSKILLS_FORMAT_CHECK=1`, + runs `robocop format --no-cache --check --diff`. Purely informational — + surfaces the proposed reformat as `additionalContext` (exit 0). Formatting + differences **never** cause exit 2. Off by default: shown a diff, the agent + reformats and re-checks a file that is otherwise correct, which cost one to + three turns per written file in the evals. - **Opt-in per-file dry run:** with `RF_AGENTSKILLS_FILE_DRYRUN=1`, a written `.robot` suite (not `.resource`, not `__init__.robot`) is also run through `robot --dryrun --output NONE --report NONE --log NONE` from the project @@ -128,6 +131,7 @@ exit-2 error diagnostic is written. Robocop is always called with |---|---|---| | `RF_AGENTSKILLS_DEPRECATION_CHECK` | `warn` (default), `off` | `off` drops all deprecation output. Any other value is treated as `warn`; no value makes a deprecation block (exit 2), and the variable never affects error findings. | | `RF_AGENTSKILLS_FILE_DRYRUN` | unset (default), `1`/`true`/`yes`/`on` | Enables the per-file dry run above. | +| `RF_AGENTSKILLS_FORMAT_CHECK` | unset (default), `1`/`true`/`yes`/`on` | Enables the formatting-drift tier above. | | `RF_AGENTSKILLS_PROJECT_VALIDATION` | unset (default), `1`/`true`/`yes`/`on` | Enables the project-wide Stop tier below. | #### Opting out per rule @@ -335,8 +339,12 @@ event produces no output; it always exits 0. or `"…something.resource"` (with bounded regex — `notes.robotic.md` does not count). Streaming chunking keeps memory bounded on long sessions. +- Also notes the last Bash `"command"` that runs Robot Framework (`robot`, + `python -m robot`, `robotcode … robot`, dry runs included). If that run + comes after the last `.robot`/`.resource` file path, the session already + ran its suite and the hook exits without output. - On hit: emits `additionalContext` reminding the user to run - the affected tests (`uv run robotcode robot -bl ""`), read the + the affected tests where they can run (`uv run robotcode robot -bl ""`), read the results with `uv run robotcode results summary --failed`, and open `results/report.html`. - On miss: stdout stays empty. diff --git a/bundles/rf-agentskills/scripts/maybe_remind_robot_tests.mjs b/bundles/rf-agentskills/scripts/maybe_remind_robot_tests.mjs index fe72b36..12634dc 100644 --- a/bundles/rf-agentskills/scripts/maybe_remind_robot_tests.mjs +++ b/bundles/rf-agentskills/scripts/maybe_remind_robot_tests.mjs @@ -5,7 +5,9 @@ // transcript for any tool result that wrote a .robot or .resource file, // and only emits an additionalContext reminder ("run robot, inspect // report.html") when at least one such file was touched. Sessions that -// produced only non-RF artifacts get no injection. +// produced only non-RF artifacts get no injection, and neither do sessions +// that ran Robot Framework (`robot`, `python -m robot`, `robotcode robot`, +// dry runs included) after their last .robot/.resource write. // // Cross-platform port of maybe_remind_robot_tests.sh. Reads the // transcript via streaming-ish chunking so very long sessions don't @@ -55,13 +57,25 @@ if (sessionId) { // Pattern: `"file_path" : "...path.robot"` (or .resource). Matches the // `grep -E` regex from the bash original. Whitespace optional between // quotes and colon. -const FILE_PATH_RE = /"file_path"\s*:\s*"[^"]*\.(?:robot|resource)"/i; +const FILE_PATH_RE = /"file_path"\s*:\s*"[^"]*\.(?:robot|resource)"/gi; +// A Bash tool call's `"command": "…"` (JSON string, escapes kept). +const COMMAND_RE = /"command"\s*:\s*"((?:[^"\\]|\\.)*)"/g; +// Robot Framework run at a command position: start, after ; & | ( or a +// `uv/poetry/pipenv run`, optionally with a path (`.venv/bin/robot`). +const RF_RUN_RE = new RegExp( + String.raw`(?:^|[;&|(]\s*|\b(?:uv|poetry|pipenv)\s+run\s+)(?:[\w.~/\\-]*[/\\])?` + + String.raw`(?:python3?(?:\.exe)?\s+-m\s+robot(?=\s|$)|robotcode(?:\.exe)?(?:\s+\S+)*?\s+robot\b|robot(?:\.exe)?(?=\s|$))`, +); +const CARRY = 4096; -function transcriptMentionsRfFile(path) { +// Offsets of the last .robot/.resource file_path and the last Robot +// Framework run command in the transcript (-1 when absent). +function lastWriteAndRun(path) { // The transcript is a JSONL file that can be tens of MB on long // sessions. Read in 64 KB chunks rather than slurping it all. const size = (() => { try { return statSync(path).size; } catch { return 0; } })(); - if (!size) return false; + const found = { write: -1, run: -1 }; + if (!size) return found; const BUF = Buffer.alloc(65536); const fd = openSync(path, "r"); let pos = 0; @@ -71,27 +85,33 @@ function transcriptMentionsRfFile(path) { const n = readSync(fd, BUF, 0, BUF.length, pos); if (n <= 0) break; // Stitch the boundary with the previous chunk so we don't miss a - // match split across the seam. + // match split across the seam (offsets are approximate, in chars). const chunk = carry + BUF.slice(0, n).toString("utf-8"); - if (FILE_PATH_RE.test(chunk)) return true; - // Keep the last ~256 bytes as carry to bridge boundary matches. - carry = chunk.length > 256 ? chunk.slice(-256) : chunk; + const base = pos - carry.length; + for (const m of chunk.matchAll(FILE_PATH_RE)) found.write = Math.max(found.write, base + m.index); + for (const m of chunk.matchAll(COMMAND_RE)) { + if (RF_RUN_RE.test(m[1])) found.run = Math.max(found.run, base + m.index); + } + carry = chunk.length > CARRY ? chunk.slice(-CARRY) : chunk; pos += n; } } finally { try { closeSync(fd); } catch {} } - return false; + return found; } -if (!transcriptMentionsRfFile(transcript)) process.exit(0); +const { write: lastWrite, run: lastRun } = lastWriteAndRun(transcript); +if (lastWrite < 0) process.exit(0); +// The suite already ran (or dry-ran) after the last write: nothing to remind. +if (lastRun > lastWrite) process.exit(0); const payload = { hookSpecificOutput: { hookEventName: "Stop", additionalContext: [ "Robot Framework artifacts detected this session. Recommended next steps:", - '\n 1. Run the affected tests: uv run robotcode robot -bl "" (or a whole directory)', + '\n 1. Run the affected tests where they can run: uv run robotcode robot -bl "" (or a whole directory)', "\n 2. Read the results: uv run robotcode results summary --failed, then robotcode results log --failed", " (not using uv: run robotcode from the project's .venv; see rf-setup)", "\n 3. Open results/report.html for the rendered report.", diff --git a/bundles/rf-agentskills/scripts/validate_robot.mjs b/bundles/rf-agentskills/scripts/validate_robot.mjs index b5c0328..4b1857e 100644 --- a/bundles/rf-agentskills/scripts/validate_robot.mjs +++ b/bundles/rf-agentskills/scripts/validate_robot.mjs @@ -35,8 +35,10 @@ // any other value (including a legacy `block`) means `warn`. // No parsable line from a failing Robocop run = tool failure = silence. // -// Tier 2 (formatting drift): `robocop format --check --diff`. Informational -// additionalContext only. +// Tier 2 (opt-in, RF_AGENTSKILLS_FORMAT_CHECK=1, formatting drift): +// `robocop format --check --diff`. Informational additionalContext only. +// Off by default: a formatting diff leads the agent to reformat and +// re-check a file that is otherwise correct. // // Tier 3 (opt-in, RF_AGENTSKILLS_FILE_DRYRUN=1): `robot --dryrun` on the // edited suite (.robot only, not __init__.robot). Advisory context only. @@ -451,7 +453,7 @@ const advisory = []; const robocopPy = STOP_RECHECK ? null : findInterpreterWith("robocop", cwd); const analyzeRun = startAnalyze(); const robocopRun = robocopPy ? startRobocop(robocopPy) : Promise.resolve(null); -const formatRun = robocopPy +const formatRun = robocopPy && isTruthy(process.env.RF_AGENTSKILLS_FORMAT_CHECK) ? spawnAsync(robocopPy, ["-m", "robocop", "format", "--no-cache", "--check", "--diff", "--no-overwrite", filePath], ROBOCOP_TIMEOUT_MS) : Promise.resolve(null); @@ -482,7 +484,7 @@ if (robocopPy) { if (text) advisory.push(text); } - // ── Tier 2: formatting drift (informational only) ── + // ── Tier 2 (opt-in): formatting drift (informational only) ── const fmt = await formatRun; if (fmt && !fmt.error && fmt.status !== 0 && fmt.status !== null && (fmt.stdout ?? "").trim()) { let diff = fmt.stdout.trim(); diff --git a/bundles/rf-agentskills/skills/rf-platynui/SKILL.md b/bundles/rf-agentskills/skills/rf-platynui/SKILL.md index 37d9511..0292014 100644 --- a/bundles/rf-agentskills/skills/rf-platynui/SKILL.md +++ b/bundles/rf-agentskills/skills/rf-platynui/SKILL.md @@ -74,6 +74,8 @@ Library PlatynUI.BareMetal auto_activate=${True} pointer_profile=${POIN Library scope is **SUITE**. The native runtime is created lazily on first keyword use (so `libdoc` and import work without a desktop). +**Check a keyword** before you use it: `robotcode doc keyword PlatynUI.BareMetal ""` prints its arguments, and `robotcode doc keywords PlatynUI.BareMetal ""` searches by name. Use them instead of parsing a libdoc file or reading the library source. + ## Essential Concepts ### The normalized desktop UI model diff --git a/docs/ci/desktop-live-check.md b/docs/ci/desktop-live-check.md new file mode 100644 index 0000000..44704d3 --- /dev/null +++ b/docs/ci/desktop-live-check.md @@ -0,0 +1,110 @@ +# rf-desktop live check + +The CI runner has no desktop, so the eval grades rf-desktop's PlatynUI suites statically: `keywords_resolve` checks that every keyword exists, and `platynui_locators` rejects locators known to fail on a real desktop. Neither runs a suite. The live check does: on a Windows desktop, it runs the suites one eval run wrote, and the skill's example, against the real Calculator, and reports how many pass per tier and arm. + +Run it by hand: + +- before a release of `rf-desktop`; +- when the `sut-platynui` fixture README, the PlatynUI tasks or the locator guidance in `rf-platynui` change; +- when the static and the live picture may differ, for example after a PlatynUI upgrade. + +It isn't part of CI, and it never writes baselines. + +## Set up the Windows machine + +You need Windows 10 or 11 with the Calculator app, a logged-in and unlocked desktop session, Python 3.12, uv and git. + +```bat +git clone https://github.com/MarketSquare/robotframework-plugins C:\workspace\robotframework-plugins +cd /d C:\workspace\robotframework-plugins +uv venv --python 3.12 .venv-desktop +uv pip install --python .venv-desktop --prerelease allow robotframework==7.5.1 robotframework-PlatynUI==0.12.0.dev330 platynui-cli==0.12.0.dev330 +``` + +These are the versions of the committed libdoc spec (`eval/fixtures/sut-platynui/specs/`). Check that PlatynUI sees Calculator: start it, then run + +```bat +.venv-desktop\Scripts\platynui-cli.exe query "//control:Text[@Id='CalculatorResults']" +``` + +It prints the result display twice (once through the desktop tree, once through the application tree). If it prints nothing, Calculator isn't ready yet; a cold start can take more than 4 seconds. + +## Get a run's artifacts + +Download the full-run artifact of an eval run on any machine with `gh`, and copy it to the Windows machine: + +```bash +gh run download -R MarketSquare/robotframework-plugins -n skill-eval-full-rf-desktop- -D live-check/ +``` + +The artifacts hold model-written suites. Treat them as untrusted: keep them in their own directory and run the script with `python -I` from the repository, not from inside the artifact directory. + +## Run it + +List what would run first. This collects the `tests/calculator.robot` of every PlatynUI session, plus the skill's example, and screens each one: + +```bat +.venv-desktop\Scripts\python.exe -I scripts\desktop-live-check.py C:\workspace\live-check\ --list +``` + +Screening skips a suite, with a reason, when it: + +- imports a library other than `PlatynUI.BareMetal`, `BuiltIn`, `Collections` or `String` (the skill's example may also import `Process`, to start Calculator); +- imports a variable file, or a resource outside its session's workspace; +- calls `Evaluate`, `Run Process`, `Start Process` (except the example), `Call Method` or an `Import …` keyword; +- has text that could run Python in an expression, such as `${{ }}`, `__`, `import` or `subprocess`. + +Then run it in the desktop session: + +```bat +.venv-desktop\Scripts\python.exe -I scripts\desktop-live-check.py C:\workspace\live-check\ --out C:\workspace\live-check\-results +``` + +For each suite it closes Calculator, starts a fresh one (the example starts its own), waits until the result display answers a query, and runs `robot` with a 150-second limit (`--timeout`). Expect about half a minute per suite, since a locator that matches nothing waits 30 seconds before it fails; the 64 suites of 2026-11-12 took 32 minutes. Don't use the mouse or keyboard meanwhile: PlatynUI moves the real pointer. + +### Over SSH + +An SSH session can't see the desktop, so UI Automation finds nothing. Run the check through a scheduled task that runs in the logged-in session (`/it`). Create the task once, as the logged-in user: + +```bat +schtasks /create /tn rf-desktop-live-check /tr C:\workspace\live-check\run.cmd /sc once /st 00:00 /sd 01.01.2030 /it /ru %USERNAME% /f +``` + +`C:\workspace\live-check\run.cmd` runs the check and marks the end of the output: + +```bat +@echo off +cd /d C:\workspace\robotframework-plugins +.venv-desktop\Scripts\python.exe -I scripts\desktop-live-check.py C:\workspace\live-check\current --out C:\workspace\live-check\current-results > C:\workspace\live-check\out.txt 2>&1 +echo DONE>> C:\workspace\live-check\out.txt +``` + +Then, over SSH: copy the artifacts to `C:\workspace\live-check\current`, delete the old `out.txt`, start the task with `schtasks /run /tn rf-desktop-live-check`, and wait until `out.txt` ends with `DONE`. The `/sd` date only keeps the task from running by itself; use the date format of the machine's locale. + +## Read the report + +``` +display language: de_DE + +tier arm suites passed skipped +example example 1 1 0 +narrow baseline 3 0 0 +narrow treatment 5 1 0 +... + +most common failures: + 15 ElementNotFoundError: No UiNode found for UiNodeDescriptor query Q within timeout of N seconds. +``` + +- **display language** matters: `@Name` values follow it (a German desktop shows `Rechner`, `Eins`, `Gleich`). A suite that finds controls by English names passes on an English desktop only. Compare runs on the same language, or note it next to the numbers. +- **suites** counts the sessions that wrote the suite; a session that wrote nothing isn't listed. +- **passed** counts suites whose every test passed against the real Calculator. +- **most common failures** groups failure messages with quoted values and numbers replaced by `Q` and `N`. + +Each suite's `output.xml`, `log.html` and `report.html` are under `--out`, in a folder named after the tier and session. + +Put the per-arm pass counts next to the static ones (the candidate baseline's `keywords_resolve` and `platynui_locators` verdicts) in the PR that changes the fixture, the tasks or the skill. If a suite passes live and fails the static gate, or the other way round, look at it: either the static check needs a rule, or the live run hit something the eval can't see. + +## Example results + +On 2026-11-12, on German Windows 11 with PlatynUI 0.12.0.dev330, the 63 suites of runs 42887127109, 42993062842 and 43167186288 all failed as written: they used the old README's `@Name='Calculator'` window and `@AutomationId` locators. The skill's example passed. Those results led to the `@Id` locators in the fixture README and the `platynui_locators` check. diff --git a/docs/ci/usage.md b/docs/ci/usage.md index d289684..d6b4b25 100644 --- a/docs/ci/usage.md +++ b/docs/ci/usage.md @@ -543,6 +543,8 @@ to prefer which. - [local-testing.md](local-testing.md) — pre-push checklist and troubleshooting. +- [desktop-live-check.md](desktop-live-check.md) — running rf-desktop's + eval suites against the real Calculator on Windows. - [faq.md](faq.md) — common questions, rate limits, rotation. - [architecture/ddd-design.md](architecture/ddd-design.md) — bounded contexts and module layout. diff --git a/eval/fixtures/sut-platynui/README.md b/eval/fixtures/sut-platynui/README.md index 5732068..9e97e12 100644 --- a/eval/fixtures/sut-platynui/README.md +++ b/eval/fixtures/sut-platynui/README.md @@ -11,10 +11,17 @@ The standard desktop calculator (Windows Calculator / GNOME Calculator). | Element | XPath (PlatynUI desktop tree) | |---|---| -| Main window | `//control:Window[@Name='Calculator']` | -| Digit buttons | `//control:Button[@Name='One']` … `//control:Button[@Name='Nine']` | -| Plus / Equals | `//control:Button[@Name='Plus']`, `//control:Button[@Name='Equals']` | -| Result display | `//control:Text[@AutomationId='CalculatorResults']` | +| Main window | `//control:Window[@Name='Calculator' or @Name='Rechner']` | +| Digit buttons | `//control:Button[@Id='num1Button']` … `//control:Button[@Id='num9Button']` | +| Plus / Equals | `//control:Button[@Id='plusButton']`, `//control:Button[@Id='equalButton']` | +| Result display | `//control:Text[@Id='CalculatorResults']` | + +PlatynUI exposes the UIA AutomationId as `@Id`, as `rf-platynui` teaches; a +plain `@AutomationId` matches nothing. `@Name` values follow the display +language (a German desktop shows `Rechner`, `Eins`, `Gleich`), so the window +locator lists both names and the controls use `@Id`. The result display's +`Name` holds the shown value, such as "Display is 3" (localised). Verified with +`platynui-cli query` on Windows 11 on 2026-11-12. `tests/example.robot` is a stub that resolves cleanly. Real execution stays a manual, local activity on a desktop machine. diff --git a/openspec/changes/fix-desktop-eval-turn-overhead/tasks.md b/openspec/changes/fix-desktop-eval-turn-overhead/tasks.md index 21f852f..37fcba8 100644 --- a/openspec/changes/fix-desktop-eval-turn-overhead/tasks.md +++ b/openspec/changes/fix-desktop-eval-turn-overhead/tasks.md @@ -6,48 +6,48 @@ - [x] 2.1 Verify on a Windows desktop which attributes PlatynUI exposes for Calculator: done 2026-11-12 on German Windows 11 with PlatynUI 0.12.0.dev330. `@Id='CalculatorResults'`, `@Id='num1Button'` and `@Id='num2Button'` match; `@AutomationId`, `@Name='One'` and `Window[@Name='Calculator']` don't. - [x] 2.2 Confirm `plusButton` and `equalButton` with `platynui-cli query` on the same machine: done 2026-11-12. `@Id='plusButton'` (Name "Plus"), `@Id='equalButton'` (Name "Gleich") and `@Id='num1Button'` (Name "Eins") each match one enabled Button, reported twice because an unanchored `//` query reaches it through both the desktop and the app tree. A cold Calculator start can take more than 4 s; queries made earlier return nothing, without an error. -- [ ] 2.3 Rewrite the element table in `eval/fixtures/sut-platynui/README.md` with the `@Id` locators and the localised-window note from D5; verify no `@AutomationId` and no button `@Name` remains (`grep`), and that the README and `rf-platynui` name the same attributes. +- [x] 2.3 Rewrite the element table in `eval/fixtures/sut-platynui/README.md` with the `@Id` locators and the localised-window note from D5; verify no `@AutomationId` and no button `@Name` remains (`grep`), and that the README and `rf-platynui` name the same attributes. ## 3. Grader: library calls through user keywords (D1) -- [ ] 3.1 Record keyword calls per owner (test, suite/test fixture, user keyword body) in `scoring/keyword_resolution.py`, and compute `counted_library_calls` by expanding user-keyword calls from tests and fixtures, memoised, with a cycle guard; `min_calls` compares against it. Verify with new tests in `tests/eval` for the four scenarios of "Static keyword resolution counts library calls through user keywords" in `specs/skill-eval-harness/spec.md`, plus an embedded-argument wrapper (`Press Digit ${digit}`). -- [ ] 3.2 Report call sites, library call sites and counted library calls in the check summary; verify a test asserts on the summary text. -- [ ] 3.3 Update the `keywords_resolve` row in `plugins/rf-core/evals/tasks/README.md` to say how `min_calls` counts; verify `scripts/check-docs-voice.py` passes. +- [x] 3.1 Record keyword calls per owner (test, suite/test fixture, user keyword body) in `scoring/keyword_resolution.py`, and compute `counted_library_calls` by expanding user-keyword calls from tests and fixtures, memoised, with a cycle guard; `min_calls` compares against it. Verify with new tests in `tests/eval` for the four scenarios of "Static keyword resolution counts library calls through user keywords" in `specs/skill-eval-harness/spec.md`, plus an embedded-argument wrapper (`Press Digit ${digit}`). +- [x] 3.2 Report call sites, library call sites and counted library calls in the check summary; verify a test asserts on the summary text. +- [x] 3.3 Update the `keywords_resolve` row in `plugins/rf-core/evals/tasks/README.md` to say how `min_calls` counts; verify `scripts/check-docs-voice.py` passes. ## 4. Grader: PlatynUI locator check (D6, D7) -- [ ] 4.1 Add `custom.platynui:locators_ok` under `src/rf_skill_eval/scoring/custom/`, taking `target` and `name_to_id`; verify with tests for the four scenarios of "PlatynUI tasks check locators statically", plus `@native:AutomationId` accepted, an `[@Id=… or @Name=…]` predicate accepted, and an `@AutomationId` in a comment ignored. -- [ ] 4.2 Let `suggests_plugin_or_passes` take `checks: [...]` (all must pass) alongside `check`; verify a test for each form, and that the other plugins' partial tasks load and score unchanged. -- [ ] 4.3 Add the locator check to `narrow-platynui-calculator-01.yaml` (gating) and to the partial task's `checks` list; verify `rf-skill-eval doctor --tasks-dir plugins/rf-desktop/evals/tasks/narrow` passes and both tasks load. -- [ ] 4.4 Re-grade offline the 72 stored rf-desktop sessions of runs 42887127109, 42993062842 and 43167186288 with tasks 3 and 4 in place; verify the static verdicts match the Windows live results of 2026-11-12 (only the 5 suites that use `@Id` for the display pass `platynui_locators`), and the 7 `min_calls` false failures no longer fail on call count. +- [x] 4.1 Add `custom.platynui:locators_ok` under `src/rf_skill_eval/scoring/custom/`, taking `target` and `name_to_id`; verify with tests for the four scenarios of "PlatynUI tasks check locators statically", plus `@native:AutomationId` accepted, an `[@Id=… or @Name=…]` predicate accepted, and an `@AutomationId` in a comment ignored. +- [x] 4.2 Let `suggests_plugin_or_passes` take `checks: [...]` (all must pass) alongside `check`; verify a test for each form, and that the other plugins' partial tasks load and score unchanged. +- [x] 4.3 Add the locator check to `narrow-platynui-calculator-01.yaml` (gating) and to the partial task's `checks` list; verify `rf-skill-eval doctor --tasks-dir plugins/rf-desktop/evals/tasks/narrow` passes and both tasks load. Done: both tasks load and `doctor` passes. The golden suite `tests/eval/golden/narrow-platynui-calculator-01/tests/calculator.robot` used the old README locators and failed the new check; it now uses `@Id`. +- [x] 4.4 Re-grade offline the 72 stored rf-desktop sessions of runs 42887127109, 42993062842 and 43167186288 with tasks 3 and 4 in place; verify the static verdicts agree with the Windows live results of 2026-11-12 (no suite that fails live passes `platynui_locators`), and the 7 `min_calls` false failures no longer fail on call count. Done: 63 sessions wrote a suite. The 7 false failures now pass `keywords_resolve` (counted 5–6 library calls), all in the skill arm: narrow skill arm 11/18 → 16/18, partial skill arm 14/18 → 16/18; baseline arms unchanged. `platynui_locators` passes 1 suite (narrow skill arm, `@Id` throughout), which also passed live. The other 4 live passes used `@Id` only for the display and English `@Name` for the buttons; they passed only after the names were translated, so the check rejects them, as the "Language-bound name rejected" scenario requires. No suite that failed live passes the check. ## 5. Hooks: formatting drift opt-in (D2) -- [ ] 5.1 In `validate_robot.mjs`, start the tier-2 formatter only when `RF_AGENTSKILLS_FORMAT_CHECK` is truthy; verify with `plugins/rf-core/tests/test_hook_scripts.py` cases for "enabled shows the diff" and "unset starts no formatter and shows no diff". -- [ ] 5.2 Document the flag in `plugins/rf-core/hooks/README.md` (tier table and env-flag table); verify the README lists it next to `RF_AGENTSKILLS_FILE_DRYRUN`. +- [x] 5.1 In `validate_robot.mjs`, start the tier-2 formatter only when `RF_AGENTSKILLS_FORMAT_CHECK` is truthy; verify with `plugins/rf-core/tests/test_hook_scripts.py` cases for "enabled shows the diff" and "unset starts no formatter and shows no diff". +- [x] 5.2 Document the flag in `plugins/rf-core/hooks/README.md` (tier table and env-flag table); verify the README lists it next to `RF_AGENTSKILLS_FILE_DRYRUN`. ## 6. Hooks: Stop reminder after a run (D3) -- [ ] 6.1 In `maybe_remind_robot_tests.mjs`, track the positions of the last `.robot`/`.resource` write and the last Robot Framework run command, and stay silent when the run comes later; change the first step's wording to "Run the affected tests where they can run". Verify with hook tests for the three scenarios in `specs/stop-hook-safety/spec.md`, and that the existing once-per-session and `stop_hook_active` tests still pass. +- [x] 6.1 In `maybe_remind_robot_tests.mjs`, track the positions of the last `.robot`/`.resource` write and the last Robot Framework run command, and stay silent when the run comes later; change the first step's wording to "Run the affected tests where they can run". Verify with hook tests for the three scenarios in `specs/stop-hook-safety/spec.md`, and that the existing once-per-session and `stop_hook_active` tests still pass. ## 7. Generated copies -- [ ] 7.1 Regenerate `bundles/rf-agentskills/` and any other generated variants from the rf-core sources; verify `scripts/check-drift.sh` and the "Check generated variants and plugin layout" check pass locally. +- [x] 7.1 Regenerate `bundles/rf-agentskills/` and any other generated variants from the rf-core sources; verify `scripts/check-drift.sh` and the "Check generated variants and plugin layout" check pass locally. ## 8. Skill (D4) -- [ ] 8.1 Add the "Check a keyword" line with `robotcode doc keyword PlatynUI.BareMetal ""` and `robotcode doc keywords PlatynUI.BareMetal ""` to the `Library Import` section of `plugins/rf-desktop/skills/rf-platynui/SKILL.md` and bump its minor version; verify `scripts/validate-skills.py` and `scripts/check-skill-keywords.py` pass. +- [x] 8.1 Add the "Check a keyword" line with `robotcode doc keyword PlatynUI.BareMetal ""` and `robotcode doc keywords PlatynUI.BareMetal ""` to the `Library Import` section of `plugins/rf-desktop/skills/rf-platynui/SKILL.md` and bump its minor version; verify `scripts/validate-skills.py` and `scripts/check-skill-keywords.py` pass. Done: no version bump, since 1.0.0 is unreleased (see 10.1); the change is in the rf-desktop 1.0.0 CHANGELOG entry. ## 9. Live check (D8) -- [ ] 9.1 Write `scripts/desktop-live-check.py` (collect, screen, run with a fresh Calculator, waiting until `CalculatorResults` answers a query rather than a fixed sleep, a 150 s limit, report per tier and arm); verify unit tests for the collection and the screening (an `OperatingSystem` import and an `Evaluate` call are skipped with a reason) run on Linux without PlatynUI. -- [ ] 9.2 Document it in `docs/` (where the rf-desktop eval docs live): Windows setup with uv and the pinned PlatynUI and platynui-cli versions, running it in the desktop session directly or over SSH through an `/it` scheduled task, and how to read the report; verify `scripts/check-docs-voice.py` passes. -- [ ] 9.3 Run it on the maintainer's Windows PC (`desktop-7800x3d`) against the artifacts of run 42993062842; verify it reproduces the 2026-11-12 results (skill example passes; all eval suites fail as written under the German display language). +- [x] 9.1 Write `scripts/desktop-live-check.py` (collect, screen, run with a fresh Calculator, waiting until `CalculatorResults` answers a query rather than a fixed sleep, a 150 s limit, report per tier and arm); verify unit tests for the collection and the screening (an `OperatingSystem` import and an `Evaluate` call are skipped with a reason) run on Linux without PlatynUI. +- [x] 9.2 Document it in `docs/` (where the rf-desktop eval docs live): Windows setup with uv and the pinned PlatynUI and platynui-cli versions, running it in the desktop session directly or over SSH through an `/it` scheduled task, and how to read the report; verify `scripts/check-docs-voice.py` passes. +- [x] 9.3 Run it on the maintainer's Windows PC (`desktop-7800x3d`) against the artifacts of run 42993062842; verify it reproduces the 2026-11-12 results (skill example passes; all eval suites fail as written under the German display language). Done 2026-11-12 (German display language): the skill example passes; all 20 suites fail as written, 19 on `//control:Window[@Name='Calculator']` not found and 1 on its own window check. None skipped by screening. The first attempt reported every suite as "Calculator not ready"; the readiness check now compares raw bytes and names the last query output when it gives up. ## 10. Tests, versions, PR -- [ ] 10.1 Bump rf-core and rf-desktop plugin versions with `scripts/bump-version.sh`; verify `scripts/validate-marketplace.py` passes. -- [ ] 10.2 Run `uv run pytest tests/eval plugins/rf-core/tests`, `ruff check src tests/eval` and `scripts/check-governance.py`; verify all pass. +- [x] 10.1 Record the rf-core and rf-desktop changes for release; verify `scripts/validate-marketplace.py` passes. Done: neither plugin has a release tag, so 1.0.0 is still unreleased; the changes go into the 1.0.0 (unreleased) CHANGELOG entries instead of a 1.1.0 bump, and `rf-platynui` stays at the plugin's 1.0.0 (validate-skills requires the two to match). +- [x] 10.2 Run `uv run pytest tests/eval plugins/rf-core/tests`, `ruff check src tests/eval` and `scripts/check-governance.py`; verify all pass. Done: `tests`, `plugins/rf-core/tests` and `plugins/rf-desktop/tests` pass, ruff and governance pass. `test_check_rf_environment_uses_project_venv` fails locally on a clean `main` checkout too (it passes in CI), so it isn't caused by this change. - [ ] 10.3 Open the PR against `main` (ask before posting); verify the required checks pass. ## 11. Re-baseline (D9) diff --git a/plugins/rf-core/CHANGELOG.md b/plugins/rf-core/CHANGELOG.md index 5d9937d..ba49bc1 100644 --- a/plugins/rf-core/CHANGELOG.md +++ b/plugins/rf-core/CHANGELOG.md @@ -9,3 +9,5 @@ First release, split out of `rf-agentskills` 3.0.0: `rf-language`, `rf-setup` an - **Library install steps moved to the library skills.** `rf-setup` keeps environments, uv, pip, Poetry, robotcode, Robocop and troubleshooting, and a table naming each library's package, install command, skill and plugin. - **Library migration tables moved** to `rf-browser` (plugin `rf-web`) and `rf-restinstance` (plugin `rf-api`); `rf-migration-guide` routes library moves there. - **Core names the plugin of every library skill**, and the subagents, hooks and `rf-setup` suggest installing the plugin when its skill is missing. +- **Formatting drift is opt-in.** The per-file hook runs `robocop format --check --diff` only with `RF_AGENTSKILLS_FORMAT_CHECK=1`. Shown a diff, agents reformatted files that were otherwise correct. +- **The Stop test reminder stays quiet once the suite has run** (`robot`, `python -m robot` or `robotcode robot`, dry runs included) after the last `.robot`/`.resource` write. Its first step now says to run the tests where they can run. diff --git a/plugins/rf-core/evals/tasks/README.md b/plugins/rf-core/evals/tasks/README.md index e87d2e3..29917f4 100644 --- a/plugins/rf-core/evals/tasks/README.md +++ b/plugins/rf-core/evals/tasks/README.md @@ -60,7 +60,7 @@ does not have. | `file_not_contains` | `path` (glob allowed), `regex` | No file may match. Missing file → `failed`. | | `robot_pass` | `target`, opt. `args`, `expected_tests`, `expected_tests_exact`, `requires` | `robot ` (cwd = workspace) exits 0 with ≥ 1 test; `expected_tests` = minimum number of **passed** tests (catches skipped tests); with `expected_tests_exact: true` exactly that many tests must exist and pass (a mismatch names expected and actual counts); `requires` = modules that must be importable, else `skipped`. | | `robot_dryrun` | `target`, opt. `args`, `expected_tests`, `expected_tests_exact`, `requires` | `robot --dryrun`: parses the suite and resolves imports and keywords; the count options work as for `robot_pass` (e.g. `args: [--include, smoke]` with an exact count checks tag selection). | -| `keywords_resolve` | `target`, `specs`, opt. `required_libraries`, `min_calls` | Static: every keyword call resolves against suite/resource user keywords, standard libraries and the given libdoc JSON specs (committed under the fixture's `specs/`). `skipped` only when a spec file is missing. Used where the library cannot run on CI (appium, platynui). | +| `keywords_resolve` | `target`, `specs`, opt. `required_libraries`, `min_calls` | Static: every keyword call resolves against suite/resource user keywords, standard libraries and the given libdoc JSON specs (committed under the fixture's `specs/`). `min_calls` (default 1) counts the spec-library calls that tests and fixtures reach, including those inside the user keywords they call; standard-library calls don't count. `skipped` only when a spec file is missing. Used where the library cannot run on CI (appium, platynui). | | `no_deprecated_keywords` | `target` | Rejects `Run Keyword If`/`Unless`, `Return From Keyword`. | | `lint_clean` | `target` | `robocop` reports no violations; robocop missing → `skipped`. | | `import_resolves` | `path` or `module` | `Library`/`Resource` imports resolve. | diff --git a/plugins/rf-core/hooks/README.md b/plugins/rf-core/hooks/README.md index 1b626e8..0879667 100644 --- a/plugins/rf-core/hooks/README.md +++ b/plugins/rf-core/hooks/README.md @@ -21,7 +21,7 @@ rather than written-and-broken. | Event | Script | Always fires? | Cost when no-op | Cost when active | Hook timeout | |---|---|---|---|---|---| | `SessionStart` | `scripts/check_rf_environment.mjs` | yes | — | ~0.5–1 s (import probes) | 15 s | -| `PostToolUse` (matcher: `Write\|Edit`) | `scripts/validate_robot.mjs` | only on Write/Edit | ~30ms (file extension check) | ~1–1.5 s on a 500-line file once robotcode's cache is warm (`robotcode analyze code`, Robocop check and format run in parallel); the first run in a project ~4 s; +0.4 s or more with the opt-in dry run | 60 s | +| `PostToolUse` (matcher: `Write\|Edit`) | `scripts/validate_robot.mjs` | only on Write/Edit | ~30ms (file extension check) | ~1–1.5 s on a 500-line file once robotcode's cache is warm (`robotcode analyze code` and Robocop check run in parallel, plus the formatter when enabled); the first run in a project ~4 s; +0.4 s or more with the opt-in dry run | 60 s | | `PostToolUse` (matcher: `Bash`) | `scripts/rf_error_hints.mjs` | only on Bash | ~30 ms (text scan) | ~30 ms | 10 s | | `UserPromptSubmit` | `scripts/maybe_inject_rf_context.mjs` | always invoked, conditional injection | ~30ms (regex over prompt) | same (≤ 450-character text) | 15 s | | `Stop` | `scripts/report_pending_errors.mjs` | always invoked, reports only files whose validation found errors or unresolved names that are still there | ~30 ms (reads the session record) | one `validate_robot.mjs` run per pending file | 60 s | @@ -106,9 +106,12 @@ Fires from `PostToolUse` after a Write/Edit of a `.robot`/`.resource` file. gets no `VAR` hint), and so does the project's Robocop config. - If Robocop is missing, times out (10 s), or prints nothing parsable while failing, the tier is silent — a tool failure is never a finding. -- **Tier 2 — formatting drift:** runs `robocop format --no-cache --check --diff`. - Purely informational — surfaces the proposed reformat as `additionalContext` - (exit 0). Formatting differences **never** cause exit 2. +- **Tier 2 — formatting drift (opt-in):** with `RF_AGENTSKILLS_FORMAT_CHECK=1`, + runs `robocop format --no-cache --check --diff`. Purely informational — + surfaces the proposed reformat as `additionalContext` (exit 0). Formatting + differences **never** cause exit 2. Off by default: shown a diff, the agent + reformats and re-checks a file that is otherwise correct, which cost one to + three turns per written file in the evals. - **Opt-in per-file dry run:** with `RF_AGENTSKILLS_FILE_DRYRUN=1`, a written `.robot` suite (not `.resource`, not `__init__.robot`) is also run through `robot --dryrun --output NONE --report NONE --log NONE` from the project @@ -128,6 +131,7 @@ exit-2 error diagnostic is written. Robocop is always called with |---|---|---| | `RF_AGENTSKILLS_DEPRECATION_CHECK` | `warn` (default), `off` | `off` drops all deprecation output. Any other value is treated as `warn`; no value makes a deprecation block (exit 2), and the variable never affects error findings. | | `RF_AGENTSKILLS_FILE_DRYRUN` | unset (default), `1`/`true`/`yes`/`on` | Enables the per-file dry run above. | +| `RF_AGENTSKILLS_FORMAT_CHECK` | unset (default), `1`/`true`/`yes`/`on` | Enables the formatting-drift tier above. | | `RF_AGENTSKILLS_PROJECT_VALIDATION` | unset (default), `1`/`true`/`yes`/`on` | Enables the project-wide Stop tier below. | #### Opting out per rule @@ -335,8 +339,12 @@ event produces no output; it always exits 0. or `"…something.resource"` (with bounded regex — `notes.robotic.md` does not count). Streaming chunking keeps memory bounded on long sessions. +- Also notes the last Bash `"command"` that runs Robot Framework (`robot`, + `python -m robot`, `robotcode … robot`, dry runs included). If that run + comes after the last `.robot`/`.resource` file path, the session already + ran its suite and the hook exits without output. - On hit: emits `additionalContext` reminding the user to run - the affected tests (`uv run robotcode robot -bl ""`), read the + the affected tests where they can run (`uv run robotcode robot -bl ""`), read the results with `uv run robotcode results summary --failed`, and open `results/report.html`. - On miss: stdout stays empty. diff --git a/plugins/rf-core/scripts/maybe_remind_robot_tests.mjs b/plugins/rf-core/scripts/maybe_remind_robot_tests.mjs index fe72b36..12634dc 100644 --- a/plugins/rf-core/scripts/maybe_remind_robot_tests.mjs +++ b/plugins/rf-core/scripts/maybe_remind_robot_tests.mjs @@ -5,7 +5,9 @@ // transcript for any tool result that wrote a .robot or .resource file, // and only emits an additionalContext reminder ("run robot, inspect // report.html") when at least one such file was touched. Sessions that -// produced only non-RF artifacts get no injection. +// produced only non-RF artifacts get no injection, and neither do sessions +// that ran Robot Framework (`robot`, `python -m robot`, `robotcode robot`, +// dry runs included) after their last .robot/.resource write. // // Cross-platform port of maybe_remind_robot_tests.sh. Reads the // transcript via streaming-ish chunking so very long sessions don't @@ -55,13 +57,25 @@ if (sessionId) { // Pattern: `"file_path" : "...path.robot"` (or .resource). Matches the // `grep -E` regex from the bash original. Whitespace optional between // quotes and colon. -const FILE_PATH_RE = /"file_path"\s*:\s*"[^"]*\.(?:robot|resource)"/i; +const FILE_PATH_RE = /"file_path"\s*:\s*"[^"]*\.(?:robot|resource)"/gi; +// A Bash tool call's `"command": "…"` (JSON string, escapes kept). +const COMMAND_RE = /"command"\s*:\s*"((?:[^"\\]|\\.)*)"/g; +// Robot Framework run at a command position: start, after ; & | ( or a +// `uv/poetry/pipenv run`, optionally with a path (`.venv/bin/robot`). +const RF_RUN_RE = new RegExp( + String.raw`(?:^|[;&|(]\s*|\b(?:uv|poetry|pipenv)\s+run\s+)(?:[\w.~/\\-]*[/\\])?` + + String.raw`(?:python3?(?:\.exe)?\s+-m\s+robot(?=\s|$)|robotcode(?:\.exe)?(?:\s+\S+)*?\s+robot\b|robot(?:\.exe)?(?=\s|$))`, +); +const CARRY = 4096; -function transcriptMentionsRfFile(path) { +// Offsets of the last .robot/.resource file_path and the last Robot +// Framework run command in the transcript (-1 when absent). +function lastWriteAndRun(path) { // The transcript is a JSONL file that can be tens of MB on long // sessions. Read in 64 KB chunks rather than slurping it all. const size = (() => { try { return statSync(path).size; } catch { return 0; } })(); - if (!size) return false; + const found = { write: -1, run: -1 }; + if (!size) return found; const BUF = Buffer.alloc(65536); const fd = openSync(path, "r"); let pos = 0; @@ -71,27 +85,33 @@ function transcriptMentionsRfFile(path) { const n = readSync(fd, BUF, 0, BUF.length, pos); if (n <= 0) break; // Stitch the boundary with the previous chunk so we don't miss a - // match split across the seam. + // match split across the seam (offsets are approximate, in chars). const chunk = carry + BUF.slice(0, n).toString("utf-8"); - if (FILE_PATH_RE.test(chunk)) return true; - // Keep the last ~256 bytes as carry to bridge boundary matches. - carry = chunk.length > 256 ? chunk.slice(-256) : chunk; + const base = pos - carry.length; + for (const m of chunk.matchAll(FILE_PATH_RE)) found.write = Math.max(found.write, base + m.index); + for (const m of chunk.matchAll(COMMAND_RE)) { + if (RF_RUN_RE.test(m[1])) found.run = Math.max(found.run, base + m.index); + } + carry = chunk.length > CARRY ? chunk.slice(-CARRY) : chunk; pos += n; } } finally { try { closeSync(fd); } catch {} } - return false; + return found; } -if (!transcriptMentionsRfFile(transcript)) process.exit(0); +const { write: lastWrite, run: lastRun } = lastWriteAndRun(transcript); +if (lastWrite < 0) process.exit(0); +// The suite already ran (or dry-ran) after the last write: nothing to remind. +if (lastRun > lastWrite) process.exit(0); const payload = { hookSpecificOutput: { hookEventName: "Stop", additionalContext: [ "Robot Framework artifacts detected this session. Recommended next steps:", - '\n 1. Run the affected tests: uv run robotcode robot -bl "" (or a whole directory)', + '\n 1. Run the affected tests where they can run: uv run robotcode robot -bl "" (or a whole directory)', "\n 2. Read the results: uv run robotcode results summary --failed, then robotcode results log --failed", " (not using uv: run robotcode from the project's .venv; see rf-setup)", "\n 3. Open results/report.html for the rendered report.", diff --git a/plugins/rf-core/scripts/validate_robot.mjs b/plugins/rf-core/scripts/validate_robot.mjs index b5c0328..4b1857e 100644 --- a/plugins/rf-core/scripts/validate_robot.mjs +++ b/plugins/rf-core/scripts/validate_robot.mjs @@ -35,8 +35,10 @@ // any other value (including a legacy `block`) means `warn`. // No parsable line from a failing Robocop run = tool failure = silence. // -// Tier 2 (formatting drift): `robocop format --check --diff`. Informational -// additionalContext only. +// Tier 2 (opt-in, RF_AGENTSKILLS_FORMAT_CHECK=1, formatting drift): +// `robocop format --check --diff`. Informational additionalContext only. +// Off by default: a formatting diff leads the agent to reformat and +// re-check a file that is otherwise correct. // // Tier 3 (opt-in, RF_AGENTSKILLS_FILE_DRYRUN=1): `robot --dryrun` on the // edited suite (.robot only, not __init__.robot). Advisory context only. @@ -451,7 +453,7 @@ const advisory = []; const robocopPy = STOP_RECHECK ? null : findInterpreterWith("robocop", cwd); const analyzeRun = startAnalyze(); const robocopRun = robocopPy ? startRobocop(robocopPy) : Promise.resolve(null); -const formatRun = robocopPy +const formatRun = robocopPy && isTruthy(process.env.RF_AGENTSKILLS_FORMAT_CHECK) ? spawnAsync(robocopPy, ["-m", "robocop", "format", "--no-cache", "--check", "--diff", "--no-overwrite", filePath], ROBOCOP_TIMEOUT_MS) : Promise.resolve(null); @@ -482,7 +484,7 @@ if (robocopPy) { if (text) advisory.push(text); } - // ── Tier 2: formatting drift (informational only) ── + // ── Tier 2 (opt-in): formatting drift (informational only) ── const fmt = await formatRun; if (fmt && !fmt.error && fmt.status !== 0 && fmt.status !== null && (fmt.stdout ?? "").trim()) { let diff = fmt.stdout.trim(); diff --git a/plugins/rf-core/tests/test_hook_scripts.py b/plugins/rf-core/tests/test_hook_scripts.py index e1442f4..5d9265a 100644 --- a/plugins/rf-core/tests/test_hook_scripts.py +++ b/plugins/rf-core/tests/test_hook_scripts.py @@ -486,6 +486,57 @@ def test_remind_does_not_match_substring_in_unrelated_path(tmp_path: Path) -> No _expect_no_injection(out) +def _tool_use(name: str, **inp: str) -> str: + return json.dumps({"type": "assistant", "message": {"role": "assistant", "content": [ + {"type": "tool_use", "name": name, "input": inp}]}}) + + +def test_remind_skips_after_a_dry_run_of_the_last_write(tmp_path: Path) -> None: + transcript = _write_transcript(tmp_path, [ + _tool_use("Write", file_path="/w/tests/calculator.robot", content="*** Test Cases ***"), + _tool_use("Bash", command="robot --dryrun tests/calculator.robot"), + ]) + out, _err, rc = _run(REMIND_SCRIPT, {"transcript_path": str(transcript)}) + assert rc == 0 + _expect_no_injection(out) + + +@pytest.mark.parametrize("command", [ + "uv run robotcode robot -bl Calculator", + "cd /w && python -m robot --outputdir results tests", + ".venv/bin/robot tests", +]) +def test_remind_skips_after_any_robot_run(tmp_path: Path, command: str) -> None: + transcript = _write_transcript(tmp_path, [ + _tool_use("Edit", file_path="/w/tests/login.robot", old_string="a", new_string="b"), + _tool_use("Bash", command=command), + ]) + out, _err, rc = _run(REMIND_SCRIPT, {"transcript_path": str(transcript)}) + _expect_no_injection(out) + + +@pytest.mark.parametrize("command", ["grep -rn robot tests", "python -m robot.libdoc PlatynUI.BareMetal x.json", + "cat tests/login.robot"]) +def test_remind_not_fooled_by_commands_that_only_mention_robot(tmp_path: Path, command: str) -> None: + transcript = _write_transcript(tmp_path, [ + _tool_use("Write", file_path="/w/tests/login.robot", content="x"), + _tool_use("Bash", command=command), + ]) + out, _err, rc = _run(REMIND_SCRIPT, {"transcript_path": str(transcript)}) + _expect_injection(out) + + +def test_remind_fires_for_a_write_after_the_last_run(tmp_path: Path) -> None: + transcript = _write_transcript(tmp_path, [ + _tool_use("Bash", command="robot tests/"), + _tool_use("Edit", file_path="/w/tests/login.robot", old_string="a", new_string="b"), + ]) + out, _err, rc = _run(REMIND_SCRIPT, {"transcript_path": str(transcript)}) + assert rc == 0 + ctx = _expect_injection(out)["additionalContext"] + assert "1. Run the affected tests where they can run:" in ctx + + # --- Stop-hook loop safety (stop_hook_active) ----------------------------- @@ -688,6 +739,7 @@ def _hook_env(**extra: str) -> dict: env = { k: v for k, v in os.environ.items() if k not in ("TOOL_INPUT", "RF_AGENTSKILLS_DEPRECATION_CHECK", "RF_AGENTSKILLS_FILE_DRYRUN", + "RF_AGENTSKILLS_FORMAT_CHECK", "RF_AGENTSKILLS_HOOK_TIMEOUT_MS") } if sys.prefix != sys.base_prefix: @@ -1795,3 +1847,45 @@ def test_keyword_written_later_is_not_reported(tmp_path: Path, pending_session: resource.write_text("*** Keywords ***\nOpen Order Page\n No Operation\n", encoding="utf-8") assert _edit(resource, tmp_path, pending_session)[2] == 0 assert _stop(tmp_path, pending_session) == ("", "", 0) + + +# --- formatting drift is opt-in (fix-desktop-eval-turn-overhead 5.1) ------------- + +_BADLY_FORMATTED = "*** Test Cases ***\nT\n Log x \n" + + +def _logging_venv(tmp_path: Path) -> tuple[Path, Path]: + """A VIRTUAL_ENV whose python logs its arguments, then runs the test interpreter.""" + venv, log = tmp_path / "logvenv", tmp_path / "python-calls.log" + (venv / "bin").mkdir(parents=True) + py = venv / "bin" / "python" + py.write_text(f'#!/bin/sh\necho "$*" >> "{log}"\nexec "{sys.executable}" "$@"\n', encoding="utf-8") + py.chmod(0o755) + return venv, log + + +@_needs_robocop +@pytest.mark.skipif(sys.platform == "win32", reason="shell wrapper interpreter") +def test_format_check_enabled_shows_the_diff(tmp_path: Path, session_id: str) -> None: + suite = tmp_path / "fmt.robot" + suite.write_text(_BADLY_FORMATTED, encoding="utf-8") + env = _hook_env(RF_AGENTSKILLS_FORMAT_CHECK="1", RF_AGENTSKILLS_DEPRECATION_CHECK="off") + out, err, rc = _run(VALIDATE_SCRIPT, _write_payload(suite, tmp_path, session_id), env=env) + assert rc == 0, err + assert "Robocop suggests formatting changes" in _context(out) + + +@_needs_robocop +@pytest.mark.skipif(sys.platform == "win32", reason="shell wrapper interpreter") +def test_format_check_unset_starts_no_formatter(tmp_path: Path, session_id: str) -> None: + suite = tmp_path / "fmt.robot" + suite.write_text(_BADLY_FORMATTED, encoding="utf-8") + venv, log = _logging_venv(tmp_path) + env = _hook_env(VIRTUAL_ENV=str(venv), RF_AGENTSKILLS_DEPRECATION_CHECK="off") + env.pop("RF_AGENTSKILLS_FORMAT_CHECK", None) + out, err, rc = _run(VALIDATE_SCRIPT, _write_payload(suite, tmp_path, session_id), env=env) + assert rc == 0, err + assert "formatting" not in _context(out) + calls = log.read_text(encoding="utf-8") + assert "robocop check" in calls # the wrapper was used for tier 1 + assert "robocop format" not in calls diff --git a/plugins/rf-desktop/CHANGELOG.md b/plugins/rf-desktop/CHANGELOG.md index 2d9314d..a38552c 100644 --- a/plugins/rf-desktop/CHANGELOG.md +++ b/plugins/rf-desktop/CHANGELOG.md @@ -7,3 +7,4 @@ The `rf-desktop` plugin, versioned on its own (`rf-desktop--v` tags); s First release, split out of `rf-agentskills` 3.0.0: `rf-platynui`. It depends on `rf-core` `^1.0`. Skill names are unchanged; their plugin prefix is now `rf-desktop:` instead of `rf-agentskills:`, for example `/rf-desktop:rf-platynui`. - **Install steps moved here** from `rf-setup`: `rf-platynui` has `references/install.md`, with the pre-release pin. +- **`rf-platynui` names `robotcode doc keyword PlatynUI.BareMetal ""`** in its Library Import section, as the way to check a keyword and its arguments. diff --git a/plugins/rf-desktop/evals/tasks/narrow/narrow-platynui-calculator-01.yaml b/plugins/rf-desktop/evals/tasks/narrow/narrow-platynui-calculator-01.yaml index 59e422c..e7d24a1 100644 --- a/plugins/rf-desktop/evals/tasks/narrow/narrow-platynui-calculator-01.yaml +++ b/plugins/rf-desktop/evals/tasks/narrow/narrow-platynui-calculator-01.yaml @@ -3,8 +3,9 @@ skill: rf-platynui tier: narrow description: | Write a PlatynUI desktop test for a calculator flow (1 + 2 = 3). No desktop - session exists on the runner, so the gating check is static: - keywords_resolve against the committed PlatynUI.BareMetal libdoc spec. + session exists on the runner, so the gating checks are static: + keywords_resolve against the committed PlatynUI.BareMetal libdoc spec, and + platynui_locators for the @Id locators that work on a real desktop. prompt: | We automate desktop apps with Robot Framework and PlatynUI. README.md lists the Calculator element XPaths. @@ -36,6 +37,12 @@ grader_checks: specs: [specs/PlatynUI.BareMetal.json] required_libraries: [PlatynUI.BareMetal] min_calls: 4 + - type: custom_python + name: platynui_locators + func_ref: rf_skill_eval.scoring.custom.platynui:locators_ok + target: tests/calculator.robot + name_to_id: {One: num1Button, Two: num2Button, Plus: plusButton, Equals: equalButton} + gating: true - type: file_contains name: targets_calculator path: tests/calculator.robot diff --git a/plugins/rf-desktop/evals/tasks/partial/partial-platynui-calculator-01.yaml b/plugins/rf-desktop/evals/tasks/partial/partial-platynui-calculator-01.yaml index 9ecdd7d..6b43de0 100644 --- a/plugins/rf-desktop/evals/tasks/partial/partial-platynui-calculator-01.yaml +++ b/plugins/rf-desktop/evals/tasks/partial/partial-platynui-calculator-01.yaml @@ -23,11 +23,17 @@ grader_checks: name: suggests_plugin_or_passes func_ref: rf_skill_eval.scoring.custom.partial_install:suggests_plugin_or_passes plugin: rf-desktop - check: - type: keywords_resolve + checks: + - type: keywords_resolve + name: platynui_keywords_resolve target: tests/calculator.robot specs: [specs/PlatynUI.BareMetal.json] required_libraries: [PlatynUI.BareMetal] min_calls: 4 + - type: custom_python + name: platynui_locators + func_ref: rf_skill_eval.scoring.custom.platynui:locators_ok + target: tests/calculator.robot + name_to_id: {One: num1Button, Two: num2Button, Plus: plusButton, Equals: equalButton} gating: true fixture: sut-platynui diff --git a/plugins/rf-desktop/skills/rf-platynui/SKILL.md b/plugins/rf-desktop/skills/rf-platynui/SKILL.md index 37d9511..0292014 100644 --- a/plugins/rf-desktop/skills/rf-platynui/SKILL.md +++ b/plugins/rf-desktop/skills/rf-platynui/SKILL.md @@ -74,6 +74,8 @@ Library PlatynUI.BareMetal auto_activate=${True} pointer_profile=${POIN Library scope is **SUITE**. The native runtime is created lazily on first keyword use (so `libdoc` and import work without a desktop). +**Check a keyword** before you use it: `robotcode doc keyword PlatynUI.BareMetal ""` prints its arguments, and `robotcode doc keywords PlatynUI.BareMetal ""` searches by name. Use them instead of parsing a libdoc file or reading the library source. + ## Essential Concepts ### The normalized desktop UI model diff --git a/scripts/desktop-live-check.py b/scripts/desktop-live-check.py new file mode 100644 index 0000000..0141e7e --- /dev/null +++ b/scripts/desktop-live-check.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +"""Run rf-desktop eval suites against the real Calculator on a Windows desktop. + +The CI eval grades PlatynUI suites statically (``keywords_resolve`` and +``platynui_locators``). This manual check runs them: it collects the suites the +PlatynUI sessions of one downloaded eval run wrote, plus the skill's example, +screens each one, and runs the safe ones with a fresh Calculator and a time +limit. It reports pass counts per tier and arm and never writes baselines. + +It must run in the logged-in desktop session, inside a venv that has +Robot Framework, robotframework-PlatynUI and platynui-cli; see +docs/ci/desktop-live-check.md. Downloaded artifacts are untrusted: run this +with ``python -I`` from outside the artifact directory. + +Usage: + python -I scripts/desktop-live-check.py [--out DIR] + python -I scripts/desktop-live-check.py --list # collect and screen only +""" + +from __future__ import annotations + +import argparse +import ast +import collections +import re +import shutil +import subprocess +import sys +import time +import xml.etree.ElementTree as ET +from dataclasses import dataclass, field +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] +SKILL_EXAMPLE = REPO / "plugins" / "rf-desktop" / "skills" / "rf-platynui" / "assets" / "examples" / "calculator.robot" + +#: Libraries a collected suite may import. Process only for the skill's +#: example, which starts Calculator itself. +ALLOWED_LIBRARIES = frozenset({"PlatynUI.BareMetal", "BuiltIn", "Collections", "String"}) +EXAMPLE_LIBRARIES = ALLOWED_LIBRARIES | {"Process"} +#: Keywords that run code or start processes (normalised names). File keywords +#: come from OperatingSystem, which no suite may import. +BANNED_KEYWORDS = frozenset( + {"evaluate", "runprocess", "startprocess", "callmethod", "importlibrary", "importresource", "importvariables"} +) +#: The skill's example starts Calculator itself. +EXAMPLE_BANNED = BANNED_KEYWORDS - {"startprocess"} +#: Text that would make a Python expression (IF, Should Be True, ${{ }}) unsafe. +UNSAFE_TEXT = re.compile(r"\$\{\{|__|\bimport\b|\bopen\s*\(|\bos\.|\bsys\.|subprocess|\bexec\b|\beval\b") +READY_QUERY = "//control:Text[@Id='CalculatorResults']" + + +@dataclass +class Suite: + tier: str # narrow | partial | example + arm: str # baseline | treatment | example + session: str + path: Path + skip_reason: str = "" + passed: bool | None = None + message: str = "" + seconds: float = 0.0 + extra: list[Path] = field(default_factory=list) + + +def collect(artifact_dir: Path, suite_name: str = "calculator.robot") -> list[Suite]: + """The task's suite (``workspace/tests/``) from each PlatynUI session under ``artifact_dir``. + + Only the file the task asks for: every workspace also holds the fixture's + stub ``tests/example.robot``. + """ + suites: list[Suite] = [] + for path in sorted(artifact_dir.rglob(f"workspace/tests/{suite_name}")): + session = path.parents[2].name + if "-platynui-" not in session: + continue + tier = session.split("-", 1)[0] + if tier not in ("narrow", "partial"): + continue + arm = "baseline" if "-baseline-" in session else "treatment" + suites.append(Suite(tier, arm, session, path)) + return suites + + +def _normalize(name: str) -> str: + return re.sub(r"[\s_]+", "", name).lower() + + +def screen( + path: Path, + *, + allowed: frozenset[str] = ALLOWED_LIBRARIES, + banned: frozenset[str] = BANNED_KEYWORDS, + root: Path | None = None, +) -> str: + """Why ``path`` must not run ('' when it may). Resources are screened too and must stay under ``root``.""" + root = (root or path.parent).resolve() + todo, seen = [path], set() + while todo: + current = todo.pop() + real = current.resolve() + if real in seen: + continue + seen.add(real) + if not real.is_relative_to(root): + return f"resource outside the session workspace: {current}" + if not current.is_file(): + return f"resource not found: {current}" + problem = _file_problem(current, allowed, banned, todo) + if problem: + return problem + return "" + + +def _file_problem(path: Path, allowed: frozenset[str], banned: frozenset[str], resources: list[Path]) -> str: + """The first reason ``path`` itself is unsafe; appends the resources it imports.""" + from robot.api import get_model, get_resource_model + from robot.parsing.model.statements import ( + KeywordCall, + LibraryImport, + ResourceImport, + Statement, + VariablesImport, + ) + + model = get_model(str(path)) if path.suffix == ".robot" else get_resource_model(str(path)) + for node in ast.walk(model): + if isinstance(node, LibraryImport) and str(node.name) not in allowed: + return f"imports library {node.name}" + if isinstance(node, VariablesImport): + return f"imports variables {node.name}" + if isinstance(node, ResourceImport) and node.name: + resources.append(path.parent / str(node.name).replace("${CURDIR}", str(path.parent))) + if ( + isinstance(node, KeywordCall) + and node.keyword + and _normalize(str(node.keyword).rpartition(".")[2]) in banned + ): + return f"calls {node.keyword} ({path.name}:{node.lineno})" + if isinstance(node, Statement): + problem = _expression_problem(node, path) + if problem: + return problem + return "" + + +def _expression_problem(node: object, path: Path) -> str: + """Text that could run Python (``${{ }}``, IF conditions, Should Be True), outside comments and docs.""" + from robot.api import Token + + if getattr(node, "type", None) in (Token.DOCUMENTATION, Token.METADATA): + return "" + for tok in getattr(node, "tokens", ()): + if tok.type not in (Token.COMMENT, Token.SEPARATOR, Token.EOL) and UNSAFE_TEXT.search(tok.value): + return f"unsafe expression {tok.value[:60]!r} ({path.name}:{tok.lineno})" + return "" + + +def _cli() -> str: + exe = Path(sys.executable).parent / ("platynui-cli.exe" if sys.platform == "win32" else "platynui-cli") + return str(exe) if exe.is_file() else (shutil.which("platynui-cli") or "platynui-cli") + + +def _robot() -> list[str]: + return [sys.executable, "-m", "robot"] + + +def stop_calculator() -> None: + subprocess.run(["taskkill", "/im", "CalculatorApp.exe", "/f"], capture_output=True, check=False) + + +def start_calculator(ready_timeout: float) -> str: + """Start Calculator and wait until its result display answers a query; '' when ready, else why not.""" + subprocess.Popen(["cmd", "/c", "start", "", "calc.exe"]) + deadline = time.monotonic() + ready_timeout + last = "no query ran" + while time.monotonic() < deadline: + r = subprocess.run([_cli(), "query", READY_QUERY], capture_output=True, check=False) + if b"CalculatorResults" in r.stdout: + return "" + out = (r.stdout or r.stderr).decode("utf-8", "replace").strip() + last = f"exit {r.returncode}: {out[:120] or 'no output'}" + time.sleep(1) + return f"Calculator not ready after {ready_timeout:.0f} s (last query: {last})" + + +def result_of(output_xml: Path) -> tuple[bool, str]: + try: + root = ET.parse(output_xml).getroot() + except (ET.ParseError, OSError): + return False, "no output.xml (killed at the time limit?)" + tests = root.findall(".//test") + if not tests: + status = root.find("suite/status") + return False, "no tests: " + ((status.text or "") if status is not None else "") + for test in tests: + status = test.find("status") + if status is not None and status.get("status") != "PASS": + return False, ((status.text or "").strip().splitlines() or [""])[0][:160] + return True, "" + + +def run_suite(suite: Suite, out: Path, *, timeout: float, ready_timeout: float) -> None: + stop_calculator() + time.sleep(1) + if suite.arm != "example": + why = start_calculator(ready_timeout) + if why: + suite.passed, suite.message = False, why + return + target = out / f"{suite.tier}__{suite.session}__{suite.path.stem}" + started = time.monotonic() + try: + subprocess.run([*_robot(), "--outputdir", str(target), "--consolecolors", "off", str(suite.path)], + capture_output=True, timeout=timeout, check=False) + suite.passed, suite.message = result_of(target / "output.xml") + except subprocess.TimeoutExpired: + suite.passed, suite.message = False, f"time limit {timeout:.0f} s" + suite.seconds = time.monotonic() - started + + +def display_language() -> str: + if sys.platform != "win32": + return "n/a (not Windows)" + import ctypes + import locale + + langid = ctypes.windll.kernel32.GetUserDefaultUILanguage() # type: ignore[attr-defined] + return locale.windows_locale.get(langid, hex(langid)) + + +def _bucket(message: str) -> str: + return re.sub(r"\d+(\.\d+)?", "N", re.sub(r"'[^']*'|\"[^\"]*\"", "Q", message))[:110] + + +def report(suites: list[Suite], *, ran: bool) -> str: + lines = [f"display language: {display_language()}", ""] + agg: dict[tuple[str, str], list[int]] = collections.defaultdict(lambda: [0, 0, 0]) + for s in suites: + row = agg[(s.tier, s.arm)] + row[0] += 1 + row[1] += bool(s.passed) + row[2] += bool(s.skip_reason) + lines.append(f"{'tier':8} {'arm':10} {'suites':>6} {'passed':>6} {'skipped':>7}") + for (tier, arm), (n, p, sk) in sorted(agg.items()): + lines.append(f"{tier:8} {arm:10} {n:6} {p if ran else '-':>6} {sk:7}") + skipped = [s for s in suites if s.skip_reason] + if skipped: + lines += ["", "skipped (not run):"] + [f" {s.session}: {s.skip_reason}" for s in skipped] + if ran: + top = collections.Counter(_bucket(s.message) for s in suites if s.passed is False and s.message) + if top: + lines += ["", "most common failures:"] + [f" {n:3} {m}" for m, n in top.most_common(8)] + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n\n", 1)[0]) + ap.add_argument("artifacts", type=Path, help="a downloaded skill-eval-full-rf-desktop- directory") + ap.add_argument("--out", type=Path, default=Path("live-check-results"), help="where robot writes its output") + ap.add_argument("--timeout", type=float, default=150.0, help="seconds per suite (default 150)") + ap.add_argument("--ready-timeout", type=float, default=30.0, help="seconds to wait for Calculator") + ap.add_argument("--suite-name", default="calculator.robot", help="the file the task asks for") + ap.add_argument("--list", action="store_true", help="collect and screen only; run nothing") + args = ap.parse_args(argv) + + suites = collect(args.artifacts, args.suite_name) + if SKILL_EXAMPLE.is_file(): + suites.append(Suite("example", "example", "skill example", SKILL_EXAMPLE)) + if not suites: + print(f"no PlatynUI suites under {args.artifacts}", file=sys.stderr) + return 1 + for s in suites: + if s.arm == "example": + s.skip_reason = screen(s.path, allowed=EXAMPLE_LIBRARIES, banned=EXAMPLE_BANNED) + else: + s.skip_reason = screen(s.path, root=s.path.parents[1]) + if not args.list: + if sys.platform != "win32": + print("the live check runs on a Windows desktop; use --list elsewhere", file=sys.stderr) + return 2 + args.out.mkdir(parents=True, exist_ok=True) + for s in (s for s in suites if not s.skip_reason): + run_suite(s, args.out, timeout=args.timeout, ready_timeout=args.ready_timeout) + print(f"{'PASS' if s.passed else 'FAIL'} {s.tier:7} {s.session} {s.message}", flush=True) + stop_calculator() + print(report(suites, ran=not args.list)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/rf_skill_eval/scoring/custom/partial_install.py b/src/rf_skill_eval/scoring/custom/partial_install.py index a888eaa..c6b7589 100644 --- a/src/rf_skill_eval/scoring/custom/partial_install.py +++ b/src/rf_skill_eval/scoring/custom/partial_install.py @@ -10,6 +10,9 @@ func_ref: rf_skill_eval.scoring.custom.partial_install:suggests_plugin_or_passes plugin: rf-web check: {type: robot_pass, target: tests/login.robot, requires: [Browser], expected_tests: 1} + +``checks: [...]`` takes several such checks instead of ``check``; all of them +must pass. """ from __future__ import annotations @@ -44,15 +47,25 @@ def suggests_plugin_or_passes(run: Run, params: dict[str, Any]) -> Verdict | dic return {"passed": False, "details": "custom check needs 'plugin'"} if suggests_plugin(assistant_text(run), plugin): return {"passed": True, "details": f"suggests installing {plugin}"} - check = dict(params.get("check") or {}) + checks = [dict(c) for c in params.get("checks") or []] + if params.get("check"): + checks.insert(0, dict(params["check"])) + if not checks: + return {"passed": False, "details": f"does not suggest installing {plugin}"} + results = [_inner(run, plugin, c) for c in checks] + detail = f"does not suggest installing {plugin}; " + "; ".join(d for _, d in results) + return {"passed": all(ok for ok, _ in results), "details": detail} + + +def _inner(run: Run, plugin: str, check: dict[str, Any]) -> tuple[bool, str]: kind = str(check.pop("type", "")) # Same alias the rubric grader applies to top-level checks. if "path" not in check and "target" in check: check["path"] = check["target"] if not kind: - return {"passed": False, "details": f"does not suggest installing {plugin}"} + return False, "fallback check has no 'type'" inner = lookup_check(kind)(run, f"{kind} (instead of suggesting {plugin})", check) - detail = f"does not suggest installing {plugin}; {kind}: {inner.status}" + detail = f"{check.get('name') or kind}: {inner.status}" if inner.details: detail += f" ({inner.details})" - return {"passed": inner.status == "passed", "details": detail} + return inner.status == "passed", detail diff --git a/src/rf_skill_eval/scoring/custom/platynui.py b/src/rf_skill_eval/scoring/custom/platynui.py new file mode 100644 index 0000000..b4a1b28 --- /dev/null +++ b/src/rf_skill_eval/scoring/custom/platynui.py @@ -0,0 +1,112 @@ +"""Static locator check for PlatynUI tasks (skill-eval-harness "PlatynUI tasks check locators statically"). + +The CI runner has no desktop, so ``keywords_resolve`` only proves that the +keywords exist. A live run on Windows (2026-11-12) showed the locators are +where suites fail: PlatynUI exposes the UIA AutomationId as ``@Id`` (a plain +``@AutomationId`` matches nothing), and ``@Name`` follows the display language. + + - type: custom_python + name: platynui_locators + func_ref: rf_skill_eval.scoring.custom.platynui:locators_ok + target: tests/calculator.robot + name_to_id: {One: num1Button, Plus: plusButton} + +The check scans every argument and variable value of the suite and the +resources it imports (comments and documentation are skipped). It fails on +``@AutomationId`` not written as ``@native:AutomationId``, and on a predicate +that matches ``@Name=''`` for a value in ``name_to_id`` without also +accepting an ``@Id`` (``[@Id='num1Button' or @Name='One']`` passes). +""" + +from __future__ import annotations + +import ast +import re +from pathlib import Path +from typing import Any + +from ...domain.run import Run +from ...domain.verdict import Verdict +from .language import _name + +_AUTOMATION_ID = re.compile(r"(? list[Path]: + """The suite plus the resources it imports, recursively.""" + from robot.api import get_model, get_resource_model + from robot.api.parsing import ResourceImport + + out: list[Path] = [] + todo = [suite] + seen: set[Path] = set() + while todo: + path = todo.pop() + real = path.resolve() + if real in seen or not path.is_file(): + continue + seen.add(real) + out.append(path) + model = get_model(str(path)) if path.suffix == ".robot" else get_resource_model(str(path)) + for node in ast.walk(model): + if isinstance(node, ResourceImport) and node.name: + raw = str(node.name).replace("${CURDIR}", str(path.parent)) + todo.append(Path(raw) if Path(raw).is_absolute() else path.parent / raw) + return out + + +def _arguments(path: Path) -> list[tuple[str, int]]: + """(value, line) of every argument token outside comments and documentation.""" + from robot.api import Token, get_model, get_resource_model + from robot.parsing.model.statements import Statement + + model = get_model(str(path)) if path.suffix == ".robot" else get_resource_model(str(path)) + values: list[tuple[str, int]] = [] + for node in ast.walk(model): + if not isinstance(node, Statement) or node.type in _SKIPPED_STATEMENTS: + continue + values.extend((t.value, t.lineno) for t in node.tokens if t.type == Token.ARGUMENT) + return values + + +def locator_problems(value: str, name_to_id: dict[str, str]) -> list[str]: + """Why ``value`` (one argument) holds a locator that fails on a real desktop.""" + problems: list[str] = [] + if _AUTOMATION_ID.search(value): + problems.append(f"{value!r} uses @AutomationId; PlatynUI exposes it as @Id") + for predicate in _PREDICATE.findall(value): + if _ID.search(predicate): + continue + for single, double in _NAME_VALUE.findall(predicate): + name = single or double + if name in name_to_id: + problems.append( + f"{value!r} finds {name!r} by its localised @Name; use @Id='{name_to_id[name]}'" + ) + return problems + + +def locators_ok(run: Run, params: dict[str, Any]) -> Verdict: + name = _name(params, "platynui_locators") + target = params.get("path") or params.get("target") + if not target: + return Verdict.errored(run.id, name, "platynui_locators needs 'target'") + try: + import robot.api # noqa: F401 + except ImportError: + return Verdict.skipped(run.id, name, "Robot Framework is not importable by the grader") + suite = run.effective_workspace / str(target) + if not suite.is_file(): + return Verdict.of(run.id, name, False, f"suite missing: {target}") + name_to_id = {str(k): str(v) for k, v in dict(params.get("name_to_id") or {}).items()} + problems: list[str] = [] + for path in _suite_files(suite): + for value, line in _arguments(path): + problems.extend(f"{path.name}:{line}: {p}" for p in locator_problems(value, name_to_id)) + if problems: + return Verdict.of(run.id, name, False, "; ".join(problems[:8])) + return Verdict.of(run.id, name, True, "locators use @Id or a language-independent form") diff --git a/src/rf_skill_eval/scoring/keyword_resolution.py b/src/rf_skill_eval/scoring/keyword_resolution.py index 5ad5071..2411576 100644 --- a/src/rf_skill_eval/scoring/keyword_resolution.py +++ b/src/rf_skill_eval/scoring/keyword_resolution.py @@ -13,6 +13,12 @@ A library import that is neither a standard library nor covered by a spec is reported as unresolved: the check cannot vouch for its keywords, so it fails. +``min_calls`` compares against the library calls the tests and suite/test +fixtures reach: a direct library call counts once, and a user-keyword call +counts the library calls in that keyword's body, recursively (memoised; a +keyword already being expanded adds nothing). User keywords that nothing calls +add nothing, and standard-library calls never count. + The approach mirrors ``scripts/check-skill-keywords.py`` (repository QA for the skills' own examples) but is self-contained: the harness package must not import repository scripts. @@ -93,9 +99,14 @@ def _libdoc(source: str) -> tuple[str, tuple[str, ...]]: return str(doc.name), tuple(str(kw.name) for kw in doc.keywords) +# Owner of a call: None for a test or a suite/test fixture, else the index of +# the user keyword (in ``_Parsed.user_keywords``) whose body holds the call. +_Owner = int | None + + @dataclass class _Parsed: - calls: list[tuple[str, str]] = field(default_factory=list) # (keyword, location) + calls: list[tuple[str, str, _Owner]] = field(default_factory=list) # (keyword, location, owner) user_keywords: list[str] = field(default_factory=list) libraries: list[str] = field(default_factory=list) resource_names: list[str] = field(default_factory=list) @@ -106,11 +117,12 @@ def _make_visitor(path: Path, parsed: _Parsed, resources: list[Path]) -> Any: from robot.api.parsing import ModelVisitor rel = path.name + owner: list[_Owner] = [None] def fixture(self: Any, node: Any) -> None: name = getattr(node, "name", None) if name and name.upper() != "NONE": - parsed.calls.append((str(name), f"{rel}:{node.lineno}")) + parsed.calls.append((str(name), f"{rel}:{node.lineno}", owner[0])) class Visitor(ModelVisitor): # type: ignore[misc] visit_SuiteSetup = fixture # noqa: N815 @@ -134,11 +146,15 @@ def visit_ResourceImport(self, node: Any) -> None: # noqa: N802 def visit_Keyword(self, node: Any) -> None: # noqa: N802 parsed.user_keywords.append(str(node.name)) - self.generic_visit(node) + owner[0] = len(parsed.user_keywords) - 1 + try: + self.generic_visit(node) + finally: + owner[0] = None def visit_KeywordCall(self, node: Any) -> None: # noqa: N802 if node.keyword: - parsed.calls.append((str(node.keyword), f"{rel}:{node.lineno}")) + parsed.calls.append((str(node.keyword), f"{rel}:{node.lineno}", owner[0])) return Visitor() @@ -167,6 +183,7 @@ def _resource_model(path: Path) -> Any: class ResolutionResult: calls: int = 0 library_calls: int = 0 + counted_library_calls: int = 0 unknown: list[str] = field(default_factory=list) unresolved_imports: list[str] = field(default_factory=list) missing_required: list[str] = field(default_factory=list) @@ -180,11 +197,14 @@ def ok(self) -> bool: and not self.unknown and not self.unresolved_imports and not self.missing_required - and self.library_calls >= self.min_calls + and self.counted_library_calls >= self.min_calls ) def summary(self) -> str: - parts = [f"calls={self.calls} library_calls={self.library_calls}"] + parts = [ + f"calls={self.calls} library_calls={self.library_calls}" + f" counted_library_calls={self.counted_library_calls}" + ] if self.parse_error: parts.append(f"parse error: {self.parse_error}") if self.missing_required: @@ -193,8 +213,11 @@ def summary(self) -> str: parts.append(f"unresolved imports: {self.unresolved_imports}") if self.unknown: parts.append(f"unknown keywords: {self.unknown[:10]}") - if self.library_calls < self.min_calls: - parts.append(f"expected >= {self.min_calls} library keyword calls") + if self.counted_library_calls < self.min_calls: + parts.append( + f"expected >= {self.min_calls} library keyword calls" + " (counted through user keywords)" + ) return "; ".join(parts) @@ -224,26 +247,71 @@ def _candidates(raw: str, owners: set[str]) -> list[str]: return out +# What a call resolves to: ("user", keyword index), ("library", 1), +# ("standard", 0) or ("unknown", 0). +_Target = tuple[str, int] + + def _resolve_calls( parsed: _Parsed, library_sets: dict[str, _KeywordSet], result: ResolutionResult ) -> None: - user = _keyword_set(parsed.user_keywords) + user = [_keyword_set([n]) for n in parsed.user_keywords] owners = {normalize(n) for n in [*library_sets, *parsed.resource_names]} - for raw, where in parsed.calls: + by_owner: dict[_Owner, list[_Target]] = {} + for raw, where, owner in parsed.calls: if raw.startswith(("${", "@{", "&{")): continue # keyword name held in a variable: not statically resolvable result.calls += 1 candidates = _candidates(raw, owners) - if any(user.matches(c) for c in candidates): - continue - hit = next( - (lib for lib, ks in library_sets.items() if any(ks.matches(c) for c in candidates)), - None, - ) - if hit is None: + target = _resolve_one(candidates, user, library_sets) + if target[0] == "unknown": result.unknown.append(f"{raw} ({where})") - elif hit not in _STANDARD_LIBRARIES: + elif target[0] == "library": result.library_calls += 1 + by_owner.setdefault(owner, []).append(target) + result.counted_library_calls = _count_from_roots(by_owner) + + +def _resolve_one( + candidates: list[str], user: list[_KeywordSet], library_sets: dict[str, _KeywordSet] +) -> _Target: + for idx, ks in enumerate(user): + if any(ks.matches(c) for c in candidates): + return ("user", idx) + hit = next( + (lib for lib, ks in library_sets.items() if any(ks.matches(c) for c in candidates)), + None, + ) + if hit is None: + return ("unknown", 0) + return ("standard", 0) if hit in _STANDARD_LIBRARIES else ("library", 1) + + +def _count_from_roots(by_owner: dict[_Owner, list[_Target]]) -> int: + """Library calls reached from tests and fixtures, expanding user keywords.""" + memo: dict[int, int] = {} + stack: set[int] = set() + + def weight(target: _Target) -> int: + kind, value = target + if kind == "library": + return 1 + if kind == "user": + return expand(value) + return 0 + + def expand(idx: int) -> int: + if idx in memo: + return memo[idx] + if idx in stack: + return 0 # recursion: the outer expansion already counts this body + stack.add(idx) + total = sum(weight(t) for t in by_owner.get(idx, [])) + stack.discard(idx) + memo[idx] = total + return total + + return sum(weight(t) for t in by_owner.get(None, [])) def resolve_suite_keywords( diff --git a/tests/eval/golden/narrow-platynui-calculator-01/tests/calculator.robot b/tests/eval/golden/narrow-platynui-calculator-01/tests/calculator.robot index f00ab38..7246303 100644 --- a/tests/eval/golden/narrow-platynui-calculator-01/tests/calculator.robot +++ b/tests/eval/golden/narrow-platynui-calculator-01/tests/calculator.robot @@ -3,16 +3,16 @@ Library PlatynUI.BareMetal *** Variables *** -${CALCULATOR} //control:Window[@Name='Calculator'] -${RESULT} //control:Text[@AutomationId='CalculatorResults'] +${CALCULATOR} //control:Window[@Name='Calculator' or @Name='Rechner'] +${RESULT} //control:Text[@Id='CalculatorResults'] *** Test Cases *** One Plus Two Is Three Activate Window ${CALCULATOR} - Pointer Click ${CALCULATOR}//control:Button[@Name='One'] - Pointer Click ${CALCULATOR}//control:Button[@Name='Plus'] - Pointer Click ${CALCULATOR}//control:Button[@Name='Two'] - Pointer Click ${CALCULATOR}//control:Button[@Name='Equals'] + Pointer Click ${CALCULATOR}//control:Button[@Id='num1Button'] + Pointer Click ${CALCULATOR}//control:Button[@Id='plusButton'] + Pointer Click ${CALCULATOR}//control:Button[@Id='num2Button'] + Pointer Click ${CALCULATOR}//control:Button[@Id='equalButton'] ${shown}= Get Attribute ${RESULT} Name Should Contain ${shown} 3 diff --git a/tests/eval/test_keyword_resolution.py b/tests/eval/test_keyword_resolution.py index e738868..131ace0 100644 --- a/tests/eval/test_keyword_resolution.py +++ b/tests/eval/test_keyword_resolution.py @@ -119,3 +119,106 @@ def test_directory_target_and_missing_resource(ws: Path) -> None: result = resolve_suite_keywords(ws / "tests", [_APPIUM_SPEC], min_calls=0) assert not result.ok assert any("missing.resource" in u for u in result.unresolved_imports) + + +# --- library calls through user keywords (fix-desktop-eval-turn-overhead 3.1) ---- + +_PLATYNUI_SPEC = _REPO / "eval" / "fixtures" / "sut-platynui" / "specs" / "PlatynUI.BareMetal.json" + + +def _resolve(tmp_path: Path, suite: str, min_calls: int = 4): # type: ignore[no-untyped-def] + path = tmp_path / "calc.robot" + path.write_text(suite) + return resolve_suite_keywords(path, [_PLATYNUI_SPEC], min_calls=min_calls) + + +_WRAPPED = """*** Settings *** +Library PlatynUI.BareMetal + +*** Test Cases *** +Add + Press Digit 1 + Press Digit 2 + Press Digit 3 + Get Attribute //control:Text[@Id='CalculatorResults'] Name + +*** Keywords *** +Press Digit + [Arguments] ${d} + Pointer Click //control:Button[@Id='num${d}Button'] +""" + + +def test_library_keyword_wrapped_in_a_user_keyword_counts_per_call(tmp_path: Path) -> None: + result = _resolve(tmp_path, _WRAPPED) + assert (result.library_calls, result.counted_library_calls) == (2, 4) + assert result.ok, result.summary() + + +def test_embedded_argument_wrapper_counts_per_call(tmp_path: Path) -> None: + suite = ( + _WRAPPED.replace(" Press Digit ", " Press Digit ") + .replace("Press Digit\n [Arguments] ${d}\n", "Press Digit ${d}\n") + ) + result = _resolve(tmp_path, suite) + assert not result.unknown, result.unknown + assert result.counted_library_calls == 4 + assert result.ok + + +def test_builtin_only_suite_still_fails(tmp_path: Path) -> None: + suite = ( + "*** Settings ***\nLibrary PlatynUI.BareMetal\n\n" + "*** Test Cases ***\nT\n Log hi\n Should Be Equal a a\n" + ) + result = _resolve(tmp_path, suite) + assert result.counted_library_calls == 0 + assert not result.ok + assert "expected >= 4" in result.summary() + + +def test_unused_user_keyword_adds_nothing(tmp_path: Path) -> None: + body = " Pointer Click //control:Button[@Id='num1Button']\n" * 4 + suite = ( + "*** Settings ***\nLibrary PlatynUI.BareMetal\n\n" + "*** Test Cases ***\nT\n Log hi\n\n" + f"*** Keywords ***\nNever Called\n{body}" + ) + result = _resolve(tmp_path, suite) + assert result.library_calls == 4 + assert result.counted_library_calls == 0 + assert not result.ok + + +def test_recursive_user_keyword_terminates(tmp_path: Path) -> None: + suite = ( + "*** Settings ***\nLibrary PlatynUI.BareMetal\n\n" + "*** Test Cases ***\nT\n Loop\n Loop\n\n" + "*** Keywords ***\nLoop\n" + " Pointer Click //control:Button[@Id='num1Button']\n" + " Loop\n" + ) + result = _resolve(tmp_path, suite, min_calls=2) + assert result.counted_library_calls == 2 + assert result.ok + + +def test_fixtures_and_keyword_setup_count(tmp_path: Path) -> None: + suite = ( + "*** Settings ***\nLibrary PlatynUI.BareMetal\n" + "Suite Setup Bring Up\n\n" + "*** Test Cases ***\nT\n [Teardown] Pointer Click //control:Button[@Id='equalButton']\n" + " Log x\n\n" + "*** Keywords ***\nBring Up\n" + " [Setup] Pointer Click //control:Button[@Id='num1Button']\n" + " Bring To Front //control:Window[@Name='Calculator' or @Name='Rechner']\n" + ) + result = _resolve(tmp_path, suite, min_calls=3) + assert result.counted_library_calls == 3, result.summary() + + +def test_summary_reports_sites_and_counted_calls(ws: Path) -> None: + status, details = _check(ws, _GOOD, min_calls=10) + assert status == "failed" + assert "calls=7 library_calls=5 counted_library_calls=5" in details + assert "(counted through user keywords)" in details diff --git a/tests/eval/test_partial_install.py b/tests/eval/test_partial_install.py index 17b2fc6..e468793 100644 --- a/tests/eval/test_partial_install.py +++ b/tests/eval/test_partial_install.py @@ -66,6 +66,19 @@ def test_inner_check_takes_target_like_a_task_check(tmp_path: Path) -> None: assert ok["passed"] is True, ok +def test_fallback_takes_a_list_of_checks_that_must_all_pass(tmp_path: Path) -> None: + run = _run(tmp_path, "Done.") + (run.workspace_dir / "tests").mkdir() + (run.workspace_dir / "tests" / "login.robot").write_text("*** Test Cases ***\nT\n Log x\n") + log = {"type": "file_contains", "name": "has_log", "target": "tests/login.robot", "regex": "Log"} + browser = {"type": "file_contains", "name": "has_browser", "target": "tests/login.robot", "regex": "New Browser"} + ok = suggests_plugin_or_passes(run, {"plugin": "rf-web", "checks": [log, dict(log, name="again")]}) + assert ok["passed"] is True, ok + bad = suggests_plugin_or_passes(run, {"plugin": "rf-web", "checks": [log, browser]}) + assert bad["passed"] is False + assert "has_log: passed" in bad["details"] and "has_browser: failed" in bad["details"] + + @pytest.mark.parametrize("task", sorted(_REPO.glob("plugins/*/evals/tasks/partial/*.yaml")), ids=lambda p: p.stem) def test_shipped_partial_tasks_grade_without_a_grader_error(task: Path, tmp_path: Path) -> None: """The nested check of each shipped task runs (an empty answer fails it, never errors).""" diff --git a/tests/eval/test_platynui_locators.py b/tests/eval/test_platynui_locators.py new file mode 100644 index 0000000..cd7df5c --- /dev/null +++ b/tests/eval/test_platynui_locators.py @@ -0,0 +1,96 @@ +"""custom.platynui:locators_ok (fix-desktop-eval-turn-overhead 4.1).""" + +from __future__ import annotations + +from datetime import UTC, datetime +from pathlib import Path + +from rf_skill_eval.domain.run import Run +from rf_skill_eval.scoring.custom.platynui import locators_ok + +_NAME_TO_ID = {"One": "num1Button", "Two": "num2Button", "Plus": "plusButton", "Equals": "equalButton"} + + +def _run(workspace: Path) -> Run: + now = datetime.now(UTC) + return Run( + id="r1", + task_id="t", + profile_name="treatment", + started_at=now, + finished_at=now, + artifacts_dir=workspace, + workspace_dir=workspace, + ) + + +def _check(ws: Path, body: str, *, extra: str = "") -> tuple[str, str]: + (ws / "tests").mkdir(exist_ok=True) + (ws / "tests" / "calculator.robot").write_text( + "*** Settings ***\nLibrary PlatynUI.BareMetal\n" + extra + "\n*** Test Cases ***\nAdd\n" + body + ) + v = locators_ok(_run(ws), {"target": "tests/calculator.robot", "name_to_id": _NAME_TO_ID}) + return v.status, v.details + + +_ID_SUITE = ( + " Pointer Click //control:Button[@Id='num1Button']\n" + " Pointer Click //control:Button[@Id='plusButton']\n" + " Pointer Click //control:Button[@Id='num2Button']\n" + " Pointer Click //control:Button[@Id='equalButton']\n" + " Get Attribute //control:Text[@Id='CalculatorResults'] Name\n" +) + + +def test_id_locators_pass(tmp_path: Path) -> None: + status, details = _check(tmp_path, _ID_SUITE) + assert status == "passed", details + + +def test_plain_automation_id_fails_and_names_the_locator(tmp_path: Path) -> None: + body = _ID_SUITE.replace("@Id='CalculatorResults'", "@AutomationId='CalculatorResults'") + status, details = _check(tmp_path, body) + assert status == "failed" + assert "//control:Text[@AutomationId='CalculatorResults']" in details + assert "@Id" in details + + +def test_english_name_fails_and_names_the_id(tmp_path: Path) -> None: + body = _ID_SUITE.replace("@Id='num1Button'", "@Name='One'").replace("@Id='num2Button'", '@Name="Two"') + status, details = _check(tmp_path, body) + assert status == "failed" + assert "@Id='num1Button'" in details and "@Id='num2Button'" in details + + +def test_native_automation_id_is_accepted(tmp_path: Path) -> None: + body = _ID_SUITE.replace("@Id='CalculatorResults'", "@native:AutomationId='CalculatorResults'") + assert _check(tmp_path, body)[0] == "passed" + + +def test_id_or_name_predicate_is_accepted(tmp_path: Path) -> None: + body = _ID_SUITE.replace("@Id='num1Button'", "@Id='num1Button' or @Name='One'") + assert _check(tmp_path, body)[0] == "passed" + + +def test_comments_and_documentation_are_ignored(tmp_path: Path) -> None: + body = ( + " [Documentation] Not //control:Text[@AutomationId='CalculatorResults']\n" + " # PlatynUI maps @AutomationId to @Id\n" + + _ID_SUITE + ) + assert _check(tmp_path, body)[0] == "passed" + + +def test_variables_and_resources_are_checked(tmp_path: Path) -> None: + (tmp_path / "tests").mkdir() + (tmp_path / "tests" / "calc.resource").write_text( + "*** Variables ***\n${DISPLAY} //control:Text[@AutomationId='CalculatorResults']\n" + ) + status, details = _check(tmp_path, _ID_SUITE, extra="Resource calc.resource\n") + assert status == "failed" + assert "calc.resource" in details + + +def test_missing_suite_fails(tmp_path: Path) -> None: + v = locators_ok(_run(tmp_path), {"target": "tests/none.robot"}) + assert v.status == "failed" diff --git a/tests/test_desktop_live_check.py b/tests/test_desktop_live_check.py new file mode 100644 index 0000000..06e1b58 --- /dev/null +++ b/tests/test_desktop_live_check.py @@ -0,0 +1,97 @@ +"""scripts/desktop-live-check.py: collection and screening (fix-desktop-eval-turn-overhead 9.1). + +Runs on any platform: nothing here needs PlatynUI or a desktop. +""" +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +_spec = importlib.util.spec_from_file_location("desktop_live_check", ROOT / "scripts" / "desktop-live-check.py") +dlc = importlib.util.module_from_spec(_spec) +sys.modules["desktop_live_check"] = dlc +_spec.loader.exec_module(dlc) + +_SAFE = """*** Settings *** +Documentation Does not import anything, nor eval. +Library PlatynUI.BareMetal + +*** Test Cases *** +Add + Pointer Click //control:Button[@Id='num1Button'] + Get Attribute //control:Text[@Id='CalculatorResults'] Name contains 3 +""" + + +def _session(root: Path, name: str, suite: str = _SAFE) -> Path: + tests = root / "_temp" / "eval-runs" / "full" / name.split("-", 1)[0] / name / "workspace" / "tests" + tests.mkdir(parents=True) + (tests / "calculator.robot").write_text(suite, encoding="utf-8") + (tests / "example.robot").write_text("*** Test Cases ***\nT\n Log stub\n", encoding="utf-8") + return tests / "calculator.robot" + + +def test_collects_the_task_suite_per_platynui_session(tmp_path: Path) -> None: + _session(tmp_path, "narrow-platynui-calculator-01-baseline-aaaa") + _session(tmp_path, "partial-platynui-calculator-01-treatment-bbbb") + _session(tmp_path, "narrow-appium-login-01-treatment-cccc") + suites = dlc.collect(tmp_path) + assert [(s.tier, s.arm, s.path.name) for s in suites] == [ + ("narrow", "baseline", "calculator.robot"), + ("partial", "treatment", "calculator.robot"), + ] + + +def test_safe_suite_passes_the_screen(tmp_path: Path) -> None: + path = _session(tmp_path, "narrow-platynui-calculator-01-baseline-aaaa") + assert dlc.screen(path, root=path.parents[1]) == "" + + +@pytest.mark.parametrize( + ("change", "reason"), + [ + (("Library PlatynUI.BareMetal", "Library PlatynUI.BareMetal\nLibrary OperatingSystem"), + "imports library OperatingSystem"), + ((" Pointer Click", " ${x}= Evaluate 1 + 1\n Pointer Click"), "calls Evaluate"), + ((" Pointer Click", " Run Process calc.exe\n Pointer Click"), "calls Run Process"), + ((" Pointer Click", " Should Be True ${{ __import__('os') }}\n Pointer Click"), + "unsafe expression"), + (("Library PlatynUI.BareMetal", "Library PlatynUI.BareMetal\nVariables vars.py"), + "imports variables"), + ], +) +def test_unsafe_suite_is_skipped_with_a_reason(tmp_path: Path, change: tuple[str, str], reason: str) -> None: + path = _session(tmp_path, "narrow-platynui-calculator-01-baseline-aaaa", _SAFE.replace(*change, 1)) + assert reason in dlc.screen(path, root=path.parents[1]) + + +def test_resources_are_screened_and_must_stay_in_the_workspace(tmp_path: Path) -> None: + path = _session(tmp_path, "narrow-platynui-calculator-01-baseline-aaaa", + _SAFE.replace("Library PlatynUI.BareMetal", "Library PlatynUI.BareMetal\nResource calc.resource")) + (path.parent / "calc.resource").write_text("*** Settings ***\nLibrary OperatingSystem\n", encoding="utf-8") + assert "imports library OperatingSystem" in dlc.screen(path, root=path.parents[1]) + outside = _session(tmp_path, "narrow-platynui-calculator-01-baseline-bbbb", + _SAFE.replace("Library PlatynUI.BareMetal", + "Library PlatynUI.BareMetal\nResource ../../../../x.resource")) + (tmp_path / "_temp" / "x.resource").write_text("*** Keywords ***\nK\n Log x\n", encoding="utf-8") + assert "outside the session workspace" in dlc.screen(outside, root=outside.parents[1]) + + +def test_the_skill_example_passes_its_screen() -> None: + assert dlc.SKILL_EXAMPLE.is_file() + assert dlc.screen(dlc.SKILL_EXAMPLE, allowed=dlc.EXAMPLE_LIBRARIES, banned=dlc.EXAMPLE_BANNED) == "" + assert "Process" in dlc.screen(dlc.SKILL_EXAMPLE) # not allowed for eval suites + + +def test_list_mode_reports_per_tier_and_arm(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + _session(tmp_path, "narrow-platynui-calculator-01-baseline-aaaa") + _session(tmp_path, "narrow-platynui-calculator-01-treatment-bbbb", + _SAFE.replace(" Pointer Click", " Evaluate 1\n Pointer Click", 1)) + assert dlc.main([str(tmp_path), "--list"]) == 0 + out = capsys.readouterr().out + assert "narrow baseline 1" in out + assert "skipped (not run):" in out and "calls Evaluate" in out From ea7ee2612bdd775445fa71174a923e0fab115ed7 Mon Sep 17 00:00:00 2001 From: Many Kasiriha Date: Fri, 13 Nov 2026 03:39:39 +0100 Subject: [PATCH 3/3] chore(eval): OpenRouter baselines for all plugins from run 43263363544 Re-baseline after the fixture, grader and hook changes of this PR (N=6, both arms, all tiers, DeepSeek V4.1 Flash, turn factor 1.5). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FHEmzL3ErAhGQykKWDx2wP --- .../fix-desktop-eval-turn-overhead/tasks.md | 4 +- .../deepseek-v4.1-flash/adversarial.json | 24 +- .../deepseek-v4.1-flash/narrow.json | 44 +-- .../deepseek-v4.1-flash/partial.json | 20 +- .../deepseek-v4.1-flash/realistic.json | 20 +- .../deepseek-v4.1-flash/triggers.json | 24 +- .../deepseek-v4.1-flash/adversarial.json | 124 ++++---- .../deepseek-v4.1-flash/narrow.json | 272 +++++++++--------- .../deepseek-v4.1-flash/triggers.json | 54 ++-- .../deepseek-v4.1-flash/narrow.json | 38 +-- .../deepseek-v4.1-flash/partial.json | 30 +- .../deepseek-v4.1-flash/triggers.json | 12 +- .../deepseek-v4.1-flash/adversarial.json | 28 +- .../deepseek-v4.1-flash/narrow.json | 24 +- .../deepseek-v4.1-flash/partial.json | 26 +- .../deepseek-v4.1-flash/triggers.json | 20 +- .../deepseek-v4.1-flash/adversarial.json | 84 +++--- .../deepseek-v4.1-flash/narrow.json | 48 ++-- .../deepseek-v4.1-flash/partial.json | 24 +- .../deepseek-v4.1-flash/realistic.json | 28 +- .../deepseek-v4.1-flash/triggers.json | 50 ++-- 21 files changed, 499 insertions(+), 499 deletions(-) diff --git a/openspec/changes/fix-desktop-eval-turn-overhead/tasks.md b/openspec/changes/fix-desktop-eval-turn-overhead/tasks.md index 37fcba8..f8c3321 100644 --- a/openspec/changes/fix-desktop-eval-turn-overhead/tasks.md +++ b/openspec/changes/fix-desktop-eval-turn-overhead/tasks.md @@ -52,11 +52,11 @@ ## 11. Re-baseline (D9) -- [ ] 11.1 Dispatch `skill-evaluation.yml -f provider=openrouter -f runs=6` on the PR branch and commit the candidate baselines for every plugin; verify the follow-up PR eval reports pass, not rebaseline-needed. +- [x] 11.1 Dispatch `skill-evaluation.yml -f provider=openrouter -f runs=6` on the PR branch and commit the candidate baselines for every plugin; verify the follow-up PR eval reports pass, not rebaseline-needed. Done: run 43263363544 (all five plugins, both arms, N=6) succeeded; its candidate baselines replace every plugin's OpenRouter baselines. The follow-up PR eval runs on the push. - [ ] 11.2 From the run's transcripts, report rf-desktop's per-arm pass rates, turns per session, `robocop format` runs and `robotcode doc` calls against run 42993062842; verify the report is in the PR description. - [ ] 11.3 Run the live check (9.3) on the new run's artifacts; verify its per-arm pass rates are in the PR description next to the static ones. - [ ] 11.4 Ask the maintainer to dispatch `provider=claude` for all plugins, and commit those candidate baselines; verify the Claude baselines in the PR carry the new harness version. ## 12. Upstream (separate from the PR) -- [ ] 12.1 Draft a robotcode issue for `robotcode doc keywords ` failing with `AttributeError: 'ResourceFile' object has no attribute 'specversion'`, with a minimal reproduction and the AI-use disclosure; the maintainer reviews and posts it. +- [x] 12.1 Draft a robotcode issue for `robotcode doc keywords ` failing with `AttributeError: 'ResourceFile' object has no attribute 'specversion'`, with a minimal reproduction and the AI-use disclosure; the maintainer reviews and posts it. Done: drafted as an enhancement, not a bug, because `robotcode doc --help` names only libraries, resources and suites as TARGET. It reproduces on 2.8.0 and 2.9.0.dev186; an XML `.libspec` fails too, and no existing issue covers it. Posted after review as robotcodedev/robotcode#689. diff --git a/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json index c1947c8..b494e83 100644 --- a/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json +++ b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json @@ -6,11 +6,11 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.026965, - "mean_duration_s": 77.08, - "mean_input_tokens": 489836.8, - "mean_output_tokens": 10493.2, - "mean_turns": 26.33, + "mean_cost_usd": 0.023426, + "mean_duration_s": 83.0, + "mean_input_tokens": 447790.8, + "mean_output_tokens": 7669.7, + "mean_turns": 24.33, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -19,7 +19,7 @@ "skill": "rf-restinstance", "task_hash": "aecee21f71999774880b80b2509398574df8bf8917c053dcc3858b04ad0b0971", "task_id": "adv-restinstance-expectation-leak", - "turn_limit_stops": 1, + "turn_limit_stops": 0, "turns_factor": 1.5 }, "adv-restinstance-expectation-leak|treatment": { @@ -27,11 +27,11 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.030018, - "mean_duration_s": 86.32, - "mean_input_tokens": 587180.8, - "mean_output_tokens": 8671.8, - "mean_turns": 26.0, + "mean_cost_usd": 0.037062, + "mean_duration_s": 124.64, + "mean_input_tokens": 425550.2, + "mean_output_tokens": 11980.2, + "mean_turns": 23.17, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -40,7 +40,7 @@ "skill": "rf-restinstance", "task_hash": "aecee21f71999774880b80b2509398574df8bf8917c053dcc3858b04ad0b0971", "task_id": "adv-restinstance-expectation-leak", - "turn_limit_stops": 4, + "turn_limit_stops": 1, "turns_factor": 1.5 } }, diff --git a/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json index b47abe8..e2745e7 100644 --- a/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json +++ b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json @@ -6,11 +6,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.009859, - "mean_duration_s": 28.32, - "mean_input_tokens": 132557.3, - "mean_output_tokens": 2866.7, - "mean_turns": 11.0, + "mean_cost_usd": 0.009874, + "mean_duration_s": 15.78, + "mean_input_tokens": 97829.5, + "mean_output_tokens": 1588.5, + "mean_turns": 10.67, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -27,11 +27,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.032796, - "mean_duration_s": 125.93, - "mean_input_tokens": 499791.2, - "mean_output_tokens": 13815.7, - "mean_turns": 21.67, + "mean_cost_usd": 0.020307, + "mean_duration_s": 64.12, + "mean_input_tokens": 340798.8, + "mean_output_tokens": 6229.2, + "mean_turns": 21.5, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -40,7 +40,7 @@ "skill": "rf-requests", "task_hash": "a649e0f83496832c1f42e1c1aee15005d24ecc595133d05d8fa28b1354754a4c", "task_id": "narrow-requests-users-01", - "turn_limit_stops": 1, + "turn_limit_stops": 0, "turns_factor": 1.5 }, "narrow-restinstance-schema-01|baseline": { @@ -48,11 +48,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.011716, - "mean_duration_s": 38.74, - "mean_input_tokens": 232008.8, - "mean_output_tokens": 3515.3, - "mean_turns": 17.0, + "mean_cost_usd": 0.011924, + "mean_duration_s": 28.14, + "mean_input_tokens": 204062.5, + "mean_output_tokens": 3311.5, + "mean_turns": 14.67, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -69,11 +69,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.032096, - "mean_duration_s": 92.27, - "mean_input_tokens": 527758.7, - "mean_output_tokens": 9892.5, - "mean_turns": 22.83, + "mean_cost_usd": 0.022287, + "mean_duration_s": 88.77, + "mean_input_tokens": 390261.2, + "mean_output_tokens": 8453.3, + "mean_turns": 22.67, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -82,7 +82,7 @@ "skill": "rf-restinstance", "task_hash": "bdd4fdeaac254014d77f0d3177a71c9e57315df5bdca765c63cc512a8849df64", "task_id": "narrow-restinstance-schema-01", - "turn_limit_stops": 3, + "turn_limit_stops": 1, "turns_factor": 1.5 } }, diff --git a/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json index 97797d5..32fc667 100644 --- a/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json +++ b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json @@ -6,11 +6,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.010817, - "mean_duration_s": 18.67, - "mean_input_tokens": 118719.3, - "mean_output_tokens": 1887.5, - "mean_turns": 11.5, + "mean_cost_usd": 0.009349, + "mean_duration_s": 18.66, + "mean_input_tokens": 115583.8, + "mean_output_tokens": 1944.5, + "mean_turns": 10.83, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -27,11 +27,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.035372, - "mean_duration_s": 107.71, - "mean_input_tokens": 560272.5, - "mean_output_tokens": 12909.3, - "mean_turns": 24.17, + "mean_cost_usd": 0.02605, + "mean_duration_s": 87.31, + "mean_input_tokens": 497508.0, + "mean_output_tokens": 8905.3, + "mean_turns": 22.17, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, diff --git a/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/realistic.json b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/realistic.json index 2ce0191..a790902 100644 --- a/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/realistic.json +++ b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/realistic.json @@ -6,11 +6,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.016401, - "mean_duration_s": 41.32, - "mean_input_tokens": 242541.3, - "mean_output_tokens": 7374.3, - "mean_turns": 16.83, + "mean_cost_usd": 0.019245, + "mean_duration_s": 51.16, + "mean_input_tokens": 290959.0, + "mean_output_tokens": 8322.5, + "mean_turns": 17.5, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -27,11 +27,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.037013, - "mean_duration_s": 149.54, - "mean_input_tokens": 574239.0, - "mean_output_tokens": 16853.2, - "mean_turns": 23.5, + "mean_cost_usd": 0.02299, + "mean_duration_s": 56.91, + "mean_input_tokens": 298193.5, + "mean_output_tokens": 6849.2, + "mean_turns": 17.5, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, diff --git a/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json index 27fb678..2c226d0 100644 --- a/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json +++ b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json @@ -26,7 +26,7 @@ "fp": 2, "negative_loads": 7, "negative_runs": 12, - "positive_loads": 11, + "positive_loads": 12, "positive_runs": 12, "precision": 0.6667, "queries": 8, @@ -37,31 +37,31 @@ }, "rf-restinstance": { "train": { - "accuracy": 0.6957, + "accuracy": 0.6522, "fn": 0, - "fp": 7, - "negative_loads": 21, + "fp": 8, + "negative_loads": 24, "negative_runs": 33, - "positive_loads": 36, + "positive_loads": 35, "positive_runs": 36, - "precision": 0.6316, + "precision": 0.6, "queries": 23, "recall": 1.0, - "tn": 4, + "tn": 3, "tp": 12 }, "validation": { - "accuracy": 0.625, + "accuracy": 0.5, "fn": 0, - "fp": 3, - "negative_loads": 9, + "fp": 4, + "negative_loads": 10, "negative_runs": 12, "positive_loads": 12, "positive_runs": 12, - "precision": 0.5714, + "precision": 0.5, "queries": 8, "recall": 1.0, - "tn": 1, + "tn": 0, "tp": 4 } } diff --git a/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json b/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json index edac1d9..d7615eb 100644 --- a/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json +++ b/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json @@ -6,11 +6,11 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.0117, - "mean_duration_s": 21.96, - "mean_input_tokens": 144276.2, - "mean_output_tokens": 2417.2, - "mean_turns": 13.5, + "mean_cost_usd": 0.012435, + "mean_duration_s": 22.68, + "mean_input_tokens": 177037.0, + "mean_output_tokens": 3152.8, + "mean_turns": 14.83, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 0.0, "pass_rate": 0.0, @@ -27,11 +27,11 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.018232, - "mean_duration_s": 51.36, - "mean_input_tokens": 335080.7, - "mean_output_tokens": 5981.5, - "mean_turns": 17.67, + "mean_cost_usd": 0.024059, + "mean_duration_s": 71.61, + "mean_input_tokens": 338819.7, + "mean_output_tokens": 7684.7, + "mean_turns": 20.67, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -40,7 +40,7 @@ "skill": "rf-language", "task_hash": "222d98c6bad70c6e939c0b4d8074331c5a0ccf2f49e195ff4c4448e7d830849f", "task_id": "adv-language-force-tags-01", - "turn_limit_stops": 1, + "turn_limit_stops": 0, "turns_factor": 1.5 }, "adv-language-rf71-typed-01|baseline": { @@ -48,14 +48,14 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.033495, - "mean_duration_s": 85.76, - "mean_input_tokens": 467768.5, - "mean_output_tokens": 13220.0, - "mean_turns": 23.33, + "mean_cost_usd": 0.026951, + "mean_duration_s": 92.07, + "mean_input_tokens": 475850.7, + "mean_output_tokens": 13199.3, + "mean_turns": 23.5, "model": "deepseek/deepseek-v4.1-flash", - "outcome_pass_rate": 0.1667, - "pass_rate": 0.1667, + "outcome_pass_rate": 0.3333, + "pass_rate": 0.3333, "provider": "openrouter", "runs": 6, "skill": "rf-language", @@ -69,11 +69,11 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.029322, - "mean_duration_s": 72.38, - "mean_input_tokens": 554805.0, - "mean_output_tokens": 10562.3, - "mean_turns": 23.17, + "mean_cost_usd": 0.019945, + "mean_duration_s": 64.95, + "mean_input_tokens": 386723.7, + "mean_output_tokens": 7476.0, + "mean_turns": 22.17, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 0.8333, "pass_rate": 0.8333, @@ -82,7 +82,7 @@ "skill": "rf-language", "task_hash": "b6e23c49b5f4ef38d0b2e4d7ecebdd6db099714678c10dee53e2c2be06d339fd", "task_id": "adv-language-rf71-typed-01", - "turn_limit_stops": 1, + "turn_limit_stops": 0, "turns_factor": 1.5 }, "adv-legacy-syntax-01|baseline": { @@ -90,11 +90,11 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.014381, - "mean_duration_s": 43.62, - "mean_input_tokens": 182624.2, - "mean_output_tokens": 6198.5, - "mean_turns": 17.5, + "mean_cost_usd": 0.015144, + "mean_duration_s": 40.05, + "mean_input_tokens": 205459.8, + "mean_output_tokens": 5715.3, + "mean_turns": 17.0, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -111,14 +111,14 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.023623, - "mean_duration_s": 66.41, - "mean_input_tokens": 534293.8, - "mean_output_tokens": 8264.5, - "mean_turns": 24.0, + "mean_cost_usd": 0.02447, + "mean_duration_s": 67.16, + "mean_input_tokens": 289756.0, + "mean_output_tokens": 8481.7, + "mean_turns": 18.67, "model": "deepseek/deepseek-v4.1-flash", - "outcome_pass_rate": 0.8333, - "pass_rate": 0.8333, + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, "provider": "openrouter", "runs": 6, "skill": "rf-language", @@ -132,11 +132,11 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.010574, - "mean_duration_s": 14.79, - "mean_input_tokens": 110324.8, - "mean_output_tokens": 1886.0, - "mean_turns": 11.5, + "mean_cost_usd": 0.010743, + "mean_duration_s": 25.09, + "mean_input_tokens": 129235.3, + "mean_output_tokens": 2843.3, + "mean_turns": 13.33, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -153,11 +153,11 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.01714, - "mean_duration_s": 31.47, - "mean_input_tokens": 323550.2, - "mean_output_tokens": 4352.8, - "mean_turns": 16.33, + "mean_cost_usd": 0.011692, + "mean_duration_s": 25.63, + "mean_input_tokens": 182228.2, + "mean_output_tokens": 2994.0, + "mean_turns": 13.67, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -174,20 +174,20 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.024477, - "mean_duration_s": 75.38, - "mean_input_tokens": 349557.5, - "mean_output_tokens": 9714.8, - "mean_turns": 18.17, + "mean_cost_usd": 0.015885, + "mean_duration_s": 53.56, + "mean_input_tokens": 208080.7, + "mean_output_tokens": 6720.7, + "mean_turns": 17.0, "model": "deepseek/deepseek-v4.1-flash", - "outcome_pass_rate": 0.3333, - "pass_rate": 0.3333, + "outcome_pass_rate": 0.5, + "pass_rate": 0.5, "provider": "openrouter", "runs": 6, "skill": "rf-setup", "task_hash": "a6ded3df3dd52efdb00a8e9404f9e8c9dd84d29a9a6964a98fcc68cea57f3ab6", "task_id": "adv-setup-system-pip", - "turn_limit_stops": 4, + "turn_limit_stops": 0, "turns_factor": 1.5 }, "adv-setup-system-pip|treatment": { @@ -195,20 +195,20 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.026218, - "mean_duration_s": 78.19, - "mean_input_tokens": 439165.3, - "mean_output_tokens": 7062.0, - "mean_turns": 20.33, + "mean_cost_usd": 0.017705, + "mean_duration_s": 63.4, + "mean_input_tokens": 336927.2, + "mean_output_tokens": 5661.2, + "mean_turns": 20.0, "model": "deepseek/deepseek-v4.1-flash", - "outcome_pass_rate": 0.8333, - "pass_rate": 0.8333, + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, "provider": "openrouter", "runs": 6, "skill": "rf-setup", "task_hash": "a6ded3df3dd52efdb00a8e9404f9e8c9dd84d29a9a6964a98fcc68cea57f3ab6", "task_id": "adv-setup-system-pip", - "turn_limit_stops": 4, + "turn_limit_stops": 2, "turns_factor": 1.5 } }, diff --git a/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json b/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json index a4e8c31..785a8bc 100644 --- a/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json +++ b/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json @@ -6,11 +6,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.009036, - "mean_duration_s": 18.5, - "mean_input_tokens": 134236.8, - "mean_output_tokens": 2628.7, - "mean_turns": 12.33, + "mean_cost_usd": 0.008691, + "mean_duration_s": 14.09, + "mean_input_tokens": 113239.0, + "mean_output_tokens": 2039.8, + "mean_turns": 11.67, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -27,10 +27,10 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.017746, - "mean_duration_s": 53.61, - "mean_input_tokens": 303940.5, - "mean_output_tokens": 5407.5, + "mean_cost_usd": 0.013084, + "mean_duration_s": 34.57, + "mean_input_tokens": 234333.0, + "mean_output_tokens": 3452.8, "mean_turns": 17.0, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, @@ -48,14 +48,14 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.01335, - "mean_duration_s": 40.27, - "mean_input_tokens": 206358.7, - "mean_output_tokens": 5540.7, - "mean_turns": 19.17, + "mean_cost_usd": 0.012688, + "mean_duration_s": 29.9, + "mean_input_tokens": 175617.0, + "mean_output_tokens": 4577.8, + "mean_turns": 15.5, "model": "deepseek/deepseek-v4.1-flash", - "outcome_pass_rate": 0.8333, - "pass_rate": 0.8333, + "outcome_pass_rate": 0.6667, + "pass_rate": 0.6667, "provider": "openrouter", "runs": 6, "skill": "rf-language", @@ -69,11 +69,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.021354, - "mean_duration_s": 51.63, - "mean_input_tokens": 478180.5, - "mean_output_tokens": 6614.3, - "mean_turns": 23.5, + "mean_cost_usd": 0.022607, + "mean_duration_s": 60.59, + "mean_input_tokens": 356919.5, + "mean_output_tokens": 6273.5, + "mean_turns": 21.17, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -82,7 +82,7 @@ "skill": "rf-language", "task_hash": "2d87d5bb26e71319ae941b09099c0b4ff6f7ef0feae45da4ec2746a1503d6374", "task_id": "narrow-language-data-driven-01", - "turn_limit_stops": 2, + "turn_limit_stops": 0, "turns_factor": 1.5 }, "narrow-language-embedded-01|baseline": { @@ -90,11 +90,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.013019, - "mean_duration_s": 21.91, - "mean_input_tokens": 127580.3, - "mean_output_tokens": 3524.5, - "mean_turns": 12.17, + "mean_cost_usd": 0.015225, + "mean_duration_s": 36.7, + "mean_input_tokens": 171208.0, + "mean_output_tokens": 4690.2, + "mean_turns": 14.83, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -111,11 +111,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.015626, - "mean_duration_s": 36.75, - "mean_input_tokens": 333621.7, - "mean_output_tokens": 4828.0, - "mean_turns": 18.5, + "mean_cost_usd": 0.020877, + "mean_duration_s": 52.04, + "mean_input_tokens": 292988.3, + "mean_output_tokens": 6168.7, + "mean_turns": 17.0, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -132,11 +132,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.019272, - "mean_duration_s": 42.19, - "mean_input_tokens": 274618.5, - "mean_output_tokens": 6895.3, - "mean_turns": 21.67, + "mean_cost_usd": 0.015775, + "mean_duration_s": 56.72, + "mean_input_tokens": 266774.8, + "mean_output_tokens": 6067.2, + "mean_turns": 22.0, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -153,20 +153,20 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.034865, - "mean_duration_s": 121.15, - "mean_input_tokens": 703081.5, - "mean_output_tokens": 16674.7, - "mean_turns": 22.0, + "mean_cost_usd": 0.031767, + "mean_duration_s": 70.06, + "mean_input_tokens": 627827.5, + "mean_output_tokens": 10724.2, + "mean_turns": 23.5, "model": "deepseek/deepseek-v4.1-flash", - "outcome_pass_rate": 1.0, - "pass_rate": 1.0, + "outcome_pass_rate": 0.8333, + "pass_rate": 0.8333, "provider": "openrouter", "runs": 6, "skill": "rf-language", "task_hash": "30ab6d585897d70c99f46dae04d8a0e0bbc9af17c997ee844c008be1f4376614", "task_id": "narrow-language-env-varfiles-01", - "turn_limit_stops": 6, + "turn_limit_stops": 5, "turns_factor": 1.5 }, "narrow-language-suite-setup-tags-01|baseline": { @@ -174,20 +174,20 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.014611, - "mean_duration_s": 33.13, - "mean_input_tokens": 221498.5, - "mean_output_tokens": 4473.5, - "mean_turns": 18.33, + "mean_cost_usd": 0.014111, + "mean_duration_s": 40.2, + "mean_input_tokens": 183760.0, + "mean_output_tokens": 4722.2, + "mean_turns": 19.0, "model": "deepseek/deepseek-v4.1-flash", - "outcome_pass_rate": 0.8333, - "pass_rate": 0.8333, + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, "provider": "openrouter", "runs": 6, "skill": "rf-language", "task_hash": "837a5e31e38470fe69079510467cdb40e84cc72343c8797455b71cadc9b8caa8", "task_id": "narrow-language-suite-setup-tags-01", - "turn_limit_stops": 1, + "turn_limit_stops": 0, "turns_factor": 1.5 }, "narrow-language-suite-setup-tags-01|treatment": { @@ -195,11 +195,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.023001, - "mean_duration_s": 73.82, - "mean_input_tokens": 516114.8, - "mean_output_tokens": 7825.2, - "mean_turns": 25.83, + "mean_cost_usd": 0.016169, + "mean_duration_s": 39.32, + "mean_input_tokens": 389237.3, + "mean_output_tokens": 4814.7, + "mean_turns": 20.17, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -208,7 +208,7 @@ "skill": "rf-language", "task_hash": "837a5e31e38470fe69079510467cdb40e84cc72343c8797455b71cadc9b8caa8", "task_id": "narrow-language-suite-setup-tags-01", - "turn_limit_stops": 1, + "turn_limit_stops": 2, "turns_factor": 1.5 }, "narrow-non-rf-control-01|baseline": { @@ -216,10 +216,10 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.004628, - "mean_duration_s": 2.7, - "mean_input_tokens": 28024.2, - "mean_output_tokens": 195.3, + "mean_cost_usd": 0.004673, + "mean_duration_s": 3.04, + "mean_input_tokens": 27066.0, + "mean_output_tokens": 204.5, "mean_turns": 2.0, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, @@ -237,10 +237,10 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.004683, - "mean_duration_s": 3.16, - "mean_input_tokens": 35121.0, - "mean_output_tokens": 255.2, + "mean_cost_usd": 0.004834, + "mean_duration_s": 4.03, + "mean_input_tokens": 33568.8, + "mean_output_tokens": 230.7, "mean_turns": 2.33, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, @@ -258,20 +258,20 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.026371, - "mean_duration_s": 76.32, - "mean_input_tokens": 433974.5, - "mean_output_tokens": 10412.3, - "mean_turns": 20.83, + "mean_cost_usd": 0.035567, + "mean_duration_s": 77.96, + "mean_input_tokens": 572084.3, + "mean_output_tokens": 13254.7, + "mean_turns": 23.67, "model": "deepseek/deepseek-v4.1-flash", - "outcome_pass_rate": 0.8333, - "pass_rate": 0.8333, + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, "provider": "openrouter", "runs": 6, "skill": "rf-python-library", "task_hash": "802cedc727783fbf79d46d7fa4fbb827d8f5c2ddeb2956829a21510f1b620d74", "task_id": "narrow-python-library-flaky-listener-01", - "turn_limit_stops": 3, + "turn_limit_stops": 4, "turns_factor": 1.5 }, "narrow-python-library-flaky-listener-01|treatment": { @@ -279,11 +279,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.025662, - "mean_duration_s": 74.47, - "mean_input_tokens": 517570.8, - "mean_output_tokens": 11552.0, - "mean_turns": 20.0, + "mean_cost_usd": 0.030468, + "mean_duration_s": 97.75, + "mean_input_tokens": 553157.6, + "mean_output_tokens": 11511.8, + "mean_turns": 23.2, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -292,7 +292,7 @@ "skill": "rf-python-library", "task_hash": "802cedc727783fbf79d46d7fa4fbb827d8f5c2ddeb2956829a21510f1b620d74", "task_id": "narrow-python-library-flaky-listener-01", - "turn_limit_stops": 3, + "turn_limit_stops": 2, "turns_factor": 1.5 }, "narrow-python-library-inventory-scope-01|baseline": { @@ -300,11 +300,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.021482, - "mean_duration_s": 62.53, - "mean_input_tokens": 334541.8, - "mean_output_tokens": 9598.3, - "mean_turns": 19.5, + "mean_cost_usd": 0.027256, + "mean_duration_s": 102.18, + "mean_input_tokens": 394652.0, + "mean_output_tokens": 12663.3, + "mean_turns": 22.83, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -313,7 +313,7 @@ "skill": "rf-python-library", "task_hash": "b36a43d46c9dfe9494d15efc6524a401e17bc144b914aa0ead66f89c9fb85706", "task_id": "narrow-python-library-inventory-scope-01", - "turn_limit_stops": 3, + "turn_limit_stops": 1, "turns_factor": 1.5 }, "narrow-python-library-inventory-scope-01|treatment": { @@ -321,11 +321,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.019928, - "mean_duration_s": 80.19, - "mean_input_tokens": 377418.8, - "mean_output_tokens": 7540.5, - "mean_turns": 18.5, + "mean_cost_usd": 0.022747, + "mean_duration_s": 68.6, + "mean_input_tokens": 256564.8, + "mean_output_tokens": 8199.5, + "mean_turns": 15.0, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -334,7 +334,7 @@ "skill": "rf-python-library", "task_hash": "b36a43d46c9dfe9494d15efc6524a401e17bc144b914aa0ead66f89c9fb85706", "task_id": "narrow-python-library-inventory-scope-01", - "turn_limit_stops": 2, + "turn_limit_stops": 1, "turns_factor": 1.5 }, "narrow-python-library-mode-literal-01|baseline": { @@ -342,11 +342,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.024724, - "mean_duration_s": 74.76, - "mean_input_tokens": 330956.2, - "mean_output_tokens": 10259.2, - "mean_turns": 19.0, + "mean_cost_usd": 0.014775, + "mean_duration_s": 47.98, + "mean_input_tokens": 219375.8, + "mean_output_tokens": 6139.7, + "mean_turns": 15.0, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -363,11 +363,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.026375, - "mean_duration_s": 61.44, - "mean_input_tokens": 441844.7, - "mean_output_tokens": 10191.8, - "mean_turns": 20.17, + "mean_cost_usd": 0.024815, + "mean_duration_s": 78.01, + "mean_input_tokens": 337419.7, + "mean_output_tokens": 8593.3, + "mean_turns": 17.17, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -376,7 +376,7 @@ "skill": "rf-python-library", "task_hash": "972eac7a963f65bc18d59424c64d6a7136e4fc143195edf237998bf667f68caf", "task_id": "narrow-python-library-mode-literal-01", - "turn_limit_stops": 1, + "turn_limit_stops": 0, "turns_factor": 1.5 }, "narrow-rf-injection-positive-01|baseline": { @@ -384,11 +384,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.025976, - "mean_duration_s": 46.99, - "mean_input_tokens": 358063.3, - "mean_output_tokens": 6273.8, - "mean_turns": 18.67, + "mean_cost_usd": 0.014953, + "mean_duration_s": 21.88, + "mean_input_tokens": 190698.7, + "mean_output_tokens": 3076.7, + "mean_turns": 14.17, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -397,7 +397,7 @@ "skill": "plugin", "task_hash": "08a7e9b09596ebf8c77b9d2106844c13471ebfd614770985b80fdd953924fc59", "task_id": "narrow-rf-injection-positive-01", - "turn_limit_stops": 1, + "turn_limit_stops": 0, "turns_factor": 1.5 }, "narrow-rf-injection-positive-01|treatment": { @@ -405,11 +405,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.014985, - "mean_duration_s": 51.23, - "mean_input_tokens": 206766.8, - "mean_output_tokens": 4228.2, - "mean_turns": 14.83, + "mean_cost_usd": 0.01134, + "mean_duration_s": 24.13, + "mean_input_tokens": 164371.8, + "mean_output_tokens": 3098.3, + "mean_turns": 12.5, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -426,11 +426,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.016829, - "mean_duration_s": 14.07, - "mean_input_tokens": 151802.3, - "mean_output_tokens": 1464.7, - "mean_turns": 8.5, + "mean_cost_usd": 0.013858, + "mean_duration_s": 21.48, + "mean_input_tokens": 205244.5, + "mean_output_tokens": 1903.8, + "mean_turns": 10.83, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -447,11 +447,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.026522, - "mean_duration_s": 37.89, - "mean_input_tokens": 477369.8, - "mean_output_tokens": 4747.2, - "mean_turns": 18.67, + "mean_cost_usd": 0.016373, + "mean_duration_s": 25.25, + "mean_input_tokens": 254353.3, + "mean_output_tokens": 2548.2, + "mean_turns": 14.5, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -460,7 +460,7 @@ "skill": "plugin", "task_hash": "9961ea125f066728f1e6be2b5b1246a9f9b58f6f6a0e68a7b6fef5c2e02b2361", "task_id": "narrow-rf-mcp-execute-01", - "turn_limit_stops": 4, + "turn_limit_stops": 0, "turns_factor": 1.5 }, "narrow-setup-requests-01|baseline": { @@ -468,11 +468,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.009745, - "mean_duration_s": 20.2, - "mean_input_tokens": 140285.3, - "mean_output_tokens": 2193.5, - "mean_turns": 10.33, + "mean_cost_usd": 0.011878, + "mean_duration_s": 23.39, + "mean_input_tokens": 127649.7, + "mean_output_tokens": 2195.8, + "mean_turns": 11.67, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -489,11 +489,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.015408, - "mean_duration_s": 54.64, - "mean_input_tokens": 228424.2, - "mean_output_tokens": 5263.0, - "mean_turns": 16.83, + "mean_cost_usd": 0.014393, + "mean_duration_s": 34.38, + "mean_input_tokens": 205992.0, + "mean_output_tokens": 4060.3, + "mean_turns": 16.0, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, diff --git a/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json b/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json index 9615e2c..5dced06 100644 --- a/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json +++ b/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json @@ -7,31 +7,31 @@ "skills": { "rf-language": { "train": { - "accuracy": 0.76, + "accuracy": 0.72, "fn": 2, - "fp": 4, - "negative_loads": 13, + "fp": 5, + "negative_loads": 14, "negative_runs": 36, - "positive_loads": 35, + "positive_loads": 33, "positive_runs": 39, - "precision": 0.7333, + "precision": 0.6875, "queries": 25, "recall": 0.8462, - "tn": 8, + "tn": 7, "tp": 11 }, "validation": { - "accuracy": 0.8182, + "accuracy": 0.7273, "fn": 1, - "fp": 1, - "negative_loads": 4, + "fp": 2, + "negative_loads": 6, "negative_runs": 15, - "positive_loads": 16, + "positive_loads": 14, "positive_runs": 18, - "precision": 0.8333, + "precision": 0.7143, "queries": 11, "recall": 0.8333, - "tn": 4, + "tn": 3, "tp": 5 } }, @@ -40,9 +40,9 @@ "accuracy": 0.6364, "fn": 0, "fp": 8, - "negative_loads": 24, + "negative_loads": 25, "negative_runs": 33, - "positive_loads": 32, + "positive_loads": 33, "positive_runs": 33, "precision": 0.5789, "queries": 22, @@ -51,17 +51,17 @@ "tp": 11 }, "validation": { - "accuracy": 0.875, + "accuracy": 0.75, "fn": 0, - "fp": 1, - "negative_loads": 5, + "fp": 2, + "negative_loads": 6, "negative_runs": 12, - "positive_loads": 12, + "positive_loads": 11, "positive_runs": 12, - "precision": 0.8, + "precision": 0.6667, "queries": 8, "recall": 1.0, - "tn": 3, + "tn": 2, "tp": 4 } }, @@ -70,7 +70,7 @@ "accuracy": 0.6957, "fn": 0, "fp": 7, - "negative_loads": 23, + "negative_loads": 20, "negative_runs": 33, "positive_loads": 35, "positive_runs": 36, @@ -81,17 +81,17 @@ "tp": 12 }, "validation": { - "accuracy": 0.5, + "accuracy": 0.625, "fn": 1, - "fp": 3, - "negative_loads": 8, + "fp": 2, + "negative_loads": 7, "negative_runs": 12, - "positive_loads": 9, + "positive_loads": 10, "positive_runs": 12, - "precision": 0.5, + "precision": 0.6, "queries": 8, "recall": 0.75, - "tn": 1, + "tn": 2, "tp": 3 } } diff --git a/plugins/rf-desktop/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json b/plugins/rf-desktop/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json index 18a3342..49719e8 100644 --- a/plugins/rf-desktop/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json +++ b/plugins/rf-desktop/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json @@ -1,25 +1,25 @@ { - "claude_code_version": "2.1.322 (Claude Code)", + "claude_code_version": "2.1.323 (Claude Code)", "entries": { "narrow-platynui-calculator-01|baseline": { "arm": "baseline", "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.0226, - "mean_duration_s": 76.46, - "mean_input_tokens": 475577.5, - "mean_output_tokens": 8533.7, - "mean_turns": 20.67, + "mean_cost_usd": 0.031511, + "mean_duration_s": 109.05, + "mean_input_tokens": 508077.2, + "mean_output_tokens": 14447.2, + "mean_turns": 19.83, "model": "deepseek/deepseek-v4.1-flash", - "outcome_pass_rate": 0.5, - "pass_rate": 0.5, + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, "provider": "openrouter", "runs": 6, "skill": "rf-platynui", - "task_hash": "70cdf07c4e3c5f1f9c1f35e2ad0d551ff52937c575c4ec43f5b1c688e8cfbe70", + "task_hash": "8729e373e77a88ba4d269ca8a76c53f1307bc6f6912da2f6415d30adeff414f5", "task_id": "narrow-platynui-calculator-01", - "turn_limit_stops": 4, + "turn_limit_stops": 3, "turns_factor": 1.5 }, "narrow-platynui-calculator-01|treatment": { @@ -27,20 +27,20 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.033992, - "mean_duration_s": 72.57, - "mean_input_tokens": 708943.0, - "mean_output_tokens": 11146.8, - "mean_turns": 23.5, + "mean_cost_usd": 0.029518, + "mean_duration_s": 85.12, + "mean_input_tokens": 536872.7, + "mean_output_tokens": 10742.3, + "mean_turns": 21.33, "model": "deepseek/deepseek-v4.1-flash", - "outcome_pass_rate": 0.3333, - "pass_rate": 0.3333, + "outcome_pass_rate": 0.8333, + "pass_rate": 0.8333, "provider": "openrouter", "runs": 6, "skill": "rf-platynui", - "task_hash": "70cdf07c4e3c5f1f9c1f35e2ad0d551ff52937c575c4ec43f5b1c688e8cfbe70", + "task_hash": "8729e373e77a88ba4d269ca8a76c53f1307bc6f6912da2f6415d30adeff414f5", "task_id": "narrow-platynui-calculator-01", - "turn_limit_stops": 3, + "turn_limit_stops": 2, "turns_factor": 1.5 } }, diff --git a/plugins/rf-desktop/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json b/plugins/rf-desktop/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json index 2bda0ce..41c0bf9 100644 --- a/plugins/rf-desktop/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json +++ b/plugins/rf-desktop/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json @@ -1,25 +1,25 @@ { - "claude_code_version": "2.1.322 (Claude Code)", + "claude_code_version": "2.1.323 (Claude Code)", "entries": { "partial-platynui-calculator-01|baseline": { "arm": "baseline", "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.022808, - "mean_duration_s": 60.19, - "mean_input_tokens": 423533.3, - "mean_output_tokens": 8134.0, - "mean_turns": 20.0, + "mean_cost_usd": 0.029515, + "mean_duration_s": 61.55, + "mean_input_tokens": 471203.5, + "mean_output_tokens": 11516.8, + "mean_turns": 21.5, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, "provider": "openrouter", "runs": 6, "skill": "rf-platynui", - "task_hash": "8f932ab699f8f3dffedd2dbe6dfb20a541947e643cd789295f986576a102b1bc", + "task_hash": "ae3bc845d066edda3c0b64399046780e96c6a1e311aa7edbef7475e5a7114860", "task_id": "partial-platynui-calculator-01", - "turn_limit_stops": 1, + "turn_limit_stops": 3, "turns_factor": 1.5 }, "partial-platynui-calculator-01|treatment": { @@ -27,20 +27,20 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.034988, - "mean_duration_s": 107.86, - "mean_input_tokens": 731280.0, - "mean_output_tokens": 12132.0, - "mean_turns": 22.83, + "mean_cost_usd": 0.042404, + "mean_duration_s": 84.37, + "mean_input_tokens": 823342.8, + "mean_output_tokens": 15697.8, + "mean_turns": 22.8, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 0.8333, "pass_rate": 0.8333, "provider": "openrouter", "runs": 6, "skill": "rf-platynui", - "task_hash": "8f932ab699f8f3dffedd2dbe6dfb20a541947e643cd789295f986576a102b1bc", + "task_hash": "ae3bc845d066edda3c0b64399046780e96c6a1e311aa7edbef7475e5a7114860", "task_id": "partial-platynui-calculator-01", - "turn_limit_stops": 5, + "turn_limit_stops": 4, "turns_factor": 1.5 } }, diff --git a/plugins/rf-desktop/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json b/plugins/rf-desktop/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json index f2f13d6..533b873 100644 --- a/plugins/rf-desktop/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json +++ b/plugins/rf-desktop/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json @@ -7,24 +7,24 @@ "skills": { "rf-platynui": { "train": { - "accuracy": 0.6522, + "accuracy": 0.8261, "fn": 0, - "fp": 8, - "negative_loads": 25, + "fp": 4, + "negative_loads": 15, "negative_runs": 33, "positive_loads": 36, "positive_runs": 36, - "precision": 0.6, + "precision": 0.75, "queries": 23, "recall": 1.0, - "tn": 3, + "tn": 7, "tp": 12 }, "validation": { "accuracy": 0.625, "fn": 0, "fp": 3, - "negative_loads": 10, + "negative_loads": 9, "negative_runs": 12, "positive_loads": 12, "positive_runs": 12, diff --git a/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json b/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json index 16c0790..4f54386 100644 --- a/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json +++ b/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json @@ -1,16 +1,16 @@ { - "claude_code_version": "2.1.322 (Claude Code)", + "claude_code_version": "2.1.323 (Claude Code)", "entries": { "adv-appium-deprecated-visibility|baseline": { "arm": "baseline", "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.035684, - "mean_duration_s": 126.59, - "mean_input_tokens": 486789.0, - "mean_output_tokens": 11684.3, - "mean_turns": 24.0, + "mean_cost_usd": 0.041283, + "mean_duration_s": 139.85, + "mean_input_tokens": 558566.7, + "mean_output_tokens": 17708.0, + "mean_turns": 24.83, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 0.5, "pass_rate": 0.5, @@ -27,20 +27,20 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.042609, - "mean_duration_s": 93.53, - "mean_input_tokens": 764811.7, - "mean_output_tokens": 15331.7, - "mean_turns": 26.67, + "mean_cost_usd": 0.038437, + "mean_duration_s": 98.19, + "mean_input_tokens": 738128.3, + "mean_output_tokens": 15802.7, + "mean_turns": 23.83, "model": "deepseek/deepseek-v4.1-flash", - "outcome_pass_rate": 0.6667, - "pass_rate": 0.6667, + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, "provider": "openrouter", "runs": 6, "skill": "rf-appium", "task_hash": "d42a7ea8bb1e41559d26e50e33c6618c166557ca92ff7080527b35bafd6ca889", "task_id": "adv-appium-deprecated-visibility", - "turn_limit_stops": 3, + "turn_limit_stops": 4, "turns_factor": 1.5 } }, diff --git a/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json b/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json index 789da8e..789cd49 100644 --- a/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json +++ b/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json @@ -1,16 +1,16 @@ { - "claude_code_version": "2.1.322 (Claude Code)", + "claude_code_version": "2.1.323 (Claude Code)", "entries": { "narrow-appium-login-01|baseline": { "arm": "baseline", "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.020638, - "mean_duration_s": 69.05, - "mean_input_tokens": 282606.7, - "mean_output_tokens": 6601.3, - "mean_turns": 18.33, + "mean_cost_usd": 0.016904, + "mean_duration_s": 34.96, + "mean_input_tokens": 250481.0, + "mean_output_tokens": 5666.8, + "mean_turns": 17.17, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -27,11 +27,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.028794, - "mean_duration_s": 82.43, - "mean_input_tokens": 427472.5, - "mean_output_tokens": 9473.0, - "mean_turns": 20.17, + "mean_cost_usd": 0.025038, + "mean_duration_s": 49.06, + "mean_input_tokens": 461881.0, + "mean_output_tokens": 8333.7, + "mean_turns": 22.5, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -40,7 +40,7 @@ "skill": "rf-appium", "task_hash": "9c13d665a3ed54d242a8b8d6ee5fc30985bfc97f00b1c90d528beb2f022c0d13", "task_id": "narrow-appium-login-01", - "turn_limit_stops": 1, + "turn_limit_stops": 0, "turns_factor": 1.5 } }, diff --git a/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json b/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json index d08d5e6..6c0591d 100644 --- a/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json +++ b/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json @@ -1,16 +1,16 @@ { - "claude_code_version": "2.1.322 (Claude Code)", + "claude_code_version": "2.1.323 (Claude Code)", "entries": { "partial-appium-login-01|baseline": { "arm": "baseline", "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.019589, - "mean_duration_s": 65.56, - "mean_input_tokens": 235691.0, - "mean_output_tokens": 7699.7, - "mean_turns": 16.5, + "mean_cost_usd": 0.022485, + "mean_duration_s": 71.39, + "mean_input_tokens": 325330.2, + "mean_output_tokens": 9397.0, + "mean_turns": 17.67, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -19,7 +19,7 @@ "skill": "rf-appium", "task_hash": "38720d9916e141cf2f3dea4874ff2b70b12ce403564779bf18d1a64fdb38c5b3", "task_id": "partial-appium-login-01", - "turn_limit_stops": 0, + "turn_limit_stops": 1, "turns_factor": 1.5 }, "partial-appium-login-01|treatment": { @@ -27,11 +27,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.037225, - "mean_duration_s": 100.02, - "mean_input_tokens": 667808.6, - "mean_output_tokens": 13806.6, - "mean_turns": 24.4, + "mean_cost_usd": 0.033973, + "mean_duration_s": 60.7, + "mean_input_tokens": 527899.0, + "mean_output_tokens": 9602.8, + "mean_turns": 22.33, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -40,7 +40,7 @@ "skill": "rf-appium", "task_hash": "38720d9916e141cf2f3dea4874ff2b70b12ce403564779bf18d1a64fdb38c5b3", "task_id": "partial-appium-login-01", - "turn_limit_stops": 3, + "turn_limit_stops": 2, "turns_factor": 1.5 } }, diff --git a/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json b/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json index dea426e..4a61314 100644 --- a/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json +++ b/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json @@ -7,31 +7,31 @@ "skills": { "rf-appium": { "train": { - "accuracy": 0.6522, + "accuracy": 0.7826, "fn": 0, - "fp": 8, - "negative_loads": 23, + "fp": 5, + "negative_loads": 14, "negative_runs": 33, "positive_loads": 36, "positive_runs": 36, - "precision": 0.6, + "precision": 0.7059, "queries": 23, "recall": 1.0, - "tn": 3, + "tn": 6, "tp": 12 }, "validation": { - "accuracy": 0.625, + "accuracy": 0.75, "fn": 0, - "fp": 3, - "negative_loads": 8, + "fp": 2, + "negative_loads": 4, "negative_runs": 12, "positive_loads": 12, "positive_runs": 12, - "precision": 0.5714, + "precision": 0.6667, "queries": 8, "recall": 1.0, - "tn": 1, + "tn": 2, "tp": 4 } } diff --git a/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json index a08a581..37ca372 100644 --- a/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json +++ b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json @@ -1,25 +1,25 @@ { - "claude_code_version": "2.1.322 (Claude Code)", + "claude_code_version": "2.1.323 (Claude Code)", "entries": { "adv-browser-deprecated-wait|baseline": { "arm": "baseline", "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.019885, - "mean_duration_s": 65.74, - "mean_input_tokens": 292573.7, - "mean_output_tokens": 6781.7, - "mean_turns": 23.0, + "mean_cost_usd": 0.018906, + "mean_duration_s": 54.32, + "mean_input_tokens": 387464.0, + "mean_output_tokens": 7496.5, + "mean_turns": 24.0, "model": "deepseek/deepseek-v4.1-flash", - "outcome_pass_rate": 0.3333, - "pass_rate": 0.3333, + "outcome_pass_rate": 0.5, + "pass_rate": 0.5, "provider": "openrouter", "runs": 6, "skill": "rf-browser", "task_hash": "f5d64210f2906670e4fb5973939a15d4fcecf5d2963843dad8e5384312023afb", "task_id": "adv-browser-deprecated-wait", - "turn_limit_stops": 0, + "turn_limit_stops": 1, "turns_factor": 1.5 }, "adv-browser-deprecated-wait|treatment": { @@ -27,20 +27,20 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.020272, - "mean_duration_s": 77.2, - "mean_input_tokens": 456359.7, - "mean_output_tokens": 5789.8, - "mean_turns": 23.17, + "mean_cost_usd": 0.029313, + "mean_duration_s": 78.22, + "mean_input_tokens": 641154.7, + "mean_output_tokens": 7527.3, + "mean_turns": 26.5, "model": "deepseek/deepseek-v4.1-flash", - "outcome_pass_rate": 0.8333, - "pass_rate": 0.8333, + "outcome_pass_rate": 0.5, + "pass_rate": 0.5, "provider": "openrouter", "runs": 6, "skill": "rf-browser", "task_hash": "f5d64210f2906670e4fb5973939a15d4fcecf5d2963843dad8e5384312023afb", "task_id": "adv-browser-deprecated-wait", - "turn_limit_stops": 2, + "turn_limit_stops": 4, "turns_factor": 1.5 }, "adv-selenium-nonexistent-kw|baseline": { @@ -48,14 +48,14 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.016414, - "mean_duration_s": 54.76, - "mean_input_tokens": 141485.3, - "mean_output_tokens": 4879.7, - "mean_turns": 11.67, + "mean_cost_usd": 0.013236, + "mean_duration_s": 45.03, + "mean_input_tokens": 207383.7, + "mean_output_tokens": 5032.7, + "mean_turns": 15.83, "model": "deepseek/deepseek-v4.1-flash", - "outcome_pass_rate": 1.0, - "pass_rate": 1.0, + "outcome_pass_rate": 0.6667, + "pass_rate": 0.6667, "provider": "openrouter", "runs": 6, "skill": "rf-selenium", @@ -69,14 +69,14 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.025679, - "mean_duration_s": 96.05, - "mean_input_tokens": 461271.2, - "mean_output_tokens": 9053.0, - "mean_turns": 24.5, + "mean_cost_usd": 0.021349, + "mean_duration_s": 81.26, + "mean_input_tokens": 447508.3, + "mean_output_tokens": 7002.7, + "mean_turns": 23.0, "model": "deepseek/deepseek-v4.1-flash", - "outcome_pass_rate": 0.8333, - "pass_rate": 0.8333, + "outcome_pass_rate": 0.5, + "pass_rate": 0.5, "provider": "openrouter", "runs": 6, "skill": "rf-selenium", @@ -90,11 +90,11 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.015992, - "mean_duration_s": 49.92, - "mean_input_tokens": 220741.7, - "mean_output_tokens": 5949.7, - "mean_turns": 17.33, + "mean_cost_usd": 0.017758, + "mean_duration_s": 47.52, + "mean_input_tokens": 305145.0, + "mean_output_tokens": 6929.3, + "mean_turns": 18.67, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -103,7 +103,7 @@ "skill": "rf-browser", "task_hash": "57639e3931b2ec11723d1524b36ad73c9fc034170c9fad3a755837b43d05d835", "task_id": "adv-web-wrong-library", - "turn_limit_stops": 0, + "turn_limit_stops": 1, "turns_factor": 1.5 }, "adv-web-wrong-library|treatment": { @@ -111,11 +111,11 @@ "cost_source": "priced", "gating": false, "incomplete": 0, - "mean_cost_usd": 0.026334, - "mean_duration_s": 93.87, - "mean_input_tokens": 460711.5, - "mean_output_tokens": 8861.3, - "mean_turns": 24.83, + "mean_cost_usd": 0.034421, + "mean_duration_s": 76.53, + "mean_input_tokens": 583977.5, + "mean_output_tokens": 9154.0, + "mean_turns": 30.17, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, diff --git a/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json index 1ae1115..1d68136 100644 --- a/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json +++ b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json @@ -1,16 +1,16 @@ { - "claude_code_version": "2.1.322 (Claude Code)", + "claude_code_version": "2.1.323 (Claude Code)", "entries": { "narrow-browser-login-01|baseline": { "arm": "baseline", "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.014097, - "mean_duration_s": 45.36, - "mean_input_tokens": 188544.3, - "mean_output_tokens": 3649.2, - "mean_turns": 15.17, + "mean_cost_usd": 0.0221, + "mean_duration_s": 41.51, + "mean_input_tokens": 259817.5, + "mean_output_tokens": 4249.5, + "mean_turns": 18.33, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -27,20 +27,20 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.02188, - "mean_duration_s": 67.12, - "mean_input_tokens": 390415.2, - "mean_output_tokens": 4727.0, - "mean_turns": 20.83, + "mean_cost_usd": 0.027451, + "mean_duration_s": 108.54, + "mean_input_tokens": 579766.5, + "mean_output_tokens": 8140.8, + "mean_turns": 23.5, "model": "deepseek/deepseek-v4.1-flash", - "outcome_pass_rate": 1.0, - "pass_rate": 1.0, + "outcome_pass_rate": 0.6667, + "pass_rate": 0.6667, "provider": "openrouter", "runs": 6, "skill": "rf-browser", "task_hash": "39ebb74ec3c1c1437cb481c44db5f8b675fccaf568b6efd283ef35a3fa05fa11", "task_id": "narrow-browser-login-01", - "turn_limit_stops": 0, + "turn_limit_stops": 2, "turns_factor": 1.5 }, "narrow-selenium-login-01|baseline": { @@ -48,11 +48,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.008738, - "mean_duration_s": 24.2, - "mean_input_tokens": 109915.0, - "mean_output_tokens": 2152.2, - "mean_turns": 11.67, + "mean_cost_usd": 0.011953, + "mean_duration_s": 26.12, + "mean_input_tokens": 121834.0, + "mean_output_tokens": 2828.8, + "mean_turns": 11.0, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -69,11 +69,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.016286, - "mean_duration_s": 58.33, - "mean_input_tokens": 304876.3, - "mean_output_tokens": 4447.3, - "mean_turns": 16.83, + "mean_cost_usd": 0.019385, + "mean_duration_s": 52.51, + "mean_input_tokens": 319132.7, + "mean_output_tokens": 4396.0, + "mean_turns": 20.5, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, diff --git a/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json index 7b263bc..077ad35 100644 --- a/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json +++ b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json @@ -1,16 +1,16 @@ { - "claude_code_version": "2.1.322 (Claude Code)", + "claude_code_version": "2.1.323 (Claude Code)", "entries": { "partial-browser-login-01|baseline": { "arm": "baseline", "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.013755, - "mean_duration_s": 42.39, - "mean_input_tokens": 180256.8, - "mean_output_tokens": 4400.5, - "mean_turns": 15.83, + "mean_cost_usd": 0.009883, + "mean_duration_s": 23.59, + "mean_input_tokens": 139337.0, + "mean_output_tokens": 2744.2, + "mean_turns": 12.0, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -27,11 +27,11 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.025019, - "mean_duration_s": 95.36, - "mean_input_tokens": 504980.2, - "mean_output_tokens": 8296.0, - "mean_turns": 23.33, + "mean_cost_usd": 0.024812, + "mean_duration_s": 87.04, + "mean_input_tokens": 458623.7, + "mean_output_tokens": 7368.5, + "mean_turns": 22.33, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -40,7 +40,7 @@ "skill": "rf-browser", "task_hash": "b25fc2a5d8a483208821b2d2af70f83e8c9a6c5af932ba538ce9a9c588a93f30", "task_id": "partial-browser-login-01", - "turn_limit_stops": 1, + "turn_limit_stops": 0, "turns_factor": 1.5 } }, diff --git a/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/realistic.json b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/realistic.json index 5d3fc2d..79394a7 100644 --- a/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/realistic.json +++ b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/realistic.json @@ -1,16 +1,16 @@ { - "claude_code_version": "2.1.322 (Claude Code)", + "claude_code_version": "2.1.323 (Claude Code)", "entries": { "realistic-browser-login-01|baseline": { "arm": "baseline", "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.012627, - "mean_duration_s": 25.91, - "mean_input_tokens": 191379.5, - "mean_output_tokens": 3273.0, - "mean_turns": 13.5, + "mean_cost_usd": 0.011813, + "mean_duration_s": 31.91, + "mean_input_tokens": 190313.5, + "mean_output_tokens": 3640.0, + "mean_turns": 14.5, "model": "deepseek/deepseek-v4.1-flash", "outcome_pass_rate": 1.0, "pass_rate": 1.0, @@ -27,20 +27,20 @@ "cost_source": "priced", "gating": true, "incomplete": 0, - "mean_cost_usd": 0.02262, - "mean_duration_s": 72.68, - "mean_input_tokens": 224201.3, - "mean_output_tokens": 7102.8, - "mean_turns": 13.83, + "mean_cost_usd": 0.016476, + "mean_duration_s": 49.13, + "mean_input_tokens": 352610.7, + "mean_output_tokens": 5195.2, + "mean_turns": 17.5, "model": "deepseek/deepseek-v4.1-flash", - "outcome_pass_rate": 1.0, - "pass_rate": 1.0, + "outcome_pass_rate": 0.8333, + "pass_rate": 0.8333, "provider": "openrouter", "runs": 6, "skill": "rf-browser", "task_hash": "1487dcd2cb2a712432868bb46d5006bd75580edfe803f09acfd5cc4e13ca40e5", "task_id": "realistic-browser-login-01", - "turn_limit_stops": 0, + "turn_limit_stops": 1, "turns_factor": 1.5 } }, diff --git a/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json index d5af87b..bb72b84 100644 --- a/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json +++ b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json @@ -7,26 +7,26 @@ "skills": { "rf-browser": { "train": { - "accuracy": 0.6087, + "accuracy": 0.8696, "fn": 0, - "fp": 9, - "negative_loads": 24, + "fp": 3, + "negative_loads": 12, "negative_runs": 33, "positive_loads": 36, "positive_runs": 36, - "precision": 0.5714, + "precision": 0.8, "queries": 23, "recall": 1.0, - "tn": 2, + "tn": 8, "tp": 12 }, "validation": { "accuracy": 0.875, "fn": 0, "fp": 1, - "negative_loads": 3, + "negative_loads": 2, "negative_runs": 12, - "positive_loads": 11, + "positive_loads": 12, "positive_runs": 12, "precision": 0.8, "queries": 8, @@ -37,32 +37,32 @@ }, "rf-selenium": { "train": { - "accuracy": 0.6957, - "fn": 0, - "fp": 7, - "negative_loads": 20, + "accuracy": 0.7826, + "fn": 5, + "fp": 0, + "negative_loads": 0, "negative_runs": 33, - "positive_loads": 36, + "positive_loads": 21, "positive_runs": 36, - "precision": 0.6316, + "precision": 1.0, "queries": 23, - "recall": 1.0, - "tn": 4, - "tp": 12 + "recall": 0.5833, + "tn": 11, + "tp": 7 }, "validation": { - "accuracy": 0.75, - "fn": 0, - "fp": 2, - "negative_loads": 6, + "accuracy": 0.625, + "fn": 3, + "fp": 0, + "negative_loads": 0, "negative_runs": 12, - "positive_loads": 12, + "positive_loads": 4, "positive_runs": 12, - "precision": 0.6667, + "precision": 1.0, "queries": 8, - "recall": 1.0, - "tn": 2, - "tp": 4 + "recall": 0.25, + "tn": 4, + "tp": 1 } } }