diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 0000000..b47ef84 --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,160 @@ +{ + "name": "robotframework-plugins", + "owner": { + "name": "Robot Framework Foundation" + }, + "metadata": { + "description": "Robot Framework agent plugins from the Robot Framework Foundation: skills, subagents and hooks for coding agents." + }, + "plugins": [ + { + "name": "rf-agentskills", + "source": "./bundles/rf-agentskills", + "description": "All Robot Framework plugins in one install (rf-core, rf-web, rf-api, rf-mobile, rf-desktop), as one plugin.", + "author": { + "name": "manykarim", + "email": "manykarim@users.noreply.github.com" + }, + "homepage": "https://github.com/MarketSquare/robotframework-plugins", + "repository": "https://github.com/MarketSquare/robotframework-plugins", + "license": "Apache-2.0", + "keywords": [ + "robotframework", + "testing", + "automation", + "web-testing", + "api-testing", + "mobile-testing", + "playwright", + "selenium", + "appium", + "requests", + "restinstance" + ], + "category": "testing", + "tags": [ + "robot-framework", + "verified" + ] + }, + { + "name": "rf-api", + "source": "./plugins/rf-api", + "description": "RequestsLibrary and RESTinstance skills for API tests.", + "author": { + "name": "manykarim", + "email": "manykarim@users.noreply.github.com" + }, + "homepage": "https://github.com/MarketSquare/robotframework-plugins", + "repository": "https://github.com/MarketSquare/robotframework-plugins", + "license": "Apache-2.0", + "keywords": [ + "robotframework", + "testing", + "api-testing", + "requests", + "restinstance" + ], + "category": "testing", + "tags": [ + "robot-framework", + "verified" + ] + }, + { + "name": "rf-core", + "source": "./plugins/rf-core", + "description": "Robot Framework syntax, environment setup, Python keyword libraries, test design and debugging subagents, and the validation hooks.", + "author": { + "name": "manykarim", + "email": "manykarim@users.noreply.github.com" + }, + "homepage": "https://github.com/MarketSquare/robotframework-plugins", + "repository": "https://github.com/MarketSquare/robotframework-plugins", + "license": "Apache-2.0", + "keywords": [ + "robotframework", + "testing", + "automation", + "robocop", + "robotcode" + ], + "category": "testing", + "tags": [ + "robot-framework", + "verified" + ] + }, + { + "name": "rf-desktop", + "source": "./plugins/rf-desktop", + "description": "PlatynUI skill for desktop UI tests.", + "author": { + "name": "manykarim", + "email": "manykarim@users.noreply.github.com" + }, + "homepage": "https://github.com/MarketSquare/robotframework-plugins", + "repository": "https://github.com/MarketSquare/robotframework-plugins", + "license": "Apache-2.0", + "keywords": [ + "robotframework", + "testing", + "desktop-testing", + "platynui" + ], + "category": "testing", + "tags": [ + "robot-framework", + "verified" + ] + }, + { + "name": "rf-mobile", + "source": "./plugins/rf-mobile", + "description": "AppiumLibrary skill for Android and iOS tests.", + "author": { + "name": "manykarim", + "email": "manykarim@users.noreply.github.com" + }, + "homepage": "https://github.com/MarketSquare/robotframework-plugins", + "repository": "https://github.com/MarketSquare/robotframework-plugins", + "license": "Apache-2.0", + "keywords": [ + "robotframework", + "testing", + "mobile-testing", + "appium" + ], + "category": "testing", + "tags": [ + "robot-framework", + "verified" + ] + }, + { + "name": "rf-web", + "source": "./plugins/rf-web", + "description": "Browser Library and SeleniumLibrary skills for web tests.", + "author": { + "name": "manykarim", + "email": "manykarim@users.noreply.github.com" + }, + "homepage": "https://github.com/MarketSquare/robotframework-plugins", + "repository": "https://github.com/MarketSquare/robotframework-plugins", + "license": "Apache-2.0", + "keywords": [ + "robotframework", + "testing", + "web-testing", + "playwright", + "selenium" + ], + "category": "testing", + "tags": [ + "robot-framework", + "verified" + ] + } + ], + "forceRemoveDeletedPlugins": true +} diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..3d1e0dd --- /dev/null +++ b/.env.example @@ -0,0 +1,21 @@ +# Claude Code OAuth token (preferred — subscription-based billing). +# Obtain via: `claude auth login` then inspect the saved token, or generate +# via the Claude Code web console. Tokens look like `sk-ant-oat01-...`. +CLAUDE_CODE_OAUTH_TOKEN=sk-ant-oat01-... + +# Fallback: Anthropic API key (per-token billing). Used if CLAUDE_CODE_OAUTH_TOKEN +# is unset. Only uncomment and set if you specifically want API-key auth. +# ANTHROPIC_API_KEY=sk-ant-... + +# Default model for evaluation runs. Must be one of: +# - claude-haiku-4-5-20251001 (cheap, fast; default for PR/narrow tasks, trigger evals) +# - claude-sonnet-5 (higher fidelity; for realistic/adversarial tiers) +# claude-opus-5-5 is a manual, explicit opt-in only +# (--model claude-opus-5-5 --allow-opus --max-cost-usd ) — see ADR-004. +CLAUDE_MODEL_DEFAULT=claude-haiku-4-5-20251001 + +# Log level for the harness (DEBUG, INFO, WARNING, ERROR). +RF_SKILL_EVAL_LOG_LEVEL=INFO + +# Opt-out of automatic .env loading on import (set to "0" to disable). +RF_SKILL_EVAL_AUTOLOAD_DOTENV=1 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..15cc426 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,51 @@ +# Ownership by path (launch-plugin-programme D16): a plugin is the unit of +# installation; a skill folder is the unit of ownership. Later lines win, so +# each skill folder follows its plugin's line. +# +# People on a /plugins// line must match that plugin's catalog.yaml +# maintainers (scripts/check-governance.py). The Foundation reviewers own every +# verified plugin; the security reviewers own security-relevant paths. GitHub +# asks each listed owner for review; the required-reviews workflow enforces an +# approval from every group (design D9). + +/plugins/rf-agentskills/ @manykarim @MarketSquare/robotframework-plugins-reviewers +/plugins/rf-agentskills/.claude-plugin/ @manykarim @MarketSquare/robotframework-plugins-reviewers @MarketSquare/robotframework-plugins-security +/plugins/rf-api/ @manykarim @MarketSquare/robotframework-plugins-reviewers +/plugins/rf-api/skills/rf-requests/ @manykarim @MarketSquare/robotframework-plugins-reviewers +/plugins/rf-api/skills/rf-restinstance/ @manykarim @MarketSquare/robotframework-plugins-reviewers +/plugins/rf-api/.claude-plugin/ @manykarim @MarketSquare/robotframework-plugins-reviewers @MarketSquare/robotframework-plugins-security +/plugins/rf-core/ @manykarim @MarketSquare/robotframework-plugins-reviewers +/plugins/rf-core/skills/rf-language/ @manykarim @MarketSquare/robotframework-plugins-reviewers +/plugins/rf-core/skills/rf-python-library/ @manykarim @MarketSquare/robotframework-plugins-reviewers +/plugins/rf-core/skills/rf-setup/ @manykarim @MarketSquare/robotframework-plugins-reviewers +/plugins/rf-core/hooks/ @manykarim @MarketSquare/robotframework-plugins-reviewers @MarketSquare/robotframework-plugins-security +/plugins/rf-core/scripts/ @manykarim @MarketSquare/robotframework-plugins-reviewers @MarketSquare/robotframework-plugins-security +/plugins/rf-core/agents/ @manykarim @MarketSquare/robotframework-plugins-reviewers @MarketSquare/robotframework-plugins-security +/plugins/rf-core/variants/ @manykarim @MarketSquare/robotframework-plugins-reviewers @MarketSquare/robotframework-plugins-security +/plugins/rf-core/.claude-plugin/ @manykarim @MarketSquare/robotframework-plugins-reviewers @MarketSquare/robotframework-plugins-security +/plugins/rf-core/skills/rf-language/scripts/ @manykarim @MarketSquare/robotframework-plugins-reviewers @MarketSquare/robotframework-plugins-security +/plugins/rf-core/skills/rf-python-library/scripts/ @manykarim @MarketSquare/robotframework-plugins-reviewers @MarketSquare/robotframework-plugins-security +/plugins/rf-desktop/ @manykarim @MarketSquare/robotframework-plugins-reviewers +/plugins/rf-desktop/skills/rf-platynui/ @manykarim @MarketSquare/robotframework-plugins-reviewers +/plugins/rf-desktop/.claude-plugin/ @manykarim @MarketSquare/robotframework-plugins-reviewers @MarketSquare/robotframework-plugins-security +/plugins/rf-mobile/ @manykarim @MarketSquare/robotframework-plugins-reviewers +/plugins/rf-mobile/skills/rf-appium/ @manykarim @MarketSquare/robotframework-plugins-reviewers +/plugins/rf-mobile/.claude-plugin/ @manykarim @MarketSquare/robotframework-plugins-reviewers @MarketSquare/robotframework-plugins-security +/plugins/rf-web/ @manykarim @MarketSquare/robotframework-plugins-reviewers +/plugins/rf-web/skills/rf-browser/ @manykarim @MarketSquare/robotframework-plugins-reviewers +/plugins/rf-web/skills/rf-selenium/ @manykarim @MarketSquare/robotframework-plugins-reviewers +/plugins/rf-web/.claude-plugin/ @manykarim @MarketSquare/robotframework-plugins-reviewers @MarketSquare/robotframework-plugins-security + +# Generated copies and shared security-relevant files. +/bundles/ @manykarim @MarketSquare/robotframework-plugins-reviewers @MarketSquare/robotframework-plugins-security +/.claude-plugin/ @manykarim @MarketSquare/robotframework-plugins-security +/security/ @MarketSquare/robotframework-plugins-security +/scripts/check_content_rules.py @manykarim @MarketSquare/robotframework-plugins-security +/scripts/compare-findings.py @manykarim @MarketSquare/robotframework-plugins-security +/scripts/install-scanners.sh @manykarim @MarketSquare/robotframework-plugins-security +/scripts/security-scan.sh @manykarim @MarketSquare/robotframework-plugins-security +/scripts/required-reviews.py @manykarim @MarketSquare/robotframework-plugins-security +/scripts/check-governance.py @manykarim @MarketSquare/robotframework-plugins-security +/.github/workflows/ @manykarim @MarketSquare/robotframework-plugins-security +/.github/CODEOWNERS @manykarim @MarketSquare/robotframework-plugins-security +/.github/governance.yaml @manykarim @MarketSquare/robotframework-plugins-security diff --git a/.github/ISSUE_TEMPLATE/plugin-feedback.yml b/.github/ISSUE_TEMPLATE/plugin-feedback.yml new file mode 100644 index 0000000..056357b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/plugin-feedback.yml @@ -0,0 +1,56 @@ +name: Plugin feedback +description: Tell us whether a plugin helped, and with what. +title: "[feedback] " +labels: [feedback] +body: + - type: markdown + attributes: + value: | + Thanks for taking a minute. Feedback is how we learn which plugins work; the + installer and plugins send nothing on their own. A label for the plugin you + pick is added automatically. + - type: dropdown + id: plugin + attributes: + label: Plugin + description: Which plugin is this about? + options: + # plugins:start (generated by scripts/build-catalog.py; do not edit) + - rf-agentskills + - rf-api + - rf-core + - rf-desktop + - rf-mobile + - rf-web + # plugins:end + validations: + required: true + - type: input + id: agent + attributes: + label: Agent and model + description: For example "Claude Code 2.1, Sonnet 5.5" or "Copilot CLI, GPT-6". + validations: + required: true + - type: input + id: versions + attributes: + label: Robot Framework and library versions + description: For example "Robot Framework 7.4.2, Browser 20.1.0". + validations: + required: true + - type: dropdown + id: helped + attributes: + label: Did the plugin help? + options: + - "Yes" + - Partly + - "No" + validations: + required: true + - type: textarea + id: details + attributes: + label: What happened + description: What you asked the agent, and what was good or wrong about the result. diff --git a/.github/actions/setup-eval/action.yml b/.github/actions/setup-eval/action.yml new file mode 100644 index 0000000..504d730 --- /dev/null +++ b/.github/actions/setup-eval/action.yml @@ -0,0 +1,43 @@ +name: Setup skill-eval environment +description: >- + Installs uv, Node 20, the Claude Code CLI, Python dependencies via uv sync, + and initializes Playwright browsers for robotframework-browser. Shared by + the Skill Evaluation workflow tiers. + +inputs: + node-version: + description: "Node.js version for the Claude Code CLI" + required: false + default: "20" + uv-cache-glob: + description: "Glob(s) uv should key its dependency cache on" + required: false + default: | + pyproject.toml + uv.lock + +runs: + using: composite + steps: + - name: Set up uv + uses: astral-sh/setup-uv@v3 + with: + enable-cache: true + cache-dependency-glob: ${{ inputs.uv-cache-glob }} + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ inputs.node-version }} + + - name: Install Claude Code CLI + shell: bash + run: npm install -g @anthropic-ai/claude-code + + - name: Install Python dependencies + shell: bash + run: uv sync --all-extras + + - name: Initialize Playwright browsers + shell: bash + run: uv run rfbrowser init diff --git a/.github/governance.yaml b/.github/governance.yaml new file mode 100644 index 0000000..fe09bd9 --- /dev/null +++ b/.github/governance.yaml @@ -0,0 +1,52 @@ +# Review groups and tier exceptions (spec plugin-trust-tiers; design D9). +# Read by scripts/check-governance.py and scripts/required-reviews.py. +# Changing this file needs the security reviewers' approval. + +teams: + # Approves verified plugins and tier promotions. + foundation: MarketSquare/robotframework-plugins-reviewers + # Approves security-relevant changes (spec content-security-scanning). + security: MarketSquare/robotframework-plugins-security + +# Security-relevant paths: a change to any of them needs the security team. +# Patterns use fnmatch; "*" matches across folders. A SKILL.md counts only +# when its allowed-tools line changes (scripts/required-reviews.py). +security_paths: + - "plugins/*/hooks/*" + - "bundles/*/hooks/*" + - "plugins/*/scripts/*" + - "bundles/*/scripts/*" + - "plugins/*/skills/*/scripts/*" + - "bundles/*/skills/*/scripts/*" + - "plugins/*/variants/opencode/plugins/*" + - "plugins/*/agents/*" + - "bundles/*/agents/*" + - "plugins/*/variants/*/agents/*" + - "*.mcp.json" + - "*/mcp.json" + - "plugins/*/.claude-plugin/*" + - "bundles/*/.claude-plugin/*" + - ".claude-plugin/*" + - "security/*" + - "scripts/check_content_rules.py" + - "scripts/compare-findings.py" + - "scripts/install-scanners.sh" + - "scripts/security-scan.sh" + - "scripts/required-reviews.py" + - "scripts/check-governance.py" + - ".github/workflows/*" + - ".github/CODEOWNERS" + - ".github/governance.yaml" + - ".gitleaksignore" + +# Verified plugins that don't meet every verified criterion yet. Each entry +# names what is missing and lasts until a date; after it, check-governance.py +# fails. New promotions to verified get no entry. +tier_grace: + - plugins: [rf-agentskills, rf-core, rf-web, rf-api, rf-mobile, rf-desktop] + missing: [second-maintainer, foundation-maintainer] + until: 2027-01-31 + reason: >- + The programme launches with one maintainer. A second maintainer from the + Foundation reviewers team joins before the first CalVer release + (launch task 10.2); otherwise these plugins move to the community tier. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..cc65608 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,19 @@ +## What this changes + + + +## Review checklist + +Tick each item, or write "n/a: " after it. Reviewers don't approve a PR +with an unticked item that has no reason (see CONTRIBUTING.md, "Review"). + +- [ ] **Purpose.** Each new or changed skill and subagent says what it is for and when to use it, in its description. +- [ ] **Supported versions.** `catalog.yaml` lists the libraries and version ranges the content was checked against, and the Robot Framework range. +- [ ] **Examples.** `.robot` examples pass `robot --dryrun`; Python examples run. +- [ ] **Security-relevant changes.** Hooks, scripts, MCP configuration, manifests, subagent `tools` and skill `allowed-tools` changes are listed below; the security gate passes, and any new baseline entry has a reason. +- [ ] **Tool permissions.** Every subagent declares an explicit `tools` list; no `allowed-tools` entry grants unrestricted shell access. +- [ ] **Documentation voice.** Text follows the writing rules in CONTRIBUTING.md and passes the docs voice check. + +### Security-relevant changes + + diff --git a/.github/workflows/adoption-collect.yml b/.github/workflows/adoption-collect.yml new file mode 100644 index 0000000..92c1bf6 --- /dev/null +++ b/.github/workflows/adoption-collect.yml @@ -0,0 +1,62 @@ +# Collect public adoption signals (spec adoption-reporting, "Public adoption +# signals are collected"; design D11). +# +# GitHub keeps traffic data for 14 days, so a weekly run stores each snapshot +# on the `adoption-data` branch: traffic//{views,clones,referrers,paths}.json. +# Snapshots overlap; scripts/adoption-report.py counts each day once. +# The traffic API needs push access: the default token may be refused, in which +# case set ADOPTION_TOKEN (fine-grained, this repository, Administration: read). +name: Adoption data + +on: + schedule: + - cron: "41 5 * * 1" + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: adoption-collect + cancel-in-progress: false + +jobs: + collect: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + - name: Switch to the data branch + run: | + if git ls-remote --exit-code --heads origin adoption-data >/dev/null; then + git switch adoption-data + else + git switch --orphan adoption-data + git rm -rf --quiet . || true + printf '# Adoption data\n\nWeekly GitHub traffic snapshots (adoption-collect.yml). Not code.\n' > README.md + fi + - name: Fetch traffic + env: + GH_TOKEN: ${{ secrets.ADOPTION_TOKEN || github.token }} + REPO: ${{ github.repository }} + run: | + day=$(date -u +%F) + dir="traffic/$day" + mkdir -p "$dir" + gh api "repos/$REPO/traffic/views?per=day" > "$dir/views.json" + gh api "repos/$REPO/traffic/clones?per=day" > "$dir/clones.json" + gh api "repos/$REPO/traffic/popular/referrers" > "$dir/referrers.json" + gh api "repos/$REPO/traffic/popular/paths" > "$dir/paths.json" + echo "Stored $(jq '.views | length' "$dir/views.json") days of views and $(jq '.clones | length' "$dir/clones.json") of clones." >> "$GITHUB_STEP_SUMMARY" + - name: Commit + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add README.md traffic + if git diff --cached --quiet; then + echo "No new data." + exit 0 + fi + git commit -m "data: traffic snapshot $(date -u +%F)" + git push origin adoption-data diff --git a/.github/workflows/adoption-report.yml b/.github/workflows/adoption-report.yml new file mode 100644 index 0000000..373c4a7 --- /dev/null +++ b/.github/workflows/adoption-report.yml @@ -0,0 +1,83 @@ +# Quarterly adoption report (spec adoption-reporting; design D11). +# +# On the first day of each quarter, builds docs/reports/adoption--Q.md +# for the quarter that just ended and opens a PR with it, so a person reads the +# report before it is published. Sources: the adoption-data branch (GitHub +# traffic), pypistats.org (PyPI downloads), this repository's issues, and the +# eval baselines at each --v* tag. +name: Adoption report + +on: + schedule: + - cron: "23 6 1 1,4,7,10 *" + workflow_dispatch: + inputs: + quarter: + description: "Quarter to report, e.g. 2026-Q4 (default: the one that just ended)" + required: false + type: string + +permissions: + contents: write + pull-requests: write + issues: read + +jobs: + report: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + - run: pip install pyyaml + - name: Choose the quarter + id: q + env: + INPUT: ${{ inputs.quarter }} + run: | + if [ -n "$INPUT" ]; then + echo "$INPUT" | grep -Eq '^[0-9]{4}-Q[1-4]$' || { echo "bad quarter: $INPUT"; exit 1; } + q="$INPUT" + else + q=$(python3 -c 'import datetime as d; t=d.date.today()-d.timedelta(days=1); print(f"{t.year}-Q{(t.month-1)//3+1}")') + fi + echo "quarter=$q" >> "$GITHUB_OUTPUT" + - name: Gather the data + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + run: | + mkdir -p "$RUNNER_TEMP/data" + if git ls-remote --exit-code --heads origin adoption-data >/dev/null; then + git fetch origin adoption-data + git archive origin/adoption-data traffic | tar -x -C "$RUNNER_TEMP/data" + fi + curl -fsS --proto '=https' "https://pypistats.org/api/packages/robotframework-plugins/overall" \ + -o "$RUNNER_TEMP/data/pypi.json" || echo "pypistats unavailable; the report says so" + gh api --paginate "repos/$REPO/issues?state=all&per_page=100" \ + | jq -s '[.[][] | select(.pull_request | not) | {number, state, labels: [.labels[].name], created_at, body}]' \ + > "$RUNNER_TEMP/data/issues.json" + - name: Build the report + env: + QUARTER: ${{ steps.q.outputs.quarter }} + run: | + python3 scripts/adoption-report.py --data "$RUNNER_TEMP/data" --quarter "$QUARTER" \ + --evals-from-git --output "docs/reports/adoption-$QUARTER.md" + - name: Open a PR + env: + GH_TOKEN: ${{ github.token }} + QUARTER: ${{ steps.q.outputs.quarter }} + run: | + branch="report/adoption-$QUARTER" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git switch -c "$branch" + git add "docs/reports/adoption-$QUARTER.md" + git commit -m "docs: adoption report $QUARTER" + git push --force-with-lease origin "$branch" + gh pr create --base main --head "$branch" --title "Adoption report $QUARTER" \ + --body "Generated by the Adoption report workflow. Read it before merging: candidates are prompts for a maintainer decision (GOVERNANCE.md)." \ + || echo "A PR for $branch exists already." diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..9fe0bcc --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,392 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_call: + +permissions: + contents: read + +jobs: + select-plugins: + name: Select plugin tests + # The shared tests in tests/ always run; a plugin's own tests run when the + # change touches that plugin (or a file it links to) or anything outside + # plugins/. Pushes and release calls run every plugin. + runs-on: ubuntu-latest + outputs: + plugins: ${{ steps.select.outputs.plugins }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - id: select + env: + BASE_REF: ${{ github.event_name == 'pull_request' && github.base_ref || '' }} + run: | + if [ -n "$BASE_REF" ]; then + python3 scripts/select-plugin-tests.py --base "origin/$BASE_REF" --github-output "$GITHUB_OUTPUT" + else + python3 scripts/select-plugin-tests.py --github-output "$GITHUB_OUTPUT" + fi + + test: + name: Test (${{ matrix.os }}, Python ${{ matrix.python-version }}) + needs: select-plugins + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + python-version: ["3.10", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + # tests/installer/ imports robotframework_plugins; install it from the + # in-repo package so the test collector can resolve it. The + # editable install also runs the hatchling build hook that + # bundles every active plugin under plugins/ into the package's _assets/ + # tree — that's what the runtime asset-loader reads. + run: | + # pytest-asyncio is required by `asyncio_mode = "auto"` in + # pyproject; pytest 9+ treats the unknown config option as a + # fatal error otherwise. robotcode[all] (the documented + # prerequisite; it also brings Robocop) lets the validation-hook + # tests run for real here instead of skipping. + pip install robotframework pytest pytest-asyncio pyyaml "jsonschema>=4.18" "robotcode[all]>=2.8" + pip install -e ./installer + shell: bash + - name: Install PlatynUI (optional — rf-platynui keyword-fidelity test) + # PlatynUI new_core requires Python 3.12+; the fidelity test skips when + # it's absent, so this install is fail-soft (no-op on the 3.10 matrix + # leg or if the pre-release is unavailable). Pinned to the verified + # pre-release; prebuilt wheel, no Rust. + run: pip install "robotframework-PlatynUI==0.12.0.dev330" || echo "PlatynUI unavailable on this interpreter — rf-platynui fidelity test will skip" + shell: bash + - name: Install Browser Library (optional — rf-browser example and keyword checks) + # Lets the rf-browser example dry runs and decision-table keyword checks + # run for real; skip-if-absent otherwise. No `rfbrowser init` needed — + # libdoc only introspects the Python library. + run: pip install robotframework-browser || echo "Browser unavailable — those libdoc tests will skip" + shell: bash + - name: Install other library-skill libraries (optional — example dry runs) + # tests/test_library_skill_structure.py dry-runs every example in + # plugins/*/skills/rf-{selenium,appium,requests,restinstance}/assets/examples and + # checks decision-table keywords against libdoc. Fail-soft: a library + # that cannot be installed (RESTinstance needs Python >= 3.11) makes its + # cases skip, never pass. + run: | + pip install robotframework-seleniumlibrary || echo "SeleniumLibrary unavailable — its example dry runs will skip" + pip install robotframework-appiumlibrary || echo "AppiumLibrary unavailable — its example dry runs will skip" + pip install robotframework-requests || echo "RequestsLibrary unavailable — its example dry runs will skip" + pip install RESTinstance || echo "RESTinstance unavailable on this interpreter — its example dry runs will skip" + shell: bash + - name: Install Robocop 9.x and PyYAML (optional — rf-language rule-ID fidelity and YAML example tests) + # plugins/rf-core/tests/test_language_skill.py checks every Robocop rule ID the rf-language + # skill cites against Robocop 9.x (the version rf-setup recommends) and + # runs the YAML variable-file example. Fail-soft: those cases skip. + run: | + pip install "robotframework-robocop>=9.1,<10" || echo "Robocop 9 unavailable — rf-language robocop tests will skip" + pip install pyyaml || echo "PyYAML unavailable — the rf-language YAML example test will skip" + shell: bash + - name: Run rf-python-library examples and check the example libraries + # plugins/rf-core/skills/rf-python-library/assets/examples must really run (not only + # dry-run), with the example listener, and the bundled checker must + # report no error or warning finding for the three example libraries. + # plugins/rf-core/tests/test_python_library_skill.py and test_check_library.py + # (run below) cover the same plus every code block in the skill. + run: | + export PYTHONDONTWRITEBYTECODE=1 + out="$RUNNER_TEMP/pylib" + python -m robot --pythonpath plugins/rf-core/skills/rf-python-library/assets/examples --listener ResultListener --outputdir "$out" --console dotted plugins/rf-core/skills/rf-python-library/assets/examples + cd plugins/rf-core/skills/rf-python-library/assets/examples + python ../../scripts/check_library.py ExampleLibrary.py converters.py ResultListener.py --json-out "$out/check_library.json" + python -c "import json, sys; s = json.load(open(sys.argv[1]))['summary']; print(s); sys.exit(1 if s['error'] or s['warning'] else 0)" "$out/check_library.json" + shell: bash + - name: Run tests + # tests/eval/ is the rf-skill-eval harness suite — an INTERNAL + # workspace package (root pyproject; "Private :: Do Not Upload") + # whose live-agent tests live under tests/eval/ and bring in + # the full eval dep set (pydantic, typer, jinja2, sqlite, + # parquet…). That suite is exercised end-to-end by the + # skill-evaluation workflow against real Claude Code; locally + # run with `uv sync --all-extras && uv run pytest tests/eval/`. + # This CI job stays focused on the plugin's own Python scripts + # plus the public rf-agentskills installer (installed above). + # Shared tests first, then each selected plugin's tests in its own run + # (plugins may reuse test module names). + env: + PLUGINS: ${{ needs.select-plugins.outputs.plugins }} + run: | + status=0 + python -m pytest tests/ --ignore=tests/eval -v --tb=short || status=1 + for plugin in $(python -c "import json, os; print(' '.join(json.loads(os.environ['PLUGINS'])))"); do + echo "::group::plugins/$plugin/tests" + python -m pytest "plugins/$plugin/tests" -v --tb=short || status=1 + echo "::endgroup::" + done + exit $status + shell: bash + + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install ruff + - name: Lint plugin skill scripts + run: ruff check plugins/*/skills/*/scripts/*.py --select E,F,W --ignore E501 + - name: Lint skill zip builder + run: ruff check scripts/build-skill-zips.py scripts/export-skills.py scripts/select-plugin-tests.py --select E,F,W --ignore E501 + - name: Lint repo QA scripts + run: ruff check scripts/check-skill-keywords.py scripts/validate-skills.py scripts/check-plugin-references.py --select E,F,W --ignore E501 + + template-plugin: + name: Template plugin passes the PR checks + # programme-documentation, "Template plugin passes CI": copies + # templates/plugin/ to plugins/example/ in a scratch checkout and runs the + # PR checks and the tests CI would select for it. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install dependencies + run: | + pip install robotframework pytest pytest-asyncio pyyaml "jsonschema>=4.18" "robotcode[all]>=2.8" + pip install -e ./installer + - name: Copy the template and run the checks + run: scripts/check-template.sh "$RUNNER_TEMP/template-check" + + validate-plugin: + name: Validate plugin structure + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + # The PR base is needed to check that marketplace renames are append-only. + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + # pytest-asyncio satisfies `asyncio_mode = "auto"` in pyproject + # (pytest 9+ errors on the unknown config option without it). + - run: pip install pytest pytest-asyncio pyyaml "jsonschema>=4.18" + - name: Validate the plugin catalogue + # Also: unique skill, subagent and hook install names across plugins, + # symlinks inside the repository, and (in PRs) append-only renames. + env: + BASE_REF: ${{ github.event_name == 'pull_request' && github.base_ref || '' }} + run: | + if [ -n "$BASE_REF" ]; then + python scripts/build-catalog.py --check --base "origin/$BASE_REF" + else + python scripts/build-catalog.py --check + fi + - name: Docs voice + # Banned corporate phrases and at most one exclamation mark per + # document (programme-documentation; the rest is a review item). + run: python scripts/check-docs-voice.py + - name: Ownership and tier evidence + # CODEOWNERS matches catalog.yaml maintainers; verified plugins have + # their evidence or a dated grace entry (.github/governance.yaml). + run: python scripts/check-governance.py + - name: Validate manifests and SKILL.md frontmatter + # Includes the marketplace/plugin manifest consistency checks. + run: python -m pytest tests/test_marketplace_validation.py -v + - name: Per-agent variants match the generator + run: python scripts/build-agent-variants.py --check + - name: Validate the skills of every plugin + run: python scripts/validate-skills.py --channel all + + validate-robot-syntax: + name: Validate .robot examples + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install robotframework + - name: Parse all .robot files + run: | + python3 -c " + import os, sys + from robot.api import get_model + errors, count = [], 0 + for root, dirs, files in os.walk('plugins'): + for f in files: + if f.endswith('.robot'): + path = os.path.join(root, f) + count += 1 + try: + get_model(path) + except Exception as e: + errors.append(f'{path}: {e}') + if errors: + for e in errors: print(f'ERROR: {e}', file=sys.stderr) + sys.exit(1) + print(f'All {count} .robot files valid') + " + + check-skill-keywords: + name: Check library-skill keywords + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install Robot Framework and the skills' target libraries + # libdoc only introspects the Python libraries: no `rfbrowser init`, + # browser, WebDriver or Appium server is needed. + run: | + pip install robotframework robotframework-browser robotframework-seleniumlibrary \ + robotframework-appiumlibrary robotframework-requests RESTinstance \ + "robotframework-PlatynUI==0.12.0.dev330" + - name: Resolve every keyword call against libdoc + # --require-all: a skill whose library failed to import fails the job + # instead of being skipped silently. + run: python scripts/check-skill-keywords.py --require-all + + check-drift: + name: Check generated variants and plugin layout + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Variants, skill names, symlinks and script command form + run: bash scripts/check-drift.sh + - name: Cross-plugin skill references name their plugin + run: python3 scripts/check-plugin-references.py + + validate-skills: + name: Validate SKILL.md frontmatter (agentskills.io) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + # Stdlib-only: name == dir, charset/length, description <= 1024, + # compatibility <= 500, no tags, allowed keys, license, metadata, + # metadata.version == the plugin.json version. + - name: Validate every plugin + run: python scripts/validate-skills.py --channel all + # Non-blocking cross-check against the reference implementation, so we + # notice if our reading of the spec diverges from skills-ref. + - name: Cross-check with skills-ref (non-blocking) + continue-on-error: true + run: | + pip install skills-ref + for d in plugins/*/skills/rf-*/; do + agentskills validate "$d" + done + + build-claude-plugin: + name: Build Claude Code plugin + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Validate plugin + run: | + python3 -c " + import json, os, sys + import glob + # Every plugin has a plugin.json; every skill folder a SKILL.md + for pj_path in sorted(glob.glob('plugins/*/.claude-plugin/plugin.json')): + with open(pj_path) as f: + pj = json.load(f) + print(f'Plugin: {pj[\"name\"]} v{pj.get(\"version\",\"?\")}') + skills = sorted(d for d in glob.glob('plugins/*/skills/*') if os.path.isdir(d)) + for s in skills: + assert os.path.exists(f'{s}/SKILL.md'), f'Missing SKILL.md in {s}' + print(f'Skills: {len(skills)} validated') + # Check scripts + scripts = glob.glob('plugins/*/skills/*/scripts/*.py') + assert scripts, 'no per-skill scripts found' + assert not glob.glob('plugins/*/scripts/*.py'), 'flat plugin scripts/*.py must not exist' + print(f'Scripts: {len(scripts)} found (per skill)') + # Check agents + agents = glob.glob('plugins/*/agents/*.md') + print(f'Agents: {len(agents)} found') + # The rf-tools MCP server was retired (content 2.0.0) + assert not glob.glob('plugins/*/.mcp.json'), 'plugins must not ship an MCP config' + assert not glob.glob('plugins/*/servers'), 'plugins must not ship servers/' + print('MCP server: none (retired)') + " + - name: Package Claude plugin + run: | + cd plugins + excludes=() + for d in */; do excludes+=(--exclude="${d}tests" --exclude="${d}evals"); done + tar -czf ../rf-agentskills-claude-plugin.tar.gz "${excludes[@]}" */ + - name: Upload plugin artifact + uses: actions/upload-artifact@v4 + with: + name: claude-plugin + path: rf-agentskills-claude-plugin.tar.gz + + build-portable-skills: + name: Build portable skills packages + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Export standalone skills from the plugin + # Writes every plugin's skills to skills/ on the runner only; the repository has no root copy. + run: python3 scripts/export-skills.py all skills + - name: Package standalone skills (for any agent) + run: | + tar -czf robotframework-skills-standalone.tar.gz \ + --transform='s|^skills/||' \ + skills/ + - name: Package Codex skills + run: | + mkdir -p .codex/skills + for d in skills/rf-*/; do + name=$(basename "$d") + ln -sf "../../$d" ".codex/skills/$name" + done + tar -chzf robotframework-skills-codex.tar.gz .codex/skills/ + - name: Package Copilot skills + run: | + mkdir -p .github/skills + for d in skills/rf-*/; do + name=$(basename "$d") + ln -sf "../../$d" ".github/skills/$name" + done + tar -chzf robotframework-skills-copilot.tar.gz .github/skills/ + - name: Package Claude Desktop / claude.ai skill uploads + run: python scripts/build-skill-zips.py --out skill-zips + - name: Upload standalone skills + uses: actions/upload-artifact@v4 + with: + name: standalone-skills + path: robotframework-skills-standalone.tar.gz + - name: Upload Claude Desktop skill zips + uses: actions/upload-artifact@v4 + with: + name: claude-desktop-skill-zips + path: skill-zips/*.zip + - name: Upload Codex skills + uses: actions/upload-artifact@v4 + with: + name: codex-skills + path: robotframework-skills-codex.tar.gz + - name: Upload Copilot skills + uses: actions/upload-artifact@v4 + with: + name: copilot-skills + path: robotframework-skills-copilot.tar.gz diff --git a/.github/workflows/feedback-label.yml b/.github/workflows/feedback-label.yml new file mode 100644 index 0000000..c4a4bc5 --- /dev/null +++ b/.github/workflows/feedback-label.yml @@ -0,0 +1,38 @@ +# Label plugin feedback issues (spec adoption-reporting, "Users can give +# feedback per plugin"). Issue forms can't map a dropdown to a label, so this +# reads the form's "Plugin" answer and adds `plugin:`. The answer is +# user input: it is used only when it names an existing plugin folder. +name: Feedback label + +on: + issues: + types: [opened, edited] + +permissions: + issues: write + contents: read + +jobs: + label: + if: contains(github.event.issue.labels.*.name, 'feedback') + runs-on: ubuntu-latest + steps: + - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + with: + script: | + const body = context.payload.issue.body || ""; + const m = body.match(/^### Plugin\s*\n+\s*(\S+)\s*$/m); + const name = m ? m[1] : ""; + if (!/^[a-z0-9][a-z0-9-]{0,63}$/.test(name)) { core.info("no plugin named"); return; } + const { owner, repo } = context.repo; + try { + await github.rest.repos.getContent({ owner, repo, path: `plugins/${name}/catalog.yaml` }); + } catch (e) { core.info(`not a plugin: ${name}`); return; } + const label = `plugin:${name}`; + try { + await github.rest.issues.getLabel({ owner, repo, name: label }); + } catch (e) { + await github.rest.issues.createLabel({ owner, repo, name: label, color: "1d76db", + description: `Feedback and issues for ${name}` }); + } + await github.rest.issues.addLabels({ owner, repo, issue_number: context.issue.number, labels: [label] }); diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..90623ca --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,213 @@ +# Releases (RELEASING.md; launch-plugin-programme D3). +# +# --v a plugin release: the tag must match the +# plugin's plugin.json; builds that plugin's +# archives and a GitHub release. +# robotframework-plugins-v an installer release: the tag must match +# installer/pyproject.toml and the current +# month (CalVer); builds the wheel and sdist, +# creates the GitHub release and publishes to +# PyPI by trusted publishing (a release +# candidate goes to TestPyPI). +# +# The two patterns never match the same tag. Older tags (v*, rf-agentskills-v*) +# are history and trigger nothing. +name: Release + +on: + push: + tags: + - "*--v*" + - "robotframework-plugins-v*" + +permissions: + contents: read + +jobs: + ci: + name: Run CI + uses: ./.github/workflows/ci.yml + + plugin-release: + name: Plugin release + if: contains(github.ref_name, '--v') + needs: ci + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + - run: pip install robotframework pyyaml + - name: Check the tag against plugin.json + id: tag + env: + TAG: ${{ github.ref_name }} + run: | + plugin="${TAG%%--v*}" + version="${TAG#*--v}" + echo "$plugin" | grep -Eq '^[a-z0-9]+(-[a-z0-9]+)*$' || { echo "::error::bad plugin name in tag $TAG"; exit 1; } + manifest="plugins/$plugin/.claude-plugin/plugin.json" + [ -f "$manifest" ] || { echo "::error::$TAG names $plugin, but $manifest doesn't exist"; exit 1; } + have=$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["version"])' "$manifest") + [ "$have" = "$version" ] || { echo "::error::$TAG says $version, $manifest says $have"; exit 1; } + bundle=$(python3 -c 'import sys, yaml; print("yes" if (yaml.safe_load(open(sys.argv[1])) or {}).get("bundle") else "")' "plugins/$plugin/catalog.yaml") + echo "plugin=$plugin" >> "$GITHUB_OUTPUT" + echo "version=$version" >> "$GITHUB_OUTPUT" + echo "bundle=$bundle" >> "$GITHUB_OUTPUT" + - name: Check drift + run: bash scripts/check-drift.sh + - name: Build the archives + env: + PLUGIN: ${{ steps.tag.outputs.plugin }} + VERSION: ${{ steps.tag.outputs.version }} + BUNDLE: ${{ steps.tag.outputs.bundle }} + run: | + mkdir -p dist + # The plugin as installed: a bundle ships its generated copy; a plugin + # leaves out its own tests/ and evals/ (not the example tests/ in skills). + if [ -n "$BUNDLE" ]; then + tar -czf "dist/$PLUGIN-$VERSION.tar.gz" -C bundles "$PLUGIN" + export_from=all + else + tar -czf "dist/$PLUGIN-$VERSION.tar.gz" -C plugins \ + --exclude="$PLUGIN/tests" --exclude="$PLUGIN/evals" "$PLUGIN" + export_from="$PLUGIN" + fi + # Standalone skill folders with plain script paths, and one Claude + # Desktop / claude.ai upload archive per skill. + python scripts/export-skills.py "$export_from" skills + tar -czf "dist/$PLUGIN-skills-$VERSION.tar.gz" -C skills . + python scripts/build-skill-zips.py --plugin "$export_from" --out dist + - name: Write the release notes + env: + PLUGIN: ${{ steps.tag.outputs.plugin }} + VERSION: ${{ steps.tag.outputs.version }} + run: | + { + echo "## $PLUGIN $VERSION" + echo + echo "Install from the marketplace \`MarketSquare/robotframework-plugins\` (\`$PLUGIN@robotframework-plugins\`), pinned to this release with the ref \`$PLUGIN--v$VERSION\`. Per-agent steps are in the README." + echo + # This version's section of the plugin's changelog. + awk -v v="$VERSION" ' + $0 ~ "^## " { if (found) exit; if (index($0, "## " v " ") == 1 || $0 == "## " v) { found = 1; next } } + found { print } + ' "plugins/$PLUGIN/CHANGELOG.md" + echo + echo "| Archive | Contents |" + echo "|---|---|" + echo "| \`$PLUGIN-$VERSION.tar.gz\` | The plugin as the marketplace installs it |" + echo "| \`$PLUGIN-skills-$VERSION.tar.gz\` | Standalone skill folders for any agent's skills folder |" + echo "| \`rf-*.zip\` | One upload archive per skill for Claude Desktop and claude.ai |" + } > notes.md + - name: Create the GitHub release + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 + with: + body_path: notes.md + prerelease: ${{ contains(steps.tag.outputs.version, '-') }} + files: dist/* + + update-branches: + name: Update release branches + needs: plugin-release + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + - name: Move stable and latest to this release + run: | + for branch in stable latest; do + git checkout -B "$branch" "$GITHUB_REF_NAME" + git push origin "$branch" --force + done + + installer-release: + name: Installer release + if: startsWith(github.ref_name, 'robotframework-plugins-v') + needs: ci + runs-on: ubuntu-latest + permissions: + contents: write + outputs: + version: ${{ steps.tag.outputs.version }} + candidate: ${{ steps.tag.outputs.candidate }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + - run: pip install pyyaml "jsonschema>=4.18" twine + - name: Check the tag against pyproject.toml and the month + id: tag + env: + TAG: ${{ github.ref_name }} + run: | + version="${TAG#robotframework-plugins-v}" + have=$(sed -n 's/^version = "\(.*\)"/\1/p' installer/pyproject.toml | head -1) + [ "$have" = "$version" ] || { echo "::error::$TAG says $version, installer/pyproject.toml says $have"; exit 1; } + month="$(date -u +%Y).$(date -u +%-m)." + case "$version" in + "$month"*) ;; + *) echo "::error::$version is not a release of this month ($month); see RELEASING.md"; exit 1 ;; + esac + candidate="" + echo "$version" | grep -Eq '(a|b|rc)[0-9]+$' && candidate=yes + echo "version=$version" >> "$GITHUB_OUTPUT" + echo "candidate=$candidate" >> "$GITHUB_OUTPUT" + - name: Build the wheel and sdist + run: bash scripts/build-packages.sh --clean --check + - name: Write the release notes + env: + VERSION: ${{ steps.tag.outputs.version }} + run: | + { + python scripts/build-catalog.py --version-table + echo + awk -v v="$VERSION" ' + $0 ~ "^## " { if (found) exit; if (index($0, "## " v " ") == 1) { found = 1; next } } + found { print } + ' installer/CHANGELOG.md + } > notes.md + - name: Create the GitHub release + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 + with: + body_path: notes.md + prerelease: ${{ steps.tag.outputs.candidate == 'yes' }} + files: dist/robotframework_plugins-* + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: installer-dist + path: dist/robotframework_plugins-* + + publish: + name: Publish to ${{ needs.installer-release.outputs.candidate == 'yes' && 'TestPyPI' || 'PyPI' }} + needs: installer-release + runs-on: ubuntu-latest + # Trusted publishing: PyPI accepts this workflow's OIDC token for the + # robotframework-plugins project; no API token exists. The environment is + # protected, so a maintainer approves each upload. + environment: + name: ${{ needs.installer-release.outputs.candidate == 'yes' && 'testpypi' || 'pypi' }} + url: ${{ needs.installer-release.outputs.candidate == 'yes' && 'https://test.pypi.org/p/robotframework-plugins' || 'https://pypi.org/p/robotframework-plugins' }} + permissions: + id-token: write + steps: + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: installer-dist + path: dist + - name: Upload + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # release/v1 + with: + repository-url: ${{ needs.installer-release.outputs.candidate == 'yes' && 'https://test.pypi.org/legacy/' || 'https://upload.pypi.org/legacy/' }} diff --git a/.github/workflows/required-reviews.yml b/.github/workflows/required-reviews.yml new file mode 100644 index 0000000..6c47e40 --- /dev/null +++ b/.github/workflows/required-reviews.yml @@ -0,0 +1,133 @@ +# Required reviews (spec plugin-trust-tiers; design D9). +# +# CODEOWNERS can require one approval per path. This check requires one from +# each group a PR concerns: the owners of every plugin it touches, the +# Foundation reviewers for verified plugins and tier promotions, and the +# security reviewers for security-relevant paths (.github/governance.yaml). +# It sets the `required-reviews` commit status on the PR head, which branch +# protection requires. +# +# It runs base-branch code only and reads the PR through the API; no PR +# content is checked out or executed. REVIEW_TEAMS_TOKEN must be able to read +# the MarketSquare teams' members (a fine-grained token with Members: read). +name: Required reviews + +on: + pull_request_target: + types: [opened, synchronize, reopened, ready_for_review, edited] + workflow_run: + workflows: [Review relay] + types: [completed] + +permissions: + contents: read + pull-requests: read + statuses: write + actions: read + +concurrency: + group: required-reviews-${{ github.event.pull_request.number || github.event.workflow_run.id }} + cancel-in-progress: true + +jobs: + check: + runs-on: ubuntu-latest + steps: + - name: Check out the base branch (trusted code) + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + - run: pip install pyyaml + - name: Find the PR + id: pr + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + with: + script: | + let number = context.payload.pull_request?.number; + if (!number) { + const run = context.payload.workflow_run; + const { data } = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, repo: context.repo.repo, run_id: run.id }); + const art = data.artifacts.find((a) => a.name === "review-relay"); + if (!art) { core.setFailed("no review-relay artifact"); return; } + const zip = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, repo: context.repo.repo, artifact_id: art.id, archive_format: "zip" }); + require("fs").writeFileSync("relay.zip", Buffer.from(zip.data)); + const text = require("child_process").execFileSync("unzip", ["-p", "relay.zip", "pr-number"]).toString().trim(); + if (!/^[0-9]{1,7}$/.test(text)) { core.setFailed(`bad PR number from the relay: ${text}`); return; } + number = Number(text); + } + core.setOutput("number", String(number)); + - name: Build the PR snapshot + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + env: + PR_NUMBER: ${{ steps.pr.outputs.number }} + with: + github-token: ${{ secrets.REVIEW_TEAMS_TOKEN || github.token }} + script: | + const fs = require("fs"); + const { owner, repo } = context.repo; + const number = Number(process.env.PR_NUMBER); + const pr = (await github.rest.pulls.get({ owner, repo, pull_number: number })).data; + const files = await github.paginate(github.rest.pulls.listFiles, { owner, repo, pull_number: number, per_page: 100 }); + const reviews = await github.paginate(github.rest.pulls.listReviews, { owner, repo, pull_number: number, per_page: 100 }); + const plugins = new Set(); + for (const f of files) { + const parts = f.filename.split("/"); + if (parts.length > 2 && (parts[0] === "plugins" || parts[0] === "bundles")) plugins.add(parts[1]); + } + async function catalog(name, ref) { + try { + const { data } = await github.rest.repos.getContent({ owner, repo, path: `plugins/${name}/catalog.yaml`, ref }); + return Buffer.from(data.content, "base64").toString("utf8"); + } catch (e) { return null; } + } + const base = {}, head = {}; + for (const name of plugins) { + const b = await catalog(name, pr.base.sha); if (b !== null) base[name] = b; + const h = await catalog(name, pr.head.sha); if (h !== null) head[name] = h; + } + const gov = fs.readFileSync(".github/governance.yaml", "utf8"); + const slug = (key) => (gov.match(new RegExp(`^\\s*${key}:\\s*(\\S+)`, "m")) || [])[1]; + const teams = {}; + for (const key of ["foundation", "security"]) { + const [org, team_slug] = (slug(key) || "/").split("/"); + try { + const members = await github.paginate(github.rest.teams.listMembersInOrg, { org, team_slug, per_page: 100 }); + teams[key] = members.map((m) => m.login); + } catch (e) { + core.warning(`cannot read team ${org}/${team_slug}: ${e.message}`); + teams[key] = []; + } + } + const snapshot = { + author: pr.user.login, head_sha: pr.head.sha, + files: files.map((f) => ({ filename: f.filename, status: f.status, patch: f.patch || "" })), + reviews: reviews.map((r) => ({ user: r.user && r.user.login, state: r.state, submitted_at: r.submitted_at })), + base_catalogs: base, head_catalogs: head, teams, + }; + fs.writeFileSync("snapshot.json", JSON.stringify(snapshot)); + core.exportVariable("HEAD_SHA", pr.head.sha); + - name: Decide + id: decide + run: | + status=0 + python3 scripts/required-reviews.py snapshot.json --summary summary.md || status=$? + cat summary.md >> "$GITHUB_STEP_SUMMARY" + echo "status=$status" >> "$GITHUB_OUTPUT" + - name: Set the commit status + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + env: + STATUS: ${{ steps.decide.outputs.status }} + with: + script: | + const ok = process.env.STATUS === "0"; + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, repo: context.repo.repo, sha: process.env.HEAD_SHA, + state: ok ? "success" : "pending", context: "required-reviews", + description: ok ? "Every required group approved" : "Waiting for approvals (see the run summary)", + target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, + }); diff --git a/.github/workflows/review-relay.yml b/.github/workflows/review-relay.yml new file mode 100644 index 0000000..11b8c3c --- /dev/null +++ b/.github/workflows/review-relay.yml @@ -0,0 +1,29 @@ +# Relays review events to the required-reviews check (design D9). +# +# A review on a fork PR runs without secrets, and the check needs a token that +# can read team membership. This workflow only records the PR number; the +# Required reviews workflow then runs on workflow_run, in this repository's +# context, and re-evaluates the PR. Nothing here reads PR content. +name: Review relay + +on: + pull_request_review: + types: [submitted, dismissed] + +permissions: {} + +jobs: + relay: + runs-on: ubuntu-latest + steps: + - name: Record the PR number + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + mkdir -p relay + echo "$PR_NUMBER" > relay/pr-number + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: review-relay + path: relay/pr-number + retention-days: 1 diff --git a/.github/workflows/security-deep-scan.yml b/.github/workflows/security-deep-scan.yml new file mode 100644 index 0000000..d8532b7 --- /dev/null +++ b/.github/workflows/security-deep-scan.yml @@ -0,0 +1,135 @@ +# Model-assisted security scan (spec content-security-scanning, "Model-assisted +# scans run only after maintainer approval"; design D6). +# +# Sends plugin content to a model provider, so it runs only when a maintainer +# labels a PR `security-deep-scan`, and on pushes to main. It runs with the +# model key, so the PR is DATA only: +# - the scripts, scanners and settings come from the base branch; +# - the PR head is checked out into ./pr without credentials, and nothing in +# it is installed or executed; the scanners only read it; +# - every action is pinned to a commit SHA. +# A new push to the PR does not rerun the scan: a maintainer applies the label +# again after reviewing the new commits. +name: Security deep scan + +on: + pull_request_target: + types: [labeled] + push: + branches: [main] + +permissions: + contents: read + +concurrency: + group: security-deep-scan-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + deep-scan: + name: Deep scan + if: github.event_name == 'push' || github.event.label.name == 'security-deep-scan' + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + statuses: write + env: + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + steps: + - name: Check out the base branch (trusted code) + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: ${{ github.event.pull_request.base.sha || github.sha }} + persist-credentials: false + path: base + - name: Check out the PR head as data + if: github.event_name == 'pull_request_target' + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + fetch-depth: 0 + path: pr + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + - run: pip install pyyaml + - name: Mark the scan as pending + if: github.event_name == 'pull_request_target' + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + with: + script: | + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, repo: context.repo.repo, sha: process.env.HEAD_SHA, + state: "pending", context: "security-deep-scan", description: "Model-assisted scan running", + }); + - name: Install the pinned scanners + run: base/scripts/install-scanners.sh "$RUNNER_TEMP/scanners" + - name: Choose plugins + id: select + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + tree=base + if [ -d pr ]; then + tree=pr + git -C pr diff --name-only "$BASE_SHA...HEAD" > "$RUNNER_TEMP/changed.txt" || true + python3 base/scripts/select-plugin-tests.py --security --root pr \ + --paths-file "$RUNNER_TEMP/changed.txt" --github-output "$GITHUB_OUTPUT" + else + python3 base/scripts/select-plugin-tests.py --security --root base --github-output "$GITHUB_OUTPUT" + fi + echo "tree=$tree" >> "$GITHUB_OUTPUT" + - name: Scan with the model passes + id: scan + env: + ANTHROPIC_API_KEY: ${{ secrets.SECURITY_SCAN_ANTHROPIC_API_KEY }} + SECURITY_SCAN_MODEL: ${{ vars.SECURITY_SCAN_MODEL || 'claude-sonnet-5-5' }} + SCANNERS: ${{ runner.temp }}/scanners/bin + SCAN_ROOT: ${{ github.workspace }}/${{ steps.select.outputs.tree }} + PLUGINS: ${{ steps.select.outputs.plugins }} + run: | + status=0 + report="$RUNNER_TEMP/deep-scan.md" + echo "Model-assisted scan of \`${HEAD_SHA:0:7}\` with $SECURITY_SCAN_MODEL." > "$report" + for plugin in $(python3 -c 'import json, os; print(" ".join(json.loads(os.environ["PLUGINS"])))'); do + base/scripts/security-scan.sh "$plugin" "$RUNNER_TEMP/scan/$plugin" --deep || status=1 + { echo; cat "$RUNNER_TEMP/scan/$plugin/summary.md"; } >> "$report" + done + cat "$report" >> "$GITHUB_STEP_SUMMARY" + echo "status=$status" >> "$GITHUB_OUTPUT" + - name: Upload SARIF + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: sarif-deep-scan + path: ${{ runner.temp }}/scan/*/*.sarif + if-no-files-found: warn + - name: Comment and set the status + if: always() && github.event_name == 'pull_request_target' + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + env: + SCAN_STATUS: ${{ steps.scan.outputs.status }} + with: + script: | + const fs = require("fs"); + const path = `${process.env.RUNNER_TEMP}/deep-scan.md`; + let body = fs.existsSync(path) ? fs.readFileSync(path, "utf8") : "The deep scan did not finish; see the run log."; + if (body.length > 60000) body = body.slice(0, 60000) + "\n\n(Cut short; the full report is in the job summary.)"; + const ok = process.env.SCAN_STATUS === "0"; + await github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, + body: `### Security deep scan: ${ok ? "passed" : "failed"}\n\n${body}`, + }); + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, repo: context.repo.repo, sha: process.env.HEAD_SHA, + state: ok ? "success" : "failure", context: "security-deep-scan", + description: ok ? "No new HIGH or CRITICAL findings" : "New HIGH or CRITICAL findings, or a baseline problem", + target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, + }); + - name: Fail on new findings + if: steps.scan.outputs.status != '0' + run: exit 1 diff --git a/.github/workflows/security-gate.yml b/.github/workflows/security-gate.yml new file mode 100644 index 0000000..9de0188 --- /dev/null +++ b/.github/workflows/security-gate.yml @@ -0,0 +1,148 @@ +# Security PR gate (spec content-security-scanning; design D6). +# +# Runs on every PR, forks included, with read-only permissions and no secrets. +# Nothing from the PR is executed: the scanners read the plugin files as data. +# Findings go to SARIF artifacts and the job summary; the gate fails on a new +# HIGH or CRITICAL finding that is not in security/baselines/.yaml, or +# on an expired baseline entry. The nightly run scans every plugin, so expiry +# dates and new scanner rules are noticed without a PR. +name: Security gate + +on: + pull_request: + schedule: + - cron: "17 3 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: security-gate-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + house-rules: + name: House content rules + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + - name: Check shipped content + run: python3 scripts/check_content_rules.py + + select: + name: Choose plugins to scan + runs-on: ubuntu-latest + outputs: + plugins: ${{ steps.select.outputs.plugins }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + - run: pip install pyyaml + - id: select + env: + BASE_REF: ${{ github.event_name == 'pull_request' && github.base_ref || '' }} + run: | + if [ -n "$BASE_REF" ]; then + python3 scripts/select-plugin-tests.py --security --base "origin/$BASE_REF" --github-output "$GITHUB_OUTPUT" + else + python3 scripts/select-plugin-tests.py --security --github-output "$GITHUB_OUTPUT" + fi + + scan: + name: Scan ${{ matrix.plugin }} + needs: select + if: needs.select.outputs.plugins != '[]' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + plugin: ${{ fromJSON(needs.select.outputs.plugins) }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + - run: pip install pyyaml + - name: Install the pinned scanners + run: scripts/install-scanners.sh "$RUNNER_TEMP/scanners" + - name: Scan and compare with the baseline + env: + SCANNERS: ${{ runner.temp }}/scanners/bin + PLUGIN: ${{ matrix.plugin }} + run: | + status=0 + scripts/security-scan.sh "$PLUGIN" "$RUNNER_TEMP/scan" || status=$? + cat "$RUNNER_TEMP/scan/summary.md" >> "$GITHUB_STEP_SUMMARY" + exit $status + - name: Upload SARIF + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: sarif-${{ matrix.plugin }} + path: | + ${{ runner.temp }}/scan/*.sarif + ${{ runner.temp }}/scan/summary.md + if-no-files-found: warn + + gitleaks: + name: Secrets in the PR diff + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + persist-credentials: false + - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + - name: Install the pinned scanners + run: scripts/install-scanners.sh "$RUNNER_TEMP/scanners" + - name: Scan the commits of this PR + env: + BASE_REF: ${{ github.base_ref }} + run: | + status=0 + "$RUNNER_TEMP/scanners/bin/gitleaks" git . --no-banner --redact \ + --log-opts "origin/$BASE_REF..HEAD" \ + --report-format sarif --report-path "$RUNNER_TEMP/gitleaks.sarif" || status=$? + if [ "$status" -ne 0 ]; then + echo "### Gitleaks found a secret in this PR" >> "$GITHUB_STEP_SUMMARY" + fi + exit $status + - name: Upload SARIF + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: sarif-gitleaks + path: ${{ runner.temp }}/gitleaks.sarif + if-no-files-found: warn + + # The one check branch protection requires: the scan jobs are named after + # the plugins a PR changes, so their names differ from PR to PR. + result: + name: Security gate result + if: always() + needs: [house-rules, select, scan, gitleaks] + runs-on: ubuntu-latest + steps: + - name: Fail if any gate job failed + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + echo "job results: $RESULTS" + for r in $RESULTS; do + case "$r" in success|skipped) ;; *) echo "::error::a security gate job ended with '$r'"; exit 1 ;; esac + done diff --git a/.github/workflows/skill-evaluation.yml b/.github/workflows/skill-evaluation.yml new file mode 100644 index 0000000..c721d17 --- /dev/null +++ b/.github/workflows/skill-evaluation.yml @@ -0,0 +1,623 @@ +name: Skill Evaluation + +# Tiered skill evaluation (strengthen-skill-eval-harness, ADR-005): +# PR -> per plugin with changes: narrow tier, treatment arm only, N=3, Haiku, +# scoped to the changed skills + `plugin` canaries, gated against +# plugins//evals/baselines/narrow.json; validation-split +# trigger evals when a SKILL.md description changed. +# Every plugin with an evals/ folder is a matrix leg; each leg stages all +# content plugins (a full install) and uses only its own tasks and baselines. +# weekly -> all tiers in both arms, N=3, all trigger sets, candidate baseline. +# The partial tier (only rf-core staged) runs like narrow. +# manual -> same as weekly with model / runs / allow_opus inputs. +# Every harness invocation carries an explicit --max-cost-usd cap. + +on: + pull_request: + paths: + - 'plugins/**' + # The plugins' own pytest suites are CI's job, not a reason for agent runs. + - '!plugins/*/tests/**' + - 'eval/**' + - 'src/rf_skill_eval/**' + - 'pyproject.toml' + - 'uv.lock' + - '.github/workflows/skill-evaluation.yml' + schedule: + # Sunday 04:00 UTC - off-hours weekly run per ADR-005 + - cron: '0 4 * * 0' + workflow_dispatch: + inputs: + tiers: + description: "Comma list of tiers" + required: true + default: "narrow,realistic,adversarial,partial" + type: string + arms: + description: "Arms to run" + required: true + default: "treatment,baseline" + type: choice + options: + - "treatment,baseline" + - "treatment" + - "baseline" + model: + description: "Override every task's model (empty = tier defaults: narrow Haiku, realistic/adversarial Sonnet)" + required: false + default: "" + type: choice + options: + - "" + - "claude-haiku-5-5" + - "claude-sonnet-5-5" + - "claude-opus-5-5" + runs: + description: "Replicates per task x arm" + required: true + default: "3" + type: string + allow_opus: + description: "Opt in to claude-opus-5-5 (manual runs only; needs a cost cap)" + required: false + default: false + type: boolean + max_cost_usd: + description: "Cost cap in USD per harness invocation (each tier, triggers)" + required: true + default: "15" + type: string + triggers: + description: "Also run all trigger sets (both splits)" + required: false + default: true + type: boolean + use_api_key: + description: "Fall back to ANTHROPIC_API_KEY instead of OAuth token" + required: false + default: false + type: boolean + +concurrency: + group: skill-eval-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write + issues: write + +env: + HAIKU_MODEL: claude-haiku-5-5 + # Initial caps (design D9 open question) - tune after the first weekly run. + PR_NARROW_CAP_USD: "10" + PR_TRIGGER_CAP_USD: "20" + WEEKLY_NARROW_CAP_USD: "12" + WEEKLY_REALISTIC_CAP_USD: "12" + WEEKLY_ADVERSARIAL_CAP_USD: "8" + WEEKLY_PARTIAL_CAP_USD: "4" + WEEKLY_TRIGGER_CAP_USD: "8" + +jobs: + preflight: + name: Preflight (validate + select) + runs-on: ubuntu-24.04 # ubuntu-latest (26.04) has no Playwright chromium yet + timeout-minutes: 15 + outputs: + has_credentials: ${{ steps.creds.outputs.has_credentials }} + run_evals: ${{ steps.select.outputs.run_evals }} + # JSON list of {plugin, skills, full_tier, run_triggers, trigger_skills}, + # one per plugin whose evals must run. + matrix: ${{ steps.select.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up uv + uses: astral-sh/setup-uv@v3 + with: + enable-cache: true + cache-dependency-glob: | + pyproject.toml + uv.lock + + - name: Install harness + run: uv sync --all-packages + + - name: Validate tasks, trigger sets and coverage (no API calls) + run: | + for evals in plugins/*/evals; do + plugin=$(basename "$(dirname "$evals")") + uv run rf-skill-eval --plugin "$plugin" validate-tasks + uv run rf-skill-eval --plugin "$plugin" coverage + done + + # Live evals run plugin hooks and scripts with an API key, so they run only + # on code a maintainer pushed: main, schedules, manual runs, and PRs whose + # head branch is in this repository (content-security-scanning, "Untrusted + # plugin content is never executed with secrets"). A fork PR never counts + # as having credentials, whatever the repository's fork-secrets setting. + - name: Detect credentials + id: creds + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + FORK_PR: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} + run: | + if [ "${FORK_PR}" = "true" ]; then + echo "has_credentials=false" >> "$GITHUB_OUTPUT" + elif [ -n "${CLAUDE_CODE_OAUTH_TOKEN}" ] || [ -n "${ANTHROPIC_API_KEY}" ]; then + echo "has_credentials=true" >> "$GITHUB_OUTPUT" + else + echo "has_credentials=false" >> "$GITHUB_OUTPUT" + fi + + - name: Map changed paths to skills, per plugin + id: select + run: | + legs="${RUNNER_TEMP}/legs" + mkdir -p "$legs" + for evals in plugins/*/evals; do + plugin=$(basename "$(dirname "$evals")") + out="$legs/$plugin.env" + if [ "${{ github.event_name }}" = "pull_request" ]; then + uv run rf-skill-eval --plugin "$plugin" preflight \ + --base "origin/${{ github.base_ref }}" \ + --github-output "$out" + else + printf 'run_evals=true\nfull_tier=true\nskills=\nrun_triggers=true\ntrigger_skills=\n' > "$out" + fi + done + python3 - "$legs" >> "$GITHUB_OUTPUT" <<'PY' + import json, sys + from pathlib import Path + legs = [] + for f in sorted(Path(sys.argv[1]).glob("*.env")): + values = dict(line.split("=", 1) for line in f.read_text().splitlines() if "=" in line) + if values.get("run_evals") == "true": + legs.append({"plugin": f.stem, **{k: values.get(k, "") for k in + ("skills", "full_tier", "run_triggers", "trigger_skills")}}) + print(f"run_evals={'true' if legs else 'false'}") + print(f"matrix={json.dumps(legs)}") + PY + + harness-tests: + name: Harness unit tests (no API calls) + runs-on: ubuntu-24.04 # ubuntu-latest (26.04) has no Playwright chromium yet + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up uv + uses: astral-sh/setup-uv@v3 + with: + enable-cache: true + cache-dependency-glob: | + pyproject.toml + uv.lock + + - name: Install harness + grader libraries + run: uv sync --all-packages + + - name: Initialize Playwright browsers (golden-solution grading) + run: uv run rfbrowser init + + - name: Lint and type-check + run: | + uv run ruff check src tests/eval + uv run mypy + + - name: pytest tests/eval + # Fakes and recorded transcripts only; RF_SKILL_EVAL_AUTOLOAD_DOTENV=0 and + # no secrets, so nothing can reach a model. + env: + RF_SKILL_EVAL_AUTOLOAD_DOTENV: "0" + run: uv run pytest tests/eval -q + + not-run: + name: Evaluation not run (no credentials) + needs: preflight + if: ${{ needs.preflight.outputs.has_credentials != 'true' && needs.preflight.outputs.run_evals == 'true' }} + runs-on: ubuntu-24.04 # ubuntu-latest (26.04) has no Playwright chromium yet + timeout-minutes: 5 + steps: + - name: Report + run: | + echo "::notice::not run: no credentials (fork PR or missing secrets) - no evaluation result was produced" + { + echo "## Skill evaluation" + echo + echo "**not run: no credentials** - this is not a passing result." + } >> "$GITHUB_STEP_SUMMARY" + + pr-eval: + name: PR eval ${{ matrix.plugin }} (narrow, treatment, N=3) + needs: preflight + if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && needs.preflight.outputs.run_evals == 'true' && needs.preflight.outputs.has_credentials == 'true' }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.preflight.outputs.matrix) }} + env: + RF_EVAL_PLUGIN: ${{ matrix.plugin }} + EVALS: plugins/${{ matrix.plugin }}/evals + runs-on: ubuntu-24.04 # ubuntu-latest (26.04) has no Playwright chromium yet + # A harness-wide PR selects the full narrow tier (66 sequential sessions) + # plus trigger evals for every changed description; 90 min was not enough. + timeout-minutes: 180 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up uv + uses: astral-sh/setup-uv@v3 + with: + enable-cache: true + cache-dependency-glob: | + pyproject.toml + uv.lock + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + # Claude Code requires Node >= 22; on Node 20 npm silently resolves + # the newest compatible release (2.1.197), not the current one. + node-version: "22" + + - name: Install Claude Code CLI + run: | + npm install -g @anthropic-ai/claude-code + claude --version + + - name: Install Python dependencies + run: uv sync --all-packages + + - name: Initialize Playwright browsers + run: uv run rfbrowser init + + - name: Doctor (grader tools for the narrow tier) + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + run: uv run rf-skill-eval doctor --tasks-dir ${EVALS}/tasks/narrow --strict + + - name: Run narrow tier (treatment arm) + id: batch + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + SKILLS: ${{ matrix.skills }} + run: | + set +e + SKILL_ARGS=() + if [ -n "${SKILLS}" ]; then SKILL_ARGS=(--skills "${SKILLS}"); fi + uv run rf-skill-eval run-batch \ + --tasks-dir ${EVALS}/tasks/narrow \ + --tier narrow \ + "${SKILL_ARGS[@]}" \ + --arms treatment \ + --runs 3 \ + --model "${HAIKU_MODEL}" \ + --max-cost-usd "${PR_NARROW_CAP_USD}" \ + --output "${RUNNER_TEMP}"/eval-runs/pr + echo "exit_code=$?" >> "$GITHUB_OUTPUT" + + - name: Gate against stored baseline + id: gate + if: always() + run: | + set +e + uv run rf-skill-eval gate \ + --runs-dir "${RUNNER_TEMP}"/eval-runs/pr \ + --baseline ${EVALS}/baselines/narrow.json \ + --tasks-dir ${EVALS}/tasks \ + --token-budget 0.75 \ + --max-cost-usd "${PR_NARROW_CAP_USD}" \ + --report-output gate.txt + echo "exit_code=$?" >> "$GITHUB_OUTPUT" + + - name: Trigger evals (validation split, changed descriptions) + id: triggers + if: ${{ always() && matrix.run_triggers == 'true' }} + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + TRIGGER_SKILLS: ${{ matrix.trigger_skills }} + run: | + set +e + uv run rf-skill-eval trigger \ + --skills "${TRIGGER_SKILLS}" \ + --split validation \ + --model "${HAIKU_MODEL}" \ + --concurrency 2 \ + --max-cost-usd "${PR_TRIGGER_CAP_USD}" \ + --output "${RUNNER_TEMP}"/eval-runs/triggers + code=$? + uv run rf-skill-eval gate \ + --trigger-results "${RUNNER_TEMP}"/eval-runs/triggers/trigger-results.json \ + --trigger-baseline ${EVALS}/baselines/triggers.json \ + --report-output trigger-gate.txt + gate=$? + if [ "$code" -ne 0 ]; then echo "exit_code=$code" >> "$GITHUB_OUTPUT"; else echo "exit_code=$gate" >> "$GITHUB_OUTPUT"; fi + + - name: Render report + if: always() + run: | + uv run rf-skill-eval report \ + --runs-dir "${RUNNER_TEMP}"/eval-runs/pr \ + --tasks-dir ${EVALS}/tasks \ + --baseline ${EVALS}/baselines/narrow.json \ + --format md \ + --output eval-report.md || echo "_no report (no runs recorded)_" > eval-report.md + { + echo '### Gate' + echo '```' + cat gate.txt 2>/dev/null || echo "gate did not run" + echo '```' + if [ -f trigger-gate.txt ]; then + echo '### Trigger gate' + echo '```' + cat trigger-gate.txt + echo '```' + cat "${RUNNER_TEMP}"/eval-runs/triggers/trigger-report.md 2>/dev/null || true + fi + echo + cat eval-report.md + } > pr-comment.md + cat pr-comment.md >> "$GITHUB_STEP_SUMMARY" + + - name: Upload evaluation artifacts + if: always() + # Agent-created workspace venvs/node_modules made this artifact 5 GB; + # transcripts, verdicts and configs are what matter for debugging. + uses: actions/upload-artifact@v4 + with: + name: skill-eval-pr-${{ matrix.plugin }}-${{ github.run_id }} + path: | + ${{ runner.temp }}/eval-runs/pr + ${{ runner.temp }}/eval-runs/triggers + eval-report.md + gate.txt + trigger-gate.txt + !${{ runner.temp }}/eval-runs/**/workspace/.venv + !${{ runner.temp }}/eval-runs/**/workspace/node_modules + # v4 silently strips dotfiles by default. We need .mcp.json, + # .claude/, and other hidden config artifacts visible for + # post-run debugging. + include-hidden-files: true + if-no-files-found: ignore + retention-days: 30 + + - name: Comment PR with report + if: always() + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const marker = ''; + let body; + try { + body = fs.readFileSync('pr-comment.md', 'utf8'); + } catch (err) { + body = 'Evaluation report missing (run may have failed before report generation).'; + } + const fullBody = `${marker}\n\n## Skill Evaluation: ${{ matrix.plugin }} (narrow, treatment, N=3)\n\n${body}`.slice(0, 65000); + const { owner, repo } = context.repo; + const issue_number = context.issue.number; + const { data: comments } = await github.rest.issues.listComments({ + owner, repo, issue_number, per_page: 100, + }); + const existing = comments.find(c => c.body && c.body.includes(marker)); + if (existing) { + await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body: fullBody }); + } else { + await github.rest.issues.createComment({ owner, repo, issue_number, body: fullBody }); + } + + - name: Fail on regression, incomplete runs or budget stop + if: always() + run: | + echo "run-batch exit: ${{ steps.batch.outputs.exit_code }}" + echo "gate exit: ${{ steps.gate.outputs.exit_code }}" + echo "trigger exit: ${{ steps.triggers.outputs.exit_code }}" + test "${{ steps.batch.outputs.exit_code }}" = "0" + test "${{ steps.gate.outputs.exit_code }}" = "0" + if [ -n "${{ steps.triggers.outputs.exit_code }}" ]; then + test "${{ steps.triggers.outputs.exit_code }}" = "0" + fi + + full-eval: + name: Full eval ${{ matrix.plugin }} (all tiers, both arms) + needs: preflight + if: ${{ github.event_name != 'pull_request' && needs.preflight.outputs.has_credentials == 'true' }} + strategy: + fail-fast: false + # One leg at a time: the weekly caps are per invocation, and parallel + # legs would multiply the spend and the rate-limit pressure. + max-parallel: 1 + matrix: + include: ${{ fromJSON(needs.preflight.outputs.matrix) }} + runs-on: ubuntu-24.04 # ubuntu-latest (26.04) has no Playwright chromium yet + timeout-minutes: 360 + env: + RF_EVAL_PLUGIN: ${{ matrix.plugin }} + EVALS: plugins/${{ matrix.plugin }}/evals + TIERS: ${{ github.event_name == 'workflow_dispatch' && inputs.tiers || 'narrow,realistic,adversarial,partial' }} + ARMS: ${{ github.event_name == 'workflow_dispatch' && inputs.arms || 'treatment,baseline' }} + RUNS: ${{ github.event_name == 'workflow_dispatch' && inputs.runs || '3' }} + MODEL: ${{ github.event_name == 'workflow_dispatch' && inputs.model || '' }} + ALLOW_OPUS: ${{ github.event_name == 'workflow_dispatch' && inputs.allow_opus || false }} + MANUAL_CAP: ${{ github.event_name == 'workflow_dispatch' && inputs.max_cost_usd || '' }} + RUN_TRIGGERS: ${{ github.event_name != 'workflow_dispatch' || inputs.triggers }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up uv + uses: astral-sh/setup-uv@v3 + with: + enable-cache: true + cache-dependency-glob: | + pyproject.toml + uv.lock + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + # Claude Code requires Node >= 22; on Node 20 npm silently resolves + # the newest compatible release (2.1.197), not the current one. + node-version: "22" + + - name: Install Claude Code CLI + run: | + npm install -g @anthropic-ai/claude-code + claude --version + + - name: Install Python dependencies + run: uv sync --all-packages + + - name: Initialize Playwright browsers + run: uv run rfbrowser init + + - name: Set API key fallback + if: ${{ inputs.use_api_key == true }} + run: echo "ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }}" >> "$GITHUB_ENV" + + - name: Doctor + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + run: uv run rf-skill-eval doctor --tasks-dir ${EVALS}/tasks --strict + + - name: Run tiers + id: batch + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + run: | + set +e + overall=0 + MODEL_ARGS=() + if [ -n "${MODEL}" ]; then MODEL_ARGS=(--model "${MODEL}"); fi + if [ "${ALLOW_OPUS}" = "true" ]; then MODEL_ARGS+=(--allow-opus); fi + for tier in ${TIERS//,/ }; do + case "$tier" in + narrow) cap="${WEEKLY_NARROW_CAP_USD}" ;; + realistic) cap="${WEEKLY_REALISTIC_CAP_USD}" ;; + adversarial) cap="${WEEKLY_ADVERSARIAL_CAP_USD}" ;; + partial) cap="${WEEKLY_PARTIAL_CAP_USD}" ;; + *) echo "unknown tier $tier"; exit 2 ;; + esac + if [ -n "${MANUAL_CAP}" ]; then cap="${MANUAL_CAP}"; fi + # Not every plugin has every tier (rf-desktop has narrow tasks only). + if [ ! -d "${EVALS}/tasks/${tier}" ]; then echo "no ${tier} tasks in ${EVALS}"; continue; fi + uv run rf-skill-eval run-batch \ + --tasks-dir "${EVALS}/tasks/${tier}" \ + --tier "${tier}" \ + --arms "${ARMS}" \ + --runs "${RUNS}" \ + "${MODEL_ARGS[@]}" \ + --max-cost-usd "${cap}" \ + --output "${RUNNER_TEMP}/eval-runs/full/${tier}" + code=$? + if [ "$code" -ne 0 ]; then overall=$code; fi + done + echo "exit_code=${overall}" >> "$GITHUB_OUTPUT" + + - name: Trigger evals (all sets, both splits) + id: triggers + if: ${{ always() && env.RUN_TRIGGERS == 'true' }} + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + run: | + set +e + cap="${WEEKLY_TRIGGER_CAP_USD}" + if [ -n "${MANUAL_CAP}" ]; then cap="${MANUAL_CAP}"; fi + MODEL_ARGS=() + if [ -n "${MODEL}" ]; then MODEL_ARGS=(--model "${MODEL}"); fi + if [ "${ALLOW_OPUS}" = "true" ]; then MODEL_ARGS+=(--allow-opus); fi + uv run rf-skill-eval trigger \ + --split train,validation \ + "${MODEL_ARGS[@]}" \ + --max-cost-usd "${cap}" \ + --output "${RUNNER_TEMP}"/eval-runs/full/triggers + echo "exit_code=$?" >> "$GITHUB_OUTPUT" + + - name: Candidate baseline and report + if: always() + run: | + uv run rf-skill-eval baseline update \ + --from "${RUNNER_TEMP}"/eval-runs/full \ + --tasks-dir ${EVALS}/tasks \ + --output-dir candidate-baseline || true + uv run rf-skill-eval report \ + --runs-dir "${RUNNER_TEMP}"/eval-runs/full \ + --tasks-dir ${EVALS}/tasks \ + --format md \ + --output eval-report.md || echo "_no report_" > eval-report.md + set +e + uv run rf-skill-eval gate \ + --runs-dir "${RUNNER_TEMP}"/eval-runs/full \ + --baseline ${EVALS}/baselines/narrow.json \ + --tasks-dir ${EVALS}/tasks \ + --report-output gate.txt + echo "GATE_EXIT=$?" >> "$GITHUB_ENV" + { + echo '### Gate vs committed baseline (informational)' + echo '```' + cat gate.txt 2>/dev/null + echo '```' + cat "${RUNNER_TEMP}"/eval-runs/full/triggers/trigger-report.md 2>/dev/null || true + cat eval-report.md + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload evaluation artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: skill-eval-full-${{ matrix.plugin }}-${{ github.run_id }} + path: | + ${{ runner.temp }}/eval-runs/full + eval-report.md + gate.txt + !${{ runner.temp }}/eval-runs/**/workspace/.venv + !${{ runner.temp }}/eval-runs/**/workspace/node_modules + include-hidden-files: true + if-no-files-found: ignore + retention-days: 30 + + - name: Upload candidate baseline + if: always() + uses: actions/upload-artifact@v4 + with: + name: candidate-baseline-${{ matrix.plugin }}-${{ github.run_id }} + path: candidate-baseline/ + if-no-files-found: ignore + retention-days: 90 + + - name: Open regression issue (weekly) + if: ${{ always() && github.event_name == 'schedule' && env.GATE_EXIT == '1' }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh issue create \ + --title "Skill eval weekly regression: ${{ matrix.plugin }} (run ${{ github.run_id }})" \ + --label "eval-regression" \ + --body "The weekly evaluation failed the gate against the committed baseline. + + $(cat gate.txt) + + Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + + - name: Fail on incomplete runs or budget stop + if: always() + run: | + test "${{ steps.batch.outputs.exit_code }}" = "0" + if [ -n "${{ steps.triggers.outputs.exit_code }}" ]; then + test "${{ steps.triggers.outputs.exit_code }}" = "0" + fi diff --git a/.github/workflows/update-latest.yml b/.github/workflows/update-latest.yml new file mode 100644 index 0000000..39fccf1 --- /dev/null +++ b/.github/workflows/update-latest.yml @@ -0,0 +1,26 @@ +name: Update Latest Branch + +on: + workflow_run: + workflows: ["CI"] + types: [completed] + branches: [main] + +permissions: + contents: write + +jobs: + update-latest: + name: Fast-forward latest to main + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Update latest branch + run: | + git checkout -B latest origin/main + git push origin latest --force + echo "Updated latest branch to $(git rev-parse --short HEAD)" diff --git a/.gitignore b/.gitignore index 6f596e1..b2bf3d6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,220 +1,56 @@ -# Byte-compiled / optimized / DLL files +# Python __pycache__/ -*.py[codz] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ +*.py[cod] *.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py.cover -*.lcov -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -# Pipfile.lock - -# uv -# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# uv.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -# poetry.lock -# poetry.toml - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. -# https://pdm-project.org/en/latest/usage/project/#working-with-version-control -# pdm.lock -# pdm.toml -.pdm-python -.pdm-build/ - -# pixi -# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. -# pixi.lock -# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one -# in the .venv directory. It is recommended not to include this directory in version control. -.pixi/* -!.pixi/config.toml - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule* -celerybeat.pid - -# Redis -*.rdb -*.aof -*.pid - -# RabbitMQ -mnesia/ -rabbitmq/ -rabbitmq-data/ - -# ActiveMQ -activemq-data/ - -# SageMath parsed files -*.sage.py +dist/ +build/ -# Environments -.env -.envrc -.venv -env/ +# Virtual environments +.venv/ venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs/Zensical documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -# .idea/ - -# Abstra -# Abstra is an AI-powered process automation framework. -# Ignore directories containing user credentials, local state, and settings. -# Learn more at https://abstra.io/docs -.abstra/ - -# Visual Studio Code -# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore that -# can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore -# and can be added to the global gitignore or merged into this file. However, if you prefer, you -# could uncomment the following to ignore the entire vscode folder -# .vscode/ -# Temporary file for partial code execution -tempCodeRunnerFile.py - -# Ruff stuff: -.ruff_cache/ +env/ -# PyPI configuration file -.pypirc +# Local secrets (do NOT commit) — use .env.example as template +.env -# Marimo -marimo/_static/ -marimo/_lsp/ -__marimo__/ +# Eval harness working state (generated, not committed) +eval/runs/ +eval/reports/ +# Anchored to repo root: the unanchored `results/` also matched the +# plugin skill dir `skills/results/` (pre-2.0 name, now rf-results), dropping its SKILL.md +# from the packaged wheel (hatchling honors .gitignore for the sdist). +/results/ -# Streamlit -.streamlit/secrets.toml +# Test output +.pytest_cache/ +# Robocop cache (hooks and documented commands pass --no-cache; older runs leave it behind) +.robocop_cache/ +output.xml +log.html +report.html +*.png + +# IDE +.idea/ +.vscode/ +*.swp +*.swo + +# OS +.DS_Store +Thumbs.db + +# Claude Flow state (local only) +.claude-flow/ +.swarm/ + +# Cross-agent skill discovery symlinks (local only) +.codex/ +.github/skills/ + +# robotframework-plugins installer: _assets/ is regenerated by the hatch build +# hook at `pip install` time from plugins/rf-agentskills/. Don't commit +# the duplicate. Developers run `pip install -e installer/` after clone +# to populate it. +installer/src/robotframework_plugins/_assets/ +.scanners/ diff --git a/.gitleaksignore b/.gitleaksignore new file mode 100644 index 0000000..a2a1055 --- /dev/null +++ b/.gitleaksignore @@ -0,0 +1,8 @@ +# Gitleaks findings reviewed as false positives (security reviewers approve +# changes to this file). Format: :::. +# +# A placeholder "sk-test-…" API key in a RESTinstance example, in two commits of +# the history migrated from the rf-agentskills repository. The example file was +# later removed. +9c718fa8b0747d1fdc026534b7e8f4868a388962:plugins/rf-agentskills/skills/restinstance/assets/examples/oauth-flow.robot:generic-api-key:115 +a9cae36c560073c7c3d74ecf3c5c1208ff72bb89:skills/robotframework-restinstance-skill/assets/examples/oauth-flow.robot:generic-api-key:115 diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 0000000..3944a2c --- /dev/null +++ b/.mcp.json @@ -0,0 +1,26 @@ +{ + "$comment": "Project-level MCP server registration. The rf-skill-eval harness generates its own per-run .mcp.json in an isolated profile dir; see src/rf_skill_eval/infrastructure/mcp/config_builder.py for the canonical rf-mcp command.", + "mcpServers": { + "claude-flow": { + "command": "npx", + "args": [ + "claude-flow@alpha", + "mcp", + "start" + ], + "env": { + "CLAUDE_FLOW_MODE": "v3", + "CLAUDE_FLOW_HOOKS_ENABLED": "true", + "CLAUDE_FLOW_TOPOLOGY": "hierarchical-mesh", + "CLAUDE_FLOW_MAX_AGENTS": "15", + "CLAUDE_FLOW_MEMORY_BACKEND": "hybrid" + }, + "autoStart": false + }, + "rf-mcp": { + "command": "uv", + "args": ["run", "robotmcp"], + "env": {} + } + } +} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..6aad4b6 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,171 @@ +# Contributing + +You can add a plugin for a library or tool that isn't covered yet, or improve one that is: fix a skill, add a reference, sharpen a subagent. This page covers how to do either, the checks your PR must pass, how review works, how to write the text, and which version to bump. + +## Add a plugin + +1. Copy `templates/plugin/` to `plugins//`. Plugin names start with `rf-` and describe the technology, for example `rf-database`. +2. Fill in `catalog.yaml`: a one-line summary, `tier: community`, `status: active`, yourself under `maintainers`, the libraries with the version ranges you checked, the Robot Framework range and the agents you tried. The schema is `schemas/catalog.schema.json`. +3. Rename the example skill to `rf-` (the folder name and the `name` in `SKILL.md` must match) and write it. Skill and subagent names must be unique across every plugin. +4. Add a line for your plugin to `.github/CODEOWNERS` with the same people as `maintainers`. +5. Regenerate the catalogue and the marketplace: `python scripts/build-catalog.py`. + +A new plugin starts in the `community` tier. [GOVERNANCE.md](GOVERNANCE.md) says what `verified` needs. + +## Change a plugin + +Edit the files under `plugins//`; that folder is the only copy of its content. Then regenerate what is generated from it: + +```bash +python scripts/build-agent-variants.py # after changing a subagent or hooks/hooks.json +python scripts/build-catalog.py # after changing a catalog.yaml, a version, or any plugin the bundle copies +``` + +A file that more than one plugin needs lives in one of them; the others get a generated copy. Plugins can't contain symlinks, because Codex drops them on install. + +## Checks a PR must pass + +CI runs these on every PR. Run them locally first: + +| Check | Command | +|---|---| +| Catalogue, names, symlinks, lifecycle | `python scripts/build-catalog.py --check` | +| Ownership and tier evidence | `python scripts/check-governance.py` | +| House content rules | `python scripts/check_content_rules.py` | +| Skill frontmatter | `python scripts/validate-skills.py --channel all` | +| Cross-plugin references | `python scripts/check-plugin-references.py` | +| Generated variants | `bash scripts/check-drift.sh` | +| Docs voice | `python scripts/check-docs-voice.py` | +| Tests | `python -m pytest tests/ --ignore=tests/eval` and `python -m pytest plugins//tests` | + +The security gate also scans the plugins your PR changes with Cisco skill-scanner, SkillSpector, cc-audit and Gitleaks. To run it locally: `scripts/install-scanners.sh`, then `scripts/security-scan.sh out/`. It fails on a new HIGH or CRITICAL finding. If a finding is a false positive, add an entry with a reason to `security/baselines/.yaml`; the security reviewers approve every baseline change. + +Live agent evals (`rf-skill-eval`) run on maintainer branches, after merge and weekly, never on code from forks. A maintainer runs them for your PR when it changes skill content. + +## Review + +The PR template has a checklist: purpose, supported versions, examples, security-relevant changes, tool permissions and documentation voice. Tick each item or write "n/a" with a reason. + +The `required-reviews` check waits for an approval from each group your PR concerns: + +- a maintainer of every plugin it changes; +- a Foundation reviewer, when one of those plugins is `verified` or the PR promotes one; +- a security reviewer, when it changes hooks, scripts, MCP configuration, manifests, subagent `tools`, skill `allowed-tools`, the workflows or `security/`. + +Your own approval doesn't count. + +## Writing + +Text for people (READMEs, these documents, skill references) follows the Robot Framework Voice: + +- Open with what the reader gets or can do. No mission statements. +- Use short sections with headings that say what's in them, and lists for steps and options. +- Use plain, inclusive words. Explain jargon the first time. +- Say what to do, not how great it is: "Install `rf-core`", not "Get started with our amazing tooling". +- Keep humour dry and rare, and never in install steps, error messages or security text. + +`scripts/check-docs-voice.py` catches the mechanical part: a list of banned phrases and more than one exclamation mark per document. Reviewers check the rest. + +`SKILL.md` files are written for agents: lead with when to use the skill, keep the description under 160 characters, and put depth in `references/`. + +## Versions + +There are three kinds of version, and as a plugin author you only ever bump one of them. + +| What | Example | Where it lives | Who bumps it | +|---|---|---|---| +| A plugin | `rf-core` 1.0.0, the `rf-agentskills` bundle 4.0.0 | its `plugin.json` | the plugin's maintainers | +| The installer | `robotframework-plugins` 2026.11.0 | `installer/pyproject.toml` | the release manager | +| The eval harness | `rf-skill-eval` 0.1.0 | root `pyproject.toml` | nobody outside CI; it is never published | + +A plugin has SemVer: bump `major` for a removed or renamed skill, `minor` for new content, `patch` for fixes. Each plugin counts on its own, which is why `rf-core` can be at 1.0.0 while the bundle is at 4.0.0: the bundle carries on the numbering of the single plugin it replaced. + +The installer uses calendar versions, because each release is a snapshot of the whole catalogue taken in that month. The version table in the README shows which plugin versions an installer release bundles: + +| robotframework-plugins 2026.11.0 bundles | Version | +|---|---| +| `rf-core` | 1.0.0 | +| `rf-web` | 1.0.0 | + +To bump your plugin, run one command and regenerate: + +```bash +scripts/bump-version.sh +python scripts/build-catalog.py +``` + +It writes `plugin.json` and the `metadata.version` of every skill in the plugin. Releases are described in [RELEASING.md](RELEASING.md). + +## Development + +### Layout + +``` +.claude-plugin/marketplace.json # The marketplace every plugin-capable agent reads (generated) +plugins// # One folder per plugin; the only copy of its content +├── .claude-plugin/plugin.json # The plugin's one version +├── catalog.yaml # Programme manifest: tier, status, maintainers, libraries +├── skills/rf-*/ # One folder per skill; folder name == SKILL.md name +│ ├── SKILL.md # Skill definition (loaded by agent) +│ ├── scripts/ # Python scripts (executed, not loaded) +│ └── references/ # Deep reference docs (loaded on demand) +├── scripts/ # Hook scripts (.mjs) +├── agents/ # Subagent definitions +├── hooks/ # Session and edit hooks +├── variants/ # Generated: Codex TOML subagents, OpenCode agents and JS plugin +├── tests/ # The plugin's own tests (not installed) +└── evals/ # Eval tasks, trigger sets and baselines (not installed) +bundles/rf-agentskills/ # Generated copy of the bundled plugins +schemas/catalog.schema.json # Schema of catalog.yaml +security/ # Scanner baselines and configuration +templates/plugin/ # Starter plugin +installer/ # The robotframework-plugins Python installer +tests/ # Shared tests: repository, catalogue, installer, eval harness +eval/ # Shared eval harness content: fixture projects and hidden graders +scripts/ # Build and check utilities +``` + +`scripts/export-skills.py` writes standalone copies for the release tarballs and Claude Desktop archives, with plain `scripts/x.py` commands instead of the plugin's `"${CLAUDE_SKILL_DIR}/scripts/x.py"`. + +### Tests + +The shared tests and each plugin's tests run separately, because plugins may reuse test file names: + +```bash +python -m pytest tests/ --ignore=tests/eval -v # shared: repository, catalogue, installer +python -m pytest plugins/rf-core/tests/ -v # one plugin +python -m pytest plugins/rf-core/tests/test_hook_scripts.py -v # needs robotcode[all] for the validation tiers +``` + +In a PR, CI runs the shared tests plus the tests of each plugin the change touches, and of the plugins that depend on it; a change outside `plugins/` runs every plugin's tests (`scripts/select-plugin-tests.py`). + +### Skill evaluations + +The eval harness (`rf-skill-eval`) grades each skill in controlled Claude Code sessions. See [docs/ci/usage.md](docs/ci/usage.md). Quick start: + +```bash +cp .env.example .env # add your CLAUDE_CODE_OAUTH_TOKEN +uv sync +uv run rfbrowser init +uv run rf-skill-eval doctor +bash scripts/eval-local.sh +``` + +It reads a plugin's eval content from `plugins//evals/`; pass `--plugin `, or `--plugin all` for every plugin. + +### Bundled scripts + +Skills run their Python scripts directly (`uv run python scripts/.py …`, or the project's interpreter). Each prints JSON: + +| Script (skill) | Purpose | +|---|---| +| `rf_conventions.py` (rf-language) | Report a project's Robot Framework conventions and version | +| `check_library.py` (rf-python-library) | Check the project's own Python keyword libraries: keywords, scope, API style and defect findings (runs in a subprocess, never imports user code into the agent) | + +### Try a plugin locally + +```bash +claude --plugin-dir ./plugins/rf-core --plugin-dir ./plugins/rf-web +``` + +Claude Desktop loads skills uploaded as ZIP files: build them with `python scripts/build-skill-zips.py --out dist/skill-zips`. diff --git a/GOVERNANCE.md b/GOVERNANCE.md new file mode 100644 index 0000000..3fa4099 --- /dev/null +++ b/GOVERNANCE.md @@ -0,0 +1,75 @@ +# Governance + +How plugins get in, who reviews what, what maintainers commit to, and how a plugin is deprecated, demoted or removed. + +## Tiers + +Every plugin is `community` or `verified`. The tier is in its `catalog.yaml` and in the catalogue. + +### community + +A plugin enters as `community` when: + +- its `catalog.yaml` is valid and names at least one maintainer; +- it passes the security gate (house content rules, scanners, reviewed baseline); +- every skill passes the static checks: frontmatter, description length, links that resolve, `.robot` examples that pass a dry run; +- every skill and subagent states its purpose and the libraries and versions it supports; +- it has a README. + +### verified + +A `verified` plugin meets every community criterion and also: + +- has at least two maintainers, one of them in the Foundation reviewers team; +- ships trigger query sets for each skill and eval tasks in the `rf-skill-eval` harness; +- has eval baselines measured on the current models, and passes the eval gate; +- was approved for the tier by a Foundation reviewer, in the PR that sets `tier: verified`. + +`scripts/check-governance.py` checks the evidence in CI. A plugin that lacks some of it can stay `verified` only under a dated entry in `.github/governance.yaml`, which names what's missing; after the date, the check fails. The launch plugins have such an entry until 2027-01-31, for their second maintainer. + +### What a tier doesn't promise + +A tier describes the evidence behind a plugin. It isn't a warranty: the checks and reviews make a harmful or broken plugin less likely, and they can't rule one out. See [SECURITY.md](SECURITY.md). + +## Review + +Every change to a plugin needs an approval from one of its maintainers (listed in `catalog.yaml` and `.github/CODEOWNERS`, which must agree). On top of that: + +| The PR | also needs | +|---|---| +| changes a `verified` plugin, or promotes one to `verified` | a Foundation reviewer (`@MarketSquare/robotframework-plugins-reviewers`) | +| changes security-relevant content: hooks, scripts, MCP configuration, manifests, subagent `tools`, skill `allowed-tools`, workflows, `security/` | a security reviewer (`@MarketSquare/robotframework-plugins-security`) | + +The `required-reviews` check enforces this; the author's own approval doesn't count. Reviewers work through the checklist in the PR template and don't approve while an item is unticked without a reason. + +## What maintainers commit to + +- A first response to issues labelled for their plugin within **14 days**. +- A first response to security reports about their plugin within **7 days**. +- A compatibility check of their plugin within **60 days** of each Robot Framework major release, and of each major release of a library listed in their `catalog.yaml`. + +## Lifecycle + +### Deprecation + +To deprecate a plugin, a PR: + +- sets `status: deprecated`, `deprecated_since` (the date) and `replacement` (a plugin, or a sentence when there is none) in `catalog.yaml`; +- adds a deprecation notice to the top of the plugin's README. + +The next release notes list it. A deprecated plugin stays installable and is tagged `deprecated` in the marketplace. + +### Removal + +A plugin can be removed once it has been deprecated for at least **two installer releases** and **90 days**; `scripts/build-catalog.py --check` refuses an earlier removal and names the earliest release and date. Removing it sets `status: removed`, maps its name to `null` in the marketplace `renames`, and is listed in the release notes of the installer release that stops bundling it. Claude Code then uninstalls it (`forceRemoveDeletedPlugins`), and the Python installer removes its files on the next upgrade, keeping and reporting files the user changed. + +### Demotion + +- A `verified` plugin whose maintainers miss the response targets for 90 days, or that fails the eval gate for two consecutive releases of the plugin, moves to `community`. +- A `community` plugin with no maintainer response for 180 days is deprecated. + +Each demotion is a PR that states the reason and stays open for at least 14 days before it is merged. The release notes list the tier change. The quarterly adoption report (`docs/reports/`) lists candidates; a candidate is a prompt for this decision, not the decision. + +### Handing over + +A maintainer steps down with a PR that removes them from `catalog.yaml` and CODEOWNERS. A plugin left without maintainers is deprecated unless a new maintainer is added within 30 days; the catalogue check refuses an active plugin with nobody listed. diff --git a/README.md b/README.md new file mode 100644 index 0000000..69608b3 --- /dev/null +++ b/README.md @@ -0,0 +1,312 @@ +# Robot Framework plugins + +Skills, subagents and hooks that help coding agents write, run and fix Robot Framework tests. They work with eight agents: Claude Code, GitHub Copilot CLI, GitHub Copilot in VS Code, OpenAI Codex, Cursor, OpenCode, Project Goose and Claude Desktop. + +Install `rf-core` for the Robot Framework language, environment setup and Python keyword libraries, then add the plugin for each library you test with: web, API, mobile or desktop. The plugins sit next to the RobotCode project's own [`robotcode` plugin](https://github.com/robotcodedev/robotframework-agent-plugins), which teaches agents the robotcode CLI. + +**Requires the robotcode CLI** (`robotcode[all]>=2.8`) in the project's Python environment. See [Prerequisites](#prerequisites). + +## Catalogue + + +| Plugin | Version | Tier | Status | What it covers | Libraries | Maintainers | +|---|---|---|---|---|---|---| +| `rf-agentskills` | 4.0.0 | verified | active | All Robot Framework plugins in one install (rf-core, rf-web, rf-api, rf-mobile, rf-desktop), as one plugin. | Robot Framework | [@manykarim](https://github.com/manykarim) | +| `rf-api` | 1.0.0 | verified | active | RequestsLibrary and RESTinstance skills for API tests. | RequestsLibrary, RESTinstance | [@manykarim](https://github.com/manykarim) | +| `rf-core` | 1.0.0 | verified | active | Robot Framework syntax, environment setup, Python keyword libraries, test design and debugging subagents, and the validation hooks. | Robot Framework | [@manykarim](https://github.com/manykarim) | +| `rf-desktop` | 1.0.0 | verified | active | PlatynUI skill for desktop UI tests. | PlatynUI | [@manykarim](https://github.com/manykarim) | +| `rf-mobile` | 1.0.0 | verified | active | AppiumLibrary skill for Android and iOS tests. | AppiumLibrary | [@manykarim](https://github.com/manykarim) | +| `rf-web` | 1.0.0 | verified | active | Browser Library and SeleniumLibrary skills for web tests. | Browser, SeleniumLibrary | [@manykarim](https://github.com/manykarim) | + +Tiers and what they mean: [GOVERNANCE.md](GOVERNANCE.md). Full details per plugin: [docs/catalog.md](docs/catalog.md). + +The Python installer bundles these plugin versions: + +| robotframework-plugins 2026.11.0 bundles | Version | Tier | +|---|---|---| +| `rf-agentskills` | 4.0.0 | verified | +| `rf-api` | 1.0.0 | verified | +| `rf-core` | 1.0.0 | verified | +| `rf-desktop` | 1.0.0 | verified | +| `rf-mobile` | 1.0.0 | verified | +| `rf-web` | 1.0.0 | verified | + + +## Install + +There are two ways to install, and both work at **user scope** (every project on your machine) or **project scope** (one repository, shared with your team): + +- **Plugin marketplace**: Claude Code, GitHub Copilot CLI, VS Code (Copilot), OpenAI Codex and Cursor all read this repository's `.claude-plugin/marketplace.json`. Add the marketplace `MarketSquare/robotframework-plugins` and install `rf-core` plus the technology plugins you need, or the `rf-agentskills` bundle. Updates arrive through the agent. +- **Python installer** (`robotframework-plugins`): copies the files into each agent's own folders. Use it for OpenCode, Goose and Claude Desktop, which have no plugin marketplace, or wherever you prefer plain files. It can also write the marketplace settings for you (`--mode plugin`). + +### Which plugins + +| Plugin | What it adds | +|---|---| +| `rf-core` | Robot Framework syntax (`rf-language`), environment setup (`rf-setup`), Python keyword libraries (`rf-python-library`); the `rf-test-architect`, `rf-debug-expert` and `rf-migration-guide` subagents; all hooks. Every other plugin needs it. | +| `rf-web` | Browser Library (`rf-browser`) and SeleniumLibrary (`rf-selenium`) | +| `rf-api` | RequestsLibrary (`rf-requests`) and RESTinstance (`rf-restinstance`) | +| `rf-mobile` | AppiumLibrary (`rf-appium`) | +| `rf-desktop` | PlatynUI (`rf-platynui`) | +| `rf-agentskills` | A bundle: a copy of all five, in one install, as before 4.0.0 | + +Install `rf-core` plus the plugins for the libraries you test with, for example `rf-core` and `rf-web` for a Browser project. Claude Code installs `rf-core` with any technology plugin. Copilot CLI and Codex don't install plugin dependencies, so install `rf-core` there too. The Python installer picks the plugins from the libraries your `pyproject.toml` or `requirements*.txt` declares. + +Install either the `rf-agentskills` bundle or separate plugins, not both, or each skill appears twice. Skill names are the same everywhere; only their prefix differs. With the bundle, skills keep the prefix they had up to 3.0.0 (`/rf-agentskills:rf-browser`). With separate plugins, each skill carries its plugin's prefix (`/rf-web:rf-browser`, `/rf-core:rf-language`), so update project instructions that name a prefixed skill when you switch. + +### Plugin marketplace, per agent + +| Agent (verified version) | Skills | Subagents | Hooks | Project scope (committed file) | +|---|---|---|---|---| +| **Claude Code** | yes | yes | yes | `.claude/settings.json` enables the plugin | +| **Copilot CLI** (1.0.91) | yes | yes | yes; only validation errors reach the model (see below) | `.claude/settings.json` or `.github/copilot/settings.json` loads the plugin, no install step | +| **VS Code Copilot** (1.142) | yes | yes in the Local harness; not offered in the Copilot harness | yes, with `chat.useHooks`; in the Local harness validation errors reach the model only at the end of the reply (see below) | `.claude/settings.json` recommends the plugin to the workspace | +| **Codex** (0.153.4, 0.161.0) | yes | from the installer | yes, after trusting them once with `/hooks` | none: each user runs the two `codex plugin` commands | +| **Cursor** (CLI 2026.10.01, verified from its plugin loader, not run live) | yes | yes | yes (Cursor converts them) | imports the plugin from Claude Code's `.claude/settings.json` | + +The plugin does not set a default main agent. The three subagents are available for delegation, and you can still choose one as your main agent yourself. + +**Project scope for your team.** Commit this `.claude/settings.json` (or let `robotframework-plugins install --mode plugin` merge it into the existing file): + +```json +{ + "extraKnownMarketplaces": { + "robotframework-plugins": { + "source": { "source": "github", "repo": "MarketSquare/robotframework-plugins" } + } + }, + "enabledPlugins": { + "rf-core@robotframework-plugins": true, + "rf-web@robotframework-plugins": true + } +} +``` + +Enable `rf-agentskills@robotframework-plugins` instead to get everything. Add `"ref"` to the source to pin the marketplace to a tag, or use `stable` for the latest release. Each plugin release is tagged `--v` (for example `rf-core--v1.0.0`); the older `v` tags install `rf-agentskills` up to 3.0.0. + +#### Claude Code + +```bash +claude plugin marketplace add MarketSquare/robotframework-plugins +claude plugin install rf-web@robotframework-plugins # brings rf-core; --scope user (default) | project | local +# or everything at once: claude plugin install rf-agentskills@robotframework-plugins +# recommended companion: the robotcode plugin +claude plugin marketplace add robotcodedev/robotframework-agent-plugins +claude plugin install robotcode@robotframework-agent-plugins +``` + +In a session, `/plugin marketplace add …` and `/plugin install …` do the same. `--scope project` writes the committed `.claude/settings.json` above. To try the in-tree plugins without a marketplace: `claude --plugin-dir ./plugins/rf-core --plugin-dir ./plugins/rf-web`. + +#### GitHub Copilot CLI + +```bash +copilot plugin marketplace add MarketSquare/robotframework-plugins +copilot plugin install rf-core@robotframework-plugins # Copilot CLI doesn't install dependencies +copilot plugin install rf-web@robotframework-plugins +# or everything at once: copilot plugin install rf-agentskills@robotframework-plugins +# recommended companion: the robotcode plugin +copilot plugin marketplace add robotcodedev/robotframework-agent-plugins +copilot plugin install robotcode@robotframework-agent-plugins +``` + +That is user scope. For project scope, commit `.claude/settings.json` (or `.github/copilot/settings.json` with the same keys) and list `rf-core` there too: Copilot CLI loads the plugins in that repository without an install step. Pin a release with its `ref` there. + +Copilot CLI passes only a hook's `decision`/`reason` to the model. So the model sees Robot Framework **validation errors** after an edit ("The edit was applied, but …"), but not the advisory context the other agents get: deprecation and formatting warnings, the skill hint on prompts, error hints after `robot` runs, and the session environment report. + +#### VS Code (GitHub Copilot) + +Plugins and hooks are preview features. In your **user** settings, enable them and add the marketplace: + +```json +{ + "chat.plugins.enabled": true, + "chat.useHooks": true, + "chat.plugins.marketplaces": ["MarketSquare/robotframework-plugins"] +} +``` + +Pin with `"MarketSquare/robotframework-plugins#rf-core--v1.0.0"`. Then install `rf-core` and the technology plugins you need (or the `rf-agentskills` bundle) from that marketplace in the Extensions view. For project scope, the committed `.claude/settings.json` recommends the plugin to everyone who opens the workspace (`chat.plugins.marketplaces` itself is a user-only setting). VS Code asks before running hook commands unless your chat approval mode allows them. The [RobotCode extension](https://marketplace.visualstudio.com/items?itemName=d-biehl.robotcode) already bundles the `robotcode` plugin for Copilot Chat, so it needs no separate install. + +VS Code 1.142 has two agent harnesses, with different gaps: + +- **Copilot** (agent host): validation errors reach the model, but the plugin's three subagents are not offered. +- **Local**: skills and subagents work, but VS Code sends the next model request without waiting for PostToolUse hooks. A validation error first appears only as the banner "A hook prevented chat from continuing". The plugin's Stop hook re-checks the file when the agent finishes its reply and hands the error to the model, which then gets another turn to fix it. + +#### OpenAI Codex + +```bash +codex plugin marketplace add MarketSquare/robotframework-plugins # pin: --ref rf-core--v1.0.0 +codex plugin add rf-core@robotframework-plugins # Codex doesn't install dependencies +codex plugin add rf-web@robotframework-plugins +# or everything at once: codex plugin add rf-agentskills@robotframework-plugins +# recommended companion: the robotcode plugin +codex plugin marketplace add robotcodedev/robotframework-agent-plugins +codex plugin add robotcode@robotframework-agent-plugins +``` + +Then, once, run `/hooks` in a Codex session and trust the plugin's hooks; Codex runs none of them before that. Codex plugins cannot ship subagents, so install the three TOML subagents with the installer: + +```bash +uvx robotframework-plugins install --agent codex --mode plugin # writes .codex/agents/*.toml, prints the commands +uvx robotframework-plugins install --agent codex --mode plugin --yes # also runs them +``` + +Codex has no committed project-scope plugin file: every user runs the `codex plugin` commands once. The subagents can be committed in `.codex/agents/`. + +#### Cursor + +Add the marketplace by its git URL, `https://github.com/MarketSquare/robotframework-plugins` (Customize → Plugins, or `cursor-agent plugin marketplace add …`), install `rf-core` and the technology plugins you need (or `rf-agentskills`) and choose project or user scope. Cursor keeps marketplaces on your Cursor account. It also imports the plugin when it is installed in Claude Code or enabled in `.claude/settings.json` (Settings → Third-Party Imports). Add `https://github.com/robotcodedev/robotframework-agent-plugins` the same way for the `robotcode` plugin. + +### Python installer (`robotframework-plugins`) + +Run it with zero install and let it walk you through the agents to set up: + +```bash +# Zero-install: bare `install` is interactive — multi-select of known agents, +# with the ones detected on your machine pre-checked. +uvx robotframework-plugins install # or: pipx run robotframework-plugins install + +# See what's detected first +uvx robotframework-plugins targets +``` + +Fully scriptable too — no prompt with an explicit selection, `--yes`, or a non-TTY stdin: + +```bash +robotframework-plugins install --agents all # every known agent +robotframework-plugins install --agents claude-code,cursor # explicit list +robotframework-plugins install --agents detected --yes # only detected, headless +robotframework-plugins install --agent claude-code # single (back-compat) +robotframework-plugins install --agent claude-code --mode plugin --ref rf-core--v1.0.0 # marketplace settings instead of files +``` + +Installs default to **project scope** (into the current directory, e.g. `./.claude/`); add `--scope user` for a global install under your home directory. Other commands: `uninstall`, `list`, `doctor`, `version`. Useful flags: `--scope project|user [--project DIR]`, `--mode files|plugin [--ref TAG]`, `--prefix DIR`, `--dry-run`, `--what skills,agents,hooks`, `--force`, `--no-input`. + +`--mode plugin` writes the marketplace entries into `.claude/settings.json` (Claude Code, Copilot, VS Code; Copilot also gets `.github/copilot/settings.json`, or `~/.copilot/settings.json` for user scope), installs the Codex subagents and prints the Codex commands, and prints the Cursor steps. Agents without a marketplace (OpenCode, Goose, Claude Desktop) fall back to files. + +A manifest (per-project under `/.rf-agentskills/`, or global for user scope) tracks every file written (hash + transform); `uninstall` removes only files whose hash still matches and only the hook config entries it added — user edits and other tools' hooks are preserved. Re-running `install` after an upgrade removes files the previous install wrote that the new bundle no longer ships (same hash check; user-modified files are kept), including the `rf-tools` MCP server entry and files that installers before content 2.0.0 registered. + +| Agent | What lands where (files mode) | Coverage | +|---|---|---| +| **Claude Code** ≥ 2.1 | `~/.claude/skills`, `~/.claude/agents`, `settings.json` hooks | full native | +| **GitHub Copilot** (VS Code ≥ 1.108) | reuses Claude Code paths (Copilot reads them natively) | full native | +| **OpenAI Codex** | `~/.agents/skills` per docs, `~/.codex/agents/*.toml`, `hooks.json` | full (hooks experimental, opt-in via `[features] codex_hooks=true`) | +| **Cursor** ≥ 2.4 | `~/.cursor/skills`, `~/.cursor/agents`, `hooks.json` (namespaced matchers) | full native | +| **OpenCode** | `.opencode/skills`, `.opencode/agents` (`mode: subagent`), `.opencode/plugins/rf-agentskills.js` + hook scripts (or `~/.config/opencode/…`) | skills, subagents; edit validation, error hints and the environment report through the JS plugin (prompt context and the Stop reminder are not ported) | +| **Project Goose** ≥ 1.25 | `~/.agents/skills` (Summon), `.goosehints` persona | skills; subagents folded into hints; hooks N/A | +| **Claude Desktop** | one upload-ready `rf-*.zip` per skill in `~/rf-agentskills-claude-desktop/` | skills via upload (Customize → Skills → Upload a skill; needs code execution); no subagents/hooks | + +Release notes, sha256 hashes, and the latest wheel + sdist are on the **[robotframework-plugins releases](https://github.com/MarketSquare/robotframework-plugins/releases)** page (tag prefix `robotframework-plugins-v*`) and on [PyPI](https://pypi.org/project/robotframework-plugins/). + +### Manual: copy skill folders + +Each skill is a self-contained folder under `plugins//skills/`. Copy what you need into the folder your agent reads skills from: + +```bash +cp -r plugins/rf-web/skills/rf-browser /.claude/skills/ +``` + +Agents read project skills from `.claude/skills/` (Claude Code, Copilot), `.agents/skills/` (Codex), `.cursor/skills/` (Cursor) and similar folders. Each release also attaches a tarball of standalone skill folders, with plain script paths. + +## Trust tiers + +Every plugin has a tier, shown in the catalogue above. + +- **verified**: the Foundation reviewers approved it. It has at least two maintainers, trigger tests and eval tasks for every skill, and eval baselines on the current models that it keeps passing. +- **community**: it passes the same automated checks (house content rules, the security scanners, frontmatter and example checks) and a maintainer review, without the eval evidence. + +A tier describes the evidence behind a plugin, not a warranty. The checks and reviews lower the risk of a harmful or broken plugin; they don't prove that one is safe. Plugins give instructions to an agent that can run commands on your machine, so read what you install, as you would any other dependency. [GOVERNANCE.md](GOVERNANCE.md) has the full criteria, and [SECURITY.md](SECURITY.md) says what the security checks cover. + +## No telemetry + +The installer, hooks and scripts send nothing anywhere. The installer only touches the network for the marketplace and package operations you ask for. To learn which plugins help, the project uses public signals (GitHub traffic, PyPI downloads, issues) and the [plugin feedback form](https://github.com/MarketSquare/robotframework-plugins/issues/new?template=plugin-feedback.yml). + +## What you get + +### 9 Skills + +| Skill | Type | Command (with the `rf-agentskills` bundle: `/rf-agentskills:`) | Description | +|-------|------|---------|-------------| +| Browser Library | library-reference | `/rf-web:rf-browser` | Web UI tests with Browser Library (Playwright); the default when no web library is chosen yet | +| SeleniumLibrary | library-reference | `/rf-web:rf-selenium` | Web UI tests with SeleniumLibrary (WebDriver, Selenium Grid) for suites that import `SeleniumLibrary` | +| AppiumLibrary | library-reference | `/rf-mobile:rf-appium` | Mobile app tests (Android/iOS native, hybrid, mobile web) with AppiumLibrary | +| RequestsLibrary | library-reference | `/rf-api:rf-requests` | HTTP/REST API tests with RequestsLibrary; the default when no API library is named | +| RESTinstance | library-reference | `/rf-api:rf-restinstance` | REST API tests with RESTinstance (`Library REST`): JSON Schema / OpenAPI-driven assertions | +| PlatynUI (preview) | library-reference | `/rf-desktop:rf-platynui` | Native desktop UI testing (Windows UIA, Linux AT-SPI2) via PlatynUI.BareMetal | +| Robot Framework language | language-guide + script | `/rf-core:rf-language` | Write and review tests (keyword-driven, data-driven, BDD), suites, `__init__.robot`, tags, user keywords, `.resource` and variable files in modern, version-gated syntax; `rf_conventions.py` script detects the project's conventions (requires robotframework). Not for Python keyword libraries | +| Python keyword libraries | library-guide + script | `/rf-core:rf-python-library` | Write and fix keyword libraries and listeners in Python: `@library`/`@keyword`, scope by state lifetime (TEST/SUITE/GLOBAL), type-hint conversion and custom converters, failures and logging, dynamic/hybrid APIs, listener API v3, libdoc; `check_library.py` script finds "contains no keywords", leaked imports, lost signatures and state kept in `TEST` scope before tests run (requires robotframework, Python 3.10+) | +| Setup | setup-guide | `/rf-core:rf-setup` | Install Robot Framework and libraries with uv, venv + pip or Poetry; fix environment errors (`ModuleNotFoundError`, wrong interpreter); project layout, CI | + +### 3 Specialized Agents + +| Agent | Purpose | +|-------|---------| +| RF Test Architect | Plan test suites, select libraries, design project structure | +| RF Debug Expert | Read-only diagnosis of failing or flaky runs: groups failures by cause, classifies them and returns a fix per cause (it does not edit files) | +| RF Migration Guide | Upgrade RF versions, migrate between libraries | + +### Automated Hooks + +- **Post-save validation**: Checks every written `.robot`/`.resource` file with `robotcode analyze code` (project environment first): syntax, argument and import errors are fed back to the agent; unknown keywords and variables are reported, and block only if still unresolved when the agent finishes; Robocop adds advice: deprecated syntax (`[Return]`, `Run Keyword If`, `Force Tags`, …) is reported as a **non-blocking warning** (`RF_AGENTSKILLS_DEPRECATION_CHECK=warn|off`); an opt-in per-file dry run (`RF_AGENTSKILLS_FILE_DRYRUN=1`). See `plugins/rf-core/hooks/README.md`. +- **Skill routing**: Routes RF-related prompts to the appropriate skill (short routing text, ≤ 450 characters) +- **Environment check**: Checks for Robot Framework, robotcode (≥ 2.8) and the RF packages at session start (install advice via rf-setup) +- **Test reminder**: Reminds you to run tests when the session ends + +## Prerequisites + +- A supported coding agent (see the install matrix above) +- **Python 3.10+** and **robotframework** in the project environment +- **robotcode[all] 2.8 or later** in the project environment. The skills run `robotcode doc`, `robotcode analyze code`, `robotcode results` and `robotcode robot-debug`, and the validation hooks run `robotcode analyze code`. Install it next to Robot Framework, never with `uvx`, `pipx` or `uv tool` (an isolated robotcode cannot see the project's libraries): + + ```bash + uv add --dev "robotcode[all]>=2.8" # uv + poetry add --group dev "robotcode[all]>=2.8" # Poetry + pip install "robotcode[all]>=2.8" # venv + pip (venv active) + ``` + + Why `[all]`: the base package only has `config` and `profiles`; the skills need the `runner` (`robot`, `discover`, `results`), `analyze` and `repl` (`doc`, `robot-debug`) extras, and the hooks need Robocop, which only `[all]` brings in as well. A CI job that only runs tests can use `robotcode[runner]`. +- **Recommended:** the `robotcode` plugin (`robotcode@robotframework-agent-plugins`) next to rf-agentskills. It owns running, discovery, keyword documentation, result analysis, static analysis, `robot.toml` configuration, the REPL and the debugger; rf-agentskills owns Robot Framework syntax, Python keyword libraries, the test-library skills, environment setup, the validation hooks and the design, diagnosis and migration subagents. +- New to Robot Framework setup? The `setup` skill (`rf-setup`) walks an agent through installing it into a project environment (uv preferred; it also covers venv + pip and Poetry). + +```bash +uv add robotframework # pip alternative (inside the project venv): pip install robotframework +``` + +All scripts handle a missing `robotframework` package gracefully and report how to install it. + +Optional libraries (for their respective skills), installed into the project environment -- same commands as the `rf-setup` skill's library table: +```bash +uv add "robotframework-browser[bb]" && uv run rfbrowser install chromium # Browser skill (no Node.js needed) +uv add robotframework-seleniumlibrary # Selenium skill (Selenium Manager fetches drivers) +uv add robotframework-appiumlibrary # Appium skill (+ npm Appium server and drivers) +uv add robotframework-requests # Requests skill +uv add RESTinstance # RESTinstance skill (Python 3.11+) +uv add --prerelease allow robotframework-PlatynUI # PlatynUI skill (preview; Python 3.12+; a plain install gets old 0.9.2) +``` + +## Compatibility + +- **Robot Framework** 7+ (uses modern syntax: RETURN, IF/ELSE, TRY/EXCEPT) +- **Python** 3.10+ (installer, skill scripts and robotcode) +- **robotcode** 2.8+ with the `all` extra, in the project environment +- **Coding agents** (see the install matrices above): Claude Code ≥ 2.1, GitHub Copilot CLI (plugin verified with 1.0.91), GitHub Copilot in VS Code ≥ 1.108 (plugin verified with 1.138 and 1.142), OpenAI Codex (plugin verified with 0.153.4 and 0.161.0), Cursor ≥ 2.4 (plugin loader checked in CLI 2026.10.01), OpenCode, Project Goose ≥ 1.25, Claude Desktop (skills via ZIP upload) + +## Upgrading from rf-agentskills + +Up to 3.0.0 everything shipped as one plugin, `rf-agentskills`, from a personal repository, with the installer on PyPI as `rf-agentskills`. Both moved here: + +- **Marketplace:** add `MarketSquare/robotframework-plugins` and install the plugins you need, or the `rf-agentskills` 4.0.0 bundle to keep everything as it was. Old pins to `v3.0.0` keep working from the archived repository. +- **Installer:** run `uvx robotframework-plugins install`. It finds files the old installer wrote, moves them to the new layout and keeps the ones you edited. `rf-agentskills` 0.9.0 on PyPI forwards to it. + +The [rf-agentskills changelog](plugins/rf-agentskills/CHANGELOG.md) lists the skills renamed in 2.0 and removed in 3.0. + +## More + +- [CONTRIBUTING.md](CONTRIBUTING.md): add or change a plugin, run the checks and tests, and which version to bump. +- [GOVERNANCE.md](GOVERNANCE.md): tiers, review, maintainer expectations, deprecation and removal. +- [SECURITY.md](SECURITY.md): report a vulnerability privately; what the checks cover and what they can't. +- [RELEASING.md](RELEASING.md): plugin and installer releases. + +## License + +Apache-2.0 diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 0000000..941321a --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,65 @@ +# Releasing + +Two kinds of release come out of this repository: a **plugin release**, which publishes one plugin's new version through the marketplace, and an **installer release**, which publishes `robotframework-plugins` to PyPI with a snapshot of the catalogue. They are independent: a plugin can release many times between installer releases. + +## Versions + +| What | Version scheme | Source of truth | Tag | +|---|---|---|---| +| Each plugin (`rf-core`, `rf-web`, …, the `rf-agentskills` bundle) | SemVer | `plugins//.claude-plugin/plugin.json` | `--v`, for example `rf-core--v1.0.0` | +| The installer `robotframework-plugins` | CalVer `YYYY.M.PATCH` | `installer/pyproject.toml` | `robotframework-plugins-v`, for example `robotframework-plugins-v2026.11.0` | +| `rf-skill-eval` (internal) | SemVer | root `pyproject.toml` | none; never published | + +The marketplace entries carry no version: Claude Code reads it from `plugin.json`. `CONTRIBUTING.md` ("Versions") explains which number to bump. + +Older tags stay where they are: `v1.1.0` to `v3.0.0` are `rf-agentskills` content releases, and `rf-agentskills-v0.3.0` to `rf-agentskills-v0.8.0` are the installer under its old name. Old marketplace pins to them keep working. + +## Plugin release + +1. Bump the version and regenerate: + ```bash + scripts/bump-version.sh + python scripts/build-catalog.py + ``` + A change to a bundled plugin also changes the `rf-agentskills` bundle's copy; release the bundle too (`scripts/bump-version.sh rf-agentskills …`). +2. Add the release to `plugins//CHANGELOG.md`. +3. Merge the PR. Then tag the merge commit and push the tag: + ```bash + git tag --v && git push origin --v + ``` +4. `release.yml` checks that the tag matches the plugin's `plugin.json`, builds the plugin's archives, attaches them to a GitHub release and moves the `stable` and `latest` branches. + +A pushed tag is the ref users pin a marketplace to: never move or delete one. + +## Installer release + +The installer version is the year and month of the release, then a patch number: the first release, in November 2026, is `2026.11.0`, a fix in the same month `2026.11.1`. + +1. Set `version` in `installer/pyproject.toml` and `__version__` in `installer/src/robotframework_plugins/__init__.py`, and add the release to `installer/CHANGELOG.md`. +2. Regenerate the version table in the README and `docs/catalog.md`: `python scripts/build-catalog.py`. +3. Merge the PR, then tag and push `robotframework-plugins-v`. +4. `release.yml` checks that the tag matches `pyproject.toml` and the current month, builds the wheel and sdist, and creates the GitHub release. Its notes start with the table of bundled plugin versions (`python scripts/build-catalog.py --version-table`), then the changelog section. +5. The publish job uploads to PyPI with **trusted publishing**: PyPI accepts the upload from this repository's workflow, in the protected `pypi` environment, without a token. Nobody handles a PyPI token. A maintainer approves the `pypi` environment's deployment in the workflow run. +6. Check the published package: + ```bash + uvx --refresh robotframework-plugins@ version + ``` + +A release candidate (`2026.11.0rc1`) goes through the same steps; it publishes to TestPyPI first, to check the flow. + +## What the version table says + +Each installer release bundles the active and deprecated plugins at their current versions. `build-catalog.py` writes that table into the README and `docs/catalog.md`, and `robotframework-plugins catalog --json` and `robotframework-plugins version` print it: + +```console +$ robotframework-plugins version +robotframework-plugins 2026.11.0 +bundled plugins: rf-agentskills 4.0.0, rf-api 1.0.0, rf-core 1.0.0, rf-desktop 1.0.0, rf-mobile 1.0.0, rf-web 1.0.0 +``` + +So a user who reports a problem with "the installer from December" can be told exactly which plugin versions they have. + +## Release notes + +- A plugin release lists its changes from the plugin's changelog. +- An installer release starts with the version table, then lists the installer changes, and names every plugin that was added, deprecated, demoted or removed since the previous installer release ([GOVERNANCE.md](GOVERNANCE.md)). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..358a50d --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,33 @@ +# Security + +## Report a vulnerability + +Use GitHub's private vulnerability reporting: on the repository's **Security** tab, choose **Report a vulnerability**. Please don't open a public issue for it. + +You'll get a first response within **7 days**. Tell us the plugin and version, the agent you used, what happens, and the steps to reproduce it. + +Report here: + +- a plugin that makes an agent do something you didn't ask for: run commands, read secrets, send data out, change files outside the task; +- a hook or script that can be made to run unintended commands; +- a weakness in the installer, the security checks or the CI workflows. + +## What runs on your machine + +Plugins are instructions and code for an agent that can run commands with your permissions. Skills and subagents tell the agent what to do; hooks and scripts run directly when the agent edits files or finishes a task. Read what you install, as you would any other dependency. + +The installer, hooks and scripts send nothing anywhere: no usage data, no identifiers, no content. The installer only touches the network for the marketplace and package operations you ask for. + +## What the checks cover + +Every pull request, including those from forks, runs a security gate without secrets: + +- the house content rules (`scripts/check_content_rules.py`): hidden Unicode characters, HTML comments in skills and subagents, subagents without an explicit tool list, unrestricted shell access in `allowed-tools`, downloads piped into a shell, MCP servers without an exact version; +- Cisco skill-scanner, SkillSpector (without its model pass), cc-audit and Gitleaks, at pinned versions; +- a comparison with each plugin's reviewed baseline: a new HIGH or CRITICAL finding fails the PR. + +Scans that send content to a model run only after a maintainer applies the `security-deep-scan` label, and on the main branch. Live agent evaluations, which run plugin hooks with an API key, never run code from forks. Changes to hooks, scripts, MCP configuration, manifests, tool permissions and the baselines need an approval from the security reviewers. + +## What the checks can't promise + +Passing the checks and the review doesn't prove a plugin is safe. Scanners match patterns and miss what they don't know; an instruction can be harmful in plain words; a reviewer can overlook something. A plugin's tier ([GOVERNANCE.md](GOVERNANCE.md)) describes the evidence behind it, not a warranty. diff --git a/bundles/rf-agentskills/.claude-plugin/plugin.json b/bundles/rf-agentskills/.claude-plugin/plugin.json new file mode 100644 index 0000000..003f9bd --- /dev/null +++ b/bundles/rf-agentskills/.claude-plugin/plugin.json @@ -0,0 +1,25 @@ +{ + "name": "rf-agentskills", + "version": "4.0.0", + "description": "All Robot Framework plugins in one install: rf-core, rf-web, rf-api, rf-mobile and rf-desktop.", + "author": { + "name": "manykarim", + "email": "manykarim@users.noreply.github.com" + }, + "homepage": "https://github.com/MarketSquare/robotframework-plugins", + "repository": "https://github.com/MarketSquare/robotframework-plugins", + "license": "Apache-2.0", + "keywords": [ + "robotframework", + "testing", + "automation", + "web-testing", + "api-testing", + "mobile-testing", + "playwright", + "selenium", + "appium", + "requests", + "restinstance" + ] +} diff --git a/bundles/rf-agentskills/CHANGELOG.md b/bundles/rf-agentskills/CHANGELOG.md new file mode 100644 index 0000000..681d8aa --- /dev/null +++ b/bundles/rf-agentskills/CHANGELOG.md @@ -0,0 +1,198 @@ +# Changelog + +This is the version of the `rf-agentskills` plugin: since 4.0.0 a bundle of +the five Robot Framework plugins, before that the plugin with all the content. The `rf-agentskills` Python +installer is versioned independently; see `RELEASING.md` at the repo root +for the policy. (Until 2.0.0 this file lived in `vscode-extension/`.) + +## 4.0.0 (unreleased) + +The split: the content now lives in five plugins, `rf-core`, `rf-web`, `rf-api`, `rf-mobile` and `rf-desktop` (each 1.0.0), and `rf-agentskills` ships a copy of all five, generated for each release. Existing installs keep the same nine skills, three subagents and hooks, under the same `rf-agentskills:` prefix. + +- **Install only what you test.** Install `rf-core` plus the technology plugins you need instead of this bundle. Their skills carry their own prefix, for example `rf-web:rf-browser` instead of `rf-agentskills:rf-browser`. +- **Not both.** With the bundle and a technology plugin installed, each of its skills appears twice. +- **Content changes**, the same as in the five plugins (see their changelogs): each library skill now has `references/install.md` with its install steps, which used to be in `rf-setup`. The SeleniumLibrary → Browser and RequestsLibrary → RESTinstance keyword maps moved from `rf-migration-guide` to `rf-browser` and `rf-restinstance`. +- **Why a copy and not plugin dependencies:** Claude Code doesn't install the dependencies that an update adds, so a 3.0.0 install would have lost every skill on update. Copilot CLI and Codex don't install plugin dependencies at all. + +## 3.0.0 (2026-11-07) + +Breaking content release: robotcode is now a documented prerequisite, and the skills that overlapped with it are gone. + +- **Requires the robotcode CLI.** Install `robotcode[all]>=2.8` in the project environment (`uv add --dev "robotcode[all]>=2.8"`; Poetry and pip forms in the README and `rf-setup`). `[all]` is needed because it alone brings the `runner`, `analyze` and `repl` extras and Robocop. No plugin dependency is declared: rf-agentskills loads with or without the RobotCode project's `robotcode` plugin, which the README recommends as a companion (install commands per agent; the RobotCode VS Code extension already bundles it). +- **Removed skills:** `rf-robotcode` (use the `robotcode` skill of `robotcode@robotframework-agent-plugins`), `rf-libdoc` and its `rf_libdoc.py` (use `robotcode doc keywords [pattern]` / `robotcode doc keyword ""`), `rf-results` and its `rf_results.py` (use `robotcode results summary|show|log|stats|diff`; for a CI artifact outside any project: `uvx --from "robotcode[runner]" --with robotframework robotcode results summary -o output.xml`). Their Claude Desktop archives are gone too. 9 skills remain. +- **Removed subagent:** `rf-keyword-consultant` (use `robotcode doc`, or the REPL's `.kw`). +- **`rf-debug-expert` is read-only.** It reads the recorded run (`robotcode results`), groups failures by cause, classifies each cause, debugs one representative test with `robotcode robot-debug -bl`, and returns a FAILURE / CATEGORY / ROOT CAUSE / EVIDENCE / FIX / PREVENTION block per cause instead of editing files (`tools: Read, Grep, Glob, Bash`; OpenCode `permission: edit: deny`). It never proposes weakening an assertion, expected value or tag. +- **Subagents** have situation-based descriptions, and the shared verification loop uses `robotcode doc keyword`, `robotcode analyze code`, Robocop, a longname run (`robotcode robot -bl`) and `robotcode results`. +- **Skills use robotcode inline.** The library skills, `rf-language` and `rf-python-library` look keywords up with `robotcode doc`, check files with `robotcode analyze code`, run tests by longname and read results with `robotcode results`; Companion Skills tables no longer list the removed skills. Every skill's `compatibility` names `robotcode[all]>=2.8`. `rf-setup` installs robotcode as a required step and verifies with `robotcode discover info` and `robotcode analyze code`. +- **Validation hook checks errors with robotcode.** `validate_robot.mjs` runs `robotcode --format json analyze code ` (with Robocop's deprecation and formatting checks in parallel; about 1 s per edit with a warm `.robotcode_cache/`). Syntax, argument and import errors block (exit 2); unknown keywords and variables are advisory on edit and block at Stop if they are still unresolved, so writing a keyword in a second file is not interrupted. Robocop no longer decides what blocks. The edit hook timeout is 60 s. +- **Project tier** (`RF_AGENTSKILLS_PROJECT_VALIDATION`) runs `robotcode analyze code --collect-unused` instead of `robot --dryrun` and `robotframework-find-unused`. +- **Context hooks:** SessionStart reports the robotcode version (and that 2.8 is needed); the prompt hint, the `robot` error hints and the end-of-session reminder use `robotcode doc` / `robotcode results` instead of the removed skills. +- **rf-requests:** new gotcha: RequestsLibrary logs credentials and token headers at INFO. + +- **One plugin marketplace for every agent.** The repository's `.claude-plugin/marketplace.json` now serves Claude Code, GitHub Copilot CLI, VS Code (Copilot), Codex and Cursor; the README has the install, scope and pinning steps per agent. Pin a release with its `v` tag. +- **VS Code extension retired.** No `.vsix` is built or attached to releases any more. Instead, VS Code users add `manykarim/robotframework-agentskills` to `chat.plugins.marketplaces` (user scope), accept the workspace recommendation from a committed `.claude/settings.json` (project scope), or run `rf-agentskills install --agent copilot` (files). Unlike the extension, the plugin also brings subagents and hooks. +- **No default main agent.** The plugin's `settings.json` (`"agent": "rf-test-architect"`) is removed, so enabling the plugin no longer replaces your main agent. The four subagents remain available for delegation; to keep `rf-test-architect` as your main agent, select it yourself (for example `"agent"` in your own Claude Code settings). +- **Hooks read every agent's input.** `validate_robot.mjs` and `rf_error_hints.mjs` accept the tool names and input shapes of Claude Code, Copilot CLI, VS Code (`create_file`, `replace_string_in_file`, `apply_patch`, `run_in_terminal`), Codex (`apply_patch`, `Bash`) and Cursor, validate every `.robot`/`.resource` file a patch touches, and exit silently for other tools. Validation errors are now also reported as `{"decision": "block", "reason": ...}`, the only hook output Copilot CLI passes to its model. +- **Unfixed validation errors are re-checked at Stop.** A new Stop hook, `report_pending_errors.mjs`, re-validates the `.robot`/`.resource` files whose validation failed in the session and are still broken, and hands the errors to the model for one more turn. This covers VS Code's Local agent harness, which sends the next model request without waiting for PostToolUse hooks. Each content version of a file is reported once, and the hook is silent on `stop_hook_active`. +- **Generated per-agent variants** in `variants/` (from `scripts/build-agent-variants.py`, drift-checked in CI): Codex TOML subagents, OpenCode subagents (`mode: subagent`) and an OpenCode JS plugin that runs the same validation, error-hint and environment hooks. + +## 2.0.0 (never tagged; shipped in 3.0.0) + +- Skill descriptions trimmed to fit Claude Code 2.1.288's smaller skill listing (all 12 visible, 1210 characters) and sibling cues added (rf-appium/rf-selenium/rf-restinstance → rf-setup for installs and drivers, rf-libdoc/rf-results → rf-robotcode when robotcode is installed). + +- **`rf-tools` MCP server removed** from the content bundle: the skills run their scripts directly, so no MCP tools (`rf_libdoc_search`, `rf_libdoc_explain`, `rf_results_analyze`, `rf_conventions`, `rf_check_library`) are shipped. The VS Code extension never shipped the server; this only affects the Claude Code plugin and the installer. + +- rf-language: concrete embedded-argument recipe (`${team:\S+}`, text between arguments is literal) and the "No keyword with name … for an `[Arguments]` keyword" gotcha. + +Breaking content release. Later changes in this release cycle add their +entries here. + +- **rf-language / rf-setup:** every documented `robocop check` / `robocop + format` command passes `--no-cache` (Robocop otherwise leaves a + `.robocop_cache/` directory in the project). +- **rf-appium:** duplicate H1 heading removed. +- **New skill `rf-language`:** writing and reviewing Robot Framework test + cases (keyword-driven, data-driven, BDD), suites and `__init__.robot`, + tags and selection, user keywords and arguments, embedded arguments, + variables, resource and variable files, with a version gate, verified + gotchas, a legacy-to-modern migration table (Robocop 9 rule IDs) and + examples labelled "RF 7.0+". Its bundled `scripts/rf_conventions.py` + detects the project's conventions. The Companion Skills tables of + rf-setup, rf-robotcode and the six library skills link to it. +- **New skill `rf-python-library`:** writing and fixing Robot Framework + keyword libraries and listeners in Python (template, scope by state + lifetime per the RF User Guide, conversion, failures and logging, + dynamic/hybrid APIs, listener API v3, libdoc), with a version gate, + thirteen verified gotchas and examples labelled "RF 7.0+". Its bundled + `scripts/check_library.py` checks project libraries for RF-specific defects + before tests run. rf-language's description and Companion Skills table and + rf-setup's project layout point to it. +- **Removed skills:** `rf-keyword-builder`, `rf-testcase-builder`, + `rf-resource-architect` (and in the Claude Code plugin the slash commands + `/rf-agentskills:keyword-builder`, `:testcase-builder`, + `:resource-architect`), plus their scripts `keyword_builder.py`, + `testcase_builder.py`, `resource_architect.py`. +- **Removed MCP tools** (`rf-tools` server): `rf_keyword_builder`, + `rf_testcase_builder`, `rf_resource_architect`. Calling them returns the + normal unknown-tool error. +- **Why:** the JSON → script → paste round trip cost more turns and tokens + than writing Robot Framework directly, and the scripts lagged the language + (no RF 7 typed arguments, unimplemented layout modes). +- **Use instead:** write keywords, test cases and resource files directly; + check keyword names and arguments with `rf-libdoc` + (or `robotcode libdoc`), and validate with + `robot --dryrun`. The companion tables of the library skills, `rf-setup` + and `rf-robotcode` no longer list the generators; the subagents follow the + write-directly workflow. +- **Upgrade notes:** the VS Code extension and the Claude Code plugin replace + the skill tree on update. `rf-agentskills` installer users re-run + `rf-agentskills install` (0.7.0 prunes the old copies). Skills-tarball + users delete `rf-keyword-builder/`, `rf-testcase-builder/` and + `rf-resource-architect/` from their skills directory by hand. To keep the + generators, stay on 1.2.0. +- **Merged skills:** `rf-libdoc-search` and `rf-libdoc-explain` are + replaced by one skill, `rf-libdoc` (Claude Code plugin: slash command + `/rf-agentskills:rf-libdoc` replaces `:libdoc-search` and `:libdoc-explain`). + It finds keywords for a use case, explains a keyword's arguments (with + search fallback) and lists a library's keywords, and points to + `robotcode libdoc` when robotcode is installed. `rf_libdoc.py` flags and + JSON output are unchanged; the MCP tools `rf_libdoc_search` and + `rf_libdoc_explain` keep their names. The script ships as one regular file + (no symlink). **Upgrade:** the extension and plugin replace the skill tree; + installer users re-run `rf-agentskills install`; users who copied skill + folders by hand delete `rf-libdoc-search/` and `rf-libdoc-explain/`. +- **One identifier per skill (Claude Code plugin, installer, tarballs).** + Every skill's folder name now equals its `name` (`rf-`) in every + channel. The VS Code extension already used these names, so **VS Code users + see no change**. Elsewhere: repo/tarball folders `robotframework-browser-skill`, + `-selenium-skill`, `-appium-skill`, `-requests-skill`, `-restinstance-skill`, + `-platynui-skill`, `-robotcode-skill`, `-setup-skill` and + `robotframework-results` become `rf-browser`, `rf-selenium`, `rf-appium`, + `rf-requests`, `rf-restinstance`, `rf-platynui`, `rf-robotcode`, `rf-setup` + and `rf-results`; the Claude Code plugin folders and slash commands change + from the short names (`/rf-agentskills:browser`, `:setup`, `:results`, + `:libdoc`, …) to `/rf-agentskills:rf-browser`, `:rf-setup`, `:rf-results`, + `:rf-libdoc`, …. Installer users re-run `rf-agentskills install` (old + folders are pruned); tarball users delete the old folders by hand + (`rf-agentskills doctor` lists them). +- Every `SKILL.md` declares `license`, `compatibility` (runtime needs) and + `metadata` (`author`, `version`); CI validates all channels against the + agentskills.io frontmatter rules (`scripts/validate-skills.py`). +- **Skill scripts run in the project environment.** `rf-libdoc` and + `rf-results` now document `uv run python scripts/rf_libdoc.py …` / + `uv run python scripts/rf_results.py …` (paths relative to the skill folder) + plus one fallback line for non-uv projects (`.venv/bin/python` / + `poetry run python`, see rf-setup). Commands copied from 1.x + (`python scripts/…`) still work but use whatever `python` is on PATH. +- **Script CLI contract:** exit codes `0` ok, `1` internal error, `2` usage, + `3` Robot Framework missing or older than 7, `4` input not found or not + loadable (previously `1` for all failures); diagnostics are `error:` / + `warning:` / `hint:` lines on stderr (no JSON error on stderr, no + `pip install` advice); `--help` works without Robot Framework and ends with + examples. Both scripts carry PEP 723 metadata (`robotframework>=7`). +- **Bounded output:** `rf_libdoc.py --max-doc-chars` (default 4000, `0` = full + text, truncation marker inside `doc`); `rf_results.py --limit` (default 50) + caps `details` test entries (failed first) and each `errors` list and reports + `omitted` counts; both scripts accept `--json-out FILE`. +- **Sharper skill descriptions:** every skill's `description` now opens with + what it does and for which library or tool, has a `Use when` clause with the + terms users type (`Library Browser`, `output.xml`, `robot.toml`, + `No module named 'Browser'`, `No keyword with name`) and ends with the + sibling to use instead: rf-browser vs rf-selenium (by import; rf-browser is + the default for new web tests), rf-requests vs rf-restinstance (by import or + JSON Schema/OpenAPI; rf-requests is the default for API tests), + rf-results / rf-libdoc vs rf-robotcode (robotcode installed or named), and + rf-setup for installing and environment errors. No more "Guide AI agents…" + phrasing. +- **Compact skill descriptions:** agents list skill descriptions only while + they fit a character budget (about 8000 characters in Claude Code for + 200k-context models), so every description is now at most 160 characters + and starts with a load-first cue ("Use first, before exploring or + answering, for Robot Framework web tests with Browser Library (Playwright): + selectors, waits."). The trigger terms, pasted errors and sibling boundaries + moved into a `## When to use` block near the top of each `SKILL.md`. +- **Standard Companion Skills tables:** every skill (now including rf-libdoc + and rf-results) has a `Need | Skill` table built from one shared catalogue, + with rows for its sibling, `rf-setup`, keyword lookup (`rf-libdoc`, or + `robotcode libdoc`) and result analysis (`rf-results`, or + `robotcode results`). +- **Library skills restructured** (rf-browser, rf-selenium, rf-appium, + rf-requests, rf-restinstance): one shared layout (Installation, Import and + defaults, a "which keyword for which situation" table, an agent workflow + with a libdoc → dry run → run → results → fix loop, verified Gotchas, a + gated reference table). Breaking for anyone who read the files directly: + every `references/keywords-reference.md` catalog is deleted (query libdoc + with `robotcode libdoc list/show` or the rf-libdoc skill + instead), and several references were merged or renamed (rf-browser + `tabs-windows.md` → `browser-context-page.md`; rf-selenium + `webdriver-setup.md` → `browser-setup.md`, `screenshots-logs.md` → + `troubleshooting.md`; rf-appium `device-capabilities.md` → + `capabilities.md`, `android-specific.md` + `ios-specific.md` → + `platform-specific.md`; rf-requests `http-methods.md` → + `request-options.md`). Examples were trimmed to suites that pass + `robot --dryrun`. + +Companion: `rf-agentskills` installer 0.7.0 ships the same content bundle +(`bundled content: 2.0.0`). + +## 1.2.0 (2026-03-17) + +- Fix: VS Code skill dirs now match the `name:` field per the Agent + Skills spec (so chat skills resolve correctly in Copilot 1.108+). +- Architecture overhaul: MCP server, script drift, hooks, tests, and + build infrastructure. + +Companion: `rf-agentskills` installer ≥ 0.3.0 ships the same content +bundle (`bundled content: 1.2.0`). + +## 1.1.0 + +- Iterative skill updates and minor fixes. + +## 1.0.0 (2026-02-27) + +- Initial release with 11 Robot Framework Agent Skills +- Skills for Browser, Selenium, Appium, Requests, RESTinstance libraries +- Keyword builder, test case builder, resource architect generators +- Library documentation search and explanation tools +- Results analysis for output.xml files diff --git a/bundles/rf-agentskills/README.md b/bundles/rf-agentskills/README.md new file mode 100644 index 0000000..467049b --- /dev/null +++ b/bundles/rf-agentskills/README.md @@ -0,0 +1,27 @@ +# rf-agentskills bundle + +Installs every Robot Framework plugin of this marketplace at once: `rf-core`, `rf-web`, `rf-api`, `rf-mobile` and `rf-desktop`. It has no content of its own. Install instructions for every agent are in the [repository README](https://github.com/MarketSquare/robotframework-plugins#install). + +## Install + +``` +/plugin install rf-agentskills --marketplace MarketSquare/robotframework-plugins +``` + +That one step works on Claude Code 2.1.275 or later. The commands for other agents and older versions are in the repository README. + +Until 3.0.0 this plugin held all nine skills, three subagents and the hooks. From 4.0.0 they are maintained in the five plugins above, so a web team can install `rf-core` and `rf-web` only. `rf-agentskills` ships a copy of all five, generated from them for each release. Projects that enable `rf-agentskills@robotframework-plugins` keep everything, under the same `rf-agentskills:` prefix as before. + +| Plugin | Skills and subagents | +|---|---| +| `rf-core` | `rf-language`, `rf-setup`, `rf-python-library`; `rf-test-architect`, `rf-debug-expert`, `rf-migration-guide`; all hooks | +| `rf-web` | `rf-browser`, `rf-selenium` | +| `rf-api` | `rf-requests`, `rf-restinstance` | +| `rf-mobile` | `rf-appium` | +| `rf-desktop` | `rf-platynui` | + +## Bundle or separate plugins, not both + +Install either this bundle or the plugins you need. With both, each skill appears twice, once as `rf-agentskills:rf-browser` and once as `rf-web:rf-browser`. Moving from the bundle to separate plugins changes the skill prefixes, so update project instructions or docs that name a prefixed skill. + +robotcode is a prerequisite of `rf-core`; see its README. diff --git a/bundles/rf-agentskills/agents/rf-debug-expert.md b/bundles/rf-agentskills/agents/rf-debug-expert.md new file mode 100644 index 0000000..6526463 --- /dev/null +++ b/bundles/rf-agentskills/agents/rf-debug-expert.md @@ -0,0 +1,104 @@ +--- +name: rf-debug-expert +description: Diagnoses failing or flaky Robot Framework runs without editing files. Use when a run failed and the cause is not obvious, tests pass locally but fail in CI, or a test fails only sometimes. Returns one report per root cause with category, evidence, fix and prevention. +tools: Read, Grep, Glob, Bash +--- + +# Robot Framework Debug Expert + +You diagnose Robot Framework failures and report them; you do not edit files. You read +the recorded results first, group the failures by cause, classify each cause, prove it +when the log is not enough, and return one report block per cause. The calling agent +applies the fix and re-runs the tests. + +## Diagnosis method (agent-owned) + +### 1. Read the recorded run before guessing + +- `robotcode results summary --failed`, then `robotcode results show --failed`: one line + per failing test with its full longname and message. +- `robotcode results log -bl "" --max-depth 2`: the keyword tree of one test. +- When a rerun is needed, keep the full run's output file under its own name + (`robotcode robot --output results/full.xml`) and read it with `-o results/full.xml`; + every later run, including the debugger, overwrites the default `output.xml`. +- Don't read or grep raw `output.xml`. + +### 2. Group failures by cause + +Failures that share a message, a keyword or a suite are one cause until proven otherwise +(`robotcode results stats --by suite --failed`). Diagnose each cause once. + +### 3. Classify each cause + +| Failure pattern | Category | Typical cause | +|---|---|---| +| element not found, strict mode violation | Locator | wrong selector, element not rendered, iframe or shadow DOM | +| timeout while waiting | Timing | page or request not finished, animation, missing wait condition | +| `!=`, "should be", assertion message | Assertion | wrong expectation, stale data, race condition | +| connection refused, DNS, 5xx | Environment | service down, wrong URL, proxy | +| "No keyword with name", "Multiple keywords" | Keyword resolution | missing import, typo, name conflict, embedded-argument mismatch | +| "contains no keywords", import error of a library | Library | Python library defect or wrong python-path | +| session or driver errors | Driver | driver or browser crash, version mismatch | +| passes and fails on the same code | Flakiness | timing, shared state, test order, external dependency | + +### 4. Prove the cause only when the log doesn't + +Debug one representative test per cause, scoped by longname and without overwriting the +kept output file: + +```bash +robotcode robot-debug -bl "" --output NONE --log NONE --report NONE +``` + +At the `(rdb)` stop use `.where`, `.vars` and `.print ${x}`, run keywords in the paused +context, and end with `.continue`, `.detach` or `.abort`. Run one debugger session at a +time. A failure that only happens in the full run (state leaking between tests) needs the +whole directory with `--break-on-failed-test` instead of one longname. + +### 5. Flakiness: structural fixes only + +| Pattern | Symptom | Fix | +|---|---|---| +| Asynchronous UI | passes locally, fails in CI | wait for the condition that proves readiness (element state, response, text) with the library's waiting keyword | +| Shared state | test B fails only after test A | isolate state in Setup/Teardown, a new browser context or session per test | +| Test data collisions | tests fail when run in parallel | unique identifiers per test, no shared records | +| Stale element | element reference invalid after re-render | re-query the element right before using it | +| External dependency | random 5xx or slow responses | stub or isolate the dependency; mark the test and report it | + +Retry loops and longer timeouts hide the cause; recommend them only as a documented +temporary measure after the root cause is named. + +## Routing + +| Work | Skill | +|------|-------| +| Name conflicts, embedded-argument mismatches, `__init__.robot` setup visibility, variable scope | `rf-language` | +| "contains no keywords", library scope losing state between tests, listener errors | `rf-python-library` | +| Locators, waits and library-specific errors | `rf-browser` / `rf-selenium` (plugin `rf-web`), `rf-requests` / `rf-restinstance` (plugin `rf-api`), `rf-appium` (plugin `rf-mobile`), `rf-platynui` (plugin `rf-desktop`) | +| Missing packages, interpreter or environment errors, robotcode missing | `rf-setup` | +| Keyword names and arguments | `robotcode doc keyword ""` | +| Recorded results, stepping through a test | `robotcode results …`, `robotcode robot-debug -bl` (robotcode plugin) | + +A library skill ships in a technology plugin that may not be installed. When the one you need is not available, name its plugin and suggest installing it; don't continue from memory for library-specific details. + +## Output format + +One block per root cause: + +``` +FAILURE: [longname(s)] +CATEGORY: [Locator | Timing | Assertion | Environment | Keyword resolution | Library | Driver | Flakiness] +ROOT CAUSE: [one-sentence explanation] +EVIDENCE: [error message, log line or debugger value] +FIX: [specific change for the calling agent to make] +PREVENTION: [pattern recommendation] +``` + +## Constraints + +- Start from the recorded results, not from assumptions. +- Name the root cause before proposing a fix; a fix without a cause is a guess. +- Never recommend `Sleep` as a fix; use the library's waiting mechanism. +- Never propose changing an assertion, expected value or tag to make a test pass. When + the evidence shows the expectation itself is wrong, say so explicitly in EVIDENCE. +- Don't write or edit files; return the report. diff --git a/bundles/rf-agentskills/agents/rf-migration-guide.md b/bundles/rf-agentskills/agents/rf-migration-guide.md new file mode 100644 index 0000000..7d53896 --- /dev/null +++ b/bundles/rf-agentskills/agents/rf-migration-guide.md @@ -0,0 +1,73 @@ +--- +name: rf-migration-guide +description: Migrates Robot Framework projects. Use when upgrading Robot Framework, replacing legacy syntax such as Run Keyword If or [Return], or moving from SeleniumLibrary to Browser or RequestsLibrary to RESTinstance. Works in phases and reports each file as migrated or unverified. +tools: Read, Write, Edit, Grep, Glob, Bash, Skill +--- + +# Robot Framework Migration Guide + +You plan and carry out migrations of Robot Framework projects: RF version upgrades, +legacy-syntax modernization and library switches. You work in phases, verify every +file with deterministic checks, and report a file as migrated only when the checks +are clean. The legacy → modern syntax mapping lives in the `rf-language` skill +(`references/migration.md`: construct, replacement, minimum RF version, Robocop rule). + +## Syntax migration procedure (agent-owned) + +1. **Inventory.** Count deprecated syntax per rule and file: + `robocop check --no-cache --select "DEPR*" --reports rules_by_id tests resources`. + Add the error check `robocop check --no-cache --threshold E tests resources`. + When the target RF version differs from the installed one, pass it: + `robocop check --no-cache --target-version 7 --select "DEPR*" tests resources`. + Select several rule groups by repeating `--select`; a comma list + (`DEPR*` and `ERR*` in one value) matches no rule and reports "No issues found". + The `rf_conventions` report of `rf-language` adds the legacy constructs Robocop does not flag. +2. **Phased plan.** Migrate shared resources first, then suites from the least to the + most dependent. Never migrate everything at once. +3. **Per-file fixes.** Rewrite one file at a time with the `rf-language` migration + table, then re-run the checks on that file. Robocop's `--fix` may apply the + mechanical rewrites, but only on a clean git working tree, followed by a review of + the diff. +4. **Exit check.** A file is migrated when it has zero `DEPR` findings, zero + error-severity findings, and `robotcode analyze code` reports no errors for the + suites that use it. + +Without Robocop, say that deterministic verification is unavailable, point to +`rf-setup` to add `robotcode[all]` (which includes Robocop) as a dev dependency, count legacy +constructs with the `rf_conventions` report the `rf-language` skill documents, and report the migration status as **unverified**. + +## Library migrations + +The keyword tables for a library switch live in the target library's skill: SeleniumLibrary → Browser in `rf-browser` (plugin `rf-web`, `references/migrate-from-selenium.md`), RequestsLibrary → RESTinstance in `rf-restinstance` (plugin `rf-api`, `references/migrate-from-requests.md`). Use the same phases as for syntax: import both libraries during the transition, move shared keywords first, then suites one by one, and remove the old import last. + +When the target library's skill is not available, say which plugin ships it and suggest installing that plugin before you change any library-specific code; don't continue from memory. + +## Routing + +| Work | Skill | +|------|-------| +| Legacy → modern syntax table, `rf_conventions` legacy counts | `rf-language` (`references/migration.md`) | +| Python libraries and listeners to modernize | `rf-python-library` | +| Keyword existence and arguments of the target library | `robotcode doc keyword ""` | +| Browser, SeleniumLibrary specifics; SeleniumLibrary → Browser table | `rf-browser` / `rf-selenium` (plugin `rf-web`) | +| RESTinstance, RequestsLibrary specifics; RequestsLibrary → RESTinstance table | `rf-restinstance` / `rf-requests` (plugin `rf-api`) | +| AppiumLibrary 3.x removed keywords | `rf-appium` (plugin `rf-mobile`) | +| Installing robotcode (with Robocop) or the new library | `rf-setup` | +| Reading results after the migrated run | `robotcode results show --failed` (robotcode plugin) | + +## Verification loop + +For every `.robot`, `.resource` or Python library file you write or change: + +1. Write the change. +2. Confirm keyword names and arguments with `robotcode doc keyword ""` (candidates: `robotcode doc keywords ""`). +3. Run `robotcode analyze code ` and fix every error. It runs nothing, so it misses undefined variables set at run time, a space before `=` in named arguments, embedded-argument mismatches and union-with-`str` conversions; the real run in step 5 catches them. +4. Run `robocop check --no-cache` on the changed files for deprecated syntax (select several rule groups by repeating `--select`, never with a comma list). +5. Run the affected tests by longname, `robotcode robot -bl ""` (longnames from `robotcode discover tests`), or a whole directory, never a single `.robot` file. +6. Read failures with `robotcode results show --failed` and `robotcode results log -bl ""`. + +## Constraints + +- Always propose a phased plan; old and new code coexist during the transition. +- Report per file: findings before, findings after, dry-run result. +- Never claim a migration is complete without a clean exit check. diff --git a/bundles/rf-agentskills/agents/rf-test-architect.md b/bundles/rf-agentskills/agents/rf-test-architect.md new file mode 100644 index 0000000..cf09bca --- /dev/null +++ b/bundles/rf-agentskills/agents/rf-test-architect.md @@ -0,0 +1,98 @@ +--- +name: rf-test-architect +description: Designs Robot Framework test projects. Use when starting a test automation project, choosing Browser or SeleniumLibrary, RequestsLibrary or RESTinstance, or restructuring suites, resources and environments. Returns the library choice with reasons, the layout and keyword layers, then writes them. +tools: Read, Write, Edit, Grep, Glob, Bash, Skill +--- + +# Robot Framework Test Architect + +You design Robot Framework test automation projects: which libraries to use, how the +project is laid out, and how keywords are layered. You decide the architecture yourself +and hand the writing of suites, keywords and libraries to the skills listed under +Routing. Load a skill before you write the part it owns; do not write from memory. + +## What this agent decides (agent-owned) + +### Web library selection + +Choose **Browser** (`rf-browser` in plugin `rf-web`, the default for new web tests) when the project needs +Playwright features: auto-waiting, network interception, shadow DOM or iframes through +selectors, several browser contexts, or the built-in assertion engine. + +Choose **SeleniumLibrary** (`rf-selenium` in plugin `rf-web`) when the team already runs Selenium or a +WebDriver grid / cloud service, depends on Selenium plugins, or must keep existing +SeleniumLibrary suites. + +### API library selection + +Choose **RequestsLibrary** (`rf-requests` in plugin `rf-api`, the default for API tests) for explicit HTTP +control: sessions, form data, file uploads, XML, and assertions on the response object. + +Choose **RESTinstance** (`rf-restinstance` in plugin `rf-api`) when JSON Schema or OpenAPI contract checks +are central and typed field assertions (`Integer`, `String`) fit the responses. + +### Project layout + +``` +project/ + tests/ # suites by area: web/, api/, mobile/, e2e/ + resources/ # common.resource (library imports) + one .resource per domain + libraries/ # project Python keyword libraries (on the python-path) + variables/ # per-environment variables: dev.yaml, staging.yaml, common.py + data/ # test data files (CSV, JSON) + results/ # output directory +``` + +Reuse an existing resource directory (`resources/`, `keywords/` or `res/`) and never +overwrite existing files. Installing Robot Framework and libraries, `robot.toml` and the +python-path belong to `rf-setup`. + +### Keyword abstraction layers + +1. Test cases read as business steps ("User Can Complete Purchase"). +2. Business keywords in domain resources ("Add Item To Cart", "Complete Checkout"). +3. Technical keywords wrap library calls (a click, filling a form field, an HTTP request). + +## Routing + +| Work | Skill | +|------|-------| +| Suites, test cases, `__init__.robot`, tags, templates, user keywords, resource and variable files | `rf-language` | +| Project keyword libraries and listeners in Python (`libraries/*.py`), checked with its library checker | `rf-python-library` | +| Installing RF and libraries, project layout mechanics, `robot.toml`, python-path | `rf-setup` | +| Web tests with Browser / SeleniumLibrary | `rf-browser` / `rf-selenium` (plugin `rf-web`) | +| API tests with RequestsLibrary / RESTinstance | `rf-requests` / `rf-restinstance` (plugin `rf-api`) | +| Mobile tests | `rf-appium` (plugin `rf-mobile`) | +| Desktop tests | `rf-platynui` (plugin `rf-desktop`) | +| Keyword names and arguments | `robotcode doc keyword ""` | +| Running, reading results, stepping through a test | `robotcode robot -bl`, `robotcode results`, `robotcode robot-debug` (robotcode plugin) | + +A library skill ships in a technology plugin that may not be installed. When the one you need is not available, name its plugin and suggest installing it; don't continue from memory for library-specific details. + +## Workflow + +1. Gather requirements: application under test, team skills, CI environment, coverage goals. +2. Select the libraries with the criteria above and state the reason. +3. Propose the layout and the keyword layers; list the files you will create. +4. Write the suites and resources with `rf-language`, the Python libraries with + `rf-python-library`, and the library calls with the matching library skill. +5. Verify with the loop below before handing the project over. + +## Verification loop + +For every `.robot`, `.resource` or Python library file you write or change: + +1. Write the change. +2. Confirm keyword names and arguments with `robotcode doc keyword ""` (candidates: `robotcode doc keywords ""`). +3. Run `robotcode analyze code ` and fix every error. It runs nothing, so it misses undefined variables set at run time, a space before `=` in named arguments, embedded-argument mismatches and union-with-`str` conversions; the real run in step 5 catches them. +4. Run `robocop check --no-cache` on the changed files for deprecated syntax (select several rule groups by repeating `--select`, never with a comma list). +5. Run the affected tests by longname, `robotcode robot -bl ""` (longnames from `robotcode discover tests`), or a whole directory, never a single `.robot` file. +6. Read failures with `robotcode results show --failed` and `robotcode results log -bl ""`. + +## Constraints + +- Write Robot Framework 7 syntax; the skills show the modern forms. +- Keep control structures out of test bodies; put logic into keywords. +- No `Sleep` for synchronization; use the library's waiting mechanism. +- Never overwrite existing project files; extend them. +- Name the library and the reason for every library choice in your plan. diff --git a/bundles/rf-agentskills/hooks/README.md b/bundles/rf-agentskills/hooks/README.md new file mode 100644 index 0000000..1b626e8 --- /dev/null +++ b/bundles/rf-agentskills/hooks/README.md @@ -0,0 +1,435 @@ +# rf-core hooks + +This plugin defines six Claude Code lifecycle hook scripts across four +events. Two (`SessionStart`, `PostToolUse`) run unconditionally; four +(`UserPromptSubmit`, and three on `Stop`) consult the session or an +environment flag before deciding to do anything. + +All are **cross-platform Node.js** scripts (`.mjs`). They run +identically on Linux, macOS, and Windows — no `.sh`/`.ps1` parity to +maintain. This follows the cross-platform-hooks guidance at +: + +> Invoke Node.js directly in your Claude Code hooks config. This works +> on Windows, Linux, and macOS. Claude Code requires Node.js, so `node` +> is always available. + +The installer probes `node` on PATH at install time; if Node is not +present, the hooks block is skipped with a clear `post_install` note +rather than written-and-broken. + +| Event | Script | Always fires? | Cost when no-op | Cost when active | Hook timeout | +|---|---|---|---|---|---| +| `SessionStart` | `scripts/check_rf_environment.mjs` | yes | — | ~0.5–1 s (import probes) | 15 s | +| `PostToolUse` (matcher: `Write\|Edit`) | `scripts/validate_robot.mjs` | only on Write/Edit | ~30ms (file extension check) | ~1–1.5 s on a 500-line file once robotcode's cache is warm (`robotcode analyze code`, Robocop check and format run in parallel); the first run in a project ~4 s; +0.4 s or more with the opt-in dry run | 60 s | +| `PostToolUse` (matcher: `Bash`) | `scripts/rf_error_hints.mjs` | only on Bash | ~30 ms (text scan) | ~30 ms | 10 s | +| `UserPromptSubmit` | `scripts/maybe_inject_rf_context.mjs` | always invoked, conditional injection | ~30ms (regex over prompt) | same (≤ 450-character text) | 15 s | +| `Stop` | `scripts/report_pending_errors.mjs` | always invoked, reports only files whose validation found errors or unresolved names that are still there | ~30 ms (reads the session record) | one `validate_robot.mjs` run per pending file | 60 s | +| `Stop` | `scripts/maybe_remind_robot_tests.mjs` | always invoked, conditional reminder | ~30ms (chunked grep over transcript) | same | default | +| `Stop` | `scripts/validate_robot_project.mjs` | only when `RF_AGENTSKILLS_PROJECT_VALIDATION` is set | ~0ms (env-flag check) | `robotcode analyze code --collect-unused` (120 s cap) | default | + +Every process a hook starts has a timeout (`robotcode analyze code` 20 s, +Robocop 10 s, per-file dry run 20 s, interpreter probes 5 s, project-wide +check 120 s). A timeout skips that +check silently. `scripts/_python_env.mjs`, `scripts/_hook_input.mjs` and +`scripts/_pending_errors.mjs` are shared helpers, not hooks. + +## Validation hooks (what catches broken Robot Framework code) + +Two scripts validate the Robot Framework files the agent writes, at two +different points in the lifecycle. They need **robotcode** in the project +environment — the documented prerequisite of rf-core — and degrade to a +silent no-op when it isn't installed. `robotcode[all]` also brings Robocop, +which the advisory tiers use: + +```bash +uv add --dev "robotcode[all]>=2.8" # project environment (see the rf-setup skill) +``` + +### Tier 1 + 2 — per file, on every write (`validate_robot.mjs`) + +Fires from `PostToolUse` after a Write/Edit of a `.robot`/`.resource` file. + +- **Tier 0 — errors: `robotcode --format json analyze code `**, run + from the project root (the event's `cwd`; a file outside it is analyzed from + its own folder), so `robot.toml`, profiles, the python-path and the installed + libraries apply. Error-severity diagnostics are sorted by code; warnings, + infos and hints are dropped: + - **Blocking errors** — everything except the two codes below: an + unterminated `FOR`/`IF`/`TRY` (`ModelError`), wrong argument counts and + other `DataError`s, failed imports, duplicate definitions. The diagnostic + goes to **stderr and the hook exits 2**, which feeds it back to the agent + (see "the exit-2 exception" below), plus the Copilot `decision` JSON. + - **Unresolved names** — `KeywordNotFound`, `VariableNotFound`. Reported as + advisory `additionalContext` (exit 0), because the agent may be about to + write that keyword or variable in another file. The file is recorded as + pending; if the name still doesn't resolve when the agent stops, the Stop + re-check blocks (below). + - Each list shows at most 5 diagnostics plus "… N more omitted." + - robotcode missing, a timeout (20 s), unparsable JSON or "0 files analyzed" + makes the tier silent — a tool failure is never a finding. + - robotcode keeps its analysis cache in `.robotcode_cache/` in the project + (the RobotCode VS Code extension uses the same folder; the rf-setup + `.gitignore` example lists it). With a warm cache the check takes under a + second; disabling the cache would make every edit take ~4 s. +- **Tier 1 — deprecations: one Robocop pass, classified by rule ID.** Runs + `robocop check --no-cache` once, with the **project's configured rule set** + and a parsable issue format + (`-c print_issues.output_format=simple --issue-format '{severity}|{rule_id}|{source}:{line}:{col}|{desc}'`), + in parallel with tier 0. Robocop's own error findings are left to tier 0; + style rules such as `DOC03` are dropped: + - **Hard deprecations** — `DEPR03` `WITH NAME`, `DEPR04` singular section + headers, `DEPR07` `Force/Default Tags`, `DEPR08` `Run Keyword If/Unless`, + `DEPR09` loop-exit keywords, `DEPR10` `Return From Keyword*`, `DEPR11` + `[Return]`. + - **Modernization hints** — `DEPR05` `Set Test/Suite/Global Variable` → + `VAR`, `DEPR06` `Create List/Dictionary` → `VAR`, and any other `DEPR` rule. + + **Deprecations only warn.** They are reported as non-blocking + `additionalContext` (exit 0) and never cause exit 2 — not even a hard + deprecation on a line the current edit wrote. Classification uses the rule + ID, not the severity (`DEPR11` is W in Robocop 8.2 and I in 9.1). + - **Touched lines** are listed individually, hard deprecations first, each + with file, line and the modern replacement (`WARN DEPR08 x.robot:12 — + Run Keyword If → use IF/ELSE/END`). Touched lines come from + `tool_response.structuredPatch` (the `+` lines of each hunk), else the + lines where an Edit's `new_string` now occurs, else the whole file (Write + create). Deprecations on other lines are summarized as a count per rule. + - **Cap:** at most 10 listed findings plus "N more omitted", at most 2,000 + characters. + - **Dedupe:** a finding listed once in a session (`session_id`; key = file, + rule and trimmed line text) is only counted afterwards. The marker is + `/rf-agentskills-depr-.json`; if it cannot be written, + only the dedupe is lost. When an error exits 2, the deprecation warnings are + neither written nor recorded, so they appear on the next clean edit. + - Robocop's gating by Robot Framework version applies (a project on RF 6.1 + gets no `VAR` hint), and so does the project's Robocop config. + - If Robocop is missing, times out (10 s), or prints nothing parsable while + failing, the tier is silent — a tool failure is never a finding. +- **Tier 2 — formatting drift:** runs `robocop format --no-cache --check --diff`. + Purely informational — surfaces the proposed reformat as `additionalContext` + (exit 0). Formatting differences **never** cause exit 2. +- **Opt-in per-file dry run:** with `RF_AGENTSKILLS_FILE_DRYRUN=1`, a written + `.robot` suite (not `.resource`, not `__init__.robot`) is also run through + `robot --dryrun --output NONE --report NONE --log NONE` from the project + directory (20 s timeout). `[ ERROR ]` lines and failures such as "No keyword + with name" are reported as advisory context, capped like the deprecation + warnings. It is off by default: a dry run imports libraries (import-time side + effects, seconds for Browser/Selenium), and a keyword the agent writes next + looks missing until it exists. + +Advisory output (deprecations, formatting, dry run) is emitted only when no +exit-2 error diagnostic is written. Robocop is always called with +`--no-cache`, so no `.robocop_cache/` directory is left in your project. + +#### Environment variables + +| Variable | Values | Effect | +|---|---|---| +| `RF_AGENTSKILLS_DEPRECATION_CHECK` | `warn` (default), `off` | `off` drops all deprecation output. Any other value is treated as `warn`; no value makes a deprecation block (exit 2), and the variable never affects error findings. | +| `RF_AGENTSKILLS_FILE_DRYRUN` | unset (default), `1`/`true`/`yes`/`on` | Enables the per-file dry run above. | +| `RF_AGENTSKILLS_PROJECT_VALIDATION` | unset (default), `1`/`true`/`yes`/`on` | Enables the project-wide Stop tier below. | + +#### Opting out per rule + +Teams that keep a legacy construct on purpose ignore the rule in their Robocop +config; the hook reads the project's config automatically: + +```toml +# pyproject.toml +[tool.robocop.lint] +ignore = ["DEPR08"] +``` + +**Selecting several rule groups on the command line:** repeat the option — +`robocop check --select "DEPR*" --select "ERR*"`. A comma list +(`--select 'DEPR*,ERR*'`) matches **no** rule in Robocop 8.2 and 9.1: Robocop +prints a warning, reports "No issues found" and exits 0. + +This replaced an earlier `robot.api.get_model` check that was effectively a +no-op: `get_model` is a lenient tokenizer that returns "OK" for unterminated +`FOR` loops, undefined keywords, missing imports — even for a file of random +prose. It never raised, so it never caught anything. (It also read the +edited path from a `TOOL_INPUT` env var; the documented contract delivers it +as `tool_input.file_path` on stdin, so the new script reads stdin first and +treats `TOOL_INPUT` only as a legacy fallback.) + +### Errors the agent never saw — re-checked at Stop (`report_pending_errors.mjs`) + +VS Code 1.142's Local agent harness sends the next model request without +waiting for `PostToolUse` hooks, so a validation error shows up only as the +banner "A hook prevented chat from continuing" and the model never sees it. +(An OpenTelemetry capture showed the next request leaving 41 ms after +`validate_robot.mjs` started; the hook takes about 0.7 s.) VS Code does wait +for `Stop` hooks, so: + +- `validate_robot.mjs` records each file with blocking errors or unresolved + names in `/rf-agentskills-pending-.json`, and removes it + when a later validation of the file is clean. +- At `Stop`, `report_pending_errors.mjs` re-validates the recorded files (by + running `validate_robot.mjs` on each with `RF_AGENTSKILLS_STOP_RECHECK=1`, + which makes unresolved names blocking and skips the advisory tiers). So a + keyword the agent called but never defined blocks here, while one it defined + in a later edit of another file clears. If any file still fails, it blocks with the + same contract as `validate_robot.mjs`: stderr + exit 2 (Claude Code, Codex, + VS Code), plus `{"decision": "block", "reason": ...}` on stdout (Copilot). The + agent then gets one more turn to fix them. +- Deleted files are dropped. Each content version of a file is reported at most + once, so a file the agent can't fix doesn't block every later turn. Like every + Stop hook here, it is a no-op when `stop_hook_active` is set. + +In agents whose `PostToolUse` feedback reaches the model, the agent normally +fixes the file right after the edit, which clears the record, and this hook +stays silent. Without a `session_id` nothing is recorded. + +### Tier 3 — whole project, end of task, opt-in (`validate_robot_project.mjs`) + +Fires from `Stop`, **only when `RF_AGENTSKILLS_PROJECT_VALIDATION` is set** +to a truthy value (`1`/`true`/`yes`/`on`). Runs one cross-file check that +only makes sense once the whole project is on disk: +`robotcode --format json analyze code --collect-unused` from the project root. +It reads `robot.toml`, and reports every error-severity diagnostic (unresolved +keywords and variables, broken imports, argument errors) and every unused +keyword and variable (`KeywordNotUsed`, `VariableNotUsed`), at most 20 of each. +The JSON is parsed; robotcode's bitmask exit code is not used. + +It is **off by default** for two reasons: (1) the analysis *loads the +project's libraries* to read their keywords, which runs their import-time +code; and (2) it scales with project size. Running them per-save would also false-alarm +constantly — a keyword you just wrote looks "unused" until something calls +it; a call into a not-yet-written resource looks "undefined". Deferring to +`Stop` (end of turn) avoids that. When findings exist the hook exits 2 so the +agent gets one more turn to fix before the turn ends. + +### The exit-2 exception + +The validation scripts (`validate_robot.mjs`, `report_pending_errors.mjs`, +`validate_robot_project.mjs`) are the **deliberate exception** to authoring +guideline #2 below ("hook scripts must always exit 0"). The Claude Code +`PostToolUse`/`Stop` contract feeds a hook's stderr back to the agent only on +**exit 2** (exit 1 / other is shown to the user, not the model). Returning a +real error to the model is the entire point of validation — the +edit→validate→feed-back→self-correct loop. So they exit 2 *specifically and +only* on a confirmed Robot Framework error, and exit 0 in every other case +(tool missing, tool timeout or unparsable output, non-Robot file, no +findings, formatting-only difference, deprecation findings of any class). + +## How `rf_error_hints.mjs` decides + +After every Bash command it scans the output (first 20 000 characters) for Robot Framework error messages. The first time each kind appears in a session, it injects one hint of at most 400 characters, naming the skill to load and the fix: + +| Error message | Hint | +|---|---| +| `No keyword with name '…' found` | embedded arguments (`Select team ${city} ${team:\S+}`; text between arguments is literal), rf-language; exact names via `robotcode doc keywords ` | +| `Multiple keywords with name '…' found` | qualify the call or use `Set Library Search Order` (rf-language) | +| `Invalid argument syntax '…'` | typed arguments `${count: int}` need RF 7.3+ (rf-language) | +| `Resolving variable '…' failed` | scope, import order, `$var` in expressions (rf-language) | +| `Importing library '…' failed` | install into the project environment (rf-setup) | + +It never blocks. It always exits 0, and unrelated or malformed input produces no output. + +## Python interpreter resolution + +`validate_robot.mjs`, `validate_robot_project.mjs`, and +`check_rf_environment.mjs` run Robot Framework tooling (`robotcode`, +`robocop`, `robot --dryrun`, `import robot` for the version probe). They share +`scripts/_python_env.mjs`, which takes the first environment that has the +needed tool, in this order (for robotcode, the `robotcode` executable in that +environment's `bin/` or `Scripts\` — robotcode has no `python -m` entry point): + +1. the active virtual environment (`$VIRTUAL_ENV`); +2. the project's `.venv` under the event's `cwd` + (`.venv/bin/python`, on Windows `.venv\Scripts\python.exe`); +3. the installer-recorded interpreter in `scripts/python_runtime.json` + (written from `sys.executable`; needed for pipx / uv tool installs); +4. `python3`, then `python` on `PATH`. + +The project environment wins so that robotcode and Robocop see the project's +libraries and Robot Framework version (version-gated rules such as the `VAR` hint follow it). If +the project environment has no Robocop, the search falls through to the +installer's interpreter, whose RF version may differ — a few version-gated +hints can then be off. The hooks never start `uv run` or another package +manager: that can create environments, sync dependencies or reach the network. +If no candidate has the required tool, the hooks exit silently — they're +non-blocking by design. + +## Why two of these are conditional + +The first version of this plugin used `type: "prompt"` for +`UserPromptSubmit` and `Stop` — Claude Code's way of statically +prepending a system note to the conversation. The text was: + +> If this request involves Robot Framework test automation, load the +> relevant skill's SKILL.md before responding... + +That seems harmless, but in headless evals against haiku-4-5 it caused +non-RF prompts to no-op: the model read the static injection as a gate +("is this RF?" → "no" → "task done") rather than as guidance. The eval +task `narrow-non-rf-control-01` (a plain JSON-authoring prompt) caught +this regression in CI run `25057802426` — see +`docs/plugin/hooks-fix-proposal.md` for the full diagnosis. + +Switching both hooks to `type: "command"` lets a small script inspect +the prompt / session and only inject context when there's a genuine +Robot Framework signal. Non-RF sessions are unaffected. + +## How `maybe_inject_rf_context.mjs` decides + +- Reads the Claude Code `UserPromptSubmit` event JSON on stdin. +- Extracts `.prompt` (the user's text). If empty / missing, exits 0 + silently. +- Tests the prompt against a case-insensitive regex covering: + - direct mentions: `robot framework`, `robot-framework` + - file extensions: `.robot`, `.resource` + - Robot Framework libraries: `SeleniumLibrary`, `BrowserLibrary`, + `AppiumLibrary`, `RequestsLibrary`, `RESTinstance`, `PlatynUI`, plus + space-separated forms (`Browser Library`, etc.) + - skill ids: `rf-language`, `rf-python-library`, `rf-setup` (rf-core); + `rf-browser`, `rf-selenium` (rf-web); `rf-requests`, `rf-restinstance` (rf-api); + `rf-appium` (rf-mobile); `rf-platynui` (rf-desktop) + - rf-core subagent ids: `rf-test-architect`, `rf-debug-expert`, + `rf-migration-guide` + - RF section headers pasted into the prompt: `*** Settings ***`, + `*** Variables ***`, `*** Test Cases ***`, `*** Tasks ***`, + `*** Keywords ***`, `*** Comments ***` + - tooling: `libdoc`, `robotidy`, `robocop`, `rfbrowser`, `robotcode`, + `robot-debug`, `robot.toml`; Python library API terms (`@keyword`, + `robot.api.deco`, `ROBOT_LIBRARY_*`, "robot listener") +- On match: emits one `{"hookSpecificOutput": {"hookEventName": + "UserPromptSubmit", "additionalContext": "..."}}` with a routing text of + at most 450 characters (about 80 tokens): tests, suites, keywords, + resources and variables → `rf-language`; Python libraries and listeners → + `rf-python-library`; installs → `rf-setup`; library usage → the + library's own skill, unnamed, and, when it is missing, a suggestion to + install its plugin (`rf-web`, `rf-api`, `rf-mobile`, `rf-desktop`); + keyword names and arguments → `robotcode doc keyword`, not memory; run + results → `robotcode results`, not `output.xml`; and a closing reminder to write RF 7 syntax (`RETURN`, `VAR`, + `IF`, `Test Tags`). Skill descriptions and subagents are already listed + by Claude Code, so the text only says which one to use when. +- On miss: stdout stays empty. + +Deliberately **not** matched: bare `RF` (too ambiguous), bare `test` +(too generic), bare `library` / `keyword` / `listener`. The parametrised +cases in `tests/test_hook_scripts.py` lock in the positive trigger list, +the negative miss list, the budget and that every `rf-*` name in the text +is a shipped skill or subagent. + +## What `check_rf_environment.mjs` reports + +At `SessionStart` it reports the Robot Framework version and interpreter +(same resolution as above, from the event's `cwd`), the robotcode version — +without robotcode (or below 2.8) the error check on edit is disabled, and it +names the robotcode plugin as the recommended companion — whether Robocop is +available (deprecation and formatting advice), and which library packages are +importable. For missing packages it shows project-environment commands +(`uv add …`, `uv add --dev "robotcode[all]"`) and points to the rf-setup skill; it never +recommends `pip install` or a library's post-install step. A missing, empty or malformed +event produces no output; it always exits 0. + +## How `maybe_remind_robot_tests.mjs` decides + +- Reads the Claude Code `Stop` event JSON on stdin. +- Extracts `.transcript_path` (path to the session JSONL). If missing + or unreadable, exits 0. +- Scans the transcript in 64 KB chunks for `"file_path": "…something.robot"` + or `"…something.resource"` (with bounded regex — `notes.robotic.md` + does not count). Streaming chunking keeps memory bounded on + long sessions. +- On hit: emits `additionalContext` reminding the user to run + the affected tests (`uv run robotcode robot -bl ""`), read the + results with `uv run robotcode results summary --failed`, and open + `results/report.html`. +- On miss: stdout stays empty. + +## Verifying the hooks fire + +Two patterns work, depending on what you can inspect: + +### Inspect side effects (most reliable) + +`stream-json` from `claude -p` includes +`hook_started/progress/response` events for `SessionStart` but **not** +for the other event types. To verify `PostToolUse` / +`UserPromptSubmit` / `Stop` fired, instrument the script itself: + +```javascript +// Prepend to any hook script for ad-hoc tracing: +import { appendFileSync } from "node:fs"; +appendFileSync(`${process.cwd()}/.hook-trace.log`, + `[${new Date().toISOString()}] hook fired\n`); +``` + +Then check `.hook-trace.log` after the session. The `rf-skill-eval` +harness exposes the workspace as `process.cwd()` for the hook +subprocess, so this works under CI as well as in interactive use. + +### Run the unit tests + +`tests/test_hook_scripts.py` invokes each `.mjs` script via `node` as a +subprocess with synthetic stdin payloads and asserts on the +(stdout, exit-code) pair. No live Claude session needed: + +```bash +uv run pytest plugins/rf-agentskills/tests/test_hook_scripts.py -v +``` + +Parametrised cases covering positive triggers, negative misses, +malformed input, and pathological transcripts. + +## Running the eval suite + +The `rf-skill-eval` harness includes two paired tasks for the +conditional injection: + +- `eval/tasks/narrow/narrow-rf-injection-positive-01.yaml` — RF + prompt, expects the agent to leverage a robotcode lookup. +- `eval/tasks/narrow/narrow-non-rf-control-01.yaml` — non-RF prompt, + expects the agent to write a plain JSON file with no rf tooling. + +Both should pass green after this hooks fix. Run with: + +```bash +uv run rf-skill-eval run \ + --task eval/tasks/narrow/narrow-rf-injection-positive-01.yaml \ + --output eval/runs/local-probe \ + --profile treatment +``` + +## Authoring guidelines for new hooks in this plugin + +1. **Default to conditional injection.** Static `type: "prompt"` + hooks bias the model on every turn — fine for project-specific + plugins where the user opted in, lethal for general-purpose + plugins like this one. +2. **Hook scripts must always exit 0** — *unless* they are intentional + validation hooks. A stray non-zero exit is interpreted as a hook + error and surfaced to the user, so wrap risky work in + `try { … } catch { process.exit(0) }`. The sole deliberate exception + is the validation tier (`validate_robot.mjs`, + `validate_robot_project.mjs`), which exits **2** on a confirmed Robot + Framework error to feed the diagnostic back to the agent — and still + exits 0 in every non-error path (see "the exit-2 exception" above). +3. **Stop/SubagentStop hooks MUST short-circuit on `stop_hook_active`.** + Add `if (event?.stop_hook_active) process.exit(0)` immediately after + parsing the event. Claude Code sets this flag when a Stop fires as a + continuation of a previous Stop-hook block; without the guard a hook + that emits any model-facing output re-fires every time and traps the + session until `CLAUDE_CODE_STOP_HOOK_BLOCK_CAP` (default 9) overrides. + **Note the trap in guideline #2:** for Stop hooks, *exit 0 is not + sufficient to be non-blocking* — model-facing output (`additionalContext` + **or** exit 2) re-invokes the model. `maybe_remind_robot_tests.mjs` once + looped exactly this way (exit 0 + `additionalContext`, no guard); it now + short-circuits on `stop_hook_active` and additionally reminds at most once + per `session_id`. +4. **Stay Node-only when possible.** Shelling out to other runtimes + (Python, bash) reintroduces the install-time dependency surface + the Node migration eliminated. If a hook genuinely needs Python + (Robot Framework parsing), resolve the interpreter with + `scripts/_python_env.mjs` (project env, then `python_runtime.json`), + not bare `python` on PATH, and give every spawned process a timeout + (`spawnOptions(...)`). +5. **Keep the regex tight.** Adding a trigger that looks ergonomic + ("test", "library") is a fast way to revive the + `narrow-non-rf-control-01` regression. The unit-test parameter + list is the canonical specification of what does and doesn't + trigger; update it deliberately. diff --git a/bundles/rf-agentskills/hooks/hooks.json b/bundles/rf-agentskills/hooks/hooks.json new file mode 100644 index 0000000..6ee8dee --- /dev/null +++ b/bundles/rf-agentskills/hooks/hooks.json @@ -0,0 +1,70 @@ +{ + "hooks": { + "PostToolUse": [ + { + "matcher": "Write|Edit", + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/scripts/validate_robot.mjs\"", + "timeout": 60 + } + ] + }, + { + "matcher": "Bash", + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/scripts/rf_error_hints.mjs\"", + "timeout": 10 + } + ] + } + ], + "UserPromptSubmit": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/scripts/maybe_inject_rf_context.mjs\"", + "timeout": 15 + } + ] + } + ], + "SessionStart": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/scripts/check_rf_environment.mjs\"", + "timeout": 15 + } + ] + } + ], + "Stop": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/scripts/report_pending_errors.mjs\"", + "timeout": 60 + }, + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/scripts/maybe_remind_robot_tests.mjs\"" + }, + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/scripts/validate_robot_project.mjs\"" + } + ] + } + ] + } +} diff --git a/bundles/rf-agentskills/scripts/_hook_input.mjs b/bundles/rf-agentskills/scripts/_hook_input.mjs new file mode 100644 index 0000000..1d25880 --- /dev/null +++ b/bundles/rf-agentskills/scripts/_hook_input.mjs @@ -0,0 +1,101 @@ +// _hook_input.mjs — one reader for every agent's hook input. +// +// The plugin's Claude-format hooks.json runs in Claude Code, Copilot CLI, +// VS Code, Codex and Cursor. Each sends the same event names but its own tool +// names and input shapes (observed 2026-10-08): +// +// Claude Code / Copilot CLI tool_name Write|Edit|MultiEdit, tool_input.file_path +// Cursor (converted hooks) tool_name Write|Shell, file_path or tool_input.file_path +// VS Code create_file / replace_string_in_file: tool_input.filePath +// multi_replace_string_in_file: tool_input.replacements[].filePath +// apply_patch: tool_input.input (patch text); run_in_terminal +// Codex apply_patch: tool_input.command (patch text); Bash +// +// VS Code ignores matchers in Claude-format hooks and runs every command for +// every tool, so callers must check `isEdit` / `isShell` and exit quietly. +import { readFileSync } from "node:fs"; +import { isAbsolute, resolve } from "node:path"; + +const EDIT_TOOLS = + /^(write|edit|multiedit|apply_patch|create_file|replace_string_in_file|multi_replace_string_in_file|insert_edit_into_file|edit_file|edit_notebook_file|create)$/i; +const SHELL_TOOLS = /^(bash|shell|run_in_terminal|powershell|exec_command|local_shell)$/i; +const PATCH_FILE_RE = /^\*\*\* (?:Add File|Update File|Move to): (.+)$/; + +function parseJson(text) { + if (typeof text !== "string" || !text.trim()) return null; + try { + const obj = JSON.parse(text); + return obj && typeof obj === "object" ? obj : null; + } catch { + return null; + } +} + +/** Paths of every file an `apply_patch` envelope adds, updates or moves to. */ +export function patchFiles(patch) { + if (typeof patch !== "string" || !patch.includes("*** ")) return []; + const files = []; + for (const line of patch.split(/\r?\n/)) { + const m = PATCH_FILE_RE.exec(line.trim()); + if (m) files.push(m[1].trim()); + } + return files; +} + +function patchText(toolInput) { + if (typeof toolInput === "string") return toolInput; + for (const key of ["input", "command", "patch"]) { + const v = toolInput?.[key]; + if (typeof v === "string" && /\*\*\* (Begin Patch|Add File:|Update File:)/.test(v)) return v; + } + return ""; +} + +/** + * Normalize one hook event. + * @returns {{event: object|null, tool: string, isEdit: boolean, isShell: boolean, + * files: string[], command: string, cwd: string}} + */ +export function normalizeHookInput(event) { + const empty = { event: null, tool: "", isEdit: false, isShell: false, files: [], command: "", cwd: process.cwd() }; + if (!event || typeof event !== "object") return empty; + const tool = String(event.tool_name ?? event.toolName ?? ""); + let toolInput = event.tool_input ?? event.toolArgs ?? {}; + if (typeof toolInput === "string") toolInput = parseJson(toolInput) ?? toolInput; + const cwd = String(event.cwd || process.cwd()); + + const raw = []; + const patch = patchText(toolInput); + raw.push(...patchFiles(patch)); + if (toolInput && typeof toolInput === "object") { + for (const key of ["file_path", "filePath"]) { + if (typeof toolInput[key] === "string") raw.push(toolInput[key]); + } + if (Array.isArray(toolInput.replacements)) { + for (const r of toolInput.replacements) if (typeof r?.filePath === "string") raw.push(r.filePath); + } + } + if (typeof event.file_path === "string") raw.push(event.file_path); + + const files = [...new Set(raw.filter(Boolean).map((f) => (isAbsolute(f) ? f : resolve(cwd, f))))]; + // No tool name (legacy TOOL_INPUT, Cursor afterFileEdit): a file path means an edit. + const isEdit = tool ? EDIT_TOOLS.test(tool) : files.length > 0; + const isShell = SHELL_TOOLS.test(tool); + const command = isShell && !patch && typeof toolInput?.command === "string" ? toolInput.command : ""; + return { event, tool, isEdit, isShell, files: isEdit ? files : [], command, cwd }; +} + +/** Read the hook event from stdin (legacy `TOOL_INPUT` env var as a fallback). */ +export function readHookInput() { + let stdin = ""; + try { + stdin = readFileSync(0, "utf-8"); + } catch { + // no stdin + } + for (const source of [stdin, process.env.TOOL_INPUT ?? ""]) { + const obj = parseJson(source); + if (obj) return normalizeHookInput(obj); + } + return normalizeHookInput(null); +} diff --git a/bundles/rf-agentskills/scripts/_pending_errors.mjs b/bundles/rf-agentskills/scripts/_pending_errors.mjs new file mode 100644 index 0000000..4fe8e82 --- /dev/null +++ b/bundles/rf-agentskills/scripts/_pending_errors.mjs @@ -0,0 +1,65 @@ +// _pending_errors.mjs — per-session record of .robot/.resource files whose +// last PostToolUse validation found real errors. +// +// Why: VS Code's Local agent mode (1.142) sends the next model request +// without waiting for PostToolUse hooks, so the validation error never +// reaches the model. Stop hooks ARE awaited, so validate_robot.mjs records +// failing files here and report_pending_errors.mjs re-checks them at Stop. +// In agents where PostToolUse feedback works, the fixing edit clears the +// entry again and the Stop hook stays silent. +// +// State: { "": { reported: "" | null } } +// in the OS temp dir, keyed by session_id. All I/O is best-effort. +import { readFileSync, writeFileSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +export function pendingPath(sessionId) { + const sid = (sessionId ?? "").toString(); + if (!sid) return ""; + return join(tmpdir(), `rf-agentskills-pending-${sid.replace(/[^A-Za-z0-9._-]/g, "_")}.json`); +} + +export function loadPending(sessionId) { + const p = pendingPath(sessionId); + if (!p) return {}; + try { + const data = JSON.parse(readFileSync(p, "utf-8")); + return data && typeof data === "object" && !Array.isArray(data) ? data : {}; + } catch { + return {}; + } +} + +export function savePending(sessionId, state) { + const p = pendingPath(sessionId); + if (!p) return; + try { + writeFileSync(p, JSON.stringify(state)); + } catch { + // Read-only temp dir: only the Stop-time re-check is lost. + } +} + +/** Record (failed=true) or clear (failed=false) one file's validation result. */ +export function recordResult(sessionId, file, failed) { + if (!pendingPath(sessionId)) return; + const state = loadPending(sessionId); + if (failed) { + if (!(file in state)) state[file] = { reported: null }; + } else if (file in state) { + delete state[file]; + } else { + return; + } + savePending(sessionId, state); +} + +export function contentHash(file) { + try { + return createHash("sha1").update(readFileSync(file)).digest("hex"); + } catch { + return ""; + } +} diff --git a/bundles/rf-agentskills/scripts/_python_env.mjs b/bundles/rf-agentskills/scripts/_python_env.mjs new file mode 100644 index 0000000..84004a1 --- /dev/null +++ b/bundles/rf-agentskills/scripts/_python_env.mjs @@ -0,0 +1,181 @@ +// _python_env.mjs — shared Python interpreter resolution for the hook scripts. +// +// Not a hook itself (the leading underscore marks a helper). Imported by +// validate_robot.mjs, validate_robot_project.mjs and check_rf_environment.mjs. +// findRobotcode() resolves the `robotcode` executable the same way: robotcode has +// no `python -m robotcode` entry point, so the executable next to each +// interpreter (bin/robotcode, Scripts\robotcode.exe) is used. +// +// Resolution order (first interpreter that can import the needed module wins): +// 1. the active virtual environment ($VIRTUAL_ENV) +// 2. the project's .venv under the event's working directory +// (POSIX: .venv/bin/python, Windows: .venv\Scripts\python.exe) +// 3. the installer-recorded interpreter (python_runtime.json next to this file, +// plus its fallbacks) +// 4. python3 / python on PATH +// +// The project environment wins so that Robocop and `robot` see the project's +// Robot Framework version. A package manager (`uv run`, `poetry run`) is never +// started: it can create environments, sync dependencies or reach the network. +// +// Every probe has a timeout (5 s by default). RF_AGENTSKILLS_HOOK_TIMEOUT_MS is a +// test-only override that caps every hook timeout (probes, Robocop, dry run). +import { existsSync, readFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { basename, dirname, join } from "node:path"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const IS_WIN = process.platform === "win32"; + +export const PROBE_TIMEOUT_MS = 5000; + +// Timeout for a spawned process: the given default, capped by the test-only +// override when it is a positive integer. +export function timeoutMs(defaultMs) { + const raw = (process.env.RF_AGENTSKILLS_HOOK_TIMEOUT_MS ?? "").trim(); + const n = /^\d+$/.test(raw) ? Number(raw) : 0; + return n > 0 ? Math.min(n, defaultMs) : defaultMs; +} + +// Options shared by every spawnSync in the hooks: bounded time and output. +export function spawnOptions(defaultTimeoutMs, extra = {}) { + return { + timeout: timeoutMs(defaultTimeoutMs), + killSignal: "SIGKILL", + maxBuffer: 1024 * 1024, + windowsHide: true, + ...extra, + }; +} + +function venvPython(root) { + return IS_WIN + ? join(root, "Scripts", "python.exe") + : join(root, "bin", "python"); +} + +function installerInterpreters() { + const out = []; + try { + const cfg = JSON.parse(readFileSync(join(HERE, "python_runtime.json"), "utf-8")); + if (typeof cfg.interpreter === "string" && cfg.interpreter) out.push(cfg.interpreter); + for (const fb of cfg.fallbacks ?? []) { + if (typeof fb === "string" && fb) out.push(fb); + } + } catch { + // Config missing or unreadable (plugin checkout) — PATH fallbacks only. + } + return out; +} + +// Ordered, de-duplicated interpreter candidates for a project directory. +export function pythonCandidates(cwd) { + const candidates = []; + const add = (py) => { + if (py && !candidates.includes(py)) candidates.push(py); + }; + const venv = (process.env.VIRTUAL_ENV ?? "").trim(); + if (venv) { + const py = venvPython(venv); + if (existsSync(py)) add(py); + } + const root = cwd || process.cwd(); + const projectPy = venvPython(join(root, ".venv")); + if (existsSync(projectPy)) add(projectPy); + for (const py of installerInterpreters()) add(py); + add("python3"); + add("python"); + return candidates; +} + +function scriptsExe(binDir, name) { + return join(binDir, IS_WIN ? `${name}.exe` : name); +} + +// Ordered, de-duplicated `robotcode` executable candidates for a project +// directory, in the same order as pythonCandidates(): the active venv, the +// project's .venv, the installer interpreter's directory, then PATH. +export function robotcodeCandidates(cwd) { + const out = []; + const add = (exe) => { + if (exe && !out.includes(exe)) out.push(exe); + }; + const venv = (process.env.VIRTUAL_ENV ?? "").trim(); + if (venv) { + const exe = scriptsExe(join(venv, IS_WIN ? "Scripts" : "bin"), "robotcode"); + if (existsSync(exe)) add(exe); + } + const root = cwd || process.cwd(); + const projectExe = scriptsExe(join(root, ".venv", IS_WIN ? "Scripts" : "bin"), "robotcode"); + if (existsSync(projectExe)) add(projectExe); + for (const py of installerInterpreters()) { + if (basename(py) === py) continue; // bare command name: covered by PATH below + const exe = scriptsExe(dirname(py), "robotcode"); + if (existsSync(exe)) add(exe); + } + add("robotcode"); + return out; +} + +// First robotcode candidate that answers `--version`, as {exe, version} +// ("2.8.0"), or null. Paths that exist are trusted without a probe unless the +// version is wanted; the bare PATH name is always probed. +export function findRobotcode(cwd, { withVersion = false } = {}) { + for (const exe of robotcodeCandidates(cwd)) { + if (!withVersion && exe !== "robotcode") return { exe, version: null }; + const r = spawnSync(exe, ["--version"], { + ...spawnOptions(PROBE_TIMEOUT_MS), + encoding: "utf-8", + stdio: ["ignore", "pipe", "ignore"], + }); + if (r.error || r.status !== 0) continue; + const m = /(\d+\.\d+(?:\.\d+)?)/.exec(r.stdout ?? ""); + return { exe, version: m ? m[1] : null }; + } + return null; +} + +// True when `version` ("2.8.0") is at least `min` ("2.8"). +export function versionAtLeast(version, min) { + const a = String(version ?? "").split(".").map(Number); + const b = String(min).split(".").map(Number); + for (let i = 0; i < Math.max(a.length, b.length); i++) { + const x = a[i] || 0; + const y = b[i] || 0; + if (x !== y) return x > y; + } + return true; +} + +// The event's working directory (falls back to the process cwd). +export function projectRoot(event) { + const cwd = event && typeof event.cwd === "string" ? event.cwd : ""; + if (cwd && existsSync(cwd)) return cwd; + return process.cwd(); +} + +// First candidate for which `import ` succeeds, or null. +export function findInterpreterWith(moduleName, cwd) { + for (const py of pythonCandidates(cwd)) { + const probe = spawnSync(py, ["-c", `import ${moduleName}`], { + ...spawnOptions(PROBE_TIMEOUT_MS), + stdio: ["ignore", "ignore", "ignore"], + }); + if (probe.error) continue; // missing interpreter, timeout, … + if (probe.status === 0) return py; + } + return null; +} + +// First candidate that exists at all (`--version` answers), or null. +export function firstPython(cwd) { + for (const py of pythonCandidates(cwd)) { + const r = spawnSync(py, ["--version"], { + ...spawnOptions(PROBE_TIMEOUT_MS), + stdio: "ignore", + }); + if (!r.error) return py; + } + return null; +} diff --git a/bundles/rf-agentskills/scripts/check_rf_environment.mjs b/bundles/rf-agentskills/scripts/check_rf_environment.mjs new file mode 100644 index 0000000..e05997b --- /dev/null +++ b/bundles/rf-agentskills/scripts/check_rf_environment.mjs @@ -0,0 +1,181 @@ +#!/usr/bin/env node +// check_rf_environment.mjs — Check the Robot Framework environment at session start. +// +// Called by the SessionStart hook. Writes a diagnostic summary to stderr so the +// agent sees the environment state. Always exits 0 (informational only — never +// blocks session start). +// +// Interpreter resolution is shared with the validation hooks (_python_env.mjs): +// VIRTUAL_ENV -> /.venv -> python_runtime.json -> python3/python. The +// SessionStart event's `cwd` selects the project (process cwd when it has none); +// a missing, empty or malformed event produces no output. Install advice follows the rf-setup skill (project environment, +// `uv add …`); it never recommends `pip install`, and leaves library post-install steps to +// the library's own skill. +import { spawnSync } from "node:child_process"; +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { + PROBE_TIMEOUT_MS, + findInterpreterWith, + findRobotcode, + firstPython, + projectRoot, + spawnOptions, + versionAtLeast, +} from "./_python_env.mjs"; + +const ROBOTCODE_MIN = "2.8"; +const MARKETPLACE = "robotframework-plugins"; + +// Libraries a technology plugin covers (detection signals only; the install +// steps live in each library's skill). Import name, label, plugin, skill. +const LIBRARIES = [ + ["Browser", "Browser Library", "rf-web", "rf-browser"], + ["SeleniumLibrary", "SeleniumLibrary", "rf-web", "rf-selenium"], + ["RequestsLibrary", "RequestsLibrary", "rf-api", "rf-requests"], + ["REST", "RESTinstance", "rf-api", "rf-restinstance"], + ["AppiumLibrary", "AppiumLibrary", "rf-mobile", "rf-appium"], + ["PlatynUI", "PlatynUI", "rf-desktop", "rf-platynui"], +]; + +// Whether the skill is already there: in this plugin tree (the rf-agentskills +// bundle, or a file-mode install's support folder next to the skills), or in +// a sibling plugin of the same marketplace cache. Best effort; unknown = no. +function skillPresent(plugin, skill) { + const root = process.env.CLAUDE_PLUGIN_ROOT; + if (!root) return false; + const candidates = [ + join(root, "skills", skill), + join(root, "..", "skills", skill), + join(root, "..", plugin, "skills", skill), + join(root, "..", "..", plugin), + ]; + return candidates.some((p) => existsSync(p)); +} + +// The command that installs a plugin in the agent running this hook, judged by +// where the agent put the plugin; a file-mode install gets the installer. +function installCommand(plugin) { + const root = (process.env.CLAUDE_PLUGIN_ROOT ?? "").replace(/\\/g, "/"); + if (root.includes("/.copilot/")) return `copilot plugin install ${plugin}@${MARKETPLACE}`; + if (root.includes("/.codex/")) return `codex plugin add ${plugin}@${MARKETPLACE}`; + if (root.includes("/.claude/plugins/")) return `/plugin install ${plugin}@${MARKETPLACE}`; + return "uvx robotframework-plugins install (adds the plugins for the libraries the project declares)"; +} + +// Missing, empty or malformed event -> no output (the hook never blocks). +let event = null; +try { + event = JSON.parse(readFileSync(0, "utf-8")); +} catch { + process.exit(0); +} +if (!event || typeof event !== "object" || Array.isArray(event)) process.exit(0); +const cwd = projectRoot(event); + +const found = []; +const missing = []; + +function pythonImportExists(py, importName) { + if (!py) return false; + const r = spawnSync(py, ["-c", `import ${importName}`], { + ...spawnOptions(PROBE_TIMEOUT_MS), + stdio: "ignore", + }); + return r.status === 0; +} + +function check(ok, label) { + (ok ? found : missing).push(label); + return ok; +} + +process.stderr.write("=== Robot Framework Environment Check ===\n"); + +// Prefer the interpreter that has Robot Framework; else the first one that runs. +const py = findInterpreterWith("robot", cwd) ?? firstPython(cwd); + +process.stderr.write("\nCore:\n"); +check(py !== null, "python3"); +const rfOk = pythonImportExists(py, "robot"); +check(rfOk, "robotframework"); +let rfVersion = "not installed"; +if (rfOk) { + const r = spawnSync(py, ["-c", "import robot; print(robot.version.VERSION)"], { + ...spawnOptions(PROBE_TIMEOUT_MS), + encoding: "utf-8", + }); + if (r.status === 0) rfVersion = (r.stdout ?? "").trim() || rfVersion; +} +process.stderr.write(` Robot Framework version: ${rfVersion}\n`); +if (py) process.stderr.write(` Interpreter: ${py}\n`); + +process.stderr.write("\nrobotcode (required by rf-core):\n"); +const rc = findRobotcode(cwd, { withVersion: true }); +const rcOk = rc !== null && versionAtLeast(rc.version, ROBOTCODE_MIN); +check(rcOk, `robotcode[all]>=${ROBOTCODE_MIN}`); +if (rcOk) { + process.stderr.write(` robotcode ${rc.version}: available (${rc.exe}); errors are checked on every .robot/.resource edit\n`); +} else { + process.stderr.write( + (rc + ? ` robotcode ${rc.version ?? "(unknown version)"} is older than ${ROBOTCODE_MIN} (robotcode doc needs ${ROBOTCODE_MIN}): ` + : " robotcode: not installed — ") + + "keyword lookup (robotcode doc), result analysis (robotcode results) and the error check on edit " + + "need it. Install it into the project environment: uv add --dev \"robotcode[all]>=2.8\" " + + "(see the rf-setup skill). Recommended companion plugin: robotcode@robotframework-agent-plugins.\n", + ); +} +const robocopOk = check(pythonImportExists(py, "robocop") || + findInterpreterWith("robocop", cwd) !== null, "robotframework-robocop"); +process.stderr.write( + robocopOk + ? " Robocop: available (deprecation and formatting advice on edit)\n" + : " Robocop: not installed — deprecation and formatting advice on edit is disabled " + + "(it comes with robotcode[all])\n", +); + +// Test libraries: report the ones the project has, and the plugin for each. +const plugins = new Map(); +for (const [importName, label, plugin, skill] of LIBRARIES) { + if (pythonImportExists(py, importName)) { + found.push(label); + if (!plugins.has(plugin)) plugins.set(plugin, { labels: [], present: true }); + const entry = plugins.get(plugin); + entry.labels.push(label); + entry.present = entry.present && skillPresent(plugin, skill); + } +} +if (plugins.size) { + process.stderr.write("\nTest libraries and their plugins:\n"); + for (const [plugin, { labels, present }] of plugins) { + process.stderr.write( + present + ? ` ${labels.join(", ")}: covered by plugin ${plugin} (installed)\n` + : ` ${labels.join(", ")}: covered by plugin ${plugin}, not installed. Install: ${installCommand(plugin)}\n`, + ); + } +} + +process.stderr.write("\n--- Summary ---\n"); +if (found.length) { + process.stderr.write(`Available: ${found.join(", ")}\n`); +} +if (missing.length) { + process.stderr.write(`Not installed: ${missing.join(", ")}\n`); + process.stderr.write( + "\nInstall only what the project needs, into the project environment. " + + "See the rf-setup skill for the full steps (uv, venv, poetry):\n", + ); + process.stderr.write(" uv add robotframework # core\n"); + process.stderr.write(" uv add --dev \"robotcode[all]>=2.8\" # required: robotcode CLI + Robocop\n"); + process.stderr.write( + " Test libraries: the library's own skill gives its install steps " + + "(plugins rf-web, rf-api, rf-mobile, rf-desktop).\n", + ); +} else { + process.stderr.write("All checked packages are installed.\n"); +} +process.stderr.write("=== End Environment Check ===\n"); + +process.exit(0); diff --git a/bundles/rf-agentskills/scripts/maybe_inject_rf_context.mjs b/bundles/rf-agentskills/scripts/maybe_inject_rf_context.mjs new file mode 100644 index 0000000..a55a393 --- /dev/null +++ b/bundles/rf-agentskills/scripts/maybe_inject_rf_context.mjs @@ -0,0 +1,74 @@ +#!/usr/bin/env node +// maybe_inject_rf_context.mjs — Conditional UserPromptSubmit hook. +// +// Reads the Claude Code UserPromptSubmit event JSON on stdin, inspects +// the user's prompt for Robot Framework signals, and only emits an +// additionalContext injection when at least one signal is present. +// The injection is a short routing text (<= 450 characters): which skill +// to load for which kind of work, keyword lookup via `robotcode doc keyword`, +// results via `robotcode results`, and a one-line RF 7 syntax reminder. +// +// Schema: +// stdin — Claude Code UserPromptSubmit event JSON. +// stdout — Either empty (no injection) or one JSON object of the form +// {"hookSpecificOutput": {"hookEventName": "UserPromptSubmit", +// "additionalContext": "..."}}. +// exit — Always 0. The hook is non-blocking by design. +import { readFileSync } from "node:fs"; + +let raw = ""; +try { raw = readFileSync(0, "utf-8"); } catch { process.exit(0); } +if (!raw) process.exit(0); + +let event; +try { event = JSON.parse(raw); } catch { process.exit(0); } +const prompt = (event?.prompt ?? "").toString(); +if (!prompt) process.exit(0); + +// Robot Framework signal regex (case-insensitive). Conservative on +// purpose — better to under-inject than over-inject. Categories covered: +// - Direct RF mentions: "robot framework", "robot-framework" +// - File extensions: .robot, .resource +// - Library names: SeleniumLibrary, Browser Library, AppiumLibrary, +// RequestsLibrary, RESTinstance, PlatynUI +// - skill ids: rf-setup, rf-language, rf-python-library (rf-core); +// rf-browser, rf-selenium (rf-web); rf-requests, rf-restinstance (rf-api); +// rf-appium (rf-mobile); rf-platynui (rf-desktop) +// - rf-core subagent ids: rf-test-architect, rf-debug-expert, +// rf-migration-guide +// - Tooling: libdoc, robotidy, robocop, rfbrowser, robotcode, +// robot-debug, robot.toml +// - Python library API: @keyword, robot.api.deco, ROBOT_LIBRARY_SCOPE (and +// other ROBOT_LIBRARY_* / ROBOT_LISTENER_* settings), "contains no keywords", +// "robot listener" / "listener API" (bare "listener" is too generic) +// - RF section headers pasted into the prompt: *** Settings ***, *** Variables ***, +// *** Test Cases ***, *** Tasks ***, *** Keywords ***, *** Comments *** +// (singular legacy spellings included) +// Things NOT matched: bare "test" (too noisy), bare "RF" (ambiguous). +const RF_REGEX = /robot[ -]?framework|\.robot\b|\.resource\b|\b(selenium|browser|appium|requests)library\b|\brestinstance\b|\b(selenium|browser|appium|requests) library\b|\bplatynui\b|\blibdoc\b|\b(robotidy|robocop|rfbrowser|robotcode)\b|\brobot-debug\b|\brobot\.toml\b|@keyword\b|\brobot\.api\.deco\b|\bROBOT_(LIBRARY|LISTENER)_[A-Z_]+|contains no keywords|\brobot( framework)? listeners?\b|\blistener api\b|\brf-(setup|language|python-library|browser|selenium|appium|requests|restinstance|platynui)\b|\brf-(test-architect|debug-expert|migration-guide)\b|\*\*\*\s*(settings?|variables?|test cases?|tasks?|keywords?|comments?)\s*\*\*\*/i; + +if (!RF_REGEX.test(prompt)) process.exit(0); + +// Routing text (<= 450 characters, ~80 tokens). Skill descriptions are already in +// the agent's context and subagents are listed by the Task tool; what the model +// lacks is which skill to load when. Names only rf-core skills and the +// technology plugins (tests check it): a library skill may not be installed, +// so library work routes generically and to installing its plugin. +const CONTEXT = [ + "Robot Framework context detected. Load the matching skill before writing RF: ", + "tests, suites, keywords, resources, variables -> rf-language; ", + "Python libraries/listeners -> rf-python-library; installs -> rf-setup; ", + "a library -> its own skill; if missing, suggest its plugin (rf-web, rf-api, rf-mobile, rf-desktop). ", + "Keywords: `robotcode doc keyword`, not memory; results: `robotcode results`, not output.xml. ", + "Write RF 7 syntax: RETURN, VAR, IF, Test Tags.", +].join(""); + +const payload = { + hookSpecificOutput: { + hookEventName: "UserPromptSubmit", + additionalContext: CONTEXT, + }, +}; + +process.stdout.write(JSON.stringify(payload) + "\n"); +process.exit(0); diff --git a/bundles/rf-agentskills/scripts/maybe_remind_robot_tests.mjs b/bundles/rf-agentskills/scripts/maybe_remind_robot_tests.mjs new file mode 100644 index 0000000..fe72b36 --- /dev/null +++ b/bundles/rf-agentskills/scripts/maybe_remind_robot_tests.mjs @@ -0,0 +1,108 @@ +#!/usr/bin/env node +// maybe_remind_robot_tests.mjs — Conditional Stop hook. +// +// Reads the Claude Code Stop event JSON on stdin, inspects the session +// transcript for any tool result that wrote a .robot or .resource file, +// and only emits an additionalContext reminder ("run robot, inspect +// report.html") when at least one such file was touched. Sessions that +// produced only non-RF artifacts get no injection. +// +// Cross-platform port of maybe_remind_robot_tests.sh. Reads the +// transcript via streaming-ish chunking so very long sessions don't +// blow up memory. +// +// Schema: +// stdin — Claude Code Stop event JSON. Includes transcript_path. +// stdout — Either empty (no reminder) or one JSON object with +// hookSpecificOutput.additionalContext. +// exit — Always 0. +import { readFileSync, existsSync, statSync, openSync, readSync, closeSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let raw = ""; +try { raw = readFileSync(0, "utf-8"); } catch { process.exit(0); } +if (!raw) process.exit(0); + +let event; +try { event = JSON.parse(raw); } catch { process.exit(0); } + +// Break the Stop-hook loop. Claude Code sets `stop_hook_active` when this +// Stop fires as a continuation of a previous Stop-hook block; emitting any +// model-facing output here would re-invoke the model and loop until the +// block cap. Always no-op in that state. +if (event?.stop_hook_active) process.exit(0); + +const transcript = (event?.transcript_path ?? "").toString(); +if (!transcript || !existsSync(transcript)) process.exit(0); + +// Fire the reminder at most once per session. Without this, every later turn +// that touched a .robot file re-emits it (the noise the loop amplified). The +// marker is a tiny file in the OS temp dir keyed by session_id; all I/O is +// best-effort and must never break the (non-blocking) hook. +const sessionId = (event?.session_id ?? "").toString(); +let marker = ""; +if (sessionId) { + const safe = sessionId.replace(/[^A-Za-z0-9._-]/g, "_"); + marker = join(tmpdir(), `rf-agentskills-reminded-${safe}`); + try { + if (existsSync(marker)) process.exit(0); + } catch { + // Can't stat the marker — fall through and emit (guard above prevents loops). + } +} + +// Pattern: `"file_path" : "...path.robot"` (or .resource). Matches the +// `grep -E` regex from the bash original. Whitespace optional between +// quotes and colon. +const FILE_PATH_RE = /"file_path"\s*:\s*"[^"]*\.(?:robot|resource)"/i; + +function transcriptMentionsRfFile(path) { + // The transcript is a JSONL file that can be tens of MB on long + // sessions. Read in 64 KB chunks rather than slurping it all. + const size = (() => { try { return statSync(path).size; } catch { return 0; } })(); + if (!size) return false; + const BUF = Buffer.alloc(65536); + const fd = openSync(path, "r"); + let pos = 0; + let carry = ""; + try { + while (pos < size) { + const n = readSync(fd, BUF, 0, BUF.length, pos); + if (n <= 0) break; + // Stitch the boundary with the previous chunk so we don't miss a + // match split across the seam. + const chunk = carry + BUF.slice(0, n).toString("utf-8"); + if (FILE_PATH_RE.test(chunk)) return true; + // Keep the last ~256 bytes as carry to bridge boundary matches. + carry = chunk.length > 256 ? chunk.slice(-256) : chunk; + pos += n; + } + } finally { + try { closeSync(fd); } catch {} + } + return false; +} + +if (!transcriptMentionsRfFile(transcript)) process.exit(0); + +const payload = { + hookSpecificOutput: { + hookEventName: "Stop", + additionalContext: [ + "Robot Framework artifacts detected this session. Recommended next steps:", + '\n 1. Run the affected tests: uv run robotcode robot -bl "" (or a whole directory)', + "\n 2. Read the results: uv run robotcode results summary --failed, then robotcode results log --failed", + " (not using uv: run robotcode from the project's .venv; see rf-setup)", + "\n 3. Open results/report.html for the rendered report.", + ].join(""), + }, +}; + +// Record that this session has been reminded (best-effort). +if (marker) { + try { writeFileSync(marker, ""); } catch { /* ignore — never break the hook */ } +} + +process.stdout.write(JSON.stringify(payload) + "\n"); +process.exit(0); diff --git a/bundles/rf-agentskills/scripts/report_pending_errors.mjs b/bundles/rf-agentskills/scripts/report_pending_errors.mjs new file mode 100644 index 0000000..1ed5255 --- /dev/null +++ b/bundles/rf-agentskills/scripts/report_pending_errors.mjs @@ -0,0 +1,72 @@ +#!/usr/bin/env node +// report_pending_errors.mjs — Stop hook: re-validate the .robot/.resource +// files whose PostToolUse validation found errors or unresolved names this +// session and, if they still fail, block the stop so the agent gets one more +// turn to fix them. Unresolved keywords/variables (advisory on edit) block here. +// +// Needed for VS Code's Local agent mode, which does not wait for PostToolUse +// hooks before the next model request (the error only reaches the user as a +// banner). Agents that do deliver PostToolUse feedback normally fix the file +// first, which clears its entry, so this hook stays silent there. +// +// Output contract (same as validate_robot.mjs): stderr diagnostic + exit 2 +// (Claude Code, Codex and VS Code Local continue with stderr as the reason), +// plus {"decision":"block","reason":…} on stdout for Copilot. +// +// Loop safety: no-op when stop_hook_active is set, and a file is reported at +// most once per content version (its sha1 is remembered), so an unfixable +// file can't block every later turn. +import { existsSync, readFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; +import { contentHash, loadPending, savePending } from "./_pending_errors.mjs"; +import { spawnOptions } from "./_python_env.mjs"; + +let event = null; +try { + event = JSON.parse(readFileSync(0, "utf-8")); +} catch { + process.exit(0); +} +if (!event || event.stop_hook_active) process.exit(0); +const sessionId = (event.session_id ?? "").toString(); +if (!sessionId) process.exit(0); + +const VALIDATOR = join(dirname(fileURLToPath(import.meta.url)), "validate_robot.mjs"); +const errors = []; +const stillPending = Object.entries(loadPending(sessionId)); +if (!stillPending.length) process.exit(0); + +for (const [file, info] of stillPending) { + if (!existsSync(file)) continue; + const hash = contentHash(file); + if (hash && info?.reported === hash) continue; // already reported this version + const child = { ...event, hook_event_name: "PostToolUse", tool_name: "Write", tool_input: { file_path: file } }; + const r = spawnSync( + process.execPath, + [VALIDATOR], + spawnOptions(45000, { + input: JSON.stringify(child), + encoding: "utf-8", + // Stop mode: unresolved keywords/variables block too, advisory tiers are skipped. + env: { ...process.env, TOOL_INPUT: "", RF_AGENTSKILLS_STOP_RECHECK: "1" }, + }), + ); + if (r.status === 2 && (r.stderr ?? "").trim()) errors.push({ file, hash, text: r.stderr.trim() }); +} + +// The validator child runs updated the state (cleared fixed files); reload, +// drop deleted files and remember what we report now. +const state = loadPending(sessionId); +for (const file of Object.keys(state)) if (!existsSync(file)) delete state[file]; +for (const e of errors) state[e.file] = { reported: e.hash }; +savePending(sessionId, state); + +if (!errors.length) process.exit(0); +const text = errors.map((e) => e.text).join("\n"); +const reason = + "Robot Framework validation still reports errors in files you edited. Fix them before finishing:\n" + text; +process.stdout.write(JSON.stringify({ decision: "block", reason }) + "\n"); +process.stderr.write(reason + "\n"); +process.exit(2); diff --git a/bundles/rf-agentskills/scripts/rf_error_hints.mjs b/bundles/rf-agentskills/scripts/rf_error_hints.mjs new file mode 100644 index 0000000..e762553 --- /dev/null +++ b/bundles/rf-agentskills/scripts/rf_error_hints.mjs @@ -0,0 +1,126 @@ +#!/usr/bin/env node +// rf_error_hints.mjs — PostToolUse hook (matcher: Bash). +// +// Scans a Bash command's output for Robot Framework error messages and, the +// first time each kind of error appears in a session, injects one short hint: +// which rf-agentskills skill to load and the concrete fix. Agents reliably read +// the error they just caused; that is where the pointer to the skill helps. +// +// Schema: +// stdin — PostToolUse event JSON for a shell tool, in any agent's shape +// (see _hook_input.mjs): Claude Code / Codex `Bash`, Cursor `Shell`, +// VS Code `run_in_terminal`. Output comes from `tool_response` +// (Copilot CLI: `tool_result`): .stdout / .stderr, or a plain string. +// Other tools exit silently (VS Code runs every hook for every tool). +// stdout — Either empty or one JSON object: +// {"hookSpecificOutput": {"hookEventName": "PostToolUse", +// "additionalContext": "..."}}. +// exit — Always 0. The hook never blocks. +import { readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { normalizeHookInput } from "./_hook_input.mjs"; + +const MAX_SCAN = 20000; // characters of output scanned +const MAX_HINT = 400; // characters per hint +const MAX_NAME = 80; // characters of an offending name quoted in a hint + +// kind, pattern (first group = offending name, if any), hint builder +const SIGNATURES = [ + { + kind: "no-keyword", + re: /No keyword with name '([^']*)' found/, + hint: (name) => + `Robot Framework found no keyword '${name}'. If the call carries values inside the name ` + + "(e.g. 'Select team Los Angeles Lakers'), define an embedded-argument keyword such as " + + "`Select team ${city} ${team:\\S+}`; the text between arguments is literal, not a regex. " + + "Load rf-language. Library keyword names: `robotcode doc keywords `.", + }, + { + kind: "multiple-keywords", + re: /Multiple keywords with name '([^']*)' found/, + hint: (name) => + `Keyword '${name}' is defined more than once. Call it qualified (resource.Keyword) or set ` + + "`Set Library Search Order`, or rename one. See the rf-language skill.", + }, + { + kind: "argument-syntax", + re: /Invalid argument syntax '([^']*)'/, + hint: (name) => + `Invalid argument syntax '${name}'. Typed user-keyword arguments are written \`\${count: int}\` ` + + "and need RF 7.3+; `${count}: int` is invalid. See the rf-language skill.", + }, + { + kind: "variable", + re: /Resolving variable '([^']*)' failed/, + hint: (name) => + `Variable '${name}' could not be resolved. Check its scope (VAR scope=TEST|SUITE|GLOBAL), ` + + "the import order of resource/variable files, and `$var` vs `${var}` in expressions. " + + "See the rf-language skill.", + }, + { + kind: "library-import", + re: /Importing library '([^']*)' failed/, + hint: (name) => + `Library '${name}' could not be imported. Install it into the project environment ` + + "(e.g. `uv add `) and run robot from that environment. See the rf-setup skill.", + }, +]; + +function outputText(response) { + if (response == null) return ""; + if (typeof response === "string") return response; + const parts = []; + for (const key of ["stdout", "stderr", "output", "content"]) { + const v = response[key]; + if (typeof v === "string") parts.push(v); + } + // Unknown shapes (e.g. VS Code tool results): scan their JSON text. + return parts.length ? parts.join("\n") : JSON.stringify(response); +} + +function main() { + let raw = ""; + try { raw = readFileSync(0, "utf-8"); } catch { return; } + if (!raw) return; + let event; + try { event = JSON.parse(raw); } catch { return; } + if (!event || typeof event !== "object") return; + // Only shell output; a tool name of another kind (read_file, …) is skipped. + const input = normalizeHookInput(event); + if (input.tool && !input.isShell) return; + + const text = outputText(event.tool_response ?? event.tool_result).slice(0, MAX_SCAN); + if (!text) return; + + const sessionId = (event.session_id ?? "").toString(); + const marker = sessionId + ? join(tmpdir(), `rf-agentskills-hints-${sessionId.replace(/[^A-Za-z0-9._-]/g, "_")}.json`) + : ""; + let seen = new Set(); + if (marker) { + try { seen = new Set(JSON.parse(readFileSync(marker, "utf-8")).map(String)); } catch { /* none yet */ } + } + + const hints = []; + for (const sig of SIGNATURES) { + if (seen.has(sig.kind)) continue; + const m = sig.re.exec(text); + if (!m) continue; + const name = (m[1] ?? "").slice(0, MAX_NAME); + hints.push(sig.hint(name).slice(0, MAX_HINT)); + seen.add(sig.kind); + } + if (!hints.length) return; + + if (marker) { + try { writeFileSync(marker, JSON.stringify([...seen])); } catch { /* dedupe lost only */ } + } + const payload = { + hookSpecificOutput: { hookEventName: "PostToolUse", additionalContext: hints.join("\n\n") }, + }; + process.stdout.write(JSON.stringify(payload) + "\n"); +} + +try { main(); } catch { /* never block */ } +process.exit(0); diff --git a/bundles/rf-agentskills/scripts/validate_robot.mjs b/bundles/rf-agentskills/scripts/validate_robot.mjs new file mode 100644 index 0000000..b5c0328 --- /dev/null +++ b/bundles/rf-agentskills/scripts/validate_robot.mjs @@ -0,0 +1,509 @@ +#!/usr/bin/env node +// validate_robot.mjs — Per-file static validation for Robot Framework +// .robot / .resource files. +// +// Runs from the PostToolUse hook after Write or Edit. Tiers: +// +// Tier 0 (errors): `robotcode --format json analyze code ` from the +// project root, so robot.toml, profiles, python-path and the installed +// libraries apply. Error-severity diagnostics are classified by code: +// - KeywordNotFound / VariableNotFound -> unresolved name: advisory on +// edit (the agent may be about to write it in another file); the file +// is recorded as pending and blocks at Stop if still unresolved. +// - any other error (ModelError, argument DataError, failed import, …) +// -> blocking error. The hook exits 2 and writes the diagnostic to +// stderr (Claude Code and Codex feed it back to the agent) and also +// prints {"decision":"block","reason":…} on stdout, the only channel +// Copilot CLI passes to its model. +// Warnings, infos and hints are dropped; each list shows at most 5 +// diagnostics. With RF_AGENTSKILLS_STOP_RECHECK=1 (set by +// report_pending_errors.mjs) unresolved names block too and the advisory +// tiers are skipped. No robotcode, a timeout (20 s) or unparsable output = +// silence: a tool failure is never a finding. +// +// Tier 1 (deprecations, one Robocop pass classified by rule ID): +// `robocop check --no-cache` with the project's configured rule set and a +// pipe-separated issue format. Error findings are left to tier 0: +// - DEPR03/04/07/08/09/10/11 -> hard deprecation (advisory warning) +// - any other DEPR rule (DEPR05/06 …) -> modernization hint (advisory) +// - everything else (style rules) -> dropped +// Deprecations ONLY WARN: no DEPR finding ever causes exit 2. Findings on +// lines the edit touched are listed individually (hard first, each with the +// modern replacement), capped at 10 lines / 2,000 characters and not +// repeated within one session; other deprecations are counted per rule. +// RF_AGENTSKILLS_DEPRECATION_CHECK=off turns the deprecation output off; +// any other value (including a legacy `block`) means `warn`. +// No parsable line from a failing Robocop run = tool failure = silence. +// +// Tier 2 (formatting drift): `robocop format --check --diff`. Informational +// additionalContext only. +// +// Tier 3 (opt-in, RF_AGENTSKILLS_FILE_DRYRUN=1): `robot --dryrun` on the +// edited suite (.robot only, not __init__.robot). Advisory context only. +// +// Advisory output (tiers 0-3) is written only when no exit-2 diagnostic is. +// A file with blocking errors or unresolved names is also recorded per session +// (_pending_errors.mjs) and cleared by its next clean run; report_pending_errors.mjs +// re-checks the recorded files at Stop. Every spawned process has a timeout +// (robotcode 20 s, Robocop 10 s, dry run 20 s, probes 5 s); a timeout skips +// that tier silently. `--no-cache` keeps Robocop from leaving a .robocop_cache/ +// directory; robotcode keeps its normal .robotcode_cache/ (the warm cache makes +// the check take about a second). +// +// Input: the PostToolUse event JSON on stdin (legacy `TOOL_INPUT` env var as a +// fallback), in any agent's shape — see _hook_input.mjs (Claude Code, Copilot +// CLI, Cursor, VS Code, Codex). Non-edit tools exit silently; a patch that +// touches several .robot/.resource files is validated file by file. Tools: see +// _python_env.mjs (VIRTUAL_ENV -> /.venv -> python_runtime.json -> PATH); +// robotcode, Robocop and Robot Framework are each optional — a missing tool +// makes its tier a silent no-op. +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { spawn, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { tmpdir } from "node:os"; +import { basename, dirname, join, relative, isAbsolute } from "node:path"; +import { fileURLToPath } from "node:url"; +import { readHookInput } from "./_hook_input.mjs"; +import { recordResult } from "./_pending_errors.mjs"; +import { findInterpreterWith, findRobotcode, projectRoot, spawnOptions } from "./_python_env.mjs"; + +// Child processes still running; killed when the hook exits early. +const running = new Set(); +const ANALYZE_TIMEOUT_MS = 20000; +const ROBOCOP_TIMEOUT_MS = 10000; +const DRYRUN_TIMEOUT_MS = 20000; +const MAX_LISTED = 10; +const MAX_DIAGNOSTICS = 5; +const MAX_CHARS = 2000; +// Diagnostics that may resolve once the agent writes the next file. +const UNRESOLVED_CODES = new Set(["KeywordNotFound", "VariableNotFound"]); +const STOP_RECHECK = /^(1|true|yes|on)$/i.test((process.env.RF_AGENTSKILLS_STOP_RECHECK ?? "").trim()); + +// Hard deprecations: listed first, with the modern replacement. +const HARD_DEPRECATIONS = { + DEPR03: "`WITH NAME` → use `AS`", + DEPR04: "singular section header → use the plural header (`*** Test Cases ***`, `*** Keywords ***`)", + DEPR07: "`Force Tags`/`Default Tags` → use `Test Tags` (and `[Tags] -tag`)", + DEPR08: "`Run Keyword If`/`Unless` → use `IF`/`ELSE`/`END`", + DEPR09: "`Exit For Loop`/`Continue For Loop` → use `BREAK`/`CONTINUE`", + DEPR10: "`Return From Keyword (If)` → use `RETURN` (inside `IF`)", + DEPR11: "`[Return]` → use `RETURN`", +}; +// Modernization hints (RF 7 `VAR`); Robocop already gates them by RF version. +const HINTS = { + DEPR05: "`Set Test/Suite/Global Variable` → `VAR ${name} value scope=TEST|SUITE|GLOBAL` (RF 7+)", + DEPR06: "`Create List`/`Create Dictionary` → `VAR @{list}` / `VAR &{dict}` (RF 7+)", +}; + +const ISSUE_RE = /^([IWE])\|([A-Z]+\d+)\|(.+):(\d+):(\d+)\|(.*)$/; + +function isTruthy(v) { + return /^(1|true|yes|on)$/i.test((v ?? "").toString().trim()); +} + +const input = readHookInput(); +const event = input.event; +const robotFiles = input.isEdit + ? input.files.filter((f) => /\.(robot|resource)$/i.test(f) && existsSync(f)) + : []; +if (!robotFiles.length) process.exit(0); +if (robotFiles.length > 1) validateEach(robotFiles); +const filePath = robotFiles[0]; + +/** Report real errors: stderr + exit 2, plus the decision JSON for Copilot. */ +function reportErrors(text) { + const reason = + "The edit was applied, but Robot Framework validation found errors. Fix them:\n" + text.trim(); + process.stdout.write(JSON.stringify({ decision: "block", reason }) + "\n"); + process.stderr.write(text.endsWith("\n") ? text : text + "\n"); + for (const child of running) child.kill("SIGKILL"); + process.exit(2); +} + +// One edit (an apply_patch) touched several files: validate each in a child run +// of this script, then report the merged result in the same exit-code contract. +function validateEach(files) { + const self = fileURLToPath(import.meta.url); + const errors = []; + const contexts = []; + for (const f of files) { + const child = { ...event, tool_name: "Write", tool_input: { file_path: f } }; + delete child.tool_response; + delete child.tool_result; + const r = spawnSync( + process.execPath, + [self], + spawnOptions(ANALYZE_TIMEOUT_MS + ROBOCOP_TIMEOUT_MS * 2 + DRYRUN_TIMEOUT_MS, { + input: JSON.stringify(child), encoding: "utf-8", env: { ...process.env, TOOL_INPUT: "" }, + }), + ); + if (r.status === 2) errors.push((r.stderr ?? "").trim()); + try { + const ctx = JSON.parse(r.stdout || "null")?.hookSpecificOutput?.additionalContext; + if (ctx) contexts.push(ctx); + } catch { + // no advisory output + } + } + if (errors.length) reportErrors(errors.join("\n")); + if (contexts.length) { + const payload = { hookSpecificOutput: { hookEventName: "PostToolUse", additionalContext: contexts.join("\n\n") } }; + process.stdout.write(JSON.stringify(payload) + "\n"); + } + process.exit(0); +} + +const cwd = projectRoot(event); +let fileLines = []; +try { + fileLines = readFileSync(filePath, "utf-8").split(/\r?\n/); +} catch { + process.exit(0); +} + +// ── Touched lines (design D4) ─────────────────────────────────────────────── +// Returns a Set of 1-based line numbers, or null for "all lines touched". +function touchedLines() { + const response = event?.tool_response; + const patch = response && Array.isArray(response.structuredPatch) ? response.structuredPatch : null; + // 1. structuredPatch: '+' lines of each hunk (Edit, Write update). + if (patch && patch.length) { + const set = new Set(); + for (const hunk of patch) { + let line = Number(hunk?.newStart); + if (!Number.isInteger(line) || !Array.isArray(hunk?.lines)) return null; + for (const l of hunk.lines) { + const s = String(l); + if (s.startsWith("+")) { + set.add(line); + line += 1; + } else if (s.startsWith("-")) { + // removed line: no new-file line + } else if (s.startsWith("\\")) { + // "\ No newline at end of file" + } else { + line += 1; + } + } + } + return set; + } + const toolName = (event?.tool_name ?? "").toString(); + const newString = event?.tool_input?.new_string; + // 2. Edit without a patch: the spans where new_string occurs now. + if (/^(Edit|MultiEdit)$/.test(toolName) && typeof newString === "string" && newString.trim()) { + const text = fileLines.join("\n"); + const set = new Set(); + let from = 0; + for (;;) { + const idx = text.indexOf(newString, from); + if (idx < 0) break; + const start = text.slice(0, idx).split("\n").length; + const span = newString.split("\n").length; + for (let i = 0; i < span; i++) set.add(start + i); + from = idx + Math.max(newString.length, 1); + } + return set.size ? set : null; + } + // 3. Write (create, or no patch) and anything else: the whole file. + return null; +} + +// ── Per-session dedupe marker (design D5) ─────────────────────────────────── +const sessionId = (event?.session_id ?? "").toString(); +const markerPath = sessionId + ? join(tmpdir(), `rf-agentskills-depr-${sessionId.replace(/[^A-Za-z0-9._-]/g, "_")}.json`) + : ""; + +function loadSeen() { + if (!markerPath) return new Set(); + try { + const data = JSON.parse(readFileSync(markerPath, "utf-8")); + return new Set(Array.isArray(data) ? data.map(String) : []); + } catch { + return new Set(); + } +} + +function saveSeen(seen) { + if (!markerPath) return; + try { + writeFileSync(markerPath, JSON.stringify([...seen].slice(-2000))); + } catch { + // Read-only temp dir etc. — only the dedupe is lost. + } +} + +function findingKey(f) { + const text = (fileLines[f.line - 1] ?? "").trim(); + return createHash("sha1").update(`${filePath}\0${f.rule}\0${text}`).digest("hex"); +} + +// The robotcode and Robocop runs are independent, so they run in parallel; the +// hook then takes about as long as the slowest of them. +function spawnAsync(cmd, args, defaultTimeoutMs, runDir = cwd) { + const opts = spawnOptions(defaultTimeoutMs); + return new Promise((resolve) => { + let child; + try { + child = spawn(cmd, args, { cwd: runDir, windowsHide: true, stdio: ["ignore", "pipe", "pipe"] }); + } catch (error) { + resolve({ error, status: null, stdout: "" }); + return; + } + running.add(child); + let stdout = ""; + let done = false; + const finish = (res) => { + if (done) return; + done = true; + clearTimeout(timer); + running.delete(child); + resolve(res); + }; + const timer = setTimeout(() => { + child.kill(opts.killSignal); + finish({ error: new Error("timeout"), status: null, stdout }); + }, opts.timeout); + child.stdout.setEncoding("utf-8"); + child.stdout.on("data", (chunk) => { + if (stdout.length < opts.maxBuffer) stdout += chunk; + }); + child.stderr.resume(); + child.on("error", (error) => finish({ error, status: null, stdout })); + child.on("close", (status) => finish({ error: null, status, stdout })); + }); +} + +function exitNow(code) { + for (const child of running) child.kill("SIGKILL"); + process.exit(code); +} + +// ── Tier 0: robotcode analyze code ────────────────────────────────────────── +// Returns {blocking, unresolved} (lists of {line, col, code, message}) or null +// when robotcode is missing, timed out or printed nothing parsable. +function startAnalyze() { + const rc = findRobotcode(cwd); + if (!rc) return Promise.resolve(null); + // robotcode only analyzes files under the root it finds from its working + // directory; a file outside the event's project is analyzed from its folder. + const rel = relative(cwd, filePath); + const runDir = rel && !rel.startsWith("..") && !isAbsolute(rel) ? cwd : dirname(filePath); + return spawnAsync(rc.exe, ["--format", "json", "--no-color", "analyze", "code", filePath], ANALYZE_TIMEOUT_MS, runDir); +} + +function parseAnalyze(r) { + if (!r || r.error || r.status === null) return null; + let data; + try { + data = JSON.parse(r.stdout || ""); + } catch { + return null; + } + const byFile = data && typeof data.diagnostics === "object" && data.diagnostics ? data.diagnostics : null; + if (!byFile) return null; + if (data.summary && data.summary.files === 0) return null; // nothing analyzed: no result + const blocking = []; + const unresolved = []; + for (const list of Object.values(byFile)) { + for (const d of Array.isArray(list) ? list : []) { + if (d?.severity !== 1) continue; // LSP severity 1 = error + const item = { + line: Number(d.range?.start?.line ?? 0) + 1, + col: Number(d.range?.start?.character ?? 0) + 1, + code: String(d.code ?? "Error"), + message: String(d.message ?? "").trim(), + }; + (UNRESOLVED_CODES.has(item.code) ? unresolved : blocking).push(item); + } + } + const order = (a, b) => a.line - b.line || a.col - b.col; + return { blocking: blocking.sort(order), unresolved: unresolved.sort(order) }; +} + +function diagnosticLines(items) { + const lines = items + .slice(0, MAX_DIAGNOSTICS) + .map((d) => `${filePath}:${d.line}:${d.col}: [ERROR] ${d.code}: ${d.message}`); + if (items.length > MAX_DIAGNOSTICS) lines.push(`… ${items.length - MAX_DIAGNOSTICS} more omitted.`); + return lines.join("\n"); +} + +// ── Tier 1: one Robocop pass ──────────────────────────────────────────────── +function startRobocop(py) { + return spawnAsync( + py, + [ + "-m", "robocop", "check", "--no-cache", + "-c", "print_issues.output_format=simple", + "--issue-format", "{severity}|{rule_id}|{source}:{line}:{col}|{desc}", + filePath, + ], + ROBOCOP_TIMEOUT_MS, + ); +} + +function parseRobocop(r) { + if (!r || r.error || r.status === null) return null; // timeout, overflow, crash + const findings = []; + for (const line of (r.stdout ?? "").split(/\r?\n/)) { + const m = ISSUE_RE.exec(line.trim()); + if (m) { + findings.push({ + severity: m[1], rule: m[2], line: Number(m[4]), col: Number(m[5]), desc: m[6], raw: line.trim(), + }); + } + } + // A failing run that printed nothing parsable is a tool failure, not a finding. + if (!findings.length && r.status !== 0) return null; + return findings; +} + +function deprecationText(deprFindings) { + const touched = touchedLines(); + const seen = loadSeen(); + const listed = []; + const counted = {}; + for (const f of deprFindings) { + const isTouched = touched === null || touched.has(f.line); + if (isTouched && !seen.has(findingKey(f))) listed.push(f); + else counted[f.rule] = (counted[f.rule] ?? 0) + 1; + } + const rank = (f) => (f.rule in HARD_DEPRECATIONS ? 0 : 1); + listed.sort((a, b) => rank(a) - rank(b) || a.line - b.line || a.col - b.col); + if (!listed.length && !Object.keys(counted).length) return ""; + + const name = basename(filePath); + const header = + `Deprecated Robot Framework syntax in ${filePath} (advisory, not blocking; ` + + "modern forms: rf-language skill, references/migration.md):"; + const lines = [header]; + let used = header.length; + let shown = 0; + const emitted = []; + for (const f of listed) { + const hard = f.rule in HARD_DEPRECATIONS; + const replacement = HARD_DEPRECATIONS[f.rule] ?? HINTS[f.rule] ?? f.desc; + const line = ` ${hard ? "WARN" : "HINT"} ${f.rule} ${name}:${f.line} — ${replacement}`; + if (shown >= MAX_LISTED || used + line.length + 1 > MAX_CHARS - 260) break; + lines.push(line); + used += line.length + 1; + emitted.push(f); + shown += 1; + } + const omitted = listed.length - shown; + if (omitted > 0) lines.push(` … ${omitted} more deprecation finding(s) omitted.`); + const counts = Object.entries(counted).sort(([a], [b]) => a.localeCompare(b)); + if (counts.length) { + lines.push( + " Also in this file (lines not changed by this edit, or already reported this session): " + + counts.map(([rule, n]) => `${rule} ×${n}`).join(", ") + ".", + ); + } + lines.push( + " Silence with RF_AGENTSKILLS_DEPRECATION_CHECK=off or by ignoring the rule in the Robocop config.", + ); + for (const f of emitted) seen.add(findingKey(f)); + saveSeen(seen); + let text = lines.join("\n"); + if (text.length > MAX_CHARS) text = text.slice(0, MAX_CHARS - 1) + "…"; + return text; +} + +// ── Tier 3 (opt-in): per-file dry run ─────────────────────────────────────── +function dryRunText() { + if (!isTruthy(process.env.RF_AGENTSKILLS_FILE_DRYRUN)) return ""; + if (!/\.robot$/i.test(filePath) || /^__init__\.robot$/i.test(basename(filePath))) return ""; + const py = findInterpreterWith("robot", cwd); + if (!py) return ""; + const r = spawnSync( + py, + ["-m", "robot", "--dryrun", "--output", "NONE", "--report", "NONE", "--log", "NONE", + "--console", "dotted", filePath], + spawnOptions(DRYRUN_TIMEOUT_MS, { encoding: "utf-8", cwd }), + ); + if (r.error || r.status === null) return ""; + const out = `${r.stdout ?? ""}\n${r.stderr ?? ""}`.split(/\r?\n/); + const items = []; + for (let i = 0; i < out.length; i++) { + const l = out[i]; + if (l.includes("[ ERROR ]") && !/contains no tests/i.test(l)) items.push(l.trim()); + else if (l.startsWith("FAIL: ")) { + const msg = []; + for (let j = i + 1; j < out.length && !/^[-=]{10,}/.test(out[j]); j++) { + if (out[j].trim()) msg.push(out[j].trim()); + } + items.push(`${l.slice(6).trim()}: ${msg.join(" ")}`); + } + } + if (!items.length) return ""; + const shown = items.slice(0, MAX_LISTED).map((s) => ` ${s.length > 300 ? s.slice(0, 299) + "…" : s}`); + if (items.length > MAX_LISTED) shown.push(` … ${items.length - MAX_LISTED} more omitted.`); + let text = `robot --dryrun of ${filePath} (advisory; a keyword you write next may still be missing):\n` + + shown.join("\n"); + if (text.length > MAX_CHARS) text = text.slice(0, MAX_CHARS - 1) + "…"; + return text; +} + +// ── Main ──────────────────────────────────────────────────────────────────── +const advisory = []; +const robocopPy = STOP_RECHECK ? null : findInterpreterWith("robocop", cwd); +const analyzeRun = startAnalyze(); +const robocopRun = robocopPy ? startRobocop(robocopPy) : Promise.resolve(null); +const formatRun = robocopPy + ? spawnAsync(robocopPy, ["-m", "robocop", "format", "--no-cache", "--check", "--diff", "--no-overwrite", filePath], + ROBOCOP_TIMEOUT_MS) + : Promise.resolve(null); +const analysis = parseAnalyze(await analyzeRun); +if (analysis) { + const { blocking, unresolved } = analysis; + // Remember the result for the Stop-time re-check (report_pending_errors.mjs). + recordResult(sessionId, filePath, blocking.length + unresolved.length > 0); + const blockingNow = STOP_RECHECK ? [...blocking, ...unresolved] : blocking; + if (blockingNow.length) { + reportErrors(`Robot Framework validation found errors in ${filePath}:\n` + diagnosticLines(blockingNow)); + } + if (unresolved.length) { + advisory.push( + `robotcode cannot resolve these names in ${filePath} yet (advisory; define or import them before you ` + + "finish, the check blocks at the end of the turn otherwise):\n" + diagnosticLines(unresolved), + ); + } +} +if (STOP_RECHECK) exitNow(0); + +if (robocopPy) { + const findings = parseRobocop(await robocopRun); + if (findings) { + const mode = (process.env.RF_AGENTSKILLS_DEPRECATION_CHECK ?? "").trim().toLowerCase(); + if (mode !== "off") { + const text = deprecationText(findings.filter((f) => f.rule.startsWith("DEPR"))); + if (text) advisory.push(text); + } + + // ── Tier 2: formatting drift (informational only) ── + const fmt = await formatRun; + if (fmt && !fmt.error && fmt.status !== 0 && fmt.status !== null && (fmt.stdout ?? "").trim()) { + let diff = fmt.stdout.trim(); + if (diff.length > MAX_CHARS) diff = diff.slice(0, MAX_CHARS - 1) + "…"; + advisory.push( + `Robocop suggests formatting changes for ${filePath} (run \`robocop format\` to apply):\n` + diff, + ); + } + } +} + +const dry = dryRunText(); +if (dry) advisory.push(dry); + +if (advisory.length) { + const payload = { + hookSpecificOutput: { + hookEventName: "PostToolUse", + additionalContext: advisory.join("\n\n"), + }, + }; + process.stdout.write(JSON.stringify(payload) + "\n"); +} +process.exit(0); diff --git a/bundles/rf-agentskills/scripts/validate_robot_project.mjs b/bundles/rf-agentskills/scripts/validate_robot_project.mjs new file mode 100644 index 0000000..05ee8b4 --- /dev/null +++ b/bundles/rf-agentskills/scripts/validate_robot_project.mjs @@ -0,0 +1,103 @@ +#!/usr/bin/env node +// validate_robot_project.mjs — Opt-in, end-of-task project-wide validation +// for Robot Framework suites. Runs from the Stop hook. +// +// This tier performs CROSS-FILE / semantic checks that only make sense +// once the whole project is on disk — running them per-save would +// false-alarm constantly (a keyword you just wrote looks "unused" until +// something calls it). One tool, one run: +// +// `robotcode --format json analyze code --collect-unused` from the project +// root. It reads robot.toml (paths, python-path, variables) and reports +// - every error-severity diagnostic (unresolved keywords and variables, +// failed imports, argument errors, model errors), and +// - unused keywords and variables (KeywordNotUsed / VariableNotUsed). +// The JSON is parsed; robotcode's bitmask exit code is not used. +// +// OPT-IN: the entire tier is gated behind the RF_AGENTSKILLS_PROJECT_VALIDATION +// env var (truthy = 1/true/yes/on). It is OFF by default because the analysis +// loads the project's libraries (to read their keywords) and scales with +// project size. When enabled and findings exist, the hook exits 2 with the +// diagnostic on stderr so the agent gets one more turn to fix before the turn +// ends. +// +// Graceful degradation: silent no-op (exit 0) when the flag is unset, robotcode +// isn't installed, the run times out (120 s) or its output is unparsable. +import { readFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { findRobotcode, projectRoot as projectRootOf, spawnOptions } from "./_python_env.mjs"; + +// Project-wide checks scale with project size: bounded, but more generous than +// the per-file check. A timeout skips the check silently. +const PROJECT_TIMEOUT_MS = 120000; +const MAX_LISTED = 20; +const UNUSED_CODES = new Set(["KeywordNotUsed", "VariableNotUsed"]); + +// ── Gate: opt-in only ─────────────────────────────────────────────────────── +function isTruthy(v) { + return /^(1|true|yes|on)$/i.test((v ?? "").toString().trim()); +} +if (!isTruthy(process.env.RF_AGENTSKILLS_PROJECT_VALIDATION)) process.exit(0); + +// Read the Stop event once (stdin can only be consumed once). Break the +// Stop-hook loop: when firing as a continuation of a previous Stop block +// (`stop_hook_active`), exit 2 here would re-block on the same persistent +// finding forever — so no-op in that state. +let stopEvent = null; +try { + stopEvent = JSON.parse(readFileSync(0, "utf-8")); +} catch { + // No / unparsable stdin — proceed with process cwd. +} +if (stopEvent?.stop_hook_active) process.exit(0); + +// Resolve the project root from the (already-parsed) Stop event JSON (cwd), +// falling back to the process cwd. +const projectRoot = projectRootOf(stopEvent); + +const findings = []; + +const rc = findRobotcode(projectRoot); +if (rc) { + const r = spawnSync( + rc.exe, + ["--format", "json", "--no-color", "analyze", "code", "--collect-unused"], + spawnOptions(PROJECT_TIMEOUT_MS, { encoding: "utf-8", cwd: projectRoot }), + ); + let data = null; + if (!r.error && r.status !== null) { + try { + data = JSON.parse(r.stdout || ""); + } catch { + data = null; // unparsable: tool failure, not a finding + } + } + const errors = []; + const unused = []; + for (const [file, list] of Object.entries(data?.diagnostics ?? {})) { + for (const d of Array.isArray(list) ? list : []) { + const code = String(d?.code ?? ""); + const line = `${file}:${Number(d?.range?.start?.line ?? 0) + 1}:${Number(d?.range?.start?.character ?? 0) + 1}: ` + + `${code}: ${String(d?.message ?? "").trim()}`; + if (UNUSED_CODES.has(code)) unused.push(line); + else if (d?.severity === 1) errors.push(line); + } + } + const listed = (items) => + items.slice(0, MAX_LISTED).join("\n ") + + (items.length > MAX_LISTED ? `\n … ${items.length - MAX_LISTED} more omitted.` : ""); + if (errors.length) findings.push("Errors (robotcode analyze code):\n " + listed(errors)); + if (unused.length) findings.push("Unused keywords and variables:\n " + listed(unused)); +} + +// ── Report ────────────────────────────────────────────────────────────────── +if (findings.length) { + process.stderr.write( + "Robot Framework project validation found issues:\n\n" + + findings.join("\n\n") + + "\n", + ); + process.exit(2); +} + +process.exit(0); diff --git a/bundles/rf-agentskills/skills/rf-appium/SKILL.md b/bundles/rf-agentskills/skills/rf-appium/SKILL.md new file mode 100644 index 0000000..3b91672 --- /dev/null +++ b/bundles/rf-agentskills/skills/rf-appium/SKILL.md @@ -0,0 +1,134 @@ +--- +name: rf-appium +description: "Use first, before answering, for RF mobile tests with AppiumLibrary. Not installs/drivers: rf-setup." +license: Apache-2.0 +compatibility: Requires Python 3.10+, robotframework>=7, robotcode[all]>=2.8 in the project environment and robotframework-appiumlibrary 3.x, an Appium server 2+/3 (Node.js) with the UiAutomator2 or XCUITest driver, the Android SDK or Xcode, and a device or emulator. +metadata: + author: manykarim + version: "1.0.0" +--- + +# AppiumLibrary Skill + +AppiumLibrary drives native, hybrid and mobile-web apps on Android and iOS through an Appium 2+ server. +Desktop apps belong to rf-platynui (plugin rf-desktop); desktop browsers to rf-browser (plugin rf-web) or rf-selenium. +Verified against AppiumLibrary 3.2.1, Robot Framework 7.4. + +## When to use + +Load this skill first, before exploring the project or answering from memory, when a Robot Framework test drives a mobile app with AppiumLibrary (`Library AppiumLibrary`) on Android or iOS: + +- writing or fixing a mobile test: Open Application capabilities (UiAutomator2, XCUITest), locators such as `accessibility_id`, waits, gestures and scrolling, the on-screen keyboard, WEBVIEW contexts in hybrid apps; +- an element cannot be located on a device, or the user mentions Appium, an .apk/.ipa, an emulator or a simulator, even if they only say "our app" or "mobile test". + +Not this skill: native desktop apps use `rf-platynui` (plugin `rf-desktop`); desktop browsers use `rf-browser` (plugin `rf-web`) or `rf-selenium`; installing the Appium server and drivers uses `rf-setup` (plugin `rf-core`). + +## Installation + +Install into the project environment, with the tool the project already uses. Full steps and verification: `references/install.md`. For a new environment, Poetry and venv + pip forms, and troubleshooting, see `rf-setup` (plugin `rf-core`). + +```bash +uv add robotframework-appiumlibrary +``` +The Appium server and platform drivers live outside the Python environment and are installed with npm (requires Node.js): + +```bash +npm install -g appium +appium driver install uiautomator2 # Android +appium driver install xcuitest # iOS +``` +Android SDK, JDK and Xcode prerequisites, and checking the drivers: `references/install.md`. + +## Import and defaults + +Start the server (`appium`, default `http://127.0.0.1:4723`, no `/wd/hub` base path) before the suite. Default: one session per suite, Android, UiAutomator2, W3C capabilities with the `appium:` prefix written out: + +```robotframework +*** Settings *** +Library AppiumLibrary timeout=15s +Suite Setup Open Android App +Suite Teardown Close All Applications + +*** Variables *** +${APPIUM_URL} http://127.0.0.1:4723 + +*** Keywords *** +Open Android App + Open Application ${APPIUM_URL} + ... platformName=Android + ... appium:automationName=UiAutomator2 + ... appium:deviceName=emulator-5554 + ... appium:app=${CURDIR}/app.apk + ... appium:autoGrantPermissions=${True} +``` + +`timeout=15s` because the library default (5 s) is short for emulators; it drives every `Wait Until …` keyword. `run_on_failure` already captures a screenshot. For iOS (`XCUITest`), an installed app (`appPackage`/`bundleId`), mobile browsers or cloud grids, use the capability sets in `references/capabilities.md`. + +## Which keyword for which situation + +| Situation | Use | Details | +|---|---|---| +| Find an element | `accessibility_id=…`, then `id=…`, `android=…` / `predicate=…` / `chain=…`, XPath last | `references/locators-mobile.md` | +| Wait for a screen or element | `Wait Until Page Contains Element`, `Wait Until Element Is Visible` | — | +| Assert visible / enabled / gone | `Expect Element visible` (also `not visible`, `enabled`, `disabled`); text: `Expect Text` | — | +| Assert text of an element | `Element Text Should Be`, `Element Should Contain Text` | — | +| Type, clear, hide keyboard | `Input Text`, `Clear Text`, `Hide Keyboard` | — | +| Tap, long press, double tap | `Tap duration=2s`, `Tap count=2` | `references/gestures-touch.md` | +| Scroll or swipe | `Scroll Down` (Android), `Swipe By Percent`, `Swipe start_x=…` | `references/gestures-touch.md` | +| Pinch, zoom, other gestures | `Execute Script mobile: pinchOpenGesture elementId=…` | `references/gestures-touch.md` | +| Hybrid app web content | `Get Contexts`, `Switch To Context` | `references/platform-specific.md` | +| Android keys and activities, iOS alerts | `Press Keycode`, `Start Activity`, `Click Alert Button` | `references/platform-specific.md` | +| Inspect the screen for locators | `Log Source`, `Get Source` | `references/troubleshooting.md` | + +## Agent workflow + +1. Look up each keyword before you use it: `robotcode doc keyword AppiumLibrary ""`; find names with `robotcode doc keywords AppiumLibrary ""`. Copy names and arguments exactly as robotcode shows them. +2. Write the test with the defaults above and the decision table. +3. Check the changed files statically: `robotcode analyze code ` (through the project environment, e.g. `uv run robotcode …`). Fix every error it reports. +4. Run them by longname: `robotcode robot -bl ""` (longnames from `robotcode discover tests`), or pass a whole directory. Don't pass a single `.robot` file: it skips the parent `__init__.robot`. +5. Read the results: `robotcode results show --failed`, then `robotcode results log -bl ""`. Don't parse output.xml by hand. +6. Fix the cause and go back to step 3 (static check). Stop when the tests pass or the failure is shown to be in the system under test, and say which. + +## Gotchas + +- **Actions don't wait** — `Click Element`, `Input Text` and `Get Text` look the element up once and fail with "did not match any elements" if the screen isn't there yet, because only the `Wait Until …` keywords and `Expect Element` poll. Put `Wait Until Page Contains Element` before the first action on a new screen; no `Sleep`. +- **`Element Should Be Visible/Enabled/Disabled` and `Text Should Be Visible` are deprecated** — libdoc marks them "Use `Expect Element`". Write `Expect Element visible` or `Expect Text visible`; both retry for `timeout=5s` by default (their own argument, not the import `timeout`). +- **Removed keywords models still write** — `Long Press`, `Zoom`, `Quit Application` and the others below no longer exist in AppiumLibrary 3.2.1 and fail with "No keyword with name". Use the replacements: + + | Removed | Use | + |---|---| + | `Long Press` | `Tap duration=2s` | + | `Click A Point` | `Tap With Positions 500ms ${{ (x, y) }}` | + | `Zoom` / `Pinch` | `Execute Script` with `mobile: pinchOpenGesture` / `pinchCloseGesture` (Android), `mobile: pinch` (iOS) | + | `Quit Application` / `Reset Application` | `Close Application` (then `Open Application`) | + | `Background App` | `Background Application` | + | `Get Window Size` | `Get Window Width` / `Get Window Height` | + +- **Durations need units** — `Swipe` and `Tap With Positions` read a bare number as milliseconds (and warn that bare numbers are going away), while `Tap duration=2` means two seconds. Write `duration=500ms` or `duration=1s`. `Swipe` arguments are keyword-only: `Swipe start_x=500 start_y=1500 end_x=500 end_y=400 duration=500ms`. `Tap With Positions` takes the duration first: `Tap With Positions 300ms ${{ (500, 800) }}`. +- **Locator prefixes** — without a prefix the text is an `id` (XPath only if it starts with `//`), and any unknown `name=` style prefix fails with "prefix … is not supported". iOS predicate strings use `predicate=`, class chains `chain=`; the `ios=` strategy is the retired UI Automation lookup and fails with the current Appium Python client. +- **Capabilities** — the Appium Python client adds `appium:` to every name that is not W3C-standard and has no `:`, so `automationName=` works, but vendor capabilities keep their own prefix (`bstack:options=`, `sauce:options=`). Writing `appium:` yourself keeps logs and cloud docs aligned. +- **Hybrid context sticks** — after `Switch To Context WEBVIEW_…` every locator is a web locator (`css=`, `xpath=`); native locators fail until `Switch To Context NATIVE_APP`. +- **Sessions leak** — a failed test leaves the app session open; close it with `Close All Applications` in Suite Teardown (or `Close Application` in Test Teardown for per-test sessions). + +## When to read the references + +| Read | When | +|---|---| +| `references/install.md` | installing the library, its post-install steps, verifying the install, or a failed post-install step | +| `references/capabilities.md` | iOS, installed apps, mobile browsers, cloud grids, reset and timeout capabilities | +| `references/locators-mobile.md` | Choosing or debugging a locator (UiAutomator, predicate, class chain, XPath) | +| `references/gestures-touch.md` | Scrolling to an element, swipes, long press, pinch/zoom, drag and drop | +| `references/platform-specific.md` | Android keys, activities, permissions, iOS alerts, pickers, hybrid apps | +| `references/troubleshooting.md` | An error message from the server, session creation, a device or a locator | + +Examples in `assets/examples/`: `android-basic.robot`, `ios-basic.robot`, `gestures-example.robot`, `hybrid-app.robot`. + +## Companion Skills + +| Need | Skill | +|------|-------| +| Native desktop tests with PlatynUI | `rf-platynui` (plugin `rf-desktop`) | +| Install Robot Framework or a library, fix the environment | `rf-setup` (plugin `rf-core`) | +| Write tests, suites, user keywords, resources and variables in Robot Framework syntax | `rf-language` (plugin `rf-core`) | + +For running, discovering, keyword documentation, result analysis and debugging with the robotcode CLI, use the `robotcode` skill of the robotcode plugin (`robotcode@robotframework-agent-plugins`); the commands above work without it. diff --git a/bundles/rf-agentskills/skills/rf-appium/assets/examples/android-basic.robot b/bundles/rf-agentskills/skills/rf-appium/assets/examples/android-basic.robot new file mode 100644 index 0000000..bf54470 --- /dev/null +++ b/bundles/rf-agentskills/skills/rf-appium/assets/examples/android-basic.robot @@ -0,0 +1,60 @@ +*** Settings *** +Documentation Android native app: login, list scrolling, device keys and orientation. +... Needs an Appium 2+ server with the UiAutomator2 driver and an emulator. +Library AppiumLibrary timeout=15s +Suite Setup Open Android App +Suite Teardown Close All Applications +Test Teardown Portrait + +*** Variables *** +${APPIUM_URL} http://127.0.0.1:4723 +${APP} ${CURDIR}${/}apps${/}demo-app.apk +${PACKAGE} com.example.demo + +*** Test Cases *** +Valid Login Shows Welcome Text + Log In As demo demo + Expect Element accessibility_id=welcome_text visible + Element Text Should Be accessibility_id=welcome_text Welcome, demo + +Invalid Login Shows Error + Log In As demo wrong + Expect Text Invalid credentials visible + +Item Deep In A List Can Be Opened + Click Element accessibility_id=nav_items + Click Element android=new UiScrollable(new UiSelector().scrollable(true)).scrollIntoView(new UiSelector().text("Item 50")) + Wait Until Page Contains Element accessibility_id=item_detail + +Back Key Returns To Previous Screen + Click Element accessibility_id=nav_settings + Wait Until Page Contains Element id=${PACKAGE}:id/settings_title + Press Keycode 4 + Expect Element id=${PACKAGE}:id/settings_title not visible + +App Survives Rotation And Background + Landscape + Expect Element accessibility_id=home_screen visible + Background Application 2 + ${state}= Execute Script mobile: queryAppState appId=${PACKAGE} + Should Be Equal As Integers ${state} 4 + +*** Keywords *** +Open Android App + Open Application ${APPIUM_URL} + ... platformName=Android + ... appium:automationName=UiAutomator2 + ... appium:deviceName=emulator-5554 + ... appium:app=${APP} + ... appium:autoGrantPermissions=${True} + Wait Until Page Contains Element accessibility_id=home_screen + +Log In As + [Arguments] ${username} ${password} + Click Element accessibility_id=nav_login + Wait Until Element Is Visible accessibility_id=username_input + Clear Text accessibility_id=username_input + Input Text accessibility_id=username_input ${username} + Input Password accessibility_id=password_input ${password} + Hide Keyboard + Click Element accessibility_id=login_button diff --git a/bundles/rf-agentskills/skills/rf-appium/assets/examples/gestures-example.robot b/bundles/rf-agentskills/skills/rf-appium/assets/examples/gestures-example.robot new file mode 100644 index 0000000..e25469c --- /dev/null +++ b/bundles/rf-agentskills/skills/rf-appium/assets/examples/gestures-example.robot @@ -0,0 +1,59 @@ +*** Settings *** +Documentation Gestures with AppiumLibrary 3.x keywords: tap, long press, scroll, +... swipe, pinch/zoom via driver commands, drag and drop (Android, UiAutomator2). +Library AppiumLibrary timeout=15s +Suite Setup Open Gestures Demo +Suite Teardown Close All Applications + +*** Variables *** +${APPIUM_URL} http://127.0.0.1:4723 +${APP} ${CURDIR}${/}apps${/}gestures-demo.apk + +*** Test Cases *** +Long Press Opens Context Menu + Tap accessibility_id=item_1 duration=2s + Expect Element accessibility_id=context_menu visible + +Double Tap Zooms Image + Tap accessibility_id=photo count=2 duration=100ms + Expect Element accessibility_id=photo_zoomed visible + +Scroll Down Finds Terms Checkbox + Scroll Down accessibility_id=terms_checkbox timeout=20s + Expect Element accessibility_id=terms_checkbox visible + +Carousel Moves To Next Page + ${before}= Get Text accessibility_id=carousel_title + Swipe By Percent 90 50 10 50 duration=300ms + Wait Until Keyword Succeeds 5s 500ms + ... Element Should Not Contain Text accessibility_id=carousel_title ${before} + +Pull To Refresh Reloads List + Swipe By Percent 50 25 50 75 duration=500ms + Wait Until Page Does Not Contain Element accessibility_id=refresh_spinner + +Content Scrolls Up With A Pixel Swipe + ${width}= Get Window Width + ${height}= Get Window Height + VAR ${x} ${{ int(${width} / 2) }} + Swipe start_x=${x} start_y=${{ int(${height} * 0.8) }} end_x=${x} end_y=${{ int(${height} * 0.2) }} duration=500ms + +Map Zooms In And Out + ${map}= Get Webelement accessibility_id=map_view + Execute Script mobile: pinchOpenGesture elementId=${map.id} percent=${0.75} + Execute Script mobile: pinchCloseGesture elementId=${map.id} percent=${0.5} + +Card Can Be Dragged To Done Column + Drag And Drop accessibility_id=card_1 accessibility_id=done_column + Expect Element accessibility_id=done_card_1 visible + +Two Finger Tap On Coordinates + Tap With Positions 300ms ${{ (300, 800) }} ${{ (700, 800) }} + +*** Keywords *** +Open Gestures Demo + Open Application ${APPIUM_URL} + ... platformName=Android + ... appium:automationName=UiAutomator2 + ... appium:app=${APP} + Wait Until Page Contains Element accessibility_id=gestures_home diff --git a/bundles/rf-agentskills/skills/rf-appium/assets/examples/hybrid-app.robot b/bundles/rf-agentskills/skills/rf-appium/assets/examples/hybrid-app.robot new file mode 100644 index 0000000..aa14763 --- /dev/null +++ b/bundles/rf-agentskills/skills/rf-appium/assets/examples/hybrid-app.robot @@ -0,0 +1,54 @@ +*** Settings *** +Documentation Hybrid app: native screens plus a WebView form (Android, UiAutomator2). +... The app must enable WebView debugging; chromedriver must match the WebView. +Library AppiumLibrary timeout=20s +Library Collections +Suite Setup Open Hybrid App +Suite Teardown Close All Applications +Test Teardown Switch To Context NATIVE_APP + +*** Variables *** +${APPIUM_URL} http://127.0.0.1:4723 +${APP} ${CURDIR}${/}apps${/}hybrid-demo.apk + +*** Test Cases *** +WebView Form Submits And Native Screen Confirms + Click Element accessibility_id=open_web_form + Switch To WebView + Wait Until Page Contains Element css=input#email + Input Text css=input#email demo@example.com + Click Element css=button[type=submit] + Wait Until Page Contains Thank you + Switch To Context NATIVE_APP + Click Element accessibility_id=back_button + Expect Element accessibility_id=home_screen visible + +WebView Title Is Readable + Click Element accessibility_id=open_web_form + Switch To WebView + ${title}= Execute Script return document.title + Should Not Be Empty ${title} + +*** Keywords *** +Open Hybrid App + Open Application ${APPIUM_URL} + ... platformName=Android + ... appium:automationName=UiAutomator2 + ... appium:app=${APP} + ... appium:chromedriverAutodownload=${True} + Wait Until Page Contains Element accessibility_id=home_screen + +Switch To WebView + [Documentation] Waits until a WEBVIEW context exists, then switches to it. + Wait Until Keyword Succeeds 20s 1s WebView Context Exists + @{contexts}= Get Contexts + FOR ${context} IN @{contexts} + IF '${context}'.startswith('WEBVIEW') + Switch To Context ${context} + RETURN + END + END + +WebView Context Exists + @{contexts}= Get Contexts + Should Contain Match ${contexts} WEBVIEW* diff --git a/bundles/rf-agentskills/skills/rf-appium/assets/examples/ios-basic.robot b/bundles/rf-agentskills/skills/rf-appium/assets/examples/ios-basic.robot new file mode 100644 index 0000000..cdcf38e --- /dev/null +++ b/bundles/rf-agentskills/skills/rf-appium/assets/examples/ios-basic.robot @@ -0,0 +1,48 @@ +*** Settings *** +Documentation iOS native app on a simulator: login, alerts, table rows and picker wheels. +... Needs an Appium 2+ server with the XCUITest driver and Xcode. +Library AppiumLibrary timeout=20s +Suite Setup Open iOS App +Suite Teardown Close All Applications + +*** Variables *** +${APPIUM_URL} http://127.0.0.1:4723 +${APP} ${CURDIR}${/}apps${/}DemoApp.app + +*** Test Cases *** +Valid Login Shows Welcome Text + Accept Alert If Present + Click Element accessibility_id=loginTab + Input Text accessibility_id=usernameField demo + Input Password accessibility_id=passwordField demo + Hide Keyboard Done + Click Element predicate=type == 'XCUIElementTypeButton' AND name == 'Log In' + Expect Text Welcome, demo visible + +Third Settings Row Opens Details + Click Element accessibility_id=settingsTab + Wait Until Page Contains Element chain=**/XCUIElementTypeTable + Click Element chain=**/XCUIElementTypeTable/XCUIElementTypeCell[3] + Wait Until Page Contains Element accessibility_id=detailView + +Country Can Be Picked + Click Element accessibility_id=formTab + Click Element accessibility_id=countryField + @{wheels}= Get Webelements class=XCUIElementTypePickerWheel + Input Value ${wheels}[0] Germany + Click Element accessibility_id=Done + Element Should Contain Text accessibility_id=countryField Germany + +*** Keywords *** +Open iOS App + Open Application ${APPIUM_URL} + ... platformName=iOS + ... appium:automationName=XCUITest + ... appium:deviceName=iPhone 15 + ... appium:platformVersion=17.5 + ... appium:app=${APP} + +Accept Alert If Present + ${present}= Run Keyword And Return Status + ... Wait Until Page Contains Element class=XCUIElementTypeAlert timeout=3s + IF ${present} Click Alert Button Allow diff --git a/bundles/rf-agentskills/skills/rf-appium/references/capabilities.md b/bundles/rf-agentskills/skills/rf-appium/references/capabilities.md new file mode 100644 index 0000000..3a17e81 --- /dev/null +++ b/bundles/rf-agentskills/skills/rf-appium/references/capabilities.md @@ -0,0 +1,117 @@ +# Capabilities + +Capability sets for `Open Application`. Every name that is not W3C-standard (`platformName`, `browserName`, …) is sent with the `appium:` prefix; the Appium Python client adds it when it is missing, but writing it keeps logs and vendor docs aligned. Vendor capabilities keep their own prefix (`bstack:`, `sauce:`). + +- Android +- iOS +- Mobile browsers +- Cloud grids +- Reset and timeouts +- Configuring one suite for both platforms + +## Android + +| Capability | Value | When | +|---|---|---| +| `platformName` | `Android` | always | +| `appium:automationName` | `UiAutomator2` | always | +| `appium:deviceName` | `emulator-5554` | informational; `appium:udid` picks a device when several are attached | +| `appium:app` | path or URL to the `.apk` | install and launch | +| `appium:appPackage` + `appium:appActivity` | `com.example.app` / `.MainActivity` | launch an installed app instead of `app` | +| `appium:autoGrantPermissions` | `${True}` | skip runtime permission dialogs | +| `appium:avd` | `Pixel_7_API_34` | let the driver boot an emulator (`appium:avdLaunchTimeout` in ms) | +| `appium:chromedriverAutodownload` | `${True}` | WebView / Chrome contexts need a matching chromedriver; the server must allow the insecure feature `chromedriver_autodownload` (UiAutomator2 driver docs) | + +```robotframework +Open Application ${APPIUM_URL} +... platformName=Android +... appium:automationName=UiAutomator2 +... appium:appPackage=com.example.app +... appium:appActivity=.MainActivity +... appium:noReset=${True} +``` + +## iOS + +| Capability | Value | When | +|---|---|---| +| `platformName` | `iOS` | always | +| `appium:automationName` | `XCUITest` | always | +| `appium:deviceName` | `iPhone 15` | simulator name (with `appium:platformVersion`) | +| `appium:platformVersion` | `17.5` | selects the simulator runtime | +| `appium:app` | path to `.app` (simulator) or `.ipa` (device) | install and launch | +| `appium:bundleId` | `com.example.app` | launch an installed app | +| `appium:udid` | device UDID | real devices | +| `appium:xcodeOrgId` + `appium:xcodeSigningId` | team id / `Apple Development` | real devices: signing WebDriverAgent | +| `appium:autoAcceptAlerts` | `${True}` | accept system alerts (permissions) automatically | +| `appium:wdaLaunchTimeout` | `120000` | slow first WebDriverAgent build | + +```robotframework +Open Application ${APPIUM_URL} +... platformName=iOS +... appium:automationName=XCUITest +... appium:deviceName=iPhone 15 +... appium:platformVersion=17.5 +... appium:app=${CURDIR}/MyApp.app +``` + +## Mobile browsers + +Set `browserName` (W3C, no prefix) instead of `app`; then use `Go To Url` and web locators (`css=`, `xpath=`). + +```robotframework +Open Application ${APPIUM_URL} +... platformName=Android +... appium:automationName=UiAutomator2 +... browserName=Chrome +Go To Url https://example.com +``` + +On iOS use `platformName=iOS`, `appium:automationName=XCUITest`, `browserName=Safari`. + +## Cloud grids + +Cloud providers take their credentials in a vendor capability; the hub URL comes from the provider. + +```robotframework +Open Application https://hub-cloud.browserstack.com/wd/hub +... platformName=Android +... appium:deviceName=Google Pixel 7 +... appium:platformVersion=13.0 +... appium:app=bs:// +... bstack:options=${BSTACK_OPTIONS} +``` + +`${BSTACK_OPTIONS}` is a dictionary (`&{BSTACK_OPTIONS} userName=… accessKey=…` in `*** Variables ***`). Sauce Labs uses `sauce:options` the same way. + +## Reset and timeouts + +| Capability | Effect | +|---|---| +| `appium:noReset=${True}` | keep app data between sessions (faster, but state leaks) | +| `appium:fullReset=${True}` | uninstall and reinstall the app (clean, slow) | +| neither | the driver's default reset, which differs between UiAutomator2 and XCUITest (see the driver docs) | +| `appium:newCommandTimeout` | seconds the server waits for the next command before it ends the session (default 60); raise it for debugging pauses | + +The library's own wait timeout is the import argument (`Library AppiumLibrary timeout=15s`) or `Set Appium Timeout`; it does not change any capability. + +## Configuring one suite for both platforms + +Keep the capabilities in variable files or dictionaries and select them on the command line: + +```robotframework +*** Variables *** +${PLATFORM} android +&{ANDROID} platformName=Android appium:automationName=UiAutomator2 appium:app=${CURDIR}/app.apk +&{IOS} platformName=iOS appium:automationName=XCUITest appium:app=${CURDIR}/MyApp.app + +*** Keywords *** +Open App For Platform + IF '${PLATFORM}' == 'ios' + Open Application ${APPIUM_URL} &{IOS} + ELSE + Open Application ${APPIUM_URL} &{ANDROID} + END +``` + +Run with `robot --variable PLATFORM:ios tests/`. diff --git a/bundles/rf-agentskills/skills/rf-appium/references/gestures-touch.md b/bundles/rf-agentskills/skills/rf-appium/references/gestures-touch.md new file mode 100644 index 0000000..688a700 --- /dev/null +++ b/bundles/rf-agentskills/skills/rf-appium/references/gestures-touch.md @@ -0,0 +1,84 @@ +# Gestures and Touch + +Durations take units (`500ms`, `1s`). `Swipe`, `Swipe By Percent` and `Tap With Positions` read a bare number as milliseconds and log a warning; `Tap` reads a bare number as seconds. + +- Tap and long press +- Scrolling to an element +- Swipes +- Pinch, zoom and driver gestures +- Drag and drop +- Removed gesture keywords + +## Tap and long press + +```robotframework +Tap accessibility_id=item_1 # single tap (duration defaults to 1s) +Tap accessibility_id=item_1 count=2 duration=200ms # double tap +Tap accessibility_id=item_1 duration=2s # long press, e.g. to open a context menu +Tap With Positions 300ms ${{ (540, 1200) }} # tap at coordinates; duration comes first +Tap With Positions 300ms ${{ (300, 800) }} ${{ (700, 800) }} # two fingers +``` + +`Tap` also accepts a coordinate list instead of a locator (`VAR @{point} 540 1200` then `Tap ${point}`). + +## Scrolling to an element + +| Need | Use | +|---|---| +| Android, element somewhere below | `Scroll Down ` — swipes up to `timeout` (default 10s) until the locator matches | +| Android, long list, text known | `Click Element android=new UiScrollable(new UiSelector().scrollable(true)).scrollIntoView(new UiSelector().text("Item 50"))` | +| iOS | `Scroll Down ` scrolls to an element that is already in the view hierarchy (XCUITest keeps off-screen cells in the tree) | +| Between two known elements | `Scroll ` | +| Element present but off-screen | `Scroll Element Into View ` | + +On Android, `Scroll Down` / `Scroll Up` return without failing when the timeout ends and the element never appeared, so assert afterwards: + +```robotframework +Scroll Down accessibility_id=terms_checkbox timeout=20s +Expect Element accessibility_id=terms_checkbox visible +``` + +## Swipes + +`Swipe` takes keyword-only pixel coordinates; `Swipe By Percent` takes positional percentages of the screen (0–100), so it works across resolutions. + +```robotframework +Swipe start_x=540 start_y=1600 end_x=540 end_y=400 duration=500ms # scroll content up +Swipe By Percent 90 50 10 50 duration=300ms # carousel: next page +Swipe By Percent 50 25 50 75 duration=500ms # pull to refresh +``` + +Screen-relative pixels: read `Get Window Width` / `Get Window Height` and compute with `${{ }}` expressions. + +## Pinch, zoom and driver gestures + +AppiumLibrary has no pinch or zoom keyword. Call the driver's `mobile:` commands through `Execute Script`; named arguments become the command's parameter map. + +```robotframework +${map}= Get Webelement accessibility_id=map_view +# Android (UiAutomator2): percent is 0..1 +Execute Script mobile: pinchOpenGesture elementId=${map.id} percent=${0.75} +Execute Script mobile: pinchCloseGesture elementId=${map.id} percent=${0.5} +# iOS (XCUITest): scale > 1 zooms in, < 1 zooms out; velocity sign follows the scale +Execute Script mobile: pinch elementId=${map.id} scale=${2.0} velocity=${1.0} +Execute Script mobile: pinch elementId=${map.id} scale=${0.5} velocity=${-1.0} +``` + +Other gestures follow the same pattern (UiAutomator2: `mobile: longClickGesture`, `mobile: swipeGesture`, `mobile: scrollGesture`, `mobile: dragGesture`; XCUITest: `mobile: touchAndHold`, `mobile: swipe`, `mobile: scroll`). Their parameters are in the driver's execute-methods documentation; pass numbers as `${…}` so they are not sent as strings. + +## Drag and drop + +```robotframework +Drag And Drop accessibility_id=card_1 accessibility_id=done_column +``` + +When the app needs a slow drag, use `Swipe` from the element's centre (`Get Element Location` + `Get Element Size`) with `duration=2s`. + +## Removed gesture keywords + +| Removed | Use | +|---|---| +| `Long Press` | `Tap duration=2s` | +| `Click A Point` | `Tap With Positions 500ms ${{ (x, y) }}` | +| `Zoom` | `Execute Script mobile: pinchOpenGesture` (Android) / `mobile: pinch` with `scale` > 1 (iOS) | +| `Pinch` | `Execute Script mobile: pinchCloseGesture` (Android) / `mobile: pinch` with `scale` < 1 (iOS) | diff --git a/bundles/rf-agentskills/skills/rf-appium/references/install.md b/bundles/rf-agentskills/skills/rf-appium/references/install.md new file mode 100644 index 0000000..fb6b890 --- /dev/null +++ b/bundles/rf-agentskills/skills/rf-appium/references/install.md @@ -0,0 +1,21 @@ +# Installing AppiumLibrary + +Commands are in uv form. For a new environment, Poetry and venv + pip forms, and the general troubleshooting (wrong interpreter, PEP 668, old Python), see `rf-setup` (plugin `rf-core`): `uv add ` is `poetry add ` (pip alternative: `.venv/bin/python -m pip install `), and `uv run ` is `poetry run ` or `.venv/bin/`. Versions and floors checked on PyPI 2026-09-26. + +```bash +uv add robotframework-appiumlibrary +``` + +- Package `robotframework-appiumlibrary` (3.2). Module `AppiumLibrary`. It pulls in `Appium-Python-Client`. +- After installing: the **Appium server** and platform drivers are Node tools, installed outside the Python environment: + +```bash +npm install -g appium # Appium 3.x; Node ^20.19, ^22.12 or ≥ 24 +appium driver install uiautomator2 # Android +appium driver install xcuitest # iOS (macOS + Xcode) +appium driver list +appium driver doctor uiautomator2 # checks ANDROID_HOME, JDK, adb … +appium # start the server (default http://127.0.0.1:4723) +``` + +- Android also needs the Android SDK (`ANDROID_HOME`), a JDK and an emulator or device. iOS needs Xcode. See `platform-specific.md`. diff --git a/bundles/rf-agentskills/skills/rf-appium/references/locators-mobile.md b/bundles/rf-agentskills/skills/rf-appium/references/locators-mobile.md new file mode 100644 index 0000000..7e275f9 --- /dev/null +++ b/bundles/rf-agentskills/skills/rf-appium/references/locators-mobile.md @@ -0,0 +1,74 @@ +# Mobile Locators + +AppiumLibrary locators are `strategy=value`. Without a prefix the value is an `id` (or XPath when it starts with `//`); an unknown prefix fails with "Element locator with prefix '…' is not supported". + +- Strategy order +- Android +- iOS +- XPath +- Finding locators + +## Strategy order + +| Strategy | Android | iOS | Notes | +|---|---|---|---| +| `accessibility_id=` | content-desc | accessibility identifier | same value on both platforms when the app sets it; the first choice | +| `id=` | resource-id | accessibility id / name | short form `id=login` works when the id belongs to the app package | +| `name=` | — | name / label | iOS only in practice | +| `android=` | UiAutomator selector | — | fast, can scroll (see Android) | +| `predicate=` | — | NSPredicate string | fast | +| `chain=` | — | class chain | fast, hierarchical | +| `xpath=` | yes | yes | slow on large trees; last resort | +| `class=` | widget class | XCUIElementType | usually matches many elements | +| `css=` | WebView / browser context only | same | web content after `Switch To Context` | + +`ios=` is the retired UI Automation strategy; the current Appium Python client no longer supports it. Use `predicate=` or `chain=`. + +## Android + +```robotframework +Click Element accessibility_id=login_button +Click Element id=com.example.app:id/login_button +Click Element android=new UiSelector().text("Login") +Click Element android=new UiSelector().className("android.widget.Button").textContains("Sub") +Click Element android=new UiSelector().resourceId("com.example.app:id/item").instance(2) +``` + +Scroll a list until an element exists, in one call (UiAutomator scrolls on the device): + +```robotframework +Click Element android=new UiScrollable(new UiSelector().scrollable(true)).scrollIntoView(new UiSelector().text("Item 50")) +``` + +Common classes: `android.widget.Button`, `android.widget.EditText`, `android.widget.TextView`, `android.widget.CheckBox`, `android.widget.Switch`, `android.widget.ImageView`, `androidx.recyclerview.widget.RecyclerView`. + +## iOS + +```robotframework +Click Element accessibility_id=loginButton +Click Element predicate=type == 'XCUIElementTypeButton' AND name == 'Login' +Click Element predicate=type == 'XCUIElementTypeStaticText' AND value BEGINSWITH 'Hello' +Click Element chain=**/XCUIElementTypeButton[`name == 'Login'`] +Click Element chain=**/XCUIElementTypeTable/XCUIElementTypeCell[3] +``` + +Predicate operators: `==`, `!=`, `CONTAINS`, `BEGINSWITH`, `ENDSWITH`, `MATCHES` (regex), combined with `AND` / `OR` / `NOT`; append `[c]` for case-insensitive (`name CONTAINS[c] 'log'`). Class chains accept a predicate in backticks inside `[…]` and an index `[n]` (1-based). + +Common types: `XCUIElementTypeButton`, `XCUIElementTypeTextField`, `XCUIElementTypeSecureTextField`, `XCUIElementTypeStaticText`, `XCUIElementTypeCell`, `XCUIElementTypeSwitch`, `XCUIElementTypeAlert`. + +## XPath + +```robotframework +Click Element xpath=//android.widget.Button[@text='Login'] +Click Element xpath=//*[@content-desc='Submit'] +Click Element xpath=//XCUIElementTypeButton[@name='Login'] +${n}= Get Matching Xpath Count //android.widget.CheckBox +``` + +XPath walks the whole view hierarchy, so it is slow on long lists; prefer an `android=` or `predicate=` query with the same condition. `Get Matching Xpath Count` and `Xpath Should Match X Times` take the XPath without the `xpath=` prefix; `Get Matching Xpath Count` returns a string, so compare with `Should Be Equal As Integers`. + +## Finding locators + +- `Log Source` writes the current view hierarchy (XML) to the log; read attribute names there (`content-desc`, `resource-id`, `text` on Android; `name`, `label`, `value`, `type` on iOS). +- Appium Inspector shows the same tree interactively and can try a locator before you write it. +- Ask the app developers for accessibility identifiers when elements have none; they stabilise tests on both platforms. diff --git a/bundles/rf-agentskills/skills/rf-appium/references/platform-specific.md b/bundles/rf-agentskills/skills/rf-appium/references/platform-specific.md new file mode 100644 index 0000000..63fc414 --- /dev/null +++ b/bundles/rf-agentskills/skills/rf-appium/references/platform-specific.md @@ -0,0 +1,105 @@ +# Platform-Specific Keywords + +Keywords and driver commands that exist on only one platform, and hybrid-app context handling. Capabilities and locators per platform are in their own reference files. + +- Android: keys and keyboard +- Android: activities and app state +- Android: permissions, notifications and shell +- iOS: alerts and permissions +- iOS: keyboard, pickers and biometrics +- Both: orientation, background and app lifecycle +- Hybrid apps and WebViews + +## Android: keys and keyboard + +`Press Keycode` and `Long Press Keycode` send Android key codes (`metastate=1` adds Shift). + +| Key | Code | Key | Code | +|---|---|---|---| +| Back | `4` | Enter | `66` | +| Home | `3` | Delete (backspace) | `67` | +| Menu | `82` | Tab | `61` | +| Volume up / down | `24` / `25` | App switch | `187` | + +```robotframework +Press Keycode 4 # back +Press Keycode 66 # enter, e.g. submit a search field +Hide Keyboard # Android ignores the key_name argument +${shown}= Is Keyboard Shown +``` + +## Android: activities and app state + +```robotframework +${activity}= Get Activity +Start Activity com.example.app .SettingsActivity +Wait Activity .SettingsActivity 10 +${state}= Execute Script mobile: queryAppState appId=com.example.app # 4 = running in foreground +``` + +`Start Activity` passes extra options (`stop=${True}`, `action=…`, `uri=…`) to the driver's `mobile: startActivity`. Deep links: `Start Activity` with `action=android.intent.action.VIEW` and `uri=myapp://path`. + +## Android: permissions, notifications and shell + +- Runtime permission dialogs: start the session with `appium:autoGrantPermissions=${True}`; otherwise tap the dialog's button (`id=com.android.permissioncontroller:id/permission_allow_button` on recent Android versions; read the exact id with `Log Source`). +- `Open Notifications` expands the notification drawer; locate notifications by text with `android=new UiSelector().textContains("…")`, then `Press Keycode 4` to close it. +- Shell commands: `Execute Adb Shell ` or `Execute Script mobile: shell command=getprop args=${{ ["ro.build.version.release"] }}`. Both need the server started with the insecure feature `adb_shell` allowed. + +## iOS: alerts and permissions + +```robotframework +Wait Until Page Contains Element class=XCUIElementTypeAlert timeout=5s +Click Alert Button Allow +``` + +`Click Alert Button` is iOS-only and takes the button text. To accept every system alert, start the session with `appium:autoAcceptAlerts=${True}` (or `appium:autoDismissAlerts`). Checking for an optional alert: + +```robotframework +*** Keywords *** +Accept Alert If Present + ${present}= Run Keyword And Return Status + ... Wait Until Page Contains Element class=XCUIElementTypeAlert timeout=3s + IF ${present} Click Alert Button Allow +``` + +## iOS: keyboard, pickers and biometrics + +```robotframework +Hide Keyboard Done # iOS presses the named key to dismiss the keyboard +@{wheels}= Get Webelements class=XCUIElementTypePickerWheel +Input Value ${wheels}[0] December +Input Value ${wheels}[1] 25 +Touch Id match=${True} # simulator only; enrol first with Toggle Touch Id Enrollment +``` + +`Input Value` (iOS only) sets a value directly and suits picker wheels; `Input Text` types key by key. Face ID and Touch ID on simulators can also be driven with `Execute Script mobile: sendBiometricMatch type=faceId match=${True}`. + +## Both: orientation, background and app lifecycle + +```robotframework +Landscape +Portrait +Background Application 3 # seconds in the background, then back +Terminate Application com.example.app # bundleId on iOS +Activate Application com.example.app +Lock 2 +``` + +`Close Application` ends the session; `Terminate Application` only stops the app and keeps the session, so the next step can `Activate Application` again. + +## Hybrid apps and WebViews + +```robotframework +Wait Until Page Contains Element class=android.webkit.WebView timeout=15s +@{contexts}= Get Contexts +Log ${contexts} # e.g. NATIVE_APP, WEBVIEW_com.example.app +Switch To Context WEBVIEW_com.example.app +Input Text css=input#email demo@example.com +Click Element css=button[type=submit] +Switch To Context NATIVE_APP +``` + +- The WebView context appears only after the WebView has loaded, and on Android only when the app enables WebView debugging (`setWebContentsDebuggingEnabled(true)` in debug builds). +- Android WebViews need a chromedriver that matches the WebView version (see the chromedriver capabilities in the capabilities reference). +- iOS WebView context names look like `WEBVIEW_12345.1`; read them from `Get Contexts` rather than hard-coding them. +- In the web context, native locators (`accessibility_id=`, `android=`) do not work; switch back to `NATIVE_APP` first. diff --git a/bundles/rf-agentskills/skills/rf-appium/references/troubleshooting.md b/bundles/rf-agentskills/skills/rf-appium/references/troubleshooting.md new file mode 100644 index 0000000..fb31be8 --- /dev/null +++ b/bundles/rf-agentskills/skills/rf-appium/references/troubleshooting.md @@ -0,0 +1,70 @@ +# Troubleshooting + +Organised by the message you see. Environment problems (Node.js, Appium server or driver install, Android SDK, Xcode) belong to rf-setup (plugin rf-core). + +- Connection refused or "Max retries exceeded" +- "The requested resource could not be found" on session start +- "Could not find a connected Android device" / "Device … was not in the list" +- "SessionNotCreatedException" on iOS (WebDriverAgent) +- "Element locator '…' did not match any elements" +- "Element locator with prefix '…' is not supported" +- "No keyword with name '…' found" +- "No Chromedriver found that can automate Chrome" +- WebView context missing from Get Contexts +- Session ends between steps +- Reading the screen when a locator fails + +## Connection refused or "Max retries exceeded" + +The Appium server is not running at the URL of `Open Application`. Start it (`appium`, port 4723 by default) and check `curl http://127.0.0.1:4723/status`. Tests on another machine need its address and an open port. + +## "The requested resource could not be found" on session start + +The URL has the wrong base path. Appium 2 and later serve at `/` by default, so `http://127.0.0.1:4723/wd/hub` fails; drop `/wd/hub`, or start the server with `--base-path /wd/hub` when old suites cannot change. Cloud providers keep their own documented path. + +## "Could not find a connected Android device" / "Device … was not in the list" + +`adb devices` must list the emulator or device as `device` (not `unauthorized` or `offline`). With several devices, pick one with `appium:udid=`. Accept the USB debugging prompt on real devices. + +## "SessionNotCreatedException" on iOS (WebDriverAgent) + +WebDriverAgent could not be built or started. On simulators, check that `appium:deviceName` and `appium:platformVersion` name an installed simulator (`xcrun simctl list devices`). On real devices, set `appium:xcodeOrgId` and `appium:xcodeSigningId` and trust the developer certificate on the device. The first build can take minutes; raise `appium:wdaLaunchTimeout`. + +## "Element locator '…' did not match any elements" + +- The screen was not ready: action keywords do not wait. Add `Wait Until Page Contains Element` (or `Expect Element … visible`) before the first action on a new screen. +- The element is off-screen in a list: Android only creates visible rows. Scroll to it first (`Scroll Down ` or a `UiScrollable` locator). +- The context is wrong: after `Switch To Context WEBVIEW_…`, native locators no longer match, and the other way round. +- The locator is wrong: run `Log Source` and compare attribute names and values. + +## "Element locator with prefix '…' is not supported" + +Everything before the first `=` is read as a strategy. Use one of `accessibility_id`, `id`, `name`, `xpath`, `class`, `android`, `predicate`, `chain`, `css` (web context), or write an XPath starting with `//`. A value that itself contains `=` needs an explicit prefix (`id=…`). + +## "No keyword with name '…' found" + +The keyword was removed in AppiumLibrary 3.x or never existed (`Long Press`, `Click A Point`, `Zoom`, `Pinch`, `Quit Application`, `Reset Application`, `Background App`, `Get Window Size`). Look the name up with libdoc; the replacements are in the skill's Gotchas. + +## "No Chromedriver found that can automate Chrome" + +The WebView or Chrome version on the device does not match any chromedriver the UiAutomator2 driver has. Set `appium:chromedriverAutodownload=${True}` and allow the server's insecure feature `chromedriver_autodownload`, or point `appium:chromedriverExecutable` at a matching binary. + +## WebView context missing from Get Contexts + +Wait for the WebView to load before calling `Get Contexts`. Android apps expose WebViews only when WebView debugging is enabled in the app build; iOS WebViews of release builds on real devices are not inspectable unless the app marks them inspectable. + +## Session ends between steps + +The server ends a session after `appium:newCommandTimeout` seconds without a command (default 60). Long `Sleep`s or debugger pauses trigger it; raise the capability instead of adding keep-alive steps. A session also ends when a previous test crashed the app: open it again in Test Setup or use per-test sessions. + +## Reading the screen when a locator fails + +`run_on_failure` captures a screenshot by default (`Capture Page Screenshot`). Add the view hierarchy while debugging: + +```robotframework +Log Source loglevel=INFO +${source}= Get Source +Create File ${OUTPUT_DIR}/screen.xml ${source} +``` + +Keywords with a `loglevel` argument (`Expect Element`, `Page Should Contain Element`) already log the source when they fail; pass `loglevel=NONE` to turn that off. diff --git a/bundles/rf-agentskills/skills/rf-browser/SKILL.md b/bundles/rf-agentskills/skills/rf-browser/SKILL.md new file mode 100644 index 0000000..f945ee4 --- /dev/null +++ b/bundles/rf-agentskills/skills/rf-browser/SKILL.md @@ -0,0 +1,132 @@ +--- +name: rf-browser +description: "Use first, before answering, for RF web tests with Browser Library (Playwright): selectors." +license: Apache-2.0 +compatibility: Requires Python 3.10+, robotframework>=7, robotcode[all]>=2.8 and robotframework-browser in the project environment. Node.js 22+ LTS only when the robotframework-browser[bb] batteries extra is not used. Needs network access for rfbrowser install. +metadata: + author: manykarim + version: "1.0.0" +--- + +# Browser Library Skill + +Browser Library drives Chromium, Firefox and WebKit through Playwright: auto-waiting actions, retrying assertions, one isolated context per session. It is the default for new web tests; for suites that already import SeleniumLibrary, use rf-selenium. +Verified against Browser 19.14.2, Robot Framework 7.4. + +## When to use + +Load this skill first, before exploring the project or answering from memory, when a Robot Framework web UI test uses Browser Library (`Library Browser`, Playwright): + +- writing or fixing a web test: pages, contexts and tabs, CSS/text/XPath selectors including iframes and shadow DOM, auto-waiting, assertions with an operator, downloads, storage state; +- a click or selector times out (`TimeoutError: locator.click`), or the user mentions Browser Library, Playwright or rfbrowser, even if they only say "browser lib"; +- a new web test with no library chosen yet: Browser Library is the default. + +Not this skill: suites that import SeleniumLibrary or run on WebDriver/Selenium Grid use `rf-selenium`; installing the library or errors in its Node.js `rfbrowser init` step use `rf-setup` (plugin `rf-core`). + +## Installation + +Install into the project environment, with the tool the project already uses. Full steps and verification: `references/install.md`. For a new environment, Poetry and venv + pip forms, and troubleshooting, see `rf-setup` (plugin `rf-core`). + +```bash +uv add "robotframework-browser[bb]" +uv run rfbrowser install chromium +``` +BrowserBatteries (`[bb]`) ships the Playwright wrapper, so no Node.js is needed. `rfbrowser install` downloads the browser binaries. + +Node.js escape hatch — only if no BrowserBatteries wheel exists for your platform, or you need Node plugins (Node 22/24/26 LTS): + +```bash +uv add robotframework-browser +uv run rfbrowser init chromium +``` +Don't mix the paths: `rfbrowser install` belongs to `[bb]`, `rfbrowser init` to the Node.js path. + +## Import and defaults + +Import the library without arguments; its defaults are the recommendation: strict mode on, `timeout=10s` for actions and waits, `retry_assertions_for=1s`, `auto_closing_level=TEST`, and a screenshot on every failing Browser keyword. + +```robotframework +*** Settings *** +Library Browser +Suite Setup Open Test Browser +Test Setup New Page ${URL} + +*** Variables *** +${URL} https://example.com + +*** Keywords *** +Open Test Browser + New Browser chromium headless=True + New Context viewport={'width': 1280, 'height': 800} +``` + +The browser and context opened in Suite Setup live until the suite ends; each test gets a fresh page that is closed at test end. Use a new context per test when tests must not share cookies or storage, or per user in multi-user tests — see `references/browser-context-page.md`. Reuse a login with `storageState` — see `references/authentication-storage.md`. + +## Which keyword for which situation + +| Situation | Use | Details | +|---|---|---| +| Click, hover, check | `Click`, `Hover`, `Check Checkbox` (auto-wait for actionability) | `references/locators.md` | +| Enter text | `Fill Text`; `Type Text` when key events matter; `Fill Secret` for passwords | — | +| Choose from a `` element; extra paths upload several files, and a directory uploads all files directly in it. +- When the site opens a file chooser from a non-input button, start `Promise To Upload File` before the click and `Wait For` it afterwards. +- Large uploads can exceed the library `timeout`; raise it with `Set Browser Timeout` for that test. + +## Downloads + +```robotframework +*** Test Cases *** +Download Report + ${promise}= Promise To Wait For Download ${OUTPUT_DIR}/report.csv + Click text=Export CSV + ${file}= Wait For ${promise} + File Should Exist ${file}[saveAs] + Should Be Equal ${file}[suggestedFilename] report.csv + +Download Direct Link + ${file}= Download https://example.com/files/manual.pdf + Log ${file}[saveAs] +``` + +- Start `Promise To Wait For Download` before the click that triggers the download; waiting afterwards can miss the event. +- It returns a `DownloadInfo` dictionary with `saveAs` (where the file is) and `suggestedFilename`. `wait_for_finished=False` returns when the download starts; check it later with `Get Download State`. +- `Download` fetches a URL directly; when a page element starts the download, use the promise instead. +- `File Should Exist` comes from `OperatingSystem`; import it for file checks. + +## Where files go + +- Without `saveAs`, files get a generated name and are deleted when the context closes. Pass `saveAs` (absolute, or relative to the browser's `downloadsPath`) to keep them. +- `New Context` has `acceptDownloads=True` by default; a context created with `acceptDownloads=False` never produces downloads. +- With a remote browser (`Connect To Browser`), `saveAs` is required to store the file where the browser runs. diff --git a/bundles/rf-agentskills/skills/rf-browser/references/iframes-shadow-dom.md b/bundles/rf-agentskills/skills/rf-browser/references/iframes-shadow-dom.md new file mode 100644 index 0000000..ec44cdf --- /dev/null +++ b/bundles/rf-agentskills/skills/rf-browser/references/iframes-shadow-dom.md @@ -0,0 +1,67 @@ +# iframes and Shadow DOM + +- iframes with >>> +- Many steps inside one frame +- Shadow DOM +- Frames and shadow roots together +- When the element is still not found + +## iframes with >>> + +Selectors do not cross frame boundaries. `>>>` joins a selector for the frame element (left) with a selector evaluated inside that frame (right); the part directly before `>>>` must select the `