V0.9.1/evals

.github/workflows/validate-skill-templates.yml

@@ -3,15 +3,40 @@ name: Validate Skill Templates
 on:
   pull_request:
 
+concurrency:
+  group: validate-skill-templates-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
 jobs:
-  validate-skill-templates:
+  validate:
+    name: Validate (${{ matrix.name }})
     runs-on: ubuntu-latest
+    timeout-minutes: 5
+    strategy:
+      fail-fast: false
+      max-parallel: 13
+      matrix:
+        include:
+          - { name: templates, script: scripts/validate-skill-templates.ps1, suite: Templates }
+          - { name: preparation, script: scripts/validate-skill-templates.ps1, suite: Preparation }
+          - { name: runners, script: scripts/validate-skill-templates.ps1, suite: Runners }
+          - { name: docfx, script: scripts/validate-skill-templates.ps1, suite: Docfx }
+          - { name: protocol, script: scripts/eval-runners/tests/test-runner-conformance.ps1, suite: Protocol }
+          - { name: codex-cli, script: scripts/eval-runners/tests/test-runner-conformance.ps1, suite: CodexCli }
+          - { name: codex-app-server, script: scripts/eval-runners/tests/test-runner-conformance.ps1, suite: CodexAppServer }
+          - { name: opencode, script: scripts/eval-runners/tests/test-runner-conformance.ps1, suite: OpenCode }
+          - { name: copilot, script: scripts/eval-runners/tests/test-runner-conformance.ps1, suite: Copilot }
+          - { name: integrity-bridge, script: scripts/eval-runners/tests/test-integrity-finalization.ps1, suite: Bridge }
+          - { name: integrity-grading, script: scripts/eval-runners/tests/test-integrity-finalization.ps1, suite: Grading }
+          - { name: integrity-application, script: scripts/eval-runners/tests/test-integrity-finalization.ps1, suite: Application }
+          - { name: integrity-finalization, script: scripts/eval-runners/tests/test-integrity-finalization.ps1, suite: Finalization }
 
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
       - name: Install DocFX CLI
+        if: matrix.name == 'docfx'
         shell: pwsh
         run: |
           if (Get-Command docfx -ErrorAction SilentlyContinue) {
@@ -23,6 +48,7 @@ jobs:
           docfx --version
 
       - name: Install Anthropic skill-creator eval assets
+        if: matrix.name == 'preparation' || matrix.name == 'runners'
         shell: pwsh
         run: |
           $sourceRoot = Join-Path $env:RUNNER_TEMP 'anthropic-skills'
@@ -42,4 +68,29 @@ jobs:
 
       - name: Run validator
         shell: pwsh
-        run: pwsh -NoProfile -File ./scripts/validate-skill-templates.ps1 -Full
+        env:
+          VALIDATION_SCRIPT: ${{ matrix.script }}
+          VALIDATION_SUITE: ${{ matrix.suite }}
+        run: |
+          $arguments = @('-NoProfile', '-NonInteractive', '-File', $env:VALIDATION_SCRIPT, '-Suite', $env:VALIDATION_SUITE)
+          if ($env:VALIDATION_SUITE -eq 'Docfx') { $arguments += '-Full' }
+          $watch = [Diagnostics.Stopwatch]::StartNew()
+          & pwsh @arguments
+          $exitCode = $LASTEXITCODE
+          "Validation suite $env:VALIDATION_SUITE completed in $([Math]::Round($watch.Elapsed.TotalSeconds, 1)) seconds (exit $exitCode)." | Tee-Object -FilePath $env:GITHUB_STEP_SUMMARY
+          exit $exitCode
+
+  # Preserve the existing required status check; every matrix entry must pass.
+  validate-skill-templates:
+    if: always()
+    needs: validate
+    runs-on: ubuntu-latest
+    steps:
+      - name: Require all validation suites
+        shell: pwsh
+        env:
+          VALIDATION_RESULT: ${{ needs.validate.result }}
+        run: |
+          if ($env:VALIDATION_RESULT -ne 'success') {
+            throw "Validation suites did not all pass: $env:VALIDATION_RESULT"
+          }

CHANGELOG.md

@@ -4,6 +4,32 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.9.1] - 2026-08-22
+
+This patch release adds harness-agnostic Eval Runner execution boundary infrastructure without changing the paired evaluation methodology or existing report schemas, while optimizing skill descriptions and refactoring repository-level authoring guidance. Prepared packages now carry `execution-profile.json`, package-local runner protocol tools, and normalized `execution-result.json` evidence. The deterministic fake runner is the conformance reference, with Codex, GitHub Copilot CLI, and OpenCode as supported real adapters. Repository automation remains model-free; only a human-directed external Eval Orchestrator may invoke the selected runner, and unsupported isolation fails closed.
+
+### Added
+
+- `scripts/eval-runners/` with the common `describe`/`preflight`/`execute` process contract, execution-profile and execution-result schemas, deterministic fake runner, Codex adapter, GitHub Copilot CLI adapter, OpenCode adapter, runner resolution, artifact/hash validation, and bridge into the existing `eval-result/2` result shape,
+- GitHub Copilot CLI as a supported Eval Runner with authentication handling (GitHub tokens, OS keychain, CLI fallback), JSONL-based event output parsing, stdin-based prompt delivery for byte fidelity, repository instruction visibility, and conformance tests covering token management and authentication source detection,
+- deterministic fake-runner conformance coverage for fresh paired sessions, prompt fidelity, isolation boundaries, candidate-skill exposure, status normalization, unavailable telemetry, event warnings, artifact references, and report compatibility,
+- runner-aware package preparation that reuses `run.json`, keeps runner selection outside `evals/evals.json`, and preserves Anthropic-compatible benchmark/report artifacts.
+
+### Changed
+
+- Eval preparation now resolves Harness + Model before writing `RUN-THIS.prompt.md`, removes the redundant portable `provider` field from `execution-profile.json` and result reporting, treats model selectors as runner-native opaque strings, and adds `scripts/Get-HarnessModels.ps1` for current model discovery with Codebelt Reference verification and free-only OpenCode filtering,
+- `AGENTS.md`, `README.md`, and `CONTRIBUTING.md` now distinguish the Eval Runner, Eval Orchestrator, Grader, and Human Reviewer and clarify that runner execution is an explicit external-handoff boundary rather than repository automation,
+- All 21 repo-managed skill descriptions refactored to lean, trigger-oriented activation metadata following progressive-disclosure principles and specification compliance,
+- `AGENTS.md` Skill Authoring section restructured for clarity, brevity, and progressive disclosure of form fields, asset handling, and dynamic defaults,
+- report timing output omits unavailable duration and token telemetry instead of writing zero placeholders,
+- GitHub Copilot runner added to the eval runner lineup alongside existing Codex and OpenCode support,
+- Runner conformance tests enhanced with additional event fixtures and isolation capability assessment.
+
+### Fixed
+
+- Codex runner now ensures the evidence directory is created before writing output files, preventing file-not-found errors,
+- Test runner conformance validation now requires the output parent directory to exist upfront with explicit error reporting when the directory structure is misconfigured.
+
 ## [0.9.0] - 2026-08-20
 
 This is a minor release that adds three .NET skills — `dotnet-test`, `dotnet-remote-testing`, and `dotnet-segregated-assets` — replaces the repository's model-backed eval benchmark workflow with deterministic, local-only validation, and finalizes the portable eval handoff. The selected external evaluator now runs the paired workers, grades their results, and invokes Anthropic's skill-creator aggregator and eval viewer without sending the user back for a second collection command. `dotnet-test` bootstraps and modernizes xUnit test projects against Codebelt conventions with role-aware fixtures; `dotnet-remote-testing` runs .NET tests inside official Microsoft SDK containers using either an existing `testenvironments.json` or zero-config, offline-safe release discovery; and `dotnet-segregated-assets` migrates ASP.NET Core applications to an artifact-first topology where `wwwroot` stays the authoring root while deployed static content is served by a separate hardened origin. Alongside those, `git-keep-a-changelog` and `git-nuget-release-notes` gained deterministic release-entity classification, and `git-visual-commits` gained an invocation routing lock. No published skill was removed or renamed, so adopting this release is non-breaking for existing installs.
@@ -612,6 +638,7 @@ This is a minor release that introduces two complementary git workflow skills, e
 
 - Improved scaffold fidelity with hidden `.bot` asset preservation, explicit UTF-8 and BOM handling, and checks aimed at preventing mojibake or incomplete generated output.
 
+[0.9.1]: https://github.com/codebeltnet/agentic/compare/v0.9.0...v0.9.1
 [0.9.0]: https://github.com/codebeltnet/agentic/compare/v0.8.2...v0.9.0
 [0.8.2]: https://github.com/codebeltnet/agentic/compare/v0.8.1...v0.8.2
 [0.8.1]: https://github.com/codebeltnet/agentic/compare/v0.8.0...v0.8.1