From ac3b68829f3d2c415fcb870ec0351de657e55f63 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:07:59 +0000 Subject: [PATCH 1/3] ci(assurance): pilot pinned proven gates and truthful test coverage --- .github/workflows/actions.lock | 7 + .github/workflows/e2e.yml | 52 +++++- .github/workflows/main-estate-audit.yml | 31 ++++ .github/workflows/proven-lock-gate.yml | 120 ++++++++++++++ .github/workflows/static-analysis-gate.yml | 8 +- docs/status/ASSURANCE-MATRIX.adoc | 122 ++++++++++++++ docs/status/TEST-NEEDS.adoc | 106 +++++------- tests/aspect_tests.sh | 179 ++++++--------------- tests/workflows/aspect_scope_test.sh | 35 ++++ 9 files changed, 459 insertions(+), 201 deletions(-) create mode 100644 .github/workflows/main-estate-audit.yml create mode 100644 .github/workflows/proven-lock-gate.yml create mode 100644 docs/status/ASSURANCE-MATRIX.adoc create mode 100644 tests/workflows/aspect_scope_test.sh diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock index 340bdcd..83fba8c 100644 --- a/.github/workflows/actions.lock +++ b/.github/workflows/actions.lock @@ -27,6 +27,8 @@ workflows: '.github/workflows/mirror.yml': [] '.github/workflows/openssf-compliance.yml': - 'actions/checkout@v7.0.1' + '.github/workflows/proven-lock-gate.yml': + - 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' '.github/workflows/push-email-notify.yml': - 'hyperpolymath/smtp-notify-action@v0.5.1' '.github/workflows/quality.yml': @@ -54,6 +56,11 @@ workflows: '.github/workflows/workflow-linter.yml': - 'actions/checkout@v7.0.1' dependencies: + 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1': + ref: 'v7.0.1' + commit: 'sha1-3d3c42e5aac5ba805825da76410c181273ba90b1' + owner_id: 44036562 + repo_id: 197814629 'actions/checkout@v7.0.1': ref: 'v7.0.1' commit: 'sha1-3d3c42e5aac5ba805825da76410c181273ba90b1' diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index c750a87..2d0308a 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -24,14 +24,22 @@ on: - 'ffi/**' - 'tests/**' - '.github/workflows/e2e.yml' + - 'build/**' + - 'Justfile' + - '.tool-versions' pull_request: branches: [main, master] paths: - 'src/**' - 'ffi/**' - 'tests/**' + - '.github/workflows/e2e.yml' + - 'build/**' + - 'Justfile' + - '.tool-versions' workflow_dispatch: -permissions: read-all +permissions: + contents: read concurrency: group: e2e-${{ github.ref }} cancel-in-progress: true @@ -42,17 +50,47 @@ jobs: e2e: name: E2E — smoke runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 45 steps: - name: Checkout uses: actions/checkout@v7.0.1 + - name: Check aspect harness silence, firing and unavailable controls + run: bash tests/workflows/aspect_scope_test.sh + - name: Check tracked source aspects + run: bash tests/aspect_tests.sh + - name: Bootstrap Idris2 0.8.0 from a verified source commit + shell: bash + run: | + set -euo pipefail + sudo apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends chezscheme libgmp-dev gcc make git ca-certificates + idir="$RUNNER_TEMP/idris2-source" + git init -q "$idir" + git -C "$idir" remote add origin https://github.com/idris-lang/Idris2.git + git -C "$idir" fetch -q --depth 1 origin 15a3e4e70843f7a34100f6470c04b791330788df + git -C "$idir" checkout -q --detach FETCH_HEAD + test "$(git -C "$idir" rev-parse HEAD)" = "15a3e4e70843f7a34100f6470c04b791330788df" + make -C "$idir" bootstrap SCHEME=chezscheme + make -C "$idir" install + "$HOME/.idris2/bin/idris2" --version | grep -F "0.8.0" + echo "$HOME/.idris2/bin" >> "$GITHUB_PATH" + - name: Install Zig 0.16.0 from sha256-verified official archive + shell: bash + run: | + set -euo pipefail + url="https://ziglang.org/download/0.16.0/zig-x86_64-linux-0.16.0.tar.xz" + archive="$RUNNER_TEMP/zig-0.16.0.tar.xz" + curl --fail --location --silent --show-error --proto '=https' "$url" -o "$archive" + printf '%s %s\n' '70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00' "$archive" | sha256sum --check - + tar -xJf "$archive" -C "$RUNNER_TEMP" + test "$("$RUNNER_TEMP/zig-x86_64-linux-0.16.0/zig" version)" = "0.16.0" + echo "$RUNNER_TEMP/zig-x86_64-linux-0.16.0" >> "$GITHUB_PATH" + - name: Declare current scope + run: echo "::notice::The harness builds Idris2 and separately tests Zig; it is not yet a linked Idris2-to-Zig P2P test or a composer E2E test." - name: Run E2E harness run: | - if [ -f tests/e2e.sh ]; then - bash tests/e2e.sh - else - echo "::notice::no tests/e2e.sh present — nothing to run" - fi + test -f tests/e2e.sh || { echo "::error::Required E2E smoke harness missing"; exit 2; } + bash tests/e2e.sh # ─── End-to-End Tests (downstream stack-specific examples) ───────── # Uncomment ONE of the following e2e job blocks matching your stack. diff --git a/.github/workflows/main-estate-audit.yml b/.github/workflows/main-estate-audit.yml new file mode 100644 index 0000000..5d37b0d --- /dev/null +++ b/.github/workflows/main-estate-audit.yml @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: MPL-2.0 +name: Central Estate CI/CD Audit + +# Thin caller of the estate audit, which lives in hyperpolymath/proven-cicd-and-workflows as +# a reusable workflow and reports five component checks: governance, +# hygiene, manifests, languages, advisory. Do not copy the audit's steps in +# here. A copy freezes this repository at whatever cicd-suite commit its +# actions.lock happened to hold, and fixes made upstream never arrive. +# +# The pin is a full SHA, advanced weekly by track-estate-audit.yml (a PR +# whose own run of this audit is the check). +# +# ⚠ DE-ONBOARDED from actions.lock, deliberately, like the secret-scanner +# caller. A reusable workflow onboarded cross-repo startup-fails with zero +# jobs under sha_pinning_required (see the header of cicd-suite's +# main-estate-audit.yml). Do not run a full-directory `gh actions-lock` that +# re-onboards this file. If one does, cicd-suite/scripts/deonboard-estate-audit.sh +# reverses it. + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +permissions: + contents: read + +jobs: + estate: + uses: hyperpolymath/proven-cicd-and-workflows/.github/workflows/main-estate-audit.yml@5b6024c0ce6ea4a97ea6f4c87b1f4e34267d69a6 # cicd-suite main diff --git a/.github/workflows/proven-lock-gate.yml b/.github/workflows/proven-lock-gate.yml new file mode 100644 index 0000000..6b30b5b --- /dev/null +++ b/.github/workflows/proven-lock-gate.yml @@ -0,0 +1,120 @@ +# SPDX-License-Identifier: MPL-2.0 +# proven-cicd: proven-lock-gate@0.3.2 sha256:07266ed730561719a7a1f6ac8859b49d1d126d45bcd8390cfaecc9f289789069 +# Stamped copy of hyperpolymath/proven-cicd-and-workflows templates/proven-lock-gate.yml. +# Do not edit by hand. A changed body fails this gate as drift. The applier +# moves the pinned suite SHA. The line after each `proven-cicd:local-next` +# marker is local and outside the stamp's hash. +# +# The gate judges this repository's actions.lock against its workflows with the +# proven lock kernel. It exits 0 for consistent, 1 for a defect or drift, and 2 +# when it could not run. A run that could not run fails this job: it is never +# skipped and never continue-on-error, so a required check cannot pass without +# a verdict. Every way the gate can fail before its verdict has its own class: +# not_run:pin PROVEN_SHA is not a 40-hex commit +# not_run:identity the suite repository is not the one this gate pins +# (repository id 1326697643 owned by account id 6759885, +# hyperpolymath, after following a rename), it moved to +# another owner, or the name no longer exists: refer, +# never re-run +# not_run:identity-unreadable the repository or owner id could not be read +# (API outage or rate limit): a re-run +# not_run:fetch-missing the pinned commit does not exist in the suite (persistent) +# not_run:fetch the suite could not be fetched after retries (transient) +# not_run:checkout the fetched commit could not be checked out +# not_run:suite-behind the pinned suite is older than this copy: its manifest +# lists an earlier proven-lock-gate (or none). Fix: move +# PROVEN_SHA to a commit whose proven/manifest.json lists +# at least this copy's version +# not_run:suite the checkout is not the pinned commit, is modified, or +# lacks a file or directory the gate runs +# and the suite's own script adds drift, toolchain, build, selftest, extract and +# verdict. The Idris2 RefC runtime is vendored in the suite (dist/), so the build +# needs no network and has no class of its own for a third-party host. +name: Proven lock gate +on: + pull_request: + push: + branches: [main] +permissions: + contents: read +jobs: + proven-lock: + name: Proven lock gate + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + # proven-cicd:local-next + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Judge actions.lock with the proven suite at its pinned commit + env: + # proven-cicd:local-next + PROVEN_SHA: "5b6024c0ce6ea4a97ea6f4c87b1f4e34267d69a6" + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + fail() { echo "PROVEN_LOCK_CLASS=not_run:$1"; [[ -z ${2:-} ]] || echo "$2" >&2; exit 2; } + [[ $PROVEN_SHA =~ ^[0-9a-f]{40}$ ]] || fail pin + repo=hyperpolymath/proven-cicd-and-workflows + # The suite is pinned by commit; the id pins the repository itself, so a + # recreated repository under the same name is refused. A transfer keeps + # the repository id and GitHub redirects the old name to it, so the + # owner id is pinned as well: a transferred repository is refused, and a + # rename within the owner (same repository id, same owner id) passes. + # Two classes: an id that cannot be read is an availability failure (a + # re-run); an id that differs is a different repository (refer). + # A rename answers 301: follow it (-L) and judge the id it leads to. + # The request and every redirect are HTTPS only (--proto, --proto-redir), + # so a redirect cannot carry the token over plain HTTP. A name that no + # longer exists (404, 410) is permanent, so it is identity; any other + # non-200 (403, 429, 5xx) is availability, a re-run. + code=$(curl -sS -L --proto '=https' --proto-redir '=https' --retry 3 -o "$RUNNER_TEMP/proven-repo.json" -w '%{http_code}' \ + -H "Authorization: Bearer $GH_TOKEN" "https://api.github.com/repos/$repo") || + fail identity-unreadable "could not reach the API for $repo" + case $code in + 200) ;; + 404|410) fail identity "$repo does not exist (HTTP $code)" ;; + *) fail identity-unreadable "the API answered HTTP $code for $repo" ;; + esac + id=$(jq -r '.id // empty' "$RUNNER_TEMP/proven-repo.json" 2>/dev/null) || fail identity-unreadable "the reply for $repo is not JSON" + [[ $id =~ ^[0-9]+$ ]] || fail identity-unreadable "the reply for $repo has no id" + [[ $id == 1326697643 ]] || fail identity "$repo has id '$id', not 1326697643" + owner=$(jq -r '.owner.id // empty' "$RUNNER_TEMP/proven-repo.json" 2>/dev/null) || fail identity-unreadable "the reply for $repo has a malformed owner" + [[ $owner =~ ^[0-9]+$ ]] || fail identity-unreadable "the reply for $repo has no owner id" + [[ $owner == 6759885 ]] || + fail identity "$repo (id $id) is owned by account $owner ($(jq -r '.owner.login // "?"' "$RUNNER_TEMP/proven-repo.json" 2>/dev/null)), not 6759885 (hyperpolymath): a transfer" + suite="$RUNNER_TEMP/proven-suite" + git init -q "$suite" + git -C "$suite" remote add origin "https://github.com/$repo.git" + git -C "$suite" config core.sparseCheckout true + printf '%s\n' dist/ scripts/ tests/fixtures/proven-lock/ /proven/manifest.json > "$suite/.git/info/sparse-checkout" + fetched=0 + for delay in 0 10 30; do + sleep "$delay" + if git -C "$suite" fetch -q --depth=1 origin "$PROVEN_SHA" 2>"$RUNNER_TEMP/proven-fetch.err"; then fetched=1; break; fi + # A commit the server does not have will not appear on a retry. + if grep -qiE "not our ref|couldn't find remote ref|unadvertised object|no such remote ref" "$RUNNER_TEMP/proven-fetch.err"; then + fail fetch-missing "$(cat "$RUNNER_TEMP/proven-fetch.err")" + fi + done + [[ $fetched == 1 ]] || fail fetch "$(cat "$RUNNER_TEMP/proven-fetch.err")" + git -C "$suite" checkout -q --detach FETCH_HEAD || fail checkout + [[ $(git -C "$suite" rev-parse HEAD) == "$PROVEN_SHA" ]] || fail suite "checked out $(git -C "$suite" rev-parse HEAD), not $PROVEN_SHA" + [[ -z $(git -C "$suite" status --porcelain) ]] || fail suite "the suite checkout is modified" + # This copy needs a suite at least as new as itself (0.3.0 needs the + # vendored runtime): a pin older than the copy is its own class, so the + # fix is plain (move the pin), not a generic refusal. + mine=$(sed -n '1,2s/^# proven-cicd: proven-lock-gate@\([0-9]*\.[0-9]*\.[0-9]*\) .*/\1/p' "$GITHUB_WORKSPACE/.github/workflows/proven-lock-gate.yml") + [[ -n $mine ]] || fail suite "this copy has no proven-lock-gate stamp" + theirs=$(jq -r '[.artifacts[] | select(.artifact == "proven-lock-gate") | .version][0] // empty' "$suite/proven/manifest.json" 2>/dev/null || true) + [[ -n $theirs && $(printf '%s\n' "$theirs" "$mine" | sort -V | head -n 1) == "$mine" ]] || + fail suite-behind "the suite at $PROVEN_SHA lists proven-lock-gate ${theirs:-nothing}, older than this copy's $mine; move PROVEN_SHA" + for f in scripts/proven-lock-gate.sh scripts/proven-stamp.sh scripts/build-proven-lock.sh \ + scripts/extract-lock-facts.sh dist/proven-lock/proven-lock-verdict.c; do + [[ -f $suite/$f ]] || fail suite "the suite at $PROVEN_SHA lacks $f" + done + for d in dist/proven-lock/runtime/refc dist/proven-lock/runtime/c tests/fixtures/proven-lock; do + [[ -d $suite/$d ]] || fail suite "the suite at $PROVEN_SHA lacks $d/" + done + bash "$suite/scripts/proven-lock-gate.sh" "$GITHUB_WORKSPACE" .github/workflows/proven-lock-gate.yml diff --git a/.github/workflows/static-analysis-gate.yml b/.github/workflows/static-analysis-gate.yml index 739cbed..0fcc061 100644 --- a/.github/workflows/static-analysis-gate.yml +++ b/.github/workflows/static-analysis-gate.yml @@ -119,7 +119,7 @@ jobs: echo "" >> "$GITHUB_STEP_SUMMARY" echo "Skipped: panic-attack not available in this environment." >> "$GITHUB_STEP_SUMMARY" - name: Upload panic-attack findings - uses: actions/upload-artifact@v7.0.2 + uses: actions/upload-artifact@v7.0.1 with: name: panic-attack-findings path: panic-attack-findings.json @@ -245,7 +245,7 @@ jobs: echo "" >> "$GITHUB_STEP_SUMMARY" echo "Skipped: Hypatia scanner not available in this environment." >> "$GITHUB_STEP_SUMMARY" - name: Upload hypatia findings - uses: actions/upload-artifact@v7.0.2 + uses: actions/upload-artifact@v7.0.1 with: name: hypatia-findings path: hypatia-findings.json @@ -326,7 +326,7 @@ jobs: echo "" >> "$GITHUB_STEP_SUMMARY" echo "Skipped: panic-attack not available in this environment." >> "$GITHUB_STEP_SUMMARY" - name: Upload bridge report - uses: actions/upload-artifact@v7.0.2 + uses: actions/upload-artifact@v7.0.1 with: name: bridge-report path: bridge-report.json @@ -418,7 +418,7 @@ jobs: echo "medium=$MEDIUM" >> "$GITHUB_OUTPUT" echo "low=$LOW" >> "$GITHUB_OUTPUT" - name: Upload unified findings (fleet scanner picks these up) - uses: actions/upload-artifact@v7.0.2 + uses: actions/upload-artifact@v7.0.1 with: name: unified-findings path: findings/unified-findings.json diff --git a/docs/status/ASSURANCE-MATRIX.adoc b/docs/status/ASSURANCE-MATRIX.adoc new file mode 100644 index 0000000..a165ecd --- /dev/null +++ b/docs/status/ASSURANCE-MATRIX.adoc @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 += Scaffoldia Evidence and Coverage Matrix (2026-11-15) +:toc: +:sectnums: + +== Source authority and evidence + +The normative taxonomy is +`metadatastician/standards/3-practice/testing-and-benchmarking/TESTING-TAXONOMY.adoc` +at `d677777725ebf3d69b7eb27534612da77ae95ed6`: **18 categories and 14 aspects**. +The proven test doctrine and downstream evidence contract belong to +`hyperpolymath/proven-tests-and-benches` at +`f90bacbe3a5fa1869239fcb9175757509464aa13`. +The reusable CI source for this pilot is +`hyperpolymath/proven-cicd-and-workflows` at +`5b6024c0ce6ea4a97ea6f4c87b1f4e34267d69a6`. + +IMPORTANT: the published Proven-Tests schema-v2 receipt has **17 x 14 = 238** +cells. Category 18 was added to the standards taxonomy afterwards. +`proven-tests-and-benches/docs/TAXONOMY-VERSION-BOUNDARY.adoc` explicitly +requires a versioned v3 migration for 18 x 14 = 252. Until then, classify +category 18 **outside** schema-v2 reports. Never change a denominator and +reinterpret previously issued evidence. + +Status vocabulary: OBSERVED = executed, nonvacuous local check; THIN = +executed but weaker than category obligations; ABSENT = no executed +admissible test; N/A = justified inapplicable subject. Proposed workflow +changes and unexecuted CI count as **pending**, not OBSERVED. + +== Test-category assessment + +[cols="1,3,1,6",options="header"] +|=== +| # | Category | Status | Actual scope / outstanding obligation + +| 1 | Unit | OBSERVED | 3 Zig-side implementation tests, not a composer +| 2 | P2P | ABSENT | Idris2 `C:rsr_*` imports do not match Zig `scaffoldia_*` exports; no linked consumer +| 3 | E2E | THIN | Existing `tests/e2e.sh` builds Idris2 then Zig independently; no mint or retrofit +| 4 | Build | OBSERVED | Local Idris2 0.8.0 and Zig 0.16.0 builds passed; pinned CI installation is pending a PR run +| 5 | Execution / runtime | THIN | Zig integration tests exercise stubbed FFI functions, not composer runtime +| 6 | Reflexive | THIN | Proven-lock selftest and aspect-harness silence/firing/void controls; not composer self-audit +| 7 | Lifecycle | THIN | Zig handle init/free tests; no repository mint/retrofit state transitions +| 8 | Smoke | OBSERVED | 3-section compiler and Zig smoke run locally; not a usable repository designer +| 9 | Property-based / generative | ABSENT | No generated composer property cases, shrinking, regression seeds +| 10 | Mutation | ABSENT | No enumerated mutants or mutation score of the subject +| 11 | Fuzz | ABSENT | No live parser, deed, path, lock or FFI fuzz harness; do not credit placeholders +| 12 | Contract / invariant | THIN | Proven lock verdict on current lock; no composer contract, deed or ABI behavioural conformance +| 13 | Regression | THIN | Aspect false-positive and lock-ref controls; no comprehensive issue-linked corpus +| 14 | Chaos / resilience | ABSENT | No interruption, partial-write, atomicity or rollback injection +| 15 | Compatibility | ABSENT | No real version/platform migration or parent-tree compatibility matrix +| 16 | Proof regression | THIN | Idris module builds; linked proof claims and Coq/Lean example artifacts not gated as evidence +| 17 | Type-safe | THIN | Idris2 compilation; no pinned negative compile-failure witnesses for composer safety +| 18 | Coupling / drift-freedom | THIN | Local generated-lock agreement checked; deed/canon/spine/ABI drift not covered. Outside v2 lattice +|=== + +== Fourteen aspect dimensions + +[cols="2,1,5",options="header"] +|=== +| Aspect | Status | What actually runs or still needs evidence + +| Dependability | THIN | Build/smoke locally; no crash and recovery tests +| Security | THIN | Basic lock and source checks; failed/missing reusable security jobs on main +| Usability | ABSENT | No real composer CLI user journey +| Interoperability | ABSENT | Missing linked Idris2/Zig P2P test; deed reader unimplemented +| Safety | THIN | Lexical proof escape guard; not a source-linked proof or path write safety proof +| Performance | THIN | Generic build timing templates only; no composer or stable benchmark baseline +| Functionality | ABSENT | Composer/mint/retrofit unimplemented +| Versability | ABSENT | No contract version migration or binary compatibility tests +| Accessibility | N/A | No working UI or CLI interface to audit; revisit when an interface exists +| Maintainability | THIN | Tool and source-scope checks; some stale validators remain +| Privacy | THIN | Security scanner intended; no repository identity/telemetry data-minimization evidence +| Observability | THIN | Lock and aspect errors classified; no execution receipt schema or runtime trace +| Reproducibility | THIN | Immutable proven suite pin; same-input composer artifact comparison absent +| Portability | THIN | Linux x86_64 local toolchain; platform and architecture matrix absent +|=== + +== Local execution receipt (base source, not a CI verdict) + +Subject: Scaffoldia `8c833fbe38c63561b7a9d7bc1c74f786c95d49d1`, +temporary clean checkout on Linux, Idris2 `0.8.0` and Zig `0.16.0`. + +* `bash tests/e2e.sh`: passed 3 build/test stages; does **not** link the ABI. +* Initial `bash tests/aspect_tests.sh`: failed on generated Zig cache and + Coq/docs commentary; existing source and proof scope was defective. +* Repaired `bash tests/workflows/aspect_scope_test.sh`: four controls + pass (one silence, two seeded firing, one NOT_RUN). +* Repaired `bash tests/aspect_tests.sh`: passes 15 tracked source files, + including 12 proof-language files, lexical only. +* `bash scripts/check-workflow-uses-lock.sh .`: baseline fails on four + `actions/upload-artifact` references mismatched with the generated lock; + corrected in this pilot. +* Published Proven Lock Gate, run against the local candidate with pinned + proven source: `PROVEN_LOCK_CLASS=consistent` (exit 0). It explicitly + marked `OwnerIdDrift` and `Moved` unjudged (no oracle snapshot). + +Every claim above is **local**; a PR must reproduce the results with its +latest commit. A skipped, missing, startup-failed or cancelled required job +is not passing evidence. + +== Next assurance obligations (priority order) + +. P0: make CI toolchains and all required workflow calls actually execute; + repair stale standards reusable coordinates and fix policy-dependent + `actions.lock` closure without disabling existing security checks. +. P0: obtain actual Idris2-to-Zig or C linked consumer evidence using the + generic Proven-Tests ABI test **after** its review and promotion; refuse + an ABI compatibility claim based on two independent builds. +. P1: implement deterministic composer plan, sandboxed mint and retrofit, + then add two independent source-derived fixtures (Julia and Rust). +. P1: build admissible silence/firing/void specimens for deed parsing, + canonical overlay precedence, traversal, symlink escape, collisions, + idempotence and three-way conflicts; contribute genuinely reusable + primitives to the proven corpus first. +. P1: apply mutation, fuzz and generative tests to the implemented + composer; retain seeds, shrinking, corpus provenance and real failure + controls. +. P2: baseline real composer benchmarks and update readiness only from + executed evidence, reviewed receipts and Hypatia's own scoring. + +Do not turn advisory proven-cicd catalogue items, experimental mother +profiles, or 17-category schema-v2 output into false mandatory coverage. diff --git a/docs/status/TEST-NEEDS.adoc b/docs/status/TEST-NEEDS.adoc index 627a105..e3ba2a6 100644 --- a/docs/status/TEST-NEEDS.adoc +++ b/docs/status/TEST-NEEDS.adoc @@ -1,64 +1,46 @@ // SPDX-License-Identifier: CC-BY-SA-4.0 -// Copyright (c) Jonathan D.A. Jewell = TEST-NEEDS: scaffoldia - -== Current State (Updated 2026-07-27 — instantiation cure) - -This file previously described the template's own dogfooding of its -now-deleted validation machinery (`scripts/validate-template.sh`, -`tests/e2e/template_instantiation_test.sh`). Those scripts existed only to -validate an *un-instantiated* RSR template and made no sense once scaffoldia -was instantiated with its own real identity — they have been removed. This -section instead describes the test infrastructure scaffoldia actually has. - -[cols="2,1,4", options="header"] -|=== -| Category | Count | Details - -| *Source modules* | 6 | 3 Idris2 ABI (Foreign, Layout, Types), 2 Zig FFI (build.zig, main.zig), 1 Zig integration test module -| *Unit tests* | 3 | Inline `test` blocks in `src/interface/ffi/src/main.zig` (lifecycle, error handling, version) -| *Integration tests* | 10 | `src/interface/ffi/test/integration_test.zig` — real tests against the exported FFI (lifecycle, operations, strings, version, build_info, error handling) -| *E2E tests* | 1 | `tests/e2e.sh` — real preflight + `idris2 --build src/interface/abi.ipkg` + `zig build`/`zig build test` -| *Aspect tests* | 1 | `tests/aspect_tests.sh` — SPDX header coverage + dangerous-pattern grep (real, pre-existing, unchanged by the cure) -| *Workflow tests* | 1 | `tests/workflows/validate_workflows_test.sh` (unchanged by the cure) -| *Benchmarks* | 3 | `benches/template_bench.sh` — Zig build, Zig tests, workflow validation -|=== - -== Zig Build & Test (Verified) - -* `zig fmt --check .` — exit 0 -* `cd src/interface/ffi && zig build` — exit 0 -* `cd src/interface/ffi && zig build test --summary all` — exit 0, **13/13 tests pass** (3 unit + 10 integration) -* `idris2 --build src/interface/abi.ipkg` — exit 0 (2 pre-existing shadowing warnings in `Abi.Layout`, unrelated to this cure, out of scope) - -== What Changed From the Template State - -* [.line-through]#`scripts/validate-template.sh` (8-phase template validation)# → REMOVED (validated an un-instantiated template; scaffoldia is instantiated) -* [.line-through]#`tests/e2e/template_instantiation_test.sh`# → REMOVED (same reason) -* `src/interface/ffi/build.zig` was a no-op stub (Zig 0.15-era comment claiming - tests "are run directly with zig build-exe -ftest-runner") → now a real - `build.zig`: static library target + `zig build test` wired to both the - main module and the integration-test module. -* `src/interface/ffi/src/main.zig` had three real Zig 0.16 compile bugs - beyond placeholder substitution (`opaque` type with fields; `callconv(.C)` - invalid casing; `std.heap.c_allocator` used without linking libc) — fixed. -* `src/interface/ffi/test/integration_test.zig` was a fully-commented-out - placeholder (`test "placeholder test - implementation required"`) → now 10 - real tests exercising every exported `scaffoldia_*` function. -* `tests/e2e.sh` was a fully-commented-out template with only a preflight - header → now runs the real Idris2 build + Zig build/test and reports - PASS/FAIL with the script's existing `check`/`green`/`red` helpers. -* `benches/template_bench.sh` — the "Template Validation" and "Template - Instantiation" benchmark sections (which called the two now-deleted - scripts) were removed; the Zig build/test and workflow-validation - benchmarks were kept and renumbered. - -== Author Attribution - -Jonathan D.A. Jewell <6759885+hyperpolymath@users.noreply.github.com> - -== Next Steps (Future Sessions) - -* [ ] Integrate `just test-all` into CI/CD workflows -* [ ] Generate test coverage reports -* [ ] Add benchmarks for the composition engine once it exists (currently design phase — see `.machine_readable/descriptiles/STATE.a2ml`) +:toc: + +== Live source and honest coverage (2026-11-15) + +The full **18-category x 14-aspect** crosswalk, test evidence and +unimplemented obligations are maintained in +link:ASSURANCE-MATRIX.adoc[ASSURANCE-MATRIX.adoc]. + +The repository remains in design phase. Its composer, mint planner, +retrofit planner and tested Idris2-to-Zig ABI are **not implemented**. +A successful build of each language separately is not linked ABI evidence. +The old record's phrase "1 E2E test" referred to a compiler/FFI smoke +harness; it never exercised a minted repository. + +The published Proven-Tests downstream receipt schema-v2 remains +17 x 14 and cannot credit the standards' new category 18. Consult +`hyperpolymath/proven-tests-and-benches/docs/TAXONOMY-VERSION-BOUNDARY.adoc` +before changing a report denominator. + +== Verified local execution + +* `bash tests/e2e.sh`: three separate build/test stages passed on a + local Linux runner with Idris2 0.8.0 and Zig 0.16.0. +* Repaired `bash tests/aspect_tests.sh`: 15 tracked files passed source-scope + and comment-filtered lexical checks. +* `bash tests/workflows/aspect_scope_test.sh`: silence, two firing, + NOT_RUN controls passed. +* Proven Lock Gate at published pinned source: exit 0, `consistent`; + remote owner drift and moved repository references were unjudged. + +These are not claims about PR CI. Every proposed workflow check must run +and pass on the latest head before promotion. Composer functionality, +proofs, property tests, fuzzing, mutation, rollback and benchmarks remain +gaps, not implicit N/A. + +== Test provenance + +The source-of-truth test rules belong to +`metadatastician/standards/3-practice/testing-and-benchmarking/TESTING-TAXONOMY.adoc`. +Before making local tests, use `hyperpolymath/proven-tests-and-benches`; +the generic workflows source is +`hyperpolymath/proven-cicd-and-workflows`. +Silence/firing and NOT_RUN controls follow the Proven-Tests +`docs/TEST-DOCTRINE.adoc`. diff --git a/tests/aspect_tests.sh b/tests/aspect_tests.sh index c29f358..dbfe37f 100755 --- a/tests/aspect_tests.sh +++ b/tests/aspect_tests.sh @@ -1,134 +1,57 @@ #!/usr/bin/env bash # SPDX-License-Identifier: MPL-2.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# RSR Standard Aspect Test Template -# -# Aspect tests validate cross-cutting architectural invariants that span -# the entire codebase. These are NOT functional tests — they verify that -# coding standards, safety rules, and structural contracts hold. -# -# Usage: -# bash tests/aspect_tests.sh -# just aspect -# -# Standard aspects (enable what applies to your project): -# 1. SPDX compliance — all source files have license headers -# 2. Dangerous patterns — no believe_me, assert_total, sorry, unsafeCoerce, etc. -# 3. ABI/FFI contract — declarations match exports -# 4. Thread safety — mutex in FFI modules -# 5. Error handling — no panic/unreachable in production paths - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" -cd "$PROJECT_DIR" - -PASS=0 -FAIL=0 -WARN=0 - -green() { printf '\033[32m%s\033[0m\n' "$*"; } -red() { printf '\033[31m%s\033[0m\n' "$*"; } -yellow(){ printf '\033[33m%s\033[0m\n' "$*"; } -bold() { printf '\033[1m%s\033[0m\n' "$*"; } - -pass() { green " PASS: $1"; PASS=$((PASS + 1)); } -fail() { red " FAIL: $1"; FAIL=$((FAIL + 1)); } -warn() { yellow " WARN: $1"; WARN=$((WARN + 1)); } - -echo "═══════════════════════════════════════════════════════════════" -echo " SCAFFOLDIA — Aspect Tests (Cross-Cutting Concerns)" -echo "═══════════════════════════════════════════════════════════════" -echo "" - -# ═══════════════════════════════════════════════════════════════════════ -# Aspect 1: SPDX License Headers -# ═══════════════════════════════════════════════════════════════════════ -bold "Aspect 1: SPDX license headers" - -MISSING_SPDX=0 -while IFS= read -r -d '' f; do - if ! head -5 "$f" | grep -q "SPDX-License-Identifier"; then - warn "Missing SPDX header: $f" - MISSING_SPDX=$((MISSING_SPDX + 1)) - fi -done < <(find src/ -type f \( -name "*.rs" -o -name "*.zig" -o -name "*.res" -o -name "*.ex" -o -name "*.exs" -o -name "*.gleam" -o -name "*.idr" -o -name "*.sh" \) -print0 2>/dev/null) - -if [ "$MISSING_SPDX" -eq 0 ]; then - pass "All source files have SPDX headers" -else - fail "$MISSING_SPDX files missing SPDX headers" +# Scoped source assessment: not a formal proof or ABI test. +set -uo pipefail +root="${1:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +if ! command -v git >/dev/null || ! command -v perl >/dev/null; then + echo 'NOT_RUN: git or perl unavailable' >&2 + exit 2 fi - -# ═══════════════════════════════════════════════════════════════════════ -# Aspect 2: Dangerous Patterns (BANNED) -# ═══════════════════════════════════════════════════════════════════════ -bold "Aspect 2: Dangerous patterns" - -# Idris2 dangerous patterns -DANGEROUS_IDRIS=$(grep -rn 'believe_me\|assert_total\|really_believe_me' src/abi/ 2>/dev/null | grep -v "^Binary" | grep -v "test" || true) -if [ -n "$DANGEROUS_IDRIS" ]; then - fail "Dangerous Idris2 patterns found:" - echo "$DANGEROUS_IDRIS" | head -5 -else - pass "No dangerous Idris2 patterns (believe_me, assert_total)" +root="$(cd "$root" 2>/dev/null && pwd)" || { echo 'NOT_RUN: missing target' >&2; exit 2; } +if ! git -C "$root" rev-parse --show-toplevel >/dev/null 2>&1; then + echo 'NOT_RUN: not a Git checkout' >&2 + exit 2 fi - -# Coq/Lean dangerous patterns -DANGEROUS_PROOF=$(grep -rn '\bAdmitted\b\|\bsorry\b\|\bunsafeCoerce\b\|\bObj\.magic\b' src/ verification/ 2>/dev/null | grep -v "test" | grep -v "comment" || true) -if [ -n "$DANGEROUS_PROOF" ]; then - fail "Dangerous proof patterns found:" - echo "$DANGEROUS_PROOF" | head -5 +fail=0 +checked=0 +proofs=0 +while IFS= read -r -d '' path; do + case "$path" in *.idr|*.zig|*.rs|*.sh|*.v|*.lean|*.agda|*.thy) ;; *) continue ;; esac + file="$root/$path" + [ -f "$file" ] || { echo "FAIL: tracked source disappeared: $path" >&2; fail=1; continue; } + checked=$((checked+1)) + if ! head -6 "$file" | grep -qE '^[[:space:]]*(--|//|#|\(\*)[[:space:]]*SPDX-License-Identifier:'; then + echo "FAIL: missing SPDX header in tracked source: $path" >&2 + fail=1 + fi + case "$path" in + *.idr|*.v|*.lean|*.agda|*.thy) + proofs=$((proofs+1)) + # Lexical guard only. The language-specific proof checker owns soundness. + if ! perl -0777 -e ' + local $/; + my $source = <>; + $source =~ s/\(\*.*?\*\)//gs; + $source =~ s/(?:--|\/\/)[^\n]*//g; + exit ($source =~ /\b(?:believe_me|really_believe_me|assert_total|Admitted|sorry|unsafeCoerce)\b/ ? 1 : 0); + ' "$file"; then + echo "FAIL: proof escape in executable source: $path" >&2 + fail=1 + fi + ;; + esac +done < <(cd "$root" && git ls-files -z -- src verification/proofs) +if [ "$checked" -eq 0 ]; then + echo 'NOT_RUN: no tracked source code to assess' >&2 + exit 2 +fi +if [ "$proofs" -eq 0 ]; then + echo 'NOT_APPLICABLE: no proof-language sources; no proof assurance claimed' else - pass "No dangerous proof patterns (Admitted, sorry, unsafeCoerce)" + echo "CHECKED: $proofs proof-language source files (lexical; not a proof)" +fi +if [ "$fail" -ne 0 ]; then + echo "FAIL: tracked-source aspect check ($checked source files)" >&2 + exit 1 fi - -# ═══════════════════════════════════════════════════════════════════════ -# Aspect 3: ABI/FFI Contract (if applicable) -# ═══════════════════════════════════════════════════════════════════════ -# Uncomment if your project has Idris2 ABI + Zig FFI: - -# bold "Aspect 3: ABI/FFI contract" -# if [ -d "src/abi" ] && [ -d "ffi/zig" ]; then -# # Check that every exported function in Idris2 ABI has a Zig FFI implementation -# ABI_EXPORTS=$(grep -h 'export' src/abi/*.idr 2>/dev/null | wc -l) -# FFI_EXPORTS=$(grep -h 'pub export fn' ffi/zig/src/*.zig 2>/dev/null | wc -l) -# if [ "$ABI_EXPORTS" -gt 0 ] && [ "$FFI_EXPORTS" -gt 0 ]; then -# pass "ABI ($ABI_EXPORTS exports) and FFI ($FFI_EXPORTS exports) both present" -# else -# fail "ABI/FFI mismatch: $ABI_EXPORTS ABI exports, $FFI_EXPORTS FFI exports" -# fi -# else -# pass "ABI/FFI not applicable (no src/abi or ffi/zig)" -# fi - -# ═══════════════════════════════════════════════════════════════════════ -# Aspect 4: Error Handling (no raw panic in production code) -# ═══════════════════════════════════════════════════════════════════════ -# Uncomment for Rust projects: - -# bold "Aspect 4: Error handling" -# UNWRAP_COUNT=$(grep -rn '\.unwrap()' src/ 2>/dev/null | grep -v "test" | grep -v "example" | wc -l) -# if [ "$UNWRAP_COUNT" -gt 20 ]; then -# warn "$UNWRAP_COUNT .unwrap() calls in src/ — consider replacing with ? or expect()" -# else -# pass "Acceptable unwrap count: $UNWRAP_COUNT" -# fi - -# ═══════════════════════════════════════════════════════════════════════ -# Summary -# ═══════════════════════════════════════════════════════════════════════ -echo "" -echo "═══════════════════════════════════════════════════════════════" -printf " Results: " -green "PASS=$PASS" | tr -d '\n' -echo -n " " -if [ "$FAIL" -gt 0 ]; then red "FAIL=$FAIL" | tr -d '\n'; else echo -n "FAIL=0"; fi -echo -n " " -if [ "$WARN" -gt 0 ]; then yellow "WARN=$WARN"; else echo "WARN=0"; fi -echo "" -echo "═══════════════════════════════════════════════════════════════" - -exit "$FAIL" +echo "PASS: $checked tracked source files; generated caches and prose excluded" diff --git a/tests/workflows/aspect_scope_test.sh b/tests/workflows/aspect_scope_test.sh new file mode 100644 index 0000000..03d715c --- /dev/null +++ b/tests/workflows/aspect_scope_test.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# Harness/payload controls: silence, firing and NOT_RUN. +set -euo pipefail +checker="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/aspect_tests.sh" +tmp="$(mktemp -d)" +trap 'rm -rf -- "$tmp"' EXIT +mkdir -p "$tmp/subject/src" "$tmp/subject/docs" "$tmp/subject/verification/proofs" "$tmp/subject/src/interface/ffi/.zig-cache" +git -C "$tmp/subject" init -q +printf '// SPDX-License-Identifier: MPL-2.0\npub fn yes() void {}\n' > "$tmp/subject/src/main.zig" +printf '// SPDX-License-Identifier: MPL-2.0\n(* Admitted, sorry, unsafeCoerce are quoted, not executed. *)\nLemma sample : True.\nProof. exact I. Qed.\n' > "$tmp/subject/verification/proofs/Safety.v" +printf 'Admitted in prose is not proof code.\n' > "$tmp/subject/docs/prose.adoc" +printf 'generated zig cache without SPDX\n' > "$tmp/subject/src/interface/ffi/.zig-cache/dependencies.zig" +git -C "$tmp/subject" add src/main.zig verification/proofs/Safety.v docs/prose.adoc +expect() { + local label="$1" want="$2" actual=0 + bash "$checker" "$3" > "$tmp/actual.log" 2>&1 || actual=$? + if [ "$actual" -ne "$want" ]; then + echo "FAIL: $label: wanted exit $want, observed $actual" >&2 + cat "$tmp/actual.log" >&2 + exit 1 + fi + echo "PASS: $label ($want)" +} +expect 'silence: clean source + generated cache + commented proof terms' 0 "$tmp/subject" +printf '// SPDX missing\npub fn yes() void {}\n' > "$tmp/subject/src/main.zig" +expect 'firing: tracked source without SPDX' 1 "$tmp/subject" +printf '// SPDX-License-Identifier: MPL-2.0\npub fn yes() void {}\n' > "$tmp/subject/src/main.zig" +printf '%s\n' '-- SPDX-License-Identifier: MPL-2.0' '%default total' 'unsafeValue = believe_me 0' > "$tmp/subject/src/Safety.idr" +git -C "$tmp/subject" add src/Safety.idr +expect 'firing: proof escape in Idris2 source' 1 "$tmp/subject" +mkdir -p "$tmp/void" +git -C "$tmp/void" init -q +expect 'void: no source means no assessment' 2 "$tmp/void" +echo 'PASS: aspect harness silence/firing/void controls' From 7e5211f05eb5242490cd6099ac3003fba5c518df Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:16:56 +0000 Subject: [PATCH 2/3] fix(governance): satisfy proven structural and wiki-format gates --- docs/architecture/README.adoc | 32 ++++++++++++++++++++ docs/{ => legacy}/wikis/0.2-AI-MANIFEST.a2ml | 0 docs/wikis/README.adoc | 17 ----------- docs/wikis/README.md | 23 ++++++++++++++ 4 files changed, 55 insertions(+), 17 deletions(-) create mode 100644 docs/architecture/README.adoc rename docs/{ => legacy}/wikis/0.2-AI-MANIFEST.a2ml (100%) delete mode 100644 docs/wikis/README.adoc create mode 100644 docs/wikis/README.md diff --git a/docs/architecture/README.adoc b/docs/architecture/README.adoc new file mode 100644 index 0000000..5f6c0c1 --- /dev/null +++ b/docs/architecture/README.adoc @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 += Scaffoldia Architecture +:toc: + +== Current implementation boundary + +Scaffoldia is a repository design and retrofit project in the design phase. +The repository presently supplies governance metadata, workflow automation +and independently buildable Idris2 and Zig modules. The actual composer, +repository mint operation, retrofit planner and linked ABI are not yet +implemented or qualified. + +== Existing directory layout + +* `src/` contains source pillars and a generic inherited foreign-function + interface scaffold; it does not contain a functioning composition engine. +* `tests/` contains existing compiler, Zig and source-scope test harnesses. +* `docs/architecture/` contains this introduction, the component topology, + and the source threat model. +* `.machine_readable/` holds repository machine metadata, some of which + is obsolete A2ML awaiting controlled migration to DEED. +* `.github/workflows/` contains automation whose individual checks + must run and report their true state. + +== Contracts and limits + +Read link:TOPOLOGY.adoc[TOPOLOGY.adoc] for the owning canon/spine/proven +boundaries, missing interfaces and the ABI mismatch. Read +link:THREAT-MODEL.adoc[THREAT-MODEL.adoc] for threats and mitigations. +The currently proposed engine decision is +link:../decisions/0002-scaffoldia-composer-architecture.adoc[ADR-0002]. +Nothing here certifies that the composer works or that its ABI is linked. diff --git a/docs/wikis/0.2-AI-MANIFEST.a2ml b/docs/legacy/wikis/0.2-AI-MANIFEST.a2ml similarity index 100% rename from docs/wikis/0.2-AI-MANIFEST.a2ml rename to docs/legacy/wikis/0.2-AI-MANIFEST.a2ml diff --git a/docs/wikis/README.adoc b/docs/wikis/README.adoc deleted file mode 100644 index 16bb1a0..0000000 --- a/docs/wikis/README.adoc +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-License-Identifier: CC-BY-SA-4.0 -// Copyright (c) Jonathan D.A. Jewell -= Project Wikis -:toc: preamble -:icons: font - -This directory contains the source files for the project wiki. It is intended for long-form documentation, deep-dives, and community-maintained knowledge. - -== Structure - -* **Core Concepts:** Fundamental architectural ideas. -* **Workflows:** Step-by-step guides for contributors. -* **Glossary:** Definitions of project-specific terminology. - -== Wiki Synchronization - -Changes made here should be synchronised with the forge-hosted wiki (GitHub/GitLab) using the project's sync scripts. diff --git a/docs/wikis/README.md b/docs/wikis/README.md new file mode 100644 index 0000000..f2b1ab0 --- /dev/null +++ b/docs/wikis/README.md @@ -0,0 +1,23 @@ + +# Scaffoldia wiki sources + +This directory is the repository's human-facing wiki area. Its current +document format is Markdown, consistent with the estate wiki policy; +long-form non-wiki technical documentation remains under docs/ in AsciiDoc. + +## Scope + +Future wiki material may explain core repository design concepts, operator +workflows, and the glossary for composition and retrofit. These are not +implemented user operations yet; see +[the architecture overview](../architecture/README.adoc) for the current state. + +## Synchronisation + +No wiki synchronisation run or published wiki is verified by this document. +Do not assume that edits here have reached GitHub, GitLab or any external forge. + +The earlier A2ML wiki-track specimen has been preserved in +[the legacy records](../legacy/wikis/0.2-AI-MANIFEST.a2ml). +It required AsciiDoc within a wiki and predates the current wiki-format policy; +it is historical input, not an active standards contract. From 007fea2dd905a0ebd7279c42b05dbf255bf94211 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Sun, 15 Nov 2026 20:05:35 +0000 Subject: [PATCH 3/3] fix(assurance): close proof lexer gaps and align wiki-format gates --- .github/workflows/e2e.yml | 2 + .github/workflows/estate-rules.yml | 4 +- scripts/check-no-md-in-docs.sh | 89 ++++++++++++---------------- tests/aspect_tests.sh | 24 ++++---- tests/scan-proof-escapes.pl | 83 ++++++++++++++++++++++++++ tests/workflows/aspect_scope_test.sh | 64 ++++++++++++++++++++ tests/workflows/wiki_format_test.sh | 34 +++++++++++ 7 files changed, 238 insertions(+), 62 deletions(-) create mode 100644 tests/scan-proof-escapes.pl create mode 100644 tests/workflows/wiki_format_test.sh diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 2d0308a..c34e8fc 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -23,6 +23,7 @@ on: - 'src/**' - 'ffi/**' - 'tests/**' + - 'verification/proofs/**' - '.github/workflows/e2e.yml' - 'build/**' - 'Justfile' @@ -33,6 +34,7 @@ on: - 'src/**' - 'ffi/**' - 'tests/**' + - 'verification/proofs/**' - '.github/workflows/e2e.yml' - 'build/**' - 'Justfile' diff --git a/.github/workflows/estate-rules.yml b/.github/workflows/estate-rules.yml index d880f81..12251c4 100644 --- a/.github/workflows/estate-rules.yml +++ b/.github/workflows/estate-rules.yml @@ -28,5 +28,7 @@ jobs: uses: actions/checkout@v7.0.1 - name: Root shape allowlist run: bash scripts/check-root-shape.sh . - - name: AsciiDoc by default (no .md under docs/) + - name: Prove wiki-format policy controls (positive and negative) + run: bash tests/workflows/wiki_format_test.sh + - name: AsciiDoc for documents, Markdown for wiki sources run: bash scripts/check-no-md-in-docs.sh . diff --git a/scripts/check-no-md-in-docs.sh b/scripts/check-no-md-in-docs.sh index 102a6b8..42d6cc6 100644 --- a/scripts/check-no-md-in-docs.sh +++ b/scripts/check-no-md-in-docs.sh @@ -1,57 +1,46 @@ #!/usr/bin/env bash # SPDX-License-Identifier: MPL-2.0 -# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# check-no-md-in-docs.sh — enforce "AsciiDoc by default for general docs". -# -# Estate rule: .adoc for general docs (TOPOLOGY, READINESS, ROADMAP, etc.); -# .md only for files GitHub's community-health rules special-case by name -# (CONTRIBUTING, CODE_OF_CONDUCT, SECURITY, CHANGELOG, etc.) — those live at -# root or in .github/, never under docs/. -# -# Fails if any .md files exist under docs/. Add justified entries to the -# ALLOWED list below if a docs/-rooted .md is genuinely needed (rare). -# -# Exit codes: -# 0 — no .md files under docs/ (or all matches are allow-listed) -# 1 — disallowed .md files found -# 2 — usage / setup error - +# Enforce canonical estate document-format split: +# docs// -> AsciiDoc (.adoc) +# docs/wiki/ and docs/wikis/ -> Markdown (.md) wiki content +# The owner of this rule is proven-cicd-and-workflows/actions/formatting-check. set -euo pipefail -REPO_ROOT="${1:-.}" -DOCS_DIR="$REPO_ROOT/docs" - -# Justified exceptions, relative to repo root. Empty by default. -ALLOWED=() - -if [ ! -d "$DOCS_DIR" ]; then - echo "PASS: no docs/ directory (nothing to check)" - exit 0 +root="$(cd "${1:-.}" 2>/dev/null && pwd)" || { + echo 'NOT_RUN: repository root does not exist' >&2 + exit 2 +} +command -v git >/dev/null || { echo 'NOT_RUN: git unavailable' >&2; exit 2; } +git -C "$root" rev-parse --show-toplevel >/dev/null 2>&1 || { + echo 'NOT_RUN: not a Git checkout' >&2 + exit 2 +} +if [ ! -d "$root/docs" ]; then + echo 'NOT_APPLICABLE: repository has no docs/ tree' + exit 0 fi -mapfile -t HITS < <(find "$DOCS_DIR" -name '*.md' -type f 2>/dev/null | sort) - -EXTRAS=() -for hit in "${HITS[@]}"; do - rel="${hit#"$REPO_ROOT/"}" - skip=0 - for allowed in "${ALLOWED[@]}"; do - if [ "$rel" = "$allowed" ]; then skip=1; break; fi - done - if [ $skip -eq 0 ]; then EXTRAS+=("$rel"); fi -done - -if [ ${#EXTRAS[@]} -eq 0 ]; then - echo "PASS: no .md files under docs/ (${#HITS[@]} total found, ${#ALLOWED[@]} allow-listed)" - exit 0 +checked=0 +bad=0 +while IFS= read -r -d '' path; do + case "$path" in + docs/*.md) + checked=$((checked+1)) + case "$path" in + docs/wiki/*.md|docs/wikis/*.md) + echo "OK: Markdown wiki source: $path" + ;; + *) + echo "FAIL: non-wiki documentation must be AsciiDoc: $path" >&2 + bad=$((bad+1)) + ;; + esac + ;; + esac +done < <(git -C "$root" ls-files -z --cached --others --exclude-standard -- docs) + +if [ "$bad" -gt 0 ]; then + echo "FAIL: $bad Markdown document(s) outside wiki roots; $checked checked" >&2 + exit 1 fi - -echo "FAIL: ${#EXTRAS[@]} .md files found under docs/ (estate rule: AsciiDoc by default):" >&2 -for e in "${EXTRAS[@]}"; do - echo " - $e" >&2 -done -echo "" >&2 -echo "Convert these to .adoc, or add a justified entry to the ALLOWED list" >&2 -echo "in scripts/check-no-md-in-docs.sh." >&2 -exit 1 +echo "PASS: $checked Markdown path(s) conform to canonical wiki-only exception" diff --git a/tests/aspect_tests.sh b/tests/aspect_tests.sh index dbfe37f..8065dc2 100755 --- a/tests/aspect_tests.sh +++ b/tests/aspect_tests.sh @@ -12,6 +12,11 @@ if ! git -C "$root" rev-parse --show-toplevel >/dev/null 2>&1; then echo 'NOT_RUN: not a Git checkout' >&2 exit 2 fi +scanner="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/scan-proof-escapes.pl" +if [ ! -r "$scanner" ]; then + echo 'NOT_RUN: required proof-source scanner absent' >&2 + exit 2 +fi fail=0 checked=0 proofs=0 @@ -27,17 +32,14 @@ while IFS= read -r -d '' path; do case "$path" in *.idr|*.v|*.lean|*.agda|*.thy) proofs=$((proofs+1)) - # Lexical guard only. The language-specific proof checker owns soundness. - if ! perl -0777 -e ' - local $/; - my $source = <>; - $source =~ s/\(\*.*?\*\)//gs; - $source =~ s/(?:--|\/\/)[^\n]*//g; - exit ($source =~ /\b(?:believe_me|really_believe_me|assert_total|Admitted|sorry|unsafeCoerce)\b/ ? 1 : 0); - ' "$file"; then - echo "FAIL: proof escape in executable source: $path" >&2 - fail=1 - fi + # Source-aware lexical guard, not a compiler or proof checker. + rc=0 + perl "$scanner" "$file" || rc=$? + case "$rc" in + 0) ;; + 1) fail=1 ;; + *) echo "NOT_RUN: proof scanner could not assess $path (exit $rc)" >&2; fail=1 ;; + esac ;; esac done < <(cd "$root" && git ls-files -z -- src verification/proofs) diff --git a/tests/scan-proof-escapes.pl b/tests/scan-proof-escapes.pl new file mode 100644 index 0000000..22e9f12 --- /dev/null +++ b/tests/scan-proof-escapes.pl @@ -0,0 +1,83 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: MPL-2.0 +# Fail-closed lexical guard for known proof escape tokens in tracked sources. +# Not a compiler, theorem prover or proof of type soundness. +use strict; +use warnings; + +if (@ARGV != 1 || !-f $ARGV[0] || !-r $ARGV[0]) { + print STDERR "NOT_RUN: expected one readable proof source file\n"; + exit 2; +} +my $file = $ARGV[0]; +my ($ext) = $file =~ /\.([^.\/]+)$/; +$ext //= ''; +my %tokens = ( + idr => qr/\b(?:believe_me|really_believe_me|assert_total)\b/, + lean => qr/\b(?:sorry|unsafeCoerce)\b/, + agda => qr/\b(?:believe_me|really_believe_me|assert_total|sorry|unsafeCoerce)\b/, + thy => qr/\b(?:sorry|Admitted)\b/, + v => qr/\b(?:Admitted\s*\.|admit\b)/, +); +if (!exists $tokens{$ext}) { + print STDERR "NOT_RUN: unsupported proof-source extension: $file\n"; + exit 2; +} +my %block = (idr => ['{-','-}'], agda => ['{-','-}'], + lean => ['/-','-/'], thy => ['(*','*)'], v => ['(*','*)']); +my %line = (idr => '--', agda => '--', lean => '--', thy => '', v => '//'); +open my $fh, '<', $file or die "NOT_RUN: unreadable $file\n"; +local $/; +my $raw = <$fh>; +defined $raw or die "NOT_RUN: unreadable $file\n"; +close $fh; +my ($start, $end) = @{$block{$ext}}; +my ($visible, $depth, $quoted) = ('', 0, 0); +my $length = length($raw); +for (my $i = 0; $i < $length;) { + my $pair = substr($raw, $i, 2); + my $c = substr($raw, $i, 1); + if ($depth) { + if ($pair eq $start) { $depth++; $visible .= ' '; $i += 2; next; } + if ($pair eq $end) { $depth--; $visible .= ' '; $i += 2; next; } + $visible .= ($c eq "\n" ? "\n" : ' '); + $i++; + next; + } + if ($quoted) { + # Coq escapes quote by doubling it; Idris/Lean/Agda also use \. + if ($ext eq 'v' && $pair eq '""') { $visible .= ' '; $i += 2; next; } + if ($c eq '\\' && $ext ne 'v' && $i + 1 < $length) { + $visible .= ' '; $i += 2; next; + } + if ($c eq '"') { $quoted = 0; } + $visible .= ($c eq "\n" ? "\n" : ' '); + $i++; + next; + } + if ($pair eq $start) { $depth = 1; $visible .= ' '; $i += 2; next; } + if ($c eq '"') { $quoted = 1; $visible .= ' '; $i++; next; } + if ($line{$ext} ne '' && substr($raw, $i, length($line{$ext})) eq $line{$ext}) { + while ($i < $length && substr($raw, $i, 1) ne "\n") { + $visible .= ' '; + $i++; + } + next; + } + $visible .= $c; + $i++; +} +if ($depth || $quoted) { + print STDERR "NOT_RUN: unterminated comment or quoted string: $file\n"; + exit 2; +} +my $failed = 0; +my $pattern = $tokens{$ext}; +while ($visible =~ /$pattern/g) { + my $offset = $-[0]; + my $prefix = substr($visible, 0, $offset); + my $line_number = 1 + ($prefix =~ tr/\n/\n/); + print STDERR "FAIL: $file:$line_number: executable proof escape token\n"; + $failed = 1; +} +exit($failed ? 1 : 0); diff --git a/tests/workflows/aspect_scope_test.sh b/tests/workflows/aspect_scope_test.sh index 03d715c..12240b4 100644 --- a/tests/workflows/aspect_scope_test.sh +++ b/tests/workflows/aspect_scope_test.sh @@ -29,6 +29,70 @@ printf '// SPDX-License-Identifier: MPL-2.0\npub fn yes() void {}\n' > "$tmp/sub printf '%s\n' '-- SPDX-License-Identifier: MPL-2.0' '%default total' 'unsafeValue = believe_me 0' > "$tmp/subject/src/Safety.idr" git -C "$tmp/subject" add src/Safety.idr expect 'firing: proof escape in Idris2 source' 1 "$tmp/subject" + +# Regression: nested Idris {- ... -} comments are NOT executable escapes. +printf '%s\n' \ + '-- SPDX-License-Identifier: MPL-2.0' \ + '{- outer {- believe_me -} assert_total -}' \ + 'safeMarker = "--"' > "$tmp/subject/src/Safety.idr" +expect 'silence: nested Idris block comment and quoted marker' 0 "$tmp/subject" + +# Regression: "--" inside a quoted string is NOT a line-comment opener. +printf '%s\n' \ + '-- SPDX-License-Identifier: MPL-2.0' \ + 'broken = let marker = "--" in believe_me 0' > "$tmp/subject/src/Safety.idr" +expect 'firing: escape after quoted comment marker' 1 "$tmp/subject" + +printf '%s\n' \ + '-- SPDX-License-Identifier: MPL-2.0' \ + 'safeMarker = "--"' > "$tmp/subject/src/Safety.idr" + +# Lean and Agda have nested block comments with different delimiters. +printf '%s\n' \ + '-- SPDX-License-Identifier: MPL-2.0' \ + '/- outer /- sorry -/ sorry -/' \ + 'def marker : String := "--"' > "$tmp/subject/src/Proof.lean" +printf '%s\n' \ + '-- SPDX-License-Identifier: MPL-2.0' \ + '{- outer {- believe_me -} -}' > "$tmp/subject/src/Proof.agda" +git -C "$tmp/subject" add src/Proof.lean src/Proof.agda +expect 'silence: nested Lean and Agda comments' 0 "$tmp/subject" +printf '%s\n' \ + '-- SPDX-License-Identifier: MPL-2.0' \ + 'def broken : Nat := let marker := "--"; sorry' > "$tmp/subject/src/Proof.lean" +expect 'firing: Lean sorry after quoted marker' 1 "$tmp/subject" +printf '%s\n' \ + '-- SPDX-License-Identifier: MPL-2.0' \ + 'def marker : String := "--"' > "$tmp/subject/src/Proof.lean" + +# Coq nested comments and string literals are comments/data, not holes. +printf '%s\n' \ + '(* SPDX-License-Identifier: MPL-2.0 *)' \ + '(* outer (* Admitted. *) no actual proof hole *)' \ + 'Definition example : string := "Admitted.".' > "$tmp/subject/verification/proofs/Safety.v" +expect 'silence: nested Coq comments and quoted strings' 0 "$tmp/subject" +printf '%s\n' \ + '(* SPDX-License-Identifier: MPL-2.0 *)' \ + 'Lemma hole : False.' \ + 'Proof. admit; idtac.' > "$tmp/subject/verification/proofs/Safety.v" +expect 'firing: Coq admit tactical' 1 "$tmp/subject" + +# Fail closed on input that cannot be safely lexed. +printf '%s\n' \ + '-- SPDX-License-Identifier: MPL-2.0' \ + '{- unterminated block' > "$tmp/subject/src/Safety.idr" +printf '%s\n' '(* SPDX-License-Identifier: MPL-2.0 *)' 'Lemma okay : True. Proof. exact I. Qed.' > "$tmp/subject/verification/proofs/Safety.v" +expect 'void: malformed proof source is not a passing assessment' 1 "$tmp/subject" + +# Workflow must run on proof-only pull requests AND proof-only pushes. +wf="$(cd "$(dirname "$checker")/.." && pwd)/.github/workflows/e2e.yml" +count="$(grep -Fc " - 'verification/proofs/**'" "$wf" || true)" +if [ "$count" -ne 2 ]; then + echo "FAIL: expected proof-only triggers under push and pull_request; saw $count" >&2 + exit 1 +fi +echo 'PASS: proof-only changes trigger both CI paths' + mkdir -p "$tmp/void" git -C "$tmp/void" init -q expect 'void: no source means no assessment' 2 "$tmp/void" diff --git a/tests/workflows/wiki_format_test.sh b/tests/workflows/wiki_format_test.sh new file mode 100644 index 0000000..32cb786 --- /dev/null +++ b/tests/workflows/wiki_format_test.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# Policy controls: permitted wiki source, forbidden other Markdown, absent root. +set -euo pipefail +checker="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../scripts" && pwd)/check-no-md-in-docs.sh" +tmp="$(mktemp -d)" +trap 'rm -rf -- "$tmp"' EXIT +git -C "$tmp" init -q +mkdir -p "$tmp/docs/wikis" "$tmp/docs/architecture" "$tmp/docs/wiki/sub" +printf '# Wiki\n' > "$tmp/docs/wikis/README.md" +printf '# Wiki page\n' > "$tmp/docs/wiki/sub/page.md" +printf '= Architecture\n' > "$tmp/docs/architecture/README.adoc" +git -C "$tmp" add docs +expect() { + local name="$1" want="$2" got=0 + bash "$checker" "$tmp" > "$tmp/check.log" 2>&1 || got=$? + if [ "$got" -ne "$want" ]; then + echo "FAIL: $name: expected $want, got $got" >&2 + cat "$tmp/check.log" >&2 + exit 1 + fi + echo "PASS: $name (exit $got)" +} +expect 'silence: wiki Markdown and architecture AsciiDoc' 0 +printf '# Wrong format\n' > "$tmp/docs/architecture/NOTES.md" +git -C "$tmp" add docs/architecture/NOTES.md +expect 'firing: non-wiki Markdown' 1 +git -C "$tmp" rm -q -f docs/architecture/NOTES.md +expect 'recovery: removing invalid format clears gate' 0 +if bash "$checker" "$tmp/does-not-exist" >/dev/null 2>&1; then + echo 'FAIL: absent repository counted as conformant' >&2 + exit 1 +fi +echo 'PASS: wiki format silence/firing/unavailable controls'