Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 79 additions & 0 deletions .github/workflows/proven-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -147,3 +147,82 @@ jobs:
set -euo pipefail
export PATH="$HOME/.cargo/bin:$PATH"
STANDARDS_DIR="$RUNNER_TEMP/standards" bash scripts/run-proven-suite.sh --require-zig --bench

# ==========================================================================
# Required status check "CI/gate" (Branch-Floor ruleset).
# The check context is this job's name. It waits for every other job in this
# workflow and runs even when one fails, is cancelled or is skipped; it exits
# 0 only when every required job succeeded (or was skipped by design behind a
# detector job that itself succeeded). Pattern: hyperpolymath/hypatia ci.yml.
# ==========================================================================
ci-gate:
name: CI/gate
runs-on: ubuntu-24.04
timeout-minutes: 5
permissions: {}
needs: [proven-suite]
if: always()
steps:
- name: Require every needed job to have succeeded
env:
NEEDS_JSON: ${{ toJSON(needs) }}
REQUIRED_JOBS: "proven-suite"
ADVISORY_JOBS: ""
DESIGNED_SKIPS: ""
run: |
set -euo pipefail
summary="${GITHUB_STEP_SUMMARY:-/dev/null}"
result_of() { local job="$1"; jq -r --arg j "$job" '.[$j].result // "missing"' <<< "$NEEDS_JSON"; }
in_list() { local needle="$1" haystack="$2"; case " $haystack " in *" $needle "*) return 0 ;; esac; return 1; }
detector_of() {
local job="$1" pair
for pair in $DESIGNED_SKIPS; do
if [ "${pair%%:*}" = "$job" ]; then echo "${pair#*:}"; return 0; fi
done
return 1
}
row() { printf '| %s | %s | %s | %s |\n' "$@" | tee -a "$summary"; }
{
echo "### CI/gate"
echo
echo "| job | result | role | verdict |"
echo "|---|---|---|---|"
} | tee -a "$summary"
fail=0
total=0
passed=0
for j in $REQUIRED_JOBS; do
total=$((total + 1))
r=$(result_of "$j")
case "$r" in
success) v="ok"; passed=$((passed + 1)) ;;
skipped)
if d=$(detector_of "$j") && [ "$(result_of "$d")" = "success" ]; then
v="ok (designed skip; detector $d succeeded)"; passed=$((passed + 1))
else
v="FAIL (skipped without a successful detector)"; fail=1
fi ;;
*) v="FAIL (result must be success)"; fail=1 ;;
esac
row "$j" "$r" required "$v"
done
for j in $ADVISORY_JOBS; do
row "$j" "$(result_of "$j")" advisory "not gating"
done
for j in $(jq -r 'keys[]' <<< "$NEEDS_JSON"); do
if ! in_list "$j" "$REQUIRED_JOBS $ADVISORY_JOBS"; then
row "$j" "$(result_of "$j")" unclassified "FAIL (job is in needs but neither required nor advisory)"
fail=1
fi
done
if [ "$total" -eq 0 ]; then
echo "::error::no required jobs configured"
fail=1
fi
msg="required jobs: $total; satisfied: $passed; advisory: $(wc -w <<< "$ADVISORY_JOBS")"
echo "$msg" | tee -a "$summary"
if [ "$fail" -ne 0 ]; then
echo "::error::CI/gate failed ($msg)"
exit 1
fi
echo "CI/gate passed ($msg)"
1 change: 1 addition & 0 deletions .machine_readable/required-contexts-allow.txt
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,4 @@
# renamed or path-filtered workflow can never silently strand pull requests
# behind an unsatisfiable ruleset check (issue #219 class).
scan / gitleaks => .github/workflows/secret-scanner.yml
CI/gate => .github/workflows/proven-tests.yml
42 changes: 21 additions & 21 deletions scripts/census/proof-holes.sh
Original file line number Diff line number Diff line change
Expand Up @@ -56,37 +56,37 @@ for d in "${EXTRA_EX[@]}"; do EXCLUDE+=(--exclude-dir="$d"); done
# (comment-regex is kept for the rule format; comments are now blanked by
# strip_comments before matching, so it is no longer consulted.)
RULES=$(cat <<'EOF'
agda~*.agda,*.lagda,*.lagda.md~^\s*--~axiom~postulate~^\s*postulate\b
agda~*.agda,*.lagda,*.lagda.md~^\s*--~axiom~postulate~^\s*postulate(?![\w'])
agda~*.agda,*.lagda,*.lagda.md~^\s*--~hole~{! !}~\{!
agda~*.agda,*.lagda,*.lagda.md~^\s*--~unsafe~TERMINATING~\{-#\s*(NON_)?TERMINATING\s*#-\}
agda~*.agda,*.lagda,*.lagda.md~^\s*--~unsafe~NO_POSITIVITY_CHECK~\{-#\s*NO_POSITIVITY_CHECK
agda~*.agda,*.lagda,*.lagda.md~^\s*--~unsafe~NO_UNIVERSE_CHECK~\{-#\s*NO_UNIVERSE_CHECK
agda~*.agda,*.lagda,*.lagda.md~^\s*--~unsafe~--type-in-type~--type-in-type
agda~*.agda,*.lagda,*.lagda.md~^\s*--~axiom~primTrustMe~\bprimTrustMe\b
idris2~*.idr,*.lidr~^\s*(--|\|\|\|)~hole~believe_me~\bbelieve_me\b
idris2~*.idr,*.lidr~^\s*(--|\|\|\|)~hole~assert_total~\bassert_total\b
idris2~*.idr,*.lidr~^\s*(--|\|\|\|)~hole~assert_smaller~\bassert_smaller\b
idris2~*.idr,*.lidr~^\s*(--|\|\|\|)~hole~idris_crash~\bidris_crash\b
agda~*.agda,*.lagda,*.lagda.md~^\s*--~axiom~primTrustMe~(?<![\w'])primTrustMe(?![\w'])
idris2~*.idr,*.lidr~^\s*(--|\|\|\|)~hole~believe_me~(?<![\w'])believe_me(?![\w'])
idris2~*.idr,*.lidr~^\s*(--|\|\|\|)~hole~assert_total~(?<![\w'])assert_total(?![\w'])
idris2~*.idr,*.lidr~^\s*(--|\|\|\|)~hole~assert_smaller~(?<![\w'])assert_smaller(?![\w'])
idris2~*.idr,*.lidr~^\s*(--|\|\|\|)~hole~idris_crash~(?<![\w'])idris_crash(?![\w'])
idris2~*.idr,*.lidr~^\s*(--|\|\|\|)~hole~?hole~(^|[\s(=])\?[a-z_][A-Za-z0-9_']*
idris2~*.idr,*.lidr~^\s*(--|\|\|\|)~unsafe~%default partial~%default\s+partial
idris2~*.idr,*.lidr~^\s*(--|\|\|\|)~unsafe~partial~^\s*partial\b
idris2~*.idr,*.lidr~^\s*(--|\|\|\|)~axiom~unsafePerformIO~\bunsafePerformIO\b
lean~*.lean~^\s*--~hole~sorry~\bsorry\b
lean~*.lean~^\s*--~hole~admit~\badmit\b
lean~*.lean~^\s*--~axiom~axiom~^\s*(private\s+|protected\s+)?axiom\b
lean~*.lean~^\s*--~axiom~native_decide~\bnative_decide\b
idris2~*.idr,*.lidr~^\s*(--|\|\|\|)~unsafe~partial~^\s*partial(?![\w'])
idris2~*.idr,*.lidr~^\s*(--|\|\|\|)~axiom~unsafePerformIO~(?<![\w'])unsafePerformIO(?![\w'])
lean~*.lean~^\s*--~hole~sorry~(?<![\w'])sorry(?![\w'])
lean~*.lean~^\s*--~hole~admit~(?<![\w'])admit(?![\w'])
lean~*.lean~^\s*--~axiom~axiom~^\s*(private\s+|protected\s+)?axiom(?![\w'])
lean~*.lean~^\s*--~axiom~native_decide~(?<![\w'])native_decide(?![\w'])
lean~*.lean~^\s*--~axiom~implemented_by~@\[implemented_by
lean~*.lean~^\s*--~unsafe~unsafe~^\s*(private\s+)?unsafe\b
lean~*.lean~^\s*--~unsafe~unsafe~^\s*(private\s+)?unsafe(?![\w'])
lean~*.lean~^\s*--~policy~import Mathlib~^\s*import\s+Mathlib
rocq~*.v~^\s*\(\*~hole~Admitted~\bAdmitted\.
rocq~*.v~^\s*\(\*~hole~admit~\badmit\.
rocq~*.v~^\s*\(\*~hole~give_up~\bgive_up\b
rocq~*.v~^\s*\(\*~axiom~Axiom~^\s*(Axiom|Axioms|Conjecture|Parameter|Parameters)\b
rocq~*.v~^\s*\(\*~hole~Admitted~(?<![\w'])Admitted\.
rocq~*.v~^\s*\(\*~hole~admit~(?<![\w'])admit\.
rocq~*.v~^\s*\(\*~hole~give_up~(?<![\w'])give_up(?![\w'])
rocq~*.v~^\s*\(\*~axiom~Axiom~^\s*(Axiom|Axioms|Conjecture|Parameter|Parameters)(?![\w'])
rocq~*.v~^\s*\(\*~unsafe~Unset Checking~Unset\s+(Guard|Positivity|Universe)\s+Checking
rocq~*.v~^\s*\(\*~unsafe~bypass_check~#\[\s*bypass_check
isabelle~*.thy~^\s*\(\*~hole~sorry~\bsorry\b
isabelle~*.thy~^\s*\(\*~hole~oops~\boops\b
isabelle~*.thy~^\s*\(\*~axiom~axiomatization~\baxiomatization\b
isabelle~*.thy~^\s*\(\*~hole~sorry~(?<![\w'])sorry(?![\w'])
isabelle~*.thy~^\s*\(\*~hole~oops~(?<![\w'])oops(?![\w'])
isabelle~*.thy~^\s*\(\*~axiom~axiomatization~(?<![\w'])axiomatization(?![\w'])
isabelle~*.thy~^\s*\(\*~unsafe~quick_and_dirty~quick_and_dirty
creusot~*.rs~^\s*//~axiom~#[trusted]~#\[\s*((creusot|creusot_std|creusot_contracts)::(macros::)?)?trusted\s*\]|cfg_attr\([^)]*,\s*((creusot|creusot_std|creusot_contracts)::(macros::)?)?trusted\s*\)
verus~*.rs~^\s*//~hole~admit()~\badmit\s*\(\s*\)
Expand All @@ -98,7 +98,7 @@ spark~*.adb,*.ads~^\s*--~hole~pragma Assume~pragma\s+Assume\b
spark~*.adb,*.ads~^\s*--~unsafe~SPARK_Mode Off~SPARK_Mode\s*(=>\s*)?\(?\s*Off
spark~*.adb,*.ads~^\s*--~unsafe~Skip_Proof~Skip_(Flow_And_)?Proof
spark~*.adb,*.ads~^\s*--~axiom~Annotate False_Positive~Annotate\s*\(\s*GNATprove\s*,\s*(False_Positive|Intentional)
dedukti~*.dk,*.lp~^\s*(//|\(;)~hole~admit~\badmit(ted)?\b
dedukti~*.dk,*.lp~^\s*(//|\(;)~hole~admit~(?<![\w'])admit(ted)?(?![\w'])
EOF
)

Expand Down
28 changes: 28 additions & 0 deletions tests/census/proof_holes_control_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,31 @@ WANT
rm "$F/Real.agda" "$F/Prose.agda"
bash "$SCAN" --fail-on any "$F" >/dev/null || { echo "FAIL: a control alone must not fail --fail-on any"; exit 1; }
echo "PASS: negative-control findings classed control, real and prose-only files still axioms"

# Identifier controls: a keyword followed by a prime is an identifier, not the
# keyword (`'` is not a word character, so a bare \b matched `partial'`). Found
# in hyperpolymath/hypatia tests/accept/ReceiptP5MissingExact.idr, which defines
# a total function `partial'` and was counted as 4 `unsafe` lines.
P=$(mktemp -d); trap 'rm -rf "$F" "$P"' EXIT
printf '%s\n' 'module Ids' "partial' : Body -> Nat" "partial' x = 0" "partial'' : Nat" 'partial_sum : Nat' 'partialApply : Nat' \
"y = believe_me' 1" "z = assert_total' 2" > "$P/Ids.idr"
printf '%s\n' "theorem t' : True := sorry'" "axiom' x : Nat" > "$P/Ids.lean"
printf '%s\n' "Axiom' f : nat." > "$P/Ids.v"
got=$(bash "$SCAN" --fail-on none "$P" | jq -r 'select(.class=="hole" or .class=="unsafe" or (.class=="axiom" and .marker!="undefined declaration")) | "\(.file):\(.line) \(.class) \(.marker)"')
[[ -z "$got" ]] || { echo "FAIL: identifiers with a trailing prime were counted:"; echo "$got"; exit 1; }
# Positive controls: the real keywords still count.
rm "$P"/*
printf '%s\n' 'module Real' 'partial' 'f : Nat -> Nat' ' partial' 'g : Nat -> Nat' 'partial h : Nat' 'y = believe_me 1' > "$P/Real.idr"
printf '%s\n' 'theorem t : True := sorry' 'axiom a : Nat' > "$P/Real.lean"
got=$(bash "$SCAN" --fail-on none "$P" | jq -r 'select(.marker!="undefined declaration") | "\(.file):\(.line) \(.class) \(.marker)"' | sort)
want=$(sort <<'WANT'
Real.idr:2 unsafe partial
Real.idr:4 unsafe partial
Real.idr:6 unsafe partial
Real.idr:7 hole believe_me
Real.lean:1 hole sorry
Real.lean:2 axiom axiom
WANT
)
[[ "$got" == "$want" ]] || { echo "FAIL: real keywords no longer counted"; diff <(printf '%s\n' "$want") <(printf '%s\n' "$got") || true; exit 1; }
echo "PASS: trailing-prime identifiers not counted, real partial/believe_me/sorry/axiom still counted"
Loading