diff --git a/.github/workflows/build-gossamer-gui.yml b/.github/workflows/build-gossamer-gui.yml index 175dd7d6..4d341d9c 100644 --- a/.github/workflows/build-gossamer-gui.yml +++ b/.github/workflows/build-gossamer-gui.yml @@ -254,16 +254,20 @@ jobs: # (which will start accepting bridge.eph once the parser # learns v2) or any of the future subcommand names that have # been floated upstream — first one to start succeeding wins. - if just --list 2>/dev/null | grep -qE '^\s*compile-affine\b'; then + # Captured, then matched: a pipe into `grep -q` can read a match as + # no match under pipefail (SIGPIPE, RE011). + just_list=$(just --list 2>/dev/null || true) + cli_help=$("$cli" --help 2>&1 || true) + if grep -qE '^\s*compile-affine\b' <<<"$just_list"; then echo "→ just compile-affine (concrete .eph path available)" just compile-affine "$src_entry" affine "$wasm_out" - elif "$cli" --help 2>&1 | grep -qE '\bcompile-eph\b'; then + elif grep -qE '\bcompile-eph\b' <<<"$cli_help"; then echo "→ ephapax compile-eph" "$cli" compile-eph "$src_entry" -o "$wasm_out" - elif "$cli" --help 2>&1 | grep -qE '\bcompile-affine\b'; then + elif grep -qE '\bcompile-affine\b' <<<"$cli_help"; then echo "→ ephapax compile-affine" "$cli" compile-affine "$src_entry" -o "$wasm_out" - elif "$cli" --help 2>&1 | grep -qE '^\s*compile\b'; then + elif grep -qE '^\s*compile\b' <<<"$cli_help"; then echo "→ ephapax compile (v1.0.0 subcommand, gated on v2 grammar)" if ! "$cli" compile "$src_entry" -o "$wasm_out" 2>/tmp/ephapax-err; then if grep -q 'Parse error\|expected EOI' /tmp/ephapax-err; then diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7728fd6..a0870bb8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -450,7 +450,7 @@ jobs: test/placeholder_network_test.exs \ test/test_module_names_test.exs 2>&1 | tee "$RUNNER_TEMP/e0-receipt-tests.log" # A skipped test here means that Nickel was not found: the contract was not checked. - if tail -n 3 "$RUNNER_TEMP/e0-receipt-tests.log" | grep -Eq '[0-9]+ skipped'; then + if grep -Eq '[0-9]+ skipped' <<<"$(tail -n 3 "$RUNNER_TEMP/e0-receipt-tests.log")"; then echo "::error::a receipt test was skipped: the Nickel contract was NOT checked" exit 1 fi diff --git a/.github/workflows/dependabot-automerge.yml b/.github/workflows/dependabot-automerge.yml index 22737d27..e7e2070a 100644 --- a/.github/workflows/dependabot-automerge.yml +++ b/.github/workflows/dependabot-automerge.yml @@ -111,7 +111,7 @@ jobs: # Dependabot adds severity labels like "severity: high", # "severity: critical". Look for those in the PR labels JSON. - if echo "$PR_LABELS" | grep -qiE '"(severity: (high|critical))"'; then + if grep -qiE '"(severity: (high|critical))"' <<<"$PR_LABELS"; then is_high_or_critical=true fi diff --git a/.github/workflows/estate-rescan.yml b/.github/workflows/estate-rescan.yml index 936a24c8..b519015f 100644 --- a/.github/workflows/estate-rescan.yml +++ b/.github/workflows/estate-rescan.yml @@ -181,7 +181,7 @@ jobs: while IFS= read -r name; do # Names come from a repository_dispatch payload — enforce a # strict charset before using them anywhere. - if ! printf '%s' "$name" | grep -Eq '^[A-Za-z0-9._-]+$'; then + if ! grep -Eq '^[A-Za-z0-9._-]+$' <<<"$name"; then echo "::warning::Skipping invalid repo name in payload: $name" continue fi diff --git a/.github/workflows/rsr-oracle.yml b/.github/workflows/rsr-oracle.yml index deb83339..d5c1b8b9 100644 --- a/.github/workflows/rsr-oracle.yml +++ b/.github/workflows/rsr-oracle.yml @@ -207,7 +207,7 @@ jobs: refs/heads/*|main|master|HEAD|develop) echo "::error::refusing standards ref '${STANDARDS_REF}': pin a commit SHA or a release tag" exit 1 ;; - *) printf '%s' "$STANDARDS_REF" | grep -Eq '^[0-9a-f]{7,40}$' || { + *) grep -Eq '^[0-9a-f]{7,40}$' <<<"$STANDARDS_REF" || { echo "::error::refusing standards ref '${STANDARDS_REF}': not a commit SHA or release tag" exit 1 } ;; @@ -297,7 +297,7 @@ jobs: case "$ref" in refs/tags/*|v[0-9]*) ok=1 ;; refs/heads/*|main|master|HEAD|develop) ok=0 ;; - *) printf '%s' "$ref" | grep -Eq '^[0-9a-f]{7,40}$' && ok=1 ;; + *) grep -Eq '^[0-9a-f]{7,40}$' <<<"$ref" && ok=1 ;; esac if [ "$repo_is_self" = "1" ]; then ok=1 @@ -306,7 +306,7 @@ jobs: echo "::error::the oracle is pinned to '${ref}' in ${WORKFLOW_REPOSITORY}, which is not an immutable ref. Pin a full commit SHA (or a release tag); '@main' is what let one broken commit break every consumer at once." exit 1 fi - if ! printf '%s' "$pinned" | grep -Eq '^[0-9a-f]{40}$'; then + if ! grep -Eq '^[0-9a-f]{40}$' <<<"$pinned"; then echo "::error::could not resolve the pinned oracle SHA (got '${pinned}'); refusing to compile an unpinned oracle" exit 1 fi @@ -360,7 +360,7 @@ jobs: # Read from the checkout, never from an expression spliced into # this script. commit=$(git -C "$GITHUB_WORKSPACE/repo" rev-parse HEAD 2>/dev/null || true) - if printf '%s' "$commit" | grep -Eq '^[0-9a-f]{40}$'; then + if grep -Eq '^[0-9a-f]{40}$' <<<"$commit"; then args+=(--source-commit "$commit") fi fi diff --git a/hooks/validate-sha-pins.sh b/hooks/validate-sha-pins.sh index 697092b5..18b3d274 100755 --- a/hooks/validate-sha-pins.sh +++ b/hooks/validate-sha-pins.sh @@ -13,7 +13,7 @@ for workflow in .github/workflows/*.yml .github/workflows/*.yaml; do while IFS= read -r line; do if [[ "$line" =~ uses:.*@ ]]; then # Check if it has a SHA (40 hex chars) - if ! echo "$line" | grep -qE '@[a-f0-9]{40}'; then + if ! grep -qE '@[a-f0-9]{40}' <<<"$line"; then echo "ERROR: Unpinned action in $workflow" echo " $line" echo " Actions must use SHA pins: uses: action/name@SHA # version" diff --git a/hooks/validate-spdx.sh b/hooks/validate-spdx.sh index cc81cf18..6ce583f3 100755 --- a/hooks/validate-spdx.sh +++ b/hooks/validate-spdx.sh @@ -11,7 +11,7 @@ for workflow in .github/workflows/*.yml .github/workflows/*.yaml; do [ -f "$workflow" ] || continue first_line=$(head -n1 "$workflow") - if ! echo "$first_line" | grep -qE "$SPDX_PATTERN"; then + if ! grep -qE "$SPDX_PATTERN" <<<"$first_line"; then echo "ERROR: Missing SPDX header in $workflow" echo " First line should be: # SPDX-License-Identifier: MPL-2.0" ERRORS=$((ERRORS + 1)) diff --git a/hypatia-cli-bash.sh b/hypatia-cli-bash.sh index a378331a..d1975ec8 100755 --- a/hypatia-cli-bash.sh +++ b/hypatia-cli-bash.sh @@ -618,7 +618,7 @@ scan_repo_structure() { if [[ -d "$dir/.github/workflows" ]]; then for wf in "$dir/.github/workflows"/*.yml "$dir/.github/workflows"/*.yaml; do [[ -f "$wf" ]] || continue - if ! head -1 "$wf" | grep -q '^# SPDX-License-Identifier:'; then + if ! grep -q '^# SPDX-License-Identifier:' <<<"$(head -1 "$wf")"; then emit_finding "medium" "repo_structure" "missing_spdx_header" \ "$wf" 1 "Workflow missing SPDX header" "CWE-1059" \ "Add '# SPDX-License-Identifier: MPL-2.0' as first line" "true" diff --git a/scripts/auto-fix-formulaic.sh b/scripts/auto-fix-formulaic.sh index 6b40c7dd..64999f2d 100755 --- a/scripts/auto-fix-formulaic.sh +++ b/scripts/auto-fix-formulaic.sh @@ -100,7 +100,7 @@ fix_agpl_references() { for f in $agpl_files; do # Skip mock/fixture files that simulate external API responses - if echo "$f" | grep -qE 'mock|fixture.*github'; then + if grep -qE 'mock|fixture.*github' <<<"$f"; then continue fi warn "AGPL-3.0 reference in: $f (should be MPL-2.0)" diff --git a/scripts/check-readme-claims.sh b/scripts/check-readme-claims.sh index dcf052f0..47a5b104 100755 --- a/scripts/check-readme-claims.sh +++ b/scripts/check-readme-claims.sh @@ -63,8 +63,10 @@ fi # The ids an AsciiDoc file defines: explicit [[id]], [[id,label]], [#id], # anchor:id[], and the default section ids (_title_in_lower_case). anchors() { - { grep -oE '\[\[[A-Za-z0-9_:.-]+|\[#[A-Za-z0-9_:-]+|anchor:[A-Za-z0-9_:.-]+\[' "$1" | sed -E 's/^(\[\[|\[#|anchor:)//; s/\[$//' - sed -nE 's/^=+ +(.*)$/\1/p' "$1" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/_/g; s/_+$//; s/^/_/' + local file=$1 + # grep exits 1 when a file has no explicit ID: no match, not an error. + { { grep -oE '\[\[[A-Za-z0-9_:.-]+|\[#[A-Za-z0-9_:-]+|anchor:[A-Za-z0-9_:.-]+\[' "$file" || [[ $? -eq 1 ]]; } | sed -E 's/^(\[\[|\[#|anchor:)//; s/\[$//' + sed -nE 's/^=+ +(.*)$/\1/p' "$file" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/_/g; s/_+$//; s/^/_/' } | sort -u } @@ -80,7 +82,11 @@ for doc in "$readme" EXPLAINME.adoc docs/README.adoc; do if [ ! -e "$dir/$path" ]; then say "$doc links $target, which does not exist"; continue; fi if [ "$target" != "$path" ] && [[ "$path" == *.adoc ]]; then frag=${target#*#} - anchors "$dir/$path" | grep -qxF -- "$frag" || say "$doc links $target, and $path defines no anchor $frag" + # Captured, then matched (RE011); an unreadable target is said, not skipped. + ids=$(anchors "$dir/$path") || { say "$doc links $target, and the anchors of $path cannot be read"; continue; } + if [[ -z "$ids" ]] || ! grep -qxF -- "$frag" <<<"$ids"; then + say "$doc links $target, and $path defines no anchor $frag" + fi fi done < <(grep -oE 'link:[^[ ]+\[' "$doc" | sed -E 's/^link:(.*)\[$/\1/') done diff --git a/scripts/ci-health/README.adoc b/scripts/ci-health/README.adoc index 3ee21a85..fc2615d5 100644 --- a/scripts/ci-health/README.adoc +++ b/scripts/ci-health/README.adoc @@ -75,6 +75,12 @@ from the conclusion alone. * `remediate.sh ` — applies B-ALLOWLIST (allow-list PUT) and D-BURN (idempotent signed PR). Guardrails: own repos only (skips forks/archived), `CI_HEALTH_DENYLIST`, idempotent, dry-run honoured. + D-BURN computes every patch before it writes, and a file is a target only + when its patch changes it, so the PR and the `FIXED` count name only files + that change. +* `burn-trigger.sh` — the one pattern for the bare trigger, sourced by + `detect.sh` (report) and `remediate.sh` (select and rewrite), so the three + cannot drift apart. `test/ci-health-test.sh` checks that they agree. * `sweep.sh` — estate driver: enumerate → detect → remediate (cap `MAX_BURN_PRS`) → upsert one rolling tracking issue. * `baton-bridge.sh` — the complement to detect/sweep: where those *detect* that diff --git a/scripts/ci-health/burn-trigger.sh b/scripts/ci-health/burn-trigger.sh new file mode 100644 index 00000000..b0465c24 --- /dev/null +++ b/scripts/ci-health/burn-trigger.sh @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: MPL-2.0 +# shellcheck shell=bash +# +# Sourced by detect.sh and remediate.sh: the bare `on: [push, pull_request]` +# trigger that D-BURN names, with an optional trailing comment. detect.sh +# reports a workflow on it, and remediate.sh selects and rewrites the same +# line, so every file the selector picks is one the rewrite changes. +# An ERE for grep -E and for awk (read there through ENVIRON, because +# awk -v would rewrite its backslashes). +# shellcheck disable=SC2034 # used by the scripts that source this file +BURN_TRIGGER_ERE='^on:[[:space:]]*\[[[:space:]]*push[[:space:]]*,[[:space:]]*pull_request[[:space:]]*\][[:space:]]*(#.*)?$' diff --git a/scripts/ci-health/detect.sh b/scripts/ci-health/detect.sh index 6c6d4fc2..4e4f6bb8 100755 --- a/scripts/ci-health/detect.sh +++ b/scripts/ci-health/detect.sh @@ -23,6 +23,8 @@ set -euo pipefail O="${OWNER:-hyperpolymath}"; R="$1" HERE="$(cd "$(dirname "$0")" && pwd)" # for action-superset.txt (allow-list coverage) +# shellcheck source=SCRIPTDIR/burn-trigger.sh +. "$HERE/burn-trigger.sh" # BURN_TRIGGER_ERE (D-BURN) emit(){ printf '%s\t%s\t%s\t%s\n' "$R" "$1" "$2" "$3"; } api() { local out @@ -59,12 +61,12 @@ for fail_id in "${fail_ids[@]}"; do emit B-TRACKER-TOKEN HIGH "Pin updater reports missing write credential (run $fail_id) → compare pins read-only before provisioning repository-scoped write access" tracker_token_seen=true fi - if [ "$billing_seen" = false ] && printf '%s' "$msg" | grep -qiE 'payments have failed|spending limit'; then + if [[ "$billing_seen" = false ]] && grep -qiE 'payments have failed|spending limit' <<<"$msg"; then emit A-BILLING CRITICAL "Actions billing/spending-limit wall blocks all billable jobs → OWNER: GitHub Settings -> Billing & plans" billing_seen=true fi - if [ "$lockfile_seen" = false ] && printf '%s' "$msg" | grep -qiE 'lockfile pin .* does not match ref|lockfile verification did not produce a result|unreachable-pin'; then - first_lock_msg=$(printf '%s\n' "$msg" | grep -iEm1 'lockfile pin .* does not match ref|lockfile verification did not produce a result|unreachable-pin') + if [[ "$lockfile_seen" = false ]] && grep -qiE 'lockfile pin .* does not match ref|lockfile verification did not produce a result|unreachable-pin' <<<"$msg"; then + first_lock_msg=$(grep -iEm1 'lockfile pin .* does not match ref|lockfile verification did not produce a result|unreachable-pin' <<<"$msg") emit B-LOCKFILE HIGH "Actions lockfile verification failed (${first_lock_msg}) → run gh actions-lock --verify, repair, then commit actions.lock" lockfile_seen=true fi @@ -106,8 +108,10 @@ fi # --- D: burn anti-pattern (bare [push, pull_request] double-trigger), via API for path in $(api "repos/$O/$R/contents/.github/workflows" --jq '.[]?|select(.name|test("\\.ya?ml$"))|.path' || true); do - if api "repos/$O/$R/contents/$path" --jq '.content' | base64 -d \ - | grep -qE '^on:[[:space:]]*\[[[:space:]]*push[[:space:]]*,[[:space:]]*pull_request[[:space:]]*\]'; then + # Captured, then matched: piping into `grep -q` under pipefail can read a + # match as no match (SIGPIPE, RE011). A failed fetch or decode is no match. + if wf_text=$(api "repos/$O/$R/contents/$path" --jq '.content' | base64 -d) \ + && grep -qE "$BURN_TRIGGER_ERE" <<<"$wf_text"; then emit D-BURN MEDIUM "ERR-WF-014: $path on bare [push,pull_request] (2x runs/PR) → scope push to default branch + concurrency-cancel" fi done diff --git a/scripts/ci-health/remediate.sh b/scripts/ci-health/remediate.sh index f1e57232..7f133a43 100755 --- a/scripts/ci-health/remediate.sh +++ b/scripts/ci-health/remediate.sh @@ -23,8 +23,41 @@ set -euo pipefail O="${OWNER:-hyperpolymath}"; R="$1"; CLASS="$2"; DRY="${3:-true}" BR="ci/ci-health-auto-remediation" HERE="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=SCRIPTDIR/burn-trigger.sh +. "$HERE/burn-trigger.sh" # BURN_TRIGGER_ERE (D-BURN), shared with detect.sh DENYLIST="${CI_HEALTH_DENYLIST:-007}" # space-separated repo names to never touch +# D-BURN rewrite of one workflow text: the first line on the bare trigger +# becomes push scoped to the default branches plus pull_request, and a +# concurrency-cancel block is added when the file has none. A text with no +# such line comes back unchanged. +burn_patch() { + local text=$1 + local blk="on: + push: + branches: [main, master] + pull_request:" + + if ! grep -q '^[[:space:]]*concurrency:' <<<"$text"; then + blk="$blk + +# Estate guardrail: scope push to default branches (PR fires once, not +# push+PR) and cancel superseded runs. Safe — read-only PR check. +concurrency: + group: \${{ github.workflow }}-\${{ github.ref }} + cancel-in-progress: true" + fi + + printf '%s' "$text" | RE="$BURN_TRIGGER_ERE" awk -v blk="$blk" ' + $0 ~ ENVIRON["RE"] && !done { + print blk + done = 1 + next + } + { print } + ' +} + for d in $DENYLIST; do [ "$R" = "$d" ] && { echo "SKIP $R/$CLASS denylisted"; exit 0; }; done meta=$(gh api "repos/$O/$R" --jq '"\(.fork) \(.archived)"' 2>/dev/null || echo "false false") read -r isfork isarch <<<"$meta" @@ -66,47 +99,28 @@ case "$CLASS" in D-BURN) if gh api "repos/$O/$R/branches/$BR" --jq '.name' >/dev/null 2>&1; then echo "SKIP $R/D-BURN branch-exists"; exit 0; fi def=$(gh api "repos/$O/$R" --jq '.default_branch'); sha=$(gh api "repos/$O/$R/git/ref/heads/$def" --jq '.object.sha') - targets=(); for p in $(gh api "repos/$O/$R/contents/.github/workflows" --jq '.[]?|select(.name|test("\\.ya?ml$"))|.path' 2>/dev/null); do - gh api "repos/$O/$R/contents/$p?ref=$def" --jq '.content' 2>/dev/null | base64 -d 2>/dev/null \ - | grep -qE '^on:[[:space:]]*\[[[:space:]]*push[[:space:]]*,[[:space:]]*pull_request' && targets+=("$p") + # Every patch is computed before anything is written, from the commit the + # branch will start at. A file is a target only when its patch changes it, + # so the PR and the FIXED count name only files that really change. + targets=(); patches=(); shas=() + for p in $(gh api "repos/$O/$R/contents/.github/workflows?ref=$sha" --jq '.[]?|select(.name|test("\\.ya?ml$"))|.path' 2>/dev/null); do + # A failed fetch or decode, or an empty file, is not a target. + cur=$(gh api "repos/$O/$R/contents/$p?ref=$sha" 2>/dev/null) || continue + ccontent=$(jq -r '.content' <<<"$cur" | tr -d '\n' | base64 -d 2>/dev/null) || continue + [[ -n "$ccontent" ]] || continue + patched=$(burn_patch "$ccontent") + [[ "$patched" != "$ccontent" ]] || continue + targets+=("$p"); patches+=("$patched"); shas+=("$(jq -r '.sha' <<<"$cur")") done [ "${#targets[@]}" -eq 0 ] && { echo "SKIP $R/D-BURN no-targets"; exit 0; } if [ "$DRY" = "true" ]; then echo "DRYRUN $R/D-BURN would patch ${#targets[@]} file(s) + open PR"; exit 0; fi gh api -X POST "repos/$O/$R/git/refs" -f ref="refs/heads/$BR" -f sha="$sha" >/dev/null - for p in "${targets[@]}"; do - cur=$(gh api "repos/$O/$R/contents/$p?ref=$BR") - ccontent=$(printf '%s' "$cur" | jq -r '.content' | tr -d '\n' | base64 -d) - csha=$(printf '%s' "$cur" | jq -r '.sha') - if [ -z "$ccontent" ]; then echo "SKIP $R/$p empty-decode (refusing to PUT an empty file)" >&2; continue; fi - - blk="on: - push: - branches: [main, master] - pull_request:" - - if ! printf '%s' "$ccontent" | grep -q '^[[:space:]]*concurrency:'; then - blk="$blk - -# Estate guardrail: scope push to default branches (PR fires once, not -# push+PR) and cancel superseded runs. Safe — read-only PR check. -concurrency: - group: \${{ github.workflow }}-\${{ github.ref }} - cancel-in-progress: true" - fi - - patched=$(printf '%s' "$ccontent" | awk -v blk="$blk" ' - /^on:[ \t]*\[[ \t]*push[ \t]*,[ \t]*pull_request[ \t]*\][ \t]*$/ && !done { - print blk - done = 1 - next - } - { print } - ') - + for i in "${!targets[@]}"; do + p=${targets[$i]} # printf '%s\n' restores the final newline the $( ) captures strip - newc=$(printf '%s\n' "$patched" | base64 | tr -d '\n') + newc=$(printf '%s\n' "${patches[$i]}" | base64 | tr -d '\n') gh api -X PUT "repos/$O/$R/contents/$p" -f message="ci: cut Actions burn in $p (scope push + concurrency-cancel)" \ - -f content="$newc" -f sha="$csha" -f branch="$BR" >/dev/null + -f content="$newc" -f sha="${shas[$i]}" -f branch="$BR" >/dev/null done url=$(gh api "repos/$O/$R/pulls" -X POST -f title="ci: cut Actions burn — scope push triggers + concurrency-cancel" \ -f head="$BR" -f base="$def" -f body="Automated by hypatia ci-health-sweep. Scopes \`push\` to the default branch (kills push+PR double-runs) and adds \`concurrency: cancel-in-progress\` to read-only PR checks. No SPDX/logic changes." --jq '.html_url') diff --git a/scripts/ci-health/sweep.sh b/scripts/ci-health/sweep.sh index b4c227d0..a6ed4f0f 100755 --- a/scripts/ci-health/sweep.sh +++ b/scripts/ci-health/sweep.sh @@ -33,7 +33,7 @@ while IFS=$'\t' read -r repo cls _sev _det; do D-BURN) if [ "$burned" -lt "$MAXPR" ]; then out=$(OWNER="$O" "$HERE/remediate.sh" "$repo" "$cls" "$DRY" || true); echo "$out" - echo "$out" | grep -q '^FIXED' && burned=$((burned+1)) + grep -q '^FIXED' <<<"$out" && burned=$((burned+1)) else echo "CAP $repo/D-BURN max-burn-prs($MAXPR) reached — deferred"; fi ;; esac done < <(sort -u "$findings") diff --git a/scripts/fix-scripts/fix-http-to-https.sh b/scripts/fix-scripts/fix-http-to-https.sh index 411b101b..1e39f783 100755 --- a/scripts/fix-scripts/fix-http-to-https.sh +++ b/scripts/fix-scripts/fix-http-to-https.sh @@ -44,7 +44,9 @@ fix_file() { [[ "$file" == */.zig-cache/* ]] && return 0 # Only process text files - file -b --mime-type "$file" 2>/dev/null | grep -q "text/" || return 0 + # Captured, then matched (RE011); a failed `file` still means "skip". + local mime + mime=$(file -b --mime-type "$file" 2>/dev/null) && grep -q "text/" <<<"$mime" || return 0 # Build sed exclusion pattern local sed_cmd="s|http://|https://|g" diff --git a/scripts/fix-scripts/fix-missing-spdx.sh b/scripts/fix-scripts/fix-missing-spdx.sh index d44a99b4..d5edbf1a 100755 --- a/scripts/fix-scripts/fix-missing-spdx.sh +++ b/scripts/fix-scripts/fix-missing-spdx.sh @@ -67,7 +67,7 @@ fix_file() { local header="${COMMENT_STYLE[$ext]}" # Handle shebang lines — insert after shebang - if head -1 "$file" | grep -q "^#!"; then + if grep -q "^#!" <<<"$(head -1 "$file")"; then sed -i "1a\\${header}" "$file" else sed -i "1i\\${header}" "$file" diff --git a/scripts/fix-scripts/fix-pin-action-sha.sh b/scripts/fix-scripts/fix-pin-action-sha.sh index 928f0bc0..158ec2cd 100755 --- a/scripts/fix-scripts/fix-pin-action-sha.sh +++ b/scripts/fix-scripts/fix-pin-action-sha.sh @@ -90,7 +90,7 @@ for wf in "${WORKFLOWS_DIR}"/*.yml "${WORKFLOWS_DIR}"/*.yaml; do [[ -z "$action_ref" ]] && continue # Already SHA-pinned (40 hex chars after @) — skip - if echo "$action_ref" | grep -qE '@[0-9a-f]{40}'; then + if grep -qE '@[0-9a-f]{40}' <<<"$action_ref"; then continue fi diff --git a/scripts/fix-scripts/fix-pin-rust-deps.sh b/scripts/fix-scripts/fix-pin-rust-deps.sh index ee491849..ec665348 100755 --- a/scripts/fix-scripts/fix-pin-rust-deps.sh +++ b/scripts/fix-scripts/fix-pin-rust-deps.sh @@ -24,10 +24,13 @@ is_binary_crate() { [[ -f "${REPO}/src/main.rs" ]] && return 0 grep -q '^\[\[bin\]\]' "${REPO}/Cargo.toml" 2>/dev/null && return 0 if grep -q '^\[workspace\]' "${REPO}/Cargo.toml" 2>/dev/null; then - find "${REPO}" -name "main.rs" \ + # Captured, then matched: `find | grep -q .` can lose a match to SIGPIPE + # under pipefail (RE011). A failed find is still no match. + local main_rs + main_rs=$(find "${REPO}" -name "main.rs" \ -not -path "*/.git/*" \ - -not -path "*/target/*" \ - | grep -q . && return 0 + -not -path "*/target/*") \ + && grep -q . <<<"$main_rs" && return 0 fi return 1 } diff --git a/scripts/fix-scripts/fix-pmpl-drift.sh b/scripts/fix-scripts/fix-pmpl-drift.sh index d8858baa..42978e8e 100755 --- a/scripts/fix-scripts/fix-pmpl-drift.sh +++ b/scripts/fix-scripts/fix-pmpl-drift.sh @@ -55,13 +55,13 @@ fi LIC_HEAD=$(head -1 "$LICENSE_PATH" | tr -d '\n') -if echo "$LIC_HEAD" | grep -q "MPL-2.0"; then +if grep -q "MPL-2.0" <<<"$LIC_HEAD"; then TARGET="MPL-2.0" CATEGORY="sole-owner" -elif echo "$LIC_HEAD" | grep -q "AGPL-3.0"; then +elif grep -q "AGPL-3.0" <<<"$LIC_HEAD"; then TARGET="AGPL-3.0-or-later" CATEGORY="son-shared" -elif echo "$LIC_HEAD" | grep -q "PMPL-1.0-or-later"; then +elif grep -q "PMPL-1.0-or-later" <<<"$LIC_HEAD"; then echo "INFO: $REPO_NAME LICENSE is PMPL-1.0-or-later (carve-out or staging) — skipping" exit 0 else diff --git a/scripts/fix-scripts/fix-sast-workflow.sh b/scripts/fix-scripts/fix-sast-workflow.sh index 3ce48e5f..4ffe05cb 100755 --- a/scripts/fix-scripts/fix-sast-workflow.sh +++ b/scripts/fix-scripts/fix-sast-workflow.sh @@ -22,11 +22,14 @@ mkdir -p "${REPO}/.github/workflows" # Detect languages for CodeQL matrix languages=() # JavaScript/TypeScript (includes ReScript compiled output) -if find "$REPO" -maxdepth 3 -name "*.js" -o -name "*.mjs" -o -name "*.res" 2>/dev/null | grep -q .; then +# Captured, then matched: `find | grep -q .` can lose a match to SIGPIPE under +# pipefail (RE011). A failed find is still no match. +if found=$(find "$REPO" -maxdepth 3 -name "*.js" -o -name "*.mjs" -o -name "*.res" 2>/dev/null) \ + && grep -q . <<<"$found"; then languages+=("javascript-typescript") fi # Ruby -if find "$REPO" -maxdepth 3 -name "*.rb" 2>/dev/null | grep -q .; then +if found=$(find "$REPO" -maxdepth 3 -name "*.rb" 2>/dev/null) && grep -q . <<<"$found"; then languages+=("ruby") fi diff --git a/scripts/fix-scripts/fix-scorecard-binary-artifacts.sh b/scripts/fix-scripts/fix-scorecard-binary-artifacts.sh index 2df0ddbf..f2a355be 100755 --- a/scripts/fix-scripts/fix-scorecard-binary-artifacts.sh +++ b/scripts/fix-scripts/fix-scorecard-binary-artifacts.sh @@ -23,7 +23,8 @@ count=0 echo "" while IFS= read -r -d '' file; do - if file "$file" | grep -qE 'ELF|PE32|Mach-O|compiled|bytecode|binary|data'; then + # Captured, then matched (RE011); a failed `file` is still no match. + if desc=$(file "$file") && grep -qE 'ELF|PE32|Mach-O|compiled|bytecode|binary|data' <<<"$desc"; then echo "BINARY: ${file#"${REPO}/"}" count=$((count + 1)) fi diff --git a/scripts/fix-scripts/fix-scorecard-ci-tests.sh b/scripts/fix-scripts/fix-scorecard-ci-tests.sh index 2fe00061..ce2f4b0c 100755 --- a/scripts/fix-scripts/fix-scorecard-ci-tests.sh +++ b/scripts/fix-scripts/fix-scorecard-ci-tests.sh @@ -14,7 +14,10 @@ REPO="${1:?Usage: fix-scorecard-ci-tests.sh }" WF_DIR="${REPO}/.github/workflows" # Already has CI if any workflow runs tests -if ls "${WF_DIR}"/*.yml "${WF_DIR}"/*.yaml 2>/dev/null | xargs grep -l 'mix test\|cargo test\|deno test\|gleam test' 2>/dev/null | grep -q .; then +# Captured, then matched (RE011). The capture keeps the writers' pipefail +# status, so a failed ls or xargs is still "no test workflow", as before. +if test_wfs=$(ls "${WF_DIR}"/*.yml "${WF_DIR}"/*.yaml 2>/dev/null | xargs grep -l 'mix test\|cargo test\|deno test\|gleam test' 2>/dev/null) \ + && grep -q . <<<"$test_wfs"; then echo "[fix-scorecard-ci-tests] CI test workflow already present" exit 0 fi diff --git a/scripts/gen-rules-praxis.sh b/scripts/gen-rules-praxis.sh index 6f00106c..1cacff3d 100755 --- a/scripts/gen-rules-praxis.sh +++ b/scripts/gen-rules-praxis.sh @@ -367,7 +367,7 @@ while IFS='~' read -r mod name f lit sev adv sf slit text; do grep -qF -- "$lit" "$f" || { echo "gen-rules-praxis: $f no longer has '$lit' ($mod/$name)" >&2; exit 1; } if [ "$sf" = "$f" ]; then # same file: beside the literal (a trailing #any: anywhere in it) n=$(grep -nF -- "$lit" "$f" | sed -n 1p | cut -d: -f1) - sed -n "$((n - 3)),$((n + 3))p" "$f" | grep -qF -- "$slit" || + grep -qF -- "$slit" <<<"$(sed -n "$((n - 3)),$((n + 3))p" "$f")" || { echo "gen-rules-praxis: $f no longer states '$slit' beside '$lit' ($mod/$name)" >&2; exit 1; } else sf=${sf%#any} diff --git a/scripts/security-check.sh b/scripts/security-check.sh index 9b334237..7892ca38 100755 --- a/scripts/security-check.sh +++ b/scripts/security-check.sh @@ -504,7 +504,7 @@ check_spdx_headers() { # Check Rust files echo "## Rust Files (.rs)" >> "$report_file" while IFS= read -r -d '' f; do - if ! head -10 "$f" | grep -q "SPDX-License-Identifier"; then + if ! grep -q "SPDX-License-Identifier" <<<"$(head -10 "$f")"; then echo " - $f" >> "$report_file" missing_count=$((missing_count + 1)) fi @@ -514,7 +514,7 @@ check_spdx_headers() { echo "" >> "$report_file" echo "## Haskell Files (.hs)" >> "$report_file" while IFS= read -r -d '' f; do - if ! head -10 "$f" | grep -q "SPDX-License-Identifier"; then + if ! grep -q "SPDX-License-Identifier" <<<"$(head -10 "$f")"; then echo " - $f" >> "$report_file" missing_count=$((missing_count + 1)) fi @@ -524,7 +524,7 @@ check_spdx_headers() { echo "" >> "$report_file" echo "## Shell Scripts (.sh)" >> "$report_file" while IFS= read -r -d '' f; do - if ! head -10 "$f" | grep -q "SPDX-License-Identifier"; then + if ! grep -q "SPDX-License-Identifier" <<<"$(head -10 "$f")"; then echo " - $f" >> "$report_file" missing_count=$((missing_count + 1)) fi @@ -534,7 +534,7 @@ check_spdx_headers() { echo "" >> "$report_file" echo "## GitHub Workflows (.yml)" >> "$report_file" while IFS= read -r -d '' f; do - if ! head -1 "$f" | grep -q "SPDX-License-Identifier"; then + if ! grep -q "SPDX-License-Identifier" <<<"$(head -1 "$f")"; then echo " - $f" >> "$report_file" missing_count=$((missing_count + 1)) fi diff --git a/scripts/standards-integration/check-standards-integration.sh b/scripts/standards-integration/check-standards-integration.sh index cc445b4d..9009901c 100755 --- a/scripts/standards-integration/check-standards-integration.sh +++ b/scripts/standards-integration/check-standards-integration.sh @@ -74,7 +74,7 @@ for f in "${CALLERS[@]}"; do [ -n "$uses_lines" ] || fail "$f declares no metadatastician/standards reusable" while IFS= read -r line; do ref=$(printf '%s' "$line" | sed -E 's/.*uses:[[:space:]]*//; s/[[:space:]].*$//') - if ! printf '%s' "$ref" | grep -qE '^metadatastician/standards/\.github/workflows/[A-Za-z0-9._-]+\.yml@[0-9a-f]{40}$'; then + if ! grep -qE '^metadatastician/standards/\.github/workflows/[A-Za-z0-9._-]+\.yml@[0-9a-f]{40}$' <<<"$ref"; then fail "$f has a non-immutable standards ref (must be a full 40-hex SHA): $ref" fi pins+=("${ref##*@}") @@ -97,7 +97,7 @@ for f in "${CALLERS[@]}"; do # Every pin used by this caller (normally exactly one) must be locked. while IFS= read -r line; do ref=$(printf '%s' "$line" | sed -E 's/.*uses:[[:space:]]*//; s/[[:space:]].*$//') - printf '%s\n' "$block" | grep -qF "metadatastician/standards@${ref##*@}" \ + grep -qF "metadatastician/standards@${ref##*@}" <<<"$block" \ || fail "actions.lock workflows entry for $f does not record ${ref}" done < # (`denylisted_pair`): `some/action@v4.38.1` is no finding. line_has_denied_token() { local tok t - if [ -n "$SHA_ALTERNATION" ] && printf '%s\n' "$1" | grep -qiE "(^|[^0-9A-Za-z])(${SHA_ALTERNATION})([^0-9A-Za-z]|\$)"; then + if [ -n "$SHA_ALTERNATION" ] && grep -qiE "(^|[^0-9A-Za-z])(${SHA_ALTERNATION})([^0-9A-Za-z]|\$)" <<<"$1"; then return 0 fi [ ${#DENY_VERSIONS[@]} -gt 0 ] || return 1 @@ -399,7 +399,7 @@ relabel_comment() { local trimmed="${body#"$lead"}" # A bare claim (no `#`, no leading space) normalises to `# vX`, as relabel/2 does. [ "$body" = "$comment" ] && [ -z "$lead" ] && lead=" " - if printf '%s' "$trimmed" | grep -qE '^v?[0-9]+(\.[0-9]+)*([[:space:]]|$)'; then + if grep -qE '^v?[0-9]+(\.[0-9]+)*([[:space:]]|$)' <<<"$trimmed"; then printf '%s%s' "#${lead}" \ "$(printf '%s' "$trimmed" | sed -E "0,/^v?[0-9]+(\.[0-9]+)*/s//v${version}/")" else diff --git a/scripts/sweeps/estate-pr-automerge.sh b/scripts/sweeps/estate-pr-automerge.sh index 078f64a2..14fd8768 100755 --- a/scripts/sweeps/estate-pr-automerge.sh +++ b/scripts/sweeps/estate-pr-automerge.sh @@ -225,6 +225,7 @@ decide_pr() { local pr="$1" local -a deltas=() local org repo num title author draft mstate head_sha created base_ref repo_archived body + local patch_filenames org=$(jq -r '.org' <<<"$pr") repo=$(jq -r '.repo' <<<"$pr") num=$(jq -r '.number' <<<"$pr") @@ -296,7 +297,7 @@ decide_pr() { only_uses=1 while IFS= read -r cl; do [ -n "$cl" ] || continue - printf '%s' "$cl" | grep -qE '^[-+]\s*-?\s*uses:' || only_uses=0 + grep -qE '^[-+]\s*-?\s*uses:' <<<"$cl" || only_uses=0 done <<<"$content_lines" [ "$only_uses" -eq 1 ] && [ -n "$content_lines" ] && pin_files=$((pin_files + 1)) [ "$only_uses" -eq 0 ] && nonpin_change=$((nonpin_change + 1)) @@ -425,8 +426,8 @@ decide_pr() { blocked_by="pin_delta_unresolvable" route="rhodibot"; pool="P2" elif [ "$pin_files" -eq 0 ] && [ "$lock_only" -eq 0 ] && - jq -r '.filename' <<<"$patches" 2>/dev/null \ - | grep -qE '(^|/)(package\.json|Cargo\.toml|Project\.toml|mix\.exs|go\.mod|pyproject\.toml|composer\.json|Gemfile|requirements[^/]*\.txt)$'; then + patch_filenames=$(jq -r '.filename' <<<"$patches" 2>/dev/null) && + grep -qE '(^|/)(package\.json|Cargo\.toml|Project\.toml|mix\.exs|go\.mod|pyproject\.toml|composer\.json|Gemfile|requirements[^/]*\.txt)$' <<<"$patch_filenames"; then change_class="bump"; disposition="flag"; safety="flag" blocked_by="dependency_manifest_not_lockfile" route="rhodibot"; pool="P2" diff --git a/scripts/sweeps/patch_phase2.sh b/scripts/sweeps/patch_phase2.sh index cd89fc3b..122016b8 100755 --- a/scripts/sweeps/patch_phase2.sh +++ b/scripts/sweeps/patch_phase2.sh @@ -40,7 +40,7 @@ END=$(awk -v s="$START" 'NR>s && /'"$NEXT_RE"'/ {print NR; exit}' "$WF" || true) [ -n "${END:-}" ] || END=$((TOTAL + 1)) # Idempotency: bail if the existing block already carries the fix. -if sed -n "${START},$((END - 1))p" "$WF" | grep -q '^ continue-on-error: true$'; then +if grep -q '^ continue-on-error: true$' <<<"$(sed -n "${START},$((END - 1))p" "$WF")"; then exit 2 fi diff --git a/test/ci-health-test.sh b/test/ci-health-test.sh index 4417dc34..d103f8f7 100755 --- a/test/ci-health-test.sh +++ b/test/ci-health-test.sh @@ -13,16 +13,16 @@ run_detect() { } lock_output=$(run_detect lockfile) -printf '%s\n' "$lock_output" | grep -q $'test-repo\tB-LOCKFILE\tHIGH\t' -printf '%s\n' "$lock_output" | grep -q '1 startup_failure conclusion(s) among the latest 3 run(s)' -if printf '%s\n' "$lock_output" | grep -q 'populate allow-list'; then +grep -q $'test-repo\tB-LOCKFILE\tHIGH\t' <<<"$lock_output" +grep -q '1 startup_failure conclusion(s) among the latest 3 run(s)' <<<"$lock_output" +if grep -q 'populate allow-list' <<<"$lock_output"; then echo "FAIL: unclassified startup failure prescribed an allow-list mutation" >&2 exit 1 fi saturated_output=$(run_detect saturated) -printf '%s\n' "$saturated_output" | grep -q '30 startup_failure conclusion(s) among the latest 30 run(s); page saturated at 30, older runs not measured' -if printf '%s\n' "$saturated_output" | grep -q $'\tB-LOCKFILE\t'; then +grep -q '30 startup_failure conclusion(s) among the latest 30 run(s); page saturated at 30, older runs not measured' <<<"$saturated_output" +if grep -q $'\tB-LOCKFILE\t' <<<"$saturated_output"; then echo "FAIL: saturated scenario was incorrectly classified as a lockfile failure" >&2 exit 1 fi @@ -67,20 +67,23 @@ for fixture in "$ROOT"/test/fixtures/ci-health/allowlist/*.json; do if [[ "$added" -eq 0 ]]; then [[ ! -s "$STATE_DIR/puts.log" ]] || { echo "FAIL [$name]: wrote although nothing was missing" >&2; exit 1; } - printf '%s\n' "$out" | grep -q '^SKIP ' || { echo "FAIL [$name]: expected SKIP, got: $out" >&2; exit 1; } + grep -q '^SKIP ' <<<"$out" || { echo "FAIL [$name]: expected SKIP, got: $out" >&2; exit 1; } else - printf '%s\n' "$out" | grep -q "^FIXED .* added $added, kept all $(jq '.expect.kept' "$fixture")" || + grep -q "^FIXED .* added $added, kept all $(jq '.expect.kept' "$fixture")" <<<"$out" || { echo "FAIL [$name]: unexpected report: $out" >&2; exit 1; } fi # Idempotent: a second run finds nothing missing and writes nothing. : > "$STATE_DIR/puts.log" - printf '%s\n' "$(remediate)" | grep -q '^SKIP ' || { echo "FAIL [$name]: second run was not a no-op" >&2; exit 1; } + grep -q '^SKIP ' <<<"$(remediate)" || { echo "FAIL [$name]: second run was not a no-op" >&2; exit 1; } [[ ! -s "$STATE_DIR/puts.log" ]] || { echo "FAIL [$name]: second run wrote" >&2; exit 1; } # Converges with detect.sh: B-ALLOWLIST no longer fires. - if PATH="$MOCK_BIN:$PATH" OWNER=test-owner MOCK_STATE="$STATE_DIR/state.json" MOCK_SCENARIO=converge \ - "$ROOT/scripts/ci-health/detect.sh" test-repo 2>/dev/null | grep -q $'\tB-ALLOWLIST\t'; then + # Captured, then matched (RE011). A failed detect.sh proves nothing, so it fails the test. + detect_out=$(PATH="$MOCK_BIN:$PATH" OWNER=test-owner MOCK_STATE="$STATE_DIR/state.json" MOCK_SCENARIO=converge \ + "$ROOT/scripts/ci-health/detect.sh" test-repo 2>/dev/null) || + { echo "FAIL [$name]: detect.sh exited $? after remediation" >&2; exit 1; } + if grep -q $'\tB-ALLOWLIST\t' <<<"$detect_out"; then echo "FAIL [$name]: detect.sh still reports B-ALLOWLIST after remediation" >&2; exit 1 fi @@ -88,7 +91,9 @@ for fixture in "$ROOT"/test/fixtures/ci-health/allowlist/*.json; do want="-- corpus: $name added=$added kept=$(jq '.expect.kept' "$fixture") result=$(jq '.expect.result_size' "$fixture")" grep -qxF -- "$want" "$model" || { echo "FAIL [$name]: model has no line '$want'" >&2; exit 1; } eqn="= ($added, $(jq '.expect.kept' "$fixture"), $(jq '.expect.result_size' "$fixture"))" - grep -A1 -xF -- "$want" "$model" | tail -n1 | grep -qF -- "$eqn" || + block=$(grep -A1 -xF -- "$want" "$model") || + { echo "FAIL [$name]: cannot read the model block under '$want'" >&2; exit 1; } + grep -qF -- "$eqn" <<<"${block##*$'\n'}" || { echo "FAIL [$name]: the equation under '$want' does not state $eqn" >&2; exit 1; } done [[ "$(grep -c '^-- corpus: ' "$model")" -eq "$(find "$ROOT/test/fixtures/ci-health/allowlist" -name '*.json' | wc -l)" ]] || @@ -108,4 +113,61 @@ if remediate remediate-mismatch >/dev/null 2>&1; then echo "FAIL: allow-list remediation accepted a mismatched read-back" >&2; exit 1 fi +# --- D-BURN: detect.sh reports, and remediate.sh selects and rewrites, the same +# files (one pattern, scripts/ci-health/burn-trigger.sh). A file the rewrite +# would leave unchanged is never a target, never written and never counted. +burn() { # [dry_run] + local dir=$1 tool=$2 + if [[ "$tool" = detect ]]; then + PATH="$MOCK_BIN:$PATH" OWNER=test-owner MOCK_SCENARIO=burn MOCK_BURN_DIR="$dir" \ + MOCK_WRITE_LOG="$STATE_DIR/writes.log" "$ROOT/scripts/ci-health/detect.sh" test-repo 2>/dev/null + else + PATH="$MOCK_BIN:$PATH" OWNER=test-owner MOCK_BURN_DIR="$dir" MOCK_WRITE_LOG="$STATE_DIR/writes.log" \ + "$ROOT/scripts/ci-health/remediate.sh" test-repo D-BURN "${3:-false}" + fi +} +mixed="$ROOT/test/fixtures/ci-health/burn/mixed" +: > "$STATE_DIR/writes.log" +burn_detect=$(burn "$mixed" detect) || { echo "FAIL [D-BURN]: detect.sh failed" >&2; exit 1; } +reported=$(grep $'\tD-BURN\t' <<<"$burn_detect" | grep -oE '\.github/workflows/[^ ]+' | sort | paste -sd' ') || true +[[ "$reported" = ".github/workflows/commented.yml .github/workflows/has-concurrency.yml" ]] || + { echo "FAIL [D-BURN]: detect.sh reported '$reported'" >&2; exit 1; } + +dry=$(burn "$mixed" remediate true) +[[ "$dry" = "DRYRUN test-repo/D-BURN would patch 2 file(s) + open PR" ]] || + { echo "FAIL [D-BURN]: dry run said '$dry'" >&2; exit 1; } +[[ ! -s "$STATE_DIR/writes.log" ]] || { echo "FAIL [D-BURN]: the dry run wrote" >&2; exit 1; } + +live=$(burn "$mixed" remediate) +[[ "$live" = "FIXED test-repo/D-BURN -> https://github.com/test-owner/test-repo/pull/1 (2 file(s))" ]] || + { echo "FAIL [D-BURN]: live run said '$live'" >&2; exit 1; } +put_paths=$(grep '^PUT ' "$STATE_DIR/writes.log" | cut -d' ' -f2 | sort | paste -sd' ') +[[ "$put_paths" = "repos/test-owner/test-repo/contents/.github/workflows/commented.yml repos/test-owner/test-repo/contents/.github/workflows/has-concurrency.yml" ]] || + { echo "FAIL [D-BURN]: wrote '$put_paths'" >&2; exit 1; } +grep -qx 'POST repos/test-owner/test-repo/git/refs | ref=refs/heads/ci/ci-health-auto-remediation | sha=cafe0001' "$STATE_DIR/writes.log" || + { echo "FAIL [D-BURN]: the branch was not cut from the commit the patches were read at" >&2; exit 1; } +put_content() { # the decoded content written to + local file=$1 line + line=$(grep "^PUT repos/test-owner/test-repo/contents/.github/workflows/$file " "$STATE_DIR/writes.log") || + { echo "FAIL [D-BURN]: no PUT for $file" >&2; exit 1; } + jq -r . <<<"$(sed -E 's/.* \| content=(".*") \| sha=.*/\1/' <<<"$line")" +} +for f in commented.yml has-concurrency.yml; do + written=$(put_content "$f") + if grep -qE "$(sed -n "s/^BURN_TRIGGER_ERE='\(.*\)'$/\1/p" "$ROOT/scripts/ci-health/burn-trigger.sh")" <<<"$written"; then + echo "FAIL [D-BURN]: $f still has the bare trigger after the rewrite" >&2; exit 1 + fi + [[ "$(grep -c '^concurrency:' <<<"$written")" -eq 1 ]] || + { echo "FAIL [D-BURN]: $f does not have exactly one concurrency block" >&2; exit 1; } + grep -qx ' branches: \[main, master\]' <<<"$written" || + { echo "FAIL [D-BURN]: $f push is not scoped to the default branches" >&2; exit 1; } +done +grep -q 'sha=sha-commented.yml | branch=ci/ci-health-auto-remediation' "$STATE_DIR/writes.log" || + { echo "FAIL [D-BURN]: the PUT did not carry the blob sha it was computed from" >&2; exit 1; } + +: > "$STATE_DIR/writes.log" +none=$(burn "$ROOT/test/fixtures/ci-health/burn/none" remediate) +[[ "$none" = "SKIP test-repo/D-BURN no-targets" ]] || { echo "FAIL [D-BURN]: no-target run said '$none'" >&2; exit 1; } +[[ ! -s "$STATE_DIR/writes.log" ]] || { echo "FAIL [D-BURN]: a run with no targets wrote" >&2; exit 1; } + echo "PASS: CI-health classification and remediation fixtures" diff --git a/test/fixtures/ci-health/burn/mixed/commented.yml b/test/fixtures/ci-health/burn/mixed/commented.yml new file mode 100644 index 00000000..4d0f986e --- /dev/null +++ b/test/fixtures/ci-health/burn/mixed/commented.yml @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: MPL-2.0 +name: commented +on: [push, pull_request] # CI +permissions: read-all +jobs: + t: + runs-on: ubuntu-latest + steps: + - run: "true" diff --git a/test/fixtures/ci-health/burn/mixed/has-concurrency.yml b/test/fixtures/ci-health/burn/mixed/has-concurrency.yml new file mode 100644 index 00000000..d2f6bc7e --- /dev/null +++ b/test/fixtures/ci-health/burn/mixed/has-concurrency.yml @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: MPL-2.0 +name: has-concurrency +on: [push, pull_request] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +permissions: read-all +jobs: + t: + runs-on: ubuntu-latest + steps: + - run: "true" diff --git a/test/fixtures/ci-health/burn/mixed/notes.txt b/test/fixtures/ci-health/burn/mixed/notes.txt new file mode 100644 index 00000000..06b66102 --- /dev/null +++ b/test/fixtures/ci-health/burn/mixed/notes.txt @@ -0,0 +1,2 @@ +Not a workflow: the selector lists only .yml and .yaml files. +on: [push, pull_request] diff --git a/test/fixtures/ci-health/burn/mixed/three-events.yml b/test/fixtures/ci-health/burn/mixed/three-events.yml new file mode 100644 index 00000000..3eece67c --- /dev/null +++ b/test/fixtures/ci-health/burn/mixed/three-events.yml @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: MPL-2.0 +name: three-events +on: [push, pull_request, workflow_dispatch] +permissions: read-all +jobs: + t: + runs-on: ubuntu-latest + steps: + - run: "true" diff --git a/test/fixtures/ci-health/burn/none/push-only.yml b/test/fixtures/ci-health/burn/none/push-only.yml new file mode 100644 index 00000000..2d5c917b --- /dev/null +++ b/test/fixtures/ci-health/burn/none/push-only.yml @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: MPL-2.0 +name: push-only +on: push +permissions: read-all +jobs: + t: + runs-on: ubuntu-latest + steps: + - run: "true" diff --git a/test/fixtures/ci-health/burn/none/three-events.yml b/test/fixtures/ci-health/burn/none/three-events.yml new file mode 100644 index 00000000..3eece67c --- /dev/null +++ b/test/fixtures/ci-health/burn/none/three-events.yml @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: MPL-2.0 +name: three-events +on: [push, pull_request, workflow_dispatch] +permissions: read-all +jobs: + t: + runs-on: ubuntu-latest + steps: + - run: "true" diff --git a/test/fixtures/ci-health/mock-gh b/test/fixtures/ci-health/mock-gh index 6c47e4ab..7260fc83 100755 --- a/test/fixtures/ci-health/mock-gh +++ b/test/fixtures/ci-health/mock-gh @@ -16,7 +16,51 @@ for i in "${!args[@]}"; do [ "${args[$i]}" = "--jq" ] && jq_filter="${args[$((i + 1))]}" done -if printf ' %s ' "$*" | grep -q ' -X PUT '; then +# D-BURN: MOCK_BURN_DIR holds the repo's workflow files on its default branch +# (main, at commit cafe0001). Writes go to MOCK_WRITE_LOG, one line per call: +# the method, the endpoint, then each -f field, a PUT's content decoded. +if [ -n "${MOCK_BURN_DIR:-}" ]; then + method=GET; target=""; fields=() + for i in "${!args[@]}"; do + case "${args[$i]}" in + -X) method=${args[$((i + 1))]} ;; + -f) fields+=("${args[$((i + 1))]}") ;; + repos/*) [ -z "$target" ] && target=${args[$i]} ;; + esac + done + [ -z "$target" ] && [[ "$endpoint" == repos/* ]] && target=$endpoint + base=repos/test-owner/test-repo + if [ "$method" != GET ]; then + line="$method $target" + for kv in "${fields[@]}"; do + [[ "$kv" == content=* ]] && kv="content=$(base64 -d <<<"${kv#content=}" | jq -Rsc .)" + line="$line | $kv" + done + printf '%s\n' "$line" >> "$MOCK_WRITE_LOG" + [[ "$target" == "$base/pulls" ]] && printf 'https://github.com/test-owner/test-repo/pull/1\n' + exit 0 + fi + case "${target%%\?*}" in + "$base") + if [ "$jq_filter" = ".default_branch" ]; then printf 'main\n'; else printf 'false false\n'; fi + exit 0 ;; + "$base/branches/ci/ci-health-auto-remediation") echo "HTTP 404" >&2; exit 1 ;; + "$base/git/ref/heads/main") printf 'cafe0001\n'; exit 0 ;; + "$base/contents/.github/workflows") + list=$(find "$MOCK_BURN_DIR" -type f -printf '%f\n' | sort | + jq -Rsc 'split("\n") | map(select(length > 0) | {name: ., path: (".github/workflows/" + .)})') + jq -r "${jq_filter:-.}" <<<"$list" + exit 0 ;; + "$base/contents/.github/workflows/"*) + p=${target%%\?*}; f="$MOCK_BURN_DIR/${p##*/}" + [ -f "$f" ] || { echo "HTTP 404" >&2; exit 1; } + doc=$(jq -nc --arg c "$(base64 < "$f")" --arg s "sha-${p##*/}" '{content: $c, sha: $s}') + jq -r "${jq_filter:-.}" <<<"$doc" + exit 0 ;; + esac +fi + +if grep -q ' -X PUT ' <<<" $* "; then body=$(cat) if [ -n "${MOCK_STATE:-}" ] && [ "${MOCK_SCENARIO:-}" != "remediate-mismatch" ]; then printf '%s\n' "$body" > "$MOCK_STATE" diff --git a/test/standards-integration-test.sh b/test/standards-integration-test.sh index 4d914573..77f711d2 100755 --- a/test/standards-integration-test.sh +++ b/test/standards-integration-test.sh @@ -17,7 +17,7 @@ expect_outcome() { # name expected_exit expected_outcome root extra... local name="$1" want_rc="$2" want_outcome="$3"; shift 3 local out rc=0 out=$("$CHECK" "$@" 2>&1) || rc=$? - if [ "$rc" != "$want_rc" ] || ! printf '%s' "$out" | grep -q "outcome=$want_outcome"; then + if [ "$rc" != "$want_rc" ] || ! grep -q "outcome=$want_outcome" <<<"$out"; then echo "FAIL($name): want rc=$want_rc outcome=$want_outcome, got rc=$rc: $out" >&2 exit 1 fi @@ -91,11 +91,11 @@ for t in bash grep sed awk sort head env; do done rc=0 out=$(PATH="$TMP/nogh" "$TMP/nogh/bash" "$CHECK" --root "$ROOT" --verify-remote 2>&1) || rc=$? -if [ "$rc" != "3" ] || ! printf '%s' "$out" | grep -q 'outcome=unavailable'; then +if [ "$rc" != "3" ] || ! grep -q 'outcome=unavailable' <<<"$out"; then echo "FAIL(unavailable-tool): want rc=3 outcome=unavailable, got rc=$rc: $out" >&2 exit 1 fi -if printf '%s' "$out" | grep -q 'outcome=pass'; then +if grep -q 'outcome=pass' <<<"$out"; then echo "FAIL(unavailable-tool): unavailable was reported as pass" >&2 exit 1 fi @@ -109,7 +109,7 @@ for t in bash grep sed awk head env; do done rc=0 out=$(PATH="$TMP/nosort" "$TMP/nosort/bash" "$CHECK" --root "$ROOT" 2>&1) || rc=$? -if [ "$rc" != "3" ] || ! printf '%s' "$out" | grep -q 'outcome=unavailable'; then +if [ "$rc" != "3" ] || ! grep -q 'outcome=unavailable' <<<"$out"; then echo "FAIL(unavailable-sort): want rc=3 outcome=unavailable, got rc=$rc: $out" >&2 exit 1 fi diff --git a/tests/e2e.sh b/tests/e2e.sh index 6c4d59f9..7e431549 100644 --- a/tests/e2e.sh +++ b/tests/e2e.sh @@ -233,7 +233,9 @@ fi # Check all documented rules have corresponding modules for rule in root_hygiene honest_completion workflow_audit cicd_rules code_safety; do - if find "$PROJECT_DIR/lib" -name "*${rule}*" -o -name "*$(echo $rule | tr '_' '-')*" 2>/dev/null | grep -q .; then + # Captured, then matched (RE011); a failed find is still no match. + if found=$(find "$PROJECT_DIR/lib" -name "*${rule}*" -o -name "*$(echo $rule | tr '_' '-')*" 2>/dev/null) \ + && grep -q . <<<"$found"; then pass "Rule module: $rule" else skip_test "Rule module: $rule" "file not found" @@ -250,7 +252,7 @@ bold "Section 4: Fixture scan detection" # The fixture has deliberately insecure patterns — scanner should find them if $HAS_RUST && [ -f "$HYPER_BIN" ]; then FINDINGS=$("$HYPER_BIN" scan "$FIXTURE_DIR" 2>/dev/null || true) - if echo "$FINDINGS" | grep -qi "unpinned\|injection\|insecure\|http\|eval\|finding\|weak\|issue" 2>/dev/null; then + if grep -qi "unpinned\|injection\|insecure\|http\|eval\|finding\|weak\|issue" <<<"$FINDINGS" 2>/dev/null; then pass "Scanner detects issues in fixture repo" else skip_test "Fixture detection" "scanner may not produce text findings" diff --git a/tests/readme-claims-test.sh b/tests/readme-claims-test.sh index bddbf66f..7c388369 100755 --- a/tests/readme-claims-test.sh +++ b/tests/readme-claims-test.sh @@ -27,7 +27,12 @@ copy() { git ls-files -z README.adoc EXPLAINME.adoc docs Justfile contractile.just lib/rules lib/mix/tasks lib/hypatia/cli.ex \ | xargs -0 -I{} sh -c 'mkdir -p "$1/$(dirname "$2")" && cp "$2" "$1/$2"' _ "$work/tree" {} # Every link target, so the copy resolves what the real tree resolves. - git ls-files | while IFS= read -r f; do [ -e "$work/tree/$f" ] || { mkdir -p "$work/tree/$(dirname "$f")"; : > "$work/tree/$f"; }; done + # AsciiDoc targets are copied whole, so their anchors resolve as well. + git ls-files | while IFS= read -r f; do + [[ -e "$work/tree/$f" ]] && continue + mkdir -p "$work/tree/$(dirname "$f")" + case "$f" in *.adoc) cp "$f" "$work/tree/$f" ;; *) : > "$work/tree/$f" ;; esac + done } expect_msg "the committed tree passes" 0 "match the tree" bash "$script" @@ -48,6 +53,16 @@ copy; sed -i '/^\[\[proof-ledger\]\]$/d' "$work/tree/EXPLAINME.adoc" expect_msg "a link to an anchor the target no longer defines is refused" 1 "defines no anchor proof-ledger" \ env README_CLAIMS_ROOT="$work/tree" bash "$script" +# A target with no explicit ID still has its section IDs. Under `set -e` a +# first grep with no match must not end the anchor list before them (#1208). +copy; printf '= Planted Title\n\n== Planted Section\n\ntext\n' > "$work/tree/docs/planted-title-only.adoc" +printf '\nlink:docs/planted-title-only.adoc#_planted_section[planted]\n' >> "$work/tree/README.adoc" +expect_msg "a section ID in a target with no explicit ID resolves" 0 "match the tree" \ + env README_CLAIMS_ROOT="$work/tree" bash "$script" +sed -i 's/#_planted_section\[/#_no_such_section[/' "$work/tree/README.adoc" +expect_msg "a missing section ID in such a target is refused" 1 "defines no anchor _no_such_section" \ + env README_CLAIMS_ROOT="$work/tree" bash "$script" + copy; touch "$work/tree/lib/mix/tasks/hypatia.planted.ex" expect_msg "a Mix task the README does not count is refused" 1 "Mix tasks" \ env README_CLAIMS_ROOT="$work/tree" bash "$script" diff --git a/tests/rules-praxis-test.sh b/tests/rules-praxis-test.sh index ffac5cc5..6e195003 100755 --- a/tests/rules-praxis-test.sh +++ b/tests/rules-praxis-test.sh @@ -85,19 +85,22 @@ expect_broken "an un-retired tombstone is refused" "$work/tomb.deed" "tombstone # Facts the code states, read back from the registry (review of #1070): a # provenance table is not a severity table, `severity: :advisory` is advisory, # and a rule that fires at several severities keeps the highest. -row() { awk -v t="\"$1\"" '$1 == ":id" { on = ($2 == t) } on { print }' "$deed"; } # one rule-def -if row hypatia/branch_protection/BP001 | grep -q ':severity high' && row hypatia/supply_chain/SC001 | grep -q ':status active'; then ok "opt-in scanner rules are registered"; else bad "BP001/SC001 missing or wrong"; fi -if row hypatia/git_state/GS007 | grep -q ':class advisory'; then ok "an advisory-severity rule is class advisory"; else bad "GS007 is not class advisory"; fi -if row hypatia/code_scanning_alerts/CSA006 | grep -q ':severity medium'; then ok "a multi-severity rule keeps its highest severity"; else bad "CSA006 is not medium"; fi +row() { local id=$1; awk -v t="\"$id\"" '$1 == ":id" { on = ($2 == t) } on { print }' "$deed"; } # one rule-def +# Whether a rule-def carries a fact. Captured, then matched: `row | grep -q` +# loses a match to SIGPIPE under pipefail when awk is still writing (RE011). +row_has() { local id=$1 fact=$2; grep -q -- "$fact" <<<"$(row "$id")"; } +if row_has hypatia/branch_protection/BP001 ':severity high' && row_has hypatia/supply_chain/SC001 ':status active'; then ok "opt-in scanner rules are registered"; else bad "BP001/SC001 missing or wrong"; fi +if row_has hypatia/git_state/GS007 ':class advisory'; then ok "an advisory-severity rule is class advisory"; else bad "GS007 is not class advisory"; fi +if row_has hypatia/code_scanning_alerts/CSA006 ':severity medium'; then ok "a multi-severity rule keeps its highest severity"; else bad "CSA006 is not medium"; fi -if row hypatia/pin_integrity/PI005 | grep -q ':class advisory' && row hypatia/disambiguation_rules/DR001 | grep -q ':class advisory'; then ok "rules the code calls advisory in words are class advisory"; else bad "PI005 or DR001 is not class advisory"; fi +if row_has hypatia/pin_integrity/PI005 ':class advisory' && row_has hypatia/disambiguation_rules/DR001 ':class advisory'; then ok "rules the code calls advisory in words are class advisory"; else bad "PI005 or DR001 is not class advisory"; fi -if row hypatia/workflow_audit/duplicate_workflow | grep -q ':severity minor' && row hypatia/cicd_rules/waste_duplicate_workflow | grep -q ':severity medium'; then ok "named atom rules carry the severity the code states"; else bad "duplicate_workflow rows lack their severity"; fi +if row_has hypatia/workflow_audit/duplicate_workflow ':severity minor' && row_has hypatia/cicd_rules/waste_duplicate_workflow ':severity medium'; then ok "named atom rules carry the severity the code states"; else bad "duplicate_workflow rows lack their severity"; fi # Severities the code computes (an if expression, a helper): the highest branch. # A severity read from run-time alert data (CSA001) has none to record. -if row hypatia/supply_chain/SC012 | grep -q ':severity high' && row hypatia/structural_drift/SD008 | grep -q ':severity critical' \ - && row hypatia/research_extensions/RE001 | grep -q ':severity high' && ! row hypatia/code_scanning_alerts/CSA001 | grep -q ':severity'; then +if row_has hypatia/supply_chain/SC012 ':severity high' && row_has hypatia/structural_drift/SD008 ':severity critical' \ + && row_has hypatia/research_extensions/RE001 ':severity high' && ! row_has hypatia/code_scanning_alerts/CSA001 ':severity'; then ok "computed severities are read; run-time severities stay empty"; else bad "computed severity rows are wrong"; fi # The names findings actually carry resolve (A5 R09). Ground truth: every @@ -140,7 +143,7 @@ done < <(awk ' ' lib/hypatia/cli.ex lib/rules/*.ex | sort -u) if [ -z "$missing_pairs" ]; then ok "every literal rule_module/type pair in the code is in the registry"; else bad "pairs in the code but not the registry:$missing_pairs"; fi if ! grep -qF '"hypatia/workflow_audit/always_after_cancel"' "$deed" && grep -qF '"hypatia/ci_failure_triage/always_after_cancel"' "$deed"; then ok "a name moved by a rule_id override is registered under its real module only"; else bad "always_after_cancel is registered under workflow_audit, or missing under ci_failure_triage"; fi -if row hypatia/structured_format/SF001 | grep -q ':class advisory' && row hypatia/cicd_rules/banned_language_file | grep -q ':severity critical' && row hypatia/implementation_inside_canon/HYP-S009 | grep -q ':status active' && row hypatia/workflow_hardening/invalid_actions_lock | grep -q ':status active'; then ok "fixed names carry the code's severity and advisory flag; HYP-S009 and invalid_actions_lock are registered"; else bad "a fixed-name row is wrong"; fi +if row_has hypatia/structured_format/SF001 ':class advisory' && row_has hypatia/cicd_rules/banned_language_file ':severity critical' && row_has hypatia/implementation_inside_canon/HYP-S009 ':status active' && row_has hypatia/workflow_hardening/invalid_actions_lock ':status active'; then ok "fixed names carry the code's severity and advisory flag; HYP-S009 and invalid_actions_lock are registered"; else bad "a fixed-name row is wrong"; fi # Portability: gawk and mawk differ (a `\[` in a -v value is valid in one and a # fatal regex error in the other, and the awk then dies without a word inside a