diff --git a/hooks/validate-sha-pins.sh b/hooks/validate-sha-pins.sh index 697092b5..5c5963dd 100755 --- a/hooks/validate-sha-pins.sh +++ b/hooks/validate-sha-pins.sh @@ -13,7 +13,7 @@ for workflow in .github/workflows/*.yml .github/workflows/*.yaml; do while IFS= read -r line; do if [[ "$line" =~ uses:.*@ ]]; then # Check if it has a SHA (40 hex chars) - if ! echo "$line" | grep -qE '@[a-f0-9]{40}'; then + if ! echo "$line" | grep -E '@[a-f0-9]{40}' >/dev/null; then echo "ERROR: Unpinned action in $workflow" echo " $line" echo " Actions must use SHA pins: uses: action/name@SHA # version" diff --git a/hooks/validate-spdx.sh b/hooks/validate-spdx.sh index cc81cf18..11517bd2 100755 --- a/hooks/validate-spdx.sh +++ b/hooks/validate-spdx.sh @@ -11,7 +11,7 @@ for workflow in .github/workflows/*.yml .github/workflows/*.yaml; do [ -f "$workflow" ] || continue first_line=$(head -n1 "$workflow") - if ! echo "$first_line" | grep -qE "$SPDX_PATTERN"; then + if ! echo "$first_line" | grep -E "$SPDX_PATTERN" >/dev/null; then echo "ERROR: Missing SPDX header in $workflow" echo " First line should be: # SPDX-License-Identifier: MPL-2.0" ERRORS=$((ERRORS + 1)) diff --git a/hypatia-cli-bash.sh b/hypatia-cli-bash.sh index a378331a..afc2afa1 100755 --- a/hypatia-cli-bash.sh +++ b/hypatia-cli-bash.sh @@ -618,7 +618,7 @@ scan_repo_structure() { if [[ -d "$dir/.github/workflows" ]]; then for wf in "$dir/.github/workflows"/*.yml "$dir/.github/workflows"/*.yaml; do [[ -f "$wf" ]] || continue - if ! head -1 "$wf" | grep -q '^# SPDX-License-Identifier:'; then + if ! head -1 "$wf" | grep '^# SPDX-License-Identifier:' >/dev/null; then emit_finding "medium" "repo_structure" "missing_spdx_header" \ "$wf" 1 "Workflow missing SPDX header" "CWE-1059" \ "Add '# SPDX-License-Identifier: MPL-2.0' as first line" "true" diff --git a/lib/data/arangodb.ex b/lib/data/arangodb.ex index 5622cdb9..82977413 100644 --- a/lib/data/arangodb.ex +++ b/lib/data/arangodb.ex @@ -57,13 +57,33 @@ defmodule Hypatia.Data.ArangoDB do "contributor_fixes" ] + @graph_name "hypatia_graph" + + @graph_edge_definitions [ + %{ + "collection" => "trusts", + "from" => ["repos", "bots", "recipes", "contributors"], + "to" => ["repos", "bots", "recipes", "contributors"] + }, + %{"collection" => "depends_on", "from" => ["repos"], "to" => ["repos"]}, + %{"collection" => "dispatches", "from" => ["findings"], "to" => ["bots"]}, + %{"collection" => "applies_to", "from" => ["recipes"], "to" => ["patterns"]}, + %{"collection" => "repo_has_finding", "from" => ["repos"], "to" => ["findings"]}, + %{"collection" => "finding_matches_pattern", "from" => ["findings"], "to" => ["patterns"]}, + %{"collection" => "bot_executes_recipe", "from" => ["bots"], "to" => ["recipes"]}, + %{"collection" => "recipe_has_outcome", "from" => ["recipes"], "to" => ["outcomes"]}, + %{"collection" => "contributor_fixes", "from" => ["contributors"], "to" => ["outcomes"]} + ] + defstruct [ :base_url, :database, :auth_header, connected: false, last_sync: nil, - sync_timer: nil + sync_timer: nil, + setup_timer: nil, + setup_attempts: 0 ] # --- GenServer API --- @@ -86,23 +106,60 @@ defmodule Hypatia.Data.ArangoDB do auth_header: "Basic #{auth}" } - # Try to connect, but don't crash if ArangoDB isn't available - state = try_connect(state) - - if state.connected do - ensure_schema(state) - timer = Process.send_after(self(), :sync_from_verisimdb, @sync_interval_ms) - Logger.info("ArangoDB connected: #{url}/#{db}") - {:ok, %{state | sync_timer: timer}} - else - Logger.warning( - "ArangoDB not available at #{url} -- running in degraded mode (flat files only)" - ) + # Never crashes when ArangoDB is unavailable: setup/1 degrades and retries. + {:ok, setup(state)} + end + + # Connects and ensures the schema. The client counts as connected only once + # both succeed; until then it stays in degraded mode and retries with an + # exponential backoff (`:arango_setup_retry_base_ms` doubling up to + # `:arango_setup_retry_max_ms`), so a transient failure such as a 503 + # during setup does not leave it disconnected for good. A retry repeats + # every step; a collection or index created by an earlier attempt answers + # 1207 and is accepted, and an existing graph (1925) is accepted once + # check_existing_graph/1 has verified it. The sync timer starts only after + # setup succeeds. + defp setup(state) do + result = + if reachable?(state) do + with {:error, reason} <- ensure_schema(state), + do: {:error, "ArangoDB schema initialisation failed: #{inspect(reason)}"} + else + {:error, "ArangoDB not available at #{state.base_url}"} + end + + case result do + :ok -> + timer = Process.send_after(self(), :sync_from_verisimdb, @sync_interval_ms) + Logger.info("ArangoDB connected: #{state.base_url}/#{state.database}") + %{state | connected: true, sync_timer: timer, setup_timer: nil, setup_attempts: 0} + + {:error, message} -> + delay = setup_retry_delay(state.setup_attempts) + level = if state.setup_attempts == 0, do: :warning, else: :debug + + Logger.log( + level, + "#{message} -- running in degraded mode (flat files only); retrying setup in #{delay} ms" + ) + + timer = Process.send_after(self(), :retry_setup, delay) - {:ok, state} + %{ + state + | connected: false, + setup_timer: timer, + setup_attempts: state.setup_attempts + 1 + } end end + defp setup_retry_delay(attempts) do + base = Application.get_env(:hypatia, :arango_setup_retry_base_ms, 5_000) + cap = Application.get_env(:hypatia, :arango_setup_retry_max_ms, 5 * 60 * 1_000) + min(base * Integer.pow(2, min(attempts, 20)), cap) + end + # --- Public API --- @doc "Check if ArangoDB is connected" @@ -314,18 +371,29 @@ defmodule Hypatia.Data.ArangoDB do def handle_cast(:sync_from_verisimdb, state) do if state.connected do - do_sync(state) - {:noreply, %{state | last_sync: DateTime.utc_now()}} + {:noreply, sync_and_record(state)} else {:noreply, state} end end + # `last_sync` moves only on a run in which every upsert succeeded; after a + # failed run it keeps the last successful timestamp. + defp sync_and_record(state) do + case do_sync(state) do + :ok -> %{state | last_sync: DateTime.utc_now()} + {:error, _failures} -> state + end + end + + def handle_info(:retry_setup, state) do + {:noreply, setup(%{state | setup_timer: nil})} + end + def handle_info(:sync_from_verisimdb, state) do if state.connected do - do_sync(state) timer = Process.send_after(self(), :sync_from_verisimdb, @sync_interval_ms) - {:noreply, %{state | last_sync: DateTime.utc_now(), sync_timer: timer}} + {:noreply, %{sync_and_record(state) | sync_timer: timer}} else timer = Process.send_after(self(), :sync_from_verisimdb, @sync_interval_ms) {:noreply, %{state | sync_timer: timer}} @@ -334,74 +402,183 @@ defmodule Hypatia.Data.ArangoDB do # --- HTTP Client --- - defp try_connect(state) do - url = "#{state.base_url}/_api/version" + # Probes the target database first, so a user who administers only that + # database (and cannot read `_system`) still connects. A 404 database-not- + # found there means the database is not created yet: fall back to the + # `_system` version probe, so that setup can try to create it. + defp reachable?(state) do + target = http_get("#{db_url(state)}/_api/version", state.auth_header) - case http_get(url, state.auth_header) do - {:ok, %{"server" => "arango"}} -> %{state | connected: true} - _ -> %{state | connected: false} + cond do + arango_version?(target) -> + true + + database_not_found?(target) -> + arango_version?(http_get("#{state.base_url}/_api/version", state.auth_header)) + + true -> + false end end + defp arango_version?({:ok, %{"server" => "arango"}}), do: true + defp arango_version?(_), do: false + + # ArangoDB answers a request for a database that does not exist with a 404 + # and errorNum 1228 (ERROR_ARANGO_DATABASE_NOT_FOUND). + defp database_not_found?({:error, {404, body}}) do + match?({:ok, %{"errorNum" => 1228}}, Jason.decode(body)) + end + + defp database_not_found?(_), do: false + + defp db_url(state), do: "#{state.base_url}/_db/#{state.database}" + defp ensure_schema(state) do - db_url = "#{state.base_url}/_db/#{state.database}" + # Every later step depends on the database, so stop here if it fails. + database = ensure_database(state) + + if schema_result_ok?(database), + do: ensure_collections_and_graph(state), + else: {:error, database} + end + + # Creates the database only when the probe says it does not exist (404, + # errorNum 1228). Creating one needs Administrate on `_system`; a + # least-privilege user that administers only a pre-provisioned target + # database gets a 403 there. Any other probe failure (a timeout, a 3xx, a + # 5xx) is no evidence of absence and is returned as it is. + defp ensure_database(state) do + probe = http_get("#{db_url(state)}/_api/database/current", state.auth_header) + + case probe do + {:ok, %{} = current} -> + {:ok, 200, current} + + _ -> + if database_not_found?(probe) do + http_post_status("#{state.base_url}/_api/database", state.auth_header, %{ + "name" => state.database + }) + else + probe + end + end + end - # Create database if it doesn't exist - http_post("#{state.base_url}/_api/database", state.auth_header, %{"name" => state.database}) + defp ensure_collections_and_graph(state) do + db_url = db_url(state) # Create document collections - Enum.each(@document_collections, fn name -> - http_post("#{db_url}/_api/collection", state.auth_header, %{"name" => name, "type" => 2}) - end) + documents = + Enum.map(@document_collections, fn name -> + http_post_status("#{db_url}/_api/collection", state.auth_header, %{ + "name" => name, + "type" => 2 + }) + end) # Create edge collections - Enum.each(@edge_collections, fn name -> - http_post("#{db_url}/_api/collection", state.auth_header, %{"name" => name, "type" => 3}) - end) + edges = + Enum.map(@edge_collections, fn name -> + http_post_status("#{db_url}/_api/collection", state.auth_header, %{ + "name" => name, + "type" => 3 + }) + end) # Create indices for common queries - create_index(state, "confidence_history", ["recipe_id", "timestamp"]) - create_index(state, "outcomes", ["timestamp"]) - create_index(state, "outcomes", ["recipe_id"]) - create_index(state, "findings", ["repo", "severity"]) - create_index(state, "findings", ["pattern_id"]) - create_index(state, "anomalies", ["resolved", "timestamp"]) - create_index(state, "repos", ["trust_score"]) - create_index(state, "bots", ["quarantined"]) + indices = [ + create_index(state, "confidence_history", ["recipe_id", "timestamp"]), + create_index(state, "outcomes", ["timestamp"]), + create_index(state, "outcomes", ["recipe_id"]), + create_index(state, "findings", ["repo", "severity"]), + create_index(state, "findings", ["pattern_id"]), + create_index(state, "anomalies", ["resolved", "timestamp"]), + create_index(state, "repos", ["trust_score"]), + create_index(state, "bots", ["quarantined"]) + ] # Create named graph - http_post("#{db_url}/_api/gharial", state.auth_header, %{ - "name" => "hypatia_graph", - "edgeDefinitions" => [ - %{ - "collection" => "trusts", - "from" => ["repos", "bots", "recipes", "contributors"], - "to" => ["repos", "bots", "recipes", "contributors"] - }, - %{"collection" => "depends_on", "from" => ["repos"], "to" => ["repos"]}, - %{"collection" => "dispatches", "from" => ["findings"], "to" => ["bots"]}, - %{"collection" => "applies_to", "from" => ["recipes"], "to" => ["patterns"]}, - %{"collection" => "repo_has_finding", "from" => ["repos"], "to" => ["findings"]}, - %{ - "collection" => "finding_matches_pattern", - "from" => ["findings"], - "to" => ["patterns"] - }, - %{"collection" => "bot_executes_recipe", "from" => ["bots"], "to" => ["recipes"]}, - %{"collection" => "recipe_has_outcome", "from" => ["recipes"], "to" => ["outcomes"]}, - %{"collection" => "contributor_fixes", "from" => ["contributors"], "to" => ["outcomes"]} - ] - }) + graph = + case http_post_status("#{db_url}/_api/gharial", state.auth_header, %{ + "name" => @graph_name, + "edgeDefinitions" => @graph_edge_definitions + }) do + {:ok, status, %{"errorNum" => 1925}} when status in 400..499 -> + check_existing_graph(state) + + created -> + created + end + + results = documents ++ edges ++ indices ++ [graph] + + case Enum.find(results, &(not schema_result_ok?(&1))) do + nil -> + Logger.info( + "ArangoDB schema ensured: #{length(@document_collections)} document + #{length(@edge_collections)} edge collections" + ) - Logger.info( - "ArangoDB schema ensured: #{length(@document_collections)} document + #{length(@edge_collections)} edge collections" - ) + :ok + + failure -> + {:error, failure} + end + end + + # A duplicate-graph answer (1925) says only that a graph of that name + # exists, not that it is ours. It is accepted when, for every required + # edge definition, the existing graph has an edge definition on the same + # collection whose `from` and `to` vertex collections are exactly the + # required ones (as sets: order and duplicates do not matter). Extra edge + # definitions and orphan collections in the existing graph are allowed. + # A mismatch is a schema failure; the graph is never replaced. + defp check_existing_graph(state) do + case http_get("#{db_url(state)}/_api/gharial/#{@graph_name}", state.auth_header) do + {:ok, %{"graph" => %{"edgeDefinitions" => existing}} = body} when is_list(existing) -> + case graph_mismatches(existing) do + [] -> {:ok, 200, body} + mismatches -> {:error, {:graph_mismatch, mismatches}} + end + + other -> + {:error, {:graph_unreadable, other}} + end + end + + defp graph_mismatches(existing) do + by_collection = Map.new(existing, &{&1["collection"], &1}) + + @graph_edge_definitions + |> Enum.reject(&same_vertices?(&1, Map.get(by_collection, &1["collection"]))) + |> Enum.map(& &1["collection"]) + end + + defp same_vertices?(required, %{} = actual) do + Enum.all?(["from", "to"], fn side -> + is_list(actual[side]) and MapSet.new(actual[side]) == MapSet.new(required[side]) + end) end + defp same_vertices?(_required, nil), do: false + + # A schema step succeeds on a 2xx, or on a 4xx that says the resource + # already exists (1207 duplicate name), which happens on every + # initialisation after the first. A duplicate graph (1925) is checked by + # check_existing_graph/1 instead. Any other status fails, even with a JSON + # body: a gateway's `{"message": "forbidden"}` is not a success. + defp schema_result_ok?({:ok, status, _}) when status in 200..299, do: true + + defp schema_result_ok?({:ok, status, %{"errorNum" => 1207}}) when status in 400..499, + do: true + + defp schema_result_ok?(_), do: false + defp create_index(state, collection, fields) do db_url = "#{state.base_url}/_db/#{state.database}" - http_post("#{db_url}/_api/index?collection=#{collection}", state.auth_header, %{ + http_post_status("#{db_url}/_api/index?collection=#{collection}", state.auth_header, %{ "type" => "persistent", "fields" => fields }) @@ -442,16 +619,45 @@ defmodule Hypatia.Data.ArangoDB do # --- verisim-data Sync --- + # Returns :ok when every upsert succeeded, otherwise {:error, failures}, + # each failure `{collection, key, reason}` (or the exception that stopped + # the run). A failed run is logged, never reported as complete. defp do_sync(state) do verisimdb_path = Application.get_env(:hypatia, :verisimdb_data_path, "data/verisim") - sync_scans(state, verisimdb_path) - sync_outcomes(state, verisimdb_path) - sync_recipes(state, verisimdb_path) + failures = + sync_scans(state, verisimdb_path) ++ + sync_outcomes(state, verisimdb_path) ++ + sync_recipes(state, verisimdb_path) + + case failures do + [] -> + Logger.info("ArangoDB sync from verisim-data complete") + :ok + + [first | _] -> + Logger.error( + "ArangoDB sync from verisim-data failed (#{length(failures)} failed upserts, " <> + "first: #{inspect(first)})" + ) - Logger.info("ArangoDB sync from verisim-data complete") + {:error, failures} + end rescue - e -> Logger.error("ArangoDB sync failed: #{inspect(e)}") + e -> + Logger.error("ArangoDB sync failed: #{inspect(e)}") + {:error, [e]} + end + + # Upserts and returns the failure, if any, as a list for flat_map. A + # cursor answer without a result list (an error body with no message) is a + # failure too. + defp sync_upsert(state, collection, key, document) do + case do_upsert(state, collection, key, document) do + {:ok, result} when is_list(result) -> [] + {:error, reason} -> [{collection, key, reason}] + other -> [{collection, key, other}] + end end defp sync_scans(state, base_path) do @@ -459,48 +665,40 @@ defmodule Hypatia.Data.ArangoDB do case File.ls(scans_dir) do {:ok, repos} -> - Enum.each(repos, fn repo_dir -> + Enum.flat_map(repos, fn repo_dir -> scan_path = Path.join([scans_dir, repo_dir, "latest.json"]) - case File.read(scan_path) do - {:ok, content} -> - case Jason.decode(content) do - {:ok, scan} -> - findings = Map.get(scan, "findings", []) - - do_upsert(state, "repos", repo_dir, %{ - "name" => repo_dir, - "finding_count" => length(findings), - "last_scanned" => Map.get(scan, "timestamp", ""), - "status" => "active" - }) - - # Sync individual findings - Enum.each(findings, fn finding -> - f_key = "#{repo_dir}_#{:erlang.phash2(finding)}" - - do_upsert( - state, - "findings", - f_key, - Map.merge(finding, %{ - "repo" => repo_dir, - "status" => "open" - }) - ) - end) - - _ -> - :skip - end - - _ -> - :skip + with {:ok, content} <- File.read(scan_path), + {:ok, scan} <- Jason.decode(content) do + findings = Map.get(scan, "findings", []) + + repo = + sync_upsert(state, "repos", repo_dir, %{ + "name" => repo_dir, + "finding_count" => length(findings), + "last_scanned" => Map.get(scan, "timestamp", ""), + "status" => "active" + }) + + # Sync individual findings + repo ++ + Enum.flat_map(findings, fn finding -> + f_key = "#{repo_dir}_#{:erlang.phash2(finding)}" + + sync_upsert( + state, + "findings", + f_key, + Map.merge(finding, %{"repo" => repo_dir, "status" => "open"}) + ) + end) + else + _ -> [] end end) _ -> - :ok + [] end end @@ -511,31 +709,28 @@ defmodule Hypatia.Data.ArangoDB do {:ok, files} -> files |> Enum.filter(&String.ends_with?(&1, ".jsonl")) - |> Enum.each(fn file -> - path = Path.join(outcomes_dir, file) - - case File.read(path) do + |> Enum.flat_map(fn file -> + case File.read(Path.join(outcomes_dir, file)) do {:ok, content} -> content |> String.split("\n", trim: true) - |> Enum.each(fn line -> + |> Enum.flat_map(fn line -> case Jason.decode(line) do {:ok, outcome} -> - key = "outcome_#{:erlang.phash2(outcome)}" - do_upsert(state, "outcomes", key, outcome) + sync_upsert(state, "outcomes", "outcome_#{:erlang.phash2(outcome)}", outcome) _ -> - :skip + [] end end) _ -> - :skip + [] end end) _ -> - :ok + [] end end @@ -546,27 +741,18 @@ defmodule Hypatia.Data.ArangoDB do {:ok, files} -> files |> Enum.filter(&String.ends_with?(&1, ".json")) - |> Enum.each(fn file -> - path = Path.join(recipes_dir, file) - - case File.read(path) do - {:ok, content} -> - case Jason.decode(content) do - {:ok, recipe} -> - key = Map.get(recipe, "id", Path.rootname(file)) - do_upsert(state, "recipes", key, recipe) - - _ -> - :skip - end - - _ -> - :skip + |> Enum.flat_map(fn file -> + with {:ok, content} <- File.read(Path.join(recipes_dir, file)), + {:ok, recipe} <- Jason.decode(content) do + key = Map.get(recipe, "id", Path.rootname(file)) + sync_upsert(state, "recipes", key, recipe) + else + _ -> [] end end) _ -> - :ok + [] end end @@ -581,14 +767,30 @@ defmodule Hypatia.Data.ArangoDB do {~c"Accept", ~c"application/json"} ] - case :httpc.request(:get, {String.to_charlist(url), headers}, [{:timeout, 10_000}], []) do + case :httpc.request( + :get, + {String.to_charlist(url), headers}, + [{:timeout, 10_000}, {:autoredirect, false}], + [] + ) do {:ok, {{_, 200, _}, _, body}} -> Jason.decode(to_string(body)) {:ok, {{_, status, _}, _, body}} -> {:error, {status, to_string(body)}} {:error, reason} -> {:error, reason} end end + # Decoded body on a 2xx or 4xx (ArangoDB reports its errors as JSON 4xx); + # `{:error, {status, body}}` on any other status, a 3xx included. defp http_post(url, auth_header, body) do + case http_post_status(url, auth_header, body) do + {:ok, _status, decoded} -> {:ok, decoded} + error -> error + end + end + + # As http_post/3, but keeps the status, so a caller can tell a 2xx from a + # 4xx whose body does not use ArangoDB's error shape. + defp http_post_status(url, auth_header, body) do :inets.start() :ssl.start() @@ -602,14 +804,13 @@ defmodule Hypatia.Data.ArangoDB do case :httpc.request( :post, {String.to_charlist(url), headers, ~c"application/json", json_body}, - [{:timeout, 10_000}], + [{:timeout, 10_000}, {:autoredirect, false}], [] ) do - {:ok, {{_, code, _}, _, resp_body}} when code in 200..299 -> - Jason.decode(to_string(resp_body)) - - {:ok, {{_, code, _}, _, resp_body}} when code in 300..499 -> - Jason.decode(to_string(resp_body)) + {:ok, {{_, code, _}, _, resp_body}} when code in 200..299 or code in 400..499 -> + with {:ok, decoded} <- Jason.decode(to_string(resp_body)) do + {:ok, code, decoded} + end {:ok, {{_, status, _}, _, resp_body}} -> {:error, {status, to_string(resp_body)}} diff --git a/lib/hypatia/reviewer.ex b/lib/hypatia/reviewer.ex index be569c2d..43341a07 100644 --- a/lib/hypatia/reviewer.ex +++ b/lib/hypatia/reviewer.ex @@ -422,15 +422,78 @@ defmodule Hypatia.Reviewer do [] end - case :httpc.request(:get, {String.to_charlist(url), headers}, [{:timeout, 15_000}], []) do - {:ok, {{_, 200, _}, _resp_headers, body}} -> - {:ok, to_string(body)} + get_same_origin(url, headers) + end - {:ok, {{_, status, _}, _, _}} -> - {:error, {:http_status, status}} + @max_redirects 3 + @timeout_ms 15_000 + + # GET that follows a redirect only to the origin (scheme, host and port) + # of the first request, at most @max_redirects times. GitHub answers a + # request for a renamed repository with a 301 to + # https://api.github.com/repositories//..., so the review still runs, + # while the token never reaches another origin or a plain-HTTP downgrade. + # A Location carrying userinfo is refused too, so a redirect cannot add + # credentials of its own. :httpc's own redirect handling stays off (#1195). + # `timeout_ms` bounds the whole chain, not each hop: every request gets + # what is left of one monotonic deadline. + @doc false + def get_same_origin(url, headers, hops \\ @max_redirects, timeout_ms \\ @timeout_ms) do + follow_same_origin(url, headers, hops, System.monotonic_time(:millisecond) + timeout_ms) + end - {:error, reason} -> - {:error, {:http_error, reason}} + defp follow_same_origin(url, headers, hops, deadline) do + remaining = deadline - System.monotonic_time(:millisecond) + + if remaining <= 0 do + {:error, {:http_error, :timeout}} + else + case :httpc.request( + :get, + {String.to_charlist(url), headers}, + [{:timeout, remaining}, {:autoredirect, false}], + [] + ) do + {:ok, {{_, 200, _}, _resp_headers, body}} -> + {:ok, to_string(body)} + + {:ok, {{_, status, _}, resp_headers, _}} when status in [301, 302, 303, 307, 308] -> + with {:ok, target} <- redirect_target(url, resp_headers), + true <- same_origin?(url, target) and hops > 0 do + follow_same_origin(target, headers, hops - 1, deadline) + else + _ -> {:error, {:http_status, status}} + end + + {:ok, {{_, status, _}, _, _}} -> + {:error, {:http_status, status}} + + {:error, reason} -> + {:error, {:http_error, reason}} + end + end + end + + defp redirect_target(url, resp_headers) do + with {_, location} <- List.keyfind(resp_headers, ~c"location", 0), + location = to_string(location), + nil <- URI.parse(location).userinfo do + {:ok, url |> URI.merge(location) |> URI.to_string()} + else + _ -> :error end end + + # Host names compare case-insensitively (ASCII), as DNS does. + defp same_origin?(from, to) do + origin(from) == origin(to) + end + + defp origin(url) do + uri = URI.parse(url) + {downcase(uri.scheme), downcase(uri.host), uri.port} + end + + defp downcase(nil), do: nil + defp downcase(text), do: String.downcase(text, :ascii) end diff --git a/mise.toml b/mise.toml index 9cd7da86..743b11c9 100644 --- a/mise.toml +++ b/mise.toml @@ -1,7 +1,10 @@ [tools] # Language runtimes # Match the OTP 27.3.4.19 / Elixir 1.19.4 lanes in .github/workflows/ci.yml and tests.yml. -erlang = "27.3.4.19" # >= 27.3.4.13: :httpc drops authorization on cross-origin redirect +# OTP 27.3.4.19 drops header and URL credentials when a redirect changes host or port. +# Scheme alone is not checked: same-host, same-port HTTPS-to-HTTP redirects are not protected. +# Keep explicit redirect guards. +erlang = "27.3.4.19" elixir = "1.19.4-otp-27" node = "latest" rust = "latest" # provides cargo, rustfmt and clippy diff --git a/scripts/auto-fix-formulaic.sh b/scripts/auto-fix-formulaic.sh index 6b40c7dd..fa7a313e 100755 --- a/scripts/auto-fix-formulaic.sh +++ b/scripts/auto-fix-formulaic.sh @@ -100,7 +100,7 @@ fix_agpl_references() { for f in $agpl_files; do # Skip mock/fixture files that simulate external API responses - if echo "$f" | grep -qE 'mock|fixture.*github'; then + if echo "$f" | grep -E 'mock|fixture.*github' >/dev/null; then continue fi warn "AGPL-3.0 reference in: $f (should be MPL-2.0)" diff --git a/scripts/check-readme-claims.sh b/scripts/check-readme-claims.sh index dcf052f0..d68da048 100755 --- a/scripts/check-readme-claims.sh +++ b/scripts/check-readme-claims.sh @@ -80,7 +80,7 @@ for doc in "$readme" EXPLAINME.adoc docs/README.adoc; do if [ ! -e "$dir/$path" ]; then say "$doc links $target, which does not exist"; continue; fi if [ "$target" != "$path" ] && [[ "$path" == *.adoc ]]; then frag=${target#*#} - anchors "$dir/$path" | grep -qxF -- "$frag" || say "$doc links $target, and $path defines no anchor $frag" + anchors "$dir/$path" | grep -xF -- "$frag" >/dev/null || say "$doc links $target, and $path defines no anchor $frag" fi done < <(grep -oE 'link:[^[ ]+\[' "$doc" | sed -E 's/^link:(.*)\[$/\1/') done diff --git a/scripts/ci-health/detect.sh b/scripts/ci-health/detect.sh index 6c6d4fc2..b8ad2890 100755 --- a/scripts/ci-health/detect.sh +++ b/scripts/ci-health/detect.sh @@ -59,11 +59,11 @@ for fail_id in "${fail_ids[@]}"; do emit B-TRACKER-TOKEN HIGH "Pin updater reports missing write credential (run $fail_id) → compare pins read-only before provisioning repository-scoped write access" tracker_token_seen=true fi - if [ "$billing_seen" = false ] && printf '%s' "$msg" | grep -qiE 'payments have failed|spending limit'; then + if [ "$billing_seen" = false ] && printf '%s' "$msg" | grep -iE 'payments have failed|spending limit' >/dev/null; then emit A-BILLING CRITICAL "Actions billing/spending-limit wall blocks all billable jobs → OWNER: GitHub Settings -> Billing & plans" billing_seen=true fi - if [ "$lockfile_seen" = false ] && printf '%s' "$msg" | grep -qiE 'lockfile pin .* does not match ref|lockfile verification did not produce a result|unreachable-pin'; then + if [ "$lockfile_seen" = false ] && printf '%s' "$msg" | grep -iE 'lockfile pin .* does not match ref|lockfile verification did not produce a result|unreachable-pin' >/dev/null; then first_lock_msg=$(printf '%s\n' "$msg" | grep -iEm1 'lockfile pin .* does not match ref|lockfile verification did not produce a result|unreachable-pin') emit B-LOCKFILE HIGH "Actions lockfile verification failed (${first_lock_msg}) → run gh actions-lock --verify, repair, then commit actions.lock" lockfile_seen=true @@ -107,7 +107,7 @@ fi # --- D: burn anti-pattern (bare [push, pull_request] double-trigger), via API for path in $(api "repos/$O/$R/contents/.github/workflows" --jq '.[]?|select(.name|test("\\.ya?ml$"))|.path' || true); do if api "repos/$O/$R/contents/$path" --jq '.content' | base64 -d \ - | grep -qE '^on:[[:space:]]*\[[[:space:]]*push[[:space:]]*,[[:space:]]*pull_request[[:space:]]*\]'; then + | grep -E '^on:[[:space:]]*\[[[:space:]]*push[[:space:]]*,[[:space:]]*pull_request[[:space:]]*\]' >/dev/null; then emit D-BURN MEDIUM "ERR-WF-014: $path on bare [push,pull_request] (2x runs/PR) → scope push to default branch + concurrency-cancel" fi done diff --git a/scripts/ci-health/remediate.sh b/scripts/ci-health/remediate.sh index f1e57232..6208478c 100755 --- a/scripts/ci-health/remediate.sh +++ b/scripts/ci-health/remediate.sh @@ -68,7 +68,7 @@ case "$CLASS" in def=$(gh api "repos/$O/$R" --jq '.default_branch'); sha=$(gh api "repos/$O/$R/git/ref/heads/$def" --jq '.object.sha') targets=(); for p in $(gh api "repos/$O/$R/contents/.github/workflows" --jq '.[]?|select(.name|test("\\.ya?ml$"))|.path' 2>/dev/null); do gh api "repos/$O/$R/contents/$p?ref=$def" --jq '.content' 2>/dev/null | base64 -d 2>/dev/null \ - | grep -qE '^on:[[:space:]]*\[[[:space:]]*push[[:space:]]*,[[:space:]]*pull_request' && targets+=("$p") + | grep -E '^on:[[:space:]]*\[[[:space:]]*push[[:space:]]*,[[:space:]]*pull_request' >/dev/null && targets+=("$p") done [ "${#targets[@]}" -eq 0 ] && { echo "SKIP $R/D-BURN no-targets"; exit 0; } if [ "$DRY" = "true" ]; then echo "DRYRUN $R/D-BURN would patch ${#targets[@]} file(s) + open PR"; exit 0; fi @@ -84,7 +84,7 @@ case "$CLASS" in branches: [main, master] pull_request:" - if ! printf '%s' "$ccontent" | grep -q '^[[:space:]]*concurrency:'; then + if ! printf '%s' "$ccontent" | grep >/dev/null '^[[:space:]]*concurrency:'; then blk="$blk # Estate guardrail: scope push to default branches (PR fires once, not diff --git a/scripts/ci-health/sweep.sh b/scripts/ci-health/sweep.sh index b4c227d0..f6518aa7 100755 --- a/scripts/ci-health/sweep.sh +++ b/scripts/ci-health/sweep.sh @@ -33,7 +33,7 @@ while IFS=$'\t' read -r repo cls _sev _det; do D-BURN) if [ "$burned" -lt "$MAXPR" ]; then out=$(OWNER="$O" "$HERE/remediate.sh" "$repo" "$cls" "$DRY" || true); echo "$out" - echo "$out" | grep -q '^FIXED' && burned=$((burned+1)) + echo "$out" | grep >/dev/null '^FIXED' && burned=$((burned+1)) else echo "CAP $repo/D-BURN max-burn-prs($MAXPR) reached — deferred"; fi ;; esac done < <(sort -u "$findings") diff --git a/scripts/fix-scripts/fix-http-to-https.sh b/scripts/fix-scripts/fix-http-to-https.sh index 411b101b..0c4385f8 100755 --- a/scripts/fix-scripts/fix-http-to-https.sh +++ b/scripts/fix-scripts/fix-http-to-https.sh @@ -44,7 +44,7 @@ fix_file() { [[ "$file" == */.zig-cache/* ]] && return 0 # Only process text files - file -b --mime-type "$file" 2>/dev/null | grep -q "text/" || return 0 + file -b --mime-type "$file" 2>/dev/null | grep >/dev/null "text/" || return 0 # Build sed exclusion pattern local sed_cmd="s|http://|https://|g" diff --git a/scripts/fix-scripts/fix-missing-spdx.sh b/scripts/fix-scripts/fix-missing-spdx.sh index d44a99b4..a99b076e 100755 --- a/scripts/fix-scripts/fix-missing-spdx.sh +++ b/scripts/fix-scripts/fix-missing-spdx.sh @@ -67,7 +67,7 @@ fix_file() { local header="${COMMENT_STYLE[$ext]}" # Handle shebang lines — insert after shebang - if head -1 "$file" | grep -q "^#!"; then + if head -1 "$file" | grep >/dev/null "^#!"; then sed -i "1a\\${header}" "$file" else sed -i "1i\\${header}" "$file" diff --git a/scripts/fix-scripts/fix-pin-action-sha.sh b/scripts/fix-scripts/fix-pin-action-sha.sh index 928f0bc0..4c82739c 100755 --- a/scripts/fix-scripts/fix-pin-action-sha.sh +++ b/scripts/fix-scripts/fix-pin-action-sha.sh @@ -90,7 +90,7 @@ for wf in "${WORKFLOWS_DIR}"/*.yml "${WORKFLOWS_DIR}"/*.yaml; do [[ -z "$action_ref" ]] && continue # Already SHA-pinned (40 hex chars after @) — skip - if echo "$action_ref" | grep -qE '@[0-9a-f]{40}'; then + if echo "$action_ref" | grep -E >/dev/null '@[0-9a-f]{40}'; then continue fi diff --git a/scripts/fix-scripts/fix-pin-rust-deps.sh b/scripts/fix-scripts/fix-pin-rust-deps.sh index ee491849..d98eee46 100755 --- a/scripts/fix-scripts/fix-pin-rust-deps.sh +++ b/scripts/fix-scripts/fix-pin-rust-deps.sh @@ -27,7 +27,7 @@ is_binary_crate() { find "${REPO}" -name "main.rs" \ -not -path "*/.git/*" \ -not -path "*/target/*" \ - | grep -q . && return 0 + | grep >/dev/null . && return 0 fi return 1 } diff --git a/scripts/fix-scripts/fix-pmpl-drift.sh b/scripts/fix-scripts/fix-pmpl-drift.sh index d8858baa..0564fda9 100755 --- a/scripts/fix-scripts/fix-pmpl-drift.sh +++ b/scripts/fix-scripts/fix-pmpl-drift.sh @@ -55,13 +55,13 @@ fi LIC_HEAD=$(head -1 "$LICENSE_PATH" | tr -d '\n') -if echo "$LIC_HEAD" | grep -q "MPL-2.0"; then +if echo "$LIC_HEAD" | grep >/dev/null "MPL-2.0"; then TARGET="MPL-2.0" CATEGORY="sole-owner" -elif echo "$LIC_HEAD" | grep -q "AGPL-3.0"; then +elif echo "$LIC_HEAD" | grep >/dev/null "AGPL-3.0"; then TARGET="AGPL-3.0-or-later" CATEGORY="son-shared" -elif echo "$LIC_HEAD" | grep -q "PMPL-1.0-or-later"; then +elif echo "$LIC_HEAD" | grep >/dev/null "PMPL-1.0-or-later"; then echo "INFO: $REPO_NAME LICENSE is PMPL-1.0-or-later (carve-out or staging) — skipping" exit 0 else diff --git a/scripts/fix-scripts/fix-sast-workflow.sh b/scripts/fix-scripts/fix-sast-workflow.sh index 3ce48e5f..ed7959ba 100755 --- a/scripts/fix-scripts/fix-sast-workflow.sh +++ b/scripts/fix-scripts/fix-sast-workflow.sh @@ -22,11 +22,11 @@ mkdir -p "${REPO}/.github/workflows" # Detect languages for CodeQL matrix languages=() # JavaScript/TypeScript (includes ReScript compiled output) -if find "$REPO" -maxdepth 3 -name "*.js" -o -name "*.mjs" -o -name "*.res" 2>/dev/null | grep -q .; then +if find "$REPO" -maxdepth 3 -name "*.js" -o -name "*.mjs" -o -name "*.res" 2>/dev/null | grep >/dev/null .; then languages+=("javascript-typescript") fi # Ruby -if find "$REPO" -maxdepth 3 -name "*.rb" 2>/dev/null | grep -q .; then +if find "$REPO" -maxdepth 3 -name "*.rb" 2>/dev/null | grep >/dev/null .; then languages+=("ruby") fi diff --git a/scripts/fix-scripts/fix-scorecard-binary-artifacts.sh b/scripts/fix-scripts/fix-scorecard-binary-artifacts.sh index 2df0ddbf..1e43d396 100755 --- a/scripts/fix-scripts/fix-scorecard-binary-artifacts.sh +++ b/scripts/fix-scripts/fix-scorecard-binary-artifacts.sh @@ -23,7 +23,7 @@ count=0 echo "" while IFS= read -r -d '' file; do - if file "$file" | grep -qE 'ELF|PE32|Mach-O|compiled|bytecode|binary|data'; then + if file "$file" | grep -E >/dev/null 'ELF|PE32|Mach-O|compiled|bytecode|binary|data'; then echo "BINARY: ${file#"${REPO}/"}" count=$((count + 1)) fi diff --git a/scripts/fix-scripts/fix-scorecard-ci-tests.sh b/scripts/fix-scripts/fix-scorecard-ci-tests.sh index 2fe00061..7128da16 100755 --- a/scripts/fix-scripts/fix-scorecard-ci-tests.sh +++ b/scripts/fix-scripts/fix-scorecard-ci-tests.sh @@ -14,7 +14,7 @@ REPO="${1:?Usage: fix-scorecard-ci-tests.sh }" WF_DIR="${REPO}/.github/workflows" # Already has CI if any workflow runs tests -if ls "${WF_DIR}"/*.yml "${WF_DIR}"/*.yaml 2>/dev/null | xargs grep -l 'mix test\|cargo test\|deno test\|gleam test' 2>/dev/null | grep -q .; then +if ls "${WF_DIR}"/*.yml "${WF_DIR}"/*.yaml 2>/dev/null | xargs grep -l 'mix test\|cargo test\|deno test\|gleam test' 2>/dev/null | grep >/dev/null .; then echo "[fix-scorecard-ci-tests] CI test workflow already present" exit 0 fi diff --git a/scripts/gen-rules-praxis.sh b/scripts/gen-rules-praxis.sh index 6f00106c..48a0bcde 100755 --- a/scripts/gen-rules-praxis.sh +++ b/scripts/gen-rules-praxis.sh @@ -367,7 +367,7 @@ while IFS='~' read -r mod name f lit sev adv sf slit text; do grep -qF -- "$lit" "$f" || { echo "gen-rules-praxis: $f no longer has '$lit' ($mod/$name)" >&2; exit 1; } if [ "$sf" = "$f" ]; then # same file: beside the literal (a trailing #any: anywhere in it) n=$(grep -nF -- "$lit" "$f" | sed -n 1p | cut -d: -f1) - sed -n "$((n - 3)),$((n + 3))p" "$f" | grep -qF -- "$slit" || + sed -n "$((n - 3)),$((n + 3))p" "$f" | grep -F >/dev/null -- "$slit" || { echo "gen-rules-praxis: $f no longer states '$slit' beside '$lit' ($mod/$name)" >&2; exit 1; } else sf=${sf%#any} diff --git a/scripts/security-check.sh b/scripts/security-check.sh index 9b334237..f553aecf 100755 --- a/scripts/security-check.sh +++ b/scripts/security-check.sh @@ -504,7 +504,7 @@ check_spdx_headers() { # Check Rust files echo "## Rust Files (.rs)" >> "$report_file" while IFS= read -r -d '' f; do - if ! head -10 "$f" | grep -q "SPDX-License-Identifier"; then + if ! head -10 "$f" | grep >/dev/null "SPDX-License-Identifier"; then echo " - $f" >> "$report_file" missing_count=$((missing_count + 1)) fi @@ -514,7 +514,7 @@ check_spdx_headers() { echo "" >> "$report_file" echo "## Haskell Files (.hs)" >> "$report_file" while IFS= read -r -d '' f; do - if ! head -10 "$f" | grep -q "SPDX-License-Identifier"; then + if ! head -10 "$f" | grep >/dev/null "SPDX-License-Identifier"; then echo " - $f" >> "$report_file" missing_count=$((missing_count + 1)) fi @@ -524,7 +524,7 @@ check_spdx_headers() { echo "" >> "$report_file" echo "## Shell Scripts (.sh)" >> "$report_file" while IFS= read -r -d '' f; do - if ! head -10 "$f" | grep -q "SPDX-License-Identifier"; then + if ! head -10 "$f" | grep >/dev/null "SPDX-License-Identifier"; then echo " - $f" >> "$report_file" missing_count=$((missing_count + 1)) fi @@ -534,7 +534,7 @@ check_spdx_headers() { echo "" >> "$report_file" echo "## GitHub Workflows (.yml)" >> "$report_file" while IFS= read -r -d '' f; do - if ! head -1 "$f" | grep -q "SPDX-License-Identifier"; then + if ! head -1 "$f" | grep >/dev/null "SPDX-License-Identifier"; then echo " - $f" >> "$report_file" missing_count=$((missing_count + 1)) fi diff --git a/scripts/standards-integration/check-standards-integration.sh b/scripts/standards-integration/check-standards-integration.sh index cc445b4d..e79059a1 100755 --- a/scripts/standards-integration/check-standards-integration.sh +++ b/scripts/standards-integration/check-standards-integration.sh @@ -74,7 +74,7 @@ for f in "${CALLERS[@]}"; do [ -n "$uses_lines" ] || fail "$f declares no metadatastician/standards reusable" while IFS= read -r line; do ref=$(printf '%s' "$line" | sed -E 's/.*uses:[[:space:]]*//; s/[[:space:]].*$//') - if ! printf '%s' "$ref" | grep -qE '^metadatastician/standards/\.github/workflows/[A-Za-z0-9._-]+\.yml@[0-9a-f]{40}$'; then + if ! printf '%s' "$ref" | grep -E >/dev/null '^metadatastician/standards/\.github/workflows/[A-Za-z0-9._-]+\.yml@[0-9a-f]{40}$'; then fail "$f has a non-immutable standards ref (must be a full 40-hex SHA): $ref" fi pins+=("${ref##*@}") @@ -97,7 +97,7 @@ for f in "${CALLERS[@]}"; do # Every pin used by this caller (normally exactly one) must be locked. while IFS= read -r line; do ref=$(printf '%s' "$line" | sed -E 's/.*uses:[[:space:]]*//; s/[[:space:]].*$//') - printf '%s\n' "$block" | grep -qF "metadatastician/standards@${ref##*@}" \ + printf '%s\n' "$block" | grep -F >/dev/null "metadatastician/standards@${ref##*@}" \ || fail "actions.lock workflows entry for $f does not record ${ref}" done </dev/null "commit: 'sha1-$PIN'" \ || fail "actions.lock dependency for standards@$PIN does not resolve to sha1-$PIN" # --- Lock transitive closure: every workflow ref and every transitive @@ -155,7 +155,7 @@ for f in governance.yml hypatia-scan.yml; do [ -n "$raw" ] || fail "$f does not pass hypatia-ref explicitly (scanner source would float)" value=$(printf '%s' "$raw" | sed -E 's/^[[:space:]]*hypatia-ref:[[:space:]]*//; s/[[:space:]]+#.*$//; s/[[:space:]]+$//') if [ "$value" != '${{ github.event.pull_request.base.sha || github.sha }}' ] \ - && ! printf '%s' "$value" | grep -qE '^[0-9a-f]{40}$'; then + && ! printf '%s' "$value" | grep -E >/dev/null '^[0-9a-f]{40}$'; then fail "$f hypatia-ref is neither the base/event SHA expression nor a 40-hex SHA: $value" fi done diff --git a/scripts/sweeps/estate-pin-integrity.sh b/scripts/sweeps/estate-pin-integrity.sh index e3a1b804..08d7c017 100755 --- a/scripts/sweeps/estate-pin-integrity.sh +++ b/scripts/sweeps/estate-pin-integrity.sh @@ -322,7 +322,7 @@ replace_site() { # # (`denylisted_pair`): `some/action@v4.38.1` is no finding. line_has_denied_token() { local tok t - if [ -n "$SHA_ALTERNATION" ] && printf '%s\n' "$1" | grep -qiE "(^|[^0-9A-Za-z])(${SHA_ALTERNATION})([^0-9A-Za-z]|\$)"; then + if [ -n "$SHA_ALTERNATION" ] && printf '%s\n' "$1" | grep -iE >/dev/null "(^|[^0-9A-Za-z])(${SHA_ALTERNATION})([^0-9A-Za-z]|\$)"; then return 0 fi [ ${#DENY_VERSIONS[@]} -gt 0 ] || return 1 @@ -399,7 +399,7 @@ relabel_comment() { local trimmed="${body#"$lead"}" # A bare claim (no `#`, no leading space) normalises to `# vX`, as relabel/2 does. [ "$body" = "$comment" ] && [ -z "$lead" ] && lead=" " - if printf '%s' "$trimmed" | grep -qE '^v?[0-9]+(\.[0-9]+)*([[:space:]]|$)'; then + if printf '%s' "$trimmed" | grep -E >/dev/null '^v?[0-9]+(\.[0-9]+)*([[:space:]]|$)'; then printf '%s%s' "#${lead}" \ "$(printf '%s' "$trimmed" | sed -E "0,/^v?[0-9]+(\.[0-9]+)*/s//v${version}/")" else diff --git a/scripts/sweeps/estate-pr-automerge.sh b/scripts/sweeps/estate-pr-automerge.sh index 078f64a2..d664bf22 100755 --- a/scripts/sweeps/estate-pr-automerge.sh +++ b/scripts/sweeps/estate-pr-automerge.sh @@ -296,7 +296,7 @@ decide_pr() { only_uses=1 while IFS= read -r cl; do [ -n "$cl" ] || continue - printf '%s' "$cl" | grep -qE '^[-+]\s*-?\s*uses:' || only_uses=0 + printf '%s' "$cl" | grep -E >/dev/null '^[-+]\s*-?\s*uses:' || only_uses=0 done <<<"$content_lines" [ "$only_uses" -eq 1 ] && [ -n "$content_lines" ] && pin_files=$((pin_files + 1)) [ "$only_uses" -eq 0 ] && nonpin_change=$((nonpin_change + 1)) @@ -426,7 +426,7 @@ decide_pr() { route="rhodibot"; pool="P2" elif [ "$pin_files" -eq 0 ] && [ "$lock_only" -eq 0 ] && jq -r '.filename' <<<"$patches" 2>/dev/null \ - | grep -qE '(^|/)(package\.json|Cargo\.toml|Project\.toml|mix\.exs|go\.mod|pyproject\.toml|composer\.json|Gemfile|requirements[^/]*\.txt)$'; then + | grep -E >/dev/null '(^|/)(package\.json|Cargo\.toml|Project\.toml|mix\.exs|go\.mod|pyproject\.toml|composer\.json|Gemfile|requirements[^/]*\.txt)$'; then change_class="bump"; disposition="flag"; safety="flag" blocked_by="dependency_manifest_not_lockfile" route="rhodibot"; pool="P2" diff --git a/scripts/sweeps/patch_phase2.sh b/scripts/sweeps/patch_phase2.sh index cd89fc3b..6aa5b908 100755 --- a/scripts/sweeps/patch_phase2.sh +++ b/scripts/sweeps/patch_phase2.sh @@ -40,7 +40,7 @@ END=$(awk -v s="$START" 'NR>s && /'"$NEXT_RE"'/ {print NR; exit}' "$WF" || true) [ -n "${END:-}" ] || END=$((TOTAL + 1)) # Idempotency: bail if the existing block already carries the fix. -if sed -n "${START},$((END - 1))p" "$WF" | grep -q '^ continue-on-error: true$'; then +if sed -n "${START},$((END - 1))p" "$WF" | grep >/dev/null '^ continue-on-error: true$'; then exit 2 fi diff --git a/test/arangodb_initialisation_test.exs b/test/arangodb_initialisation_test.exs new file mode 100644 index 00000000..4bf4563a --- /dev/null +++ b/test/arangodb_initialisation_test.exs @@ -0,0 +1,531 @@ +# SPDX-License-Identifier: MPL-2.0 + +defmodule Hypatia.Data.ArangoDBInitialisationTest do + use ExUnit.Case, async: false + import ExUnit.CaptureLog + alias Hypatia.Data.ArangoDB + + defmodule Stub do + def init(arg), do: arg + + # Every response carries `location: /redirect-target`, and that path + # answers a terminal 200 that would read as success, so a client that + # followed a redirect would pass a 3xx test. Any request to it is + # reported to the test process. + def call(%{request_path: "/redirect-target"} = conn, {parent, _respond}) do + send(parent, {:redirect_target, conn.method}) + + conn + |> Plug.Conn.put_resp_content_type("application/json") + |> Plug.Conn.send_resp(200, Jason.encode!(%{"error" => false, "result" => []})) + end + + def call(conn, {_parent, respond}) do + {:ok, _body, conn} = Plug.Conn.read_body(conn) + # A route the test does not describe answers 404 database-not-found + # (errorNum 1228), as ArangoDB does for an unknown database, so the + # probes of the target database fall through to `_system` and to + # creating it. + {status, body} = + try do + respond.(conn.method, conn.request_path) + rescue + FunctionClauseError -> {404, %{"error" => true, "errorNum" => 1228}} + end + + conn + |> Plug.Conn.put_resp_header("location", "/redirect-target") + |> Plug.Conn.put_resp_content_type("application/json") + |> Plug.Conn.send_resp(status, Jason.encode!(body)) + end + end + + setup do + previous = System.get_env("HYPATIA_ARANGO_URL") + + on_exit(fn -> + if previous, + do: System.put_env("HYPATIA_ARANGO_URL", previous), + else: System.delete_env("HYPATIA_ARANGO_URL") + end) + + :ok + end + + for status <- [301, 302, 303, 307, 308] do + test "POST rejects #{status} with valid JSON" do + url = serve(fn _, _ -> {unquote(status), %{"result" => []}} end) + + state = %ArangoDB{ + base_url: url, + database: "test", + auth_header: "Basic test-dummy", + connected: true + } + + assert {:reply, {:error, {unquote(status), _}}, ^state} = + ArangoDB.handle_call({:query, "RETURN 1", %{}}, nil, state) + + refute_received {:redirect_target, _} + end + end + + test "POST preserves decoded 4xx errors" do + body = %{"error" => true, "errorMessage" => "test denied"} + url = serve(fn _, _ -> {403, body} end) + + state = %ArangoDB{ + base_url: url, + database: "test", + auth_header: "Basic test-dummy", + connected: true + } + + assert {:reply, {:error, "test denied"}, ^state} = + ArangoDB.handle_call({:query, "RETURN 1", %{}}, nil, state) + end + + for path <- ["/_api/database", "/_api/collection", "/_api/index", "/_api/gharial"] do + test "redirect at #{path} leaves initialisation degraded" do + serve(fn + "GET", "/_api/version" -> + {200, %{"server" => "arango"}} + + "POST", path -> + if String.ends_with?(path, unquote(path)), + do: {307, %{"result" => true}}, + else: {201, %{"error" => false}} + end) + + log = + capture_log(fn -> + assert {:ok, state} = ArangoDB.init([]) + refute state.connected + assert state.sync_timer == nil + end) + + assert log =~ "schema initialisation failed" + refute log =~ "schema ensured" + refute log =~ "ArangoDB connected:" + refute_received {:redirect_target, _} + end + end + + for existing? <- [false, true] do + test "schema succeeds with existing resources: #{existing?}" do + serve(fn + "GET", "/_api/version" -> + {200, %{"server" => "arango"}} + + "POST", path -> + if unquote(existing?) do + number = if String.ends_with?(path, "/gharial"), do: 1925, else: 1207 + {409, %{"error" => true, "errorNum" => number}} + else + {201, %{"error" => false}} + end + + "GET", "/_db/hypatia/_api/gharial/hypatia_graph" -> + {200, graph(required_edge_definitions())} + end) + + capture_log(fn -> + assert {:ok, state} = ArangoDB.init([]) + assert state.connected + assert is_reference(state.sync_timer) + Process.cancel_timer(state.sync_timer) + end) + end + end + + test "schema API errors enter degraded mode" do + serve(fn + "GET", _ -> {200, %{"server" => "arango"}} + "POST", _ -> {401, %{"error" => true, "errorNum" => 11}} + end) + + capture_log(fn -> + assert {:ok, state} = ArangoDB.init([]) + refute state.connected + assert state.sync_timer == nil + end) + end + + # A gateway or proxy in front of ArangoDB can answer with a JSON 4xx that + # has no `"error": true`. Only a 2xx, or an existing-resource 4xx (1207, + # 1925), counts as success. + test "schema 4xx without the error flag enters degraded mode" do + serve(fn + "GET", _ -> {200, %{"server" => "arango"}} + "POST", _ -> {403, %{"message" => "forbidden"}} + end) + + log = + capture_log(fn -> + assert {:ok, state} = ArangoDB.init([]) + refute state.connected + assert state.sync_timer == nil + end) + + refute log =~ "schema ensured" + end + + # A user with Administrate on a pre-provisioned target database but not on + # `_system` cannot create databases. The client must see that the target + # exists and not ask `_system` to create it. + test "existing target database is not recreated through _system" do + parent = self() + + serve(fn + "GET", "/_api/version" -> + {200, %{"server" => "arango"}} + + "GET", "/_db/hypatia/_api/version" -> + {200, %{"server" => "arango"}} + + "GET", path -> + if String.ends_with?(path, "/_api/database/current"), + do: {200, %{"error" => false, "result" => %{"name" => "hypatia"}}}, + else: {404, %{"error" => true}} + + "POST", "/_api/database" -> + send(parent, :created_database) + {403, %{"error" => true, "errorNum" => 11}} + + "POST", _ -> + {201, %{"error" => false}} + end) + + capture_log(fn -> + assert {:ok, state} = ArangoDB.init([]) + assert state.connected + Process.cancel_timer(state.sync_timer) + end) + + refute_received :created_database + end + + # Without a readable target database the client still creates it, and a + # refused creation leaves it degraded. + test "absent target database is created, and a refusal degrades" do + parent = self() + + serve(fn + "GET", "/_api/version" -> + {200, %{"server" => "arango"}} + + "GET", _ -> + {404, %{"error" => true, "errorNum" => 1228}} + + "POST", "/_api/database" -> + send(parent, :created_database) + {403, %{"error" => true, "errorNum" => 11}} + + "POST", _ -> + {201, %{"error" => false}} + end) + + capture_log(fn -> + assert {:ok, state} = ArangoDB.init([]) + refute state.connected + end) + + assert_received :created_database + end + + # The connectivity probe asks the target database, so a user who + # administers only that database connects although `_system` refuses it. + test "connectivity is probed on the target database, not _system" do + parent = self() + + serve(fn + "GET", "/_db/hypatia/_api/version" -> + {200, %{"server" => "arango"}} + + "GET", "/_api/version" -> + send(parent, :probed_system) + {403, %{"error" => true, "errorNum" => 11}} + + "GET", "/_db/hypatia/_api/database/current" -> + {200, %{"error" => false, "result" => %{"name" => "hypatia"}}} + + "POST", _ -> + {201, %{"error" => false}} + end) + + capture_log(fn -> + assert {:ok, state} = ArangoDB.init([]) + assert state.connected + Process.cancel_timer(state.sync_timer) + end) + + refute_received :probed_system + end + + # Only a 404 database-not-found on the target falls back to `_system`; any + # other failure leaves the client disconnected. + test "a failing target probe other than 404 does not fall back to _system" do + parent = self() + + serve(fn + "GET", "/_db/hypatia/_api/version" -> + {503, %{"error" => true, "errorNum" => 503}} + + "GET", "/_api/version" -> + send(parent, :probed_system) + {200, %{"server" => "arango"}} + end) + + capture_log(fn -> + assert {:ok, state} = ArangoDB.init([]) + refute state.connected + cancel(state.setup_timer) + end) + + refute_received :probed_system + end + + # A failed probe of the target database is no evidence that it is absent: + # only a 404 with errorNum 1228 leads to creating it. + for status <- [307, 500, 503] do + test "a #{status} from the database probe fails setup without creating it" do + parent = self() + + serve(fn + "GET", "/_db/hypatia/_api/version" -> + {200, %{"server" => "arango"}} + + "GET", "/_db/hypatia/_api/database/current" -> + {unquote(status), %{"error" => true, "errorNum" => 1228}} + + "POST", path -> + send(parent, {:post, path}) + {201, %{"error" => false}} + end) + + log = + capture_log(fn -> + assert {:ok, state} = ArangoDB.init([]) + refute state.connected + cancel(state.setup_timer) + end) + + assert log =~ "schema initialisation failed" + refute_received {:post, _} + refute_received {:redirect_target, _} + end + end + + # When the database step fails, none of the collection, index or graph + # requests is sent. + test "a failed database step stops setup before the dependent requests" do + parent = self() + + serve(fn + "GET", "/_api/version" -> + {200, %{"server" => "arango"}} + + "POST", "/_api/database" -> + {403, %{"error" => true, "errorNum" => 11}} + + "POST", path -> + send(parent, {:post, path}) + {201, %{"error" => false}} + end) + + capture_log(fn -> + assert {:ok, state} = ArangoDB.init([]) + refute state.connected + cancel(state.setup_timer) + end) + + refute_received {:post, _} + end + + # A duplicate graph (1925) is accepted only when the existing graph has + # the required edge definitions; order within from/to and extra edge + # definitions do not matter. + test "an existing graph with the required edge definitions is accepted" do + existing = + [%{"collection" => "extra", "from" => ["repos"], "to" => ["bots"]}] ++ + Enum.map(required_edge_definitions(), fn d -> + %{d | "from" => Enum.reverse(d["from"]), "to" => Enum.reverse(d["to"])} + end) + + serve_existing_graph(existing) + + capture_log(fn -> + assert {:ok, state} = ArangoDB.init([]) + assert state.connected + Process.cancel_timer(state.sync_timer) + end) + + assert_received {:graph_write, "POST"} + refute_received {:graph_write, _} + end + + for {name, change} <- [ + {"different vertex collections", + quote( + do: fn d -> + if d["collection"] == "dispatches", do: %{d | "to" => ["repos"]}, else: d + end + )}, + {"a missing edge collection", + quote(do: fn d -> if d["collection"] == "trusts", do: nil, else: d end)} + ] do + test "an existing graph with #{name} is a schema failure, not replaced" do + existing = + required_edge_definitions() + |> Enum.map(unquote(change)) + |> Enum.reject(&is_nil/1) + + serve_existing_graph(existing) + + log = + capture_log(fn -> + assert {:ok, state} = ArangoDB.init([]) + refute state.connected + assert state.sync_timer == nil + cancel(state.setup_timer) + end) + + assert log =~ "graph_mismatch" + refute log =~ "schema ensured" + assert_received {:graph_write, "POST"} + refute_received {:graph_write, _} + end + end + + # A transient failure during setup leaves the client disconnected, with a + # retry scheduled and no sync timer; the retry accepts what the first + # attempt created (1207) and then connects and starts syncing. + test "a transient schema failure is retried until setup succeeds" do + put_retry_env(10, 40) + first = :atomics.new(1, []) + + serve(fn + "GET", "/_api/version" -> + {200, %{"server" => "arango"}} + + "GET", "/_db/hypatia/_api/database/current" -> + {200, %{"error" => false, "result" => %{"name" => "hypatia"}}} + + "POST", "/_db/hypatia/_api/collection" -> + if :atomics.exchange(first, 1, 1) == 0, + do: {503, %{"error" => true, "errorNum" => 503}}, + else: {409, %{"error" => true, "errorNum" => 1207}} + + "POST", _ -> + {201, %{"error" => false}} + end) + + capture_log(fn -> + {:ok, pid} = GenServer.start(ArangoDB, []) + on_exit(fn -> if Process.alive?(pid), do: GenServer.stop(pid) end) + + assert %{connected: false, sync_timer: nil, setup_timer: retry} = :sys.get_state(pid) + assert is_reference(retry) + + assert eventually(fn -> GenServer.call(pid, :connected?) end) + assert %{sync_timer: sync, setup_timer: nil, setup_attempts: 0} = :sys.get_state(pid) + assert is_reference(sync) + end) + end + + test "an unreachable server keeps retrying with backoff and never syncs" do + put_retry_env(5, 20) + serve(fn "GET", _ -> {503, %{"error" => true}} end) + + capture_log(fn -> + {:ok, pid} = GenServer.start(ArangoDB, []) + on_exit(fn -> if Process.alive?(pid), do: GenServer.stop(pid) end) + + assert eventually(fn -> :sys.get_state(pid).setup_attempts >= 4 end) + assert %{connected: false, sync_timer: nil} = :sys.get_state(pid) + end) + end + + defp serve_existing_graph(existing) do + parent = self() + + serve(fn + "GET", "/_api/version" -> + {200, %{"server" => "arango"}} + + "GET", "/_db/hypatia/_api/gharial/hypatia_graph" -> + {200, graph(existing)} + + method, "/_db/hypatia/_api/gharial" <> _ when method != "GET" -> + send(parent, {:graph_write, method}) + {409, %{"error" => true, "errorNum" => 1925}} + + "POST", _ -> + {201, %{"error" => false}} + end) + end + + defp graph(edge_definitions) do + %{ + "error" => false, + "code" => 200, + "graph" => %{ + "name" => "hypatia_graph", + "edgeDefinitions" => edge_definitions, + "orphanCollections" => [] + } + } + end + + defp required_edge_definitions do + vertices = ["repos", "bots", "recipes", "contributors"] + + [ + %{"collection" => "trusts", "from" => vertices, "to" => vertices}, + %{"collection" => "depends_on", "from" => ["repos"], "to" => ["repos"]}, + %{"collection" => "dispatches", "from" => ["findings"], "to" => ["bots"]}, + %{"collection" => "applies_to", "from" => ["recipes"], "to" => ["patterns"]}, + %{"collection" => "repo_has_finding", "from" => ["repos"], "to" => ["findings"]}, + %{"collection" => "finding_matches_pattern", "from" => ["findings"], "to" => ["patterns"]}, + %{"collection" => "bot_executes_recipe", "from" => ["bots"], "to" => ["recipes"]}, + %{"collection" => "recipe_has_outcome", "from" => ["recipes"], "to" => ["outcomes"]}, + %{"collection" => "contributor_fixes", "from" => ["contributors"], "to" => ["outcomes"]} + ] + end + + defp put_retry_env(base, max) do + for {key, value} <- [arango_setup_retry_base_ms: base, arango_setup_retry_max_ms: max] do + previous = Application.fetch_env(:hypatia, key) + Application.put_env(:hypatia, key, value) + + on_exit(fn -> + case previous do + {:ok, old} -> Application.put_env(:hypatia, key, old) + :error -> Application.delete_env(:hypatia, key) + end + end) + end + end + + defp eventually(check, tries \\ 100) do + cond do + check.() -> true + tries == 0 -> false + true -> Process.sleep(10) && eventually(check, tries - 1) + end + end + + # init/1 called from a test schedules its retry on the test process. + defp cancel(timer) when is_reference(timer), do: Process.cancel_timer(timer) + defp cancel(_), do: :ok + + defp serve(respond) do + pid = + start_supervised!({Bandit, plug: {Stub, {self(), respond}}, ip: {127, 0, 0, 1}, port: 0}) + + {:ok, {_, port}} = ThousandIsland.listener_info(pid) + url = "http://127.0.0.1:#{port}" + System.put_env("HYPATIA_ARANGO_URL", url) + url + end +end diff --git a/test/arangodb_redirect_status_test.exs b/test/arangodb_redirect_status_test.exs new file mode 100644 index 00000000..e277880a --- /dev/null +++ b/test/arangodb_redirect_status_test.exs @@ -0,0 +1,106 @@ +# SPDX-License-Identifier: MPL-2.0 + +defmodule Hypatia.Data.ArangoDBRedirectStatusTest do + # Hypatia.Data.ArangoDB sends Basic auth and does not follow redirects, so a + # 3xx reaches its own status handling. A 3xx with a JSON body must be an + # error, never `{:ok, map}`: otherwise a query or upsert that did not run + # reads as a success. A loopback server stands in for ArangoDB. It answers + # the version probe and schema setup successfully, then query POSTs with + # 307 and the body `{}`. + use ExUnit.Case, async: false + + setup do + {:ok, listen} = + :gen_tcp.listen(0, [:binary, active: false, reuseaddr: true, ip: {127, 0, 0, 1}]) + + {:ok, port} = :inet.port(listen) + server = spawn_link(fn -> serve(listen) end) + previous = System.get_env("HYPATIA_ARANGO_URL") + System.put_env("HYPATIA_ARANGO_URL", "http://127.0.0.1:#{port}") + + on_exit(fn -> + if previous, + do: System.put_env("HYPATIA_ARANGO_URL", previous), + else: System.delete_env("HYPATIA_ARANGO_URL") + end) + + {:ok, pid} = GenServer.start(Hypatia.Data.ArangoDB, []) + + on_exit(fn -> + if Process.alive?(pid), do: GenServer.stop(pid) + Process.unlink(server) + Process.exit(server, :kill) + :gen_tcp.close(listen) + end) + + {:ok, pid: pid} + end + + test "the fake server is reached: the client reports connected", %{pid: pid} do + assert GenServer.call(pid, :connected?) + end + + test "a 307 with a JSON body from a query is an error", %{pid: pid} do + assert {:error, {307, "{}"}} = GenServer.call(pid, {:query, "RETURN 1", %{}}) + end + + defp serve(listen) do + case :gen_tcp.accept(listen, 10_000) do + {:ok, sock} -> + head = read_head(sock, "") + + reply = + cond do + String.starts_with?(head, "GET ") and String.contains?(head, "/_api/version ") -> + respond(200, ~s({"server":"arango","version":"3.12.0"}), "") + + String.starts_with?(head, "GET ") -> + respond(200, ~s({"error":false,"result":{"name":"hypatia"}}), "") + + String.starts_with?(head, "POST ") and String.contains?(head, "/_api/cursor ") -> + respond(307, "{}", "Location: http://127.0.0.1:1/elsewhere\r\n") + + true -> + respond(201, ~s({"error":false}), "") + end + + :gen_tcp.send(sock, reply) + :gen_tcp.close(sock) + serve(listen) + + {:error, _} -> + :ok + end + end + + defp respond(status, body, extra) do + "HTTP/1.1 #{status} X\r\nContent-Type: application/json\r\n" <> + extra <> "Content-Length: #{byte_size(body)}\r\nConnection: close\r\n\r\n" <> body + end + + # Reads the request head and any Content-Length body, and returns the head. + defp read_head(sock, acc) do + case :binary.match(acc, "\r\n\r\n") do + {pos, _} -> + head = binary_part(acc, 0, pos) + + want = + case Regex.run(~r/content-length:\s*(\d+)/i, head) do + [_, n] -> String.to_integer(n) + nil -> 0 + end + + if byte_size(acc) - pos - 4 >= want, do: head, else: more(sock, acc) + + :nomatch -> + more(sock, acc) + end + end + + defp more(sock, acc) do + case :gen_tcp.recv(sock, 0, 5_000) do + {:ok, data} -> read_head(sock, acc <> data) + {:error, _} -> acc + end + end +end diff --git a/test/arangodb_sync_test.exs b/test/arangodb_sync_test.exs new file mode 100644 index 00000000..271b43d4 --- /dev/null +++ b/test/arangodb_sync_test.exs @@ -0,0 +1,102 @@ +# SPDX-License-Identifier: MPL-2.0 + +defmodule Hypatia.Data.ArangoDBSyncTest do + # A sync run counts as complete only when every upsert succeeded. After a + # failed upsert the run is logged as failed and `last_sync` keeps the last + # successful timestamp. A Bandit stub stands in for ArangoDB's cursor API + # and fails the upserts into the collections a test names. + use ExUnit.Case, async: false + import ExUnit.CaptureLog + alias Hypatia.Data.ArangoDB + + defmodule Stub do + def init(failing), do: failing + + def call(conn, failing) do + {:ok, body, conn} = Plug.Conn.read_body(conn) + collection = get_in(Jason.decode!(body), ["bindVars", "@collection"]) + + {status, reply} = + if collection in failing, + do: {500, %{"error" => true, "errorNum" => 4}}, + else: {201, %{"error" => false, "result" => [%{"_key" => "k"}]}} + + conn + |> Plug.Conn.put_resp_content_type("application/json") + |> Plug.Conn.send_resp(status, Jason.encode!(reply)) + end + end + + @previous ~U[2026-01-01 00:00:00Z] + + setup do + dir = Path.join(System.tmp_dir!(), "arangodb-sync-#{System.unique_integer([:positive])}") + File.mkdir_p!(Path.join([dir, "scans", "repo-a"])) + File.mkdir_p!(Path.join(dir, "outcomes")) + File.mkdir_p!(Path.join(dir, "recipes")) + + File.write!( + Path.join([dir, "scans", "repo-a", "latest.json"]), + Jason.encode!(%{"findings" => [%{"rule" => "r1"}]}) + ) + + File.write!(Path.join([dir, "outcomes", "2026-01.jsonl"]), ~s({"recipe_id":"x"}\n)) + File.write!(Path.join([dir, "recipes", "x.json"]), ~s({"id":"x"})) + + previous = Application.get_env(:hypatia, :verisimdb_data_path) + Application.put_env(:hypatia, :verisimdb_data_path, dir) + + on_exit(fn -> + Application.put_env(:hypatia, :verisimdb_data_path, previous) + File.rm_rf!(dir) + end) + + :ok + end + + for failing <- [["repos"], ["findings"], ["outcomes"], ["recipes"]] do + test "a failed #{hd(failing)} upsert keeps last_sync (cast and timer)" do + state = state(unquote(failing)) + + log = + capture_log(fn -> + assert {:noreply, after_cast} = ArangoDB.handle_cast(:sync_from_verisimdb, state) + assert after_cast.last_sync == @previous + + assert {:noreply, after_timer} = ArangoDB.handle_info(:sync_from_verisimdb, state) + assert after_timer.last_sync == @previous + assert is_reference(after_timer.sync_timer) + Process.cancel_timer(after_timer.sync_timer) + end) + + assert log =~ "sync from verisim-data failed (1 failed upserts" + assert log =~ inspect(hd(unquote(failing))) + refute log =~ "sync from verisim-data complete" + end + end + + test "a run in which every upsert succeeds advances last_sync" do + state = state([]) + + log = + capture_log(fn -> + assert {:noreply, after_cast} = ArangoDB.handle_cast(:sync_from_verisimdb, state) + assert DateTime.compare(after_cast.last_sync, @previous) == :gt + end) + + assert log =~ "sync from verisim-data complete" + end + + defp state(failing) do + pid = start_supervised!({Bandit, plug: {Stub, failing}, ip: {127, 0, 0, 1}, port: 0}) + {:ok, {_, port}} = ThousandIsland.listener_info(pid) + + %ArangoDB{ + base_url: "http://127.0.0.1:#{port}", + database: "test", + auth_header: "Basic test-dummy", + connected: true, + last_sync: @previous + } + end +end diff --git a/test/ci-health-test.sh b/test/ci-health-test.sh index 4417dc34..368128fe 100755 --- a/test/ci-health-test.sh +++ b/test/ci-health-test.sh @@ -13,16 +13,16 @@ run_detect() { } lock_output=$(run_detect lockfile) -printf '%s\n' "$lock_output" | grep -q $'test-repo\tB-LOCKFILE\tHIGH\t' -printf '%s\n' "$lock_output" | grep -q '1 startup_failure conclusion(s) among the latest 3 run(s)' -if printf '%s\n' "$lock_output" | grep -q 'populate allow-list'; then +printf '%s\n' "$lock_output" | grep >/dev/null $'test-repo\tB-LOCKFILE\tHIGH\t' +printf '%s\n' "$lock_output" | grep >/dev/null '1 startup_failure conclusion(s) among the latest 3 run(s)' +if printf '%s\n' "$lock_output" | grep >/dev/null 'populate allow-list'; then echo "FAIL: unclassified startup failure prescribed an allow-list mutation" >&2 exit 1 fi saturated_output=$(run_detect saturated) -printf '%s\n' "$saturated_output" | grep -q '30 startup_failure conclusion(s) among the latest 30 run(s); page saturated at 30, older runs not measured' -if printf '%s\n' "$saturated_output" | grep -q $'\tB-LOCKFILE\t'; then +printf '%s\n' "$saturated_output" | grep >/dev/null '30 startup_failure conclusion(s) among the latest 30 run(s); page saturated at 30, older runs not measured' +if printf '%s\n' "$saturated_output" | grep >/dev/null $'\tB-LOCKFILE\t'; then echo "FAIL: saturated scenario was incorrectly classified as a lockfile failure" >&2 exit 1 fi @@ -67,20 +67,20 @@ for fixture in "$ROOT"/test/fixtures/ci-health/allowlist/*.json; do if [[ "$added" -eq 0 ]]; then [[ ! -s "$STATE_DIR/puts.log" ]] || { echo "FAIL [$name]: wrote although nothing was missing" >&2; exit 1; } - printf '%s\n' "$out" | grep -q '^SKIP ' || { echo "FAIL [$name]: expected SKIP, got: $out" >&2; exit 1; } + printf '%s\n' "$out" | grep >/dev/null '^SKIP ' || { echo "FAIL [$name]: expected SKIP, got: $out" >&2; exit 1; } else - printf '%s\n' "$out" | grep -q "^FIXED .* added $added, kept all $(jq '.expect.kept' "$fixture")" || + printf '%s\n' "$out" | grep >/dev/null "^FIXED .* added $added, kept all $(jq '.expect.kept' "$fixture")" || { echo "FAIL [$name]: unexpected report: $out" >&2; exit 1; } fi # Idempotent: a second run finds nothing missing and writes nothing. : > "$STATE_DIR/puts.log" - printf '%s\n' "$(remediate)" | grep -q '^SKIP ' || { echo "FAIL [$name]: second run was not a no-op" >&2; exit 1; } + printf '%s\n' "$(remediate)" | grep >/dev/null '^SKIP ' || { echo "FAIL [$name]: second run was not a no-op" >&2; exit 1; } [[ ! -s "$STATE_DIR/puts.log" ]] || { echo "FAIL [$name]: second run wrote" >&2; exit 1; } # Converges with detect.sh: B-ALLOWLIST no longer fires. if PATH="$MOCK_BIN:$PATH" OWNER=test-owner MOCK_STATE="$STATE_DIR/state.json" MOCK_SCENARIO=converge \ - "$ROOT/scripts/ci-health/detect.sh" test-repo 2>/dev/null | grep -q $'\tB-ALLOWLIST\t'; then + "$ROOT/scripts/ci-health/detect.sh" test-repo 2>/dev/null | grep >/dev/null $'\tB-ALLOWLIST\t'; then echo "FAIL [$name]: detect.sh still reports B-ALLOWLIST after remediation" >&2; exit 1 fi @@ -88,7 +88,7 @@ for fixture in "$ROOT"/test/fixtures/ci-health/allowlist/*.json; do want="-- corpus: $name added=$added kept=$(jq '.expect.kept' "$fixture") result=$(jq '.expect.result_size' "$fixture")" grep -qxF -- "$want" "$model" || { echo "FAIL [$name]: model has no line '$want'" >&2; exit 1; } eqn="= ($added, $(jq '.expect.kept' "$fixture"), $(jq '.expect.result_size' "$fixture"))" - grep -A1 -xF -- "$want" "$model" | tail -n1 | grep -qF -- "$eqn" || + grep -A1 -xF -- "$want" "$model" | tail -n1 | grep -F >/dev/null -- "$eqn" || { echo "FAIL [$name]: the equation under '$want' does not state $eqn" >&2; exit 1; } done [[ "$(grep -c '^-- corpus: ' "$model")" -eq "$(find "$ROOT/test/fixtures/ci-health/allowlist" -name '*.json' | wc -l)" ]] || diff --git a/test/fixtures/ci-health/mock-gh b/test/fixtures/ci-health/mock-gh index 6c47e4ab..d6d6322f 100755 --- a/test/fixtures/ci-health/mock-gh +++ b/test/fixtures/ci-health/mock-gh @@ -16,7 +16,7 @@ for i in "${!args[@]}"; do [ "${args[$i]}" = "--jq" ] && jq_filter="${args[$((i + 1))]}" done -if printf ' %s ' "$*" | grep -q ' -X PUT '; then +if printf ' %s ' "$*" | grep >/dev/null ' -X PUT '; then body=$(cat) if [ -n "${MOCK_STATE:-}" ] && [ "${MOCK_SCENARIO:-}" != "remediate-mismatch" ]; then printf '%s\n' "$body" > "$MOCK_STATE" diff --git a/test/httpc_redirect_credentials_test.exs b/test/httpc_redirect_credentials_test.exs index 861e72c7..763a51f3 100644 --- a/test/httpc_redirect_credentials_test.exs +++ b/test/httpc_redirect_credentials_test.exs @@ -2,56 +2,108 @@ defmodule Hypatia.HttpcRedirectCredentialsTest do # Guards the Erlang/OTP pin (mise.toml and the CI test lanes), not Hypatia's - # code. OTP releases before 27.3.4.13 make `:httpc` resend the - # `authorization` header to whatever host a redirect names; measured on - # 2026-11-09, 27.3.4.4 and 27.3.4.12 forwarded it to another port and to - # another host for 301, 302, 303, 307 and 308, GET and POST, while - # 27.3.4.13, 27.3.4.19 and 28.5.0.8 dropped it. The two echidnabot senders - # already set `autoredirect: false`; this test fails if the runtime itself - # regresses, so every other `:httpc` caller stays covered. + # code. Older OTP releases make `:httpc` resend credentials to whatever host + # a redirect names. Measured on loopback for 301, 302, 303, 307 and 308, GET + # and POST, to another port and to another host: + # + # * an explicit `authorization` header: forwarded by 27.3.4.4 and + # 27.3.4.12 (2026-11-09), dropped from 27.3.4.13; + # * credentials in the URL (`http://user:pass@host`, sent as Basic auth): + # still forwarded by 27.3.4.13, dropped by 27.3.4.19 and 28.5.0.8 + # (2026-11-17). + # + # So 27.3.4.19 is the floor on the 27 line. Credentialed callers in lib/ + # also set `autoredirect: false` (checked by the last test here), so they + # do not depend on the pin alone. # # Each case runs a redirector and a target on loopback. The redirector - # answers with STATUS and a Location on the target; the target reports the - # request line and any authorization header it receives. + # answers with STATUS and a Location on the target, and reports the + # authorization header it received, which must be the expected credential + # (otherwise an empty list at the target would prove nothing). The target + # reports the request line and any authorization header it receives. use ExUnit.Case, async: true @token "Bearer test-dummy-redirect-probe" + @userinfo "test-dummy-user:test-dummy-pass" setup_all do {:ok, _} = Application.ensure_all_started(:inets) :ok end - for status <- [301, 302, 303, 307, 308], method <- [:get, :post] do - test "#{method} #{status} to another origin drops the authorization header" do - {line, auth} = follow(unquote(method), unquote(status), :other_origin) + # :other_port keeps the host (127.0.0.1) and changes the port, so a runtime + # that compared hosts only would fail it. :other_host keeps the port and + # moves to 127.0.0.2, so a runtime that compared ports only would fail it. + for where <- [:other_port, :other_host], + status <- [301, 302, 303, 307, 308], + method <- [:get, :post] do + test "#{method} #{status} to #{where} drops the authorization header" do + {line, auth} = follow(unquote(method), unquote(status), unquote(where)) + assert line =~ "/final" + assert auth == [] + end + + test "#{method} #{status} to #{where} drops credentials from the URL" do + {line, auth} = follow(unquote(method), unquote(status), unquote(where), :userinfo) assert line =~ "/final" assert auth == [] end end - # Positive control: the target does see the header when the redirect stays - # on the same origin, so an empty list above is the runtime dropping it, not - # the harness failing to observe it. + # Positive controls: the target does see the credentials when the redirect + # stays on the same origin, so an empty list above is the runtime dropping + # them, not the harness failing to observe them. test "same-origin redirect keeps the authorization header" do {_line, auth} = follow(:get, 302, :same_origin) assert auth == ["authorization: #{@token}"] end - defp follow(method, status, where) do - {:ok, front} = listen() - {:ok, back} = listen() + test "same-origin redirect keeps credentials from the URL" do + {_line, auth} = follow(:get, 302, :same_origin, :userinfo) + assert auth == ["authorization: Basic " <> Base.encode64(@userinfo)] + end + + # Every lib/ module that sends an Authorization header must turn off + # automatic redirects on each :httpc request, so a credential never depends + # on the runtime pin alone (for example on the OTP 28.3 CI lanes). + test "credentialed :httpc callers in lib/ disable automatic redirects" do + offenders = + Path.wildcard(Path.expand("../lib/**/*.ex", __DIR__)) + |> Enum.filter(fn path -> + src = File.read!(path) + String.match?(src, ~r/authorization/i) and String.contains?(src, ":httpc.request") + end) + |> Enum.reject(fn path -> + src = File.read!(path) + calls = length(String.split(src, ":httpc.request")) - 1 + guarded = length(String.split(src, "{:autoredirect, false}")) - 1 + guarded >= calls + end) + |> Enum.map(&Path.relative_to_cwd/1) + + assert offenders == [] + end + + defp follow(method, status, where, cred \\ :header) do + {:ok, front} = listen({127, 0, 0, 1}) {:ok, front_port} = :inet.port(front) + + {:ok, back} = + if where == :other_host, + do: listen({127, 0, 0, 2}, front_port), + else: listen({127, 0, 0, 1}) + {:ok, back_port} = :inet.port(back) target = if where == :same_origin, do: front, else: back target_port = if where == :same_origin, do: front_port, else: back_port - location = "http://127.0.0.1:#{target_port}/final" + target_host = if where == :other_host, do: "127.0.0.2", else: "127.0.0.1" + location = "http://#{target_host}:#{target_port}/final" parent = self() server = spawn_link(fn -> {:ok, s1} = :gen_tcp.accept(front, 5_000) - _ = read_request(s1) + send(parent, {:start, s1 |> read_request() |> authorization()}) :ok = :gen_tcp.send( @@ -62,15 +114,22 @@ defmodule Hypatia.HttpcRedirectCredentialsTest do :gen_tcp.close(s1) {:ok, s2} = :gen_tcp.accept(target, 5_000) - [line | headers] = s2 |> read_request() |> String.split("\r\n") - auth = Enum.filter(headers, &String.starts_with?(String.downcase(&1), "authorization:")) - send(parent, {:seen, line, auth}) + request = read_request(s2) + [line | _] = String.split(request, "\r\n") + send(parent, {:seen, line, authorization(request)}) :gen_tcp.send(s2, "HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") :gen_tcp.close(s2) end) - url = ~c"http://127.0.0.1:#{front_port}/start" - headers = [{~c"authorization", String.to_charlist(@token)}] + {url, headers} = + case cred do + :header -> + {~c"http://127.0.0.1:#{front_port}/start", + [{~c"authorization", String.to_charlist(@token)}]} + + :userinfo -> + {~c"http://#{@userinfo}@127.0.0.1:#{front_port}/start", []} + end request = case method do @@ -81,14 +140,28 @@ defmodule Hypatia.HttpcRedirectCredentialsTest do {:ok, {{_, 200, _}, _, _}} = :httpc.request(method, request, [timeout: 5_000, autoredirect: true], []) + # The probe is valid only if the credentials did reach the redirector. + assert_receive {:start, start_auth}, 5_000 + assert start_auth == [expected_authorization(cred)] assert_receive {:seen, line, auth}, 5_000 Process.unlink(server) Enum.each([front, back], &:gen_tcp.close/1) {line, auth} end - defp listen do - :gen_tcp.listen(0, [:binary, active: false, reuseaddr: true, ip: {127, 0, 0, 1}]) + defp expected_authorization(:header), do: "authorization: #{@token}" + defp expected_authorization(:userinfo), do: "authorization: Basic " <> Base.encode64(@userinfo) + + # The authorization header lines of a request head. + defp authorization(request) do + request + |> String.split("\r\n") + |> tl() + |> Enum.filter(&String.starts_with?(String.downcase(&1), "authorization:")) + end + + defp listen(ip, port \\ 0) do + :gen_tcp.listen(port, [:binary, active: false, reuseaddr: true, ip: ip]) end # Reads up to the end of the headers, then any Content-Length body. diff --git a/test/reviewer_redirect_test.exs b/test/reviewer_redirect_test.exs new file mode 100644 index 00000000..3d661e7c --- /dev/null +++ b/test/reviewer_redirect_test.exs @@ -0,0 +1,135 @@ +# SPDX-License-Identifier: MPL-2.0 + +defmodule Hypatia.ReviewerRedirectTest do + # Hypatia.Reviewer sends GITHUB_TOKEN, so :httpc's own redirect handling is + # off. It follows a redirect itself only to the origin of the first + # request (GitHub's renamed-repository 301 stays on api.github.com), at + # most three times. + use ExUnit.Case, async: true + alias Hypatia.Reviewer + + @headers [{~c"authorization", ~c"token test-dummy-reviewer"}] + + defmodule Stub do + def init(respond), do: respond + + def call(conn, {name, parent, respond}) do + auth = Plug.Conn.get_req_header(conn, "authorization") + send(parent, {:hit, name, conn.request_path, auth}) + + {status, location, body} = + if is_function(respond, 2), + do: respond.(conn.request_path, conn.port), + else: respond.(conn.request_path) + + conn + |> then(fn c -> + if location, do: Plug.Conn.put_resp_header(c, "location", location), else: c + end) + |> Plug.Conn.send_resp(status, body) + end + end + + setup_all do + {:ok, _} = Application.ensure_all_started(:inets) + :ok + end + + test "a same-origin redirect is followed with the token" do + a = + serve(:a, fn + "/start" -> {301, "/repositories/1/pulls/2", ""} + "/repositories/1/pulls/2" -> {200, nil, "diff --git a/x b/x"} + end) + + assert {:ok, "diff --git a/x b/x"} = Reviewer.get_same_origin("#{a}/start", @headers) + assert_received {:hit, :a, "/repositories/1/pulls/2", ["token test-dummy-reviewer"]} + end + + test "a redirect to another origin is refused and never contacted" do + b = serve(:b, fn _ -> {200, nil, "should not be reached"} end) + a = serve(:a, fn "/start" -> {301, "#{b}/final", ""} end) + + assert {:error, {:http_status, 301}} = Reviewer.get_same_origin("#{a}/start", @headers) + refute_received {:hit, :b, _, _} + end + + test "a redirect loop stops after three hops" do + a = serve(:a, fn "/loop" -> {302, "/loop", ""} end) + + assert {:error, {:http_status, 302}} = Reviewer.get_same_origin("#{a}/loop", @headers) + + hits = for _ <- 1..4, do: assert_received({:hit, :a, "/loop", _}) + assert length(hits) == 4 + refute_received {:hit, :a, _, _} + end + + # Only the scheme changes (same host, same explicit port): a downgrade or + # upgrade is another origin, so the 301 is returned without a second request. + test "a redirect that changes only the scheme is refused" do + a = serve(:a, fn "/start", port -> {301, "https://127.0.0.1:#{port}/final", ""} end) + + assert {:error, {:http_status, 301}} = Reviewer.get_same_origin("#{a}/start", @headers) + assert_received {:hit, :a, "/start", _} + refute_received {:hit, :a, _, _} + end + + test "host names compare case-insensitively" do + a = + serve(:a, fn + "/start", port -> {301, "http://LocalHost:#{port}/final", ""} + "/final", _ -> {200, nil, "diff"} + end) + + url = String.replace(a, "127.0.0.1", "localhost") + assert {:ok, "diff"} = Reviewer.get_same_origin("#{url}/start", @headers) + assert_received {:hit, :a, "/final", ["token test-dummy-reviewer"]} + end + + test "a same-origin Location carrying userinfo is refused" do + a = + serve(:a, fn + "/start", port -> + {302, "http://test-dummy-user:test-dummy-pass@127.0.0.1:#{port}/final", ""} + + "/final", _ -> + {200, nil, "should not be reached"} + end) + + assert {:error, {:http_status, 302}} = Reviewer.get_same_origin("#{a}/start", @headers) + assert_received {:hit, :a, "/start", _} + refute_received {:hit, :a, _, _} + end + + # Each hop takes 150 ms and fits a 400 ms budget on its own; the chain of + # four requests does not, so the budget must span the redirects. + test "the timeout covers the whole redirect chain" do + a = + serve(:a, fn path -> + Process.sleep(150) + + case path do + "/1" -> {302, "/2", ""} + "/2" -> {302, "/3", ""} + "/3" -> {302, "/4", ""} + "/4" -> {200, nil, "late"} + end + end) + + assert {:error, {:http_error, :timeout}} = + Reviewer.get_same_origin("#{a}/1", @headers, 3, 400) + + assert {:ok, "late"} = Reviewer.get_same_origin("#{a}/1", @headers, 3, 5_000) + end + + defp serve(name, respond) do + pid = + start_supervised!( + {Bandit, plug: {Stub, {name, self(), respond}}, ip: {127, 0, 0, 1}, port: 0}, + id: name + ) + + {:ok, {_, port}} = ThousandIsland.listener_info(pid) + "http://127.0.0.1:#{port}" + end +end diff --git a/test/standards-integration-test.sh b/test/standards-integration-test.sh index 4d914573..171b8874 100755 --- a/test/standards-integration-test.sh +++ b/test/standards-integration-test.sh @@ -17,7 +17,7 @@ expect_outcome() { # name expected_exit expected_outcome root extra... local name="$1" want_rc="$2" want_outcome="$3"; shift 3 local out rc=0 out=$("$CHECK" "$@" 2>&1) || rc=$? - if [ "$rc" != "$want_rc" ] || ! printf '%s' "$out" | grep -q "outcome=$want_outcome"; then + if [ "$rc" != "$want_rc" ] || ! printf '%s' "$out" | grep >/dev/null "outcome=$want_outcome"; then echo "FAIL($name): want rc=$want_rc outcome=$want_outcome, got rc=$rc: $out" >&2 exit 1 fi @@ -91,11 +91,11 @@ for t in bash grep sed awk sort head env; do done rc=0 out=$(PATH="$TMP/nogh" "$TMP/nogh/bash" "$CHECK" --root "$ROOT" --verify-remote 2>&1) || rc=$? -if [ "$rc" != "3" ] || ! printf '%s' "$out" | grep -q 'outcome=unavailable'; then +if [ "$rc" != "3" ] || ! printf '%s' "$out" | grep >/dev/null 'outcome=unavailable'; then echo "FAIL(unavailable-tool): want rc=3 outcome=unavailable, got rc=$rc: $out" >&2 exit 1 fi -if printf '%s' "$out" | grep -q 'outcome=pass'; then +if printf '%s' "$out" | grep >/dev/null 'outcome=pass'; then echo "FAIL(unavailable-tool): unavailable was reported as pass" >&2 exit 1 fi @@ -109,7 +109,7 @@ for t in bash grep sed awk head env; do done rc=0 out=$(PATH="$TMP/nosort" "$TMP/nosort/bash" "$CHECK" --root "$ROOT" 2>&1) || rc=$? -if [ "$rc" != "3" ] || ! printf '%s' "$out" | grep -q 'outcome=unavailable'; then +if [ "$rc" != "3" ] || ! printf '%s' "$out" | grep >/dev/null 'outcome=unavailable'; then echo "FAIL(unavailable-sort): want rc=3 outcome=unavailable, got rc=$rc: $out" >&2 exit 1 fi diff --git a/tests/e2e.sh b/tests/e2e.sh index 6c4d59f9..a07ecc6c 100644 --- a/tests/e2e.sh +++ b/tests/e2e.sh @@ -233,7 +233,7 @@ fi # Check all documented rules have corresponding modules for rule in root_hygiene honest_completion workflow_audit cicd_rules code_safety; do - if find "$PROJECT_DIR/lib" -name "*${rule}*" -o -name "*$(echo $rule | tr '_' '-')*" 2>/dev/null | grep -q .; then + if find "$PROJECT_DIR/lib" -name "*${rule}*" -o -name "*$(echo $rule | tr '_' '-')*" 2>/dev/null | grep >/dev/null .; then pass "Rule module: $rule" else skip_test "Rule module: $rule" "file not found" @@ -250,7 +250,7 @@ bold "Section 4: Fixture scan detection" # The fixture has deliberately insecure patterns — scanner should find them if $HAS_RUST && [ -f "$HYPER_BIN" ]; then FINDINGS=$("$HYPER_BIN" scan "$FIXTURE_DIR" 2>/dev/null || true) - if echo "$FINDINGS" | grep -qi "unpinned\|injection\|insecure\|http\|eval\|finding\|weak\|issue" 2>/dev/null; then + if echo "$FINDINGS" | grep -i >/dev/null "unpinned\|injection\|insecure\|http\|eval\|finding\|weak\|issue" 2>/dev/null; then pass "Scanner detects issues in fixture repo" else skip_test "Fixture detection" "scanner may not produce text findings" diff --git a/tests/rules-praxis-test.sh b/tests/rules-praxis-test.sh index ffac5cc5..c1bb6dbf 100755 --- a/tests/rules-praxis-test.sh +++ b/tests/rules-praxis-test.sh @@ -86,18 +86,18 @@ expect_broken "an un-retired tombstone is refused" "$work/tomb.deed" "tombstone # provenance table is not a severity table, `severity: :advisory` is advisory, # and a rule that fires at several severities keeps the highest. row() { awk -v t="\"$1\"" '$1 == ":id" { on = ($2 == t) } on { print }' "$deed"; } # one rule-def -if row hypatia/branch_protection/BP001 | grep -q ':severity high' && row hypatia/supply_chain/SC001 | grep -q ':status active'; then ok "opt-in scanner rules are registered"; else bad "BP001/SC001 missing or wrong"; fi -if row hypatia/git_state/GS007 | grep -q ':class advisory'; then ok "an advisory-severity rule is class advisory"; else bad "GS007 is not class advisory"; fi -if row hypatia/code_scanning_alerts/CSA006 | grep -q ':severity medium'; then ok "a multi-severity rule keeps its highest severity"; else bad "CSA006 is not medium"; fi +if row hypatia/branch_protection/BP001 | grep >/dev/null ':severity high' && row hypatia/supply_chain/SC001 | grep >/dev/null ':status active'; then ok "opt-in scanner rules are registered"; else bad "BP001/SC001 missing or wrong"; fi +if row hypatia/git_state/GS007 | grep >/dev/null ':class advisory'; then ok "an advisory-severity rule is class advisory"; else bad "GS007 is not class advisory"; fi +if row hypatia/code_scanning_alerts/CSA006 | grep >/dev/null ':severity medium'; then ok "a multi-severity rule keeps its highest severity"; else bad "CSA006 is not medium"; fi -if row hypatia/pin_integrity/PI005 | grep -q ':class advisory' && row hypatia/disambiguation_rules/DR001 | grep -q ':class advisory'; then ok "rules the code calls advisory in words are class advisory"; else bad "PI005 or DR001 is not class advisory"; fi +if row hypatia/pin_integrity/PI005 | grep >/dev/null ':class advisory' && row hypatia/disambiguation_rules/DR001 | grep >/dev/null ':class advisory'; then ok "rules the code calls advisory in words are class advisory"; else bad "PI005 or DR001 is not class advisory"; fi -if row hypatia/workflow_audit/duplicate_workflow | grep -q ':severity minor' && row hypatia/cicd_rules/waste_duplicate_workflow | grep -q ':severity medium'; then ok "named atom rules carry the severity the code states"; else bad "duplicate_workflow rows lack their severity"; fi +if row hypatia/workflow_audit/duplicate_workflow | grep >/dev/null ':severity minor' && row hypatia/cicd_rules/waste_duplicate_workflow | grep >/dev/null ':severity medium'; then ok "named atom rules carry the severity the code states"; else bad "duplicate_workflow rows lack their severity"; fi # Severities the code computes (an if expression, a helper): the highest branch. # A severity read from run-time alert data (CSA001) has none to record. -if row hypatia/supply_chain/SC012 | grep -q ':severity high' && row hypatia/structural_drift/SD008 | grep -q ':severity critical' \ - && row hypatia/research_extensions/RE001 | grep -q ':severity high' && ! row hypatia/code_scanning_alerts/CSA001 | grep -q ':severity'; then +if row hypatia/supply_chain/SC012 | grep >/dev/null ':severity high' && row hypatia/structural_drift/SD008 | grep >/dev/null ':severity critical' \ + && row hypatia/research_extensions/RE001 | grep >/dev/null ':severity high' && ! row hypatia/code_scanning_alerts/CSA001 | grep >/dev/null ':severity'; then ok "computed severities are read; run-time severities stay empty"; else bad "computed severity rows are wrong"; fi # The names findings actually carry resolve (A5 R09). Ground truth: every @@ -140,7 +140,7 @@ done < <(awk ' ' lib/hypatia/cli.ex lib/rules/*.ex | sort -u) if [ -z "$missing_pairs" ]; then ok "every literal rule_module/type pair in the code is in the registry"; else bad "pairs in the code but not the registry:$missing_pairs"; fi if ! grep -qF '"hypatia/workflow_audit/always_after_cancel"' "$deed" && grep -qF '"hypatia/ci_failure_triage/always_after_cancel"' "$deed"; then ok "a name moved by a rule_id override is registered under its real module only"; else bad "always_after_cancel is registered under workflow_audit, or missing under ci_failure_triage"; fi -if row hypatia/structured_format/SF001 | grep -q ':class advisory' && row hypatia/cicd_rules/banned_language_file | grep -q ':severity critical' && row hypatia/implementation_inside_canon/HYP-S009 | grep -q ':status active' && row hypatia/workflow_hardening/invalid_actions_lock | grep -q ':status active'; then ok "fixed names carry the code's severity and advisory flag; HYP-S009 and invalid_actions_lock are registered"; else bad "a fixed-name row is wrong"; fi +if row hypatia/structured_format/SF001 | grep >/dev/null ':class advisory' && row hypatia/cicd_rules/banned_language_file | grep >/dev/null ':severity critical' && row hypatia/implementation_inside_canon/HYP-S009 | grep >/dev/null ':status active' && row hypatia/workflow_hardening/invalid_actions_lock | grep >/dev/null ':status active'; then ok "fixed names carry the code's severity and advisory flag; HYP-S009 and invalid_actions_lock are registered"; else bad "a fixed-name row is wrong"; fi # Portability: gawk and mawk differ (a `\[` in a -v value is valid in one and a # fatal regex error in the other, and the awk then dies without a word inside a