From a3bf1a1028d2124a9d0ec3ce10c805c8771545d8 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 00:17:22 +0000 Subject: [PATCH 01/12] fix(http): credentialed callers never follow redirects; URL-credential floor Follow-up to #1035, answering its two Codex findings. - reviewer.ex (GITHUB_TOKEN) and arangodb.ex (Basic auth, GET and POST) now pass autoredirect: false, like the two echidnabot senders. A credential no longer depends on the runtime pin alone. A redirect becomes an HTTP-status error, which both already handle. - The mise.toml comment claimed 27.3.4.13 as the floor. Measured on 2026-11-17, 27.3.4.13 still forwards credentials given in the URL (http://user:pass@host, sent as Basic auth) to another origin in all 20 cases; 27.3.4.19 and 28.5.0.8 forward none. The pin was already 27.3.4.19; the comment now says why. - The redirect test adds the 10 URL-credential cases and a same-origin control, plus a source check: any lib/ module that sends an Authorization header must set autoredirect: false on every :httpc.request. On OTP 27.3.4.19: 23 tests in the file pass. On 27.3.4.13 the 10 URL-credential cases fail. With the guard removed from reviewer.ex the source check names that file. Full suite on Elixir 1.19.4 / OTP 27.3.4.19: 3599 tests, 0 failures, 3 skipped, 246 excluded. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01J4QvDgzT72Tq2HTJmikyNg --- lib/data/arangodb.ex | 9 ++- lib/hypatia/reviewer.ex | 7 ++- mise.toml | 2 +- test/httpc_redirect_credentials_test.exs | 74 +++++++++++++++++++----- 4 files changed, 75 insertions(+), 17 deletions(-) diff --git a/lib/data/arangodb.ex b/lib/data/arangodb.ex index 5622cdb9..fb6beb33 100644 --- a/lib/data/arangodb.ex +++ b/lib/data/arangodb.ex @@ -581,7 +581,12 @@ defmodule Hypatia.Data.ArangoDB do {~c"Accept", ~c"application/json"} ] - case :httpc.request(:get, {String.to_charlist(url), headers}, [{:timeout, 10_000}], []) do + case :httpc.request( + :get, + {String.to_charlist(url), headers}, + [{:timeout, 10_000}, {:autoredirect, false}], + [] + ) do {:ok, {{_, 200, _}, _, body}} -> Jason.decode(to_string(body)) {:ok, {{_, status, _}, _, body}} -> {:error, {status, to_string(body)}} {:error, reason} -> {:error, reason} @@ -602,7 +607,7 @@ defmodule Hypatia.Data.ArangoDB do case :httpc.request( :post, {String.to_charlist(url), headers, ~c"application/json", json_body}, - [{:timeout, 10_000}], + [{:timeout, 10_000}, {:autoredirect, false}], [] ) do {:ok, {{_, code, _}, _, resp_body}} when code in 200..299 -> diff --git a/lib/hypatia/reviewer.ex b/lib/hypatia/reviewer.ex index be569c2d..8106f583 100644 --- a/lib/hypatia/reviewer.ex +++ b/lib/hypatia/reviewer.ex @@ -422,7 +422,12 @@ defmodule Hypatia.Reviewer do [] end - case :httpc.request(:get, {String.to_charlist(url), headers}, [{:timeout, 15_000}], []) do + case :httpc.request( + :get, + {String.to_charlist(url), headers}, + [{:timeout, 15_000}, {:autoredirect, false}], + [] + ) do {:ok, {{_, 200, _}, _resp_headers, body}} -> {:ok, to_string(body)} diff --git a/mise.toml b/mise.toml index 9cd7da86..8f5a6fb7 100644 --- a/mise.toml +++ b/mise.toml @@ -1,7 +1,7 @@ [tools] # Language runtimes # Match the OTP 27.3.4.19 / Elixir 1.19.4 lanes in .github/workflows/ci.yml and tests.yml. -erlang = "27.3.4.19" # >= 27.3.4.13: :httpc drops authorization on cross-origin redirect +erlang = "27.3.4.19" # floor: from 27.3.4.19 :httpc drops header and URL credentials on cross-origin redirect elixir = "1.19.4-otp-27" node = "latest" rust = "latest" # provides cargo, rustfmt and clippy diff --git a/test/httpc_redirect_credentials_test.exs b/test/httpc_redirect_credentials_test.exs index 861e72c7..8edd5edd 100644 --- a/test/httpc_redirect_credentials_test.exs +++ b/test/httpc_redirect_credentials_test.exs @@ -2,13 +2,19 @@ defmodule Hypatia.HttpcRedirectCredentialsTest do # Guards the Erlang/OTP pin (mise.toml and the CI test lanes), not Hypatia's - # code. OTP releases before 27.3.4.13 make `:httpc` resend the - # `authorization` header to whatever host a redirect names; measured on - # 2026-11-09, 27.3.4.4 and 27.3.4.12 forwarded it to another port and to - # another host for 301, 302, 303, 307 and 308, GET and POST, while - # 27.3.4.13, 27.3.4.19 and 28.5.0.8 dropped it. The two echidnabot senders - # already set `autoredirect: false`; this test fails if the runtime itself - # regresses, so every other `:httpc` caller stays covered. + # code. Older OTP releases make `:httpc` resend credentials to whatever host + # a redirect names. Measured on loopback for 301, 302, 303, 307 and 308, GET + # and POST, to another port and to another host: + # + # * an explicit `authorization` header: forwarded by 27.3.4.4 and + # 27.3.4.12 (2026-11-09), dropped from 27.3.4.13; + # * credentials in the URL (`http://user:pass@host`, sent as Basic auth): + # still forwarded by 27.3.4.13, dropped by 27.3.4.19 and 28.5.0.8 + # (2026-11-17). + # + # So 27.3.4.19 is the floor on the 27 line. Credentialed callers in lib/ + # also set `autoredirect: false` (checked by the last test here), so they + # do not depend on the pin alone. # # Each case runs a redirector and a target on loopback. The redirector # answers with STATUS and a Location on the target; the target reports the @@ -16,6 +22,7 @@ defmodule Hypatia.HttpcRedirectCredentialsTest do use ExUnit.Case, async: true @token "Bearer test-dummy-redirect-probe" + @userinfo "test-dummy-user:test-dummy-pass" setup_all do {:ok, _} = Application.ensure_all_started(:inets) @@ -30,15 +37,49 @@ defmodule Hypatia.HttpcRedirectCredentialsTest do end end - # Positive control: the target does see the header when the redirect stays - # on the same origin, so an empty list above is the runtime dropping it, not - # the harness failing to observe it. + for status <- [301, 302, 303, 307, 308], method <- [:get, :post] do + test "#{method} #{status} to another origin drops credentials from the URL" do + {line, auth} = follow(unquote(method), unquote(status), :other_origin, :userinfo) + assert line =~ "/final" + assert auth == [] + end + end + + # Positive controls: the target does see the credentials when the redirect + # stays on the same origin, so an empty list above is the runtime dropping + # them, not the harness failing to observe them. test "same-origin redirect keeps the authorization header" do {_line, auth} = follow(:get, 302, :same_origin) assert auth == ["authorization: #{@token}"] end - defp follow(method, status, where) do + test "same-origin redirect keeps credentials from the URL" do + {_line, auth} = follow(:get, 302, :same_origin, :userinfo) + assert auth == ["authorization: Basic " <> Base.encode64(@userinfo)] + end + + # Every lib/ module that sends an Authorization header must turn off + # automatic redirects on each :httpc request, so a credential never depends + # on the runtime pin alone (for example on the OTP 28.3 CI lanes). + test "credentialed :httpc callers in lib/ disable automatic redirects" do + offenders = + Path.wildcard(Path.expand("../lib/**/*.ex", __DIR__)) + |> Enum.filter(fn path -> + src = File.read!(path) + String.match?(src, ~r/authorization/i) and String.contains?(src, ":httpc.request") + end) + |> Enum.reject(fn path -> + src = File.read!(path) + calls = length(String.split(src, ":httpc.request")) - 1 + guarded = length(String.split(src, "{:autoredirect, false}")) - 1 + guarded >= calls + end) + |> Enum.map(&Path.relative_to_cwd/1) + + assert offenders == [] + end + + defp follow(method, status, where, cred \\ :header) do {:ok, front} = listen() {:ok, back} = listen() {:ok, front_port} = :inet.port(front) @@ -69,8 +110,15 @@ defmodule Hypatia.HttpcRedirectCredentialsTest do :gen_tcp.close(s2) end) - url = ~c"http://127.0.0.1:#{front_port}/start" - headers = [{~c"authorization", String.to_charlist(@token)}] + {url, headers} = + case cred do + :header -> + {~c"http://127.0.0.1:#{front_port}/start", + [{~c"authorization", String.to_charlist(@token)}]} + + :userinfo -> + {~c"http://#{@userinfo}@127.0.0.1:#{front_port}/start", []} + end request = case method do From b5cb9e3c5ec01fad978f512810669c5ab43919e6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 00:51:37 +0000 Subject: [PATCH 02/12] fix(arangodb): a 3xx from a POST is an error, not a decoded success With redirects no longer followed, a 3xx reached http_post's 300..499 clause, which decoded the body. A 307 with the body `{}` returned {:ok, %{}}, and do_query passed that on, so a query that never ran read as a success. Only 4xx bodies are decoded now; a 3xx falls through to {:error, {status, body}}. test/arangodb_redirect_status_test.exs runs the client against a loopback server that answers every POST with 307 and `{}`. With the old clause it fails (left {:error, {307, "{}"}}, right {:ok, %{}}); with the fix both tests pass. mise.toml: OTP 27.3.4.19 decides "cross-origin" on host and port only (httpc_response.erl compares {Host, Port}), so the comment now says so. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01J4QvDgzT72Tq2HTJmikyNg --- lib/data/arangodb.ex | 2 +- mise.toml | 2 +- test/arangodb_redirect_status_test.exs | 97 ++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 test/arangodb_redirect_status_test.exs diff --git a/lib/data/arangodb.ex b/lib/data/arangodb.ex index fb6beb33..1f460a20 100644 --- a/lib/data/arangodb.ex +++ b/lib/data/arangodb.ex @@ -613,7 +613,7 @@ defmodule Hypatia.Data.ArangoDB do {:ok, {{_, code, _}, _, resp_body}} when code in 200..299 -> Jason.decode(to_string(resp_body)) - {:ok, {{_, code, _}, _, resp_body}} when code in 300..499 -> + {:ok, {{_, code, _}, _, resp_body}} when code in 400..499 -> Jason.decode(to_string(resp_body)) {:ok, {{_, status, _}, _, resp_body}} -> diff --git a/mise.toml b/mise.toml index 8f5a6fb7..b95b0d4b 100644 --- a/mise.toml +++ b/mise.toml @@ -1,7 +1,7 @@ [tools] # Language runtimes # Match the OTP 27.3.4.19 / Elixir 1.19.4 lanes in .github/workflows/ci.yml and tests.yml. -erlang = "27.3.4.19" # floor: from 27.3.4.19 :httpc drops header and URL credentials on cross-origin redirect +erlang = "27.3.4.19" # floor: from 27.3.4.19 :httpc drops header and URL credentials when a redirect changes host or port (not scheme alone) elixir = "1.19.4-otp-27" node = "latest" rust = "latest" # provides cargo, rustfmt and clippy diff --git a/test/arangodb_redirect_status_test.exs b/test/arangodb_redirect_status_test.exs new file mode 100644 index 00000000..76b611a5 --- /dev/null +++ b/test/arangodb_redirect_status_test.exs @@ -0,0 +1,97 @@ +# SPDX-License-Identifier: MPL-2.0 + +defmodule Hypatia.Data.ArangoDBRedirectStatusTest do + # Hypatia.Data.ArangoDB sends Basic auth and does not follow redirects, so a + # 3xx reaches its own status handling. A 3xx with a JSON body must be an + # error, never `{:ok, map}`: otherwise a query or upsert that did not run + # reads as a success. A loopback server stands in for ArangoDB. It answers + # the version probe with 200 and every POST with 307 and the body `{}`. + use ExUnit.Case, async: false + + setup do + {:ok, listen} = + :gen_tcp.listen(0, [:binary, active: false, reuseaddr: true, ip: {127, 0, 0, 1}]) + + {:ok, port} = :inet.port(listen) + server = spawn_link(fn -> serve(listen) end) + previous = System.get_env("HYPATIA_ARANGO_URL") + System.put_env("HYPATIA_ARANGO_URL", "http://127.0.0.1:#{port}") + + on_exit(fn -> + if previous, + do: System.put_env("HYPATIA_ARANGO_URL", previous), + else: System.delete_env("HYPATIA_ARANGO_URL") + end) + + {:ok, pid} = GenServer.start(Hypatia.Data.ArangoDB, []) + + on_exit(fn -> + if Process.alive?(pid), do: GenServer.stop(pid) + Process.unlink(server) + Process.exit(server, :kill) + :gen_tcp.close(listen) + end) + + {:ok, pid: pid} + end + + test "the fake server is reached: the client reports connected", %{pid: pid} do + assert GenServer.call(pid, :connected?) + end + + test "a 307 with a JSON body from a query is an error", %{pid: pid} do + assert {:error, {307, "{}"}} = GenServer.call(pid, {:query, "RETURN 1", %{}}) + end + + defp serve(listen) do + case :gen_tcp.accept(listen, 10_000) do + {:ok, sock} -> + head = read_head(sock, "") + + reply = + if String.starts_with?(head, "GET /_api/version") do + respond(200, ~s({"server":"arango","version":"3.12.0"}), "") + else + respond(307, "{}", "Location: http://127.0.0.1:1/elsewhere\r\n") + end + + :gen_tcp.send(sock, reply) + :gen_tcp.close(sock) + serve(listen) + + {:error, _} -> + :ok + end + end + + defp respond(status, body, extra) do + "HTTP/1.1 #{status} X\r\nContent-Type: application/json\r\n" <> + extra <> "Content-Length: #{byte_size(body)}\r\nConnection: close\r\n\r\n" <> body + end + + # Reads the request head and any Content-Length body, and returns the head. + defp read_head(sock, acc) do + case :binary.match(acc, "\r\n\r\n") do + {pos, _} -> + head = binary_part(acc, 0, pos) + + want = + case Regex.run(~r/content-length:\s*(\d+)/i, head) do + [_, n] -> String.to_integer(n) + nil -> 0 + end + + if byte_size(acc) - pos - 4 >= want, do: head, else: more(sock, acc) + + :nomatch -> + more(sock, acc) + end + end + + defp more(sock, acc) do + case :gen_tcp.recv(sock, 0, 5_000) do + {:ok, data} -> read_head(sock, acc <> data) + {:error, _} -> acc + end + end +end From 66b99f913ecc9f5f80eb6f456e422f28d02e689b Mon Sep 17 00:00:00 2001 From: "coderabbitai[bot]" <136622811+coderabbitai[bot]@users.noreply.github.com> Date: Tue, 17 Nov 2026 00:53:26 +0000 Subject: [PATCH 03/12] fix(arangodb): reject POST redirects and handle schema setup failures --- lib/data/arangodb.ex | 121 +++++++++++++++-------- mise.toml | 4 +- test/arangodb_initialisation_test.exs | 134 ++++++++++++++++++++++++++ 3 files changed, 216 insertions(+), 43 deletions(-) create mode 100644 test/arangodb_initialisation_test.exs diff --git a/lib/data/arangodb.ex b/lib/data/arangodb.ex index fb6beb33..ca127f54 100644 --- a/lib/data/arangodb.ex +++ b/lib/data/arangodb.ex @@ -89,8 +89,21 @@ defmodule Hypatia.Data.ArangoDB do # Try to connect, but don't crash if ArangoDB isn't available state = try_connect(state) + state = + if state.connected do + case ensure_schema(state) do + :ok -> + state + + {:error, reason} -> + Logger.warning("ArangoDB schema initialisation failed: #{inspect(reason)}") + %{state | connected: false} + end + else + state + end + if state.connected do - ensure_schema(state) timer = Process.send_after(self(), :sync_from_verisimdb, @sync_interval_ms) Logger.info("ArangoDB connected: #{url}/#{db}") {:ok, %{state | sync_timer: timer}} @@ -347,57 +360,81 @@ defmodule Hypatia.Data.ArangoDB do db_url = "#{state.base_url}/_db/#{state.database}" # Create database if it doesn't exist - http_post("#{state.base_url}/_api/database", state.auth_header, %{"name" => state.database}) + database = + http_post("#{state.base_url}/_api/database", state.auth_header, %{"name" => state.database}) # Create document collections - Enum.each(@document_collections, fn name -> - http_post("#{db_url}/_api/collection", state.auth_header, %{"name" => name, "type" => 2}) - end) + documents = + Enum.map(@document_collections, fn name -> + http_post("#{db_url}/_api/collection", state.auth_header, %{"name" => name, "type" => 2}) + end) # Create edge collections - Enum.each(@edge_collections, fn name -> - http_post("#{db_url}/_api/collection", state.auth_header, %{"name" => name, "type" => 3}) - end) + edges = + Enum.map(@edge_collections, fn name -> + http_post("#{db_url}/_api/collection", state.auth_header, %{"name" => name, "type" => 3}) + end) # Create indices for common queries - create_index(state, "confidence_history", ["recipe_id", "timestamp"]) - create_index(state, "outcomes", ["timestamp"]) - create_index(state, "outcomes", ["recipe_id"]) - create_index(state, "findings", ["repo", "severity"]) - create_index(state, "findings", ["pattern_id"]) - create_index(state, "anomalies", ["resolved", "timestamp"]) - create_index(state, "repos", ["trust_score"]) - create_index(state, "bots", ["quarantined"]) + indices = [ + create_index(state, "confidence_history", ["recipe_id", "timestamp"]), + create_index(state, "outcomes", ["timestamp"]), + create_index(state, "outcomes", ["recipe_id"]), + create_index(state, "findings", ["repo", "severity"]), + create_index(state, "findings", ["pattern_id"]), + create_index(state, "anomalies", ["resolved", "timestamp"]), + create_index(state, "repos", ["trust_score"]), + create_index(state, "bots", ["quarantined"]) + ] # Create named graph - http_post("#{db_url}/_api/gharial", state.auth_header, %{ - "name" => "hypatia_graph", - "edgeDefinitions" => [ - %{ - "collection" => "trusts", - "from" => ["repos", "bots", "recipes", "contributors"], - "to" => ["repos", "bots", "recipes", "contributors"] - }, - %{"collection" => "depends_on", "from" => ["repos"], "to" => ["repos"]}, - %{"collection" => "dispatches", "from" => ["findings"], "to" => ["bots"]}, - %{"collection" => "applies_to", "from" => ["recipes"], "to" => ["patterns"]}, - %{"collection" => "repo_has_finding", "from" => ["repos"], "to" => ["findings"]}, - %{ - "collection" => "finding_matches_pattern", - "from" => ["findings"], - "to" => ["patterns"] - }, - %{"collection" => "bot_executes_recipe", "from" => ["bots"], "to" => ["recipes"]}, - %{"collection" => "recipe_has_outcome", "from" => ["recipes"], "to" => ["outcomes"]}, - %{"collection" => "contributor_fixes", "from" => ["contributors"], "to" => ["outcomes"]} - ] - }) + graph = + http_post("#{db_url}/_api/gharial", state.auth_header, %{ + "name" => "hypatia_graph", + "edgeDefinitions" => [ + %{ + "collection" => "trusts", + "from" => ["repos", "bots", "recipes", "contributors"], + "to" => ["repos", "bots", "recipes", "contributors"] + }, + %{"collection" => "depends_on", "from" => ["repos"], "to" => ["repos"]}, + %{"collection" => "dispatches", "from" => ["findings"], "to" => ["bots"]}, + %{"collection" => "applies_to", "from" => ["recipes"], "to" => ["patterns"]}, + %{"collection" => "repo_has_finding", "from" => ["repos"], "to" => ["findings"]}, + %{ + "collection" => "finding_matches_pattern", + "from" => ["findings"], + "to" => ["patterns"] + }, + %{"collection" => "bot_executes_recipe", "from" => ["bots"], "to" => ["recipes"]}, + %{"collection" => "recipe_has_outcome", "from" => ["recipes"], "to" => ["outcomes"]}, + %{"collection" => "contributor_fixes", "from" => ["contributors"], "to" => ["outcomes"]} + ] + }) + + results = [database] ++ documents ++ edges ++ indices ++ [graph] + + case Enum.find(results, &(not schema_result_ok?(&1))) do + nil -> + Logger.info( + "ArangoDB schema ensured: #{length(@document_collections)} document + #{length(@edge_collections)} edge collections" + ) - Logger.info( - "ArangoDB schema ensured: #{length(@document_collections)} document + #{length(@edge_collections)} edge collections" - ) + :ok + + failure -> + {:error, failure} + end end + # Repeated initialisation may encounter resources that already exist. + defp schema_result_ok?({:ok, %{"error" => true, "errorNum" => number}}) + when number in [1207, 1925], do: true + + defp schema_result_ok?({:ok, %{"error" => true}}), do: false + defp schema_result_ok?({:ok, _}), do: true + defp schema_result_ok?(_), do: false + defp create_index(state, collection, fields) do db_url = "#{state.base_url}/_db/#{state.database}" @@ -613,7 +650,7 @@ defmodule Hypatia.Data.ArangoDB do {:ok, {{_, code, _}, _, resp_body}} when code in 200..299 -> Jason.decode(to_string(resp_body)) - {:ok, {{_, code, _}, _, resp_body}} when code in 300..499 -> + {:ok, {{_, code, _}, _, resp_body}} when code in 400..499 -> Jason.decode(to_string(resp_body)) {:ok, {{_, status, _}, _, resp_body}} -> diff --git a/mise.toml b/mise.toml index 8f5a6fb7..49701150 100644 --- a/mise.toml +++ b/mise.toml @@ -1,7 +1,9 @@ [tools] # Language runtimes # Match the OTP 27.3.4.19 / Elixir 1.19.4 lanes in .github/workflows/ci.yml and tests.yml. -erlang = "27.3.4.19" # floor: from 27.3.4.19 :httpc drops header and URL credentials on cross-origin redirect +# OTP 27.3.4.19 drops header and URL credentials on redirects across host or port. +# Same-host, same-port HTTPS-to-HTTP redirects are not protected; keep explicit redirect guards. +erlang = "27.3.4.19" elixir = "1.19.4-otp-27" node = "latest" rust = "latest" # provides cargo, rustfmt and clippy diff --git a/test/arangodb_initialisation_test.exs b/test/arangodb_initialisation_test.exs new file mode 100644 index 00000000..f0c886ec --- /dev/null +++ b/test/arangodb_initialisation_test.exs @@ -0,0 +1,134 @@ +# SPDX-License-Identifier: MPL-2.0 + +defmodule Hypatia.Data.ArangoDBInitialisationTest do + use ExUnit.Case, async: false + import ExUnit.CaptureLog + alias Hypatia.Data.ArangoDB + + defmodule Stub do + def init(respond), do: respond + + def call(conn, respond) do + {:ok, _body, conn} = Plug.Conn.read_body(conn) + {status, body} = respond.(conn.method, conn.request_path) + + conn + |> Plug.Conn.put_resp_header("location", "/redirect-target") + |> Plug.Conn.put_resp_content_type("application/json") + |> Plug.Conn.send_resp(status, Jason.encode!(body)) + end + end + + setup do + previous = System.get_env("HYPATIA_ARANGO_URL") + + on_exit(fn -> + if previous, + do: System.put_env("HYPATIA_ARANGO_URL", previous), + else: System.delete_env("HYPATIA_ARANGO_URL") + end) + + :ok + end + + for status <- [301, 302, 303, 307, 308] do + test "POST rejects #{status} with valid JSON" do + url = serve(fn _, _ -> {unquote(status), %{"result" => []}} end) + + state = %ArangoDB{ + base_url: url, + database: "test", + auth_header: "Basic test-dummy", + connected: true + } + + assert {:reply, {:error, {unquote(status), _}}, ^state} = + ArangoDB.handle_call({:query, "RETURN 1", %{}}, nil, state) + end + end + + test "POST preserves decoded 4xx errors" do + body = %{"error" => true, "errorMessage" => "test denied"} + url = serve(fn _, _ -> {403, body} end) + + state = %ArangoDB{ + base_url: url, + database: "test", + auth_header: "Basic test-dummy", + connected: true + } + + assert {:reply, {:error, "test denied"}, ^state} = + ArangoDB.handle_call({:query, "RETURN 1", %{}}, nil, state) + end + + for path <- ["/_api/database", "/_api/collection", "/_api/index", "/_api/gharial"] do + test "redirect at #{path} leaves initialisation degraded" do + serve(fn + "GET", "/_api/version" -> + {200, %{"server" => "arango"}} + + "POST", path -> + if String.ends_with?(path, unquote(path)), + do: {307, %{"result" => true}}, + else: {201, %{"error" => false}} + end) + + log = + capture_log(fn -> + assert {:ok, state} = ArangoDB.init([]) + refute state.connected + assert state.sync_timer == nil + end) + + assert log =~ "schema initialisation failed" + refute log =~ "schema ensured" + refute log =~ "ArangoDB connected:" + end + end + + for existing? <- [false, true] do + test "schema succeeds with existing resources: #{existing?}" do + serve(fn + "GET", "/_api/version" -> + {200, %{"server" => "arango"}} + + "POST", path -> + if unquote(existing?) do + number = if String.ends_with?(path, "/gharial"), do: 1925, else: 1207 + {409, %{"error" => true, "errorNum" => number}} + else + {201, %{"error" => false}} + end + end) + + capture_log(fn -> + assert {:ok, state} = ArangoDB.init([]) + assert state.connected + assert is_reference(state.sync_timer) + Process.cancel_timer(state.sync_timer) + end) + end + end + + test "schema API errors enter degraded mode" do + serve(fn + "GET", _ -> {200, %{"server" => "arango"}} + "POST", _ -> {401, %{"error" => true, "errorNum" => 11}} + end) + + capture_log(fn -> + assert {:ok, state} = ArangoDB.init([]) + refute state.connected + assert state.sync_timer == nil + end) + end + + defp serve(respond) do + pid = start_supervised!({Bandit, plug: {Stub, respond}, ip: {127, 0, 0, 1}, port: 0}) + {:ok, {_, port}} = ThousandIsland.listener_info(pid) + url = "http://127.0.0.1:#{port}" + System.put_env("HYPATIA_ARANGO_URL", url) + url + end +end From e97366e48889d4e5f3665dbfc55fa3f1f0ade883 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 01:16:32 +0000 Subject: [PATCH 04/12] test(httpc): cross-origin cases also change the host, not only the port The cross-origin cases kept 127.0.0.1 and changed the port, so a runtime that stripped credentials on a port change but not on a host change would still pass. Each case now runs twice: :other_port (127.0.0.1, new port) and :other_host (target on 127.0.0.2). On OTP 27.3.4.19: 43 tests, 0 failures. On OTP 27.3.4.13: 20 failures, exactly the URL-credential cases for both :other_port and :other_host. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01J4QvDgzT72Tq2HTJmikyNg --- test/httpc_redirect_credentials_test.exs | 28 ++++++++++++++---------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/test/httpc_redirect_credentials_test.exs b/test/httpc_redirect_credentials_test.exs index 8edd5edd..e2bc6bdb 100644 --- a/test/httpc_redirect_credentials_test.exs +++ b/test/httpc_redirect_credentials_test.exs @@ -29,17 +29,19 @@ defmodule Hypatia.HttpcRedirectCredentialsTest do :ok end - for status <- [301, 302, 303, 307, 308], method <- [:get, :post] do - test "#{method} #{status} to another origin drops the authorization header" do - {line, auth} = follow(unquote(method), unquote(status), :other_origin) + # :other_port keeps the host (127.0.0.1) and changes the port; :other_host + # moves to 127.0.0.2, so a runtime that compared ports only would fail it. + for where <- [:other_port, :other_host], + status <- [301, 302, 303, 307, 308], + method <- [:get, :post] do + test "#{method} #{status} to #{where} drops the authorization header" do + {line, auth} = follow(unquote(method), unquote(status), unquote(where)) assert line =~ "/final" assert auth == [] end - end - for status <- [301, 302, 303, 307, 308], method <- [:get, :post] do - test "#{method} #{status} to another origin drops credentials from the URL" do - {line, auth} = follow(unquote(method), unquote(status), :other_origin, :userinfo) + test "#{method} #{status} to #{where} drops credentials from the URL" do + {line, auth} = follow(unquote(method), unquote(status), unquote(where), :userinfo) assert line =~ "/final" assert auth == [] end @@ -80,13 +82,15 @@ defmodule Hypatia.HttpcRedirectCredentialsTest do end defp follow(method, status, where, cred \\ :header) do - {:ok, front} = listen() - {:ok, back} = listen() + back_ip = if where == :other_host, do: {127, 0, 0, 2}, else: {127, 0, 0, 1} + {:ok, front} = listen({127, 0, 0, 1}) + {:ok, back} = listen(back_ip) {:ok, front_port} = :inet.port(front) {:ok, back_port} = :inet.port(back) target = if where == :same_origin, do: front, else: back target_port = if where == :same_origin, do: front_port, else: back_port - location = "http://127.0.0.1:#{target_port}/final" + target_host = if where == :other_host, do: "127.0.0.2", else: "127.0.0.1" + location = "http://#{target_host}:#{target_port}/final" parent = self() server = @@ -135,8 +139,8 @@ defmodule Hypatia.HttpcRedirectCredentialsTest do {line, auth} end - defp listen do - :gen_tcp.listen(0, [:binary, active: false, reuseaddr: true, ip: {127, 0, 0, 1}]) + defp listen(ip) do + :gen_tcp.listen(0, [:binary, active: false, reuseaddr: true, ip: ip]) end # Reads up to the end of the headers, then any Content-Length body. From b53a3ac7f39deba955a7c1beffacc3a9492136d2 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 02:05:35 +0000 Subject: [PATCH 05/12] fix(arangodb): schema setup keeps the HTTP status of each step http_post/3 decodes 4xx bodies, so schema setup could not tell a 2xx from a 4xx whose JSON lacks ArangoDB's "error": true. A gateway answering 403 {"message":"forbidden"} counted as a created collection. The schema steps now call http_post_status/3, which keeps the status. A step succeeds only on a 2xx, or on a 4xx with errorNum 1207 or 1925 (already exists). http_post/3 wraps it unchanged for queries; there is still one :httpc.request call in the module. test(httpc): the :other_host target now listens on 127.0.0.2 at the redirector's own port, so only the host differs. Before, the ports differed too, and a runtime comparing ports only would have passed it. New test: schema 4xx without the error flag enters degraded mode. It fails on the previous commit and passes here. Full suite on Elixir 1.19.4 / OTP 27.3.4.19: 3635 tests, 0 failures, 3 skipped, 246 excluded. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01J4QvDgzT72Tq2HTJmikyNg --- lib/data/arangodb.ex | 52 +++++++++++++++++------- test/arangodb_initialisation_test.exs | 19 +++++++++ test/httpc_redirect_credentials_test.exs | 15 ++++--- 3 files changed, 66 insertions(+), 20 deletions(-) diff --git a/lib/data/arangodb.ex b/lib/data/arangodb.ex index ca127f54..1ef6a980 100644 --- a/lib/data/arangodb.ex +++ b/lib/data/arangodb.ex @@ -361,18 +361,26 @@ defmodule Hypatia.Data.ArangoDB do # Create database if it doesn't exist database = - http_post("#{state.base_url}/_api/database", state.auth_header, %{"name" => state.database}) + http_post_status("#{state.base_url}/_api/database", state.auth_header, %{ + "name" => state.database + }) # Create document collections documents = Enum.map(@document_collections, fn name -> - http_post("#{db_url}/_api/collection", state.auth_header, %{"name" => name, "type" => 2}) + http_post_status("#{db_url}/_api/collection", state.auth_header, %{ + "name" => name, + "type" => 2 + }) end) # Create edge collections edges = Enum.map(@edge_collections, fn name -> - http_post("#{db_url}/_api/collection", state.auth_header, %{"name" => name, "type" => 3}) + http_post_status("#{db_url}/_api/collection", state.auth_header, %{ + "name" => name, + "type" => 3 + }) end) # Create indices for common queries @@ -389,7 +397,7 @@ defmodule Hypatia.Data.ArangoDB do # Create named graph graph = - http_post("#{db_url}/_api/gharial", state.auth_header, %{ + http_post_status("#{db_url}/_api/gharial", state.auth_header, %{ "name" => "hypatia_graph", "edgeDefinitions" => [ %{ @@ -427,18 +435,22 @@ defmodule Hypatia.Data.ArangoDB do end end - # Repeated initialisation may encounter resources that already exist. - defp schema_result_ok?({:ok, %{"error" => true, "errorNum" => number}}) - when number in [1207, 1925], do: true + # A schema step succeeds on a 2xx, or on a 4xx that says the resource + # already exists (1207 duplicate name, 1925 duplicate graph), which happens + # on every initialisation after the first. Any other status fails, even with + # a JSON body: a gateway's `{"message": "forbidden"}` is not a success. + defp schema_result_ok?({:ok, status, _}) when status in 200..299, do: true + + defp schema_result_ok?({:ok, status, %{"errorNum" => number}}) + when status in 400..499 and number in [1207, 1925], + do: true - defp schema_result_ok?({:ok, %{"error" => true}}), do: false - defp schema_result_ok?({:ok, _}), do: true defp schema_result_ok?(_), do: false defp create_index(state, collection, fields) do db_url = "#{state.base_url}/_db/#{state.database}" - http_post("#{db_url}/_api/index?collection=#{collection}", state.auth_header, %{ + http_post_status("#{db_url}/_api/index?collection=#{collection}", state.auth_header, %{ "type" => "persistent", "fields" => fields }) @@ -630,7 +642,18 @@ defmodule Hypatia.Data.ArangoDB do end end + # Decoded body on a 2xx or 4xx (ArangoDB reports its errors as JSON 4xx); + # `{:error, {status, body}}` on any other status, a 3xx included. defp http_post(url, auth_header, body) do + case http_post_status(url, auth_header, body) do + {:ok, _status, decoded} -> {:ok, decoded} + error -> error + end + end + + # As http_post/3, but keeps the status, so a caller can tell a 2xx from a + # 4xx whose body does not use ArangoDB's error shape. + defp http_post_status(url, auth_header, body) do :inets.start() :ssl.start() @@ -647,11 +670,10 @@ defmodule Hypatia.Data.ArangoDB do [{:timeout, 10_000}, {:autoredirect, false}], [] ) do - {:ok, {{_, code, _}, _, resp_body}} when code in 200..299 -> - Jason.decode(to_string(resp_body)) - - {:ok, {{_, code, _}, _, resp_body}} when code in 400..499 -> - Jason.decode(to_string(resp_body)) + {:ok, {{_, code, _}, _, resp_body}} when code in 200..299 or code in 400..499 -> + with {:ok, decoded} <- Jason.decode(to_string(resp_body)) do + {:ok, code, decoded} + end {:ok, {{_, status, _}, _, resp_body}} -> {:error, {status, to_string(resp_body)}} diff --git a/test/arangodb_initialisation_test.exs b/test/arangodb_initialisation_test.exs index f0c886ec..3ad8e1f4 100644 --- a/test/arangodb_initialisation_test.exs +++ b/test/arangodb_initialisation_test.exs @@ -124,6 +124,25 @@ defmodule Hypatia.Data.ArangoDBInitialisationTest do end) end + # A gateway or proxy in front of ArangoDB can answer with a JSON 4xx that + # has no `"error": true`. Only a 2xx, or an existing-resource 4xx (1207, + # 1925), counts as success. + test "schema 4xx without the error flag enters degraded mode" do + serve(fn + "GET", _ -> {200, %{"server" => "arango"}} + "POST", _ -> {403, %{"message" => "forbidden"}} + end) + + log = + capture_log(fn -> + assert {:ok, state} = ArangoDB.init([]) + refute state.connected + assert state.sync_timer == nil + end) + + refute log =~ "schema ensured" + end + defp serve(respond) do pid = start_supervised!({Bandit, plug: {Stub, respond}, ip: {127, 0, 0, 1}, port: 0}) {:ok, {_, port}} = ThousandIsland.listener_info(pid) diff --git a/test/httpc_redirect_credentials_test.exs b/test/httpc_redirect_credentials_test.exs index e2bc6bdb..145668d9 100644 --- a/test/httpc_redirect_credentials_test.exs +++ b/test/httpc_redirect_credentials_test.exs @@ -29,7 +29,8 @@ defmodule Hypatia.HttpcRedirectCredentialsTest do :ok end - # :other_port keeps the host (127.0.0.1) and changes the port; :other_host + # :other_port keeps the host (127.0.0.1) and changes the port, so a runtime + # that compared hosts only would fail it. :other_host keeps the port and # moves to 127.0.0.2, so a runtime that compared ports only would fail it. for where <- [:other_port, :other_host], status <- [301, 302, 303, 307, 308], @@ -82,10 +83,14 @@ defmodule Hypatia.HttpcRedirectCredentialsTest do end defp follow(method, status, where, cred \\ :header) do - back_ip = if where == :other_host, do: {127, 0, 0, 2}, else: {127, 0, 0, 1} {:ok, front} = listen({127, 0, 0, 1}) - {:ok, back} = listen(back_ip) {:ok, front_port} = :inet.port(front) + + {:ok, back} = + if where == :other_host, + do: listen({127, 0, 0, 2}, front_port), + else: listen({127, 0, 0, 1}) + {:ok, back_port} = :inet.port(back) target = if where == :same_origin, do: front, else: back target_port = if where == :same_origin, do: front_port, else: back_port @@ -139,8 +144,8 @@ defmodule Hypatia.HttpcRedirectCredentialsTest do {line, auth} end - defp listen(ip) do - :gen_tcp.listen(0, [:binary, active: false, reuseaddr: true, ip: ip]) + defp listen(ip, port \\ 0) do + :gen_tcp.listen(port, [:binary, active: false, reuseaddr: true, ip: ip]) end # Reads up to the end of the headers, then any Content-Length body. From 8c60abbe98feb5b4b1c89cbc4ed2c2e985a68c8b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 02:30:26 +0000 Subject: [PATCH 06/12] fix(arangodb): skip database creation when the target already exists Creating a database needs Administrate on _system. A least-privilege user that administers only a pre-provisioned HYPATIA_ARANGO_DB gets a 403 from POST /_api/database. Since ensure_schema now checks every step, that 403 left the client degraded on every start, where it had previously been ignored. ensure_schema first reads /_db//_api/database/current, which needs only access to the target database, and creates the database only when that read fails. A refused creation of an absent database still degrades. The test stub answers 404 for routes a test does not describe, as ArangoDB does for an unknown database. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01J4QvDgzT72Tq2HTJmikyNg --- lib/data/arangodb.ex | 16 ++++-- test/arangodb_initialisation_test.exs | 70 ++++++++++++++++++++++++++- 2 files changed, 81 insertions(+), 5 deletions(-) diff --git a/lib/data/arangodb.ex b/lib/data/arangodb.ex index 1ef6a980..c323e04c 100644 --- a/lib/data/arangodb.ex +++ b/lib/data/arangodb.ex @@ -359,11 +359,19 @@ defmodule Hypatia.Data.ArangoDB do defp ensure_schema(state) do db_url = "#{state.base_url}/_db/#{state.database}" - # Create database if it doesn't exist + # Create the database only when it is absent. Creating one needs + # Administrate on `_system`; a least-privilege user that administers only + # a pre-provisioned target database gets a 403 there, so probe first. database = - http_post_status("#{state.base_url}/_api/database", state.auth_header, %{ - "name" => state.database - }) + case http_get("#{db_url}/_api/database/current", state.auth_header) do + {:ok, %{} = current} -> + {:ok, 200, current} + + _absent_or_unreadable -> + http_post_status("#{state.base_url}/_api/database", state.auth_header, %{ + "name" => state.database + }) + end # Create document collections documents = diff --git a/test/arangodb_initialisation_test.exs b/test/arangodb_initialisation_test.exs index 3ad8e1f4..9fb2af0a 100644 --- a/test/arangodb_initialisation_test.exs +++ b/test/arangodb_initialisation_test.exs @@ -10,7 +10,15 @@ defmodule Hypatia.Data.ArangoDBInitialisationTest do def call(conn, respond) do {:ok, _body, conn} = Plug.Conn.read_body(conn) - {status, body} = respond.(conn.method, conn.request_path) + # A route the test does not describe answers 404, as ArangoDB does for + # an unknown database or path, so the probe of the target database + # falls through to creating it. + {status, body} = + try do + respond.(conn.method, conn.request_path) + rescue + FunctionClauseError -> {404, %{"error" => true, "errorNum" => 1228}} + end conn |> Plug.Conn.put_resp_header("location", "/redirect-target") @@ -143,6 +151,66 @@ defmodule Hypatia.Data.ArangoDBInitialisationTest do refute log =~ "schema ensured" end + # A user with Administrate on a pre-provisioned target database but not on + # `_system` cannot create databases. The client must see that the target + # exists and not ask `_system` to create it. + test "existing target database is not recreated through _system" do + parent = self() + + serve(fn + "GET", "/_api/version" -> + {200, %{"server" => "arango"}} + + "GET", path -> + if String.ends_with?(path, "/_api/database/current"), + do: {200, %{"error" => false, "result" => %{"name" => "hypatia"}}}, + else: {404, %{"error" => true}} + + "POST", "/_api/database" -> + send(parent, :created_database) + {403, %{"error" => true, "errorNum" => 11}} + + "POST", _ -> + {201, %{"error" => false}} + end) + + capture_log(fn -> + assert {:ok, state} = ArangoDB.init([]) + assert state.connected + Process.cancel_timer(state.sync_timer) + end) + + refute_received :created_database + end + + # Without a readable target database the client still creates it, and a + # refused creation leaves it degraded. + test "absent target database is created, and a refusal degrades" do + parent = self() + + serve(fn + "GET", "/_api/version" -> + {200, %{"server" => "arango"}} + + "GET", _ -> + {404, %{"error" => true, "errorNum" => 1228}} + + "POST", "/_api/database" -> + send(parent, :created_database) + {403, %{"error" => true, "errorNum" => 11}} + + "POST", _ -> + {201, %{"error" => false}} + end) + + capture_log(fn -> + assert {:ok, state} = ArangoDB.init([]) + refute state.connected + end) + + assert_received :created_database + end + defp serve(respond) do pid = start_supervised!({Bandit, plug: {Stub, respond}, ip: {127, 0, 0, 1}, port: 0}) {:ok, {_, port}} = ThousandIsland.listener_info(pid) From c3047c84ab53db3c79f7b16c2fae86e971aa969a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 02:32:24 +0000 Subject: [PATCH 07/12] fix(reviewer): follow same-origin redirects, at most three With autoredirect off, a pull request in a renamed repository failed review: GitHub answers the old name with a 301 to https://api.github.com/repositories//..., and review_pr/3 returned {:error, {:http_status, 301}}. get_same_origin/3 follows a redirect only when the target has the scheme, host and port of the first request, at most three times. The token therefore never reaches another origin or a plain-HTTP downgrade. The file keeps one :httpc.request with autoredirect off. Loopback tests: same-origin redirect followed with the token; another origin refused and never contacted; a loop stops after three hops. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01J4QvDgzT72Tq2HTJmikyNg --- lib/hypatia/reviewer.ex | 34 +++++++++++++++ test/reviewer_redirect_test.exs | 73 +++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 test/reviewer_redirect_test.exs diff --git a/lib/hypatia/reviewer.ex b/lib/hypatia/reviewer.ex index 8106f583..c54f7b7b 100644 --- a/lib/hypatia/reviewer.ex +++ b/lib/hypatia/reviewer.ex @@ -422,6 +422,19 @@ defmodule Hypatia.Reviewer do [] end + get_same_origin(url, headers) + end + + @max_redirects 3 + + # GET that follows a redirect only to the origin (scheme, host and port) + # of the first request, at most @max_redirects times. GitHub answers a + # request for a renamed repository with a 301 to + # https://api.github.com/repositories//..., so the review still runs, + # while the token never reaches another origin or a plain-HTTP downgrade. + # :httpc's own redirect handling stays off (#1195). + @doc false + def get_same_origin(url, headers, hops \\ @max_redirects) do case :httpc.request( :get, {String.to_charlist(url), headers}, @@ -431,6 +444,14 @@ defmodule Hypatia.Reviewer do {:ok, {{_, 200, _}, _resp_headers, body}} -> {:ok, to_string(body)} + {:ok, {{_, status, _}, resp_headers, _}} when status in [301, 302, 303, 307, 308] -> + with {:ok, target} <- redirect_target(url, resp_headers), + true <- same_origin?(url, target) and hops > 0 do + get_same_origin(target, headers, hops - 1) + else + _ -> {:error, {:http_status, status}} + end + {:ok, {{_, status, _}, _, _}} -> {:error, {:http_status, status}} @@ -438,4 +459,17 @@ defmodule Hypatia.Reviewer do {:error, {:http_error, reason}} end end + + defp redirect_target(url, resp_headers) do + case List.keyfind(resp_headers, ~c"location", 0) do + {_, location} -> {:ok, url |> URI.merge(to_string(location)) |> URI.to_string()} + nil -> :error + end + end + + defp same_origin?(from, to) do + a = URI.parse(from) + b = URI.parse(to) + {a.scheme, a.host, a.port} == {b.scheme, b.host, b.port} + end end diff --git a/test/reviewer_redirect_test.exs b/test/reviewer_redirect_test.exs new file mode 100644 index 00000000..4a250dd7 --- /dev/null +++ b/test/reviewer_redirect_test.exs @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: MPL-2.0 + +defmodule Hypatia.ReviewerRedirectTest do + # Hypatia.Reviewer sends GITHUB_TOKEN, so :httpc's own redirect handling is + # off. It follows a redirect itself only to the origin of the first + # request (GitHub's renamed-repository 301 stays on api.github.com), at + # most three times. + use ExUnit.Case, async: true + alias Hypatia.Reviewer + + @headers [{~c"authorization", ~c"token test-dummy-reviewer"}] + + defmodule Stub do + def init(respond), do: respond + + def call(conn, {name, parent, respond}) do + auth = Plug.Conn.get_req_header(conn, "authorization") + send(parent, {:hit, name, conn.request_path, auth}) + {status, location, body} = respond.(conn.request_path) + + conn + |> then(fn c -> + if location, do: Plug.Conn.put_resp_header(c, "location", location), else: c + end) + |> Plug.Conn.send_resp(status, body) + end + end + + setup_all do + {:ok, _} = Application.ensure_all_started(:inets) + :ok + end + + test "a same-origin redirect is followed with the token" do + a = + serve(:a, fn + "/start" -> {301, "/repositories/1/pulls/2", ""} + "/repositories/1/pulls/2" -> {200, nil, "diff --git a/x b/x"} + end) + + assert {:ok, "diff --git a/x b/x"} = Reviewer.get_same_origin("#{a}/start", @headers) + assert_received {:hit, :a, "/repositories/1/pulls/2", ["token test-dummy-reviewer"]} + end + + test "a redirect to another origin is refused and never contacted" do + b = serve(:b, fn _ -> {200, nil, "should not be reached"} end) + a = serve(:a, fn "/start" -> {301, "#{b}/final", ""} end) + + assert {:error, {:http_status, 301}} = Reviewer.get_same_origin("#{a}/start", @headers) + refute_received {:hit, :b, _, _} + end + + test "a redirect loop stops after three hops" do + a = serve(:a, fn "/loop" -> {302, "/loop", ""} end) + + assert {:error, {:http_status, 302}} = Reviewer.get_same_origin("#{a}/loop", @headers) + + hits = for _ <- 1..4, do: assert_received({:hit, :a, "/loop", _}) + assert length(hits) == 4 + refute_received {:hit, :a, _, _} + end + + defp serve(name, respond) do + pid = + start_supervised!( + {Bandit, plug: {Stub, {name, self(), respond}}, ip: {127, 0, 0, 1}, port: 0}, + id: name + ) + + {:ok, {_, port}} = ThousandIsland.listener_info(pid) + "http://127.0.0.1:#{port}" + end +end From 322b62d8f6f1c488e5192dd78097ac19330f86c5 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 15:23:09 +0000 Subject: [PATCH 08/12] fix(reviewer): bound the redirect chain and refuse userinfo redirects get_same_origin/4 now takes one monotonic deadline (15 s by default) for the whole redirect chain and gives each hop only what is left of it; when the budget is spent it returns {:error, {:http_error, :timeout}}. A Location that carries URL userinfo is refused before anything is resent, and the original redirect status is returned. Origins compare scheme and host case-insensitively (ASCII), port exactly. New tests: a scheme-only change (http to https, same host and port) is refused without a second request; a mixed-case host is followed; a same-origin Location with userinfo is refused; four 150 ms hops overrun a 400 ms budget. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01J4QvDgzT72Tq2HTJmikyNg --- lib/hypatia/reviewer.ex | 80 +++++++++++++++++++++------------ test/reviewer_redirect_test.exs | 64 +++++++++++++++++++++++++- 2 files changed, 115 insertions(+), 29 deletions(-) diff --git a/lib/hypatia/reviewer.ex b/lib/hypatia/reviewer.ex index c54f7b7b..43341a07 100644 --- a/lib/hypatia/reviewer.ex +++ b/lib/hypatia/reviewer.ex @@ -426,50 +426,74 @@ defmodule Hypatia.Reviewer do end @max_redirects 3 + @timeout_ms 15_000 # GET that follows a redirect only to the origin (scheme, host and port) # of the first request, at most @max_redirects times. GitHub answers a # request for a renamed repository with a 301 to # https://api.github.com/repositories//..., so the review still runs, # while the token never reaches another origin or a plain-HTTP downgrade. - # :httpc's own redirect handling stays off (#1195). + # A Location carrying userinfo is refused too, so a redirect cannot add + # credentials of its own. :httpc's own redirect handling stays off (#1195). + # `timeout_ms` bounds the whole chain, not each hop: every request gets + # what is left of one monotonic deadline. @doc false - def get_same_origin(url, headers, hops \\ @max_redirects) do - case :httpc.request( - :get, - {String.to_charlist(url), headers}, - [{:timeout, 15_000}, {:autoredirect, false}], - [] - ) do - {:ok, {{_, 200, _}, _resp_headers, body}} -> - {:ok, to_string(body)} - - {:ok, {{_, status, _}, resp_headers, _}} when status in [301, 302, 303, 307, 308] -> - with {:ok, target} <- redirect_target(url, resp_headers), - true <- same_origin?(url, target) and hops > 0 do - get_same_origin(target, headers, hops - 1) - else - _ -> {:error, {:http_status, status}} - end + def get_same_origin(url, headers, hops \\ @max_redirects, timeout_ms \\ @timeout_ms) do + follow_same_origin(url, headers, hops, System.monotonic_time(:millisecond) + timeout_ms) + end - {:ok, {{_, status, _}, _, _}} -> - {:error, {:http_status, status}} + defp follow_same_origin(url, headers, hops, deadline) do + remaining = deadline - System.monotonic_time(:millisecond) - {:error, reason} -> - {:error, {:http_error, reason}} + if remaining <= 0 do + {:error, {:http_error, :timeout}} + else + case :httpc.request( + :get, + {String.to_charlist(url), headers}, + [{:timeout, remaining}, {:autoredirect, false}], + [] + ) do + {:ok, {{_, 200, _}, _resp_headers, body}} -> + {:ok, to_string(body)} + + {:ok, {{_, status, _}, resp_headers, _}} when status in [301, 302, 303, 307, 308] -> + with {:ok, target} <- redirect_target(url, resp_headers), + true <- same_origin?(url, target) and hops > 0 do + follow_same_origin(target, headers, hops - 1, deadline) + else + _ -> {:error, {:http_status, status}} + end + + {:ok, {{_, status, _}, _, _}} -> + {:error, {:http_status, status}} + + {:error, reason} -> + {:error, {:http_error, reason}} + end end end defp redirect_target(url, resp_headers) do - case List.keyfind(resp_headers, ~c"location", 0) do - {_, location} -> {:ok, url |> URI.merge(to_string(location)) |> URI.to_string()} - nil -> :error + with {_, location} <- List.keyfind(resp_headers, ~c"location", 0), + location = to_string(location), + nil <- URI.parse(location).userinfo do + {:ok, url |> URI.merge(location) |> URI.to_string()} + else + _ -> :error end end + # Host names compare case-insensitively (ASCII), as DNS does. defp same_origin?(from, to) do - a = URI.parse(from) - b = URI.parse(to) - {a.scheme, a.host, a.port} == {b.scheme, b.host, b.port} + origin(from) == origin(to) end + + defp origin(url) do + uri = URI.parse(url) + {downcase(uri.scheme), downcase(uri.host), uri.port} + end + + defp downcase(nil), do: nil + defp downcase(text), do: String.downcase(text, :ascii) end diff --git a/test/reviewer_redirect_test.exs b/test/reviewer_redirect_test.exs index 4a250dd7..3d661e7c 100644 --- a/test/reviewer_redirect_test.exs +++ b/test/reviewer_redirect_test.exs @@ -16,7 +16,11 @@ defmodule Hypatia.ReviewerRedirectTest do def call(conn, {name, parent, respond}) do auth = Plug.Conn.get_req_header(conn, "authorization") send(parent, {:hit, name, conn.request_path, auth}) - {status, location, body} = respond.(conn.request_path) + + {status, location, body} = + if is_function(respond, 2), + do: respond.(conn.request_path, conn.port), + else: respond.(conn.request_path) conn |> then(fn c -> @@ -60,6 +64,64 @@ defmodule Hypatia.ReviewerRedirectTest do refute_received {:hit, :a, _, _} end + # Only the scheme changes (same host, same explicit port): a downgrade or + # upgrade is another origin, so the 301 is returned without a second request. + test "a redirect that changes only the scheme is refused" do + a = serve(:a, fn "/start", port -> {301, "https://127.0.0.1:#{port}/final", ""} end) + + assert {:error, {:http_status, 301}} = Reviewer.get_same_origin("#{a}/start", @headers) + assert_received {:hit, :a, "/start", _} + refute_received {:hit, :a, _, _} + end + + test "host names compare case-insensitively" do + a = + serve(:a, fn + "/start", port -> {301, "http://LocalHost:#{port}/final", ""} + "/final", _ -> {200, nil, "diff"} + end) + + url = String.replace(a, "127.0.0.1", "localhost") + assert {:ok, "diff"} = Reviewer.get_same_origin("#{url}/start", @headers) + assert_received {:hit, :a, "/final", ["token test-dummy-reviewer"]} + end + + test "a same-origin Location carrying userinfo is refused" do + a = + serve(:a, fn + "/start", port -> + {302, "http://test-dummy-user:test-dummy-pass@127.0.0.1:#{port}/final", ""} + + "/final", _ -> + {200, nil, "should not be reached"} + end) + + assert {:error, {:http_status, 302}} = Reviewer.get_same_origin("#{a}/start", @headers) + assert_received {:hit, :a, "/start", _} + refute_received {:hit, :a, _, _} + end + + # Each hop takes 150 ms and fits a 400 ms budget on its own; the chain of + # four requests does not, so the budget must span the redirects. + test "the timeout covers the whole redirect chain" do + a = + serve(:a, fn path -> + Process.sleep(150) + + case path do + "/1" -> {302, "/2", ""} + "/2" -> {302, "/3", ""} + "/3" -> {302, "/4", ""} + "/4" -> {200, nil, "late"} + end + end) + + assert {:error, {:http_error, :timeout}} = + Reviewer.get_same_origin("#{a}/1", @headers, 3, 400) + + assert {:ok, "late"} = Reviewer.get_same_origin("#{a}/1", @headers, 3, 5_000) + end + defp serve(name, respond) do pid = start_supervised!( From d42b37a86c63478a37a30da492674b1906a8284a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 15:26:26 +0000 Subject: [PATCH 09/12] fix(arangodb): probe the target database, verify setup, retry with backoff Connectivity is probed on the target database (/_db//_api/version), so a user who administers only that database connects; only a 404 database-not-found (errorNum 1228) there falls back to the _system version probe, so that setup can create the database. The database is created only when its probe answers 404 with errorNum 1228; a timeout, redirect, 5xx or other failure is returned as it is. If the database step fails, setup stops before the collection, index and graph requests. A duplicate-graph answer (1925) is no longer accepted by its errorNum alone: the existing graph is read back, and accepted only if every required edge definition is present on the same collection with exactly the required from and to vertex collections (as sets). Extra edge definitions are allowed. A mismatch is a schema failure and the graph is not replaced. A failed setup keeps the client disconnected and schedules a retry with exponential backoff (5 s doubling to 5 min, configurable through :arango_setup_retry_base_ms and :arango_setup_retry_max_ms). The sync timer starts only after setup succeeds. Tests: the Bandit stub now answers /redirect-target with a terminal 200 and reports any request to it, so the redirect tests fail if the client follows a redirect. New tests cover the target-database probe, the probe failures that must not create the database, the early stop, the graph check (match, changed vertices, missing collection) and the retry. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01J4QvDgzT72Tq2HTJmikyNg --- lib/data/arangodb.ex | 251 +++++++++++++------ test/arangodb_initialisation_test.exs | 322 ++++++++++++++++++++++++- test/arangodb_redirect_status_test.exs | 5 +- 3 files changed, 502 insertions(+), 76 deletions(-) diff --git a/lib/data/arangodb.ex b/lib/data/arangodb.ex index c323e04c..0dbc18b0 100644 --- a/lib/data/arangodb.ex +++ b/lib/data/arangodb.ex @@ -57,13 +57,33 @@ defmodule Hypatia.Data.ArangoDB do "contributor_fixes" ] + @graph_name "hypatia_graph" + + @graph_edge_definitions [ + %{ + "collection" => "trusts", + "from" => ["repos", "bots", "recipes", "contributors"], + "to" => ["repos", "bots", "recipes", "contributors"] + }, + %{"collection" => "depends_on", "from" => ["repos"], "to" => ["repos"]}, + %{"collection" => "dispatches", "from" => ["findings"], "to" => ["bots"]}, + %{"collection" => "applies_to", "from" => ["recipes"], "to" => ["patterns"]}, + %{"collection" => "repo_has_finding", "from" => ["repos"], "to" => ["findings"]}, + %{"collection" => "finding_matches_pattern", "from" => ["findings"], "to" => ["patterns"]}, + %{"collection" => "bot_executes_recipe", "from" => ["bots"], "to" => ["recipes"]}, + %{"collection" => "recipe_has_outcome", "from" => ["recipes"], "to" => ["outcomes"]}, + %{"collection" => "contributor_fixes", "from" => ["contributors"], "to" => ["outcomes"]} + ] + defstruct [ :base_url, :database, :auth_header, connected: false, last_sync: nil, - sync_timer: nil + sync_timer: nil, + setup_timer: nil, + setup_attempts: 0 ] # --- GenServer API --- @@ -86,36 +106,60 @@ defmodule Hypatia.Data.ArangoDB do auth_header: "Basic #{auth}" } - # Try to connect, but don't crash if ArangoDB isn't available - state = try_connect(state) - - state = - if state.connected do - case ensure_schema(state) do - :ok -> - state - - {:error, reason} -> - Logger.warning("ArangoDB schema initialisation failed: #{inspect(reason)}") - %{state | connected: false} - end + # Never crashes when ArangoDB is unavailable: setup/1 degrades and retries. + {:ok, setup(state)} + end + + # Connects and ensures the schema. The client counts as connected only once + # both succeed; until then it stays in degraded mode and retries with an + # exponential backoff (`:arango_setup_retry_base_ms` doubling up to + # `:arango_setup_retry_max_ms`), so a transient failure such as a 503 + # during setup does not leave it disconnected for good. A retry repeats + # every step; a collection or index created by an earlier attempt answers + # 1207 and is accepted, and an existing graph (1925) is accepted once + # check_existing_graph/1 has verified it. The sync timer starts only after + # setup succeeds. + defp setup(state) do + result = + if reachable?(state) do + with {:error, reason} <- ensure_schema(state), + do: {:error, "ArangoDB schema initialisation failed: #{inspect(reason)}"} else - state + {:error, "ArangoDB not available at #{state.base_url}"} end - if state.connected do - timer = Process.send_after(self(), :sync_from_verisimdb, @sync_interval_ms) - Logger.info("ArangoDB connected: #{url}/#{db}") - {:ok, %{state | sync_timer: timer}} - else - Logger.warning( - "ArangoDB not available at #{url} -- running in degraded mode (flat files only)" - ) + case result do + :ok -> + timer = Process.send_after(self(), :sync_from_verisimdb, @sync_interval_ms) + Logger.info("ArangoDB connected: #{state.base_url}/#{state.database}") + %{state | connected: true, sync_timer: timer, setup_timer: nil, setup_attempts: 0} + + {:error, message} -> + delay = setup_retry_delay(state.setup_attempts) + level = if state.setup_attempts == 0, do: :warning, else: :debug + + Logger.log( + level, + "#{message} -- running in degraded mode (flat files only); retrying setup in #{delay} ms" + ) - {:ok, state} + timer = Process.send_after(self(), :retry_setup, delay) + + %{ + state + | connected: false, + setup_timer: timer, + setup_attempts: state.setup_attempts + 1 + } end end + defp setup_retry_delay(attempts) do + base = Application.get_env(:hypatia, :arango_setup_retry_base_ms, 5_000) + cap = Application.get_env(:hypatia, :arango_setup_retry_max_ms, 5 * 60 * 1_000) + min(base * Integer.pow(2, min(attempts, 20)), cap) + end + # --- Public API --- @doc "Check if ArangoDB is connected" @@ -334,6 +378,10 @@ defmodule Hypatia.Data.ArangoDB do end end + def handle_info(:retry_setup, state) do + {:noreply, setup(%{state | setup_timer: nil})} + end + def handle_info(:sync_from_verisimdb, state) do if state.connected do do_sync(state) @@ -347,31 +395,72 @@ defmodule Hypatia.Data.ArangoDB do # --- HTTP Client --- - defp try_connect(state) do - url = "#{state.base_url}/_api/version" + # Probes the target database first, so a user who administers only that + # database (and cannot read `_system`) still connects. A 404 database-not- + # found there means the database is not created yet: fall back to the + # `_system` version probe, so that setup can try to create it. + defp reachable?(state) do + target = http_get("#{db_url(state)}/_api/version", state.auth_header) + + cond do + arango_version?(target) -> + true - case http_get(url, state.auth_header) do - {:ok, %{"server" => "arango"}} -> %{state | connected: true} - _ -> %{state | connected: false} + database_not_found?(target) -> + arango_version?(http_get("#{state.base_url}/_api/version", state.auth_header)) + + true -> + false end end + defp arango_version?({:ok, %{"server" => "arango"}}), do: true + defp arango_version?(_), do: false + + # ArangoDB answers a request for a database that does not exist with a 404 + # and errorNum 1228 (ERROR_ARANGO_DATABASE_NOT_FOUND). + defp database_not_found?({:error, {404, body}}) do + match?({:ok, %{"errorNum" => 1228}}, Jason.decode(body)) + end + + defp database_not_found?(_), do: false + + defp db_url(state), do: "#{state.base_url}/_db/#{state.database}" + defp ensure_schema(state) do - db_url = "#{state.base_url}/_db/#{state.database}" + # Every later step depends on the database, so stop here if it fails. + database = ensure_database(state) + + if schema_result_ok?(database), + do: ensure_collections_and_graph(state), + else: {:error, database} + end - # Create the database only when it is absent. Creating one needs - # Administrate on `_system`; a least-privilege user that administers only - # a pre-provisioned target database gets a 403 there, so probe first. - database = - case http_get("#{db_url}/_api/database/current", state.auth_header) do - {:ok, %{} = current} -> - {:ok, 200, current} + # Creates the database only when the probe says it does not exist (404, + # errorNum 1228). Creating one needs Administrate on `_system`; a + # least-privilege user that administers only a pre-provisioned target + # database gets a 403 there. Any other probe failure (a timeout, a 3xx, a + # 5xx) is no evidence of absence and is returned as it is. + defp ensure_database(state) do + probe = http_get("#{db_url(state)}/_api/database/current", state.auth_header) - _absent_or_unreadable -> + case probe do + {:ok, %{} = current} -> + {:ok, 200, current} + + _ -> + if database_not_found?(probe) do http_post_status("#{state.base_url}/_api/database", state.auth_header, %{ "name" => state.database }) - end + else + probe + end + end + end + + defp ensure_collections_and_graph(state) do + db_url = db_url(state) # Create document collections documents = @@ -405,30 +494,18 @@ defmodule Hypatia.Data.ArangoDB do # Create named graph graph = - http_post_status("#{db_url}/_api/gharial", state.auth_header, %{ - "name" => "hypatia_graph", - "edgeDefinitions" => [ - %{ - "collection" => "trusts", - "from" => ["repos", "bots", "recipes", "contributors"], - "to" => ["repos", "bots", "recipes", "contributors"] - }, - %{"collection" => "depends_on", "from" => ["repos"], "to" => ["repos"]}, - %{"collection" => "dispatches", "from" => ["findings"], "to" => ["bots"]}, - %{"collection" => "applies_to", "from" => ["recipes"], "to" => ["patterns"]}, - %{"collection" => "repo_has_finding", "from" => ["repos"], "to" => ["findings"]}, - %{ - "collection" => "finding_matches_pattern", - "from" => ["findings"], - "to" => ["patterns"] - }, - %{"collection" => "bot_executes_recipe", "from" => ["bots"], "to" => ["recipes"]}, - %{"collection" => "recipe_has_outcome", "from" => ["recipes"], "to" => ["outcomes"]}, - %{"collection" => "contributor_fixes", "from" => ["contributors"], "to" => ["outcomes"]} - ] - }) - - results = [database] ++ documents ++ edges ++ indices ++ [graph] + case http_post_status("#{db_url}/_api/gharial", state.auth_header, %{ + "name" => @graph_name, + "edgeDefinitions" => @graph_edge_definitions + }) do + {:ok, status, %{"errorNum" => 1925}} when status in 400..499 -> + check_existing_graph(state) + + created -> + created + end + + results = documents ++ edges ++ indices ++ [graph] case Enum.find(results, &(not schema_result_ok?(&1))) do nil -> @@ -443,15 +520,51 @@ defmodule Hypatia.Data.ArangoDB do end end + # A duplicate-graph answer (1925) says only that a graph of that name + # exists, not that it is ours. It is accepted when, for every required + # edge definition, the existing graph has an edge definition on the same + # collection whose `from` and `to` vertex collections are exactly the + # required ones (as sets: order and duplicates do not matter). Extra edge + # definitions and orphan collections in the existing graph are allowed. + # A mismatch is a schema failure; the graph is never replaced. + defp check_existing_graph(state) do + case http_get("#{db_url(state)}/_api/gharial/#{@graph_name}", state.auth_header) do + {:ok, %{"graph" => %{"edgeDefinitions" => existing}} = body} when is_list(existing) -> + case graph_mismatches(existing) do + [] -> {:ok, 200, body} + mismatches -> {:error, {:graph_mismatch, mismatches}} + end + + other -> + {:error, {:graph_unreadable, other}} + end + end + + defp graph_mismatches(existing) do + by_collection = Map.new(existing, &{&1["collection"], &1}) + + @graph_edge_definitions + |> Enum.reject(&same_vertices?(&1, Map.get(by_collection, &1["collection"]))) + |> Enum.map(& &1["collection"]) + end + + defp same_vertices?(required, %{} = actual) do + Enum.all?(["from", "to"], fn side -> + is_list(actual[side]) and MapSet.new(actual[side]) == MapSet.new(required[side]) + end) + end + + defp same_vertices?(_required, nil), do: false + # A schema step succeeds on a 2xx, or on a 4xx that says the resource - # already exists (1207 duplicate name, 1925 duplicate graph), which happens - # on every initialisation after the first. Any other status fails, even with - # a JSON body: a gateway's `{"message": "forbidden"}` is not a success. + # already exists (1207 duplicate name), which happens on every + # initialisation after the first. A duplicate graph (1925) is checked by + # check_existing_graph/1 instead. Any other status fails, even with a JSON + # body: a gateway's `{"message": "forbidden"}` is not a success. defp schema_result_ok?({:ok, status, _}) when status in 200..299, do: true - defp schema_result_ok?({:ok, status, %{"errorNum" => number}}) - when status in 400..499 and number in [1207, 1925], - do: true + defp schema_result_ok?({:ok, status, %{"errorNum" => 1207}}) when status in 400..499, + do: true defp schema_result_ok?(_), do: false diff --git a/test/arangodb_initialisation_test.exs b/test/arangodb_initialisation_test.exs index 9fb2af0a..4bf4563a 100644 --- a/test/arangodb_initialisation_test.exs +++ b/test/arangodb_initialisation_test.exs @@ -6,13 +6,26 @@ defmodule Hypatia.Data.ArangoDBInitialisationTest do alias Hypatia.Data.ArangoDB defmodule Stub do - def init(respond), do: respond + def init(arg), do: arg - def call(conn, respond) do + # Every response carries `location: /redirect-target`, and that path + # answers a terminal 200 that would read as success, so a client that + # followed a redirect would pass a 3xx test. Any request to it is + # reported to the test process. + def call(%{request_path: "/redirect-target"} = conn, {parent, _respond}) do + send(parent, {:redirect_target, conn.method}) + + conn + |> Plug.Conn.put_resp_content_type("application/json") + |> Plug.Conn.send_resp(200, Jason.encode!(%{"error" => false, "result" => []})) + end + + def call(conn, {_parent, respond}) do {:ok, _body, conn} = Plug.Conn.read_body(conn) - # A route the test does not describe answers 404, as ArangoDB does for - # an unknown database or path, so the probe of the target database - # falls through to creating it. + # A route the test does not describe answers 404 database-not-found + # (errorNum 1228), as ArangoDB does for an unknown database, so the + # probes of the target database fall through to `_system` and to + # creating it. {status, body} = try do respond.(conn.method, conn.request_path) @@ -52,6 +65,8 @@ defmodule Hypatia.Data.ArangoDBInitialisationTest do assert {:reply, {:error, {unquote(status), _}}, ^state} = ArangoDB.handle_call({:query, "RETURN 1", %{}}, nil, state) + + refute_received {:redirect_target, _} end end @@ -92,6 +107,7 @@ defmodule Hypatia.Data.ArangoDBInitialisationTest do assert log =~ "schema initialisation failed" refute log =~ "schema ensured" refute log =~ "ArangoDB connected:" + refute_received {:redirect_target, _} end end @@ -108,6 +124,9 @@ defmodule Hypatia.Data.ArangoDBInitialisationTest do else {201, %{"error" => false}} end + + "GET", "/_db/hypatia/_api/gharial/hypatia_graph" -> + {200, graph(required_edge_definitions())} end) capture_log(fn -> @@ -161,6 +180,9 @@ defmodule Hypatia.Data.ArangoDBInitialisationTest do "GET", "/_api/version" -> {200, %{"server" => "arango"}} + "GET", "/_db/hypatia/_api/version" -> + {200, %{"server" => "arango"}} + "GET", path -> if String.ends_with?(path, "/_api/database/current"), do: {200, %{"error" => false, "result" => %{"name" => "hypatia"}}}, @@ -211,8 +233,296 @@ defmodule Hypatia.Data.ArangoDBInitialisationTest do assert_received :created_database end + # The connectivity probe asks the target database, so a user who + # administers only that database connects although `_system` refuses it. + test "connectivity is probed on the target database, not _system" do + parent = self() + + serve(fn + "GET", "/_db/hypatia/_api/version" -> + {200, %{"server" => "arango"}} + + "GET", "/_api/version" -> + send(parent, :probed_system) + {403, %{"error" => true, "errorNum" => 11}} + + "GET", "/_db/hypatia/_api/database/current" -> + {200, %{"error" => false, "result" => %{"name" => "hypatia"}}} + + "POST", _ -> + {201, %{"error" => false}} + end) + + capture_log(fn -> + assert {:ok, state} = ArangoDB.init([]) + assert state.connected + Process.cancel_timer(state.sync_timer) + end) + + refute_received :probed_system + end + + # Only a 404 database-not-found on the target falls back to `_system`; any + # other failure leaves the client disconnected. + test "a failing target probe other than 404 does not fall back to _system" do + parent = self() + + serve(fn + "GET", "/_db/hypatia/_api/version" -> + {503, %{"error" => true, "errorNum" => 503}} + + "GET", "/_api/version" -> + send(parent, :probed_system) + {200, %{"server" => "arango"}} + end) + + capture_log(fn -> + assert {:ok, state} = ArangoDB.init([]) + refute state.connected + cancel(state.setup_timer) + end) + + refute_received :probed_system + end + + # A failed probe of the target database is no evidence that it is absent: + # only a 404 with errorNum 1228 leads to creating it. + for status <- [307, 500, 503] do + test "a #{status} from the database probe fails setup without creating it" do + parent = self() + + serve(fn + "GET", "/_db/hypatia/_api/version" -> + {200, %{"server" => "arango"}} + + "GET", "/_db/hypatia/_api/database/current" -> + {unquote(status), %{"error" => true, "errorNum" => 1228}} + + "POST", path -> + send(parent, {:post, path}) + {201, %{"error" => false}} + end) + + log = + capture_log(fn -> + assert {:ok, state} = ArangoDB.init([]) + refute state.connected + cancel(state.setup_timer) + end) + + assert log =~ "schema initialisation failed" + refute_received {:post, _} + refute_received {:redirect_target, _} + end + end + + # When the database step fails, none of the collection, index or graph + # requests is sent. + test "a failed database step stops setup before the dependent requests" do + parent = self() + + serve(fn + "GET", "/_api/version" -> + {200, %{"server" => "arango"}} + + "POST", "/_api/database" -> + {403, %{"error" => true, "errorNum" => 11}} + + "POST", path -> + send(parent, {:post, path}) + {201, %{"error" => false}} + end) + + capture_log(fn -> + assert {:ok, state} = ArangoDB.init([]) + refute state.connected + cancel(state.setup_timer) + end) + + refute_received {:post, _} + end + + # A duplicate graph (1925) is accepted only when the existing graph has + # the required edge definitions; order within from/to and extra edge + # definitions do not matter. + test "an existing graph with the required edge definitions is accepted" do + existing = + [%{"collection" => "extra", "from" => ["repos"], "to" => ["bots"]}] ++ + Enum.map(required_edge_definitions(), fn d -> + %{d | "from" => Enum.reverse(d["from"]), "to" => Enum.reverse(d["to"])} + end) + + serve_existing_graph(existing) + + capture_log(fn -> + assert {:ok, state} = ArangoDB.init([]) + assert state.connected + Process.cancel_timer(state.sync_timer) + end) + + assert_received {:graph_write, "POST"} + refute_received {:graph_write, _} + end + + for {name, change} <- [ + {"different vertex collections", + quote( + do: fn d -> + if d["collection"] == "dispatches", do: %{d | "to" => ["repos"]}, else: d + end + )}, + {"a missing edge collection", + quote(do: fn d -> if d["collection"] == "trusts", do: nil, else: d end)} + ] do + test "an existing graph with #{name} is a schema failure, not replaced" do + existing = + required_edge_definitions() + |> Enum.map(unquote(change)) + |> Enum.reject(&is_nil/1) + + serve_existing_graph(existing) + + log = + capture_log(fn -> + assert {:ok, state} = ArangoDB.init([]) + refute state.connected + assert state.sync_timer == nil + cancel(state.setup_timer) + end) + + assert log =~ "graph_mismatch" + refute log =~ "schema ensured" + assert_received {:graph_write, "POST"} + refute_received {:graph_write, _} + end + end + + # A transient failure during setup leaves the client disconnected, with a + # retry scheduled and no sync timer; the retry accepts what the first + # attempt created (1207) and then connects and starts syncing. + test "a transient schema failure is retried until setup succeeds" do + put_retry_env(10, 40) + first = :atomics.new(1, []) + + serve(fn + "GET", "/_api/version" -> + {200, %{"server" => "arango"}} + + "GET", "/_db/hypatia/_api/database/current" -> + {200, %{"error" => false, "result" => %{"name" => "hypatia"}}} + + "POST", "/_db/hypatia/_api/collection" -> + if :atomics.exchange(first, 1, 1) == 0, + do: {503, %{"error" => true, "errorNum" => 503}}, + else: {409, %{"error" => true, "errorNum" => 1207}} + + "POST", _ -> + {201, %{"error" => false}} + end) + + capture_log(fn -> + {:ok, pid} = GenServer.start(ArangoDB, []) + on_exit(fn -> if Process.alive?(pid), do: GenServer.stop(pid) end) + + assert %{connected: false, sync_timer: nil, setup_timer: retry} = :sys.get_state(pid) + assert is_reference(retry) + + assert eventually(fn -> GenServer.call(pid, :connected?) end) + assert %{sync_timer: sync, setup_timer: nil, setup_attempts: 0} = :sys.get_state(pid) + assert is_reference(sync) + end) + end + + test "an unreachable server keeps retrying with backoff and never syncs" do + put_retry_env(5, 20) + serve(fn "GET", _ -> {503, %{"error" => true}} end) + + capture_log(fn -> + {:ok, pid} = GenServer.start(ArangoDB, []) + on_exit(fn -> if Process.alive?(pid), do: GenServer.stop(pid) end) + + assert eventually(fn -> :sys.get_state(pid).setup_attempts >= 4 end) + assert %{connected: false, sync_timer: nil} = :sys.get_state(pid) + end) + end + + defp serve_existing_graph(existing) do + parent = self() + + serve(fn + "GET", "/_api/version" -> + {200, %{"server" => "arango"}} + + "GET", "/_db/hypatia/_api/gharial/hypatia_graph" -> + {200, graph(existing)} + + method, "/_db/hypatia/_api/gharial" <> _ when method != "GET" -> + send(parent, {:graph_write, method}) + {409, %{"error" => true, "errorNum" => 1925}} + + "POST", _ -> + {201, %{"error" => false}} + end) + end + + defp graph(edge_definitions) do + %{ + "error" => false, + "code" => 200, + "graph" => %{ + "name" => "hypatia_graph", + "edgeDefinitions" => edge_definitions, + "orphanCollections" => [] + } + } + end + + defp required_edge_definitions do + vertices = ["repos", "bots", "recipes", "contributors"] + + [ + %{"collection" => "trusts", "from" => vertices, "to" => vertices}, + %{"collection" => "depends_on", "from" => ["repos"], "to" => ["repos"]}, + %{"collection" => "dispatches", "from" => ["findings"], "to" => ["bots"]}, + %{"collection" => "applies_to", "from" => ["recipes"], "to" => ["patterns"]}, + %{"collection" => "repo_has_finding", "from" => ["repos"], "to" => ["findings"]}, + %{"collection" => "finding_matches_pattern", "from" => ["findings"], "to" => ["patterns"]}, + %{"collection" => "bot_executes_recipe", "from" => ["bots"], "to" => ["recipes"]}, + %{"collection" => "recipe_has_outcome", "from" => ["recipes"], "to" => ["outcomes"]}, + %{"collection" => "contributor_fixes", "from" => ["contributors"], "to" => ["outcomes"]} + ] + end + + defp put_retry_env(base, max) do + for {key, value} <- [arango_setup_retry_base_ms: base, arango_setup_retry_max_ms: max] do + previous = Application.fetch_env(:hypatia, key) + Application.put_env(:hypatia, key, value) + + on_exit(fn -> + case previous do + {:ok, old} -> Application.put_env(:hypatia, key, old) + :error -> Application.delete_env(:hypatia, key) + end + end) + end + end + + defp eventually(check, tries \\ 100) do + cond do + check.() -> true + tries == 0 -> false + true -> Process.sleep(10) && eventually(check, tries - 1) + end + end + + # init/1 called from a test schedules its retry on the test process. + defp cancel(timer) when is_reference(timer), do: Process.cancel_timer(timer) + defp cancel(_), do: :ok + defp serve(respond) do - pid = start_supervised!({Bandit, plug: {Stub, respond}, ip: {127, 0, 0, 1}, port: 0}) + pid = + start_supervised!({Bandit, plug: {Stub, {self(), respond}}, ip: {127, 0, 0, 1}, port: 0}) + {:ok, {_, port}} = ThousandIsland.listener_info(pid) url = "http://127.0.0.1:#{port}" System.put_env("HYPATIA_ARANGO_URL", url) diff --git a/test/arangodb_redirect_status_test.exs b/test/arangodb_redirect_status_test.exs index 74883893..e277880a 100644 --- a/test/arangodb_redirect_status_test.exs +++ b/test/arangodb_redirect_status_test.exs @@ -51,9 +51,12 @@ defmodule Hypatia.Data.ArangoDBRedirectStatusTest do reply = cond do - String.starts_with?(head, "GET /_api/version") -> + String.starts_with?(head, "GET ") and String.contains?(head, "/_api/version ") -> respond(200, ~s({"server":"arango","version":"3.12.0"}), "") + String.starts_with?(head, "GET ") -> + respond(200, ~s({"error":false,"result":{"name":"hypatia"}}), "") + String.starts_with?(head, "POST ") and String.contains?(head, "/_api/cursor ") -> respond(307, "{}", "Location: http://127.0.0.1:1/elsewhere\r\n") From 4cbc9acf5c2047f3e91c13bfc4e015a963b3bac7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 15:27:35 +0000 Subject: [PATCH 10/12] fix(arangodb): a sync run with a failed upsert is not recorded as complete sync_scans/2, sync_outcomes/2 and sync_recipes/2 now return the upserts that failed ({collection, key, reason}) instead of discarding the results, and do_sync/1 returns :ok or {:error, failures}. A cursor answer without a result list counts as a failure too. On any failure the run is logged as failed with the count and the first failure, "complete" is not logged, and last_sync keeps the last successful timestamp, for both the cast and the timer path. Automatic redirects stay disabled. New test file test/arangodb_sync_test.exs: a failure in each of the four collections keeps last_sync, and a clean run advances it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01J4QvDgzT72Tq2HTJmikyNg --- lib/data/arangodb.ex | 160 ++++++++++++++++++++---------------- test/arangodb_sync_test.exs | 102 +++++++++++++++++++++++ 2 files changed, 190 insertions(+), 72 deletions(-) create mode 100644 test/arangodb_sync_test.exs diff --git a/lib/data/arangodb.ex b/lib/data/arangodb.ex index 0dbc18b0..82977413 100644 --- a/lib/data/arangodb.ex +++ b/lib/data/arangodb.ex @@ -371,22 +371,29 @@ defmodule Hypatia.Data.ArangoDB do def handle_cast(:sync_from_verisimdb, state) do if state.connected do - do_sync(state) - {:noreply, %{state | last_sync: DateTime.utc_now()}} + {:noreply, sync_and_record(state)} else {:noreply, state} end end + # `last_sync` moves only on a run in which every upsert succeeded; after a + # failed run it keeps the last successful timestamp. + defp sync_and_record(state) do + case do_sync(state) do + :ok -> %{state | last_sync: DateTime.utc_now()} + {:error, _failures} -> state + end + end + def handle_info(:retry_setup, state) do {:noreply, setup(%{state | setup_timer: nil})} end def handle_info(:sync_from_verisimdb, state) do if state.connected do - do_sync(state) timer = Process.send_after(self(), :sync_from_verisimdb, @sync_interval_ms) - {:noreply, %{state | last_sync: DateTime.utc_now(), sync_timer: timer}} + {:noreply, %{sync_and_record(state) | sync_timer: timer}} else timer = Process.send_after(self(), :sync_from_verisimdb, @sync_interval_ms) {:noreply, %{state | sync_timer: timer}} @@ -612,16 +619,45 @@ defmodule Hypatia.Data.ArangoDB do # --- verisim-data Sync --- + # Returns :ok when every upsert succeeded, otherwise {:error, failures}, + # each failure `{collection, key, reason}` (or the exception that stopped + # the run). A failed run is logged, never reported as complete. defp do_sync(state) do verisimdb_path = Application.get_env(:hypatia, :verisimdb_data_path, "data/verisim") - sync_scans(state, verisimdb_path) - sync_outcomes(state, verisimdb_path) - sync_recipes(state, verisimdb_path) + failures = + sync_scans(state, verisimdb_path) ++ + sync_outcomes(state, verisimdb_path) ++ + sync_recipes(state, verisimdb_path) - Logger.info("ArangoDB sync from verisim-data complete") + case failures do + [] -> + Logger.info("ArangoDB sync from verisim-data complete") + :ok + + [first | _] -> + Logger.error( + "ArangoDB sync from verisim-data failed (#{length(failures)} failed upserts, " <> + "first: #{inspect(first)})" + ) + + {:error, failures} + end rescue - e -> Logger.error("ArangoDB sync failed: #{inspect(e)}") + e -> + Logger.error("ArangoDB sync failed: #{inspect(e)}") + {:error, [e]} + end + + # Upserts and returns the failure, if any, as a list for flat_map. A + # cursor answer without a result list (an error body with no message) is a + # failure too. + defp sync_upsert(state, collection, key, document) do + case do_upsert(state, collection, key, document) do + {:ok, result} when is_list(result) -> [] + {:error, reason} -> [{collection, key, reason}] + other -> [{collection, key, other}] + end end defp sync_scans(state, base_path) do @@ -629,48 +665,40 @@ defmodule Hypatia.Data.ArangoDB do case File.ls(scans_dir) do {:ok, repos} -> - Enum.each(repos, fn repo_dir -> + Enum.flat_map(repos, fn repo_dir -> scan_path = Path.join([scans_dir, repo_dir, "latest.json"]) - case File.read(scan_path) do - {:ok, content} -> - case Jason.decode(content) do - {:ok, scan} -> - findings = Map.get(scan, "findings", []) - - do_upsert(state, "repos", repo_dir, %{ - "name" => repo_dir, - "finding_count" => length(findings), - "last_scanned" => Map.get(scan, "timestamp", ""), - "status" => "active" - }) - - # Sync individual findings - Enum.each(findings, fn finding -> - f_key = "#{repo_dir}_#{:erlang.phash2(finding)}" - - do_upsert( - state, - "findings", - f_key, - Map.merge(finding, %{ - "repo" => repo_dir, - "status" => "open" - }) - ) - end) - - _ -> - :skip - end - - _ -> - :skip + with {:ok, content} <- File.read(scan_path), + {:ok, scan} <- Jason.decode(content) do + findings = Map.get(scan, "findings", []) + + repo = + sync_upsert(state, "repos", repo_dir, %{ + "name" => repo_dir, + "finding_count" => length(findings), + "last_scanned" => Map.get(scan, "timestamp", ""), + "status" => "active" + }) + + # Sync individual findings + repo ++ + Enum.flat_map(findings, fn finding -> + f_key = "#{repo_dir}_#{:erlang.phash2(finding)}" + + sync_upsert( + state, + "findings", + f_key, + Map.merge(finding, %{"repo" => repo_dir, "status" => "open"}) + ) + end) + else + _ -> [] end end) _ -> - :ok + [] end end @@ -681,31 +709,28 @@ defmodule Hypatia.Data.ArangoDB do {:ok, files} -> files |> Enum.filter(&String.ends_with?(&1, ".jsonl")) - |> Enum.each(fn file -> - path = Path.join(outcomes_dir, file) - - case File.read(path) do + |> Enum.flat_map(fn file -> + case File.read(Path.join(outcomes_dir, file)) do {:ok, content} -> content |> String.split("\n", trim: true) - |> Enum.each(fn line -> + |> Enum.flat_map(fn line -> case Jason.decode(line) do {:ok, outcome} -> - key = "outcome_#{:erlang.phash2(outcome)}" - do_upsert(state, "outcomes", key, outcome) + sync_upsert(state, "outcomes", "outcome_#{:erlang.phash2(outcome)}", outcome) _ -> - :skip + [] end end) _ -> - :skip + [] end end) _ -> - :ok + [] end end @@ -716,27 +741,18 @@ defmodule Hypatia.Data.ArangoDB do {:ok, files} -> files |> Enum.filter(&String.ends_with?(&1, ".json")) - |> Enum.each(fn file -> - path = Path.join(recipes_dir, file) - - case File.read(path) do - {:ok, content} -> - case Jason.decode(content) do - {:ok, recipe} -> - key = Map.get(recipe, "id", Path.rootname(file)) - do_upsert(state, "recipes", key, recipe) - - _ -> - :skip - end - - _ -> - :skip + |> Enum.flat_map(fn file -> + with {:ok, content} <- File.read(Path.join(recipes_dir, file)), + {:ok, recipe} <- Jason.decode(content) do + key = Map.get(recipe, "id", Path.rootname(file)) + sync_upsert(state, "recipes", key, recipe) + else + _ -> [] end end) _ -> - :ok + [] end end diff --git a/test/arangodb_sync_test.exs b/test/arangodb_sync_test.exs new file mode 100644 index 00000000..271b43d4 --- /dev/null +++ b/test/arangodb_sync_test.exs @@ -0,0 +1,102 @@ +# SPDX-License-Identifier: MPL-2.0 + +defmodule Hypatia.Data.ArangoDBSyncTest do + # A sync run counts as complete only when every upsert succeeded. After a + # failed upsert the run is logged as failed and `last_sync` keeps the last + # successful timestamp. A Bandit stub stands in for ArangoDB's cursor API + # and fails the upserts into the collections a test names. + use ExUnit.Case, async: false + import ExUnit.CaptureLog + alias Hypatia.Data.ArangoDB + + defmodule Stub do + def init(failing), do: failing + + def call(conn, failing) do + {:ok, body, conn} = Plug.Conn.read_body(conn) + collection = get_in(Jason.decode!(body), ["bindVars", "@collection"]) + + {status, reply} = + if collection in failing, + do: {500, %{"error" => true, "errorNum" => 4}}, + else: {201, %{"error" => false, "result" => [%{"_key" => "k"}]}} + + conn + |> Plug.Conn.put_resp_content_type("application/json") + |> Plug.Conn.send_resp(status, Jason.encode!(reply)) + end + end + + @previous ~U[2026-01-01 00:00:00Z] + + setup do + dir = Path.join(System.tmp_dir!(), "arangodb-sync-#{System.unique_integer([:positive])}") + File.mkdir_p!(Path.join([dir, "scans", "repo-a"])) + File.mkdir_p!(Path.join(dir, "outcomes")) + File.mkdir_p!(Path.join(dir, "recipes")) + + File.write!( + Path.join([dir, "scans", "repo-a", "latest.json"]), + Jason.encode!(%{"findings" => [%{"rule" => "r1"}]}) + ) + + File.write!(Path.join([dir, "outcomes", "2026-01.jsonl"]), ~s({"recipe_id":"x"}\n)) + File.write!(Path.join([dir, "recipes", "x.json"]), ~s({"id":"x"})) + + previous = Application.get_env(:hypatia, :verisimdb_data_path) + Application.put_env(:hypatia, :verisimdb_data_path, dir) + + on_exit(fn -> + Application.put_env(:hypatia, :verisimdb_data_path, previous) + File.rm_rf!(dir) + end) + + :ok + end + + for failing <- [["repos"], ["findings"], ["outcomes"], ["recipes"]] do + test "a failed #{hd(failing)} upsert keeps last_sync (cast and timer)" do + state = state(unquote(failing)) + + log = + capture_log(fn -> + assert {:noreply, after_cast} = ArangoDB.handle_cast(:sync_from_verisimdb, state) + assert after_cast.last_sync == @previous + + assert {:noreply, after_timer} = ArangoDB.handle_info(:sync_from_verisimdb, state) + assert after_timer.last_sync == @previous + assert is_reference(after_timer.sync_timer) + Process.cancel_timer(after_timer.sync_timer) + end) + + assert log =~ "sync from verisim-data failed (1 failed upserts" + assert log =~ inspect(hd(unquote(failing))) + refute log =~ "sync from verisim-data complete" + end + end + + test "a run in which every upsert succeeds advances last_sync" do + state = state([]) + + log = + capture_log(fn -> + assert {:noreply, after_cast} = ArangoDB.handle_cast(:sync_from_verisimdb, state) + assert DateTime.compare(after_cast.last_sync, @previous) == :gt + end) + + assert log =~ "sync from verisim-data complete" + end + + defp state(failing) do + pid = start_supervised!({Bandit, plug: {Stub, failing}, ip: {127, 0, 0, 1}, port: 0}) + {:ok, {_, port}} = ThousandIsland.listener_info(pid) + + %ArangoDB{ + base_url: "http://127.0.0.1:#{port}", + database: "test", + auth_header: "Basic test-dummy", + connected: true, + last_sync: @previous + } + end +end From 2004b0092c0b3046d9d8af12a53849df4c84335e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 15:27:59 +0000 Subject: [PATCH 11/12] test(httpc): check each redirect probe sends its credentials first The redirector now reports the authorization header it receives on /start, and every probe asserts it is the expected credential: the Bearer token for :header, and Basic with the base64 userinfo for :userinfo. An empty authorization list at /final then shows the runtime dropped the credentials, not that the probe never sent any. The cross-origin assertion that /final receives none is unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01J4QvDgzT72Tq2HTJmikyNg --- test/httpc_redirect_credentials_test.exs | 28 +++++++++++++++++++----- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/test/httpc_redirect_credentials_test.exs b/test/httpc_redirect_credentials_test.exs index 145668d9..763a51f3 100644 --- a/test/httpc_redirect_credentials_test.exs +++ b/test/httpc_redirect_credentials_test.exs @@ -17,8 +17,10 @@ defmodule Hypatia.HttpcRedirectCredentialsTest do # do not depend on the pin alone. # # Each case runs a redirector and a target on loopback. The redirector - # answers with STATUS and a Location on the target; the target reports the - # request line and any authorization header it receives. + # answers with STATUS and a Location on the target, and reports the + # authorization header it received, which must be the expected credential + # (otherwise an empty list at the target would prove nothing). The target + # reports the request line and any authorization header it receives. use ExUnit.Case, async: true @token "Bearer test-dummy-redirect-probe" @@ -101,7 +103,7 @@ defmodule Hypatia.HttpcRedirectCredentialsTest do server = spawn_link(fn -> {:ok, s1} = :gen_tcp.accept(front, 5_000) - _ = read_request(s1) + send(parent, {:start, s1 |> read_request() |> authorization()}) :ok = :gen_tcp.send( @@ -112,9 +114,9 @@ defmodule Hypatia.HttpcRedirectCredentialsTest do :gen_tcp.close(s1) {:ok, s2} = :gen_tcp.accept(target, 5_000) - [line | headers] = s2 |> read_request() |> String.split("\r\n") - auth = Enum.filter(headers, &String.starts_with?(String.downcase(&1), "authorization:")) - send(parent, {:seen, line, auth}) + request = read_request(s2) + [line | _] = String.split(request, "\r\n") + send(parent, {:seen, line, authorization(request)}) :gen_tcp.send(s2, "HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") :gen_tcp.close(s2) end) @@ -138,12 +140,26 @@ defmodule Hypatia.HttpcRedirectCredentialsTest do {:ok, {{_, 200, _}, _, _}} = :httpc.request(method, request, [timeout: 5_000, autoredirect: true], []) + # The probe is valid only if the credentials did reach the redirector. + assert_receive {:start, start_auth}, 5_000 + assert start_auth == [expected_authorization(cred)] assert_receive {:seen, line, auth}, 5_000 Process.unlink(server) Enum.each([front, back], &:gen_tcp.close/1) {line, auth} end + defp expected_authorization(:header), do: "authorization: #{@token}" + defp expected_authorization(:userinfo), do: "authorization: Basic " <> Base.encode64(@userinfo) + + # The authorization header lines of a request head. + defp authorization(request) do + request + |> String.split("\r\n") + |> tl() + |> Enum.filter(&String.starts_with?(String.downcase(&1), "authorization:")) + end + defp listen(ip, port \\ 0) do :gen_tcp.listen(port, [:binary, active: false, reuseaddr: true, ip: ip]) end From d05df322ce70cbfafad74bb6a91e2a37f91124ec Mon Sep 17 00:00:00 2001 From: "coderabbitai[bot]" <136622811+coderabbitai[bot]@users.noreply.github.com> Date: Wed, 18 Nov 2026 19:01:56 +0000 Subject: [PATCH 12/12] fix(shell): avoid SIGPIPE failures from grep -q in pipelines --- hooks/validate-sha-pins.sh | 2 +- hooks/validate-spdx.sh | 2 +- hypatia-cli-bash.sh | 2 +- scripts/auto-fix-formulaic.sh | 2 +- scripts/check-readme-claims.sh | 2 +- scripts/ci-health/detect.sh | 6 +++--- scripts/ci-health/remediate.sh | 4 ++-- scripts/ci-health/sweep.sh | 2 +- scripts/fix-scripts/fix-http-to-https.sh | 2 +- scripts/fix-scripts/fix-missing-spdx.sh | 2 +- scripts/fix-scripts/fix-pin-action-sha.sh | 2 +- scripts/fix-scripts/fix-pin-rust-deps.sh | 2 +- scripts/fix-scripts/fix-pmpl-drift.sh | 6 +++--- scripts/fix-scripts/fix-sast-workflow.sh | 4 ++-- .../fix-scorecard-binary-artifacts.sh | 2 +- scripts/fix-scripts/fix-scorecard-ci-tests.sh | 2 +- scripts/gen-rules-praxis.sh | 2 +- scripts/security-check.sh | 8 ++++---- .../check-standards-integration.sh | 8 ++++---- scripts/sweeps/estate-pin-integrity.sh | 4 ++-- scripts/sweeps/estate-pr-automerge.sh | 4 ++-- scripts/sweeps/patch_phase2.sh | 2 +- test/ci-health-test.sh | 20 +++++++++---------- test/fixtures/ci-health/mock-gh | 2 +- test/standards-integration-test.sh | 8 ++++---- tests/e2e.sh | 4 ++-- tests/rules-praxis-test.sh | 16 +++++++-------- 27 files changed, 61 insertions(+), 61 deletions(-) diff --git a/hooks/validate-sha-pins.sh b/hooks/validate-sha-pins.sh index 697092b5..5c5963dd 100755 --- a/hooks/validate-sha-pins.sh +++ b/hooks/validate-sha-pins.sh @@ -13,7 +13,7 @@ for workflow in .github/workflows/*.yml .github/workflows/*.yaml; do while IFS= read -r line; do if [[ "$line" =~ uses:.*@ ]]; then # Check if it has a SHA (40 hex chars) - if ! echo "$line" | grep -qE '@[a-f0-9]{40}'; then + if ! echo "$line" | grep -E '@[a-f0-9]{40}' >/dev/null; then echo "ERROR: Unpinned action in $workflow" echo " $line" echo " Actions must use SHA pins: uses: action/name@SHA # version" diff --git a/hooks/validate-spdx.sh b/hooks/validate-spdx.sh index cc81cf18..11517bd2 100755 --- a/hooks/validate-spdx.sh +++ b/hooks/validate-spdx.sh @@ -11,7 +11,7 @@ for workflow in .github/workflows/*.yml .github/workflows/*.yaml; do [ -f "$workflow" ] || continue first_line=$(head -n1 "$workflow") - if ! echo "$first_line" | grep -qE "$SPDX_PATTERN"; then + if ! echo "$first_line" | grep -E "$SPDX_PATTERN" >/dev/null; then echo "ERROR: Missing SPDX header in $workflow" echo " First line should be: # SPDX-License-Identifier: MPL-2.0" ERRORS=$((ERRORS + 1)) diff --git a/hypatia-cli-bash.sh b/hypatia-cli-bash.sh index a378331a..afc2afa1 100755 --- a/hypatia-cli-bash.sh +++ b/hypatia-cli-bash.sh @@ -618,7 +618,7 @@ scan_repo_structure() { if [[ -d "$dir/.github/workflows" ]]; then for wf in "$dir/.github/workflows"/*.yml "$dir/.github/workflows"/*.yaml; do [[ -f "$wf" ]] || continue - if ! head -1 "$wf" | grep -q '^# SPDX-License-Identifier:'; then + if ! head -1 "$wf" | grep '^# SPDX-License-Identifier:' >/dev/null; then emit_finding "medium" "repo_structure" "missing_spdx_header" \ "$wf" 1 "Workflow missing SPDX header" "CWE-1059" \ "Add '# SPDX-License-Identifier: MPL-2.0' as first line" "true" diff --git a/scripts/auto-fix-formulaic.sh b/scripts/auto-fix-formulaic.sh index 6b40c7dd..fa7a313e 100755 --- a/scripts/auto-fix-formulaic.sh +++ b/scripts/auto-fix-formulaic.sh @@ -100,7 +100,7 @@ fix_agpl_references() { for f in $agpl_files; do # Skip mock/fixture files that simulate external API responses - if echo "$f" | grep -qE 'mock|fixture.*github'; then + if echo "$f" | grep -E 'mock|fixture.*github' >/dev/null; then continue fi warn "AGPL-3.0 reference in: $f (should be MPL-2.0)" diff --git a/scripts/check-readme-claims.sh b/scripts/check-readme-claims.sh index dcf052f0..d68da048 100755 --- a/scripts/check-readme-claims.sh +++ b/scripts/check-readme-claims.sh @@ -80,7 +80,7 @@ for doc in "$readme" EXPLAINME.adoc docs/README.adoc; do if [ ! -e "$dir/$path" ]; then say "$doc links $target, which does not exist"; continue; fi if [ "$target" != "$path" ] && [[ "$path" == *.adoc ]]; then frag=${target#*#} - anchors "$dir/$path" | grep -qxF -- "$frag" || say "$doc links $target, and $path defines no anchor $frag" + anchors "$dir/$path" | grep -xF -- "$frag" >/dev/null || say "$doc links $target, and $path defines no anchor $frag" fi done < <(grep -oE 'link:[^[ ]+\[' "$doc" | sed -E 's/^link:(.*)\[$/\1/') done diff --git a/scripts/ci-health/detect.sh b/scripts/ci-health/detect.sh index 6c6d4fc2..b8ad2890 100755 --- a/scripts/ci-health/detect.sh +++ b/scripts/ci-health/detect.sh @@ -59,11 +59,11 @@ for fail_id in "${fail_ids[@]}"; do emit B-TRACKER-TOKEN HIGH "Pin updater reports missing write credential (run $fail_id) → compare pins read-only before provisioning repository-scoped write access" tracker_token_seen=true fi - if [ "$billing_seen" = false ] && printf '%s' "$msg" | grep -qiE 'payments have failed|spending limit'; then + if [ "$billing_seen" = false ] && printf '%s' "$msg" | grep -iE 'payments have failed|spending limit' >/dev/null; then emit A-BILLING CRITICAL "Actions billing/spending-limit wall blocks all billable jobs → OWNER: GitHub Settings -> Billing & plans" billing_seen=true fi - if [ "$lockfile_seen" = false ] && printf '%s' "$msg" | grep -qiE 'lockfile pin .* does not match ref|lockfile verification did not produce a result|unreachable-pin'; then + if [ "$lockfile_seen" = false ] && printf '%s' "$msg" | grep -iE 'lockfile pin .* does not match ref|lockfile verification did not produce a result|unreachable-pin' >/dev/null; then first_lock_msg=$(printf '%s\n' "$msg" | grep -iEm1 'lockfile pin .* does not match ref|lockfile verification did not produce a result|unreachable-pin') emit B-LOCKFILE HIGH "Actions lockfile verification failed (${first_lock_msg}) → run gh actions-lock --verify, repair, then commit actions.lock" lockfile_seen=true @@ -107,7 +107,7 @@ fi # --- D: burn anti-pattern (bare [push, pull_request] double-trigger), via API for path in $(api "repos/$O/$R/contents/.github/workflows" --jq '.[]?|select(.name|test("\\.ya?ml$"))|.path' || true); do if api "repos/$O/$R/contents/$path" --jq '.content' | base64 -d \ - | grep -qE '^on:[[:space:]]*\[[[:space:]]*push[[:space:]]*,[[:space:]]*pull_request[[:space:]]*\]'; then + | grep -E '^on:[[:space:]]*\[[[:space:]]*push[[:space:]]*,[[:space:]]*pull_request[[:space:]]*\]' >/dev/null; then emit D-BURN MEDIUM "ERR-WF-014: $path on bare [push,pull_request] (2x runs/PR) → scope push to default branch + concurrency-cancel" fi done diff --git a/scripts/ci-health/remediate.sh b/scripts/ci-health/remediate.sh index f1e57232..6208478c 100755 --- a/scripts/ci-health/remediate.sh +++ b/scripts/ci-health/remediate.sh @@ -68,7 +68,7 @@ case "$CLASS" in def=$(gh api "repos/$O/$R" --jq '.default_branch'); sha=$(gh api "repos/$O/$R/git/ref/heads/$def" --jq '.object.sha') targets=(); for p in $(gh api "repos/$O/$R/contents/.github/workflows" --jq '.[]?|select(.name|test("\\.ya?ml$"))|.path' 2>/dev/null); do gh api "repos/$O/$R/contents/$p?ref=$def" --jq '.content' 2>/dev/null | base64 -d 2>/dev/null \ - | grep -qE '^on:[[:space:]]*\[[[:space:]]*push[[:space:]]*,[[:space:]]*pull_request' && targets+=("$p") + | grep -E '^on:[[:space:]]*\[[[:space:]]*push[[:space:]]*,[[:space:]]*pull_request' >/dev/null && targets+=("$p") done [ "${#targets[@]}" -eq 0 ] && { echo "SKIP $R/D-BURN no-targets"; exit 0; } if [ "$DRY" = "true" ]; then echo "DRYRUN $R/D-BURN would patch ${#targets[@]} file(s) + open PR"; exit 0; fi @@ -84,7 +84,7 @@ case "$CLASS" in branches: [main, master] pull_request:" - if ! printf '%s' "$ccontent" | grep -q '^[[:space:]]*concurrency:'; then + if ! printf '%s' "$ccontent" | grep >/dev/null '^[[:space:]]*concurrency:'; then blk="$blk # Estate guardrail: scope push to default branches (PR fires once, not diff --git a/scripts/ci-health/sweep.sh b/scripts/ci-health/sweep.sh index b4c227d0..f6518aa7 100755 --- a/scripts/ci-health/sweep.sh +++ b/scripts/ci-health/sweep.sh @@ -33,7 +33,7 @@ while IFS=$'\t' read -r repo cls _sev _det; do D-BURN) if [ "$burned" -lt "$MAXPR" ]; then out=$(OWNER="$O" "$HERE/remediate.sh" "$repo" "$cls" "$DRY" || true); echo "$out" - echo "$out" | grep -q '^FIXED' && burned=$((burned+1)) + echo "$out" | grep >/dev/null '^FIXED' && burned=$((burned+1)) else echo "CAP $repo/D-BURN max-burn-prs($MAXPR) reached — deferred"; fi ;; esac done < <(sort -u "$findings") diff --git a/scripts/fix-scripts/fix-http-to-https.sh b/scripts/fix-scripts/fix-http-to-https.sh index 411b101b..0c4385f8 100755 --- a/scripts/fix-scripts/fix-http-to-https.sh +++ b/scripts/fix-scripts/fix-http-to-https.sh @@ -44,7 +44,7 @@ fix_file() { [[ "$file" == */.zig-cache/* ]] && return 0 # Only process text files - file -b --mime-type "$file" 2>/dev/null | grep -q "text/" || return 0 + file -b --mime-type "$file" 2>/dev/null | grep >/dev/null "text/" || return 0 # Build sed exclusion pattern local sed_cmd="s|http://|https://|g" diff --git a/scripts/fix-scripts/fix-missing-spdx.sh b/scripts/fix-scripts/fix-missing-spdx.sh index d44a99b4..a99b076e 100755 --- a/scripts/fix-scripts/fix-missing-spdx.sh +++ b/scripts/fix-scripts/fix-missing-spdx.sh @@ -67,7 +67,7 @@ fix_file() { local header="${COMMENT_STYLE[$ext]}" # Handle shebang lines — insert after shebang - if head -1 "$file" | grep -q "^#!"; then + if head -1 "$file" | grep >/dev/null "^#!"; then sed -i "1a\\${header}" "$file" else sed -i "1i\\${header}" "$file" diff --git a/scripts/fix-scripts/fix-pin-action-sha.sh b/scripts/fix-scripts/fix-pin-action-sha.sh index 928f0bc0..4c82739c 100755 --- a/scripts/fix-scripts/fix-pin-action-sha.sh +++ b/scripts/fix-scripts/fix-pin-action-sha.sh @@ -90,7 +90,7 @@ for wf in "${WORKFLOWS_DIR}"/*.yml "${WORKFLOWS_DIR}"/*.yaml; do [[ -z "$action_ref" ]] && continue # Already SHA-pinned (40 hex chars after @) — skip - if echo "$action_ref" | grep -qE '@[0-9a-f]{40}'; then + if echo "$action_ref" | grep -E >/dev/null '@[0-9a-f]{40}'; then continue fi diff --git a/scripts/fix-scripts/fix-pin-rust-deps.sh b/scripts/fix-scripts/fix-pin-rust-deps.sh index ee491849..d98eee46 100755 --- a/scripts/fix-scripts/fix-pin-rust-deps.sh +++ b/scripts/fix-scripts/fix-pin-rust-deps.sh @@ -27,7 +27,7 @@ is_binary_crate() { find "${REPO}" -name "main.rs" \ -not -path "*/.git/*" \ -not -path "*/target/*" \ - | grep -q . && return 0 + | grep >/dev/null . && return 0 fi return 1 } diff --git a/scripts/fix-scripts/fix-pmpl-drift.sh b/scripts/fix-scripts/fix-pmpl-drift.sh index d8858baa..0564fda9 100755 --- a/scripts/fix-scripts/fix-pmpl-drift.sh +++ b/scripts/fix-scripts/fix-pmpl-drift.sh @@ -55,13 +55,13 @@ fi LIC_HEAD=$(head -1 "$LICENSE_PATH" | tr -d '\n') -if echo "$LIC_HEAD" | grep -q "MPL-2.0"; then +if echo "$LIC_HEAD" | grep >/dev/null "MPL-2.0"; then TARGET="MPL-2.0" CATEGORY="sole-owner" -elif echo "$LIC_HEAD" | grep -q "AGPL-3.0"; then +elif echo "$LIC_HEAD" | grep >/dev/null "AGPL-3.0"; then TARGET="AGPL-3.0-or-later" CATEGORY="son-shared" -elif echo "$LIC_HEAD" | grep -q "PMPL-1.0-or-later"; then +elif echo "$LIC_HEAD" | grep >/dev/null "PMPL-1.0-or-later"; then echo "INFO: $REPO_NAME LICENSE is PMPL-1.0-or-later (carve-out or staging) — skipping" exit 0 else diff --git a/scripts/fix-scripts/fix-sast-workflow.sh b/scripts/fix-scripts/fix-sast-workflow.sh index 3ce48e5f..ed7959ba 100755 --- a/scripts/fix-scripts/fix-sast-workflow.sh +++ b/scripts/fix-scripts/fix-sast-workflow.sh @@ -22,11 +22,11 @@ mkdir -p "${REPO}/.github/workflows" # Detect languages for CodeQL matrix languages=() # JavaScript/TypeScript (includes ReScript compiled output) -if find "$REPO" -maxdepth 3 -name "*.js" -o -name "*.mjs" -o -name "*.res" 2>/dev/null | grep -q .; then +if find "$REPO" -maxdepth 3 -name "*.js" -o -name "*.mjs" -o -name "*.res" 2>/dev/null | grep >/dev/null .; then languages+=("javascript-typescript") fi # Ruby -if find "$REPO" -maxdepth 3 -name "*.rb" 2>/dev/null | grep -q .; then +if find "$REPO" -maxdepth 3 -name "*.rb" 2>/dev/null | grep >/dev/null .; then languages+=("ruby") fi diff --git a/scripts/fix-scripts/fix-scorecard-binary-artifacts.sh b/scripts/fix-scripts/fix-scorecard-binary-artifacts.sh index 2df0ddbf..1e43d396 100755 --- a/scripts/fix-scripts/fix-scorecard-binary-artifacts.sh +++ b/scripts/fix-scripts/fix-scorecard-binary-artifacts.sh @@ -23,7 +23,7 @@ count=0 echo "" while IFS= read -r -d '' file; do - if file "$file" | grep -qE 'ELF|PE32|Mach-O|compiled|bytecode|binary|data'; then + if file "$file" | grep -E >/dev/null 'ELF|PE32|Mach-O|compiled|bytecode|binary|data'; then echo "BINARY: ${file#"${REPO}/"}" count=$((count + 1)) fi diff --git a/scripts/fix-scripts/fix-scorecard-ci-tests.sh b/scripts/fix-scripts/fix-scorecard-ci-tests.sh index 2fe00061..7128da16 100755 --- a/scripts/fix-scripts/fix-scorecard-ci-tests.sh +++ b/scripts/fix-scripts/fix-scorecard-ci-tests.sh @@ -14,7 +14,7 @@ REPO="${1:?Usage: fix-scorecard-ci-tests.sh }" WF_DIR="${REPO}/.github/workflows" # Already has CI if any workflow runs tests -if ls "${WF_DIR}"/*.yml "${WF_DIR}"/*.yaml 2>/dev/null | xargs grep -l 'mix test\|cargo test\|deno test\|gleam test' 2>/dev/null | grep -q .; then +if ls "${WF_DIR}"/*.yml "${WF_DIR}"/*.yaml 2>/dev/null | xargs grep -l 'mix test\|cargo test\|deno test\|gleam test' 2>/dev/null | grep >/dev/null .; then echo "[fix-scorecard-ci-tests] CI test workflow already present" exit 0 fi diff --git a/scripts/gen-rules-praxis.sh b/scripts/gen-rules-praxis.sh index 6f00106c..48a0bcde 100755 --- a/scripts/gen-rules-praxis.sh +++ b/scripts/gen-rules-praxis.sh @@ -367,7 +367,7 @@ while IFS='~' read -r mod name f lit sev adv sf slit text; do grep -qF -- "$lit" "$f" || { echo "gen-rules-praxis: $f no longer has '$lit' ($mod/$name)" >&2; exit 1; } if [ "$sf" = "$f" ]; then # same file: beside the literal (a trailing #any: anywhere in it) n=$(grep -nF -- "$lit" "$f" | sed -n 1p | cut -d: -f1) - sed -n "$((n - 3)),$((n + 3))p" "$f" | grep -qF -- "$slit" || + sed -n "$((n - 3)),$((n + 3))p" "$f" | grep -F >/dev/null -- "$slit" || { echo "gen-rules-praxis: $f no longer states '$slit' beside '$lit' ($mod/$name)" >&2; exit 1; } else sf=${sf%#any} diff --git a/scripts/security-check.sh b/scripts/security-check.sh index 9b334237..f553aecf 100755 --- a/scripts/security-check.sh +++ b/scripts/security-check.sh @@ -504,7 +504,7 @@ check_spdx_headers() { # Check Rust files echo "## Rust Files (.rs)" >> "$report_file" while IFS= read -r -d '' f; do - if ! head -10 "$f" | grep -q "SPDX-License-Identifier"; then + if ! head -10 "$f" | grep >/dev/null "SPDX-License-Identifier"; then echo " - $f" >> "$report_file" missing_count=$((missing_count + 1)) fi @@ -514,7 +514,7 @@ check_spdx_headers() { echo "" >> "$report_file" echo "## Haskell Files (.hs)" >> "$report_file" while IFS= read -r -d '' f; do - if ! head -10 "$f" | grep -q "SPDX-License-Identifier"; then + if ! head -10 "$f" | grep >/dev/null "SPDX-License-Identifier"; then echo " - $f" >> "$report_file" missing_count=$((missing_count + 1)) fi @@ -524,7 +524,7 @@ check_spdx_headers() { echo "" >> "$report_file" echo "## Shell Scripts (.sh)" >> "$report_file" while IFS= read -r -d '' f; do - if ! head -10 "$f" | grep -q "SPDX-License-Identifier"; then + if ! head -10 "$f" | grep >/dev/null "SPDX-License-Identifier"; then echo " - $f" >> "$report_file" missing_count=$((missing_count + 1)) fi @@ -534,7 +534,7 @@ check_spdx_headers() { echo "" >> "$report_file" echo "## GitHub Workflows (.yml)" >> "$report_file" while IFS= read -r -d '' f; do - if ! head -1 "$f" | grep -q "SPDX-License-Identifier"; then + if ! head -1 "$f" | grep >/dev/null "SPDX-License-Identifier"; then echo " - $f" >> "$report_file" missing_count=$((missing_count + 1)) fi diff --git a/scripts/standards-integration/check-standards-integration.sh b/scripts/standards-integration/check-standards-integration.sh index cc445b4d..e79059a1 100755 --- a/scripts/standards-integration/check-standards-integration.sh +++ b/scripts/standards-integration/check-standards-integration.sh @@ -74,7 +74,7 @@ for f in "${CALLERS[@]}"; do [ -n "$uses_lines" ] || fail "$f declares no metadatastician/standards reusable" while IFS= read -r line; do ref=$(printf '%s' "$line" | sed -E 's/.*uses:[[:space:]]*//; s/[[:space:]].*$//') - if ! printf '%s' "$ref" | grep -qE '^metadatastician/standards/\.github/workflows/[A-Za-z0-9._-]+\.yml@[0-9a-f]{40}$'; then + if ! printf '%s' "$ref" | grep -E >/dev/null '^metadatastician/standards/\.github/workflows/[A-Za-z0-9._-]+\.yml@[0-9a-f]{40}$'; then fail "$f has a non-immutable standards ref (must be a full 40-hex SHA): $ref" fi pins+=("${ref##*@}") @@ -97,7 +97,7 @@ for f in "${CALLERS[@]}"; do # Every pin used by this caller (normally exactly one) must be locked. while IFS= read -r line; do ref=$(printf '%s' "$line" | sed -E 's/.*uses:[[:space:]]*//; s/[[:space:]].*$//') - printf '%s\n' "$block" | grep -qF "metadatastician/standards@${ref##*@}" \ + printf '%s\n' "$block" | grep -F >/dev/null "metadatastician/standards@${ref##*@}" \ || fail "actions.lock workflows entry for $f does not record ${ref}" done </dev/null "commit: 'sha1-$PIN'" \ || fail "actions.lock dependency for standards@$PIN does not resolve to sha1-$PIN" # --- Lock transitive closure: every workflow ref and every transitive @@ -155,7 +155,7 @@ for f in governance.yml hypatia-scan.yml; do [ -n "$raw" ] || fail "$f does not pass hypatia-ref explicitly (scanner source would float)" value=$(printf '%s' "$raw" | sed -E 's/^[[:space:]]*hypatia-ref:[[:space:]]*//; s/[[:space:]]+#.*$//; s/[[:space:]]+$//') if [ "$value" != '${{ github.event.pull_request.base.sha || github.sha }}' ] \ - && ! printf '%s' "$value" | grep -qE '^[0-9a-f]{40}$'; then + && ! printf '%s' "$value" | grep -E >/dev/null '^[0-9a-f]{40}$'; then fail "$f hypatia-ref is neither the base/event SHA expression nor a 40-hex SHA: $value" fi done diff --git a/scripts/sweeps/estate-pin-integrity.sh b/scripts/sweeps/estate-pin-integrity.sh index e3a1b804..08d7c017 100755 --- a/scripts/sweeps/estate-pin-integrity.sh +++ b/scripts/sweeps/estate-pin-integrity.sh @@ -322,7 +322,7 @@ replace_site() { # # (`denylisted_pair`): `some/action@v4.38.1` is no finding. line_has_denied_token() { local tok t - if [ -n "$SHA_ALTERNATION" ] && printf '%s\n' "$1" | grep -qiE "(^|[^0-9A-Za-z])(${SHA_ALTERNATION})([^0-9A-Za-z]|\$)"; then + if [ -n "$SHA_ALTERNATION" ] && printf '%s\n' "$1" | grep -iE >/dev/null "(^|[^0-9A-Za-z])(${SHA_ALTERNATION})([^0-9A-Za-z]|\$)"; then return 0 fi [ ${#DENY_VERSIONS[@]} -gt 0 ] || return 1 @@ -399,7 +399,7 @@ relabel_comment() { local trimmed="${body#"$lead"}" # A bare claim (no `#`, no leading space) normalises to `# vX`, as relabel/2 does. [ "$body" = "$comment" ] && [ -z "$lead" ] && lead=" " - if printf '%s' "$trimmed" | grep -qE '^v?[0-9]+(\.[0-9]+)*([[:space:]]|$)'; then + if printf '%s' "$trimmed" | grep -E >/dev/null '^v?[0-9]+(\.[0-9]+)*([[:space:]]|$)'; then printf '%s%s' "#${lead}" \ "$(printf '%s' "$trimmed" | sed -E "0,/^v?[0-9]+(\.[0-9]+)*/s//v${version}/")" else diff --git a/scripts/sweeps/estate-pr-automerge.sh b/scripts/sweeps/estate-pr-automerge.sh index 078f64a2..d664bf22 100755 --- a/scripts/sweeps/estate-pr-automerge.sh +++ b/scripts/sweeps/estate-pr-automerge.sh @@ -296,7 +296,7 @@ decide_pr() { only_uses=1 while IFS= read -r cl; do [ -n "$cl" ] || continue - printf '%s' "$cl" | grep -qE '^[-+]\s*-?\s*uses:' || only_uses=0 + printf '%s' "$cl" | grep -E >/dev/null '^[-+]\s*-?\s*uses:' || only_uses=0 done <<<"$content_lines" [ "$only_uses" -eq 1 ] && [ -n "$content_lines" ] && pin_files=$((pin_files + 1)) [ "$only_uses" -eq 0 ] && nonpin_change=$((nonpin_change + 1)) @@ -426,7 +426,7 @@ decide_pr() { route="rhodibot"; pool="P2" elif [ "$pin_files" -eq 0 ] && [ "$lock_only" -eq 0 ] && jq -r '.filename' <<<"$patches" 2>/dev/null \ - | grep -qE '(^|/)(package\.json|Cargo\.toml|Project\.toml|mix\.exs|go\.mod|pyproject\.toml|composer\.json|Gemfile|requirements[^/]*\.txt)$'; then + | grep -E >/dev/null '(^|/)(package\.json|Cargo\.toml|Project\.toml|mix\.exs|go\.mod|pyproject\.toml|composer\.json|Gemfile|requirements[^/]*\.txt)$'; then change_class="bump"; disposition="flag"; safety="flag" blocked_by="dependency_manifest_not_lockfile" route="rhodibot"; pool="P2" diff --git a/scripts/sweeps/patch_phase2.sh b/scripts/sweeps/patch_phase2.sh index cd89fc3b..6aa5b908 100755 --- a/scripts/sweeps/patch_phase2.sh +++ b/scripts/sweeps/patch_phase2.sh @@ -40,7 +40,7 @@ END=$(awk -v s="$START" 'NR>s && /'"$NEXT_RE"'/ {print NR; exit}' "$WF" || true) [ -n "${END:-}" ] || END=$((TOTAL + 1)) # Idempotency: bail if the existing block already carries the fix. -if sed -n "${START},$((END - 1))p" "$WF" | grep -q '^ continue-on-error: true$'; then +if sed -n "${START},$((END - 1))p" "$WF" | grep >/dev/null '^ continue-on-error: true$'; then exit 2 fi diff --git a/test/ci-health-test.sh b/test/ci-health-test.sh index 4417dc34..368128fe 100755 --- a/test/ci-health-test.sh +++ b/test/ci-health-test.sh @@ -13,16 +13,16 @@ run_detect() { } lock_output=$(run_detect lockfile) -printf '%s\n' "$lock_output" | grep -q $'test-repo\tB-LOCKFILE\tHIGH\t' -printf '%s\n' "$lock_output" | grep -q '1 startup_failure conclusion(s) among the latest 3 run(s)' -if printf '%s\n' "$lock_output" | grep -q 'populate allow-list'; then +printf '%s\n' "$lock_output" | grep >/dev/null $'test-repo\tB-LOCKFILE\tHIGH\t' +printf '%s\n' "$lock_output" | grep >/dev/null '1 startup_failure conclusion(s) among the latest 3 run(s)' +if printf '%s\n' "$lock_output" | grep >/dev/null 'populate allow-list'; then echo "FAIL: unclassified startup failure prescribed an allow-list mutation" >&2 exit 1 fi saturated_output=$(run_detect saturated) -printf '%s\n' "$saturated_output" | grep -q '30 startup_failure conclusion(s) among the latest 30 run(s); page saturated at 30, older runs not measured' -if printf '%s\n' "$saturated_output" | grep -q $'\tB-LOCKFILE\t'; then +printf '%s\n' "$saturated_output" | grep >/dev/null '30 startup_failure conclusion(s) among the latest 30 run(s); page saturated at 30, older runs not measured' +if printf '%s\n' "$saturated_output" | grep >/dev/null $'\tB-LOCKFILE\t'; then echo "FAIL: saturated scenario was incorrectly classified as a lockfile failure" >&2 exit 1 fi @@ -67,20 +67,20 @@ for fixture in "$ROOT"/test/fixtures/ci-health/allowlist/*.json; do if [[ "$added" -eq 0 ]]; then [[ ! -s "$STATE_DIR/puts.log" ]] || { echo "FAIL [$name]: wrote although nothing was missing" >&2; exit 1; } - printf '%s\n' "$out" | grep -q '^SKIP ' || { echo "FAIL [$name]: expected SKIP, got: $out" >&2; exit 1; } + printf '%s\n' "$out" | grep >/dev/null '^SKIP ' || { echo "FAIL [$name]: expected SKIP, got: $out" >&2; exit 1; } else - printf '%s\n' "$out" | grep -q "^FIXED .* added $added, kept all $(jq '.expect.kept' "$fixture")" || + printf '%s\n' "$out" | grep >/dev/null "^FIXED .* added $added, kept all $(jq '.expect.kept' "$fixture")" || { echo "FAIL [$name]: unexpected report: $out" >&2; exit 1; } fi # Idempotent: a second run finds nothing missing and writes nothing. : > "$STATE_DIR/puts.log" - printf '%s\n' "$(remediate)" | grep -q '^SKIP ' || { echo "FAIL [$name]: second run was not a no-op" >&2; exit 1; } + printf '%s\n' "$(remediate)" | grep >/dev/null '^SKIP ' || { echo "FAIL [$name]: second run was not a no-op" >&2; exit 1; } [[ ! -s "$STATE_DIR/puts.log" ]] || { echo "FAIL [$name]: second run wrote" >&2; exit 1; } # Converges with detect.sh: B-ALLOWLIST no longer fires. if PATH="$MOCK_BIN:$PATH" OWNER=test-owner MOCK_STATE="$STATE_DIR/state.json" MOCK_SCENARIO=converge \ - "$ROOT/scripts/ci-health/detect.sh" test-repo 2>/dev/null | grep -q $'\tB-ALLOWLIST\t'; then + "$ROOT/scripts/ci-health/detect.sh" test-repo 2>/dev/null | grep >/dev/null $'\tB-ALLOWLIST\t'; then echo "FAIL [$name]: detect.sh still reports B-ALLOWLIST after remediation" >&2; exit 1 fi @@ -88,7 +88,7 @@ for fixture in "$ROOT"/test/fixtures/ci-health/allowlist/*.json; do want="-- corpus: $name added=$added kept=$(jq '.expect.kept' "$fixture") result=$(jq '.expect.result_size' "$fixture")" grep -qxF -- "$want" "$model" || { echo "FAIL [$name]: model has no line '$want'" >&2; exit 1; } eqn="= ($added, $(jq '.expect.kept' "$fixture"), $(jq '.expect.result_size' "$fixture"))" - grep -A1 -xF -- "$want" "$model" | tail -n1 | grep -qF -- "$eqn" || + grep -A1 -xF -- "$want" "$model" | tail -n1 | grep -F >/dev/null -- "$eqn" || { echo "FAIL [$name]: the equation under '$want' does not state $eqn" >&2; exit 1; } done [[ "$(grep -c '^-- corpus: ' "$model")" -eq "$(find "$ROOT/test/fixtures/ci-health/allowlist" -name '*.json' | wc -l)" ]] || diff --git a/test/fixtures/ci-health/mock-gh b/test/fixtures/ci-health/mock-gh index 6c47e4ab..d6d6322f 100755 --- a/test/fixtures/ci-health/mock-gh +++ b/test/fixtures/ci-health/mock-gh @@ -16,7 +16,7 @@ for i in "${!args[@]}"; do [ "${args[$i]}" = "--jq" ] && jq_filter="${args[$((i + 1))]}" done -if printf ' %s ' "$*" | grep -q ' -X PUT '; then +if printf ' %s ' "$*" | grep >/dev/null ' -X PUT '; then body=$(cat) if [ -n "${MOCK_STATE:-}" ] && [ "${MOCK_SCENARIO:-}" != "remediate-mismatch" ]; then printf '%s\n' "$body" > "$MOCK_STATE" diff --git a/test/standards-integration-test.sh b/test/standards-integration-test.sh index 4d914573..171b8874 100755 --- a/test/standards-integration-test.sh +++ b/test/standards-integration-test.sh @@ -17,7 +17,7 @@ expect_outcome() { # name expected_exit expected_outcome root extra... local name="$1" want_rc="$2" want_outcome="$3"; shift 3 local out rc=0 out=$("$CHECK" "$@" 2>&1) || rc=$? - if [ "$rc" != "$want_rc" ] || ! printf '%s' "$out" | grep -q "outcome=$want_outcome"; then + if [ "$rc" != "$want_rc" ] || ! printf '%s' "$out" | grep >/dev/null "outcome=$want_outcome"; then echo "FAIL($name): want rc=$want_rc outcome=$want_outcome, got rc=$rc: $out" >&2 exit 1 fi @@ -91,11 +91,11 @@ for t in bash grep sed awk sort head env; do done rc=0 out=$(PATH="$TMP/nogh" "$TMP/nogh/bash" "$CHECK" --root "$ROOT" --verify-remote 2>&1) || rc=$? -if [ "$rc" != "3" ] || ! printf '%s' "$out" | grep -q 'outcome=unavailable'; then +if [ "$rc" != "3" ] || ! printf '%s' "$out" | grep >/dev/null 'outcome=unavailable'; then echo "FAIL(unavailable-tool): want rc=3 outcome=unavailable, got rc=$rc: $out" >&2 exit 1 fi -if printf '%s' "$out" | grep -q 'outcome=pass'; then +if printf '%s' "$out" | grep >/dev/null 'outcome=pass'; then echo "FAIL(unavailable-tool): unavailable was reported as pass" >&2 exit 1 fi @@ -109,7 +109,7 @@ for t in bash grep sed awk head env; do done rc=0 out=$(PATH="$TMP/nosort" "$TMP/nosort/bash" "$CHECK" --root "$ROOT" 2>&1) || rc=$? -if [ "$rc" != "3" ] || ! printf '%s' "$out" | grep -q 'outcome=unavailable'; then +if [ "$rc" != "3" ] || ! printf '%s' "$out" | grep >/dev/null 'outcome=unavailable'; then echo "FAIL(unavailable-sort): want rc=3 outcome=unavailable, got rc=$rc: $out" >&2 exit 1 fi diff --git a/tests/e2e.sh b/tests/e2e.sh index 6c4d59f9..a07ecc6c 100644 --- a/tests/e2e.sh +++ b/tests/e2e.sh @@ -233,7 +233,7 @@ fi # Check all documented rules have corresponding modules for rule in root_hygiene honest_completion workflow_audit cicd_rules code_safety; do - if find "$PROJECT_DIR/lib" -name "*${rule}*" -o -name "*$(echo $rule | tr '_' '-')*" 2>/dev/null | grep -q .; then + if find "$PROJECT_DIR/lib" -name "*${rule}*" -o -name "*$(echo $rule | tr '_' '-')*" 2>/dev/null | grep >/dev/null .; then pass "Rule module: $rule" else skip_test "Rule module: $rule" "file not found" @@ -250,7 +250,7 @@ bold "Section 4: Fixture scan detection" # The fixture has deliberately insecure patterns — scanner should find them if $HAS_RUST && [ -f "$HYPER_BIN" ]; then FINDINGS=$("$HYPER_BIN" scan "$FIXTURE_DIR" 2>/dev/null || true) - if echo "$FINDINGS" | grep -qi "unpinned\|injection\|insecure\|http\|eval\|finding\|weak\|issue" 2>/dev/null; then + if echo "$FINDINGS" | grep -i >/dev/null "unpinned\|injection\|insecure\|http\|eval\|finding\|weak\|issue" 2>/dev/null; then pass "Scanner detects issues in fixture repo" else skip_test "Fixture detection" "scanner may not produce text findings" diff --git a/tests/rules-praxis-test.sh b/tests/rules-praxis-test.sh index ffac5cc5..c1bb6dbf 100755 --- a/tests/rules-praxis-test.sh +++ b/tests/rules-praxis-test.sh @@ -86,18 +86,18 @@ expect_broken "an un-retired tombstone is refused" "$work/tomb.deed" "tombstone # provenance table is not a severity table, `severity: :advisory` is advisory, # and a rule that fires at several severities keeps the highest. row() { awk -v t="\"$1\"" '$1 == ":id" { on = ($2 == t) } on { print }' "$deed"; } # one rule-def -if row hypatia/branch_protection/BP001 | grep -q ':severity high' && row hypatia/supply_chain/SC001 | grep -q ':status active'; then ok "opt-in scanner rules are registered"; else bad "BP001/SC001 missing or wrong"; fi -if row hypatia/git_state/GS007 | grep -q ':class advisory'; then ok "an advisory-severity rule is class advisory"; else bad "GS007 is not class advisory"; fi -if row hypatia/code_scanning_alerts/CSA006 | grep -q ':severity medium'; then ok "a multi-severity rule keeps its highest severity"; else bad "CSA006 is not medium"; fi +if row hypatia/branch_protection/BP001 | grep >/dev/null ':severity high' && row hypatia/supply_chain/SC001 | grep >/dev/null ':status active'; then ok "opt-in scanner rules are registered"; else bad "BP001/SC001 missing or wrong"; fi +if row hypatia/git_state/GS007 | grep >/dev/null ':class advisory'; then ok "an advisory-severity rule is class advisory"; else bad "GS007 is not class advisory"; fi +if row hypatia/code_scanning_alerts/CSA006 | grep >/dev/null ':severity medium'; then ok "a multi-severity rule keeps its highest severity"; else bad "CSA006 is not medium"; fi -if row hypatia/pin_integrity/PI005 | grep -q ':class advisory' && row hypatia/disambiguation_rules/DR001 | grep -q ':class advisory'; then ok "rules the code calls advisory in words are class advisory"; else bad "PI005 or DR001 is not class advisory"; fi +if row hypatia/pin_integrity/PI005 | grep >/dev/null ':class advisory' && row hypatia/disambiguation_rules/DR001 | grep >/dev/null ':class advisory'; then ok "rules the code calls advisory in words are class advisory"; else bad "PI005 or DR001 is not class advisory"; fi -if row hypatia/workflow_audit/duplicate_workflow | grep -q ':severity minor' && row hypatia/cicd_rules/waste_duplicate_workflow | grep -q ':severity medium'; then ok "named atom rules carry the severity the code states"; else bad "duplicate_workflow rows lack their severity"; fi +if row hypatia/workflow_audit/duplicate_workflow | grep >/dev/null ':severity minor' && row hypatia/cicd_rules/waste_duplicate_workflow | grep >/dev/null ':severity medium'; then ok "named atom rules carry the severity the code states"; else bad "duplicate_workflow rows lack their severity"; fi # Severities the code computes (an if expression, a helper): the highest branch. # A severity read from run-time alert data (CSA001) has none to record. -if row hypatia/supply_chain/SC012 | grep -q ':severity high' && row hypatia/structural_drift/SD008 | grep -q ':severity critical' \ - && row hypatia/research_extensions/RE001 | grep -q ':severity high' && ! row hypatia/code_scanning_alerts/CSA001 | grep -q ':severity'; then +if row hypatia/supply_chain/SC012 | grep >/dev/null ':severity high' && row hypatia/structural_drift/SD008 | grep >/dev/null ':severity critical' \ + && row hypatia/research_extensions/RE001 | grep >/dev/null ':severity high' && ! row hypatia/code_scanning_alerts/CSA001 | grep >/dev/null ':severity'; then ok "computed severities are read; run-time severities stay empty"; else bad "computed severity rows are wrong"; fi # The names findings actually carry resolve (A5 R09). Ground truth: every @@ -140,7 +140,7 @@ done < <(awk ' ' lib/hypatia/cli.ex lib/rules/*.ex | sort -u) if [ -z "$missing_pairs" ]; then ok "every literal rule_module/type pair in the code is in the registry"; else bad "pairs in the code but not the registry:$missing_pairs"; fi if ! grep -qF '"hypatia/workflow_audit/always_after_cancel"' "$deed" && grep -qF '"hypatia/ci_failure_triage/always_after_cancel"' "$deed"; then ok "a name moved by a rule_id override is registered under its real module only"; else bad "always_after_cancel is registered under workflow_audit, or missing under ci_failure_triage"; fi -if row hypatia/structured_format/SF001 | grep -q ':class advisory' && row hypatia/cicd_rules/banned_language_file | grep -q ':severity critical' && row hypatia/implementation_inside_canon/HYP-S009 | grep -q ':status active' && row hypatia/workflow_hardening/invalid_actions_lock | grep -q ':status active'; then ok "fixed names carry the code's severity and advisory flag; HYP-S009 and invalid_actions_lock are registered"; else bad "a fixed-name row is wrong"; fi +if row hypatia/structured_format/SF001 | grep >/dev/null ':class advisory' && row hypatia/cicd_rules/banned_language_file | grep >/dev/null ':severity critical' && row hypatia/implementation_inside_canon/HYP-S009 | grep >/dev/null ':status active' && row hypatia/workflow_hardening/invalid_actions_lock | grep >/dev/null ':status active'; then ok "fixed names carry the code's severity and advisory flag; HYP-S009 and invalid_actions_lock are registered"; else bad "a fixed-name row is wrong"; fi # Portability: gawk and mawk differ (a `\[` in a -v value is valid in one and a # fatal regex error in the other, and the awk then dies without a word inside a