diff --git a/lib/hypatia/cli.ex b/lib/hypatia/cli.ex index 6f25224c..40c09275 100644 --- a/lib/hypatia/cli.ex +++ b/lib/hypatia/cli.ex @@ -1018,17 +1018,7 @@ defmodule Hypatia.CLI do if :secret_scanning_alerts in rules do case Hypatia.Rules.SecretScanningAlerts.scan_from_path(repo_path) do {:ok, %{findings: findings}} -> - normalized = - Enum.map(findings, fn f -> - %{ - rule_module: "secret_scanning_alerts", - severity: to_string(f.severity), - type: f.rule, - file: Map.get(f, :file, ""), - reason: f.reason, - action: to_string(f.action) - } - end) + normalized = Enum.map(findings, &normalize_secret_alert/1) results ++ normalized @@ -1178,6 +1168,23 @@ defmodule Hypatia.CLI do |> Enum.map(&Hypatia.Verdict.annotate/1) end + @doc false + # A secret-scanning finding in the scan's shared shape. The located line is + # kept when there is one, so the verdict span and SARIF point at the secret, + # and so is `detail`: the alert number, its URL and, when the finding is + # repository-level, why no file location was available. + def normalize_secret_alert(f) do + %{ + rule_module: "secret_scanning_alerts", + severity: to_string(f.severity), + type: f.rule, + file: Map.get(f, :file, ""), + reason: f.reason, + action: to_string(f.action) + } + |> Map.merge(Map.take(f, [:line, :detail])) + end + # ─── Opt-in rule modules ───────────────────────────────────────────── # # These 4 families had detectors and tests but no scan caller. They run diff --git a/lib/hypatia/scanner_suppression.ex b/lib/hypatia/scanner_suppression.ex index 95e4c29d..8d104ffe 100644 --- a/lib/hypatia/scanner_suppression.ex +++ b/lib/hypatia/scanner_suppression.ex @@ -201,6 +201,21 @@ defmodule Hypatia.ScannerSuppression do end end + # Secret-scanning alerts come from GitHub, not from this scan's content + # patterns: each is a credential GitHub found committed to the repository. + # Their `file` is the secret's real location, which may be vendored or + # generated (`node_modules/`, `dist/`, `test/`). The universal excludes and + # default exemptions exist to quiet local pattern noise in such paths, and + # must not hide a leaked credential. Only a `.hypatia-ignore` entry that + # names this module explicitly suppresses one; GitHub's own dismissal + # vocabulary (`revoked`, `used_in_tests`, ...) is the usual way to accept + # an alert. + def suppressed?(file, "secret_scanning_alerts" = rule_module, rule_type, opts) do + repo_path = Keyword.get(opts, :repo_path, nil) + explicit = Map.take(user_exemptions(repo_path), [rule_module]) + match_exemptions?(relative(file, repo_path), explicit, rule_module, rule_type) + end + def suppressed?(file, rule_module, rule_type, opts) do repo_path = Keyword.get(opts, :repo_path, nil) rel = relative(file, repo_path) diff --git a/lib/rules/secret_scanning_alerts.ex b/lib/rules/secret_scanning_alerts.ex index 91aec1e5..248b849d 100644 --- a/lib/rules/secret_scanning_alerts.ex +++ b/lib/rules/secret_scanning_alerts.ex @@ -22,6 +22,15 @@ defmodule Hypatia.Rules.SecretScanningAlerts do `HYPATIA_SECRET_ALERTS_TOKEN` first, then `GITHUB_TOKEN`, so a dedicated read-only token never has to be exported under the general name. + ## Locations + + The alert list carries only a `locations_url`. For every alert that becomes a + file-scoped finding (SSA001, SSA003, SSA004) that URL is read once, and the + finding's `file` and `line` come from the first location with a path. A + secret found outside the repository's files (an issue, a pull-request + comment) or a failed lookup gives the repository-level file `"."`, and + `detail.location` records why. The repo summary (SSA002) is always `"."`. + ## Errors The per-rule functions (`ssa001_open_alerts/2` and so on) log a fetch @@ -48,6 +57,13 @@ defmodule Hypatia.Rules.SecretScanningAlerts do # Dismissal reasons that are accepted by policy without further review. @accepted_resolutions ~w(revoked used_in_tests pattern_deleted pattern_edited) + # `file` of a finding that names no file: the repository as a whole. + @repo_level_file "." + # Locations read per alert (one page, no pagination: the first file wins). + @max_locations_per_alert 100 + # Pages of locations read before the alert is reported at repository level. + @max_location_pages 10 + # ─── SSA001: Open secret-scanning alerts ─────────────────────────────── @doc """ @@ -58,7 +74,7 @@ defmodule Hypatia.Rules.SecretScanningAlerts do fixable (rotate + remove) vs. a documented test fixture. """ def ssa001_open_alerts(owner, repo) do - case fetch_alerts(owner, repo) do + case fetch_alerts(owner, repo, &open?/1) do {:ok, alerts} -> open_alert_findings(alerts) @@ -80,7 +96,6 @@ defmodule Hypatia.Rules.SecretScanningAlerts do %{ rule: "SSA001", - file: secret_type, severity: :critical, reason: build_alert_reason(secret_type, age_days, is_stale), action: :escalate, @@ -95,6 +110,7 @@ defmodule Hypatia.Rules.SecretScanningAlerts do locations_url: alert["locations_url"] } } + |> locate(alert) end) end @@ -106,7 +122,7 @@ defmodule Hypatia.Rules.SecretScanningAlerts do so the dashboard can highlight the repo, not just the individual alert. """ def ssa002_severity_summary(owner, repo) do - case fetch_alerts(owner, repo) do + case fetch_alerts(owner, repo, fn _ -> false end) do {:ok, alerts} -> summary_findings(owner, repo, alerts) {:error, _} -> [] end @@ -121,11 +137,12 @@ defmodule Hypatia.Rules.SecretScanningAlerts do [ %{ rule: "SSA002", - file: "#{owner}/#{repo}", + file: @repo_level_file, severity: :critical, reason: "#{count} open secret-scanning alert(s) -- rotate and purge from history", action: :escalate, detail: %{ + repo: "#{owner}/#{repo}", total: count, by_type: open @@ -148,7 +165,7 @@ defmodule Hypatia.Rules.SecretScanningAlerts do but staleness is surfaced in the reason for triage prioritisation. """ def ssa003_stale_alerts(owner, repo) do - case fetch_alerts(owner, repo) do + case fetch_alerts(owner, repo, &stale_open?/1) do {:ok, alerts} -> stale_findings(alerts) {:error, _} -> [] end @@ -167,7 +184,6 @@ defmodule Hypatia.Rules.SecretScanningAlerts do %{ rule: "SSA003", - file: secret_type, severity: :critical, reason: "Secret-scanning alert for #{secret_type} is #{age} days old " <> @@ -180,6 +196,7 @@ defmodule Hypatia.Rules.SecretScanningAlerts do threshold_days: @stale_threshold_days } } + |> locate(alert) end) end @@ -192,7 +209,7 @@ defmodule Hypatia.Rules.SecretScanningAlerts do vocabulary; anything else (including nil) is policy-suspicious. """ def ssa004_dismissed_without_fix(owner, repo) do - case fetch_alerts(owner, repo) do + case fetch_alerts(owner, repo, &suspicious_resolved?/1) do {:ok, alerts} -> dismissed_findings(alerts) {:error, _} -> [] end @@ -211,7 +228,6 @@ defmodule Hypatia.Rules.SecretScanningAlerts do %{ rule: "SSA004", - file: secret_type, severity: :high, reason: "Secret-scanning alert for #{secret_type} resolved as '#{resolution}' " <> @@ -225,6 +241,7 @@ defmodule Hypatia.Rules.SecretScanningAlerts do resolution_comment: alert["resolution_comment"] } } + |> locate(alert) end) end @@ -238,11 +255,16 @@ defmodule Hypatia.Rules.SecretScanningAlerts do if resolve_token() == nil do {:error, missing_token_message()} else + # One fetch (and one location lookup per alert) for all four rules. findings = - ssa001_open_alerts(owner, repo) ++ - ssa002_severity_summary(owner, repo) ++ - ssa003_stale_alerts(owner, repo) ++ - ssa004_dismissed_without_fix(owner, repo) + case fetch_alerts(owner, repo) do + {:ok, alerts} -> + findings_from_alerts(owner, repo, alerts) + + {:error, reason} -> + Logger.warning("SSA001-SSA004: Failed to fetch secret-scanning alerts: #{reason}") + [] + end {:ok, result(findings)} end @@ -313,7 +335,9 @@ defmodule Hypatia.Rules.SecretScanningAlerts do "cannot query secret-scanning alerts" end - defp fetch_alerts(owner, repo) do + # `locate?` picks the alerts whose file location is read, one request each: + # only those a rule reports at a file. The repo summary (SSA002) reads none. + defp fetch_alerts(owner, repo, locate? \\ &needs_location?/1) do case resolve_token() do nil -> {:error, "GITHUB_TOKEN not set"} @@ -323,7 +347,11 @@ defmodule Hypatia.Rules.SecretScanningAlerts do "#{@github_api_base}/repos/#{owner}/#{repo}/secret-scanning/alerts" <> "?per_page=#{@max_alerts_per_repo}" - collect_pages(url, &request(&1, token)) + request = &request(&1, token) + + with {:ok, alerts} <- collect_pages(url, request) do + {:ok, attach_locations(alerts, request, locate?)} + end end end @@ -356,6 +384,124 @@ defmodule Hypatia.Rules.SecretScanningAlerts do end end + @doc false + # Read the location of every alert that becomes a file-scoped finding + # (SSA001/SSA003 open, SSA004 resolved without an accepted reason) and + # record it on the alert under "hypatia_location". A failed lookup is + # recorded, not fatal: the finding still reports the alert, at repository + # level, with the reason. + def attach_locations(alerts, request, locate? \\ &needs_location?/1) do + Enum.map(alerts, fn alert -> + if locate?.(alert) do + Map.put(alert, "hypatia_location", fetch_location(alert["locations_url"], request)) + else + alert + end + end) + end + + defp needs_location?(alert), do: open?(alert) or suspicious_resolved?(alert) + + defp open?(alert), do: alert["state"] == "open" + + defp stale_open?(alert), + do: open?(alert) and age_in_days(alert["created_at"]) > @stale_threshold_days + + defp suspicious_resolved?(alert), + do: alert["state"] == "resolved" and alert["resolution"] not in @accepted_resolutions + + @doc false + # Reads pages of locations until one names a file, following the `next` + # link (the first commit location can sit behind many wiki or comment + # locations), up to @max_location_pages pages. The token goes with every + # request, so every URL must be on the API host. + def fetch_location(url, request) when is_binary(url) do + separator = if String.contains?(url, "?"), do: "&", else: "?" + find_location(url <> separator <> "per_page=#{@max_locations_per_alert}", request, 1) + end + + def fetch_location(_url, _request), do: %{"error" => "the alert has no locations_url"} + + defp find_location(_url, _request, page) when page > @max_location_pages do + %{ + "error" => + "no location names a file in the repository in the first " <> + "#{@max_location_pages * @max_locations_per_alert} locations" + } + end + + defp find_location(url, request, page) do + if String.starts_with?(url, @github_api_base <> "/") do + with {:ok, output} <- request.(url), + {:ok, locations, next} <- parse_page(output) do + case location_from(locations) do + %{"path" => _} = found -> found + not_found when next == nil -> not_found + _ -> find_location(next, request, page + 1) + end + else + {:error, reason} -> %{"error" => reason} + end + else + %{"error" => "refusing to follow a locations URL off #{@github_api_base}"} + end + end + + @doc false + # The first `commit` location with a file path: only a commit location + # names a file of the scanned repository. A `wiki_commit` location also + # has a path, but in the separate `.wiki.git`; issue bodies, + # pull-request comments and discussions name no file. GitHub writes the + # path from the repository root with a leading slash (`/config/app.env`); + # the finding's `file` is repository-relative, like every other rule's. + def location_from(locations) do + Enum.find_value( + locations, + %{"error" => "no location names a file in the repository"}, + fn + %{"type" => "commit", "details" => %{"path" => path} = details} + when is_binary(path) -> + case String.trim_leading(path, "/") do + "" -> nil + rel -> %{"path" => rel, "start_line" => details["start_line"]} + end + + _ -> + nil + end + ) + end + + # Set a finding's `file` and `line` from its alert's location, falling + # back to the repository level with the reason in `detail.location`. + defp locate(finding, alert) do + case alert["hypatia_location"] do + %{"path" => path, "start_line" => line} -> + finding + |> Map.put(:file, path) + |> put_line(line) + |> put_location(%{status: "found", path: path, start_line: line}) + + %{"error" => reason} -> + finding + |> Map.put(:file, @repo_level_file) + |> put_location(%{status: "unavailable", reason: reason}) + + _ -> + finding + |> Map.put(:file, @repo_level_file) + |> put_location(%{status: "not_fetched"}) + end + end + + defp put_line(finding, line) when is_integer(line) and line >= 1, + do: Map.put(finding, :line, line) + + defp put_line(finding, _line), do: finding + + defp put_location(finding, location), + do: Map.update!(finding, :detail, &Map.put(&1, :location, location)) + # No `-f`: on an HTTP error GitHub's JSON body carries the reason (for # example "Secret scanning is disabled on this repository"), and `-f` would # discard it. `-i` keeps the headers for the `Link` pagination header, and diff --git a/test/secret_scanning_alerts_test.exs b/test/secret_scanning_alerts_test.exs index f3b2d5cb..c4c72b2d 100644 --- a/test/secret_scanning_alerts_test.exs +++ b/test/secret_scanning_alerts_test.exs @@ -241,7 +241,7 @@ defmodule Hypatia.Rules.SecretScanningAlertsTest do by_rule = Enum.group_by(findings, & &1.rule) assert length(by_rule["SSA001"]) == 2 - assert [%{file: "acme/demo", detail: %{total: 2}}] = by_rule["SSA002"] + assert [%{file: ".", detail: %{total: 2, repo: "acme/demo"}}] = by_rule["SSA002"] # Only the 30-day-old alert is past the 7-day threshold. assert [%{detail: %{alert_number: 1}}] = by_rule["SSA003"] # `wont_fix` is not an accepted resolution; `revoked` is. @@ -253,6 +253,313 @@ defmodule Hypatia.Rules.SecretScanningAlertsTest do end end + describe "alert locations" do + @locations "https://api.github.com/repos/acme/demo/secret-scanning/alerts/1/locations" + + defp body(json), do: "HTTP/2 200\r\n\r\n" <> json <> "\n200" + + test "the first location with a path gives the file and line" do + locations = [ + %{"type" => "issue_body", "details" => %{"issue_body_url" => "https://x"}}, + %{"type" => "commit", "details" => %{"path" => "config/prod.env", "start_line" => 12}}, + %{"type" => "commit", "details" => %{"path" => "later.env", "start_line" => 3}} + ] + + assert SecretScanningAlerts.location_from(locations) == + %{"path" => "config/prod.env", "start_line" => 12} + end + + test "no location with a path is an error, not a guessed file" do + assert %{"error" => "no location names a file" <> _} = + SecretScanningAlerts.location_from([ + %{"type" => "pull_request_comment", "details" => %{}} + ]) + + assert %{"error" => _} = SecretScanningAlerts.location_from([]) + end + + test "reads one page of locations from the API host" do + request = fn url -> + assert url == @locations <> "?per_page=100" + + {:ok, body(~s([{"type":"commit","details":{"path":"a/b.ex","start_line":4}}]))} + end + + assert SecretScanningAlerts.fetch_location(@locations, request) == + %{"path" => "a/b.ex", "start_line" => 4} + end + + test "never sends the token to a locations URL off the API host" do + request = fn _ -> flunk("must not request off the API host") end + + assert %{"error" => "refusing to follow a locations URL off https://api.github.com"} = + SecretScanningAlerts.fetch_location("https://evil.example.com/x", request) + end + + test "a failed lookup and a missing URL are recorded as errors" do + request = fn _ -> {:error, "GitHub API request timed out after 30s"} end + + assert SecretScanningAlerts.fetch_location(@locations, request) == + %{"error" => "GitHub API request timed out after 30s"} + + assert %{"error" => "the alert has no locations_url"} = + SecretScanningAlerts.fetch_location(nil, request) + end + + test "only alerts that become file-scoped findings are looked up" do + {:ok, asked} = Agent.start_link(fn -> [] end) + + request = fn url -> + Agent.update(asked, &[url | &1]) + {:ok, body(~s([{"type":"commit","details":{"path":"f.txt","start_line":1}}]))} + end + + alerts = + for {n, state, resolution} <- [ + {1, "open", nil}, + {2, "resolved", "wont_fix"}, + {3, "resolved", "revoked"} + ] do + %{ + "number" => n, + "state" => state, + "resolution" => resolution, + "locations_url" => "https://api.github.com/repos/acme/demo/l/#{n}" + } + end + + located = SecretScanningAlerts.attach_locations(alerts, request) + + assert Agent.get(asked, &Enum.sort/1) == [ + "https://api.github.com/repos/acme/demo/l/1?per_page=100", + "https://api.github.com/repos/acme/demo/l/2?per_page=100" + ] + + assert [%{"hypatia_location" => _}, %{"hypatia_location" => _}, third] = located + refute Map.has_key?(third, "hypatia_location") + end + + test "findings carry the location, or the repository level with the reason" do + created = DateTime.utc_now() |> DateTime.add(-30 * 86_400) |> DateTime.to_iso8601() + + alerts = [ + %{ + "number" => 1, + "state" => "open", + "secret_type" => "github_personal_access_token", + "created_at" => created, + "hypatia_location" => %{"path" => "config/prod.env", "start_line" => 12} + }, + %{ + "number" => 2, + "state" => "open", + "secret_type" => "aws_access_key_id", + "created_at" => created, + "hypatia_location" => %{"error" => "no location names a file in the repository"} + }, + %{ + "number" => 3, + "state" => "resolved", + "resolution" => "wont_fix", + "secret_type" => "slack_api_token" + } + ] + + by_key = + "acme" + |> SecretScanningAlerts.findings_from_alerts("demo", alerts) + |> Map.new(fn f -> {{f.rule, f.detail[:alert_number]}, f} end) + + for rule <- ["SSA001", "SSA003"] do + located = by_key[{rule, 1}] + assert located.file == "config/prod.env" + assert located.line == 12 + assert located.detail.location.status == "found" + assert located.detail.secret_type == "github_personal_access_token" + end + + unavailable = by_key[{"SSA001", 2}] + assert unavailable.file == "." + refute Map.has_key?(unavailable, :line) + + assert unavailable.detail.location == + %{status: "unavailable", reason: "no location names a file in the repository"} + + not_fetched = by_key[{"SSA004", 3}] + assert not_fetched.file == "." + assert not_fetched.detail.location == %{status: "not_fetched"} + end + + test "the verdict span points at the located file and line" do + [finding] = + SecretScanningAlerts.open_alert_findings([ + %{ + "number" => 1, + "state" => "open", + "secret_type" => "github_personal_access_token", + "hypatia_location" => %{"path" => "config/prod.env", "start_line" => 12} + } + ]) + + token = + finding + |> Map.put(:rule_module, "secret_scanning_alerts") + |> Map.put(:type, finding.rule) + |> Hypatia.Verdict.token() + + assert token.span == %{file: "config/prod.env", start_line: 12, end_line: 12} + end + end + + describe "review follow-ups (#1184)" do + test "a wiki_commit location is not a file of the scanned repository" do + locations = [ + %{"type" => "wiki_commit", "details" => %{"path" => "Home.md", "start_line" => 3}}, + %{"type" => "commit", "details" => %{"path" => "src/app.env", "start_line" => 9}} + ] + + assert SecretScanningAlerts.location_from(locations) == + %{"path" => "src/app.env", "start_line" => 9} + + assert %{"error" => _} = + SecretScanningAlerts.location_from([ + %{"type" => "wiki_commit", "details" => %{"path" => "Home.md"}} + ]) + end + + test "a rule that reports no file looks up no locations" do + request = fn _ -> flunk("no location may be requested") end + + alerts = [ + %{"number" => 1, "state" => "open", "locations_url" => "https://api.github.com/x/1"} + ] + + assert [alert] = SecretScanningAlerts.attach_locations(alerts, request, fn _ -> false end) + refute Map.has_key?(alert, "hypatia_location") + end + + test "path exclusions do not hide a leaked credential" do + repo = Path.join(System.tmp_dir!(), "ssa-suppress-#{System.unique_integer([:positive])}") + File.mkdir_p!(repo) + # A generic (any-rule) exclusion of node_modules must not reach secret alerts. + File.write!(Path.join(repo, ".hypatia-ignore"), "node_modules/\n") + on_exit(fn -> File.rm_rf!(repo) end) + + for file <- ["node_modules/pkg/.env", "dist/app.js", "test/fixtures/key.pem", "deps/x/c.ex"] do + refute Hypatia.ScannerSuppression.suppressed?( + file, + "secret_scanning_alerts", + "SSA001", + repo_path: repo + ), + "#{file} must not be suppressed" + end + + # An entry that names the module explicitly still applies. + File.write!(Path.join(repo, ".hypatia-ignore"), "secret_scanning_alerts/SSA001:vendor/\n") + + assert Hypatia.ScannerSuppression.suppressed?( + "vendor/lib/key.pem", + "secret_scanning_alerts", + "SSA001", + repo_path: repo + ) + end + + test "the main scan path keeps the located line" do + [finding] = + SecretScanningAlerts.open_alert_findings([ + %{ + "number" => 1, + "state" => "open", + "secret_type" => "github_personal_access_token", + "hypatia_location" => %{"path" => "config/prod.env", "start_line" => 12} + } + ]) + + normalized = Hypatia.CLI.normalize_secret_alert(finding) + assert normalized.file == "config/prod.env" + assert normalized.line == 12 + + assert Hypatia.Verdict.token(normalized).span == + %{file: "config/prod.env", start_line: 12, end_line: 12} + end + + test "GitHub's leading-slash path becomes repository-relative" do + assert SecretScanningAlerts.location_from([ + %{"type" => "commit", "details" => %{"path" => "/example/secrets.txt"}} + ]) == %{"path" => "example/secrets.txt", "start_line" => nil} + + assert %{"error" => _} = + SecretScanningAlerts.location_from([ + %{"type" => "commit", "details" => %{"path" => "/"}} + ]) + end + + test "a commit location on a later page is found" do + first = "https://api.github.com/repos/acme/demo/l/1?per_page=100" + second = "https://api.github.com/repositories/1/l/1?page=2" + + request = fn + ^first -> + {:ok, + "HTTP/2 200\r\nlink: <#{second}>; rel=\"next\"\r\n\r\n" <> + ~s([{"type":"wiki_commit","details":{"path":"Home.md"}}]) <> "\n200"} + + ^second -> + {:ok, + "HTTP/2 200\r\n\r\n" <> + ~s([{"type":"commit","details":{"path":"/src/a.env","start_line":2}}]) <> "\n200"} + end + + assert SecretScanningAlerts.fetch_location( + "https://api.github.com/repos/acme/demo/l/1", + request + ) == %{"path" => "src/a.env", "start_line" => 2} + end + + test "a later page off the API host is refused" do + first = "https://api.github.com/repos/acme/demo/l/1?per_page=100" + + request = fn + ^first -> + {:ok, + "HTTP/2 200\r\nlink: ; rel=\"next\"\r\n\r\n" <> + ~s([{"type":"issue_body","details":{}}]) <> "\n200"} + + _ -> + flunk("must not request off the API host") + end + + assert %{"error" => "refusing to follow a locations URL off " <> _} = + SecretScanningAlerts.fetch_location( + "https://api.github.com/repos/acme/demo/l/1", + request + ) + end + + test "the main scan path keeps why a finding has no file" do + [finding] = + SecretScanningAlerts.open_alert_findings([ + %{ + "number" => 7, + "state" => "open", + "secret_type" => "slack_api_token", + "hypatia_location" => %{"error" => "no location names a file in the repository"} + } + ]) + + normalized = Hypatia.CLI.normalize_secret_alert(finding) + assert normalized.file == "." + assert normalized.detail.alert_number == 7 + + assert normalized.detail.location == + %{status: "unavailable", reason: "no location names a file in the repository"} + + assert {:ok, _} = Jason.encode(normalized) + end + end + describe "scan_from_path/1" do test "returns error when remote is not a github URL" do tmp = Path.join(System.tmp_dir!(), "ssa-test-#{System.unique_integer([:positive])}")