From fb20d6552ce577598ce0816ff3bc1682a377ce95 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Mon, 16 Nov 2026 09:19:00 +0000 Subject: [PATCH 1/2] fix(rules): reconcile RE001 secret inheritance and endpoint evidence --- CHANGELOG.adoc | 21 ++ lib/rules/research_extensions.ex | 107 +++++----- ...arch_extensions_review_regression_test.exs | 199 ++++++++++++++++++ 3 files changed, 274 insertions(+), 53 deletions(-) create mode 100644 test/research_extensions_review_regression_test.exs diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index 0dba05f6..f6d86899 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -12,6 +12,27 @@ https://semver.org/[Semantic Versioning]. === Changed +==== RE001 post-merge review corrections (2026-11-16) + +RE001 excludes workflow environment secrets overridden by a job, including +when deciding whether fetched code shares a step with a credential. Existing +alias-use locations and job traversal order are preserved. + +Endpoint discovery distinguishes numeric loopback addresses from `127.*` +hostnames, keeps full punycode names and explicit Docker registry hosts, and +treats hashes in environment and action-input values as data. Run scripts now +use `Workflow.shell_mask/1`, including its command-substitution handling, +instead of a second shell-comment walker. GitHub wildcard allow entries stay +within the GitHub exemption; unknown runtime ports cannot establish an +allowance for every port. + +Ten new regression tests reproduce the eight outstanding #1036 findings on +the previous implementation. The repaired implementation passes 189 focused +tests, including the existing located-diagnostic, representation-equivalence +and bounded Egress decision tests. RE001 findings are unchanged on snapshots +of Hypatia `b93f0615`, standards `305743d2` and Echidna `478b7519` (zero before +and after on each). These checks do not complete #971 or prove runtime egress. + ==== The oracle and CI toolchain move back to OTP 27.3.4.19 (2026-11-13) `rsr-oracle.yml` now defaults to OTP `27.3.4.19`, and the two `ci.yml` diff --git a/lib/rules/research_extensions.ex b/lib/rules/research_extensions.ex index 31c853ce..f3b4cfb3 100644 --- a/lib/rules/research_extensions.ex +++ b/lib/rules/research_extensions.ex @@ -418,18 +418,14 @@ defmodule Hypatia.Rules.ResearchExtensions do defp re001_jobs(rel, %Workflow{} = workflow) do untrusted_events = Enum.filter(workflow.triggers, &(&1 in @re001_untrusted_events)) - # Secrets in the workflow's top-level `env:` are inherited by every job, - # so each job reads them too, and a declaration must list them. - workflow_refs = node_secret_refs(workflow.env) - workflow_env_credentials = env_credentials(workflow.env) - for job <- workflow.jobs, # Reusable jobs delegate their runtime to the source workflow: only a # job's own `runs-on` makes it local. job.runs_on != nil, + inherited_env = job_env_entries(workflow.env, job.env), refs = Enum.reject( - node_secret_refs(job.node) ++ workflow_refs, + node_secret_refs(job.node) ++ inherited_workflow_refs(workflow.env, job.env), &(elem(&1, 0) in @re001_ambient_secrets) ), refs != [] do @@ -437,7 +433,7 @@ defmodule Hypatia.Rules.ResearchExtensions do # A credential in the job's or the workflow's `env:` is in every step's # environment, unless the step sets the same variable itself. - env_credentials = env_credentials(job.env) ++ workflow_env_credentials + env_credentials = env_credentials(inherited_env) job_untrusted_events = case job_event_gate(job) do @@ -524,13 +520,24 @@ defmodule Hypatia.Rules.ResearchExtensions do # The `env:` variables of a workflow or job that hold a real secret, as # {NAME, line of the read}. - defp env_credentials(env) do - for %{key: key, value: value} <- Workflow.entries(env), + defp env_credentials(entries) do + for %{key: key, value: value} <- entries, {name, line} <- node_secret_refs(value), name not in @re001_ambient_secrets, do: {key.value, line} end + # Keep the job's original traversal order and alias locations, adding only + # workflow environment values that its own environment does not override. + defp inherited_workflow_refs(workflow_env, job_env) do + keys = MapSet.new(Workflow.entries(job_env), &Workflow.text(&1.key)) + + workflow_env + |> Workflow.entries() + |> Enum.reject(&(Workflow.text(&1.key) in keys)) + |> Enum.flat_map(&node_secret_refs(&1.value)) + end + # The events a job-level `if:` restricts the job to, when the condition is # nothing but `github.event_name == ''` tests joined by `||`. # Anything more complex is not interpreted, and the job is judged on all the @@ -765,11 +772,12 @@ defmodule Hypatia.Rules.ResearchExtensions do # A host a workflow can name: a dotted domain name, or a literal IPv4 # address. A single-label name (`localhost`, a service container such as # `postgres`) is not egress and is not compared. - @host ~S"[A-Za-z0-9](?:[A-Za-z0-9.-]*[A-Za-z0-9])?\.[A-Za-z]{2,}|(?:\d{1,3}\.){3}\d{1,3}(?![\w-]|\.\w)" + @label ~S"[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?" + @host "(?:(?:#{@label}\\.)+(?:xn--#{@label}|[A-Za-z]{2,})|(?:\\d{1,3}\\.){3}\\d{1,3})\\.?(?![\\w.-])" # A URL's scheme (in any case, as clients read it), host, and port: digits, # or an expression (`:${{ vars.PORT }}`), whose value is known only at run time. # The schemes are the web's, and the databases' and brokers' a job reaches. - @url_host ~r<\b(https?|wss?|ssh|s?ftp|ftps|git|postgres(?:ql)?|mysql|mariadb|rediss?|mongodb(?:\+srv)?|amqps?|mqtts?|ldaps?|smtps?)://(?:[^@/\s"'`]+@)?(#{@host})(?::(\d{1,5})|(:\$\{\{))?>i + @url_host ~r<\b(https?|wss?|ssh|s?ftp|ftps|git|docker|postgres(?:ql)?|mysql|mariadb|rediss?|mongodb(?:\+srv)?|amqps?|mqtts?|ldaps?|smtps?)://(?:[^@/\s"'`]+@)?(#{@host})(?::(\d{1,5})|(:\$\{\{))?>i # A `user@host:path` remote. The path is not blank (`a@example.com: failed` # is prose) and not a port (`user:pass@host:5432/db` is a URL's userinfo), # though it may start with a digit (`git@gitlab.com:123team/x.git`). @@ -778,6 +786,7 @@ defmodule Hypatia.Rules.ResearchExtensions do "http" => 80, "ws" => 80, "https" => 443, + "docker" => 443, "wss" => 443, "ssh" => 22, "git" => 9418, @@ -810,10 +819,17 @@ defmodule Hypatia.Rules.ResearchExtensions do steps |> Enum.flat_map(fn step -> own = MapSet.new(Workflow.entries(step.env), &Workflow.text(&1.key)) - Enum.reject(inherited, &(Workflow.text(&1.key) in own)) ++ step_entries(step) + + inherited_nodes = + inherited + |> Enum.reject(&(Workflow.text(&1.key) in own)) + |> Enum.map(&{&1.value, false}) + + inherited_nodes ++ + Enum.map(step_entries(step), &{&1.value, Workflow.text(&1.key) == "run"}) end) |> Enum.uniq() - |> Enum.flat_map(&named_hosts(&1.value)) + |> Enum.flat_map(fn {node, shell?} -> named_hosts(node, shell?) end) end # The `env:` entries a job passes to its steps: its own, and the @@ -831,50 +847,26 @@ defmodule Hypatia.Rules.ResearchExtensions do do: Enum.reject(Workflow.entries(step.node), &(Workflow.text(&1.key) in @step_labels)) # The hosts outside GitHub a node names literally, as {host, port, line}. - defp named_hosts(node) do + defp named_hosts(node, shell?) do for {text, line_of} <- Workflow.strings(node), + # Shared byte-preserving mask: endpoint positions still refer to the + # original scalar, including folded values and alias use sites. + text = if(shell?, do: Workflow.shell_mask(text), else: text), {host, port, at} <- url_hosts(text) ++ scp_hosts(text), - not in_shell_comment?(text, at), - host = String.downcase(host), + host = host |> String.downcase() |> String.trim_trailing("."), not Regex.match?(@github_hosts, host), not loopback?(host), do: {host, port, line_of.(at)} end - defp loopback?(host), do: Regex.match?(~r/\A(?:127\.|0\.0\.0\.0\z)/, host) - - # Is offset `at` inside a `#` comment: after a `#` that starts a word (at - # the start of the line, or after a blank or a control or redirection - # operator: `; & | ( ) < >`), outside quotes? (A URL fragment, `/#x`, is - # not one, nor is the `#` of `git commit -m "fix #12"`.) - defp in_shell_comment?(text, at) do - line_start = - case :binary.matches(binary_part(text, 0, at), "\n") do - [] -> 0 - matches -> matches |> List.last() |> elem(0) |> Kernel.+(1) - end - - comment_opens?(binary_part(text, line_start, at - line_start), nil, true) + defp loopback?(host) do + case :inet.parse_address(String.to_charlist(host)) do + {:ok, {127, _, _, _}} -> true + {:ok, {0, 0, 0, 0}} -> true + _ -> false + end end - # Walks one line: `quote` is the open quote character or nil, `word_start?` - # whether the next character starts a word. A backslash escapes the next - # character outside single quotes, as the shell reads it. - @word_breaks [?\s, ?\t, ?;, ?&, ?|, ?(, ?), ?<, ?>] - defp comment_opens?(<<>>, _quote, _word_start?), do: false - defp comment_opens?(<>, nil, true), do: true - - defp comment_opens?(<>, nil, _) when c in [?', ?"], - do: comment_opens?(rest, c, false) - - defp comment_opens?(<>, quote, _) when quote != ?', - do: comment_opens?(rest, quote, false) - - defp comment_opens?(<>, c, _), do: comment_opens?(rest, nil, false) - - defp comment_opens?(<>, quote, _), - do: comment_opens?(rest, quote, quote == nil and c in @word_breaks) - # A port written as an expression is unknown (nil), not the scheme's default. defp url_hosts(text) do for [{at, _}, {s, sl}, {h, hl} | port] <- Regex.scan(@url_host, text, return: :index) do @@ -938,7 +930,7 @@ defmodule Hypatia.Rules.ResearchExtensions do # Does a {pattern, port} cover a host and port? `*.x` covers the # subdomains of x (as harden-runner reads it), not x itself; an endpoint - # without a port (nil or `:any`) covers every port. A port known only at + # without a port (`:any`) covers every port. A port known only at # run time (nil) is covered by any port of the host; every port (`:any`) # only by an endpoint without a port. defp covers?({pattern, want}, host, port) do @@ -948,11 +940,15 @@ defmodule Hypatia.Rules.ResearchExtensions do _ -> pattern == host end - host_ok and (want in [nil, :any] or port == nil or want == port) + host_ok and (want == :any or (port != :any and (want == nil or port == nil or want == port))) end defp re001_endpoint_findings(job, deed_file, decl) do - declared = decl.endpoints |> Enum.map(&literal_endpoint/1) |> Enum.reject(&is_nil/1) + declared = + decl.endpoints + |> Enum.map(&literal_endpoint/1) + |> Enum.reject(&is_nil/1) + |> Enum.map(&any_port/1) uncovered = job.named_hosts @@ -970,7 +966,7 @@ defmodule Hypatia.Rules.ResearchExtensions do defp allowed_endpoint_findings(%{allowed_endpoints: nil}, _deed_file, _declared), do: [] defp allowed_endpoint_findings(job, deed_file, declared) do - allowed = Enum.reject(job.allowed_endpoints, &Regex.match?(@github_hosts, elem(&1, 0))) + allowed = Enum.reject(job.allowed_endpoints, &github_endpoint?(elem(&1, 0))) # What the hardener lets out must be declared. A declared literal host # must be let out, or block mode stops the job; a declared wildcard is a @@ -983,7 +979,7 @@ defmodule Hypatia.Rules.ResearchExtensions do # allowed entry without a port lets it out. blocked = Enum.reject(declared, fn {h, p} -> - String.starts_with?(h, "*.") or Regex.match?(@github_hosts, h) or + String.starts_with?(h, "*.") or github_endpoint?(h) or Enum.any?(allowed, &covers?(&1, h, p || :any)) end) @@ -994,6 +990,11 @@ defmodule Hypatia.Rules.ResearchExtensions do end end + # A wildcard restricted to a GitHub domain is still a GitHub endpoint; + # stripping only the leading wildcard preserves the domain boundary. + defp github_endpoint?("*." <> suffix), do: Regex.match?(@github_hosts, "probe." <> suffix) + defp github_endpoint?(host), do: Regex.match?(@github_hosts, host) + defp endpoint_text({host, port}) when port in [nil, :any], do: "\"#{host}\"" defp endpoint_text({host, port}), do: "\"#{host}:#{port}\"" diff --git a/test/research_extensions_review_regression_test.exs b/test/research_extensions_review_regression_test.exs new file mode 100644 index 00000000..cee0924e --- /dev/null +++ b/test/research_extensions_review_regression_test.exs @@ -0,0 +1,199 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) + +defmodule Hypatia.Rules.ResearchExtensionsReviewRegressionTest do + use ExUnit.Case, async: true + + alias Hypatia.Rules.ResearchExtensions + + defp indent(text, width) do + text + |> String.trim_trailing() + |> String.split("\n") + |> Enum.map_join("\n", &(String.duplicate(" ", width) <> &1)) + |> Kernel.<>("\n") + end + + defp findings(steps, endpoints \\ [], options \\ []) do + repo = Path.join(System.tmp_dir!(), "re001_review_#{System.unique_integer([:positive])}") + File.mkdir_p!(Path.join(repo, ".github/workflows")) + on_exit(fn -> File.rm_rf!(repo) end) + + workflow_env = + if env = options[:workflow_env], do: "env:\n" <> indent(env, 2), else: "" + + job_env = if env = options[:job_env], do: " env:\n" <> indent(env, 6), else: "" + + File.write!( + Path.join(repo, ".github/workflows/ship.yml"), + "on: push\n" <> + workflow_env <> + "jobs:\n ship:\n runs-on: ubuntu-latest\n" <> + job_env <> + " steps:\n" <> indent(steps, 6) + ) + + quoted = fn values -> Enum.map_join(values, " ", &inspect/1) end + + File.write!(Path.join(repo, "test_chora.deed"), """ + ;; SPDX-License-Identifier: CC-BY-SA-4.0 + (repo-deed + :schema-version "1.0.0" + :canonical-name "test" + :repo-uuid #u8"chora-registry:github.com/o/test" + :beholding-chora #u8"chora:estate/github.com/o" + (egress + (job :workflow ".github/workflows/ship.yml" :job "ship" + :secrets (#{quoted.(options[:secrets] || ["TOKEN"])}) + :endpoints (#{quoted.(endpoints)}) :enforcement none))) + """) + + ResearchExtensions.re001_missing_harden_runner(repo) + end + + defp run(script), + do: "- run: |\n" <> indent(script, 4) <> " env:\n K: ${{ secrets.TOKEN }}\n" + + test "only numeric loopback addresses are exempt, not 127-prefixed hostnames" do + assert [finding] = findings(run("curl https://127.example.com")) + assert finding.reason =~ ~s|"127.example.com:443"| + assert findings(run("curl https://127.0.0.1 https://127.255.255.254 https://0.0.0.0")) == [] + assert [_] = findings(run("curl https://127.999.0.1")) + end + + test "GitHub wildcard allows are exempt only within GitHub's domain boundary" do + steps = """ + - uses: step-security/harden-runner@v2 + with: + egress-policy: block + allowed-endpoints: '*.github.com:443 *.githubusercontent.com:443' + - run: echo ${{ secrets.TOKEN }} + """ + + assert findings(steps) == [] + + assert [finding] = + findings( + String.replace(steps, "*.github.com:443", "*.github.com.attacker.invalid:443") + ) + + assert finding.reason =~ "github.com.attacker.invalid" + end + + test "punycode hostnames and their ports are matched in full" do + steps = run("curl https://example.xn--p1ai:8443/upload") + assert [finding] = findings(steps) + assert finding.reason =~ ~s|"example.xn--p1ai:8443"| + assert findings(steps, ["example.xn--p1ai:8443"]) == [] + assert [_] = findings(steps, ["example.xn:8443"]) + + assert findings(run("curl https://EXAMPLE.XN--P1AI.:8443/upload"), ["example.xn--p1ai:8443"]) == + [] + + assert findings(run("curl https://example.com.:8443/upload"), ["example.com:8443"]) == [] + end + + test "Docker action references name explicit external registries" do + for {image, endpoint} <- [ + {"registry.example.com/org/action:tag", "registry.example.com:443"}, + {"registry.example.com:5000/org/action:tag", "registry.example.com:5000"} + ] do + steps = "- uses: docker://#{image}\n env:\n K: ${{ secrets.TOKEN }}\n" + assert [finding] = findings(steps) + assert finding.reason =~ endpoint + assert findings(steps, [endpoint]) == [] + end + end + + test "job environment overrides remove the workflow's shadowed secret" do + assert findings("- run: echo ready", [], + workflow_env: "TOKEN: ${{ secrets.OLD }}", + job_env: "TOKEN: ${{ secrets.NEW }}", + secrets: ["NEW"] + ) == [] + + assert [finding] = + findings("- run: echo ready", [], + workflow_env: "TOKEN: ${{ secrets.OLD }}", + job_env: "OTHER: ${{ secrets.NEW }}", + secrets: ["NEW"] + ) + + assert finding.reason =~ ~s|reads undeclared "OLD"| + end + + test "job overrides also remove shadowed credentials from code exposure" do + assert [finding] = + findings("- run: npm ci\n- run: send\n env:\n K: ${{ secrets.TOKEN }}", [], + workflow_env: "SHADOW: ${{ secrets.OLD }}", + job_env: "SHADOW: public", + secrets: ["DECLARED"] + ) + + assert finding.severity == :warn + assert finding.reason =~ ~s|reads undeclared "TOKEN"| + refute finding.reason =~ "OLD" + refute finding.reason =~ "in the same step" + end + + test "a command substitution closes inside its word before a literal hash" do + for prefix <- [ + "echo $(printf x)#tag;", + ~S|echo $(printf '%s' "$(printf x)")#tag;|, + ~S|echo "$(printf x)"#tag;| + ] do + assert [finding] = findings(run(prefix <> " curl https://upload.example.com")) + assert finding.reason =~ "upload.example.com:443" + end + + assert findings(run("echo $(printf x) # curl https://comment.example.com")) == [] + assert findings(run("(echo x)# curl https://comment.example.com")) == [] + end + + test "an unknown allowed port is not evidence that every port is allowed" do + steps = """ + - uses: step-security/harden-runner@v2 + with: + egress-policy: block + allowed-endpoints: api.example.com:${{ vars.PORT }} + - run: echo ${{ secrets.TOKEN }} + """ + + assert [finding] = findings(steps, ["api.example.com"]) + assert finding.reason =~ ~s|does not allow declared "api.example.com"| + assert findings(String.replace(steps, ":${{ vars.PORT }}", ""), ["api.example.com"]) == [] + end + + test "hashes in environment values are data, including inherited values" do + for options <- [ + [workflow_env: "UPLOAD: '# production https://upload.example.com'"], + [job_env: "UPLOAD: '# production https://upload.example.com'"] + ] do + assert [finding] = findings(run("curl $UPLOAD"), [], options) + assert finding.reason =~ "upload.example.com:443" + end + + steps = """ + - run: curl "$UPLOAD" + env: + K: ${{ secrets.TOKEN }} + UPLOAD: '# production https://upload.example.com' + """ + + assert [finding] = findings(steps) + assert finding.reason =~ "upload.example.com:443" + end + + test "hashes in action input values are data, including an input called run" do + steps = """ + - uses: owner/action@v1 + with: + run: '# production https://upload.example.com' + token: ${{ secrets.TOKEN }} + """ + + assert [finding] = findings(steps) + assert finding.reason =~ "upload.example.com:443" + assert findings(steps, ["upload.example.com:443"]) == [] + end +end From 84f719fc2b227da930a412e358fd644dfb37e1e8 Mon Sep 17 00:00:00 2001 From: "coderabbitai[bot]" <136622811+coderabbitai[bot]@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:00:11 +0000 Subject: [PATCH 2/2] fix(rules): exclude Docker images without registries from endpoints --- lib/rules/research_extensions.ex | 3 ++- test/research_extensions_review_regression_test.exs | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/rules/research_extensions.ex b/lib/rules/research_extensions.ex index f3b4cfb3..2811ed64 100644 --- a/lib/rules/research_extensions.ex +++ b/lib/rules/research_extensions.ex @@ -777,7 +777,8 @@ defmodule Hypatia.Rules.ResearchExtensions do # A URL's scheme (in any case, as clients read it), host, and port: digits, # or an expression (`:${{ vars.PORT }}`), whose value is known only at run time. # The schemes are the web's, and the databases' and brokers' a job reaches. - @url_host ~r<\b(https?|wss?|ssh|s?ftp|ftps|git|docker|postgres(?:ql)?|mysql|mariadb|rediss?|mongodb(?:\+srv)?|amqps?|mqtts?|ldaps?|smtps?)://(?:[^@/\s"'`]+@)?(#{@host})(?::(\d{1,5})|(:\$\{\{))?>i + # Docker names a registry only when a slash follows its host and optional port. + @url_host ~r<\b(docker(?=://#{@host}(?::\d{1,5})?/)|https?|wss?|ssh|s?ftp|ftps|git|postgres(?:ql)?|mysql|mariadb|rediss?|mongodb(?:\+srv)?|amqps?|mqtts?|ldaps?|smtps?)://(?:[^@/\s"'`]+@)?(#{@host})(?::(\d{1,5})|(:\$\{\{))?>i # A `user@host:path` remote. The path is not blank (`a@example.com: failed` # is prose) and not a port (`user:pass@host:5432/db` is a URL's userinfo), # though it may start with a digit (`git@gitlab.com:123team/x.git`). diff --git a/test/research_extensions_review_regression_test.exs b/test/research_extensions_review_regression_test.exs index cee0924e..34ef445c 100644 --- a/test/research_extensions_review_regression_test.exs +++ b/test/research_extensions_review_regression_test.exs @@ -105,6 +105,11 @@ defmodule Hypatia.Rules.ResearchExtensionsReviewRegressionTest do end end + test "a dotted Docker image without an explicit registry is not an endpoint" do + steps = "- uses: docker://my.action:1\n env:\n K: ${{ secrets.TOKEN }}\n" + assert findings(steps) == [] + end + test "job environment overrides remove the workflow's shadowed secret" do assert findings("- run: echo ready", [], workflow_env: "TOKEN: ${{ secrets.OLD }}",