From 9ddd5199c6b3bef4e83b1ec383253b4920618736 Mon Sep 17 00:00:00 2001 From: Jatin <84621253+h0x0er@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:55:24 +0530 Subject: [PATCH 1/4] revert github dns experiment when global feature flag is enabled --- agent.go | 4 ++ github_dns_experiment.go | 95 ++++++++++++++++++++++++++++++++++++++++ global_feature_flags.go | 1 + 3 files changed, 100 insertions(+) create mode 100644 github_dns_experiment.go diff --git a/agent.go b/agent.go index 152417a..9ae1ef9 100644 --- a/agent.go +++ b/agent.go @@ -82,6 +82,10 @@ func Run(ctx context.Context, configFilePath string, hostDNSServer DNSServer, WriteLog("\n") globalFlags := GetGlobalFeatureFlags() + if globalFlags.RevertGithubDNSExperiment { + RevertGithubDNSExperiment() + } + globalBlocklistResponse, err := apiclient.getGlobalBlocklist() globalBlocklist := NewGlobalBlocklist(globalBlocklistResponse) if err != nil { diff --git a/github_dns_experiment.go b/github_dns_experiment.go new file mode 100644 index 0000000..df86323 --- /dev/null +++ b/github_dns_experiment.go @@ -0,0 +1,95 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "strings" +) + +// GitHub is experimenting with a DNS resolver and an outbound proxy on hosted runners, +// which bypasses the agent's DNS server and results in allowed connections being dropped. +// ref: https://github.com/step-security/harden-runner/issues/708 +const ( + githubDNSDropInPath = "/etc/systemd/resolved.conf.d/90-egress-proxy.conf" + githubDNSResolverAddress = "127.0.0.100" + etcResolvConfPath = "/etc/resolv.conf" + stubResolvConfPath = "/run/systemd/resolve/stub-resolv.conf" +) + +var githubDNSExperimentServices = []string{"egress-dns-resolver.service", "egress-outbound-proxy.service"} + +// RevertGithubDNSExperiment reverts github's dns experiment, if it is present on the runner +func RevertGithubDNSExperiment() { + if !isGithubDNSExperimentPresent() { + WriteLog("[github-dns-experiment] not detected") + return + } + + WriteLog("[github-dns-experiment] detected, reverting") + if err := revertGithubDNSExperiment(); err != nil { + WriteLog(fmt.Sprintf("[github-dns-experiment] error reverting: %v", err)) + return + } + WriteLog("[github-dns-experiment] reverted") +} + +func isGithubDNSExperimentPresent() bool { + if _, err := os.Stat(githubDNSDropInPath); err == nil { + return true + } + + for _, service := range githubDNSExperimentServices { + if isServiceActive(service) { + return true + } + } + + resolvConf, err := os.ReadFile(etcResolvConfPath) + return err == nil && strings.Contains(string(resolvConf), githubDNSResolverAddress) +} + +func revertGithubDNSExperiment() error { + for _, service := range githubDNSExperimentServices { + if !isServiceActive(service) { + continue + } + if err := exec.Command("sudo", "systemctl", "stop", service).Run(); err != nil { + return fmt.Errorf("error stopping %s: %v", service, err) + } + if isServiceActive(service) { + return fmt.Errorf("%s is still active", service) + } + } + + if err := os.Remove(githubDNSDropInPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("error removing %s: %v", githubDNSDropInPath, err) + } + + if err := exec.Command("sudo", "systemctl", "restart", "systemd-resolved").Run(); err != nil { + return fmt.Errorf("error restarting systemd-resolved: %v", err) + } + + if _, err := os.Stat(stubResolvConfPath); err != nil { + return fmt.Errorf("error finding %s: %v", stubResolvConfPath, err) + } + + if err := os.Remove(etcResolvConfPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("error removing %s: %v", etcResolvConfPath, err) + } + + if err := os.Symlink(stubResolvConfPath, etcResolvConfPath); err != nil { + return fmt.Errorf("error linking %s to %s: %v", etcResolvConfPath, stubResolvConfPath, err) + } + + if err := exec.Command("sudo", "resolvectl", "flush-caches").Run(); err != nil { + WriteLog(fmt.Sprintf("[github-dns-experiment] error flushing dns caches: %v", err)) + } + + return nil +} + +func isServiceActive(service string) bool { + output, _ := exec.Command("systemctl", "is-active", service).Output() + return strings.TrimSpace(string(output)) == "active" +} diff --git a/global_feature_flags.go b/global_feature_flags.go index 81201aa..eca7ce5 100644 --- a/global_feature_flags.go +++ b/global_feature_flags.go @@ -15,6 +15,7 @@ type GlobalFeatureFlags struct { EnableArmour bool `json:"enable_armour"` EnableCustomDetectionRules bool `json:"enable_custom_detection_rules"` DisableKillEnforcement bool `json:"disable_kill_enforcement,omitempty"` + RevertGithubDNSExperiment bool `json:"revert_github_dns_experiment,omitempty"` } // GlobalFeatureFlagManager manages fetching and caching of global feature flags. From dd821b44cf373e369c780e5cc02bc78c5ed095a3 Mon Sep 17 00:00:00 2001 From: Jatin <84621253+h0x0er@users.noreply.github.com> Date: Tue, 27 Oct 2026 13:08:03 +0530 Subject: [PATCH 2/4] also detect and remove github dns experiment drop-in under /run --- github_dns_experiment.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/github_dns_experiment.go b/github_dns_experiment.go index df86323..6a14671 100644 --- a/github_dns_experiment.go +++ b/github_dns_experiment.go @@ -11,7 +11,6 @@ import ( // which bypasses the agent's DNS server and results in allowed connections being dropped. // ref: https://github.com/step-security/harden-runner/issues/708 const ( - githubDNSDropInPath = "/etc/systemd/resolved.conf.d/90-egress-proxy.conf" githubDNSResolverAddress = "127.0.0.100" etcResolvConfPath = "/etc/resolv.conf" stubResolvConfPath = "/run/systemd/resolve/stub-resolv.conf" @@ -19,6 +18,9 @@ const ( var githubDNSExperimentServices = []string{"egress-dns-resolver.service", "egress-outbound-proxy.service"} +// systemd-resolved reads drop-ins from both /etc and /run +var githubDNSDropInPaths = []string{"/etc/systemd/resolved.conf.d/90-egress-proxy.conf", "/run/systemd/resolved.conf.d/90-egress-proxy.conf"} + // RevertGithubDNSExperiment reverts github's dns experiment, if it is present on the runner func RevertGithubDNSExperiment() { if !isGithubDNSExperimentPresent() { @@ -35,8 +37,10 @@ func RevertGithubDNSExperiment() { } func isGithubDNSExperimentPresent() bool { - if _, err := os.Stat(githubDNSDropInPath); err == nil { - return true + for _, dropInPath := range githubDNSDropInPaths { + if _, err := os.Stat(dropInPath); err == nil { + return true + } } for _, service := range githubDNSExperimentServices { @@ -62,8 +66,10 @@ func revertGithubDNSExperiment() error { } } - if err := os.Remove(githubDNSDropInPath); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("error removing %s: %v", githubDNSDropInPath, err) + for _, dropInPath := range githubDNSDropInPaths { + if err := os.Remove(dropInPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("error removing %s: %v", dropInPath, err) + } } if err := exec.Command("sudo", "systemctl", "restart", "systemd-resolved").Run(); err != nil { From 2293ed099d0f046f09694ac04a8f784583a1ed7d Mon Sep 17 00:00:00 2001 From: Jatin <84621253+h0x0er@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:06:08 +0530 Subject: [PATCH 3/4] chore: bump go to 1.27.3 --- .github/workflows/int.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/test.yml | 2 +- go.mod | 2 +- release-monitor.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml index c33cf43..a700d8a 100644 --- a/.github/workflows/int.yml +++ b/.github/workflows/int.yml @@ -23,7 +23,7 @@ jobs: - name: Set up Go uses: actions/setup-go@424fc82d43fa5a37540bae62709ddcc23d9520d4 with: - go-version: 1.26.2 + go-version: 1.27.3 - name: Configure .netrc run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 902ab60..0df8365 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,7 +29,7 @@ jobs: - name: Set up Go uses: actions/setup-go@424fc82d43fa5a37540bae62709ddcc23d9520d4 with: - go-version: 1.26.2 + go-version: 1.27.3 - name: Configure .netrc run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6dd66b5..22b99bb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,7 +21,7 @@ jobs: - name: Set up Go uses: actions/setup-go@424fc82d43fa5a37540bae62709ddcc23d9520d4 with: - go-version: 1.26.2 + go-version: 1.27.3 - name: Configure .netrc run: | diff --git a/go.mod b/go.mod index 267f570..613714b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/step-security/agent -go 1.26.2 +go 1.27.3 require ( github.com/coreos/go-iptables v0.6.0 diff --git a/release-monitor.yml b/release-monitor.yml index dbaaf08..84d4614 100644 --- a/release-monitor.yml +++ b/release-monitor.yml @@ -7,7 +7,7 @@ release-process: - artifact: agent_{{.Version}}_linux_amd64.tar.gz binary: agent build-command: go build -trimpath -ldflags="-s -w -X main.version={{.Version}} -X main.commit={{.FullCommit}} -X main.date=123 -X main.ReleaseTag={{.Tag}}" - go-version: 1.19.8 + go-version: 1.27.3 pipeline: github-action: repo: step-security/agent From 0669b572e791080adf095c58cff42f2b555ed016 Mon Sep 17 00:00:00 2001 From: Jatin <84621253+h0x0er@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:57:02 +0530 Subject: [PATCH 4/4] revert github dns experiment in fail-safe order and replace resolv.conf symlink atomically --- github_dns_experiment.go | 47 +++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/github_dns_experiment.go b/github_dns_experiment.go index 6a14671..fe755a1 100644 --- a/github_dns_experiment.go +++ b/github_dns_experiment.go @@ -54,44 +54,55 @@ func isGithubDNSExperimentPresent() bool { } func revertGithubDNSExperiment() error { + // repoint dns away from github's resolver before stopping it, so a failure at any step leaves dns working + for _, dropInPath := range githubDNSDropInPaths { + if err := os.Remove(dropInPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("error removing %s: %v", dropInPath, err) + } + } + + if err := exec.Command("sudo", "systemctl", "restart", "systemd-resolved").Run(); err != nil { + return fmt.Errorf("error restarting systemd-resolved: %v", err) + } + + if err := linkResolvConfToStub(); err != nil { + return err + } + + // github's services are no longer in the dns path, so failing to stop them is not fatal for _, service := range githubDNSExperimentServices { if !isServiceActive(service) { continue } if err := exec.Command("sudo", "systemctl", "stop", service).Run(); err != nil { - return fmt.Errorf("error stopping %s: %v", service, err) - } - if isServiceActive(service) { - return fmt.Errorf("%s is still active", service) + WriteLog(fmt.Sprintf("[github-dns-experiment] error stopping %s: %v", service, err)) } } - for _, dropInPath := range githubDNSDropInPaths { - if err := os.Remove(dropInPath); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("error removing %s: %v", dropInPath, err) - } + if err := exec.Command("sudo", "resolvectl", "flush-caches").Run(); err != nil { + WriteLog(fmt.Sprintf("[github-dns-experiment] error flushing dns caches: %v", err)) } - if err := exec.Command("sudo", "systemctl", "restart", "systemd-resolved").Run(); err != nil { - return fmt.Errorf("error restarting systemd-resolved: %v", err) - } + return nil +} +// linkResolvConfToStub atomically points /etc/resolv.conf to systemd-resolved's stub resolver +func linkResolvConfToStub() error { if _, err := os.Stat(stubResolvConfPath); err != nil { return fmt.Errorf("error finding %s: %v", stubResolvConfPath, err) } - if err := os.Remove(etcResolvConfPath); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("error removing %s: %v", etcResolvConfPath, err) + tmpPath := etcResolvConfPath + ".tmp" + os.Remove(tmpPath) + if err := os.Symlink(stubResolvConfPath, tmpPath); err != nil { + return fmt.Errorf("error creating %s: %v", tmpPath, err) } - if err := os.Symlink(stubResolvConfPath, etcResolvConfPath); err != nil { + if err := os.Rename(tmpPath, etcResolvConfPath); err != nil { + os.Remove(tmpPath) return fmt.Errorf("error linking %s to %s: %v", etcResolvConfPath, stubResolvConfPath, err) } - if err := exec.Command("sudo", "resolvectl", "flush-caches").Run(); err != nil { - WriteLog(fmt.Sprintf("[github-dns-experiment] error flushing dns caches: %v", err)) - } - return nil }