fix: patch Python and Go trust stores for eMudhra

.github/workflows/ci.yml

@@ -33,7 +33,9 @@ jobs:
       - name: Install dev deps
         run: |
           python -m pip install --upgrade pip
-          pip install -e ".[dev,k8s]"
+          pip install --require-hashes -r dev-requirements.txt
+          pip install -e ".[k8s]"
+          pip check
 
       - name: Lint (flake8)
         run: flake8 --per-file-ignores=l9gpu/_version.py:F401 l9gpu
@@ -63,7 +65,7 @@ jobs:
 
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.25"
+          go-version: "1.27.3"
           cache-dependency-path: ${{ matrix.module }}/go.sum
 
       - run: go mod download
@@ -78,8 +80,8 @@ jobs:
           working-directory: ${{ matrix.module }}
           args: --timeout=5m
           # golangci-lint v1.64 ships with Go 1.24 — skip lint until a
-          # newer release supports the Go 1.25 toolchain required by the
-          # upstream OTel collector v0.150 dependencies.
+          # newer release supports the Go 1.27 toolchain used for the
+          # crypto/x509 eMudhra certificate blocklist.
         continue-on-error: true
 
   helm:
@@ -168,7 +170,7 @@ jobs:
 
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.25"
+          go-version: "1.27.3"
 
       - name: Install OpenTelemetry Collector Builder
         env:

.github/workflows/collector-release.yml

@@ -15,7 +15,7 @@ concurrency:
   cancel-in-progress: false
 
 env:
-  GO_VERSION: "1.25"
+  GO_VERSION: "1.27.3"
   OCB_VERSION: "v0.150.0"
 
 jobs:
@@ -44,7 +44,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/go/bin/builder
-          key: ocb-${{ runner.os }}-${{ env.OCB_VERSION }}
+          key: ocb-${{ runner.os }}-${{ env.GO_VERSION }}-${{ env.OCB_VERSION }}
 
       - name: Install OpenTelemetry Collector Builder
         if: steps.cache-ocb.outputs.cache-hit != 'true'

SECURITY.md

@@ -37,3 +37,39 @@ Out of scope:
 - Vulnerabilities requiring physical access to GPU nodes or root on the host
 
 Thank you for helping keep the project and its users safe.
+
+## eMudhra trust-store updates (October 2026)
+
+Python installations require `certifi>=2026.10.19`. The development lockfile
+also includes patched `pip>=26.2.2`, which carries its own CA bundle. Go
+modules and collector builds require Go 1.27.3 or newer. See the
+[incident and patched-version list](https://emudhra.we-have-a-problem.com/)
+and the [Go security announcement](https://groups.google.com/g/golang-announce/c/66pojueaQek).
+
+After installing the locked dependencies, run this network smoke check:
+
+```bash
+python scripts/check_tls_trust.py
+```
+
+It must accept a trusted HTTPS endpoint and reject both the direct and
+cross-signed compromised test chains with certificate verification errors.
+Connection failures do not count as successful rejection. Add `--system` to
+check the host or container system CA bundle too.
+
+Rebuild Python and collector release artifacts with the updated dependencies
+and Go compiler before publishing a new release. Rebuild container images
+with `--pull --no-cache` to refresh base-image CA bundles and dependency
+layers; existing images and installed packages do not receive these changes
+automatically. The Python image refreshes Debian's `ca-certificates` package
+in both stages. Verify a rebuilt Python image before publishing:
+
+```bash
+docker build --pull --no-cache -f docker/Dockerfile -t l9gpu:tls-check .
+docker run --rm --entrypoint python \
+  -v "$PWD/scripts:/checks:ro" l9gpu:tls-check \
+  /checks/check_tls_trust.py --system
+```
+
+Publish rebuilt packages and images through the normal release workflows.
+Do not disable certificate verification to restore connectivity.

docker/Dockerfile

@@ -25,7 +25,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1
 
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends build-essential && \
+    apt-get install -y --no-install-recommends build-essential ca-certificates && \
     rm -rf /var/lib/apt/lists/*
 
 WORKDIR /build
@@ -56,6 +56,10 @@ RUN if [ "$INSTALL_AMD" = "1" ]; then \
 # ---------------------------------------------------------------------------
 FROM python:3.10-slim
 
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends ca-certificates && \
+    rm -rf /var/lib/apt/lists/*
+
 ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1 \
     PATH="/usr/local/slurm/bin:${PATH}"

k8shelper/go.mod

@@ -1,6 +1,6 @@
 module github.com/last9/gpu-telemetry/k8shelper
 
-go 1.24.3
+go 1.27.3
 
 require (
 	k8s.io/api v0.32.0

k8sprocessor/go.mod

@@ -1,6 +1,6 @@
 module github.com/last9/gpu-telemetry/k8sprocessor
 
-go 1.25.0
+go 1.27.3
 
 require (
 	github.com/last9/gpu-telemetry/k8shelper v0.2.0

pyproject.toml

@@ -18,7 +18,7 @@ dependencies = [
   "pydantic==2.13.4",
   "omegaconf~=2.2",
   "tomli~=2.0",
-  "certifi>=2024.02.02",
+  "certifi>=2026.10.19",
   "opentelemetry-api",
   "opentelemetry-sdk",
   "opentelemetry-exporter-otlp",

scripts/check_tls_trust.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+"""Check eMudhra rejection and trusted HTTPS using real TLS handshakes."""
+
+import argparse
+import socket
+import ssl
+
+import certifi
+
+
+def check_context(label: str, context: ssl.SSLContext) -> bool:
+    success = True
+    for host, should_trust in (
+        ("example.com", True),
+        ("dv-g3b-emudhra.we-have-a-problem.com", False),
+        ("dv-g3b-aaa-emudhra.we-have-a-problem.com", False),
+    ):
+        try:
+            with socket.create_connection((host, 443), timeout=15) as connection:
+                with context.wrap_socket(connection, server_hostname=host):
+                    pass
+        except ssl.SSLCertVerificationError as error:
+            if should_trust:
+                print(f"FAIL {label}: {host}: {error}")
+                success = False
+            else:
+                print(f"PASS {label}: rejected {host}: {error.verify_message}")
+        except OSError as error:
+            # DNS failures, timeouts, and connection errors are inconclusive.
+            print(f"FAIL {label}: {host}: {error}")
+            success = False
+        else:
+            if should_trust:
+                print(f"PASS {label}: trusted {host}")
+            else:
+                print(f"FAIL {label}: trusted compromised chain for {host}")
+                success = False
+    return success
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--system", action="store_true", help="also check the system CA bundle"
+    )
+    args = parser.parse_args()
+    success = check_context(
+        f"certifi {certifi.__version__}",
+        ssl.create_default_context(cafile=certifi.where()),
+    )
+    if args.system:
+        success = check_context("system", ssl.create_default_context()) and success
+    return 0 if success else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

shelper/go.mod

@@ -1,6 +1,6 @@
 module github.com/last9/gpu-telemetry/shelper
 
-go 1.24.3
+go 1.27.3
 
 require github.com/stretchr/testify v1.10.0
 

slurmprocessor/go.mod

@@ -1,6 +1,6 @@
 module github.com/last9/gpu-telemetry/slurmprocessor
 
-go 1.25.0
+go 1.27.3
 
 require (
 	github.com/last9/gpu-telemetry/shelper v0.2.0