From 4ff7363fb170d0d67273850b58c1f9b2f51547a2 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:34:39 +1000 Subject: [PATCH 1/2] Use Podman API for container stats Amp-Thread-ID: https://ampcode.com/threads/T-019fc533-c395-765b-8ba4-b88857972e27 Co-authored-by: Amp --- agent/README.md | 9 +- agent/internal/container/runtime.go | 7 + agent/internal/container/stats.go | 262 ++++++++++-------------- agent/internal/container/stats_test.go | 265 +++++++++++++++---------- docs/agents/setup.mdx | 8 +- web/public/setup.sh | 15 +- 6 files changed, 297 insertions(+), 269 deletions(-) diff --git a/agent/README.md b/agent/README.md index a193d024..a4a0cb15 100644 --- a/agent/README.md +++ b/agent/README.md @@ -55,6 +55,7 @@ The agent downloads the release binary, verifies its checksum, installs it, and ```bash sudo apt update && sudo apt upgrade -y sudo apt install wireguard wireguard-tools podman -y +sudo systemctl enable --now podman.socket curl -sSL https://railpack.com/install.sh | sh sudo ln -s ~/.railpack/bin/railpack /usr/local/bin/railpack @@ -67,6 +68,7 @@ curl -sSL https://github.com/moby/buildkit/releases/download/v0.26.3/buildkit-v0 ```bash sudo apt update && sudo apt upgrade -y sudo apt install wireguard wireguard-tools podman -y +sudo systemctl enable --now podman.socket curl -sSL https://railpack.com/install.sh | sh sudo ln -s ~/.railpack/bin/railpack /usr/local/bin/railpack @@ -191,7 +193,8 @@ Worker node: ```ini [Unit] Description=Techulus Cloud Agent -After=network.target buildkitd.service +After=network.target podman.socket buildkitd.service +Wants=podman.socket [Service] Type=simple @@ -208,7 +211,8 @@ Proxy node: ```ini [Unit] Description=Techulus Cloud Agent -After=network.target traefik.service buildkitd.service +After=network.target podman.socket traefik.service buildkitd.service +Wants=podman.socket [Service] Type=simple @@ -222,6 +226,7 @@ WantedBy=multi-user.target ``` `KillMode=process` ensures only the agent process is killed on restart, not container processes. +The rootful Podman API socket at `/run/podman/podman.sock` is required for container metrics collection. ```bash sudo systemctl daemon-reload diff --git a/agent/internal/container/runtime.go b/agent/internal/container/runtime.go index 3671b0d7..a63be5b3 100644 --- a/agent/internal/container/runtime.go +++ b/agent/internal/container/runtime.go @@ -366,6 +366,13 @@ func CheckPrerequisites() error { if _, err := exec.LookPath("podman"); err != nil { return fmt.Errorf("podman not found: %w", err) } + socket, err := os.Stat(podmanSocketPath) + if err != nil { + return fmt.Errorf("podman API socket unavailable at %s; enable podman.socket: %w", podmanSocketPath, err) + } + if socket.Mode()&os.ModeSocket == 0 { + return fmt.Errorf("podman API endpoint at %s is not a Unix socket", podmanSocketPath) + } return nil } diff --git a/agent/internal/container/stats.go b/agent/internal/container/stats.go index 46133268..b5e1f795 100644 --- a/agent/internal/container/stats.go +++ b/agent/internal/container/stats.go @@ -1,18 +1,17 @@ package container import ( - "bufio" "bytes" "context" + "encoding/json" "fmt" - "log" + "io" "math" - "os/exec" - "strconv" + "net" + "net/http" "strings" "sync" "time" - "unicode" ) type ResourceStats struct { @@ -30,16 +29,40 @@ type ResourceStats struct { } type podmanStatsSample struct { - containerID string - cpuNano uint64 - systemNano uint64 - cpuCountersValid bool - memoryUsage string - memoryUsagePercent string - networkIO string + ContainerID string `json:"ContainerID"` + CPUNano uint64 `json:"CPUNano"` + SystemNano uint64 `json:"SystemNano"` + MemUsage uint64 `json:"MemUsage"` + MemPerc float64 `json:"MemPerc"` + NetInput uint64 `json:"NetInput"` + NetOutput uint64 `json:"NetOutput"` + Network *map[string]podmanNetworkStats `json:"Network"` } -const podmanStatsFormat = "{{.ContainerID}}\t{{.CPUNano}}\t{{.SystemNano}}\t{{.MemUsage}}\t{{.MemPerc}}\t{{.NetIO}}" +type podmanNetworkStats struct { + RxBytes uint64 `json:"RxBytes"` + TxBytes uint64 `json:"TxBytes"` +} + +type podmanStatsReport struct { + Error json.RawMessage `json:"Error"` + Stats []podmanStatsSample `json:"Stats"` +} + +const ( + podmanSocketPath = "/run/podman/podman.sock" + podmanStatsEndpoint = "http://podman/v4.0.0/libpod/containers/stats" +) + +var ( + podmanStatsClient = &http.Client{Transport: &http.Transport{ + DisableCompression: true, + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", podmanSocketPath) + }, + }} + podmanStatsURL = podmanStatsEndpoint +) var previousResourceSamples = struct { sync.Mutex @@ -60,18 +83,13 @@ func CollectResourceStats() ([]ResourceStats, error) { } running := make([]Container, 0, len(containers)) - args := []string{ - "stats", - "--no-stream", - "--no-trunc", - "--format", podmanStatsFormat, - } + containerIDs := make([]string, 0, len(containers)) for _, c := range containers { if c.State != "running" || c.ServiceID == "" || c.DeploymentID == "" { continue } running = append(running, c) - args = append(args, c.ID) + containerIDs = append(containerIDs, c.ID) } if len(running) == 0 { return nil, nil @@ -79,14 +97,7 @@ func CollectResourceStats() ([]ResourceStats, error) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - cmd := exec.CommandContext(ctx, "podman", args...) - var stderr bytes.Buffer - cmd.Stderr = &stderr - output, err := cmd.Output() - if err != nil { - return nil, fmt.Errorf("failed to collect container stats: %s: %w", stderr.String(), err) - } - samples, err := parsePodmanStatsSamples(output) + samples, err := fetchPodmanStats(ctx, podmanStatsClient, podmanStatsURL, containerIDs) if err != nil { return nil, err } @@ -101,7 +112,7 @@ func CollectResourceStats() ([]ResourceStats, error) { } stats := make([]ResourceStats, 0, len(samples)) for _, sample := range samples { - container := findStatsContainerByID(sample.containerID, running) + container := findStatsContainerByID(sample.ContainerID, running) if container == nil { continue } @@ -113,6 +124,46 @@ func CollectResourceStats() ([]ResourceStats, error) { return stats, nil } +func fetchPodmanStats(ctx context.Context, client *http.Client, endpoint string, containerIDs []string) ([]podmanStatsSample, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, fmt.Errorf("failed to create container stats request: %w", err) + } + query := req.URL.Query() + query.Set("stream", "false") + for _, containerID := range containerIDs { + query.Add("containers", containerID) + } + req.URL.RawQuery = query.Encode() + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to collect container stats: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + message, _ := io.ReadAll(io.LimitReader(resp.Body, 4*1024)) + return nil, fmt.Errorf("failed to collect container stats: podman returned %s: %s", resp.Status, strings.TrimSpace(string(message))) + } + + var report podmanStatsReport + decoder := json.NewDecoder(resp.Body) + if err := decoder.Decode(&report); err != nil { + return nil, fmt.Errorf("failed to decode container stats: %w", err) + } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + if err == nil { + return nil, fmt.Errorf("failed to decode container stats: unexpected additional response") + } + return nil, fmt.Errorf("failed to decode container stats: %w", err) + } + if value := bytes.TrimSpace(report.Error); len(value) > 0 && !bytes.Equal(value, []byte("null")) { + return nil, fmt.Errorf("failed to collect container stats: podman report error: %.1024s", value) + } + return report.Stats, nil +} + func findStatsContainerByID(value string, containers []Container) *Container { value = strings.TrimSpace(value) if value == "" { @@ -131,159 +182,46 @@ func findStatsContainerByID(value string, containers []Container) *Container { return nil } -func parsePodmanStatsSamples(output []byte) ([]podmanStatsSample, error) { - samples := make([]podmanStatsSample, 0) - skipped := 0 - scanner := bufio.NewScanner(bytes.NewReader(output)) - for scanner.Scan() { - if strings.TrimSpace(scanner.Text()) == "" { - continue - } - sample, err := parsePodmanStatsSample(scanner.Text()) - if err != nil { - skipped++ - continue - } - samples = append(samples, sample) - } - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("failed to read container stats: %w", err) - } - if skipped > 0 { - log.Printf("[metrics] skipped %d malformed container stats rows", skipped) - } - return samples, nil -} - -func parsePodmanStatsSample(line string) (podmanStatsSample, error) { - parts := strings.Split(line, "\t") - if len(parts) != 6 { - return podmanStatsSample{}, fmt.Errorf("failed to parse podman stats row: expected 6 fields, got %d", len(parts)) - } - cpuNano, cpuErr := strconv.ParseUint(strings.TrimSpace(parts[1]), 10, 64) - systemNano, systemErr := strconv.ParseUint(strings.TrimSpace(parts[2]), 10, 64) - return podmanStatsSample{ - containerID: strings.TrimSpace(parts[0]), - cpuNano: cpuNano, - systemNano: systemNano, - cpuCountersValid: cpuErr == nil && systemErr == nil, - memoryUsage: parts[3], - memoryUsagePercent: parts[4], - networkIO: parts[5], - }, nil -} - func resourceStatsFromSamples(container Container, previous, current podmanStatsSample) ResourceStats { cpuUsagePercent := 0.0 // Podman SystemNano is a wall-clock timestamp, so CPU nanoseconds divided // by its delta yields used cores; the metrics sender converts percent to cores. cpuUsageValid := - previous.cpuCountersValid && - current.cpuCountersValid && - current.cpuNano >= previous.cpuNano && - current.systemNano > previous.systemNano + previous.SystemNano > 0 && + current.CPUNano >= previous.CPUNano && + current.SystemNano > previous.SystemNano if cpuUsageValid { - cpuUsagePercent = 100 * float64(current.cpuNano-previous.cpuNano) / - float64(current.systemNano-previous.systemNano) + cpuUsagePercent = 100 * float64(current.CPUNano-previous.CPUNano) / + float64(current.SystemNano-previous.SystemNano) cpuUsageValid = isFinite(cpuUsagePercent) } - memoryUsagePercent, memoryUsageValid := parsePercent(current.memoryUsagePercent) - memoryUsedBytes, memoryUsedValid := parseMemUsed(current.memoryUsage) - rx, tx := parseNetIO(current.networkIO) + networkReceiveBytes, networkTransmitBytes := current.networkTotals() return ResourceStats{ ContainerID: container.ID, ServiceID: container.ServiceID, DeploymentID: container.DeploymentID, CPUUsagePercent: cpuUsagePercent, CPUUsageValid: cpuUsageValid, - MemoryUsagePercent: memoryUsagePercent, - MemoryUsageValid: memoryUsageValid, - MemoryUsedBytes: memoryUsedBytes, - MemoryUsedValid: memoryUsedValid, - NetworkReceiveBytes: rx, - NetworkTransmitBytes: tx, - } -} - -func parsePercent(value string) (float64, bool) { - value = strings.TrimSpace(strings.TrimSuffix(value, "%")) - if value == "" || value == "--" { - return 0, false - } - parsed, err := strconv.ParseFloat(value, 64) - if err != nil || !isFinite(parsed) { - return 0, false - } - return parsed, true -} - -func parseMemUsed(value string) (float64, bool) { - parts := strings.Split(value, "/") - if len(parts) == 0 { - return 0, false + MemoryUsagePercent: current.MemPerc, + MemoryUsageValid: isFinite(current.MemPerc), + MemoryUsedBytes: float64(current.MemUsage), + MemoryUsedValid: true, + NetworkReceiveBytes: float64(networkReceiveBytes), + NetworkTransmitBytes: float64(networkTransmitBytes), } - return parseByteQuantityValue(parts[0]) } -func parseNetIO(value string) (float64, float64) { - parts := strings.Split(value, "/") - if len(parts) != 2 { - return 0, 0 +func (sample podmanStatsSample) networkTotals() (uint64, uint64) { + if sample.Network == nil { + return sample.NetInput, sample.NetOutput } - rx, _ := parseByteQuantityValue(parts[0]) - tx, _ := parseByteQuantityValue(parts[1]) - return rx, tx -} - -func parseByteQuantity(value string) float64 { - parsed, _ := parseByteQuantityValue(value) - return parsed -} - -func parseByteQuantityValue(value string) (float64, bool) { - value = strings.TrimSpace(value) - if value == "" || value == "--" { - return 0, false - } - - compact := strings.ReplaceAll(value, " ", "") - splitAt := len(compact) - for i, r := range compact { - if !(unicode.IsDigit(r) || r == '.' || r == '-') { - splitAt = i - break - } - } - - numberText := compact[:splitAt] - unit := strings.ToLower(compact[splitAt:]) - parsed, err := strconv.ParseFloat(numberText, 64) - if err != nil || !isFinite(parsed) { - return 0, false - } - - switch unit { - case "", "b": - return parsed, true - case "kb", "k", "kib", "ki": - return parsed * unitMultiplier(unit, 1), true - case "mb", "m", "mib", "mi": - return parsed * unitMultiplier(unit, 2), true - case "gb", "g", "gib", "gi": - return parsed * unitMultiplier(unit, 3), true - case "tb", "t", "tib", "ti": - return parsed * unitMultiplier(unit, 4), true - default: - return 0, false - } -} -func unitMultiplier(unit string, power float64) float64 { - base := 1000.0 - if strings.Contains(unit, "i") { - base = 1024.0 + var receive, transmit uint64 + for _, network := range *sample.Network { + receive += network.RxBytes + transmit += network.TxBytes } - return math.Pow(base, power) + return receive, transmit } func isFinite(value float64) bool { diff --git a/agent/internal/container/stats_test.go b/agent/internal/container/stats_test.go index d9a6fe00..bd7b65d6 100644 --- a/agent/internal/container/stats_test.go +++ b/agent/internal/container/stats_test.go @@ -1,45 +1,26 @@ package container import ( + "encoding/json" "math" + "net/http" + "net/http/httptest" "os" "path/filepath" - "strconv" "strings" "testing" ) -func TestParsePodmanStatsSample(t *testing.T) { - sample, err := parsePodmanStatsSample("abcdef123456\t1000000000\t2000000000\t64MiB / 512MiB\t12.50%\t1.5MB / 2.5MB") - if err != nil { - t.Fatalf("parse sample: %v", err) - } - if sample.containerID != "abcdef123456" || sample.cpuNano != 1_000_000_000 || sample.systemNano != 2_000_000_000 || !sample.cpuCountersValid { - t.Fatalf("unexpected sample: %#v", sample) - } -} - -func TestParsePodmanStatsSamplesSkipsMalformedRows(t *testing.T) { - containerID := strings.Repeat("e", 64) - samples, err := parsePodmanStatsSamples([]byte("malformed\n" + statsLine(containerID, 100, 1000))) - if err != nil { - t.Fatalf("parse samples: %v", err) - } - if len(samples) != 1 || samples[0].containerID != containerID { - t.Fatalf("expected valid row to survive malformed peer, got %+v", samples) - } -} - func TestResourceStatsFromSamplesUsesRecentCPUInterval(t *testing.T) { container := Container{ID: "container-1", ServiceID: "service-1", DeploymentID: "deployment-1"} - previous := podmanStatsSample{cpuNano: 1_000_000_000, systemNano: 10_000_000_000, cpuCountersValid: true} + previous := podmanStatsSample{CPUNano: 1_000_000_000, SystemNano: 10_000_000_000} current := podmanStatsSample{ - cpuNano: 1_500_000_000, - systemNano: 11_000_000_000, - cpuCountersValid: true, - memoryUsage: "64MiB / 512MiB", - memoryUsagePercent: "12.50%", - networkIO: "1.5MB / 2.5MB", + CPUNano: 1_500_000_000, + SystemNano: 11_000_000_000, + MemUsage: 64 * 1024 * 1024, + MemPerc: 12.5, + NetInput: 1_500_000, + NetOutput: 2_500_000, } stats := resourceStatsFromSamples(container, previous, current) @@ -52,70 +33,124 @@ func TestResourceStatsFromSamplesUsesRecentCPUInterval(t *testing.T) { if !stats.MemoryUsageValid || stats.MemoryUsagePercent != 12.5 { t.Fatalf("memory percent = %f, valid=%v", stats.MemoryUsagePercent, stats.MemoryUsageValid) } - if stats.NetworkReceiveBytes != 1.5*1000*1000 || stats.NetworkTransmitBytes != 2.5*1000*1000 { + if stats.NetworkReceiveBytes != 1_500_000 || stats.NetworkTransmitBytes != 2_500_000 { t.Fatalf("network stats = %f/%f", stats.NetworkReceiveBytes, stats.NetworkTransmitBytes) } } -func TestResourceStatsFromSamplesKeepsInvalidValuesMissing(t *testing.T) { +func TestResourceStatsFromSamplesKeepsGenuineZeroValid(t *testing.T) { container := Container{ID: "container-1", ServiceID: "service-1", DeploymentID: "deployment-1"} stats := resourceStatsFromSamples(container, - podmanStatsSample{cpuNano: 2, systemNano: 2, cpuCountersValid: true}, + podmanStatsSample{CPUNano: 1, SystemNano: 1}, podmanStatsSample{ - cpuNano: 1, - systemNano: 3, - cpuCountersValid: true, - memoryUsage: "-- / 512MiB", - memoryUsagePercent: "NaN%", - networkIO: "-- / --", + CPUNano: 1, + SystemNano: 2, + MemUsage: 0, + MemPerc: 0, }, ) - if stats.CPUUsageValid || stats.MemoryUsageValid || stats.MemoryUsedValid { - t.Fatalf("invalid observations marked valid: %#v", stats) + if !stats.CPUUsageValid || !stats.MemoryUsageValid || !stats.MemoryUsedValid { + t.Fatalf("zero observations marked invalid: %#v", stats) } } -func TestResourceStatsFromSamplesKeepsGenuineZeroValid(t *testing.T) { - container := Container{ID: "container-1", ServiceID: "service-1", DeploymentID: "deployment-1"} - stats := resourceStatsFromSamples(container, - podmanStatsSample{cpuNano: 1, systemNano: 1, cpuCountersValid: true}, - podmanStatsSample{ - cpuNano: 1, - systemNano: 2, - cpuCountersValid: true, - memoryUsage: "0B / 512MiB", - memoryUsagePercent: "0%", +func TestFetchPodmanStatsUsesVersionedEndpointAndContainerIDs(t *testing.T) { + containerIDs := []string{strings.Repeat("a", 64), strings.Repeat("b", 64)} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v4.0.0/libpod/containers/stats" { + t.Errorf("path = %q", r.URL.Path) + } + if r.URL.Query().Get("stream") != "false" { + t.Errorf("stream = %q", r.URL.Query().Get("stream")) + } + if got := r.URL.Query()["containers"]; len(got) != 2 || got[0] != containerIDs[0] || got[1] != containerIDs[1] { + t.Errorf("containers = %#v", got) + } + writeStatsReport(t, w, []podmanStatsSample{{ContainerID: containerIDs[0], CPUNano: 100, SystemNano: 1_000}}) + })) + defer server.Close() + + samples, err := fetchPodmanStats(t.Context(), server.Client(), server.URL+"/v4.0.0/libpod/containers/stats", containerIDs) + if err != nil { + t.Fatalf("fetch stats: %v", err) + } + if len(samples) != 1 || samples[0].ContainerID != containerIDs[0] { + t.Fatalf("samples = %#v", samples) + } +} + +func TestFetchPodmanStatsDecodesPodmanNetworkShapes(t *testing.T) { + tests := []struct { + name string + body string + wantReceive uint64 + wantTransmit uint64 + }{ + { + name: "Podman 4 aggregate fields", + body: `{"Error":null,"Stats":[{"ContainerID":"container","NetInput":100,"NetOutput":200}]}`, + wantReceive: 100, + wantTransmit: 200, }, - ) - if !stats.CPUUsageValid || !stats.MemoryUsageValid || !stats.MemoryUsedValid { - t.Fatalf("zero observations marked invalid: %#v", stats) + { + name: "Podman 5 per-interface fields", + body: `{"Error":null,"Stats":[{"ContainerID":"container","Network":{"eth0":{"RxBytes":100,"TxBytes":200},"eth1":{"RxBytes":30,"TxBytes":40}}}]}`, + wantReceive: 130, + wantTransmit: 240, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(tt.body)) + })) + defer server.Close() + + samples, err := fetchPodmanStats(t.Context(), server.Client(), server.URL, []string{"container"}) + if err != nil { + t.Fatalf("fetch stats: %v", err) + } + receive, transmit := samples[0].networkTotals() + if receive != tt.wantReceive || transmit != tt.wantTransmit { + t.Fatalf("network totals = %d/%d, want %d/%d", receive, transmit, tt.wantReceive, tt.wantTransmit) + } + }) } } -func TestParseByteQuantity(t *testing.T) { - tests := map[string]float64{ - "42B": 42, - "1 kB": 1000, - "1KiB": 1024, - "1.5GB": 1.5 * 1000 * 1000 * 1000, - "2 MiB": 2 * 1024 * 1024, - "--": 0, - "broken": 0, +func TestFetchPodmanStatsRejectsInvalidResponses(t *testing.T) { + tests := []struct { + name string + status int + body string + }{ + {name: "HTTP error", status: http.StatusInternalServerError, body: `{"cause":"failed"}`}, + {name: "malformed JSON", status: http.StatusOK, body: `{`}, + {name: "in-band error", status: http.StatusOK, body: `{"Error":{},"Stats":null}`}, } - for input, expected := range tests { - if actual := parseByteQuantity(input); actual != expected { - t.Fatalf("%q = %f, want %f", input, actual, expected) - } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(tt.status) + _, _ = w.Write([]byte(tt.body)) + })) + defer server.Close() + + if _, err := fetchPodmanStats(t.Context(), server.Client(), server.URL, []string{"container"}); err == nil { + t.Fatal("expected error") + } + }) } } func TestCollectResourceStatsUsesCounterDeltas(t *testing.T) { - statsOutput := installFakeStatsPodman(t, []string{strings.Repeat("a", 64)}) - resetPreviousResourceSamples(t) containerID := strings.Repeat("a", 64) + api := installFakeStatsEnvironment(t, []string{containerID}) + resetPreviousResourceSamples(t) - writeStatsOutput(t, statsOutput, statsLine(containerID, 1_000_000_000, 10_000_000_000)) + api.setSamples(t, statsSample(containerID, 1_000_000_000, 10_000_000_000)) first, err := CollectResourceStats() if err != nil { t.Fatalf("first collection failed: %v", err) @@ -130,7 +165,7 @@ func TestCollectResourceStatsUsesCounterDeltas(t *testing.T) { t.Fatal("expected memory values to remain valid on first sample") } - writeStatsOutput(t, statsOutput, statsLine(containerID, 2_000_000_000, 12_000_000_000)) + api.setSamples(t, statsSample(containerID, 2_000_000_000, 12_000_000_000)) second, err := CollectResourceStats() if err != nil { t.Fatalf("second collection failed: %v", err) @@ -141,8 +176,8 @@ func TestCollectResourceStatsUsesCounterDeltas(t *testing.T) { } func TestCollectResourceStatsRejectsInvalidCounterDeltas(t *testing.T) { - statsOutput := installFakeStatsPodman(t, []string{strings.Repeat("b", 64)}) containerID := strings.Repeat("b", 64) + api := installFakeStatsEnvironment(t, []string{containerID}) tests := []struct { name string firstCPU uint64 @@ -158,11 +193,11 @@ func TestCollectResourceStatsRejectsInvalidCounterDeltas(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { resetPreviousResourceSamples(t) - writeStatsOutput(t, statsOutput, statsLine(containerID, tt.firstCPU, tt.firstTime)) + api.setSamples(t, statsSample(containerID, tt.firstCPU, tt.firstTime)) if _, err := CollectResourceStats(); err != nil { t.Fatalf("first collection failed: %v", err) } - writeStatsOutput(t, statsOutput, statsLine(containerID, tt.secondCPU, tt.secondTime)) + api.setSamples(t, statsSample(containerID, tt.secondCPU, tt.secondTime)) stats, err := CollectResourceStats() if err != nil { t.Fatalf("second collection failed: %v", err) @@ -177,26 +212,21 @@ func TestCollectResourceStatsRejectsInvalidCounterDeltas(t *testing.T) { func TestCollectResourceStatsPreservesBaselinesAndPrunesStoppedContainers(t *testing.T) { firstID := strings.Repeat("c", 64) missingID := strings.Repeat("d", 64) - statsOutput := installFakeStatsPodman(t, []string{firstID, missingID}) + api := installFakeStatsEnvironment(t, []string{firstID, missingID}) resetPreviousResourceSamples(t) - writeStatsOutput(t, statsOutput, statsLine(firstID, 100, 1000)+statsLine(missingID, 100, 1000)) + api.setSamples(t, statsSample(firstID, 100, 1000), statsSample(missingID, 100, 1000)) if _, err := CollectResourceStats(); err != nil { t.Fatalf("first collection failed: %v", err) } - failPath := statsOutput + ".fail" - if err := os.WriteFile(failPath, nil, 0o600); err != nil { - t.Fatalf("create failure marker: %v", err) - } + api.status = http.StatusInternalServerError if _, err := CollectResourceStats(); err == nil { t.Fatal("expected Podman failure") } - if err := os.Remove(failPath); err != nil { - t.Fatalf("remove failure marker: %v", err) - } - writeStatsOutput(t, statsOutput, statsLine(firstID, 300, 2000)) + api.status = http.StatusOK + api.setSamples(t, statsSample(firstID, 300, 2000)) stats, err := CollectResourceStats() if err != nil { t.Fatalf("collection after failure failed: %v", err) @@ -213,8 +243,8 @@ func TestCollectResourceStatsPreservesBaselinesAndPrunesStoppedContainers(t *tes t.Fatalf("expected baselines for running containers to survive partial output: retained=%v missingRetained=%v", retained, missingRetained) } - writeContainersOutput(t, filepath.Join(filepath.Dir(statsOutput), "containers-output"), []string{firstID}) - writeStatsOutput(t, statsOutput, statsLine(firstID, 400, 3000)) + writeContainersOutput(t, api.containersOutput, []string{firstID}) + api.setSamples(t, statsSample(firstID, 400, 3000)) if _, err := CollectResourceStats(); err != nil { t.Fatalf("collection after container removal failed: %v", err) } @@ -226,28 +256,62 @@ func TestCollectResourceStatsPreservesBaselinesAndPrunesStoppedContainers(t *tes } } -func installFakeStatsPodman(t *testing.T, containerIDs []string) string { +type fakeStatsAPI struct { + server *httptest.Server + status int + body []byte + containersOutput string +} + +func installFakeStatsEnvironment(t *testing.T, containerIDs []string) *fakeStatsAPI { t.Helper() dir := t.TempDir() - statsOutput := filepath.Join(dir, "stats-output") containersOutput := filepath.Join(dir, "containers-output") script := `#!/bin/sh if [ "$1" = "ps" ]; then cat "$PODMAN_CONTAINERS_OUTPUT" -elif [ -f "$PODMAN_STATS_OUTPUT.fail" ]; then - exit 1 else - cat "$PODMAN_STATS_OUTPUT" + exit 1 fi ` if err := os.WriteFile(filepath.Join(dir, "podman"), []byte(script), 0o700); err != nil { t.Fatalf("write fake podman: %v", err) } writeContainersOutput(t, containersOutput, containerIDs) - t.Setenv("PODMAN_STATS_OUTPUT", statsOutput) t.Setenv("PODMAN_CONTAINERS_OUTPUT", containersOutput) t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) - return statsOutput + + api := &fakeStatsAPI{status: http.StatusOK, containersOutput: containersOutput} + api.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(api.status) + _, _ = w.Write(api.body) + })) + previousClient := podmanStatsClient + previousURL := podmanStatsURL + podmanStatsClient = api.server.Client() + podmanStatsURL = api.server.URL + t.Cleanup(func() { + podmanStatsClient = previousClient + podmanStatsURL = previousURL + api.server.Close() + }) + return api +} + +func (api *fakeStatsAPI) setSamples(t *testing.T, samples ...podmanStatsSample) { + t.Helper() + data, err := json.Marshal(podmanStatsReport{Stats: samples}) + if err != nil { + t.Fatalf("marshal stats report: %v", err) + } + api.body = data +} + +func writeStatsReport(t *testing.T, w http.ResponseWriter, samples []podmanStatsSample) { + t.Helper() + if err := json.NewEncoder(w).Encode(podmanStatsReport{Stats: samples}); err != nil { + t.Fatalf("encode stats report: %v", err) + } } func writeContainersOutput(t *testing.T, path string, containerIDs []string) { @@ -277,13 +341,14 @@ func resetPreviousResourceSamples(t *testing.T) { }) } -func writeStatsOutput(t *testing.T, path, output string) { - t.Helper() - if err := os.WriteFile(path, []byte(output), 0o600); err != nil { - t.Fatalf("write stats output: %v", err) +func statsSample(containerID string, cpuNano, systemNano uint64) podmanStatsSample { + return podmanStatsSample{ + ContainerID: containerID, + CPUNano: cpuNano, + SystemNano: systemNano, + MemUsage: 100_000_000, + MemPerc: 10, + NetInput: 1_000_000, + NetOutput: 2_000_000, } } - -func statsLine(containerID string, cpuNano, systemNano uint64) string { - return containerID + "\t" + strconv.FormatUint(cpuNano, 10) + "\t" + strconv.FormatUint(systemNano, 10) + "\t100 MB / 1 GB\t10%\t1 MB / 2 MB\n" -} diff --git a/docs/agents/setup.mdx b/docs/agents/setup.mdx index fc77ac0c..fddd0ed0 100644 --- a/docs/agents/setup.mdx +++ b/docs/agents/setup.mdx @@ -60,6 +60,7 @@ After registration, the token is invalidated. Subsequent runs do not require a t ```bash sudo apt update && sudo apt upgrade -y sudo apt install wireguard wireguard-tools podman git -y +sudo systemctl enable --now podman.socket # Install Railpack curl -sSL https://railpack.com/install.sh | sh @@ -105,7 +106,8 @@ Create `/etc/systemd/system/techulus-agent.service`: ```ini [Unit] Description=Techulus Cloud Agent -After=network.target buildkitd.service +After=network.target podman.socket buildkitd.service +Wants=podman.socket [Service] Type=simple @@ -123,7 +125,8 @@ WantedBy=multi-user.target ```ini [Unit] Description=Techulus Cloud Agent -After=network.target traefik.service buildkitd.service +After=network.target podman.socket traefik.service buildkitd.service +Wants=podman.socket [Service] Type=simple @@ -146,6 +149,7 @@ sudo systemctl start techulus-agent `KillMode=process` ensures only the agent process is stopped on restart, not the containers it manages. + The rootful Podman API socket at `/run/podman/podman.sock` is required for container metrics collection. ## Troubleshooting diff --git a/web/public/setup.sh b/web/public/setup.sh index 1509fe3a..a01b6036 100644 --- a/web/public/setup.sh +++ b/web/public/setup.sh @@ -251,6 +251,15 @@ if ! podman --version &>/dev/null; then fi echo "✓ Podman verified" +step "Enabling Podman API socket..." +if ! systemctl enable --now podman.socket; then + error "Failed to enable the rootful Podman API socket" +fi +if ! systemctl is-active --quiet podman.socket || [ ! -S /run/podman/podman.sock ]; then + error "Podman API socket is unavailable at /run/podman/podman.sock" +fi +echo "✓ Podman API socket running" + if [ "$IS_PROXY" = "true" ]; then step "Installing Traefik (proxy mode)..." TRAEFIK_VERSION="v3.6.6" @@ -684,16 +693,16 @@ if [ "$IS_PROXY" = "true" ]; then fi if [ "$IS_PROXY" = "true" ]; then - AFTER_SERVICES="network-online.target crowdsec.service traefik.service buildkitd.service" + AFTER_SERVICES="network-online.target podman.socket crowdsec.service traefik.service buildkitd.service" else - AFTER_SERVICES="network-online.target buildkitd.service" + AFTER_SERVICES="network-online.target podman.socket buildkitd.service" fi cat > /etc/systemd/system/techulus-agent.service << EOF [Unit] Description=Techulus Cloud Agent After=${AFTER_SERVICES} -Wants=network-online.target +Wants=network-online.target podman.socket [Service] Type=simple From fa0eab2948bbe1d4a664e0d2b1c9579592532f08 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:51:33 +1000 Subject: [PATCH 2/2] Self-heal the Podman API socket Amp-Thread-ID: https://ampcode.com/threads/T-019fc533-c395-765b-8ba4-b88857972e27 Co-authored-by: Amp --- agent/internal/container/runtime.go | 33 ++++++++++- agent/internal/container/runtime_test.go | 72 ++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 3 deletions(-) diff --git a/agent/internal/container/runtime.go b/agent/internal/container/runtime.go index a63be5b3..ee4d08fb 100644 --- a/agent/internal/container/runtime.go +++ b/agent/internal/container/runtime.go @@ -366,12 +366,39 @@ func CheckPrerequisites() error { if _, err := exec.LookPath("podman"); err != nil { return fmt.Errorf("podman not found: %w", err) } - socket, err := os.Stat(podmanSocketPath) + return ensurePodmanSocket(podmanSocketPath, func() ([]byte, error) { + return exec.Command("systemctl", "enable", "--now", "podman.socket").CombinedOutput() + }) +} + +func ensurePodmanSocket(socketPath string, enable func() ([]byte, error)) error { + if err := validatePodmanSocket(socketPath); err == nil { + return nil + } + + output, err := enable() + if err != nil { + if len(output) > 4*1024 { + output = output[:4*1024] + } + if detail := strings.TrimSpace(string(output)); detail != "" { + return fmt.Errorf("failed to enable podman.socket: %s: %w", detail, err) + } + return fmt.Errorf("failed to enable podman.socket: %w", err) + } + if err := validatePodmanSocket(socketPath); err != nil { + return fmt.Errorf("podman API socket unavailable after enabling podman.socket: %w", err) + } + return nil +} + +func validatePodmanSocket(socketPath string) error { + socket, err := os.Stat(socketPath) if err != nil { - return fmt.Errorf("podman API socket unavailable at %s; enable podman.socket: %w", podmanSocketPath, err) + return fmt.Errorf("podman API socket unavailable at %s: %w", socketPath, err) } if socket.Mode()&os.ModeSocket == 0 { - return fmt.Errorf("podman API endpoint at %s is not a Unix socket", podmanSocketPath) + return fmt.Errorf("podman API endpoint at %s is not a Unix socket", socketPath) } return nil } diff --git a/agent/internal/container/runtime_test.go b/agent/internal/container/runtime_test.go index f9b09853..f983aba5 100644 --- a/agent/internal/container/runtime_test.go +++ b/agent/internal/container/runtime_test.go @@ -1,7 +1,12 @@ package container import ( + "errors" + "net" + "os" + "path/filepath" "slices" + "strings" "testing" ) @@ -104,3 +109,70 @@ func TestBuildPodmanRunArgsDoesNotPublishStaticIPPortsByDefault(t *testing.T) { t.Fatalf("args unexpectedly publish ports: %+v", args) } } + +func TestEnsurePodmanSocketDoesNotEnableExistingSocket(t *testing.T) { + socketPath := testPodmanSocketPath(t) + listener, err := net.Listen("unix", socketPath) + if err != nil { + t.Fatalf("listen on test socket: %v", err) + } + defer listener.Close() + + called := false + err = ensurePodmanSocket(socketPath, func() ([]byte, error) { + called = true + return nil, nil + }) + if err != nil { + t.Fatalf("ensure socket: %v", err) + } + if called { + t.Fatal("activation called for an existing socket") + } +} + +func TestEnsurePodmanSocketRepairsMissingSocket(t *testing.T) { + socketPath := testPodmanSocketPath(t) + var listener net.Listener + err := ensurePodmanSocket(socketPath, func() ([]byte, error) { + var err error + listener, err = net.Listen("unix", socketPath) + return nil, err + }) + if listener != nil { + defer listener.Close() + } + if err != nil { + t.Fatalf("ensure socket: %v", err) + } +} + +func TestEnsurePodmanSocketReportsActivationFailure(t *testing.T) { + socketPath := testPodmanSocketPath(t) + err := ensurePodmanSocket(socketPath, func() ([]byte, error) { + return []byte("permission denied"), errors.New("exit status 1") + }) + if err == nil || !strings.Contains(err.Error(), "failed to enable podman.socket: permission denied") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestEnsurePodmanSocketReportsInvalidSocketAfterActivation(t *testing.T) { + socketPath := testPodmanSocketPath(t) + err := ensurePodmanSocket(socketPath, func() ([]byte, error) { + return nil, os.WriteFile(socketPath, nil, 0o600) + }) + if err == nil || !strings.Contains(err.Error(), "is not a Unix socket") { + t.Fatalf("unexpected error: %v", err) + } +} + +func testPodmanSocketPath(t *testing.T) string { + t.Helper() + dir, err := os.MkdirTemp("/tmp", "podman-socket-") + if err != nil { + t.Fatalf("create socket test directory: %v", err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + return filepath.Join(dir, "podman.sock") +}