Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
416 changes: 416 additions & 0 deletions docs/uptimerobot-monitoring-design.md

Large diffs are not rendered by default.

142 changes: 117 additions & 25 deletions internal/apidocs/openapi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,25 @@ components:
type: apiKey
in: cookie
name: vtafarm_admin
MonitorBearerAuth:
type: http
scheme: bearer
bearerFormat: opaque
description: Environment-scoped MONITOR_TOKEN for external uptime polling.

schemas:
PublicServiceStatus:
type: object
additionalProperties: false
required: [status, checked_at]
properties:
status:
type: string
enum: [operational, unavailable]
checked_at:
type: string
format: date-time

TrustRegistryState:
type: object
required: [enabled, state, phase, image, did, error]
Expand Down Expand Up @@ -1026,24 +1043,100 @@ paths:
type: string
example: ok

# ── Public customer status ─────────────────────────────────────────────────

/api/v1/status/farm:
get:
summary: Public VTA Farm control-plane availability
description: |
Reports whether the public API and the dependencies required for core
Farm management operations are available. The response deliberately
contains no dependency or infrastructure detail.
tags: [System]
security: []
responses:
"200":
description: VTA Farm is operational
content:
application/json:
schema: { $ref: "#/components/schemas/PublicServiceStatus" }
"503":
description: VTA Farm is unavailable; the body remains generic
content:
application/json:
schema: { $ref: "#/components/schemas/PublicServiceStatus" }
"429":
description: Shared public-status rate limit exceeded
content:
application/json:
schema: { $ref: "#/components/schemas/Error" }

/api/v1/status/vta:
get:
summary: Public VTA connectivity
description: |
Checks the platform stack's VTA through its public HTTPS health
endpoint. The caller cannot select the target.
tags: [System]
security: []
responses:
"200":
description: The VTA data plane is operational
content:
application/json:
schema: { $ref: "#/components/schemas/PublicServiceStatus" }
"503":
description: The platform VTA is unavailable
content:
application/json:
schema: { $ref: "#/components/schemas/PublicServiceStatus" }
"429":
description: Shared public-status rate limit exceeded
content:
application/json:
schema: { $ref: "#/components/schemas/Error" }

/api/v1/status/platform:
get:
summary: Public shared-platform connectivity
description: |
Checks the platform stack's mediator, DID hosting, and VTC through
their public HTTPS health endpoints. VTA connectivity is reported by
`/api/v1/status/vta`; the response does not identify the failing
component.
tags: [System]
security: []
responses:
"200":
description: All required shared-platform components are operational
content:
application/json:
schema: { $ref: "#/components/schemas/PublicServiceStatus" }
"503":
description: A required shared-platform component is unavailable
content:
application/json:
schema: { $ref: "#/components/schemas/PublicServiceStatus" }
"429":
description: Shared public-status rate limit exceeded
content:
application/json:
schema: { $ref: "#/components/schemas/Error" }

# ── Monitor (external uptime poller) ────────────────────────────────────────
# Polled by UptimeRobot: healthy → 200, anything wrong → 503 with the same
# body shape. Gated by the MONITOR_TOKEN shared secret (?token=); the routes
# answer 404 until the token is configured.
# body shape. Gated by the MONITOR_TOKEN shared secret in the Authorization
# Bearer header; the routes answer 404 until the token is configured.

/api/v1/monitor/health:
get:
summary: "(token) Dependency health — DB, K8s API, Vault"
summary: "(monitor bearer) Dependency health — DB, K8s API, Vault"
description: |
Checks the database, the Kubernetes API server, and Vault (including
its sealed state). `vault: disabled` (VAULT_ADDR unset) is not an alarm.
tags: [System]
parameters:
- name: token
in: query
required: true
schema: { type: string }
description: MONITOR_TOKEN shared secret
security:
- MonitorBearerAuth: []
responses:
"200":
description: All dependencies healthy
Expand All @@ -1062,23 +1155,21 @@ paths:
"503":
description: One or more dependencies unhealthy (same body shape, status=alarm)
"401":
description: Missing or wrong token
description: Missing or wrong Bearer token
"404":
description: Monitoring is disabled because MONITOR_TOKEN is not configured

/api/v1/monitor/workloads:
get:
summary: "(token) Unhealthy pods in user + infra namespaces"
summary: "(monitor bearer) Unhealthy pods in user + infra namespaces"
description: |
Scans `{K8S_NAMESPACE_PREFIX}-*` plus MONITOR_EXTRA_NAMESPACES for pods
that are crash-looping, failing image pulls, restarted within the last
MONITOR_RESTART_WINDOW_MIN minutes, or stuck Pending / not-Ready past
MONITOR_PENDING_GRACE_MIN minutes. Job-owned pods are ignored.
tags: [System]
parameters:
- name: token
in: query
required: true
schema: { type: string }
description: MONITOR_TOKEN shared secret
security:
- MonitorBearerAuth: []
responses:
"200":
description: No unhealthy pods
Expand All @@ -1092,23 +1183,22 @@ paths:
"503":
description: Unhealthy pods found — `pods` lists namespace/pod/reason
"401":
description: Missing or wrong token
description: Missing or wrong Bearer token
"404":
description: Monitoring is disabled because MONITOR_TOKEN is not configured

/api/v1/monitor/capacity:
get:
summary: "(token) Node CPU/memory/storage usage vs alarm thresholds"
summary: "(monitor bearer) Node CPU/memory/storage usage vs alarm thresholds"
description: |
Alarms when any node's live CPU or memory usage, or any Longhorn node's
disk usage, crosses its threshold — and also when metrics-server or
Longhorn stats are unreadable ("can't see" is never "all green").
Thresholds default from MONITOR_*_PCT and can be overridden per poll.
tags: [System]
security:
- MonitorBearerAuth: []
parameters:
- name: token
in: query
required: true
schema: { type: string }
description: MONITOR_TOKEN shared secret
- name: cpu
in: query
schema: { type: integer, minimum: 1, maximum: 100 }
Expand Down Expand Up @@ -1140,7 +1230,9 @@ paths:
"503":
description: Threshold breached or stats source unreadable — see `issues`
"401":
description: Missing or wrong token
description: Missing or wrong Bearer token
"404":
description: Monitoring is disabled because MONITOR_TOKEN is not configured

# ── Auth — Admin ────────────────────────────────────────────────────────────

Expand Down
2 changes: 1 addition & 1 deletion internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ type Config struct {
// MonitorConfig configures the token-gated /api/v1/monitor/* endpoints polled
// by an external uptime service (UptimeRobot). Empty Token disables them.
type MonitorConfig struct {
Token string // shared secret, passed as ?token= by the poller
Token string // shared secret, passed as an Authorization Bearer token by the poller
CPUPercent int // per-node CPU used/allocatable alarm threshold
MemPercent int // per-node memory used/allocatable alarm threshold
StoragePercent int // per-node Longhorn disk usage alarm threshold
Expand Down
23 changes: 20 additions & 3 deletions internal/handler/monitor.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"log"
"net/http"
"strconv"
"strings"
"time"

"github.com/gin-gonic/gin"
Expand Down Expand Up @@ -45,20 +46,36 @@ func NewMonitorHandler(db *gorm.DB, k8sClient *k8s.Client, cfg MonitorConfig) *M

// TokenRequired gates the monitor group. These endpoints expose internal
// cluster state yet must be reachable without a login (the poller can't do
// passkeys), so a shared secret in the URL is the gate. No MONITOR_TOKEN
// configured → 404, indistinguishable from the routes not existing.
// passkeys), so MONITOR_TOKEN is supplied as an Authorization Bearer token.
// No MONITOR_TOKEN configured → 404, indistinguishable from the routes not
// existing.
func (h *MonitorHandler) TokenRequired(c *gin.Context) {
if h.cfg.Token == "" {
c.AbortWithStatusJSON(http.StatusNotFound, gin.H{"error": "not found"})
return
}
if subtle.ConstantTimeCompare([]byte(c.Query("token")), []byte(h.cfg.Token)) != 1 {
authorization := c.GetHeader("Authorization")
token, ok := monitorBearerToken(authorization)
if authorization == "" {
// Undocumented cutover bridge: existing pollers still send the exposed
// query token. Remove this fallback immediately after they move to Bearer.
token, ok = c.GetQuery("token")
}
if !ok || subtle.ConstantTimeCompare([]byte(token), []byte(h.cfg.Token)) != 1 {
c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"})
return
}
c.Next()
}

func monitorBearerToken(header string) (string, bool) {
parts := strings.Fields(header)
if len(parts) != 2 || !strings.EqualFold(parts[0], "Bearer") {
return "", false
}
return parts[1], true
}

// Health — GET /api/v1/monitor/health. Dependency checks: DB, K8s API, Vault
// (including sealed state). Vault reports "disabled" (not an alarm) when
// VAULT_ADDR is unset so local dev without Vault doesn't alarm forever.
Expand Down
57 changes: 57 additions & 0 deletions internal/handler/monitor_test.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,68 @@
package handler

import (
"net/http"
"net/http/httptest"
"testing"

"github.com/gin-gonic/gin"
"github.com/ic3software/vtafarm-api/internal/k8s"
)

func TestMonitorTokenRequired(t *testing.T) {
gin.SetMode(gin.TestMode)

tests := []struct {
name string
configured string
authorize string
queryToken string
wantStatus int
wantReached bool
}{
{name: "disabled", configured: "", authorize: "Bearer monitor-secret", wantStatus: http.StatusNotFound},
{name: "valid bearer", configured: "monitor-secret", authorize: "Bearer monitor-secret", wantStatus: http.StatusNoContent, wantReached: true},
{name: "case insensitive scheme", configured: "monitor-secret", authorize: "bearer monitor-secret", wantStatus: http.StatusNoContent, wantReached: true},
{name: "missing header", configured: "monitor-secret", wantStatus: http.StatusUnauthorized},
{name: "wrong token", configured: "monitor-secret", authorize: "Bearer wrong-secret", wantStatus: http.StatusUnauthorized},
{name: "wrong scheme", configured: "monitor-secret", authorize: "Basic monitor-secret", wantStatus: http.StatusUnauthorized},
{name: "malformed bearer", configured: "monitor-secret", authorize: "Bearer monitor-secret extra", wantStatus: http.StatusUnauthorized},
{name: "legacy query token accepted during cutover", configured: "monitor-secret", queryToken: "monitor-secret", wantStatus: http.StatusNoContent, wantReached: true},
{name: "invalid bearer does not fall back to query", configured: "monitor-secret", authorize: "Bearer wrong-secret", queryToken: "monitor-secret", wantStatus: http.StatusUnauthorized},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
h := NewMonitorHandler(nil, nil, MonitorConfig{Token: tt.configured})
reached := false
router := gin.New()
router.GET("/monitor", h.TokenRequired, func(c *gin.Context) {
reached = true
c.Status(http.StatusNoContent)
})

req := httptest.NewRequest(http.MethodGet, "/monitor", nil)
if tt.authorize != "" {
req.Header.Set("Authorization", tt.authorize)
}
if tt.queryToken != "" {
query := req.URL.Query()
query.Set("token", tt.queryToken)
req.URL.RawQuery = query.Encode()
}
resp := httptest.NewRecorder()
router.ServeHTTP(resp, req)

if resp.Code != tt.wantStatus {
t.Fatalf("status = %d, want %d; body = %s", resp.Code, tt.wantStatus, resp.Body.String())
}
if reached != tt.wantReached {
t.Fatalf("downstream reached = %v, want %v", reached, tt.wantReached)
}
})
}
}

func TestCapacityIssues(t *testing.T) {
healthy := func() *k8s.ClusterStats {
return &k8s.ClusterStats{
Expand Down
Loading
Loading