feat: one-call, env-driven observability and audit instrumentation

docs/auditlog.md

@@ -16,7 +16,46 @@ acknowledged the record, or with the error that prevented it.** Nothing is
 buffered; a nil return means written. High-volume events can opt into
 `ModeBatch` — see [Delivery modes](#delivery-modes).
 
-## Set up
+## The short way: env + one call
+
+```go
+audit, err := auditlog.FromEnv(ctx) // GOPHERAGENT_AUDIT_ENDPOINT unset → nil client: auditing off
+if err != nil { /* set but invalid configuration */ }
+defer audit.Close(ctx)
+
+audit.Instrument(loop) // tools, HITL approvals, permission verdicts, hook rejections
+```
+
+```bash
+export GOPHERAGENT_AUDIT_ENDPOINT=collector:4317
+export GOPHERAGENT_AUDIT_INSECURE=true          # local collectors only
+export GOPHERAGENT_AUDIT_HMAC_KEY=...            # optional: signed records, keyed fingerprints
+```
+
+A nil `*Client` is a valid, disabled client — every method is a no-op — so
+the same code runs with auditing on or off. Call `Instrument` last, after
+`ConfirmHITL`, `Permissions`, and the tools are set, and once per loop;
+anything attached later is named in a log warning on the loop's first event.
+The client's health metrics (`gopheragent.audit.records{result=failed|dropped}`)
+go to the global OTel meter, so with telemetry enabled they need no wiring.
+Records fall back to `GOPHERAGENT_AUDIT_TENANT_ID` (default `"default"`)
+when the request ctx carries no `Actor`; multi-tenant apps set the actor per
+request with `auditlog.WithActor`.
+
+| Variable | Meaning |
+|---|---|
+| `GOPHERAGENT_AUDIT_ENDPOINT` | `host:port` — set it to turn auditing on |
+| `GOPHERAGENT_AUDIT_REQUIRED` | `true`: a missing (or misspelled) endpoint fails `FromEnv` instead of turning auditing off — set it wherever audit is mandatory |
+| `GOPHERAGENT_AUDIT_PROTOCOL` | `grpc` (default) or `http/protobuf` |
+| `GOPHERAGENT_AUDIT_INSECURE` | `true` for plaintext |
+| `GOPHERAGENT_AUDIT_SERVICE_NAME` | default `OTEL_SERVICE_NAME`, then `gopheragent` |
+| `GOPHERAGENT_AUDIT_TENANT_ID` | tenant when ctx has no `Actor`; default `default` |
+| `GOPHERAGENT_AUDIT_HMAC_KEY` | ≥16 bytes: keyed fingerprints and signed records |
+| `GOPHERAGENT_AUDIT_HEADERS` | `k=v,k2=v2`, values percent-encoded |
+| `GOPHERAGENT_AUDIT_MODE` | `blocking` (default) or `batch` |
+| `GOPHERAGENT_AUDIT_CA_FILE` / `_CERT_FILE` / `_KEY_FILE` | TLS / mTLS |
+
+## Set up in code
 
 ```go
 c, err := auditlog.New(ctx, auditlog.Config{

docs/observability.md

@@ -59,6 +59,29 @@ if err != nil { /* ... */ }
 defer shutdown(context.Background())
 ```
 
+### The short way: one call
+
+```go
+tel, shutdown, err := otelsetup.Setup(ctx, otelsetup.Config{ServiceName: "my-agent"})
+if err != nil { /* invalid OTEL_* configuration */ }
+defer shutdown(context.Background())
+
+tel.Instrument(loop, otelllm.WithSystem("anthropic"), otelllm.WithModel(model))
+```
+
+`Instrument` wires the loop (run/iteration spans, run metrics), its LLM (chat
+spans, token metrics), and every registered tool (execute_tool spans) — for a
+loop from `agent.New` or the YAML builder alike. Call it after the tools are
+registered (tools registered later are named in a log warning on the loop's
+first event); calling it twice is harmless. With no endpoint configured it does
+nothing, so call it unconditionally and enable telemetry purely with env:
+
+```bash
+export OTEL_EXPORTER_OTLP_ENDPOINT=localhost:4317   # host:port or http://host:4317
+```
+
+The sections below show the same wiring piece by piece, for finer control.
+
 ### Loops built with `agent.New`
 
 ```go

examples/demo/main.go

@@ -17,21 +17,25 @@ import (
 	"go.opentelemetry.io/otel/propagation"
 
 	"github.com/hung12ct/gopheragent/pkg/agent"
+	"github.com/hung12ct/gopheragent/pkg/auditlog"
 	"github.com/hung12ct/gopheragent/pkg/builder"
 	"github.com/hung12ct/gopheragent/pkg/history"
 	"github.com/hung12ct/gopheragent/pkg/llm/anthropic"
 	"github.com/hung12ct/gopheragent/pkg/llm/gemini"
 	"github.com/hung12ct/gopheragent/pkg/llm/openai"
 	"github.com/hung12ct/gopheragent/pkg/telemetry/otelllm"
 	"github.com/hung12ct/gopheragent/pkg/telemetry/otelsetup"
-	"github.com/hung12ct/gopheragent/pkg/telemetry/oteltools"
 	"github.com/hung12ct/gopheragent/pkg/tools/builtin"
 )
 
 // telProviders holds the OTel Tracer/Meter when export is enabled; nil disables
 // all demo instrumentation.
 var telProviders *otelsetup.Providers
 
+// auditClient ships audit events when GOPHERAGENT_AUDIT_ENDPOINT is set; nil
+// (a disabled client) otherwise.
+var auditClient *auditlog.Client
+
 var myAgentApp *agent.AgentLoop
 
 // agentInfo exposes the loaded YAML config to the frontend so the UI can
@@ -68,48 +72,42 @@ func loadEnvFiles() {
 	}
 }
 
-// instrumentLLM wraps p with the OTel decorator when telemetry is enabled,
-// tagging spans/metrics with the vendor and model. A no-op otherwise.
-func instrumentLLM(p agent.LLMProvider, system, model string) agent.LLMProvider {
-	if telProviders == nil {
-		return p
-	}
-	return otelllm.NewProvider(p,
-		otelllm.WithSystem(system), otelllm.WithModel(model),
-		otelllm.WithTracer(telProviders.Tracer), otelllm.WithMeter(telProviders.Meter))
-}
-
-func buildProvider() agent.LLMProvider {
+// buildProvider returns the configured LLM plus its vendor and model, which
+// label the telemetry.
+func buildProvider() (p agent.LLMProvider, system, model string) {
 	provider := strings.ToLower(strings.TrimSpace(os.Getenv("LLM_PROVIDER")))
 	log.Printf("LLM_PROVIDER=%q", provider)
 	switch provider {
 	case "openai":
 		model := strings.TrimSpace(os.Getenv("OPENAI_MODEL"))
-		p, err := openai.New("", model)
+		op, err := openai.New("", model)
+		p = op
 		if err == nil {
 			log.Printf("Using OpenAI provider")
-			return instrumentLLM(p, "openai", model)
+			return p, "openai", model
 		}
 		log.Printf("Warning: OpenAI provider init failed: %v", err)
 	case "anthropic", "claude":
 		model := strings.TrimSpace(os.Getenv("ANTHROPIC_MODEL"))
-		p, err := anthropic.New("", model)
+		ap, err := anthropic.New("", model)
+		p = ap
 		if err == nil {
 			log.Printf("Using Anthropic provider")
-			return instrumentLLM(p, "anthropic", model)
+			return p, "anthropic", model
 		}
 		log.Printf("Warning: Anthropic provider init failed: %v", err)
 	case "gemini":
 		model := strings.TrimSpace(os.Getenv("GEMINI_MODEL"))
-		p, err := gemini.New("", model)
+		gp, err := gemini.New("", model)
+		p = gp
 		if err == nil {
 			log.Printf("Using Gemini provider")
-			return instrumentLLM(p, "gemini", model)
+			return p, "gemini", model
 		}
 		log.Printf("Warning: Gemini provider init failed: %v", err)
 	}
 	log.Printf("Using Mock provider (set LLM_PROVIDER=openai|anthropic|gemini to use real model)")
-	return agent.NewMockProvider()
+	return agent.NewMockProvider(), "mock", ""
 }
 
 func SecurityHook(ctx context.Context, sessionKey string, userInput string) error {
@@ -370,14 +368,6 @@ func getStream(sessionKey string) chan<- agent.StreamEvent {
 func initApp() {
 	catalog := builder.NewGlobalCatalog()
 
-	// Instrument every catalog tool with per-Execute spans + latency metrics.
-	// Must run before the loop is built so tools are wrapped at registration.
-	if telProviders != nil {
-		catalog.Use(oteltools.Instrument(
-			oteltools.WithTracer(telProviders.Tracer),
-			oteltools.WithMeter(telProviders.Meter)))
-	}
-
 	// Web tools — Register* helpers use tools.RegisterFunc internally
 	// so the catalog gets a typed-args funcTool without the per-tool
 	// struct boilerplate.
@@ -434,7 +424,8 @@ func initApp() {
 		log.Printf("Background summarizer enabled (gpt-4o-mini)")
 	}
 
-	loop, _, _, err := builder.BuildFromYAMLWithSession(yamlPath, catalog, buildProvider(), SecurityHook, sm)
+	llm, llmSystem, llmModel := buildProvider()
+	loop, _, _, err := builder.BuildFromYAMLWithSession(yamlPath, catalog, llm, SecurityHook, sm)
 	if err != nil {
 		log.Fatalf("Failed to build demo agent from YAML (%s): %v", yamlPath, err)
 	}
@@ -444,11 +435,10 @@ func initApp() {
 	loop.ConfirmPlan = buildConfirmPlan()
 	planModeDefault = os.Getenv("PLAN_MODE") == "true"
 
-	// Open a run/iteration span per turn so LLM (via the decorated provider) and
-	// tool (via catalog middleware) spans nest into one trace per conversation.
-	if telProviders != nil {
-		loop.Configure(agent.WithTracer(telProviders.Tracer), agent.WithMeter(telProviders.Meter))
-	}
+	// One call each, after the loop is fully configured. Both are no-ops when
+	// their endpoint isn't set (telProviders / auditClient are nil).
+	telProviders.Instrument(loop, otelllm.WithSystem(llmSystem), otelllm.WithModel(llmModel))
+	auditClient.Instrument(loop)
 
 	myAgentApp = loop
 	log.Printf("Demo loaded: %s", yamlPath)
@@ -587,10 +577,8 @@ func ChatHandler(w http.ResponseWriter, r *http.Request) {
 // setupTelemetry wires OpenTelemetry OTLP export. otelsetup.Setup stays off
 // unless OTEL_EXPORTER_OTLP_ENDPOINT (or a per-signal endpoint) is set, and
 // honors OTEL_SDK_DISABLED / OTEL_*_EXPORTER=none / OTEL_EXPORTER_OTLP_PROTOCOL.
-// Returns a shutdown func to flush exporters, or nil when telemetry is disabled.
-// It sets telProviders; the rest of the wiring (provider decorator, catalog tool
-// middleware, loop.Configure) reads that global so a YAML-built agent still gets
-// fully nested invoke_agent → iteration → chat → execute_tool spans.
+// Returns a shutdown func to flush exporters, or nil when telemetry is
+// disabled. initApp instruments the loop with one telProviders.Instrument call.
 func setupTelemetry() func(context.Context) error {
 	tel, shutdown, err := otelsetup.Setup(context.Background(), otelsetup.Config{
 		ServiceName: "gopheragent-demo",
@@ -627,6 +615,12 @@ func main() {
 	if shutdown != nil {
 		defer func() { _ = shutdown(context.Background()) }()
 	}
+	// Auditing is on only when GOPHERAGENT_AUDIT_ENDPOINT is set.
+	var err error
+	if auditClient, err = auditlog.FromEnv(context.Background()); err != nil {
+		log.Fatalf("audit log config: %v", err)
+	}
+	defer func() { _ = auditClient.Close(context.Background()) }()
 	initApp()
 	http.Handle("/", http.FileServer(http.Dir("./frontend")))
 	http.HandleFunc("/api/chat", withTraceContext(ChatHandler))

internal/latewire/latewire.go

@@ -0,0 +1,55 @@
+// Package latewire catches the one way the one-call Instrument helpers fail
+// silently: something attached to the loop after Instrument ran.
+package latewire
+
+import (
+	"context"
+	"log"
+	"slices"
+	"sync"
+
+	"github.com/hung12ct/gopheragent/pkg/agent"
+	"github.com/hung12ct/gopheragent/pkg/tools"
+)
+
+// Watch snapshots what Instrument covered and, on the loop's first event —
+// once, so it costs nothing afterwards — logs a warning naming anything
+// attached later: tools, and with hooks, the HITL callback and permission
+// policy. Compared by name, so wrapping a tool in more middleware is not a
+// false alarm.
+func Watch(loop *agent.AgentLoop, component string, hooks bool) {
+	known := names(loop.Tools)
+	hadHITL, hadPerms := loop.ConfirmHITL != nil, loop.Permissions != nil
+	var once sync.Once
+	loop.OnEvent(func(context.Context, string, agent.StreamEvent) {
+		once.Do(func() {
+			var late []string
+			for _, n := range names(loop.Tools) {
+				if !slices.Contains(known, n) {
+					late = append(late, "tool "+n)
+				}
+			}
+			if hooks && !hadHITL && loop.ConfirmHITL != nil {
+				late = append(late, "ConfirmHITL")
+			}
+			if hooks && !hadPerms && loop.Permissions != nil {
+				late = append(late, "Permissions")
+			}
+			if len(late) > 0 {
+				log.Printf("[gopheragent] %s: %v attached after Instrument and not covered — call Instrument last", component, late)
+			}
+		})
+	})
+}
+
+func names(r *tools.Registry) []string {
+	if r == nil {
+		return nil
+	}
+	all := r.All()
+	out := make([]string, 0, len(all))
+	for _, t := range all {
+		out = append(out, t.Descriptor().Name)
+	}
+	return out
+}

internal/latewire/latewire_test.go

@@ -0,0 +1,73 @@
+package latewire
+
+import (
+	"bytes"
+	"context"
+	"log"
+	"strings"
+	"testing"
+
+	"github.com/hung12ct/gopheragent/pkg/agent"
+	"github.com/hung12ct/gopheragent/pkg/history"
+	"github.com/hung12ct/gopheragent/pkg/tools"
+)
+
+type answerLLM struct{}
+
+func (answerLLM) GenerateStream(context.Context, []history.Message, *tools.Registry, chan<- agent.StreamEvent) (agent.LLMResult, error) {
+	return agent.LLMResult{Content: "ok"}, nil
+}
+
+type namedTool string
+
+func (n namedTool) Descriptor() tools.ToolDescriptor { return tools.ToolDescriptor{Name: string(n)} }
+func (namedTool) Execute(context.Context, string) (tools.Result, error) {
+	return tools.Text("ok"), nil
+}
+
+// captureLog runs fn and returns what it logged.
+func captureLog(t *testing.T, fn func()) string {
+	t.Helper()
+	var buf bytes.Buffer
+	prev, flags := log.Writer(), log.Flags()
+	log.SetOutput(&buf)
+	t.Cleanup(func() { log.SetOutput(prev); log.SetFlags(flags) })
+	fn()
+	return buf.String()
+}
+
+func newLoop(toolNames ...string) *agent.AgentLoop {
+	reg := tools.NewRegistry()
+	for _, n := range toolNames {
+		reg.Register(namedTool(n))
+	}
+	return agent.New(history.NewInMemSessionManager("sys"), reg, answerLLM{})
+}
+
+func TestWatch_WarnsOnceAboutLateAttachments(t *testing.T) {
+	loop := newLoop("early")
+	Watch(loop, "auditlog", true)
+	loop.Tools.Register(namedTool("late"))
+	loop.ConfirmHITL = func(context.Context, string, string) bool { return true }
+
+	out := captureLog(t, func() {
+		_, _ = loop.RunIteration(context.Background(), "s", "a")
+		_, _ = loop.RunIteration(context.Background(), "s", "b")
+	})
+	if !strings.Contains(out, "tool late") || !strings.Contains(out, "ConfirmHITL") || strings.Contains(out, "tool early") {
+		t.Fatalf("warning = %q, want it to name the late tool and ConfirmHITL only", out)
+	}
+	if strings.Count(out, "attached after Instrument") != 1 {
+		t.Fatalf("warned %d times, want once", strings.Count(out, "attached after Instrument"))
+	}
+}
+
+func TestWatch_SilentWhenInstrumentIsLast(t *testing.T) {
+	loop := newLoop("a", "b")
+	loop.ConfirmHITL = func(context.Context, string, string) bool { return true }
+	Watch(loop, "auditlog", true)
+	loop.Tools.Use(func(next tools.Tool) tools.Tool { return next }) // more wrapping is not "late"
+	if out := captureLog(t, func() { _, _ = loop.RunIteration(context.Background(), "s", "q") }); out != "" {
+		t.Fatalf("unexpected warning: %q", out)
+	}
+}

pkg/auditlog/agent.go

@@ -135,6 +135,9 @@ func ToolMiddleware(e Emitter, opts ...Option) tools.Middleware {
 	o := newOptions(opts)
 	fp, _ := e.(Fingerprinter)
 	return func(next tools.Tool) tools.Tool {
+		if _, done := next.(*auditedTool); done {
+			return next // already audited: wrapping again would double every record
+		}
 		desc := next.Descriptor()
 		level := LevelMetadata
 		if o.policy != nil {