feat(go): report source context for recorded errors

e2e/go-plugin/README.md

@@ -20,6 +20,48 @@ cd observability-sdk/e2e/go-plugin
 export LAUNCHDARKLY_SDK_KEY="your-sdk-key-here"
 ```
 
+## Running against a backend on localhost
+
+The examples export to LaunchDarkly by default. The HTTP example also honours the
+standard endpoint variable, so it can be pointed at a stack running locally. The
+SDK exports OTLP over HTTP, which the local collector receives on port 4318 — not
+4317, which is the gRPC port other SDKs use:
+
+```bash
+export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
+export LAUNCHDARKLY_SDK_KEY="key-from-the-local-project"
+```
+
+Data is attributed to the project the SDK key belongs to, so this has to be a key
+from the local stack rather than a real one, or the errors land somewhere you are
+not looking.
+
+Then start the example and record one error of each shape:
+
+```bash
+go run ./cmd/http &
+until curl -s -o /dev/null localhost:8080/; do sleep 1; done
+for route in plain stack wrapped goroutine; do curl -s "localhost:8080/error/$route"; done
+```
+
+The example is waited for because it has to compile and then wait for the
+LaunchDarkly client before it starts serving. The errors reach the local backend a
+few seconds later, once the SDK batches its spans out, and appear in the errors UI
+as four separate error groups.
+
+What to look for on each of them:
+
+- The first frame is the application's own code. It should never be
+  `RecordError`, `recordSpanError` or anything else belonging to the SDK.
+- Frames read as `package.(*Receiver).Method`, and a closure reads as
+  `Method.func1` rather than as a bare number.
+- Each frame shows the source line that ran with the lines around it. A frame
+  reporting no source at all is expected for any code whose file is not on this
+  machine, which for a Go binary is the normal case anywhere but a dev box.
+
+If nothing arrives, the example logs its export failures, so check its output
+first: a wrong port shows up there as `connection refused`.
+
 ## Examples
 
 This project contains several examples, each demonstrating the LaunchDarkly Observability plugin with different Go frameworks and libraries:
@@ -100,6 +142,10 @@ go run cmd/http/http.go
 
 **Endpoints:**
 - `GET /rolldice` - Rolls a dice and returns the result, with verbose output controlled by the `verbose-response` feature flag
+- `GET /error/plain` - Records an error with no stack of its own, so the SDK has to capture one. The reported frames should start at the handler's helper, not inside the SDK
+- `GET /error/stack` - Records a `pkg/errors` error, whose stack was captured where the error was created, three calls below the handler
+- `GET /error/wrapped` - Records a wrapped error. The frames should reach the root cause, not stop at the wrap
+- `GET /error/goroutine` - Records an error raised in nested closures on another goroutine, with no span to attach it to
 
 ### 5. Logrus Example (`cmd/logrus/`)
 
@@ -143,4 +189,3 @@ go run cmd/preinit/preinit.go
 - Advanced initialization scenarios where you need observability before the LaunchDarkly client
 - Using observability features without a LaunchDarkly client (some features will be unavailable)
 - Custom initialization timing requirements
-

e2e/go-plugin/cmd/http/http.go

@@ -19,6 +19,7 @@ import (
 
 	appcontext "dice/internal/context"
 	"dice/internal/dice"
+	"dice/internal/failure"
 	"dice/internal/version"
 )
 
@@ -33,14 +34,21 @@ func run() (err error) {
 	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
 	defer stop()
 
+	options := []ldobserve.Option{
+		ldobserve.WithEnvironment("test"),
+		ldobserve.WithServiceName("go-plugin-example"),
+		ldobserve.WithServiceVersion(version.Commit),
+	}
+	// The SDK sets its endpoint explicitly, so the standard environment variable
+	// has to be applied by hand for this example to reach a local stack.
+	if endpoint := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"); endpoint != "" {
+		options = append(options, ldobserve.WithOTLPEndpoint(endpoint))
+	}
+
 	client, _ := ld.MakeCustomClient(os.Getenv("LAUNCHDARKLY_SDK_KEY"),
 		ld.Config{
 			Plugins: []ldplugins.Plugin{
-				ldobserve.NewObservabilityPlugin(
-					ldobserve.WithEnvironment("test"),
-					ldobserve.WithServiceName("go-plugin-example"),
-					ldobserve.WithServiceVersion(version.Commit),
-				),
+				ldobserve.NewObservabilityPlugin(options...),
 			},
 		}, 5*time.Second)
 
@@ -92,6 +100,10 @@ func newHTTPHandler() http.Handler {
 
 	// Register handlers.Als
 	handleFunc("/rolldice", dice.Rolldice)
+	handleFunc("/error/plain", failure.Plain)
+	handleFunc("/error/stack", failure.WithStack)
+	handleFunc("/error/wrapped", failure.Wrapped)
+	handleFunc("/error/goroutine", failure.InGoroutine)
 
 	// Add HTTP instrumentation for the whole server.
 	handler := otelhttp.NewHandler(

e2e/go-plugin/internal/failure/failure.go

@@ -0,0 +1,113 @@
+// Package failure records the error shapes the SDK has to describe, so that the
+// frames and source context reaching the UI can be checked against a running
+// application rather than only a unit test.
+package failure
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"sync"
+
+	"github.com/pkg/errors"
+
+	ldobserve "github.com/launchdarkly/observability-sdk/go"
+)
+
+// workerSettings stands in for configuration loaded at startup. The missing
+// timeout is what every route here trips over, so the source lines reported for
+// the innermost frame are the same few lines of real code each time.
+var workerSettings = map[string]int{"retries": 3}
+
+// Plain records an error that carries no stack of its own, which is what
+// fmt.Errorf and the standard errors package produce. The SDK has to capture a
+// stack while recording, so the innermost frame should be readTimeout below
+// rather than the SDK's own recording code.
+func Plain(w http.ResponseWriter, r *http.Request) {
+	if _, err := readTimeout(); err != nil {
+		ldobserve.RecordError(r.Context(), err)
+		respond(w, "recorded an error with no stack of its own")
+		return
+	}
+	respond(w, "worker config was complete, nothing to record")
+}
+
+func readTimeout() (int, error) {
+	timeout, ok := workerSettings["timeout"]
+	if !ok {
+		return 0, fmt.Errorf("worker config has no timeout")
+	}
+	return timeout, nil
+}
+
+// WithStack records an error from pkg/errors, which captures a stack when the
+// error is created. The SDK reports that stack instead of the one it could
+// capture here, so the frames should start at connectWorker, three calls below
+// this handler.
+func WithStack(w http.ResponseWriter, r *http.Request) {
+	if _, err := startWorker(); err != nil {
+		ldobserve.RecordError(r.Context(), err)
+		respond(w, "recorded an error carrying its own stack")
+		return
+	}
+	respond(w, "worker started, nothing to record")
+}
+
+func startWorker() (int, error) {
+	return connectWorker()
+}
+
+func connectWorker() (int, error) {
+	timeout, ok := workerSettings["timeout"]
+	if !ok {
+		return 0, errors.New("worker config has no timeout")
+	}
+	return timeout, nil
+}
+
+// Wrapped records a wrapped error, Go's answer to a chained exception: one error
+// for the root cause and another for the context around it. Both messages belong
+// in the text stack trace, and the frames come from where the cause was created.
+func Wrapped(w http.ResponseWriter, r *http.Request) {
+	if err := runWorker(); err != nil {
+		ldobserve.RecordError(r.Context(), err)
+		respond(w, "recorded a wrapped error")
+		return
+	}
+	respond(w, "worker ran, nothing to record")
+}
+
+func runWorker() error {
+	if _, err := connectWorker(); err != nil {
+		return errors.Wrap(err, "could not start worker")
+	}
+	return nil
+}
+
+// InGoroutine records an error from nested closures on another goroutine, the
+// shape whose frames used to arrive named after bare ordinals ("2") or with a
+// trailing "in goroutine N". Nothing carries the request span onto that
+// goroutine, so the SDK also has to start a span of its own here.
+func InGoroutine(w http.ResponseWriter, r *http.Request) {
+	var workers sync.WaitGroup
+	workers.Add(1)
+	go func() {
+		defer workers.Done()
+		func() {
+			if _, err := connectWorker(); err != nil {
+				ldobserve.RecordError(context.Background(), errors.Wrap(err, "worker goroutine failed"))
+			}
+		}()
+	}()
+	workers.Wait()
+
+	respond(w, "recorded an error raised on another goroutine")
+}
+
+func respond(w http.ResponseWriter, message string) {
+	if _, err := io.WriteString(w, message+"\n"); err != nil {
+		log.Printf("Write failed: %v\n", err)
+	}
+}

go/singleton.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"reflect"
 
-	"github.com/pkg/errors"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/log"
 	semconv "go.opentelemetry.io/otel/semconv/v1.34.0"
@@ -39,6 +38,13 @@ func EndSpan(span trace.Span) {
 // If there is no active recording span, then a new span is created and the error is recorded in it.
 // If this function starts a span, then that span will be ended after the error is recorded.
 func RecordError(ctx context.Context, err error, tags ...attribute.KeyValue) context.Context {
+	if err == nil {
+		// Nothing to record, as in the other SDKs. Recording used to reach
+		// span.RecordError, which ignores a nil error but only after a span had
+		// been started for it.
+		return ctx
+	}
+
 	span := trace.SpanFromContext(ctx)
 	if span.IsRecording() {
 		recordSpanError(span, err, tags...)
@@ -56,22 +62,33 @@ func RecordError(ctx context.Context, err error, tags ...attribute.KeyValue) con
 }
 
 func recordSpanError(span trace.Span, err error, tags ...attribute.KeyValue) {
-	type withStackTrace interface {
-		StackTrace() errors.StackTrace
-	}
-
-	if stackErr, ok := err.(withStackTrace); ok {
-		stackTrace := fmt.Sprintf("%+v", stackErr.StackTrace())
-		attributes := make([]attribute.KeyValue, 0, 3+len(tags))
+	// A stack the error carries was captured where the error was created, which
+	// is closer to the failure than anything this function could capture.
+	if stack, ok := stackTraceOf(err); ok {
+		stackTrace := fmt.Sprintf("%+v", stack)
+		attributes := make([]attribute.KeyValue, 0, 4+len(tags))
 		attributes = append(attributes,
 			semconv.ExceptionTypeKey.String(reflect.TypeOf(err).String()),
 			semconv.ExceptionMessageKey.String(err.Error()),
 			semconv.ExceptionStacktraceKey.String(stackTrace),
 		)
+		carried := structuredFrames(callersFromStackTrace(stack), err.Error())
+		if structured, ok := structuredStacktraceAttribute(carried); ok {
+			attributes = append(attributes, structured)
+		}
 		attributes = append(attributes, tags...)
 		span.AddEvent(semconv.ExceptionEventName, trace.WithAttributes(attributes...))
 	} else {
-		span.RecordError(err, trace.WithStackTrace(true))
+		options := []trace.EventOption{trace.WithStackTrace(true)}
+		attributes := make([]attribute.KeyValue, 0, 1+len(tags))
+		if structured, ok := structuredStacktraceAttribute(framesAtRecordTime(err.Error())); ok {
+			attributes = append(attributes, structured)
+		}
+		attributes = append(attributes, tags...)
+		if len(attributes) > 0 {
+			options = append(options, trace.WithAttributes(attributes...))
+		}
+		span.RecordError(err, options...)
 	}
 }