From 04cf35053c9d989a0e36ffe942e7c1743302c28a Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Fri, 24 Jul 2026 13:50:36 -0700 Subject: [PATCH 1/6] docs(e2e/android): document --include-sources for mapping uploads Explains why Android needs --source-path at all (an R8 mapping records no file paths, unlike a dSYM's DWARF), how files are keyed by their declared package so a Kotlin file in a mismatched directory still resolves, the skipped directories and size limits, and that the flag is off by default because it stores source in LaunchDarkly. Also notes the side benefit: R8's 'SourceFile' placeholder forces the backend to guess .java, and a resolved source replaces that guess with the real path, so Kotlin frames finally show .kt. Co-authored-by: Cursor --- e2e/android/SYMBOLICATION.md | 46 ++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/e2e/android/SYMBOLICATION.md b/e2e/android/SYMBOLICATION.md index 5c1d538405..bf1bbffeea 100644 --- a/e2e/android/SYMBOLICATION.md +++ b/e2e/android/SYMBOLICATION.md @@ -93,6 +93,52 @@ ldcli symbols upload \ You can also pass `--symbols-id ` explicitly, or `--app-version ` to key by version instead (the Version Lane) if you don't inject a symbols id. +### Optional — also upload your sources (`--include-sources`) + +Retracing alone gets you `CartPricing.computeTotal` at `SymbolicationDemo.kt:42`. +Adding `--include-sources` also uploads your `.java`/`.kt` files, so the errors +page shows the **code around each frame** instead of just the location: + +```bash +# from e2e/android/ +ldcli symbols upload \ + --type android \ + --path ./app/build/symbols/composeRelease \ + --project \ + --backend-url http://localhost:8082/private \ + --access-token \ + --include-sources \ + --source-path ./app/src/main +``` + +``` +Built source bundle from ./app/src/main (14 files, 21504 bytes) +``` + +Unlike Apple, an R8 mapping records no file paths at all — only +(class, method, line) — so `ldcli` has to be told where your sources are. +`--source-path` defaults to the current directory, which works from a project +root; pointing it at a source directory is faster and avoids bundling test or +sample code. Files under `build/`, `.gradle/`, `.git/`, `.idea/` and +`node_modules/` are always skipped. + +The files are packed into a `sources.srcbundle` uploaded beside `mapping.txt` +under the same symbols id, so sources are matched to a build exactly as the +mapping is. Each file is keyed by its **declared package** plus its file name +(`com/example/androidobservability/SymbolicationDemo.kt`), which is what a +retraced frame's fully-qualified class name reconstructs — so a Kotlin file in a +directory that doesn't mirror its package still resolves. + +- **Off by default, because it stores your source code in LaunchDarkly.** +- Individual files over 2 MiB, and a bundle over 64 MiB, are skipped. +- If no sources are found it says so and uploads the mapping alone, so this flag + can't break an upload that would otherwise work. + +A side benefit: R8 usually reports the placeholder `SourceFile`, leaving the +backend to guess `.java` for every frame. When a source bundle resolves the +frame, the guess is replaced with the real path — so Kotlin frames finally show +`.kt` and get Kotlin syntax highlighting. + ## 3. Trigger an obfuscated error In the app, tap **Trigger Obfuscated Error**. It throws deep inside an From 6155b12729033c217defb692b2666ea3ea06ffad Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 28 Jul 2026 12:42:32 -0700 Subject: [PATCH 2/6] feat(python): report source context for recorded exceptions A Python traceback carries one source line per frame, so errors recorded by this SDK could only ever show that single line in the errors UI. Read the surrounding lines at capture time and report them as exception.structured_stacktrace, the same attribute and frame shape the Ruby SDK already reports; ingestion prefers it over the raw text stacktrace. Frames are reported innermost first (matching how the backend stores and truncates traces) with 5 lines of context on each side, capped at 20 frames and 1000 characters per line to match the backend's own limits. Source is read through linecache without checkcache, so a file rewritten under a long-running process cannot pair an old frame's line numbers with the new file's text, plus a loader fallback for modules imported from an archive (zipapp, PEX). Frames whose source is not on the running host report no context, which is the behavior callers get today. Co-authored-by: Cursor --- .../ldobserve/_source_context.py | 178 ++++++++++++++++++ .../observability-python/ldobserve/observe.py | 4 +- .../ldobserve/test_observe_instance.py | 22 +++ .../ldobserve/test_source_context.py | 154 +++++++++++++++ 4 files changed, 357 insertions(+), 1 deletion(-) create mode 100644 sdk/@launchdarkly/observability-python/ldobserve/_source_context.py create mode 100644 sdk/@launchdarkly/observability-python/ldobserve/test_source_context.py diff --git a/sdk/@launchdarkly/observability-python/ldobserve/_source_context.py b/sdk/@launchdarkly/observability-python/ldobserve/_source_context.py new file mode 100644 index 0000000000..1f61094f07 --- /dev/null +++ b/sdk/@launchdarkly/observability-python/ldobserve/_source_context.py @@ -0,0 +1,178 @@ +"""Source context for recorded exceptions. + +A Python traceback carries exactly one source line per frame -- whatever +``traceback.format_exception`` chose to print -- so an error recorded with only +the standard ``exception.stacktrace`` attribute can never show more than that +single line, no matter what the errors UI is willing to render. + +This module reads the surrounding lines off disk at capture time and reports them +as ``exception.structured_stacktrace``: a JSON array of frames, innermost first, +each carrying ``linesBefore`` / ``lineContent`` / ``linesAfter``. Ingestion +prefers that attribute over the raw text stacktrace, so the frames land in the +errors UI as-is. The Ruby SDK reports the same attribute in the same shape. + +Everything here is best-effort. Frames whose source is not on the running host +(``exec``-compiled code, source-stripped or frozen bundles, bytecode compiled at +a different path) simply report no context, which is the behavior callers already +get today. No failure in this module is allowed to interfere with recording the +exception itself. +""" + +import json +import linecache +import traceback +import types +import typing + +# Lines of context on each side of the reported line. Matches the window the +# backend produces for every other language it symbolicates. +_CONTEXT_LINES = 5 + +# Deepest frames reported per exception. The backend keeps the leading frames of +# the trace (which are the innermost ones, see _structured_stacktrace) and drops +# the rest, so this only bounds payload size. +_MAX_FRAMES = 20 + +# Per-line cap. The backend trims each field to the same length, so anything +# longer is bytes on the wire that would be discarded on arrival. +_MAX_LINE_LENGTH = 1000 + +_STRUCTURED_STACKTRACE_ATTRIBUTE = "exception.structured_stacktrace" + + +def exception_attributes(error: BaseException) -> typing.Dict[str, str]: + """Build the span attributes describing ``error``'s frames and source. + + Returns an empty dict when no structured stacktrace can be built, so the + result is safe to merge directly into an attribute dict. + """ + try: + frames = _structured_stacktrace(error) + if not frames: + return {} + return {_STRUCTURED_STACKTRACE_ATTRIBUTE: json.dumps(frames)} + except Exception: + # Source context is a nicety; recording the exception is not. Any failure + # here (unreadable file, unexpected frame shape, non-serializable value) + # degrades to the plain text stacktrace. + return {} + + +def _structured_stacktrace( + error: BaseException, +) -> typing.Optional[typing.List[typing.Dict[str, typing.Any]]]: + frames = list(_walk_traceback(error.__traceback__)) + if not frames: + return None + + message = _error_message(error) + # A traceback runs outermost -> innermost, while the backend stores and + # displays frames innermost first (which is also the order it reverses the + # parsed text form into). Take the deepest _MAX_FRAMES, then reverse, so the + # frames nearest the raise survive both this cap and the backend's. + return [ + _build_frame(frame, lineno, message) + for frame, lineno in reversed(frames[-_MAX_FRAMES:]) + ] + + +def _walk_traceback( + tb: typing.Optional[types.TracebackType], +) -> typing.Iterator[typing.Tuple[types.FrameType, int]]: + while tb is not None: + # tb_lineno, not frame.f_lineno: the former is where this frame was + # executing when the exception passed through it, the latter is wherever + # the frame is now. + yield tb.tb_frame, tb.tb_lineno + tb = tb.tb_next + + +def _build_frame( + frame: types.FrameType, lineno: int, message: str +) -> typing.Dict[str, typing.Any]: + code = frame.f_code + built: typing.Dict[str, typing.Any] = { + "fileName": code.co_filename, + "lineNumber": lineno, + "functionName": code.co_name, + "error": message, + } + context = _read_source_context(frame, lineno) + if context: + built.update(context) + return built + + +def _read_source_context( + frame: types.FrameType, lineno: int +) -> typing.Optional[typing.Dict[str, str]]: + if lineno <= 0: + return None + + lines = _file_lines(frame) + if not lines or lineno > len(lines): + return None + + index = lineno - 1 + context = {"lineContent": _clean_line(lines[index])} + before = [ + _clean_line(line) for line in lines[max(0, index - _CONTEXT_LINES) : index] + ] + if before: + context["linesBefore"] = "\n".join(before) + after = [ + _clean_line(line) for line in lines[index + 1 : index + 1 + _CONTEXT_LINES] + ] + if after: + context["linesAfter"] = "\n".join(after) + return context + + +def _file_lines(frame: types.FrameType) -> typing.List[str]: + """Return the raw source lines of ``frame``'s file, or an empty list. + + ``linecache.checkcache`` is deliberately not called. Leaving the cache alone + keeps the text captured when the module was imported, so a file rewritten + under a long-running process (an in-place deploy, a dev reloader) cannot pair + an old frame's line numbers with the new file's contents and report source + that never raised anything. + """ + lines = linecache.getlines(frame.f_code.co_filename) + if lines: + return lines + return _loader_lines(frame) + + +def _loader_lines(frame: types.FrameType) -> typing.List[str]: + """Source for frames whose file cannot be opened by path but whose module can + still produce it -- code imported from an archive (zipapp, PEX, shiv), where + ``co_filename`` names a path *inside* the zip. + """ + loader = frame.f_globals.get("__loader__") + module = frame.f_globals.get("__name__") + if loader is None or module is None or not hasattr(loader, "get_source"): + return [] + try: + source = loader.get_source(module) + except Exception: + return [] + if not source: + return [] + return source.splitlines() + + +def _clean_line(line: str) -> str: + # Strip only the line terminator: leading whitespace is indentation, and the + # rendered preview needs it. + return line.rstrip("\r\n")[:_MAX_LINE_LENGTH] + + +def _error_message(error: BaseException) -> str: + """The exception's type and message, as the trailing line of a traceback.""" + try: + formatted = traceback.format_exception_only(type(error), error) + except Exception: + formatted = [] + if formatted: + return formatted[-1].strip() + return str(error) diff --git a/sdk/@launchdarkly/observability-python/ldobserve/observe.py b/sdk/@launchdarkly/observability-python/ldobserve/observe.py index 3734766815..590d93ec4b 100644 --- a/sdk/@launchdarkly/observability-python/ldobserve/observe.py +++ b/sdk/@launchdarkly/observability-python/ldobserve/observe.py @@ -9,6 +9,7 @@ from opentelemetry.util.types import Attributes from opentelemetry._logs import get_logger_provider from ldobserve._otel.configuration import _OTELConfiguration +from ldobserve._source_context import exception_attributes from ldobserve._util.dict import flatten_dict from opentelemetry.metrics import ( @@ -60,7 +61,8 @@ def record_exception( ctx = self.start_span(_ERROR_NAME) with ctx as span: - attrs = {} + # Source context first, so caller-supplied attributes win on conflict. + attrs = exception_attributes(error) if attributes: addedAttributes = flatten_dict(attributes, sep=".") attrs.update(addedAttributes) diff --git a/sdk/@launchdarkly/observability-python/ldobserve/test_observe_instance.py b/sdk/@launchdarkly/observability-python/ldobserve/test_observe_instance.py index 42bbc5e839..1d6ae9d82b 100644 --- a/sdk/@launchdarkly/observability-python/ldobserve/test_observe_instance.py +++ b/sdk/@launchdarkly/observability-python/ldobserve/test_observe_instance.py @@ -1,3 +1,4 @@ +import json import logging import pytest import typing @@ -240,6 +241,27 @@ def test_record_exception_with_attributes(self): assert attrs_dict.get("error.code") == 400 assert attrs_dict.get("user.id") == "12345" + def test_record_exception_adds_structured_stacktrace(self): + """Test recorded exceptions carry source context for their frames""" + try: + raise RuntimeError("recorded with source context") + except RuntimeError as exception: + with self.test_otel_config.tracer.start_as_current_span( + "test-span" + ) as span: + self.observe_instance.record_exception(exception) + + exception_event = span.events[-1] + attrs_dict = dict(exception_event.attributes) + frames = json.loads(attrs_dict["exception.structured_stacktrace"]) + + assert frames + assert ( + frames[0]["lineContent"].strip() + == 'raise RuntimeError("recorded with source context")' + ) + assert frames[0]["linesBefore"] + def test_record_exception_with_null_exception(self): """Test recording None exception doesn't crash""" # Should not raise an exception diff --git a/sdk/@launchdarkly/observability-python/ldobserve/test_source_context.py b/sdk/@launchdarkly/observability-python/ldobserve/test_source_context.py new file mode 100644 index 0000000000..8dce4a6631 --- /dev/null +++ b/sdk/@launchdarkly/observability-python/ldobserve/test_source_context.py @@ -0,0 +1,154 @@ +import json +import os + +from ldobserve._source_context import ( + _CONTEXT_LINES, + _MAX_FRAMES, + _MAX_LINE_LENGTH, + _STRUCTURED_STACKTRACE_ATTRIBUTE, + exception_attributes, +) + + +def _frames(error: BaseException): + """The structured frames reported for ``error``.""" + attributes = exception_attributes(error) + assert _STRUCTURED_STACKTRACE_ATTRIBUTE in attributes + return json.loads(attributes[_STRUCTURED_STACKTRACE_ATTRIBUTE]) + + +def _raise_with_context(): + important_value = 42 + doubled = important_value * 2 + raise ValueError("context failure") + + +def test_includes_source_context(): + try: + _raise_with_context() + except ValueError as error: + frames = _frames(error) + + innermost = frames[0] + assert innermost["functionName"] == "_raise_with_context" + assert os.path.basename(innermost["fileName"]) == "test_source_context.py" + assert innermost["lineContent"].strip() == 'raise ValueError("context failure")' + assert "important_value = 42" in innermost["linesBefore"] + assert innermost["error"] == "ValueError: context failure" + + +def test_frames_are_innermost_first(): + try: + _raise_with_context() + except ValueError as error: + frames = _frames(error) + + assert [frame["functionName"] for frame in frames] == [ + "_raise_with_context", + "test_frames_are_innermost_first", + ] + + +def test_line_content_keeps_indentation(): + try: + _raise_with_context() + except ValueError as error: + frames = _frames(error) + + # The preview is rendered as code, so leading whitespace has to survive. + assert frames[0]["lineContent"].startswith(" ") + + +def _raise_in_padded_region(): + first = 1 + second = 2 + third = 3 + fourth = 4 + fifth = 5 + raise ValueError("padded") + + +def test_context_window_is_bounded(): + try: + _raise_in_padded_region() + except ValueError as error: + frames = _frames(error) + + innermost = frames[0] + assert len(innermost["linesBefore"].splitlines()) == _CONTEXT_LINES + assert len(innermost["linesAfter"].splitlines()) == _CONTEXT_LINES + assert innermost["linesBefore"].splitlines()[0].strip() == "first = 1" + + +def _recurse(depth: int): + if depth == 0: + raise RuntimeError("deep") + _recurse(depth - 1) + + +def test_limits_frame_count_keeping_deepest(): + try: + _recurse(_MAX_FRAMES + 10) + except RuntimeError as error: + frames = _frames(error) + + assert len(frames) == _MAX_FRAMES + # Innermost first, so the raise survives the cap and this test's own frame + # (the outermost one) is what gets dropped. + assert frames[0]["lineContent"].strip() == 'raise RuntimeError("deep")' + assert all(frame["functionName"] == "_recurse" for frame in frames) + + +def test_missing_file_reports_frame_without_source(): + source = "def explode():\n raise RuntimeError('missing file')\n" + namespace: dict = {} + exec(compile(source, "/definitely/missing/file.py", "exec"), namespace) + + try: + namespace["explode"]() + except RuntimeError as error: + frames = _frames(error) + + innermost = frames[0] + assert innermost["fileName"] == "/definitely/missing/file.py" + assert innermost["lineNumber"] == 2 + assert innermost["functionName"] == "explode" + assert "lineContent" not in innermost + assert "linesBefore" not in innermost + assert "linesAfter" not in innermost + + +def test_truncates_long_source_lines(tmp_path): + long_line = "padding = '" + "y" * (_MAX_LINE_LENGTH * 2) + "'" + module = tmp_path / "long_lines.py" + module.write_text( + "def explode():\n" f" {long_line}\n" " raise RuntimeError('long')\n" + ) + + namespace: dict = {} + exec(compile(module.read_text(), str(module), "exec"), namespace) + + try: + namespace["explode"]() + except RuntimeError as error: + frames = _frames(error) + + innermost = frames[0] + assert len(innermost["lineContent"]) <= _MAX_LINE_LENGTH + assert innermost["linesBefore"] + for line in innermost["linesBefore"].splitlines(): + assert len(line) <= _MAX_LINE_LENGTH + + +def test_exception_without_traceback_reports_nothing(): + # Never raised, so there are no frames to describe. + assert exception_attributes(ValueError("unraised")) == {} + + +def test_error_message_uses_exception_type_and_message(): + try: + raise KeyError("missing-key") + except KeyError as error: + frames = _frames(error) + + assert frames[0]["error"] == "KeyError: 'missing-key'" From b3ed834722a93d44a3e54e265916c4e790610304 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 28 Jul 2026 12:59:27 -0700 Subject: [PATCH 3/6] fix(python): report source context for chained exceptions Ingestion prefers exception.structured_stacktrace over the raw text stacktrace, and the text form is what carried the frames of a chained exception's root cause: the parser collects both tracebacks either side of the "During handling of the above exception" separator. Describing only error.__traceback__ therefore dropped root cause frames from the errors UI for every "raise ... from" and implicit context chain. Walk the chain with traceback's own precedence (__cause__ wins, __context__ is skipped once "from None" suppressed it), attribute each frame to the exception it belongs to, and guard against cycles in the writable chain attributes. Frames are now capped per exception as well as in total, so a deep stack on the raised exception cannot crowd its root cause out of the payload, and the total matches the backend's own frame cap. Co-authored-by: Cursor --- .../ldobserve/_source_context.py | 72 +++++++++--- .../ldobserve/test_source_context.py | 105 +++++++++++++++++- 2 files changed, 162 insertions(+), 15 deletions(-) diff --git a/sdk/@launchdarkly/observability-python/ldobserve/_source_context.py b/sdk/@launchdarkly/observability-python/ldobserve/_source_context.py index 1f61094f07..14eb2c1cf2 100644 --- a/sdk/@launchdarkly/observability-python/ldobserve/_source_context.py +++ b/sdk/@launchdarkly/observability-python/ldobserve/_source_context.py @@ -9,7 +9,9 @@ as ``exception.structured_stacktrace``: a JSON array of frames, innermost first, each carrying ``linesBefore`` / ``lineContent`` / ``linesAfter``. Ingestion prefers that attribute over the raw text stacktrace, so the frames land in the -errors UI as-is. The Ruby SDK reports the same attribute in the same shape. +errors UI as-is -- and so the whole exception chain has to be described here, +because the text form that would otherwise have carried the root cause is no +longer read. The Ruby SDK reports the same attribute in the same shape. Everything here is best-effort. Frames whose source is not on the running host (``exec``-compiled code, source-stripped or frozen bundles, bytecode compiled at @@ -28,10 +30,14 @@ # backend produces for every other language it symbolicates. _CONTEXT_LINES = 5 -# Deepest frames reported per exception. The backend keeps the leading frames of -# the trace (which are the innermost ones, see _structured_stacktrace) and drops -# the rest, so this only bounds payload size. -_MAX_FRAMES = 20 +# Deepest frames reported for each exception in the chain, so that a deep stack +# on the raised exception cannot crowd out its root cause entirely. +_MAX_FRAMES_PER_EXCEPTION = 20 + +# Total frames reported. Matches the backend's own frame cap: it keeps the +# leading frames (the innermost ones, see _structured_stacktrace) and drops the +# rest, so anything past this would be discarded on arrival. +_MAX_FRAMES = 64 # Per-line cap. The backend trims each field to the same length, so anything # longer is bytes on the wire that would be discarded on arrival. @@ -61,21 +67,61 @@ def exception_attributes(error: BaseException) -> typing.Dict[str, str]: def _structured_stacktrace( error: BaseException, ) -> typing.Optional[typing.List[typing.Dict[str, typing.Any]]]: - frames = list(_walk_traceback(error.__traceback__)) - if not frames: + collected: typing.List[typing.Tuple[types.FrameType, int, str]] = [] + for exception in _exception_chain(error): + message = _error_message(exception) + frames = list(_walk_traceback(exception.__traceback__)) + collected.extend( + (frame, lineno, message) + for frame, lineno in frames[-_MAX_FRAMES_PER_EXCEPTION:] + ) + if not collected: return None - message = _error_message(error) - # A traceback runs outermost -> innermost, while the backend stores and - # displays frames innermost first (which is also the order it reverses the - # parsed text form into). Take the deepest _MAX_FRAMES, then reverse, so the - # frames nearest the raise survive both this cap and the backend's. + # ``collected`` now runs the way a traceback prints: root cause first, each + # exception outermost -> innermost. The backend stores and displays the + # reverse of that (it reverses the parsed text form to get there), so flip + # the whole list. The raise site of the final exception leads, root cause + # frames trail, and the backend's tail truncation drops the least relevant + # frames rather than the most relevant ones. + collected.reverse() return [ _build_frame(frame, lineno, message) - for frame, lineno in reversed(frames[-_MAX_FRAMES:]) + for frame, lineno, message in collected[:_MAX_FRAMES] ] +def _exception_chain(error: BaseException) -> typing.List[BaseException]: + """The exceptions to describe, root cause first. + + An exception raised while handling another one carries the first as its + ``__cause__`` (explicit ``raise ... from``) or ``__context__`` (implicit), + and ``traceback.format_exception`` prints the whole chain. Ingestion prefers + the structured stacktrace over that text, so leaving the chain out here would + drop the root cause frames from the errors UI entirely. + """ + chain: typing.List[BaseException] = [] + seen: typing.Set[int] = set() + current: typing.Optional[BaseException] = error + while current is not None and id(current) not in seen: + seen.add(id(current)) + chain.append(current) + current = _chained_cause(current) + # Printed order: deepest cause first, the exception that was raised last. + chain.reverse() + return chain + + +def _chained_cause(error: BaseException) -> typing.Optional[BaseException]: + # The same precedence ``traceback`` applies: an explicit "raise ... from" + # wins, and an implicit context is skipped once "from None" suppressed it. + if error.__cause__ is not None: + return error.__cause__ + if error.__context__ is not None and not error.__suppress_context__: + return error.__context__ + return None + + def _walk_traceback( tb: typing.Optional[types.TracebackType], ) -> typing.Iterator[typing.Tuple[types.FrameType, int]]: diff --git a/sdk/@launchdarkly/observability-python/ldobserve/test_source_context.py b/sdk/@launchdarkly/observability-python/ldobserve/test_source_context.py index 8dce4a6631..810a3c51f1 100644 --- a/sdk/@launchdarkly/observability-python/ldobserve/test_source_context.py +++ b/sdk/@launchdarkly/observability-python/ldobserve/test_source_context.py @@ -4,6 +4,7 @@ from ldobserve._source_context import ( _CONTEXT_LINES, _MAX_FRAMES, + _MAX_FRAMES_PER_EXCEPTION, _MAX_LINE_LENGTH, _STRUCTURED_STACKTRACE_ATTRIBUTE, exception_attributes, @@ -88,17 +89,117 @@ def _recurse(depth: int): def test_limits_frame_count_keeping_deepest(): try: - _recurse(_MAX_FRAMES + 10) + _recurse(_MAX_FRAMES_PER_EXCEPTION + 10) except RuntimeError as error: frames = _frames(error) - assert len(frames) == _MAX_FRAMES + assert len(frames) == _MAX_FRAMES_PER_EXCEPTION # Innermost first, so the raise survives the cap and this test's own frame # (the outermost one) is what gets dropped. assert frames[0]["lineContent"].strip() == 'raise RuntimeError("deep")' assert all(frame["functionName"] == "_recurse" for frame in frames) +def _raise_root_cause(): + marker = "root cause marker" + raise KeyError("root cause") + + +def test_explicit_cause_frames_are_reported(): + try: + try: + _raise_root_cause() + except KeyError as cause: + raise ValueError("wrapper") from cause + except ValueError as error: + frames = _frames(error) + + # The raised exception leads, innermost first; the cause's frames trail it. + assert frames[0]["functionName"] == "test_explicit_cause_frames_are_reported" + assert frames[0]["lineContent"].strip() == 'raise ValueError("wrapper") from cause' + assert [frame["functionName"] for frame in frames[1:]] == [ + "_raise_root_cause", + "test_explicit_cause_frames_are_reported", + ] + assert frames[1]["lineContent"].strip() == 'raise KeyError("root cause")' + assert "root cause marker" in frames[1]["linesBefore"] + + +def test_each_frame_carries_its_own_exception_message(): + try: + try: + _raise_root_cause() + except KeyError as cause: + raise ValueError("wrapper") from cause + except ValueError as error: + frames = _frames(error) + + assert frames[0]["error"] == "ValueError: wrapper" + assert frames[1]["error"] == "KeyError: 'root cause'" + + +def test_implicit_context_frames_are_reported(): + try: + try: + _raise_root_cause() + except KeyError: + raise ValueError("during handling") + except ValueError as error: + frames = _frames(error) + + assert [frame["functionName"] for frame in frames] == [ + "test_implicit_context_frames_are_reported", + "_raise_root_cause", + "test_implicit_context_frames_are_reported", + ] + + +def test_suppressed_context_is_not_reported(): + try: + try: + _raise_root_cause() + except KeyError: + raise ValueError("standalone") from None + except ValueError as error: + frames = _frames(error) + + # "from None" means the context is deliberately hidden, as in a traceback. + assert [frame["functionName"] for frame in frames] == [ + "test_suppressed_context_is_not_reported" + ] + + +def test_chain_frame_count_is_capped_in_total(): + try: + try: + _recurse(_MAX_FRAMES) + except RuntimeError as cause: + raise ValueError("wrapper") from cause + except ValueError as error: + frames = _frames(error) + + assert len(frames) <= _MAX_FRAMES + # The cause is capped per exception, so it still gets its deepest frames in + # rather than being crowded out by the raised exception's stack. + assert any(frame["functionName"] == "_recurse" for frame in frames) + + +def test_cyclic_exception_chain_terminates(): + error = ValueError("self-referential") + try: + raise error + except ValueError: + pass + # A cycle cannot arise from normal raising, but the attributes are writable + # and a walk that trusted them would not terminate. + error.__context__ = error + + frames = _frames(error) + assert [frame["functionName"] for frame in frames] == [ + "test_cyclic_exception_chain_terminates" + ] + + def test_missing_file_reports_frame_without_source(): source = "def explode():\n raise RuntimeError('missing file')\n" namespace: dict = {} From 12a563a65ba2b9308f66e6716354513f7babc4ea Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 28 Jul 2026 13:08:06 -0700 Subject: [PATCH 4/6] test(e2e/python): exercise chained exception source context The Flask app's routes only raised flat exceptions, so neither the source context nor the exception chain the SDK now reports had a route that showed them. Add one that raises through a "raise ... from" chain and records it explicitly, since only observe.record_exception carries source context (an uncaught exception is recorded by the Flask instrumentation instead). Also fix the site-packages wipe in `make install`, which never ran: make expanded $(poetry env info --path) as a make variable rather than a shell substitution, leaving find with no path to search and erroring out. That wipe exists precisely so local SDK changes get picked up, which is what this app is for. Co-authored-by: Cursor --- e2e/python-plugin-flask/Makefile | 2 +- e2e/python-plugin-flask/README.md | 26 ++++++++++++++++++- e2e/python-plugin-flask/flask_app/__init__.py | 16 ++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/e2e/python-plugin-flask/Makefile b/e2e/python-plugin-flask/Makefile index 5348842ed2..5ecf7300b6 100644 --- a/e2e/python-plugin-flask/Makefile +++ b/e2e/python-plugin-flask/Makefile @@ -9,7 +9,7 @@ help: # This is a pretty aggressive install that removes all the site packages. # This ensures that we get the latest version when working on changes to the plugin. install: - find $(poetry env info --path) -type d -name site-packages -exec rm -rf {} \; ; poetry lock && poetry install + find $$(poetry env info --path) -type d -name site-packages -exec rm -rf {} \; ; poetry lock && poetry install run: poetry run flask run diff --git a/e2e/python-plugin-flask/README.md b/e2e/python-plugin-flask/README.md index 357f214de6..63bd231a25 100644 --- a/e2e/python-plugin-flask/README.md +++ b/e2e/python-plugin-flask/README.md @@ -19,4 +19,28 @@ make install Run the application: ```bash make run -``` \ No newline at end of file +``` + +# Pointing at a local stack + +The SDK exports to LaunchDarkly by default. To send to a local observability +stack, point it at the local collector — OTLP over gRPC, port 4317: + +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 +``` + +Data is attributed to the project the SDK key belongs to, so use a key from the +environment the data should land in. + +# Routes + +| Route | Exercises | +| --- | --- | +| `/` | Nothing; a liveness check | +| `/crash`, `/raise-exception` | An uncaught exception, recorded by the Flask instrumentation | +| `/manual-record-exception` | `observe.record_exception`, including source context | +| `/manual-record-chained-exception` | A `raise ... from` chain, whose root cause frames and source must survive | +| `/manual-span` | `observe.start_span` and a flag evaluation | +| `/manual-record-log` | `observe.record_log` | +| `/record-metrics` | Counters, gauges, and histograms | \ No newline at end of file diff --git a/e2e/python-plugin-flask/flask_app/__init__.py b/e2e/python-plugin-flask/flask_app/__init__.py index f22656e015..41848b0c0d 100644 --- a/e2e/python-plugin-flask/flask_app/__init__.py +++ b/e2e/python-plugin-flask/flask_app/__init__.py @@ -57,6 +57,22 @@ def manual_record_exception(): def raise_exception(): raise Exception("uncaught exception") +def _load_worker_config(): + settings = {"retries": 3} + # The root cause. Its frames and source context have to survive the chain. + return settings["timeout"] + +@app.route('/manual-record-chained-exception') +def manual_record_chained_exception(): + try: + try: + _load_worker_config() + except KeyError as cause: + raise RuntimeError("could not start worker") from cause + except RuntimeError as e: + observe.record_exception(e) + return 'Manual record chained exception' + @app.route('/manual-record-log') def manual_record_log(): observe.record_log("manual-record-log", logging.INFO, {"custom": "value"}) From 8fb20b07124b7c077cfa4f92eeb694aaaf9775e3 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Wed, 29 Jul 2026 10:06:05 -0700 Subject: [PATCH 5/6] fixes --- .../ldobserve/_source_context.py | 76 +++++++-- .../ldobserve/test_source_context.py | 153 ++++++++++++++++++ 2 files changed, 218 insertions(+), 11 deletions(-) diff --git a/sdk/@launchdarkly/observability-python/ldobserve/_source_context.py b/sdk/@launchdarkly/observability-python/ldobserve/_source_context.py index 14eb2c1cf2..7e8277bc7f 100644 --- a/sdk/@launchdarkly/observability-python/ldobserve/_source_context.py +++ b/sdk/@launchdarkly/observability-python/ldobserve/_source_context.py @@ -43,6 +43,11 @@ # longer is bytes on the wire that would be discarded on arrival. _MAX_LINE_LENGTH = 1000 +# How far into nested exception groups to look. Groups this deep are describing +# the machinery that gathered the failures rather than the failures, and the walk +# needs a bound it cannot get from cycle detection alone. +_MAX_GROUP_DEPTH = 8 + _STRUCTURED_STACKTRACE_ATTRIBUTE = "exception.structured_stacktrace" @@ -68,7 +73,11 @@ def _structured_stacktrace( error: BaseException, ) -> typing.Optional[typing.List[typing.Dict[str, typing.Any]]]: collected: typing.List[typing.Tuple[types.FrameType, int, str]] = [] - for exception in _exception_chain(error): + # Every exception contributes at least one frame, so only the last _MAX_FRAMES + # of them can survive the cap below (the list is reversed before it is cut). + # Without this, the source for every member of a wide group is read and then + # thrown away. + for exception in _exception_chain(error)[-_MAX_FRAMES:]: message = _error_message(exception) frames = list(_walk_traceback(exception.__traceback__)) collected.extend( @@ -92,24 +101,59 @@ def _structured_stacktrace( def _exception_chain(error: BaseException) -> typing.List[BaseException]: - """The exceptions to describe, root cause first. + """The exceptions to describe, in the order a traceback prints them. An exception raised while handling another one carries the first as its - ``__cause__`` (explicit ``raise ... from``) or ``__context__`` (implicit), - and ``traceback.format_exception`` prints the whole chain. Ingestion prefers - the structured stacktrace over that text, so leaving the chain out here would - drop the root cause frames from the errors UI entirely. + ``__cause__`` (explicit ``raise ... from``) or ``__context__`` (implicit), and + an exception group carries several at once. ``traceback.format_exception`` + prints all of them; ingestion prefers the structured stacktrace over that + text, so anything left out here is missing from the errors UI entirely. """ chain: typing.List[BaseException] = [] - seen: typing.Set[int] = set() + _collect_exceptions(error, chain, set(), 0) + return chain + + +def _collect_exceptions( + error: BaseException, + chain: typing.List[BaseException], + seen: typing.Set[int], + depth: int, +) -> None: + # The cause walk is iterative because the chain has no bound: raising inside + # an except block in a loop adds a link per iteration. Recursion is used only + # for group nesting, which _MAX_GROUP_DEPTH bounds. + causes: typing.List[BaseException] = [] current: typing.Optional[BaseException] = error while current is not None and id(current) not in seen: seen.add(id(current)) - chain.append(current) + causes.append(current) current = _chained_cause(current) # Printed order: deepest cause first, the exception that was raised last. - chain.reverse() - return chain + causes.reverse() + + for exception in causes: + chain.append(exception) + if depth < _MAX_GROUP_DEPTH: + # A group's own traceback is the machinery that gathered the + # failures; the failures themselves are its members, and a member can + # be another group. + for member in _group_members(exception): + _collect_exceptions(member, chain, seen, depth + 1) + + +def _group_members(error: BaseException) -> typing.List[BaseException]: + """The exceptions held by ``error`` when it is a group, otherwise nothing. + + Groups are recognized by shape rather than by type. ``BaseExceptionGroup`` is + new in 3.11, so on 3.10 the name cannot even be referred to, and the same + shape arrives there from the ``exceptiongroup`` backport that anyio and trio + depend on. + """ + members = getattr(error, "exceptions", None) + if not isinstance(members, (tuple, list)): + return [] + return [member for member in members if isinstance(member, BaseException)] def _chained_cause(error: BaseException) -> typing.Optional[BaseException]: @@ -214,11 +258,21 @@ def _clean_line(line: str) -> str: def _error_message(error: BaseException) -> str: - """The exception's type and message, as the trailing line of a traceback.""" + """The exception's type and message, as a traceback prints it. + + ``format_exception_only`` returns more than that one line, and what surrounds + it differs by version, so the line is picked out by position rather than by + taking an end of the list. A ``SyntaxError`` is preceded by its file, source + and caret, all indented; notes added by ``add_note`` (3.11+) follow it. The + line wanted is the first one written flush left. + """ try: formatted = traceback.format_exception_only(type(error), error) except Exception: formatted = [] + for entry in formatted: + if entry and not entry[0].isspace(): + return entry.strip() if formatted: return formatted[-1].strip() return str(error) diff --git a/sdk/@launchdarkly/observability-python/ldobserve/test_source_context.py b/sdk/@launchdarkly/observability-python/ldobserve/test_source_context.py index 810a3c51f1..f2585f5a62 100644 --- a/sdk/@launchdarkly/observability-python/ldobserve/test_source_context.py +++ b/sdk/@launchdarkly/observability-python/ldobserve/test_source_context.py @@ -1,5 +1,8 @@ import json import os +import sys + +import pytest from ldobserve._source_context import ( _CONTEXT_LINES, @@ -253,3 +256,153 @@ def test_error_message_uses_exception_type_and_message(): frames = _frames(error) assert frames[0]["error"] == "KeyError: 'missing-key'" + + +def test_syntax_error_message_survives_its_source_preview(): + try: + compile("x ===\n", "", "exec") + except SyntaxError as error: + frames = _frames(error) + + # A SyntaxError is formatted with its file, source and caret before the + # message, so the message is not the first line printed. + assert frames[0]["error"].startswith("SyntaxError:") + + +@pytest.mark.skipif( + sys.version_info < (3, 11), reason="add_note was added in Python 3.11" +) +def test_note_does_not_replace_the_exception_message(): + try: + raise KeyError("noted") + except KeyError as error: + error.add_note("worker was retrying the config load") + frames = _frames(error) + + # Notes are formatted after the message, so the message is not the last line + # printed either. + assert frames[0]["error"] == "KeyError: 'noted'" + + +def _two_failures(): + """Two exceptions that have been raised, ready to be put in a group.""" + caught = [] + for raiser in (_raise_root_cause, _raise_with_context): + try: + raiser() + except (KeyError, ValueError) as error: + caught.append(error) + return caught + + +@pytest.mark.skipif( + sys.version_info < (3, 11), reason="exception groups were added in Python 3.11" +) +def test_exception_group_members_are_reported(): + try: + raise ExceptionGroup("worker failures", _two_failures()) + except BaseException as error: + frames = _frames(error) + + functions = [frame["functionName"] for frame in frames] + assert "_raise_root_cause" in functions + assert "_raise_with_context" in functions + + # A member's frames carry that member's message, not the group's. + messages = {frame["error"] for frame in frames} + assert "KeyError: 'root cause'" in messages + assert "ValueError: context failure" in messages + assert any(message.startswith("ExceptionGroup:") for message in messages) + + +@pytest.mark.skipif( + sys.version_info < (3, 11), reason="exception groups were added in Python 3.11" +) +def test_exception_group_members_report_source_context(): + try: + raise ExceptionGroup("worker failures", _two_failures()) + except BaseException as error: + frames = _frames(error) + + member = next( + frame for frame in frames if frame["functionName"] == "_raise_root_cause" + ) + assert member["lineContent"].strip() == 'raise KeyError("root cause")' + assert "root cause marker" in member["linesBefore"] + + +@pytest.mark.skipif( + sys.version_info < (3, 11), reason="exception groups were added in Python 3.11" +) +def test_nested_exception_group_members_are_reported(): + inner = ExceptionGroup("inner failures", _two_failures()) + try: + raise ExceptionGroup("outer failures", [inner]) + except BaseException as error: + frames = _frames(error) + + functions = [frame["functionName"] for frame in frames] + assert "_raise_root_cause" in functions + assert "_raise_with_context" in functions + + +class _BackportedGroup(Exception): + """A group as the ``exceptiongroup`` backport presents one. + + That backport is how anyio and trio raise groups on 3.10, where the builtin + does not exist, and it is why members are found by shape. + """ + + def __init__(self, message, exceptions): + super().__init__(message) + self.exceptions = tuple(exceptions) + + +def test_group_members_are_found_by_shape(): + try: + raise _BackportedGroup("worker failures", _two_failures()) + except _BackportedGroup as error: + frames = _frames(error) + + functions = [frame["functionName"] for frame in frames] + assert "_raise_root_cause" in functions + assert "_raise_with_context" in functions + + +def test_group_member_count_is_capped_in_total(): + members = _two_failures() * 100 + try: + raise _BackportedGroup("many failures", members) + except _BackportedGroup as error: + frames = _frames(error) + + assert len(frames) <= _MAX_FRAMES + + +def test_cyclic_group_membership_terminates(): + try: + raise _BackportedGroup("self-referential", []) + except _BackportedGroup as error: + group = error + # A group cannot normally contain itself, but the attribute is writable and a + # walk that trusted it would not terminate. + group.exceptions = (group,) + + frames = _frames(group) + assert [frame["functionName"] for frame in frames] == [ + "test_cyclic_group_membership_terminates" + ] + + +def test_non_group_exceptions_attribute_is_ignored(): + class Confusing(Exception): + exceptions = "not a group at all" + + try: + raise Confusing("just an exception") + except Confusing as error: + frames = _frames(error) + + assert [frame["functionName"] for frame in frames] == [ + "test_non_group_exceptions_attribute_is_ignored" + ] From 48aa9261578aae737a530b5a8f54a502ea550cdb Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Wed, 29 Jul 2026 18:41:27 -0700 Subject: [PATCH 6/6] fix(python): keep the raise site when a group's members were never raised The chain was trimmed to its last _MAX_FRAMES entries before any frame was read, on the assumption that every exception contributes at least one. A group member built at the raise site rather than caught has no traceback and contributes none, so a group wider than the cap pushed the group itself -- and any root cause behind it -- off the front of the chain while keeping members that describe nothing. That left either no structured stacktrace at all, though the raise site was there to report, or one that omits the root cause while ingestion still prefers it over the text form. Frames are now produced lazily in the order they are reported, so the walk stops once the cap is met and nothing is dropped for having contributed nothing. Co-authored-by: Cursor --- .../ldobserve/_source_context.py | 52 +++++++++++-------- .../ldobserve/test_source_context.py | 43 ++++++++++++++- 2 files changed, 71 insertions(+), 24 deletions(-) diff --git a/sdk/@launchdarkly/observability-python/ldobserve/_source_context.py b/sdk/@launchdarkly/observability-python/ldobserve/_source_context.py index 7e8277bc7f..4bbd71a16f 100644 --- a/sdk/@launchdarkly/observability-python/ldobserve/_source_context.py +++ b/sdk/@launchdarkly/observability-python/ldobserve/_source_context.py @@ -20,6 +20,7 @@ exception itself. """ +import itertools import json import linecache import traceback @@ -72,34 +73,41 @@ def exception_attributes(error: BaseException) -> typing.Dict[str, str]: def _structured_stacktrace( error: BaseException, ) -> typing.Optional[typing.List[typing.Dict[str, typing.Any]]]: - collected: typing.List[typing.Tuple[types.FrameType, int, str]] = [] - # Every exception contributes at least one frame, so only the last _MAX_FRAMES - # of them can survive the cap below (the list is reversed before it is cut). - # Without this, the source for every member of a wide group is read and then - # thrown away. - for exception in _exception_chain(error)[-_MAX_FRAMES:]: - message = _error_message(exception) - frames = list(_walk_traceback(exception.__traceback__)) - collected.extend( - (frame, lineno, message) - for frame, lineno in frames[-_MAX_FRAMES_PER_EXCEPTION:] - ) + # Lazily, so that a wide group costs the frames reported rather than the + # frames held: the walk stops as soon as the cap is met. Trimming the chain + # up front instead would assume every exception contributes a frame, and a + # group member that was built rather than raised contributes none. + collected = list(itertools.islice(_chain_frames(error), _MAX_FRAMES)) if not collected: return None - - # ``collected`` now runs the way a traceback prints: root cause first, each - # exception outermost -> innermost. The backend stores and displays the - # reverse of that (it reverses the parsed text form to get there), so flip - # the whole list. The raise site of the final exception leads, root cause - # frames trail, and the backend's tail truncation drops the least relevant - # frames rather than the most relevant ones. - collected.reverse() return [ - _build_frame(frame, lineno, message) - for frame, lineno, message in collected[:_MAX_FRAMES] + _build_frame(frame, lineno, message) for frame, lineno, message in collected ] +def _chain_frames( + error: BaseException, +) -> typing.Iterator[typing.Tuple[types.FrameType, int, str]]: + """Every frame of the exception chain, in the order the backend stores them. + + A traceback prints the root cause first and each exception outermost -> + innermost. The backend stores and displays the reverse of that (it reverses + the parsed text form to get there), which is why both walks here run + backwards: the raise site of the exception raised last comes first and the + root cause frames trail. A caller that keeps a prefix therefore keeps the + most relevant frames, the same way the backend's own truncation does. + """ + for exception in reversed(_exception_chain(error)): + frames = list(_walk_traceback(exception.__traceback__)) + # A never-raised exception -- a group member built at the raise site + # rather than caught -- has no traceback and nothing to describe. + if not frames: + continue + message = _error_message(exception) + for frame, lineno in reversed(frames[-_MAX_FRAMES_PER_EXCEPTION:]): + yield frame, lineno, message + + def _exception_chain(error: BaseException) -> typing.List[BaseException]: """The exceptions to describe, in the order a traceback prints them. diff --git a/sdk/@launchdarkly/observability-python/ldobserve/test_source_context.py b/sdk/@launchdarkly/observability-python/ldobserve/test_source_context.py index f2585f5a62..96b992b590 100644 --- a/sdk/@launchdarkly/observability-python/ldobserve/test_source_context.py +++ b/sdk/@launchdarkly/observability-python/ldobserve/test_source_context.py @@ -370,13 +370,52 @@ def test_group_members_are_found_by_shape(): def test_group_member_count_is_capped_in_total(): - members = _two_failures() * 100 + members = [failure for _ in range(50) for failure in _two_failures()] try: raise _BackportedGroup("many failures", members) except _BackportedGroup as error: frames = _frames(error) - assert len(frames) <= _MAX_FRAMES + assert len(frames) == _MAX_FRAMES + + +def _unraised_failures(count): + """Exceptions built rather than caught, as a group's members often are. + + ``raise ExceptionGroup("...", [ValueError("a"), ValueError("b")])`` puts + exceptions that were never raised in the group, so they have no traceback. + """ + return [ValueError("failure {}".format(number)) for number in range(count)] + + +def test_wide_group_of_unraised_members_reports_the_raise_site(): + members = _unraised_failures(_MAX_FRAMES * 2) + try: + raise _BackportedGroup("many failures", members) + except _BackportedGroup as error: + frames = _frames(error) + + # The members carry no frames, so the group's own are all there is to report + # -- and there are more members than the frame cap, which must not be spent + # on them. + assert [frame["functionName"] for frame in frames] == [ + "test_wide_group_of_unraised_members_reports_the_raise_site" + ] + + +def test_unraised_group_members_do_not_crowd_out_the_root_cause(): + try: + try: + _raise_root_cause() + except KeyError as cause: + raise _BackportedGroup( + "many failures", _unraised_failures(_MAX_FRAMES * 2) + ) from cause + except _BackportedGroup as error: + frames = _frames(error) + + functions = [frame["functionName"] for frame in frames] + assert "_raise_root_cause" in functions def test_cyclic_group_membership_terminates():